ЛР 5 - БВТ-2302 - Каменский - на 5 #38
@@ -16,5 +16,9 @@ backups
|
||||
.venv
|
||||
__pycache__
|
||||
node_modules
|
||||
**/bin
|
||||
**/obj
|
||||
evidence
|
||||
.DS_Store
|
||||
|
||||
.local
|
||||
|
||||
+3
-1
@@ -11,7 +11,7 @@ certs/private/
|
||||
*.pem
|
||||
*.tfstate*
|
||||
volumes/
|
||||
data/
|
||||
/data/
|
||||
backups/
|
||||
# Runtime/build artifacts
|
||||
.venv/
|
||||
@@ -30,3 +30,5 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
|
||||
/.local/
|
||||
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
FROM mcr.microsoft.com/dotnet/sdk:9.0.306 AS build
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
COPY src/Messenger.Api/Messenger.Api.csproj src/Messenger.Api/
|
||||
RUN dotnet restore src/Messenger.Api/Messenger.Api.csproj
|
||||
|
||||
COPY src/Messenger.Api/ src/Messenger.Api/
|
||||
|
||||
RUN dotnet publish src/Messenger.Api/Messenger.Api.csproj \
|
||||
-c Release \
|
||||
--no-restore \
|
||||
-o /app/publish
|
||||
|
||||
|
||||
FROM mcr.microsoft.com/dotnet/aspnet:9.0.10 AS runtime
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
ARG BUILD_REVISION=local
|
||||
LABEL org.opencontainers.image.revision=$BUILD_REVISION
|
||||
|
||||
COPY --from=build /app/publish .
|
||||
|
||||
ENV APP_HOST=0.0.0.0
|
||||
ENV APP_PORT=8080
|
||||
ENV APP_ENV=lab
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
USER app
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD curl --fail --silent http://127.0.0.1:8080/health/ready > /dev/null || exit 1
|
||||
|
||||
ENTRYPOINT ["dotnet", "Messenger.Api.dll"]
|
||||
@@ -1,5 +1,5 @@
|
||||
PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
PYTHON ?= .venv/bin/python
|
||||
BASE_URL ?= http://localhost:18085
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
@@ -28,3 +28,49 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: setup up down build restore-api stop-worker start-worker replay reconcile
|
||||
setup:
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-dev.txt -r requirements-student.txt -r requirements-research.txt
|
||||
./scripts/init-secrets.sh
|
||||
up:
|
||||
./scripts/init-secrets.sh
|
||||
./scripts/compose.sh build api
|
||||
./scripts/compose.sh up -d --wait --wait-timeout 150
|
||||
build:
|
||||
dotnet build src/Messenger.Api
|
||||
restore-api:
|
||||
./scripts/compose.sh up -d --wait api lb
|
||||
stop-worker:
|
||||
./scripts/compose.sh stop worker
|
||||
start-worker:
|
||||
./scripts/compose.sh up -d worker
|
||||
replay:
|
||||
@test -n "$(JOB_ID)" || (echo "Set JOB_ID"; exit 2)
|
||||
./scripts/compose.sh exec -T dispatcher dotnet Messenger.Api.dll --replay "$(JOB_ID)"
|
||||
reconcile:
|
||||
./scripts/compose.sh exec -T dispatcher dotnet Messenger.Api.dll --reconcile
|
||||
down:
|
||||
./scripts/compose.sh down
|
||||
|
||||
.PHONY: student-test
|
||||
student-test:
|
||||
$(PYTHON) student-tests/pipeline_tests.py --base-url "$(BASE_URL)"
|
||||
|
||||
.PHONY: research-grade5
|
||||
research-grade5:
|
||||
$(PYTHON) student-tests/overload_research.py --base-url "$(BASE_URL)"
|
||||
$(PYTHON) scripts/plot_overload.py
|
||||
|
||||
.PHONY: share stop-share operations-test
|
||||
share:
|
||||
python3 scripts/share.py
|
||||
stop-share:
|
||||
python3 scripts/share.py --stop
|
||||
operations-test:
|
||||
$(PYTHON) student-tests/operations_panel.py
|
||||
|
||||
.PHONY: operator
|
||||
operator:
|
||||
python3 scripts/configure-operator.py "$(USERNAME)"
|
||||
|
||||
@@ -1,37 +1,305 @@
|
||||
# Отчёт — лабораторная 5
|
||||
# Лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
Автор: Каменский Никита, БВТ-2302.
|
||||
Целевая оценка: 5. Трек 1: очередь под перегрузкой.
|
||||
PR: [ЛР 5 - БВТ-2302 - Каменский - на 5](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/38).
|
||||
|
||||
## Запуск из чистого clone
|
||||
## Запуск
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Desktop, Python 3.9+ и доступ в интернет для сборки.
|
||||
Для сборки вне Docker нужен .NET SDK 9.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
make setup
|
||||
make up
|
||||
make check
|
||||
make validate
|
||||
make test
|
||||
make student-test
|
||||
make research-grade5
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
make restore-api
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Сайт: http://localhost:18085/. Lab5 использует отдельную базу и volumes.
|
||||
Для входа сначала зарегистрировать аккаунт с паролем 12–128 символов.
|
||||
Остановка с сохранением данных: make down.
|
||||
Намеренное удаление учебных данных: ./scripts/compose.sh down -v.
|
||||
Секреты создаёт scripts/init-secrets.py в исключённой папке secrets/.
|
||||
Повторный setup сохраняет существующие ключи. Общий volume RabbitMQ закреплён
|
||||
за hostname broker; миграции запускаются отдельно с блокировкой PostgreSQL.
|
||||
|
||||
## Доказательства критериев
|
||||
Версии: .NET SDK 9.0.306/runtime 9.0.10, PostgreSQL 17.6, Redis 7.4.2,
|
||||
HAProxy 3.2.7-alpine, SeaweedFS 3.97, RabbitMQ 4.2.0-alpine,
|
||||
RabbitMQ.Client 7.1.2, AWSSDK.S3 4.0.0, SkiaSharp 3.119.1.
|
||||
Pillow 12.0.0 используется только для тестовых изображений и проверки результатов.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Изолированная сборка из копии исходников без .env, secrets, bin/obj и старых
|
||||
volumes проверена отдельно: evidence/clean-build.txt. Созданы новые secrets,
|
||||
проверен Compose config и собран Docker-образ.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
## Архитектура
|
||||
|
||||
## Эксперименты и отказы
|
||||
Перенесены собственные исходники и интерфейс Lab4, Dockerfile, конфигурация
|
||||
HAProxy и операционные скрипты. Skeleton, contracts и публичные tests взяты
|
||||
из Lab5. Сопоставление архитектуры: docs/ARCHITECTURE.md.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Две API за HAProxy обслуживают общий PostgreSQL, Redis и приватный S3.
|
||||
Добавлены отдельные RabbitMQ, worker и outbox dispatcher. Один C# образ
|
||||
используется для нескольких ролей, но worker и API являются разными процессами.
|
||||
Интерфейс опрашивает метаданные; queued/processing показываются пользователю,
|
||||
после ready работает прежняя галерея с оригиналами. В браузере проверен переход
|
||||
при остановленном и затем запущенном worker: evidence/ui-queued.jpg и ui-ready.jpg.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
## Границы сохранности
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
API полностью принимает PNG/JPEG до 10 MiB, проверяет MIME, имя и максимум
|
||||
40 млн пикселей. Проверка декодируемости остаётся на API для совместимости
|
||||
с ошибкой 415; resize выполняет только worker. Копирование ограничено буфером
|
||||
64 KiB и временным файлом. Один upload/decoder на API, deadline 90 секунд.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Под блокировкой приёма проверяется количество queued/processing jobs. Если
|
||||
лимит PIPELINE_CAPACITY достигнут, возвращается 429 и Retry-After: 3, без
|
||||
записи нового объекта. По умолчанию лимит 128. Входящие HTTP-передачи и
|
||||
валидация занимают ограниченные ресурсы до этой проверки; лимит защищает
|
||||
принятую работу, а не означает отсутствие стоимости отклонённых запросов.
|
||||
|
||||
Полный оригинал сохраняется в приватный staging/<attachment UUID>. В одной
|
||||
транзакции PostgreSQL записываются Attachment queued, job и outbox. Только
|
||||
после S3 PUT и COMMIT API возвращает снимок 202 queued. Даже если worker уже
|
||||
успел завершиться, ответ upload остаётся queued по контракту. Broker и worker
|
||||
не входят в readiness API: при их отказе работа принимается до лимита outbox.
|
||||
PostgreSQL и S3 остаются критичными для синхронного приёма.
|
||||
|
||||
Outbox dispatcher выбирает доступную запись FOR UPDATE SKIP LOCKED, отправляет
|
||||
persistent сообщение в durable quorum queue attachments.v1 с mandatory и
|
||||
publisher confirms. Отметка published_at записывается только после confirm.
|
||||
Сбой между publish и COMMIT приводит к дублю, но не теряет намерение.
|
||||
Брокер получает только поля jobs.schema.json, без байтов и credentials.
|
||||
Job ID сохраняется при управляемых повторах и redelivery. Реальный carrier
|
||||
проверен по jobs.schema.json с FormatChecker: evidence/job-envelope.json.
|
||||
|
||||
Worker получает job с ручным ack и проверяет состояние в PostgreSQL.
|
||||
Для job используется session advisory lock; параллельный дубль не начинает
|
||||
второй эффект. queued атомарно становится processing с attempt и lease_token.
|
||||
Работа скачивается из доверенного staging_key в БД, проверяются размер и SHA-256.
|
||||
Клиентский filename не используется как путь или ключ.
|
||||
|
||||
Имена результатов определены attachment_id/pipeline_version/variant.
|
||||
Оригинал: originals/<id>/v1; варианты: variants/<id>/v1/<name>. Оригинальные
|
||||
байты не перекодируются. После записи всех объектов одна транзакция публикует
|
||||
метаданные вариантов и ready. Фиксация проверяет lease_token и срок аренды.
|
||||
Ack выполняется после устойчивого результата или устойчивого решения о retry/failed.
|
||||
Дубль ready не меняет completed_at, attempts или варианты.
|
||||
|
||||
## Изображения и ограничения worker
|
||||
|
||||
Thumbnail: JPEG 85, вписан в 256×256 без увеличения. Crop: центральный квадрат
|
||||
128×128, для маленьких изображений возможно увеличение. Optimized: WebP 80,
|
||||
вписан в 1280×1280 без увеличения. Размеры округляются вниз с минимумом 1 px.
|
||||
EXIF orientation применяется перед расчётом размеров и crop. Alpha компонуется
|
||||
на белый. Перекодирование из bitmap удаляет EXIF. Для fixture 320×200 thumbnail
|
||||
имеет 256×160. Не гарантируется, что WebP каждого маленького файла меньше PNG.
|
||||
|
||||
Worker повторно проверяет bytes/checksum, заголовок, пиксели и декодирование.
|
||||
Нативное преобразование идёт в дочернем процессе, который принудительно
|
||||
останавливается по JOB_TIMEOUT_SECONDS, по умолчанию 30. Аренда дольше на 5 s.
|
||||
Один decoder хранит RGBA bitmap до 160 млн байт и небольшие выходные bitmap.
|
||||
Варианты формируются по очереди. Общие CPU/RAM worker ограничены Docker:
|
||||
1 CPU, 1024 MiB. Время/память ограничены и при ошибках нативной библиотеки.
|
||||
Переменные WORKER_CONCURRENCY и WORKER_PREFETCH по умолчанию 2 и 1;
|
||||
каждый канал обрабатывает одно сообщение одновременно, prefetch ограничивает
|
||||
число полученных, но ещё не подтверждённых сообщений на канале.
|
||||
|
||||
## Ошибки, повторы и replay
|
||||
|
||||
Невалидный или потерянный staging становится failed с безопасным invalid_image.
|
||||
Временные ошибки S3/БД/таймаута приводят к queued и следующей попытке.
|
||||
Backoff: min(30, 2^attempt) секунд плюс случайный jitter [0,1).
|
||||
По умолчанию JOB_MAX_ATTEMPTS=5, затем failed с retry_exhausted.
|
||||
Реестр failed находится в attachment_jobs, журнал в attachment_job_events.
|
||||
Секреты, внутренние адреса и stack trace не выдаются клиенту.
|
||||
|
||||
```sh
|
||||
make replay JOB_ID=<UUID задания>
|
||||
```
|
||||
|
||||
Replay разрешён только для failed с ещё сохранённым staging. Он открывает
|
||||
новый цикл попыток, увеличивает replay_count и сохраняет audit-событие;
|
||||
идентификатор логического задания и ключ результата остаются прежними.
|
||||
Replay уже ready является no-op. Контроль доступа к replay осуществляется
|
||||
операторским доступом к Docker, публичного HTTP endpoint replay нет.
|
||||
|
||||
После гибели worker просроченное processing восстанавливается dispatcher.
|
||||
Прежде чем менять его, проверяется advisory lock: работа живого владельца
|
||||
не захватывается вторым worker. Lease ограничивает восстановление зависшего
|
||||
состояния; нативный subprocess имеет отдельный принудительный timeout.
|
||||
Потерянная доставка queued восстанавливается из сохранённого намерения по
|
||||
истечении 30 s. Это at-least-once доставка с идемпотентным эффектом, не exactly-once.
|
||||
|
||||
## Очистка staging
|
||||
|
||||
После ready dispatcher удаляет staging только под блокировкой job и после
|
||||
проверки устойчивого завершения. Отказ между DELETE и отметкой повторяется
|
||||
безопасно. Failed staging сохраняется 86400 секунд для replay; минимум
|
||||
retention 3600. Текущие queued/processing не очищаются.
|
||||
|
||||
```sh
|
||||
make reconcile
|
||||
./scripts/compose.sh exec -T dispatcher dotnet Messenger.Api.dll --reconcile --apply
|
||||
```
|
||||
|
||||
Сверка untracked staging имеет grace 1 час и общую блокировку с приёмом upload.
|
||||
Она не удаляет staging, известный job, в том числе текущую работу. Это закрывает
|
||||
сбой S3 PUT до COMMIT приёма. Автоматически копировать отсутствующие байты
|
||||
без резервной копии невозможно; такое задание получает failed.
|
||||
|
||||
## Проверки
|
||||
|
||||
| Критерий | Сценарий | Доказательство |
|
||||
| --- | --- | --- |
|
||||
| Весь обязательный HTTP, 202, оригинал, thumbnail | make test | evidence/checks.txt |
|
||||
| Worker остановлен, API и broker перезапущены | make student-test | evidence/pipeline.json |
|
||||
| Сохранность опубликованного задания в broker volume | make student-test | evidence/pipeline.json |
|
||||
| Crop/WebP, EXIF orientation, alpha, отсутствие EXIF | make student-test | evidence/pipeline.json |
|
||||
| Bytes/pixels/MIME/filename и worker pixel guard | make student-test | evidence/pipeline.json |
|
||||
| COMMIT без publish, confirm без отметки outbox | make student-test | evidence/pipeline.json |
|
||||
| Повтор готовой job, crash после S3 и после COMMIT до ack | make student-test | evidence/pipeline.json |
|
||||
| Ограниченные retries, failed, явный replay и no-op ready | make student-test | evidence/pipeline.json |
|
||||
| Очистка готового staging и retention failed | make student-test | evidence/pipeline.json |
|
||||
| Три настройки, два размера, lag/p95/RSS | make research-grade5 | evidence/overload.json, docs/OVERLOAD.md |
|
||||
| Backpressure и восстановление принятой работы | make research-grade5 | evidence/overload.json |
|
||||
| Сессия и оригинал после restart всех хранилищ | make test-persistence | evidence/persistence.txt |
|
||||
| Переключение API и общая идемпотентность | make test-failover | evidence/failover.txt |
|
||||
|
||||
Fault injection включается только оператором через PIPELINE_FAULT_FILE для
|
||||
worker/dispatcher. Локальный файл содержит выбранный job ID и точку остановки;
|
||||
он одноразовый. Публичный клиент не может включить паузу. Тест ждёт marker,
|
||||
убивает настоящий контейнер и проверяет восстановление. По умолчанию путь
|
||||
пустой и инструментация не исполняется. Тесты создают только синтетические данные.
|
||||
|
||||
## Исследование
|
||||
|
||||
Гипотеза: при одинаковых CPU/RAM рост concurrency может повысить расход памяти
|
||||
без пропорционального роста скорости. Методика: 1/1, 2/2, 4/4 для concurrency /
|
||||
prefetch; два размера PNG; три повтора каждой точки. Порядок меняется между
|
||||
повторами. Каждая точка прогревается одним изображением. Затем batch 8 файлов,
|
||||
четыре HTTP-клиента, ожидание ready всех восьми. Метрики и ограничения измерения
|
||||
описаны в docs/OVERLOAD.md, индивидуальные job/HTTP и выборки в overload.json.
|
||||
|
||||
Отдельная серия останавливает worker, устанавливает лимит 4 и отправляет
|
||||
8 файлов одновременно. Проверяются ровно 4 принятых и 4 отклонённых с 429,
|
||||
непревышение лимита в БД и последующая готовность всех принятых после запуска.
|
||||
Для крупных PNG медиана throughput составила 0.808, 0.951 и 0.813 готовых/с
|
||||
для профилей 1/1, 2/2 и 4/4. Их медианы p95 готовности: 7.693, 6.520 и
|
||||
8.308 s; RSS процессов worker: 189.7, 268.6 и 378.8 MiB. У маленьких PNG
|
||||
лучше оказался 1/1. Рост concurrency не оказался универсальным ускорением.
|
||||
Профили меняют concurrency и prefetch вместе; отдельный эффект каждого параметра
|
||||
из этой серии не выводится. По умолчанию оставлено 2/1 как ограниченный
|
||||
рабочий вариант; этот профиль отдельно не измерен и не объявляется оптимальным.
|
||||
|
||||
Пилотная серия выявила нехватку свободных томов S3 при настройке volume.max=8,
|
||||
перенесённой из маленького стенда Lab4. До окончательных измерений лимит
|
||||
поднят до 64; volumes сохранены, неудавшиеся пилотные задания восстановлены
|
||||
явным replay. Пилотные результаты отделены от основной серии и не включаются
|
||||
в её статистику. Это изменение ёмкости хранилища, не результат ускорения worker.
|
||||
|
||||
Отдельный запуск публичного smoke во время намеренного рестарта API получил
|
||||
503; этот вывод сохранён в evidence/checks-during-restart.txt. Финальные проверки
|
||||
на готовом стенде записаны в evidence/checks.txt.
|
||||
|
||||
## Ограничения и настройки
|
||||
|
||||
Один Docker-host и единственные PostgreSQL, S3, RabbitMQ и HAProxy являются
|
||||
точками отказа. Несколько API и durable queue не защищают от потери host/диска.
|
||||
Для исходников и broker volume требуются резервные копии. Внешний доступ
|
||||
и настройка туннеля не входят в Lab5; наружу опубликован только localhost:18085.
|
||||
|
||||
Из .env.example используются APP_HOST/PORT, SESSION_TTL_SECONDS, API_REPLICAS,
|
||||
WORKER_CONCURRENCY, JOB_MAX_ATTEMPTS, JOB_TIMEOUT_SECONDS и STAGING_RETENTION_SECONDS.
|
||||
BROKER_URL_FILE сопоставлен BROKER_HOST и BROKER_PASSWORD_FILE.
|
||||
S3_ACCESS_KEY_ID_FILE/S3_SECRET_ACCESS_KEY_FILE сопоставлены S3_ACCESS_KEY_FILE/
|
||||
S3_SECRET_KEY_FILE. Endpoint S3 http://s3:8333, регион us-east-1, bucket messenger.
|
||||
Redis остаётся кэшем, PostgreSQL источником сессий. Byte/pixel/Idempotency TTL
|
||||
фиксированы контрактом в коде. Отдельно реализованы WORKER_PREFETCH и PIPELINE_CAPACITY.
|
||||
|
||||
Источники: контракт Lab5; официальная документация RabbitMQ publisher confirms,
|
||||
consumer acknowledgements и .NET Client API Guide; AWS SDK for .NET; SkiaSharp.
|
||||
ChatGPT использован при реализации и проверках.
|
||||
|
||||
## Панель обработки изображений
|
||||
|
||||
Кнопка «Обработка» под изображением открывает состояния, число попыток, время всего цикла и события задания из PostgreSQL. После готовности доступны оригинал, миниатюра, crop и WebP с размерами и скачиванием. Endpoint GET /api/v1/attachments/{id}/processing доступен только участникам чата; ключи S3 и служебные lease не возвращаются. Панель обновляется только пока открыта и задание выполняется. Проверка: .venv/bin/python student-tests/processing_panel.py. Результаты нагрузочного эксперимента выше относятся к версии до добавления этого интерфейса.
|
||||
|
||||
## Панель системы и внешний доступ
|
||||
|
||||
Кнопка «Система» доступна только аккаунту, UUID которого задан в OPS_USER_ID.
|
||||
После регистрации владельца: `make operator USERNAME=ваш_логин`. Команда записывает
|
||||
UUID в локальный .env и обновляет API. Если OPS_USER_ID не настроен, панель закрыта
|
||||
для всех. Клиентское скрытие кнопки дополнено проверкой каждой серверной операции.
|
||||
|
||||
Панель показывает две API-реплики по heartbeat Redis, PostgreSQL, Redis, S3,
|
||||
RabbitMQ, worker и dispatcher. Heartbeat процессов хранится в pipeline_presence;
|
||||
срок свежести 10 секунд. Это признак работающего процесса, а не гарантия успешной
|
||||
обработки любого задания. Проверка HAProxy основана на ответе текущего адреса.
|
||||
Графики содержат последние 60 замеров открытой панели: активные задания в БД
|
||||
и длительность запроса проверки зависимостей. CPU и память здесь не измеряются.
|
||||
При ошибке запроса старые состояния отмечаются как неизвестные.
|
||||
|
||||
Последние 30 заданий и событий доступны владельцу. Кнопка «Повторить» использует
|
||||
существующий механизм replay: только failed с сохранённым staging. Проверка состояния
|
||||
и изменение задания выполняются в одной транзакции под advisory lock; второй
|
||||
запрос не создаёт ещё один replay. Обычный пользователь получает 404, анонимный 401.
|
||||
Проверка: `OPS_TEST_PASSWORD=пароль_владельца make operations-test` (логин по умолчанию
|
||||
n1k0der, другой задаётся OPS_TEST_USERNAME). Тест использует контролируемый failed
|
||||
fixture и проверяет завершение настоящим worker, восстанавливая worker в finally.
|
||||
|
||||
Создать чат можно по @логину или UUID. Поиск выполняется только по точному логину,
|
||||
после входа; возвращает публичный User DTO без password_hash.
|
||||
|
||||
Временный внешний доступ: установить cloudflared, затем `make share`.
|
||||
Отключение: `make stop-share`. Ссылка и состояние процесса остаются в .local вне Git.
|
||||
Ссылка работает, пока включён компьютер, запущены Docker и туннель; при следующем
|
||||
запуске адрес может измениться. Снаружи доступен только HTTP-вход мессенджера,
|
||||
порты БД и RabbitMQ не публикуются. Собеседник регистрирует свой аккаунт и создаёт
|
||||
чат с @логином владельца. Учётная запись собеседника не получает права оператора.
|
||||
|
||||
Через HTTPS-туннель пустой POST входа может приходить с Transfer-Encoding: chunked.
|
||||
API проверяет фактическое отсутствие тела (с таймаутом 3 секунды), а не отклоняет
|
||||
такой способ передачи целиком. Непустое тело по-прежнему возвращает 400.
|
||||
Проверка: `.venv/bin/python student-tests/chunked_login.py`.
|
||||
Внешняя регистрация, вход, точный поиск, создание общего чата и сообщение проверены
|
||||
через публичный HTTPS-адрес; результат без токенов записан в evidence/operations.txt.
|
||||
|
||||
## Оформление интерфейса
|
||||
|
||||
Общие стили мессенджера находятся в wwwroot/design.css, панель системы в
|
||||
operations.css и operations.js. Добавлены 16 SVG на общей сетке и спрайт,
|
||||
две новые иллюстрации персонажа в WebP с прозрачностью. Использование графики
|
||||
описано в docs/ASSETS.md. Изображения загружаются в размерах 320/640, без
|
||||
исходных PNG в webroot. На ширине 1366 и 390 проверены отсутствие горизонтальной
|
||||
прокрутки, загрузка иллюстраций, навигация панели и доступность поля сообщения.
|
||||
В браузере проверены реальные состояния восьми карточек API/зависимостей.
|
||||
SVG разобраны XML-парсером, все новые asset URL отвечают 200. make check,
|
||||
make validate и публичный smoke-сценарий прошли после изменений.
|
||||
|
||||
## Удобство и устойчивость интерфейса
|
||||
|
||||
В чате добавлены копирование текста, вставка PNG/JPEG из буфера и перенос файла
|
||||
мышью. Поиск чатов открывается сочетанием ⌘K/Ctrl+K, результаты можно очистить
|
||||
кнопкой или Escape. В панели задания фильтруются среди последних 30 записей;
|
||||
состояние кнопки повтора не перерисовывается без изменения данных. При смене
|
||||
пользователя содержимое панели удаляется из страницы. Анимации уважают настройку
|
||||
уменьшения движения.
|
||||
|
||||
Формы входа и регистрации ожидают загрузки скриптов; их резервный метод POST не
|
||||
помещает поля в адрес страницы. HAProxy проверяет API с тайм-аутом 2 секунды и
|
||||
интервалом 1 секунда. Docker healthcheck RabbitMQ проверяет TCP-порт 5672 без
|
||||
запуска Erlang-процесса на каждый опрос; проверка панели дополнительно выполняет
|
||||
AMQP-запрос. Лимит RabbitMQ по умолчанию 512 МиБ, настраивается BROKER_MEMORY.
|
||||
При прежней проверке регулярные Erlang-процессы исчерпывали лимит при запуске.
|
||||
|
||||
Результаты повторных проверок и переключения API: evidence/ui-reliability.txt.
|
||||
Предыдущие измерения нагрузки относятся к настройкам и версии, указанным в их
|
||||
исходных результатах; повторный эксперимент для новых параметров не проводился.
|
||||
|
||||
+263
@@ -0,0 +1,263 @@
|
||||
name: mtusi2026lab5
|
||||
|
||||
x-api: &api
|
||||
image: messenger-lab5:local
|
||||
build: .
|
||||
environment:
|
||||
OPS_USER_ID: ${OPS_USER_ID:-}
|
||||
BROKER_HOST: broker
|
||||
BROKER_PASSWORD_FILE: /run/secrets/broker_password
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
REDIS_URL: redis:6379
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
DB_POOL_SIZE: ${DB_POOL_SIZE:-20}
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_BUCKET: messenger
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret
|
||||
PIPELINE_CAPACITY: ${PIPELINE_CAPACITY:-128}
|
||||
MESSAGE_BATCHING: ${MESSAGE_BATCHING:-true}
|
||||
Logging__LogLevel__Default: Warning
|
||||
Logging__LogLevel__Microsoft.Hosting.Lifetime: Information
|
||||
secrets:
|
||||
- broker_password
|
||||
- database_url
|
||||
- s3_access
|
||||
- s3_secret
|
||||
cpus: ${API_CPUS:-1}
|
||||
mem_limit: ${API_MEMORY:-512m}
|
||||
stop_grace_period: 10s
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "--silent", "--max-time", "1", "http://127.0.0.1:8080/health/ready"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 5
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.6
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
cpus: ${POSTGRES_CPUS:-0.5}
|
||||
mem_limit: 512m
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
restart: unless-stopped
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "64mb", "--maxmemory-policy", "allkeys-lru"]
|
||||
cpus: 0.25
|
||||
mem_limit: 96m
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
restart: unless-stopped
|
||||
|
||||
migrate:
|
||||
image: messenger-lab5:local
|
||||
command: ["--migrate"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets:
|
||||
- database_url
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
restart: "no"
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:3.97
|
||||
command: ["server", "-dir=/data", "-ip=s3", "-master.volumeSizeLimitMB=100", "-volume.max=64", "-filer", "-s3", "-s3.config=/run/secrets/s3_config"]
|
||||
secrets:
|
||||
- s3_config
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q '403'"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 40
|
||||
start_period: 35s
|
||||
|
||||
s3-bootstrap:
|
||||
image: messenger-lab5:local
|
||||
command: ["--bootstrap"]
|
||||
environment:
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_admin_access
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_admin_secret
|
||||
secrets:
|
||||
- s3_admin_access
|
||||
- s3_admin_secret
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: on-failure
|
||||
healthcheck:
|
||||
disable: true
|
||||
ulimits:
|
||||
core: 0
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
|
||||
broker:
|
||||
image: rabbitmq:4.2.0-alpine
|
||||
hostname: broker
|
||||
environment:
|
||||
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2 +sbwt none"
|
||||
secrets:
|
||||
- source: broker_config
|
||||
target: /etc/rabbitmq/conf.d/20-messenger.conf
|
||||
volumes:
|
||||
- broker_data:/var/lib/rabbitmq
|
||||
cpus: 0.5
|
||||
mem_limit: ${BROKER_MEMORY:-512m}
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/5672'"]
|
||||
interval: 3s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
start_period: 15s
|
||||
|
||||
dispatcher:
|
||||
image: messenger-lab5:local
|
||||
command: ["--dispatcher"]
|
||||
environment: &pipeline-env
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_BUCKET: messenger
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret
|
||||
BROKER_HOST: broker
|
||||
BROKER_PASSWORD_FILE: /run/secrets/broker_password
|
||||
PIPELINE_FAULT_FILE: ${PIPELINE_FAULT_FILE:-}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-30}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
volumes:
|
||||
- ./.local/faults:/run/faults
|
||||
secrets: &pipeline-secrets
|
||||
- database_url
|
||||
- s3_access
|
||||
- s3_secret
|
||||
- broker_password
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
disable: true
|
||||
cpus: 0.25
|
||||
mem_limit: 192m
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
|
||||
worker:
|
||||
image: messenger-lab5:local
|
||||
command: ["--worker"]
|
||||
environment:
|
||||
<<: *pipeline-env
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
|
||||
volumes:
|
||||
- ./.local/faults:/run/faults
|
||||
secrets: *pipeline-secrets
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
disable: true
|
||||
cpus: 1
|
||||
mem_limit: 1024m
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
ulimits:
|
||||
core: 0
|
||||
|
||||
api:
|
||||
<<: *api
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
|
||||
lb:
|
||||
image: haproxy:3.2.7-alpine
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
- "127.0.0.1:${API_PORT:-18085}:8080"
|
||||
cpus: 0.5
|
||||
mem_limit: 96m
|
||||
stop_grace_period: 10s
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
|
||||
secrets:
|
||||
broker_config:
|
||||
file: ./secrets/broker.conf
|
||||
broker_password:
|
||||
file: ./secrets/broker_password
|
||||
s3_config:
|
||||
file: ./secrets/s3_config
|
||||
s3_admin_access:
|
||||
file: ./secrets/s3_admin_access
|
||||
s3_admin_secret:
|
||||
file: ./secrets/s3_admin_secret
|
||||
s3_access:
|
||||
file: ./secrets/s3_access
|
||||
s3_secret:
|
||||
file: ./secrets/s3_secret
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
|
||||
volumes:
|
||||
broker_data:
|
||||
s3_data:
|
||||
postgres_data:
|
||||
@@ -0,0 +1,38 @@
|
||||
# Переход Lab4 → Lab5
|
||||
|
||||
| Часть | Lab4 | Lab5 |
|
||||
| --- | --- | --- |
|
||||
| Приём файла | S3 PUT и 201 ready | Приватный staging, метаданные, job и outbox, затем 202 queued |
|
||||
| Преобразования | Нет | Worker: JPEG thumbnail, JPEG crop, WebP optimized |
|
||||
| Брокер | Нет | RabbitMQ, durable quorum queue, persistent messages, publisher confirms |
|
||||
| Надёжность публикации | Не требуется | OutboxDispatcher читает PostgreSQL и публикует с подтверждением |
|
||||
| Повторная доставка | Только отправка сообщений | Дополнительно job_id, состояние, advisory lock, lease, детерминированные S3 keys |
|
||||
| Ошибки | Синхронный HTTP | Ограниченные попытки, backoff/jitter, failed registry и явный replay |
|
||||
| Интерфейс | Превью готового оригинала | Опрос queued/processing, затем прежняя галерея и скачивание |
|
||||
|
||||
## Процессы
|
||||
|
||||
Один образ C# запускается в нескольких ролях. API обрабатывает HTTP. Worker
|
||||
запускается с --worker, dispatcher с --dispatcher. Это отдельные контейнеры
|
||||
и процессы. Миграции и создание bucket выполняются одноразовыми сервисами.
|
||||
Worker запускает короткоживущий дочерний процесс --render-image для SkiaSharp:
|
||||
его можно принудительно остановить при превышении времени декодирования.
|
||||
|
||||
PostgreSQL хранит метаданные, состояния, аренду работы, outbox и журнал событий.
|
||||
SeaweedFS хранит staging, оригинал и варианты. Redis сохраняет прежнюю роль
|
||||
кэша сессий. RabbitMQ переносит только идентификаторы и служебные поля.
|
||||
|
||||
## Файлы
|
||||
|
||||
- Services/AttachmentService.cs: приём, проверка, лимит очереди, staging и транзакция приёма.
|
||||
- Data/Migrations/005_pipeline.sql: jobs, outbox, журнал, новые статусы и варианты.
|
||||
- Pipeline/OutboxDispatcher.cs: надёжная публикация в RabbitMQ.
|
||||
- Pipeline/PipelineWorker.cs: получение, ограниченная обработка, запись S3 и результата.
|
||||
- Pipeline/ImageRenderer.cs: EXIF orientation, resize, crop, JPEG/WebP.
|
||||
- Pipeline/JobStore.cs: захват работы, lease, повтор, восстановление и replay.
|
||||
- Pipeline/PipelineMaintenance.cs: очистка staging с проверкой состояния.
|
||||
- Pipeline/FaultInjection.cs: выключенные по умолчанию локальные точки остановки для тестов.
|
||||
- wwwroot/attachments.js: ожидание обработки и просмотр изображения.
|
||||
|
||||
Program.cs выбирает режим процесса и подключает зависимости. HTTP-контракт,
|
||||
авторизация, доступ к чатам и идемпотентность сообщений продолжают работать.
|
||||
@@ -0,0 +1,41 @@
|
||||
# Графика интерфейса
|
||||
|
||||
Набор: src/Messenger.Api/wwwroot/assets/.
|
||||
|
||||
## SVG
|
||||
|
||||
16 отдельных файлов в icons/: api, database, queue, storage, worker, dispatcher,
|
||||
cache, proxy, browser, chat, image, activity, check, clock, error, overview.
|
||||
Размерная сетка 48×48, обводка 2, скруглённые окончания, прозрачный фон.
|
||||
Для обычной картинки: `<img src="/assets/icons/database.svg" alt="База данных">`.
|
||||
Для управления цветом через CSS используйте спрайт:
|
||||
|
||||
```html
|
||||
<svg viewBox="0 0 48 48" width="24" height="24" aria-hidden="true"
|
||||
style="color:#FFE135">
|
||||
<use href="/assets/system-icons.svg#database"></use>
|
||||
</svg>
|
||||
```
|
||||
|
||||
Файл system-icons.svg должен находиться на том же origin. В другом проекте
|
||||
измените путь к спрайту. Для кнопки задайте текст или aria-label; декоративному
|
||||
SVG оставьте aria-hidden. Исходные SVG не зависят от JavaScript или библиотеки.
|
||||
conversation-grid.svg используется как малоконтрастный фон переписки.
|
||||
|
||||
## Иллюстрации
|
||||
|
||||
kolovertysh-message-320.webp / kolovertysh-message-640.webp: пустой экран чатов
|
||||
и боковая панель системы. kolovertysh-reconnect-320.webp: ошибка подключения.
|
||||
Все изображения содержат alpha; прозрачность сохранена при экспорте.
|
||||
Созданы с помощью ChatGPT по существующему образу персонажа. Для UI используются
|
||||
уменьшенные WebP, исходные PNG оставлены локально вне webroot.
|
||||
|
||||
## Цвета и движение
|
||||
|
||||
Основной фон #262626, акцент #FFE135, основной текст #EEEEE2.
|
||||
Рабочие поверхности #282A25 и #30312C, вторичный текст #A8AC9B.
|
||||
Состояния: зелёный #A6CF9A, ошибка #EEB09F.
|
||||
Анимация точек между узлами запускается после реального ответа панели.
|
||||
Выбранный API определяется по X-Instance-Id. При prefers-reduced-motion
|
||||
переходы, появление окон и движение точек отключены. Графики строят прямые
|
||||
отрезки между измерениями, без сглаживания или искусственных данных.
|
||||
@@ -0,0 +1,35 @@
|
||||
# Очередь под нагрузкой
|
||||
|
||||
Три повтора; одинаковые CPU/RAM worker; batch 8, четыре HTTP-клиента.
|
||||
|
||||
| Файл | Concurrency / prefetch | Готовых/с, медиана | p95 до ready, s | p95 ожидания, s | RSS процессов, MiB |
|
||||
| --- | --- | ---: | ---: | ---: | ---: |
|
||||
| small | 1/1 | 4.411 | 1.524 | 1.291 | 157.6 |
|
||||
| small | 2/2 | 3.625 | 1.675 | 1.319 | 165.2 |
|
||||
| small | 4/4 | 3.904 | 1.788 | 1.052 | 281.9 |
|
||||
| large | 1/1 | 0.808 | 7.693 | 6.868 | 189.7 |
|
||||
| large | 2/2 | 0.951 | 6.520 | 4.749 | 268.6 |
|
||||
| large | 4/4 | 0.813 | 8.308 | 4.477 | 378.8 |
|
||||
|
||||
p95: nearest rank; при batch 8 это максимальная задержка. RSS: сумма VmRSS
|
||||
процессов dotnet внутри worker, включая дочерние декодеры; общие страницы могут
|
||||
считаться несколько раз. memory.current измерен отдельно. Период выборок 200 ms
|
||||
плюс docker exec; короткие пики могут быть пропущены. Время включает загрузку
|
||||
batch и ожидание последнего ready, без регистрации и прогрева. Queue lag: от
|
||||
создания записи задания до первой processing. Время до ready: по часам PostgreSQL.
|
||||
|
||||
Backpressure: при остановленном worker лимит 4 принял ровно 4 из 8 файлов;
|
||||
остальные получили 429. После запуска обработались все четыре принятых задания.
|
||||
|
||||
Исходные результаты: evidence/overload.json. Один Docker-host, небольшие batch,
|
||||
параллельно могут работать другие локальные приложения. Эти значения не являются
|
||||
пределом производительности сервера. Вывод сравнения относится к этому профилю.
|
||||
|
||||

|
||||
|
||||
В этом профиле для small один обработчик оказался быстрее двух и четырёх.
|
||||
Для large вариант 2/2 дал лучшую медиану throughput и меньшую p95 готовности,
|
||||
чем 1/1, при большем RSS. Четыре обработчика уменьшили время ожидания начала,
|
||||
но увеличили полное время обработки и память. По умолчанию оставлены 2/1:
|
||||
два параллельных обработчика с небольшим prefetch. Пара 2/1 отдельно не входит
|
||||
в сравнение, поэтому не заявляется как измеренный оптимум.
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
ЛР 5 - БВТ-2302 - Каменский - на 5
|
||||
|
||||
Автор / группа: Каменский Никита, БВТ-2302.
|
||||
Целевая оценка: 5. Трек 1: очередь под перегрузкой.
|
||||
|
||||
Из Lab4 перенесены собственные API, PostgreSQL, Redis, S3, HAProxy и интерфейс.
|
||||
Загрузка теперь возвращает 202 queued после сохранения staging и транзакции
|
||||
Attachment/job/outbox. Отдельные RabbitMQ, dispatcher и worker выполняют
|
||||
доставку и обработку: thumbnail JPEG, crop JPEG, optimized WebP.
|
||||
|
||||
Добавлены ограниченные retries с backoff/jitter, failed registry, replay,
|
||||
lease recovery и очистка staging после устойчивого результата. Дубли не
|
||||
создают второй набор вариантов. UI показывает ожидание и обработку изображения.
|
||||
|
||||
Чистый запуск: make setup, make up. Проверки: make check, make validate,
|
||||
make test, make student-test, make research-grade5; persistence/failover
|
||||
по REPORT.md. Результаты: evidence/, docs/OVERLOAD.md и REPORT.md.
|
||||
|
||||
Публичный контракт, tests и README задания сохранены. Один Docker-host,
|
||||
единственные БД, S3, RabbitMQ и HAProxy остаются точками отказа. Измерения
|
||||
получены на локальном стенде с ограниченными ресурсами и небольшими batch.
|
||||
|
||||
ChatGPT использован при реализации и проверках.
|
||||
@@ -0,0 +1,9 @@
|
||||
.venv/bin/python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
.venv/bin/python tests/smoke.py --base-url "http://localhost:18085"
|
||||
FAIL: GET /api/v1/attachments/bb21267f-b7da-491b-96b3-7f55f8c3d884: expected [200], received 503
|
||||
make: *** [test] Error 1
|
||||
.venv/bin/python tests/smoke.py --base-url "http://localhost:18085"
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,6 @@
|
||||
.venv/bin/python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
.venv/bin/python tests/smoke.py --base-url "http://localhost:18085"
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,67 @@
|
||||
Local secrets ready
|
||||
#0 building with "desktop-linux" instance using docker driver
|
||||
|
||||
#1 [internal] load build definition from Dockerfile
|
||||
#1 transferring dockerfile: 969B done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [internal] load metadata for mcr.microsoft.com/dotnet/sdk:9.0.306
|
||||
#2 DONE 0.6s
|
||||
|
||||
#3 [internal] load metadata for mcr.microsoft.com/dotnet/aspnet:9.0.10
|
||||
#3 DONE 0.6s
|
||||
|
||||
#4 [internal] load .dockerignore
|
||||
#4 transferring context: 218B done
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [build 1/6] FROM mcr.microsoft.com/dotnet/sdk:9.0.306@sha256:81f6d622fe21ed9d31375167f62a3538ff4d6835f9d5e6da9c2defa8a84b7687
|
||||
#5 resolve mcr.microsoft.com/dotnet/sdk:9.0.306@sha256:81f6d622fe21ed9d31375167f62a3538ff4d6835f9d5e6da9c2defa8a84b7687 0.0s done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [runtime 1/4] FROM mcr.microsoft.com/dotnet/aspnet:9.0.10@sha256:3dcb33395722d14c80d19107158293ed677b2c07841100d51df07275ae2b2682
|
||||
#6 resolve mcr.microsoft.com/dotnet/aspnet:9.0.10@sha256:3dcb33395722d14c80d19107158293ed677b2c07841100d51df07275ae2b2682 0.0s done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [internal] load build context
|
||||
#7 transferring context: 420.67kB 0.0s done
|
||||
#7 DONE 0.0s
|
||||
|
||||
#8 [build 3/6] COPY src/Messenger.Api/Messenger.Api.csproj src/Messenger.Api/
|
||||
#8 CACHED
|
||||
|
||||
#9 [runtime 2/4] RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
|
||||
#9 CACHED
|
||||
|
||||
#10 [runtime 3/4] WORKDIR /app
|
||||
#10 CACHED
|
||||
|
||||
#11 [build 4/6] RUN dotnet restore src/Messenger.Api/Messenger.Api.csproj
|
||||
#11 CACHED
|
||||
|
||||
#12 [build 6/6] RUN dotnet publish src/Messenger.Api/Messenger.Api.csproj -c Release --no-restore -o /app/publish
|
||||
#12 CACHED
|
||||
|
||||
#13 [build 2/6] WORKDIR /src
|
||||
#13 CACHED
|
||||
|
||||
#14 [build 5/6] COPY src/Messenger.Api/ src/Messenger.Api/
|
||||
#14 CACHED
|
||||
|
||||
#15 [runtime 4/4] COPY --from=build /app/publish .
|
||||
#15 CACHED
|
||||
|
||||
#16 exporting to image
|
||||
#16 exporting layers done
|
||||
#16 exporting manifest sha256:8032c23ca82abca0a98189e25912a472fa9f9766aaadfb1dc72e3a8ffd87797d done
|
||||
#16 exporting config sha256:6d5f902dbe672c56753a47a14e780cedd15d0a6134b91f7e1bfcb0da75d784e6 0.0s done
|
||||
#16 exporting attestation manifest sha256:1c91c52f053bf7e2022c725dfdc1906fc181127ae049e472f4f61cbc5255c452 0.0s done
|
||||
#16 exporting manifest list sha256:bc7b3964de3dcbf7f81a4313518695817ae781561b99f22cf02e8114b41271ba 0.0s done
|
||||
#16 naming to docker.io/library/messenger-lab5:clean-check done
|
||||
#16 unpacking to docker.io/library/messenger-lab5:clean-check
|
||||
#16 unpacking to docker.io/library/messenger-lab5:clean-check done
|
||||
#16 DONE 0.1s
|
||||
|
||||
View build details: docker-desktop://dashboard/build/desktop-linux/desktop-linux/mwlwenbjkzp9ukzg8a92tezj0
|
||||
|
||||
PASS: isolated source context, fresh generated secrets, compose config and Docker build. No existing .env, volumes or build outputs copied.
|
||||
@@ -0,0 +1,55 @@
|
||||
.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18085"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 118, "failed_attempts": 1, "first_recovery_seconds": 1.87}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab5-api-2 messenger-lab5:local "dotnet Messenger.Ap…" api 3 minutes ago Up 30 seconds (healthy) 8080/tcp
|
||||
mtusi2026lab5-broker-1 rabbitmq:4.2.0-alpine "docker-entrypoint.s…" broker 15 minutes ago Up 54 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
mtusi2026lab5-dispatcher-1 messenger-lab5:local "dotnet Messenger.Ap…" dispatcher About a minute ago Up 30 seconds 8080/tcp
|
||||
mtusi2026lab5-lb-1 haproxy:3.2.7-alpine "docker-entrypoint.s…" lb 15 minutes ago Up 30 seconds (healthy) 127.0.0.1:18085->8080/tcp
|
||||
mtusi2026lab5-postgres-1 postgres:17.6 "docker-entrypoint.s…" postgres 15 minutes ago Up 55 seconds (healthy) 5432/tcp
|
||||
mtusi2026lab5-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 15 minutes ago Up 55 seconds (healthy) 6379/tcp
|
||||
mtusi2026lab5-s3-1 chrislusf/seaweedfs:3.97 "/entrypoint.sh serv…" s3 7 minutes ago Up 52 seconds (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
mtusi2026lab5-worker-1 messenger-lab5:local "dotnet Messenger.Ap…" worker About a minute ago Up 30 seconds 8080/tcp
|
||||
./scripts/compose.sh up -d --wait api lb
|
||||
Container mtusi2026lab5-redis-1 Running
|
||||
Container mtusi2026lab5-api-2 Running
|
||||
Container mtusi2026lab5-s3-1 Running
|
||||
Container mtusi2026lab5-lb-1 Running
|
||||
Container mtusi2026lab5-postgres-1 Running
|
||||
Container mtusi2026lab5-s3-1 Waiting
|
||||
Container mtusi2026lab5-postgres-1 Waiting
|
||||
Container mtusi2026lab5-s3-1 Healthy
|
||||
Container mtusi2026lab5-postgres-1 Healthy
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Starting
|
||||
Container mtusi2026lab5-migrate-1 Starting
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Started
|
||||
Container mtusi2026lab5-migrate-1 Started
|
||||
Container mtusi2026lab5-migrate-1 Waiting
|
||||
Container mtusi2026lab5-redis-1 Waiting
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-redis-1 Healthy
|
||||
Container mtusi2026lab5-migrate-1 Exited
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-api-1 Starting
|
||||
Container mtusi2026lab5-api-1 Started
|
||||
Container mtusi2026lab5-api-2 Waiting
|
||||
Container mtusi2026lab5-api-1 Waiting
|
||||
Container mtusi2026lab5-api-2 Healthy
|
||||
Container mtusi2026lab5-api-1 Healthy
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab5-postgres-1 Waiting
|
||||
Container mtusi2026lab5-redis-1 Waiting
|
||||
Container mtusi2026lab5-api-2 Waiting
|
||||
Container mtusi2026lab5-api-1 Waiting
|
||||
Container mtusi2026lab5-migrate-1 Waiting
|
||||
Container mtusi2026lab5-s3-1 Waiting
|
||||
Container mtusi2026lab5-lb-1 Waiting
|
||||
Container mtusi2026lab5-migrate-1 Exited
|
||||
Container mtusi2026lab5-lb-1 Healthy
|
||||
Container mtusi2026lab5-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab5-redis-1 Healthy
|
||||
Container mtusi2026lab5-s3-1 Healthy
|
||||
Container mtusi2026lab5-postgres-1 Healthy
|
||||
Container mtusi2026lab5-api-2 Healthy
|
||||
Container mtusi2026lab5-api-1 Healthy
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"job_id": "3590b5cc-4392-4c8e-819f-a44e67ca7c64",
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": "80fc41c9-e25b-4bcb-8475-e08d62822e78",
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": "2026-11-09T22:22:38.7816100Z"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
Проверки дополнения панели системы:
|
||||
- Сборка .NET: 0 ошибок, 0 предупреждений.
|
||||
- make check, make validate, make test: PASS.
|
||||
- operations_panel.py: PASS: operator-only metrics and replay; anonymous 401; regular user 404; exact login lookup; one replay completes with real worker.
|
||||
- Повторный POST replay для queued: 409; replay_count остался 1.
|
||||
- Интерфейс проверен в браузере: экземпляры API, службы, графики, события и повторный запуск доступны владельцу.
|
||||
Первый прогон теста после рестарта API встретил временный 503 при ожидании ready; само задание завершилось. Ожидание учитывает 503 до общего таймаута, финальное состояние ready остаётся обязательным.
|
||||
PASS: public HTTPS registration, login, lookup @n1k0der, shared chat and message; operations denied to guest
|
||||
PASS: empty chunked login accepted; nonempty chunked login rejected (student-tests/chunked_login.py).
|
||||
@@ -0,0 +1,20 @@
|
||||
.venv/bin/python student-tests/overload_research.py --base-url "http://localhost:18085"
|
||||
run 1/18: small, 1/1, 4.41 ready/s
|
||||
run 2/18: small, 2/2, 3.62 ready/s
|
||||
run 3/18: small, 4/4, 3.91 ready/s
|
||||
run 4/18: large, 1/1, 0.78 ready/s
|
||||
run 5/18: large, 2/2, 0.95 ready/s
|
||||
run 6/18: large, 4/4, 0.81 ready/s
|
||||
run 7/18: large, 2/2, 0.99 ready/s
|
||||
run 8/18: large, 4/4, 0.97 ready/s
|
||||
run 9/18: large, 1/1, 0.88 ready/s
|
||||
run 10/18: small, 2/2, 3.57 ready/s
|
||||
run 11/18: small, 4/4, 3.62 ready/s
|
||||
run 12/18: small, 1/1, 4.65 ready/s
|
||||
run 13/18: small, 4/4, 3.90 ready/s
|
||||
run 14/18: small, 1/1, 3.86 ready/s
|
||||
run 15/18: small, 2/2, 4.66 ready/s
|
||||
run 16/18: large, 4/4, 0.69 ready/s
|
||||
run 17/18: large, 1/1, 0.81 ready/s
|
||||
run 18/18: large, 2/2, 0.73 ready/s
|
||||
PASS: overload research and bounded admission
|
||||
@@ -0,0 +1,14 @@
|
||||
.venv/bin/python student-tests/overload_research.py --base-url "http://localhost:18085"
|
||||
run 1/18: small, 1/1, 4.60 ready/s
|
||||
run 2/18: small, 2/2, 4.06 ready/s
|
||||
run 3/18: small, 4/4, 0.84 ready/s
|
||||
run 4/18: large, 1/1, 0.86 ready/s
|
||||
FAIL: Infrastructure command failed: 2cad4d5d_mtusi2026lab5-api-2 Stopping
|
||||
Container 74e89bf6ba55_mtusi2026lab5-worker-1 Recreate
|
||||
Container 5c6d2cad4d5d_mtusi2026lab5-api-2 Stopped
|
||||
Container 5c6d2cad4d5d_mtusi2026lab5-api-2 Removing
|
||||
Container mtusi2026lab5-api-2 Error response from daemon: No such container: 5c6d2cad4d5db9f150fce5fc6f432103bddadddad8b1bf0d5f723466384f4843
|
||||
Container mtusi2026lab5-api-2 Removing
|
||||
Error response from daemon: No such container: 5c6d2cad4d5db9f150fce5fc6f432103bddadddad8b1bf0d5f723466384f4843
|
||||
|
||||
make: *** [research-grade5] Error 1
|
||||
@@ -0,0 +1,693 @@
|
||||
{
|
||||
"hypothesis": "With fixed worker CPU, more simultaneous image processes need not improve throughput and can increase RSS; bounded admission protects the queue when the worker stops.",
|
||||
"profile": {
|
||||
"settings": [
|
||||
[
|
||||
1,
|
||||
1
|
||||
],
|
||||
[
|
||||
2,
|
||||
2
|
||||
],
|
||||
[
|
||||
4,
|
||||
4
|
||||
]
|
||||
],
|
||||
"repeats": 3,
|
||||
"batch": 8,
|
||||
"upload_clients": 4,
|
||||
"worker_cpu": 1,
|
||||
"worker_memory_limit_mib": 1024,
|
||||
"warmup_images": 1,
|
||||
"image_sizes_bytes": {
|
||||
"small": 111817,
|
||||
"large": 3319965
|
||||
}
|
||||
},
|
||||
"runs": [
|
||||
{
|
||||
"repeat": 1,
|
||||
"image": "small",
|
||||
"concurrency": 1,
|
||||
"prefetch": 1,
|
||||
"successful_jobs": 8,
|
||||
"errors": 0,
|
||||
"seconds": 1.7385662909946404,
|
||||
"throughput": 4.601492644507199,
|
||||
"p95_ready_seconds": 1.355825,
|
||||
"p95_queue_lag_seconds": 1.221817,
|
||||
"peak_rss_bytes": 152162304,
|
||||
"peak_cgroup_memory_bytes": 53309440,
|
||||
"worker_cpu_seconds": 1.329851,
|
||||
"requests": [
|
||||
{
|
||||
"upload_seconds": 0.04113916699861875,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.10876779200043529,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.12739412499649916,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.1493414579963428,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.11392083399550756,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.05546583300019847,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.04235250000056112,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.02778862499690149,
|
||||
"status": 202
|
||||
}
|
||||
],
|
||||
"jobs": [
|
||||
{
|
||||
"id": "d2b88786-66d5-44f9-8398-afd799333b0d",
|
||||
"ready_seconds": 0.27748,
|
||||
"queue_lag_seconds": 0.115927,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "746abbed-2f2a-44a7-8054-8e2fb4e40d60",
|
||||
"ready_seconds": 0.405159,
|
||||
"queue_lag_seconds": 0.269961,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "a4fbcad3-77a5-402f-8ce9-a286c3c50998",
|
||||
"ready_seconds": 0.52426,
|
||||
"queue_lag_seconds": 0.395915,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "bbc83e93-f179-4f84-a5a1-4bfb334ecf5d",
|
||||
"ready_seconds": 0.629692,
|
||||
"queue_lag_seconds": 0.459954,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "8d5f7573-9a93-4f75-9871-fe57beaafa15",
|
||||
"ready_seconds": 0.909593,
|
||||
"queue_lag_seconds": 0.620331,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "8280ed8d-da55-40a0-bde3-36ee21893e72",
|
||||
"ready_seconds": 1.068921,
|
||||
"queue_lag_seconds": 0.902328,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "d8dbae97-6243-4f95-b9cc-4a3e8fdfefbb",
|
||||
"ready_seconds": 1.225459,
|
||||
"queue_lag_seconds": 1.068507,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "dcd72436-5215-4aca-900d-74767f0385b2",
|
||||
"ready_seconds": 1.355825,
|
||||
"queue_lag_seconds": 1.221817,
|
||||
"attempts": 1
|
||||
}
|
||||
],
|
||||
"samples": [
|
||||
{
|
||||
"time": 1794262118.743776,
|
||||
"memory_bytes": 42041344,
|
||||
"cpu_usec": 795694,
|
||||
"rss_bytes": 103636992
|
||||
},
|
||||
{
|
||||
"time": 1794262119.0674748,
|
||||
"memory_bytes": 48332800,
|
||||
"cpu_usec": 1134009,
|
||||
"rss_bytes": 127852544
|
||||
},
|
||||
{
|
||||
"time": 1794262119.545117,
|
||||
"memory_bytes": 50176000,
|
||||
"cpu_usec": 1544976,
|
||||
"rss_bytes": 152162304
|
||||
},
|
||||
{
|
||||
"time": 1794262119.89533,
|
||||
"memory_bytes": 53309440,
|
||||
"cpu_usec": 1889921,
|
||||
"rss_bytes": 147939328
|
||||
},
|
||||
{
|
||||
"time": 1794262120.2800698,
|
||||
"memory_bytes": 46026752,
|
||||
"cpu_usec": 2125545,
|
||||
"rss_bytes": 107089920
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"repeat": 1,
|
||||
"image": "small",
|
||||
"concurrency": 2,
|
||||
"prefetch": 2,
|
||||
"successful_jobs": 8,
|
||||
"errors": 0,
|
||||
"seconds": 1.969336124995607,
|
||||
"throughput": 4.062282663919469,
|
||||
"p95_ready_seconds": 1.525365,
|
||||
"p95_queue_lag_seconds": 1.016688,
|
||||
"peak_rss_bytes": 219119616,
|
||||
"peak_cgroup_memory_bytes": 62783488,
|
||||
"worker_cpu_seconds": 1.534243,
|
||||
"requests": [
|
||||
{
|
||||
"upload_seconds": 0.03250100000150269,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.044958750004298054,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.05604845799825853,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.06452524999622256,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.0502204169970355,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.04115625000122236,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.03952566699445015,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.0422521250002319,
|
||||
"status": 202
|
||||
}
|
||||
],
|
||||
"jobs": [
|
||||
{
|
||||
"id": "9353ef12-3388-4f17-a52e-7788d9631d5e",
|
||||
"ready_seconds": 0.394761,
|
||||
"queue_lag_seconds": 0.116528,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "5ff9fdf3-e9e6-446e-95c7-978c9b3fd22b",
|
||||
"ready_seconds": 0.404182,
|
||||
"queue_lag_seconds": 0.129876,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "314ef166-fd5e-4c85-abf7-15185aea7576",
|
||||
"ready_seconds": 0.692893,
|
||||
"queue_lag_seconds": 0.388414,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "52a91f37-b649-4896-8051-5f39802fb3ad",
|
||||
"ready_seconds": 0.751849,
|
||||
"queue_lag_seconds": 0.382157,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "052cd532-0568-47b6-a95c-e35bc68c6d81",
|
||||
"ready_seconds": 0.976998,
|
||||
"queue_lag_seconds": 0.669662,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "5df94c7d-97a6-4ca1-a3bb-c663ea5a7e41",
|
||||
"ready_seconds": 1.032436,
|
||||
"queue_lag_seconds": 0.738607,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "926d6d55-a9b6-408f-a7e2-2bf2b98871b0",
|
||||
"ready_seconds": 1.523118,
|
||||
"queue_lag_seconds": 1.005413,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "dad6cf86-7200-4468-b927-aca54ae14db9",
|
||||
"ready_seconds": 1.525365,
|
||||
"queue_lag_seconds": 1.016688,
|
||||
"attempts": 1
|
||||
}
|
||||
],
|
||||
"samples": [
|
||||
{
|
||||
"time": 1794262122.505246,
|
||||
"memory_bytes": 42008576,
|
||||
"cpu_usec": 782744,
|
||||
"rss_bytes": 103116800
|
||||
},
|
||||
{
|
||||
"time": 1794262122.863539,
|
||||
"memory_bytes": 54919168,
|
||||
"cpu_usec": 1204770,
|
||||
"rss_bytes": 161333248
|
||||
},
|
||||
{
|
||||
"time": 1794262123.2467399,
|
||||
"memory_bytes": 57008128,
|
||||
"cpu_usec": 1566113,
|
||||
"rss_bytes": 175915008
|
||||
},
|
||||
{
|
||||
"time": 1794262123.8038552,
|
||||
"memory_bytes": 62783488,
|
||||
"cpu_usec": 2036485,
|
||||
"rss_bytes": 219119616
|
||||
},
|
||||
{
|
||||
"time": 1794262124.115168,
|
||||
"memory_bytes": 45998080,
|
||||
"cpu_usec": 2316987,
|
||||
"rss_bytes": 106729472
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"repeat": 1,
|
||||
"image": "small",
|
||||
"concurrency": 4,
|
||||
"prefetch": 4,
|
||||
"successful_jobs": 8,
|
||||
"errors": 0,
|
||||
"seconds": 9.549317416000122,
|
||||
"throughput": 0.8377562135065066,
|
||||
"p95_ready_seconds": 9.209337,
|
||||
"p95_queue_lag_seconds": 7.286448,
|
||||
"peak_rss_bytes": 289599488,
|
||||
"peak_cgroup_memory_bytes": 87416832,
|
||||
"worker_cpu_seconds": 5.107425,
|
||||
"requests": [
|
||||
{
|
||||
"upload_seconds": 0.03872108399809804,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.06579587499436457,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.08007283299957635,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.09368379199440824,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.040345542001887225,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.07635662500251783,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.032701916999940295,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.05977749999874504,
|
||||
"status": 202
|
||||
}
|
||||
],
|
||||
"jobs": [
|
||||
{
|
||||
"id": "c70a41eb-b3da-40f7-9669-fbc0adf914d1",
|
||||
"ready_seconds": 9.074966,
|
||||
"queue_lag_seconds": 7.278132,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "e799ce82-e00a-4e15-aacc-daa6fa757335",
|
||||
"ready_seconds": 9.150763,
|
||||
"queue_lag_seconds": 7.224741,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "d248b714-1eeb-4668-b497-4a30b3b0aad8",
|
||||
"ready_seconds": 9.16808,
|
||||
"queue_lag_seconds": 7.267137,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "a540b150-be0e-4d38-9077-92802293c12d",
|
||||
"ready_seconds": 9.209337,
|
||||
"queue_lag_seconds": 7.286448,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "d62a309a-a54e-4a7d-aebc-864fd251bed4",
|
||||
"ready_seconds": 7.164243,
|
||||
"queue_lag_seconds": 0.116455,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "063093ae-38a4-4fc4-a841-d3e5db5624b3",
|
||||
"ready_seconds": 7.164926,
|
||||
"queue_lag_seconds": 0.135483,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "06613954-2472-4efe-b2a7-42644a1d8a30",
|
||||
"ready_seconds": 7.149471,
|
||||
"queue_lag_seconds": 0.127605,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "f593fbcd-f166-4e3e-9d89-09faeaa592bb",
|
||||
"ready_seconds": 7.127115,
|
||||
"queue_lag_seconds": 0.132045,
|
||||
"attempts": 1
|
||||
}
|
||||
],
|
||||
"samples": [
|
||||
{
|
||||
"time": 1794262126.910836,
|
||||
"memory_bytes": 44044288,
|
||||
"cpu_usec": 1063223,
|
||||
"rss_bytes": 143413248
|
||||
},
|
||||
{
|
||||
"time": 1794262127.9235861,
|
||||
"memory_bytes": 87416832,
|
||||
"cpu_usec": 1873587,
|
||||
"rss_bytes": 284831744
|
||||
},
|
||||
{
|
||||
"time": 1794262130.018199,
|
||||
"memory_bytes": 42848256,
|
||||
"cpu_usec": 3399041,
|
||||
"rss_bytes": 106557440
|
||||
},
|
||||
{
|
||||
"time": 1794262130.52949,
|
||||
"memory_bytes": 43167744,
|
||||
"cpu_usec": 3541243,
|
||||
"rss_bytes": 106573824
|
||||
},
|
||||
{
|
||||
"time": 1794262131.362358,
|
||||
"memory_bytes": 42827776,
|
||||
"cpu_usec": 3715963,
|
||||
"rss_bytes": 106573824
|
||||
},
|
||||
{
|
||||
"time": 1794262133.449707,
|
||||
"memory_bytes": 44789760,
|
||||
"cpu_usec": 4203236,
|
||||
"rss_bytes": 108687360
|
||||
},
|
||||
{
|
||||
"time": 1794262133.8703759,
|
||||
"memory_bytes": 47022080,
|
||||
"cpu_usec": 4272996,
|
||||
"rss_bytes": 108298240
|
||||
},
|
||||
{
|
||||
"time": 1794262134.622085,
|
||||
"memory_bytes": 77737984,
|
||||
"cpu_usec": 5000530,
|
||||
"rss_bytes": 289599488
|
||||
},
|
||||
{
|
||||
"time": 1794262135.440066,
|
||||
"memory_bytes": 55914496,
|
||||
"cpu_usec": 5808140,
|
||||
"rss_bytes": 108367872
|
||||
},
|
||||
{
|
||||
"time": 1794262135.956406,
|
||||
"memory_bytes": 44863488,
|
||||
"cpu_usec": 6170648,
|
||||
"rss_bytes": 109416448
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"repeat": 1,
|
||||
"image": "large",
|
||||
"concurrency": 1,
|
||||
"prefetch": 1,
|
||||
"successful_jobs": 8,
|
||||
"errors": 0,
|
||||
"seconds": 9.295377750000625,
|
||||
"throughput": 0.8606428071198572,
|
||||
"p95_ready_seconds": 7.433391,
|
||||
"p95_queue_lag_seconds": 6.388493,
|
||||
"peak_rss_bytes": 193171456,
|
||||
"peak_cgroup_memory_bytes": 102318080,
|
||||
"worker_cpu_seconds": 7.074789,
|
||||
"requests": [
|
||||
{
|
||||
"upload_seconds": 0.7639207920001354,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.9345893749996321,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 1.1469821249993402,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 1.436994916999538,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.7003262910002377,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.9740857910001068,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.39281854200089583,
|
||||
"status": 202
|
||||
},
|
||||
{
|
||||
"upload_seconds": 0.8782962080003927,
|
||||
"status": 202
|
||||
}
|
||||
],
|
||||
"jobs": [
|
||||
{
|
||||
"id": "2be2644d-861c-43c4-8085-5fbba90a4d64",
|
||||
"ready_seconds": 1.712623,
|
||||
"queue_lag_seconds": 0.363585,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "ad0083b2-3bf8-4f42-8043-4a57ad7b84a9",
|
||||
"ready_seconds": 2.629908,
|
||||
"queue_lag_seconds": 1.699148,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "407c2a2e-e5aa-4437-ab5b-49a2ec05829b",
|
||||
"ready_seconds": 3.021709,
|
||||
"queue_lag_seconds": 2.407417,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "854309f9-5d4f-4305-ad65-bcb005b3844f",
|
||||
"ready_seconds": 3.852419,
|
||||
"queue_lag_seconds": 2.844126,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "fae78450-5058-4101-8cd9-4dcbcecd9e2c",
|
||||
"ready_seconds": 4.70857,
|
||||
"queue_lag_seconds": 3.753701,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "fe283bf1-cdff-4ee2-867d-369338758dd4",
|
||||
"ready_seconds": 5.265006,
|
||||
"queue_lag_seconds": 4.502767,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "86228db3-70a0-4118-bd9c-1f0aeb13ccf4",
|
||||
"ready_seconds": 6.477127,
|
||||
"queue_lag_seconds": 5.104128,
|
||||
"attempts": 1
|
||||
},
|
||||
{
|
||||
"id": "3fea2d31-ad7b-4282-b815-23adaa068c7a",
|
||||
"ready_seconds": 7.433391,
|
||||
"queue_lag_seconds": 6.388493,
|
||||
"attempts": 1
|
||||
}
|
||||
],
|
||||
"samples": [
|
||||
{
|
||||
"time": 1794262140.0249028,
|
||||
"memory_bytes": 41619456,
|
||||
"cpu_usec": 1623573,
|
||||
"rss_bytes": 106139648
|
||||
},
|
||||
{
|
||||
"time": 1794262140.4498188,
|
||||
"memory_bytes": 41545728,
|
||||
"cpu_usec": 1659252,
|
||||
"rss_bytes": 106139648
|
||||
},
|
||||
{
|
||||
"time": 1794262140.9149399,
|
||||
"memory_bytes": 60805120,
|
||||
"cpu_usec": 2022910,
|
||||
"rss_bytes": 150097920
|
||||
},
|
||||
{
|
||||
"time": 1794262141.32444,
|
||||
"memory_bytes": 88137728,
|
||||
"cpu_usec": 2454146,
|
||||
"rss_bytes": 191004672
|
||||
},
|
||||
{
|
||||
"time": 1794262141.7649221,
|
||||
"memory_bytes": 54976512,
|
||||
"cpu_usec": 2788341,
|
||||
"rss_bytes": 106950656
|
||||
},
|
||||
{
|
||||
"time": 1794262142.153526,
|
||||
"memory_bytes": 82583552,
|
||||
"cpu_usec": 3145199,
|
||||
"rss_bytes": 180039680
|
||||
},
|
||||
{
|
||||
"time": 1794262142.4781618,
|
||||
"memory_bytes": 98701312,
|
||||
"cpu_usec": 3485405,
|
||||
"rss_bytes": 192589824
|
||||
},
|
||||
{
|
||||
"time": 1794262142.818176,
|
||||
"memory_bytes": 54738944,
|
||||
"cpu_usec": 3670691,
|
||||
"rss_bytes": 106897408
|
||||
},
|
||||
{
|
||||
"time": 1794262143.150275,
|
||||
"memory_bytes": 92516352,
|
||||
"cpu_usec": 4041841,
|
||||
"rss_bytes": 187322368
|
||||
},
|
||||
{
|
||||
"time": 1794262143.4721,
|
||||
"memory_bytes": 58019840,
|
||||
"cpu_usec": 4330075,
|
||||
"rss_bytes": 106909696
|
||||
},
|
||||
{
|
||||
"time": 1794262143.816778,
|
||||
"memory_bytes": 91193344,
|
||||
"cpu_usec": 4691490,
|
||||
"rss_bytes": 185057280
|
||||
},
|
||||
{
|
||||
"time": 1794262144.319894,
|
||||
"memory_bytes": 54292480,
|
||||
"cpu_usec": 4990675,
|
||||
"rss_bytes": 106909696
|
||||
},
|
||||
{
|
||||
"time": 1794262144.7009401,
|
||||
"memory_bytes": 66936832,
|
||||
"cpu_usec": 5267047,
|
||||
"rss_bytes": 147943424
|
||||
},
|
||||
{
|
||||
"time": 1794262145.049203,
|
||||
"memory_bytes": 98369536,
|
||||
"cpu_usec": 5612259,
|
||||
"rss_bytes": 192471040
|
||||
},
|
||||
{
|
||||
"time": 1794262145.4414349,
|
||||
"memory_bytes": 63508480,
|
||||
"cpu_usec": 5901715,
|
||||
"rss_bytes": 109088768
|
||||
},
|
||||
{
|
||||
"time": 1794262145.81682,
|
||||
"memory_bytes": 92336128,
|
||||
"cpu_usec": 6171315,
|
||||
"rss_bytes": 182304768
|
||||
},
|
||||
{
|
||||
"time": 1794262146.1910982,
|
||||
"memory_bytes": 75481088,
|
||||
"cpu_usec": 6539191,
|
||||
"rss_bytes": 109092864
|
||||
},
|
||||
{
|
||||
"time": 1794262146.612237,
|
||||
"memory_bytes": 73048064,
|
||||
"cpu_usec": 6912398,
|
||||
"rss_bytes": 154517504
|
||||
},
|
||||
{
|
||||
"time": 1794262147.0016918,
|
||||
"memory_bytes": 96288768,
|
||||
"cpu_usec": 7288823,
|
||||
"rss_bytes": 190681088
|
||||
},
|
||||
{
|
||||
"time": 1794262147.491589,
|
||||
"memory_bytes": 92078080,
|
||||
"cpu_usec": 7759923,
|
||||
"rss_bytes": 179814400
|
||||
},
|
||||
{
|
||||
"time": 1794262147.9121048,
|
||||
"memory_bytes": 64536576,
|
||||
"cpu_usec": 8079635,
|
||||
"rss_bytes": 128561152
|
||||
},
|
||||
{
|
||||
"time": 1794262148.251782,
|
||||
"memory_bytes": 102318080,
|
||||
"cpu_usec": 8385623,
|
||||
"rss_bytes": 193171456
|
||||
},
|
||||
{
|
||||
"time": 1794262148.6732469,
|
||||
"memory_bytes": 67047424,
|
||||
"cpu_usec": 8698362,
|
||||
"rss_bytes": 115306496
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 107 KiB |
File diff suppressed because it is too large
Load Diff
|
After Width: | Height: | Size: 85 KiB |
@@ -0,0 +1,23 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab5-api-1 messenger-lab5:local "dotnet Messenger.Ap…" api 2 minutes ago Up About a minute (healthy) 8080/tcp
|
||||
mtusi2026lab5-api-2 messenger-lab5:local "dotnet Messenger.Ap…" api 2 minutes ago Up About a minute (healthy) 8080/tcp
|
||||
mtusi2026lab5-broker-1 rabbitmq:4.2.0-alpine "docker-entrypoint.s…" broker 14 minutes ago Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
mtusi2026lab5-dispatcher-1 messenger-lab5:local "dotnet Messenger.Ap…" dispatcher 2 seconds ago Up Less than a second 8080/tcp
|
||||
mtusi2026lab5-lb-1 haproxy:3.2.7-alpine "docker-entrypoint.s…" lb 14 minutes ago Up 13 minutes (healthy) 127.0.0.1:18085->8080/tcp
|
||||
mtusi2026lab5-postgres-1 postgres:17.6 "docker-entrypoint.s…" postgres 14 minutes ago Up 14 minutes (healthy) 5432/tcp
|
||||
mtusi2026lab5-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 14 minutes ago Up 14 minutes (healthy) 6379/tcp
|
||||
mtusi2026lab5-s3-1 chrislusf/seaweedfs:3.97 "/entrypoint.sh serv…" s3 6 minutes ago Up 6 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
mtusi2026lab5-worker-1 messenger-lab5:local "dotnet Messenger.Ap…" worker 2 seconds ago Up Less than a second 8080/tcp
|
||||
.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18085"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab5-api-1 messenger-lab5:local "dotnet Messenger.Ap…" api 2 minutes ago Up 7 seconds (healthy) 8080/tcp
|
||||
mtusi2026lab5-api-2 messenger-lab5:local "dotnet Messenger.Ap…" api 2 minutes ago Up 7 seconds (healthy) 8080/tcp
|
||||
mtusi2026lab5-broker-1 rabbitmq:4.2.0-alpine "docker-entrypoint.s…" broker 15 minutes ago Up 30 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
mtusi2026lab5-dispatcher-1 messenger-lab5:local "dotnet Messenger.Ap…" dispatcher 44 seconds ago Up 7 seconds 8080/tcp
|
||||
mtusi2026lab5-lb-1 haproxy:3.2.7-alpine "docker-entrypoint.s…" lb 15 minutes ago Up 7 seconds (healthy) 127.0.0.1:18085->8080/tcp
|
||||
mtusi2026lab5-postgres-1 postgres:17.6 "docker-entrypoint.s…" postgres 15 minutes ago Up 32 seconds (healthy) 5432/tcp
|
||||
mtusi2026lab5-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 15 minutes ago Up 32 seconds (healthy) 6379/tcp
|
||||
mtusi2026lab5-s3-1 chrislusf/seaweedfs:3.97 "/entrypoint.sh serv…" s3 7 minutes ago Up 29 seconds (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
mtusi2026lab5-worker-1 messenger-lab5:local "dotnet Messenger.Ap…" worker 44 seconds ago Up 7 seconds 8080/tcp
|
||||
@@ -0,0 +1,2 @@
|
||||
.venv/bin/python student-tests/pipeline_tests.py --base-url "http://localhost:18085"
|
||||
PASS: 11 pipeline scenarios
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"checks": [
|
||||
"HTTP size/MIME/corruption/filename validation remains compatible with Lab4",
|
||||
"202 while worker stopped; durable outbox survives API restart; published queue survives broker restart; same ID becomes ready",
|
||||
"thumbnail/crop/WebP bytes and dimensions; no EXIF; orientation applied; transparent small image remains small with white background; private variants",
|
||||
"three duplicate ready deliveries are no-op",
|
||||
"after_objects: worker killed, same logical job recovered, one published result",
|
||||
"after_commit: worker killed, same logical job recovered, one published result",
|
||||
"publisher confirm before outbox COMMIT crash recovers without lost task or duplicate effect",
|
||||
"invalid staging checksum becomes failed after one attempt; failed attachment cannot be linked",
|
||||
"worker independently rejects >40 million pixels before bitmap allocation",
|
||||
"transient failure retries with bounded attempts then failed; explicit replay succeeds; replay of ready is no-op",
|
||||
"ready staging cleaned after durable publication; failed staging retained for replay"
|
||||
],
|
||||
"broker_queue_before_restart": "[\n{\"name\":\"attachments.v1\",\"messages_ready\":1}\n]\n",
|
||||
"retry_test_max_attempts": 3,
|
||||
"default_max_attempts": 5
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 42 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 48 KiB |
@@ -0,0 +1,19 @@
|
||||
Проверка после изменений интерфейса и настроек сервисов:
|
||||
|
||||
make check: PASS
|
||||
make validate: PASS
|
||||
make test: PASS
|
||||
student-tests/chunked_login.py: PASS
|
||||
student-tests/processing_panel.py: PASS
|
||||
make operations-test: PASS (повторное задание обработано worker)
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh:
|
||||
observed_instances_before: 2
|
||||
successful_writes: 109
|
||||
failed_attempts: 1
|
||||
first_recovery_seconds: 2.756
|
||||
PASS: подтверждённые данные и сессия сохранены.
|
||||
|
||||
Первый запуск failover остановился до сценария из-за тайм-аута регистрации,
|
||||
пока RabbitMQ запускался. После готовности сервисов повторный запуск прошёл.
|
||||
В браузере проверены поиск, копирование сообщения, вставка PNG, превью и удаление
|
||||
черновика, фильтр ошибок и отображение состояний сервисов.
|
||||
@@ -0,0 +1,40 @@
|
||||
global
|
||||
maxconn 2000
|
||||
stats socket ipv4@127.0.0.1:9999 level admin
|
||||
hard-stop-after 8s
|
||||
|
||||
defaults
|
||||
mode http
|
||||
timeout connect 2s
|
||||
timeout check 2s
|
||||
timeout client 15s
|
||||
timeout server 12s
|
||||
timeout http-request 5s
|
||||
timeout http-keep-alive 5s
|
||||
timeout queue 2s
|
||||
retries 1
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 2
|
||||
timeout resolve 500ms
|
||||
timeout retry 500ms
|
||||
hold valid 1s
|
||||
hold nx 1s
|
||||
hold other 1s
|
||||
hold refused 1s
|
||||
hold timeout 1s
|
||||
hold obsolete 0s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend messenger
|
||||
|
||||
backend messenger
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host localhost
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2
|
||||
server-template api 1-8 api:8080 resolvers docker resolve-prefer ipv4 init-addr none
|
||||
@@ -0,0 +1 @@
|
||||
matplotlib==3.11.2
|
||||
@@ -0,0 +1 @@
|
||||
Pillow==12.0.0
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
root="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
exec docker compose --project-name mtusi2026lab5 --project-directory "$root" "$@"
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grant the local operations UI to an existing messenger account."""
|
||||
import re,subprocess,sys
|
||||
from pathlib import Path
|
||||
root=Path(__file__).resolve().parents[1]
|
||||
if len(sys.argv)!=2 or not re.fullmatch('[a-z0-9_]{3,32}',sys.argv[1]):raise SystemExit('Укажите логин: make operator USERNAME=n1k0der')
|
||||
result=subprocess.check_output([str(root/'scripts/compose.sh'),'exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-c',"SELECT id FROM users WHERE username='"+sys.argv[1]+"'"],cwd=root,text=True).strip()
|
||||
if not result:raise SystemExit('Сначала зарегистрируйте этот аккаунт в мессенджере')
|
||||
p=root/'.env';lines=p.read_text().splitlines() if p.exists() else []
|
||||
lines=[line for line in lines if not line.startswith('OPS_USER_ID=')];lines.append('OPS_USER_ID='+result);p.write_text('\n'.join(lines)+'\n');p.chmod(0o600)
|
||||
subprocess.run([str(root/'scripts/compose.sh'),'up','-d','--no-deps','--wait','api'],cwd=root,check=True)
|
||||
print('Панель доступна аккаунту '+sys.argv[1]+'. Перезайдите в мессенджер.')
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env python3
|
||||
from pathlib import Path
|
||||
import secrets,json,os
|
||||
os.umask(0o077)
|
||||
p=Path(__file__).resolve().parents[1]/'secrets';p.mkdir(exist_ok=True)
|
||||
def secret(name,size=24):
|
||||
f=p/name
|
||||
if not f.exists(): f.write_text(secrets.token_hex(size))
|
||||
return f.read_text().strip()
|
||||
pw=secret('postgres_password')
|
||||
f=p/'database_url'
|
||||
if not f.exists(): f.write_text('Host=postgres;Port=5432;Database=messenger;Username=messenger;Password='+pw)
|
||||
admin=secret('s3_admin_access',12); admin_secret=secret('s3_admin_secret')
|
||||
access=secret('s3_access',12); key=secret('s3_secret')
|
||||
config={'identities':[
|
||||
{'name':'bootstrap','credentials':[{'accessKey':admin,'secretKey':admin_secret}],'actions':['Admin','Read','Write','List','Tagging']},
|
||||
{'name':'messenger','credentials':[{'accessKey':access,'secretKey':key}],'actions':['Read:messenger','Write:messenger','List:messenger']}]}
|
||||
(p/'s3_config').write_text(json.dumps(config))
|
||||
print('Local secrets ready')
|
||||
|
||||
broker_password=secret('broker_password')
|
||||
(p/'broker.conf').write_text('default_user = messenger\ndefault_pass = '+broker_password+'\ndefault_vhost = messenger\n')
|
||||
|
||||
(p.parent/'.local/faults').mkdir(parents=True,exist_ok=True)
|
||||
|
||||
(p.parent/'.local/faults').chmod(0o777)
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
exec python3 "$(dirname "$0")/init-secrets.py"
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env python3
|
||||
from pathlib import Path
|
||||
import json,statistics
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
runs=json.loads((ROOT/'evidence/overload.json').read_text())['runs']
|
||||
fig,axes=plt.subplots(1,3,figsize=(13,4.3),layout='constrained')
|
||||
settings=[1,2,4]
|
||||
for label,color in [('small','#4b6fba'),('large','#719036')]:
|
||||
for ax,key,title,unit in zip(axes,['throughput','p95_ready_seconds','peak_rss_bytes'],['Скорость обработки','Время до ready: p95 batch','Память процессов worker'],['готовых файлов/с','секунды','MiB']):
|
||||
groups=[[r[key]/(1048576 if key=='peak_rss_bytes' else 1) for r in runs if r['image']==label and r['concurrency']==n] for n in settings]
|
||||
mid=[statistics.median(g) for g in groups]
|
||||
ax.errorbar(range(3),mid,yerr=[[m-min(g) for m,g in zip(mid,groups)],[max(g)-m for m,g in zip(mid,groups)]],color=color,marker='o',capsize=4,label=label)
|
||||
ax.set_xticks(range(3),['1 / 1','2 / 2','4 / 4']);ax.set_xlabel('Concurrency / prefetch');ax.set_ylabel(unit);ax.set_title(title,fontsize=11);ax.set_ylim(bottom=0);ax.grid(alpha=.2)
|
||||
for ax,key in zip(axes,['throughput','p95_ready_seconds','peak_rss_bytes']):
|
||||
ax.set_ylim(0,max(r[key] for r in runs)/(1048576 if key=='peak_rss_bytes' else 1)*1.12)
|
||||
axes[0].legend(title='Размер PNG')
|
||||
fig.suptitle('Lab5: одинаковые 1 CPU / 1024 MiB worker; медианы и диапазон 3 повторов',fontsize=12)
|
||||
fig.savefig(ROOT/'evidence/overload.png',dpi=170)
|
||||
fig.savefig(ROOT/'evidence/overload.svg')
|
||||
print('Saved evidence/overload.png and overload.svg')
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
./scripts/compose.sh restart postgres redis s3 broker
|
||||
./scripts/compose.sh up -d --wait --wait-timeout 60 postgres redis s3 broker
|
||||
./scripts/compose.sh restart api lb dispatcher worker
|
||||
./scripts/compose.sh up -d --wait --wait-timeout 60 api lb dispatcher worker
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Manage a temporary HTTPS tunnel for this project's HTTP entry point."""
|
||||
import fcntl,os,re,shutil,signal,socket,subprocess,sys,time,urllib.request
|
||||
from pathlib import Path
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
STATE=ROOT/'.local';STATE.mkdir(exist_ok=True)
|
||||
PID=STATE/'share.pid';URL=STATE/'share.url';origin='http://127.0.0.1:18085'
|
||||
def running():
|
||||
if not PID.exists():return None
|
||||
try:pid=int(PID.read_text())
|
||||
except ValueError:return None
|
||||
command=subprocess.run(['ps','-p',str(pid),'-o','command='],capture_output=True,text=True).stdout
|
||||
return pid if 'cloudflared tunnel' in command and '--url '+origin in command else None
|
||||
with (STATE/'share.lock').open('w') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX)
|
||||
pid=running()
|
||||
if '--stop' in sys.argv:
|
||||
if pid:
|
||||
os.kill(pid,signal.SIGTERM)
|
||||
for _ in range(50):
|
||||
if not running():break
|
||||
time.sleep(.1)
|
||||
else:
|
||||
try:os.kill(pid,signal.SIGKILL)
|
||||
except ProcessLookupError:pass
|
||||
PID.unlink(missing_ok=True);URL.unlink(missing_ok=True)
|
||||
print('Внешний доступ отключён. Localhost продолжает работать.');raise SystemExit
|
||||
if pid:
|
||||
print(URL.read_text().strip() if URL.exists() else 'Туннель запущен, ссылка ещё создаётся.');raise SystemExit
|
||||
binary=shutil.which('cloudflared')
|
||||
if not binary:raise SystemExit('Установите cloudflared: brew install cloudflared')
|
||||
try:urllib.request.urlopen(origin+'/health/ready',timeout=5).close()
|
||||
except Exception:raise SystemExit('Сначала запустите мессенджер: make up')
|
||||
URL.unlink(missing_ok=True)
|
||||
(STATE/'share.log').unlink(missing_ok=True)
|
||||
with (STATE/'share.log').open('w') as log:
|
||||
edge=socket.gethostbyname('region1.v2.argotunnel.com')
|
||||
process=subprocess.Popen([binary,'tunnel','--no-autoupdate','--protocol','http2','--edge',edge+':7844','--url',origin],stdout=log,stderr=subprocess.STDOUT,start_new_session=True,env={**os.environ,'GODEBUG':'netdns=cgo'})
|
||||
PID.write_text(str(process.pid));PID.chmod(0o600)
|
||||
deadline=time.monotonic()+120
|
||||
while time.monotonic()<deadline and process.poll() is None:
|
||||
text=(STATE/'share.log').read_text()
|
||||
match=re.search(r'https://[a-z0-9-]+\.trycloudflare\.com',text)
|
||||
if match:URL.write_text(match.group(0)+'\n')
|
||||
if match and 'Registered tunnel connection' in text:
|
||||
check=subprocess.run(['curl','--fail','--silent','--max-time','8',match.group(0)+'/health/ready'],capture_output=True)
|
||||
if check.returncode==0:
|
||||
print(match.group(0));print('Отключение: make stop-share');raise SystemExit
|
||||
time.sleep(.25)
|
||||
if process.poll() is None:process.terminate()
|
||||
PID.unlink(missing_ok=True)
|
||||
raise SystemExit('Не удалось получить ссылку. Подробности в .local/share.log')
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
test -n "${TARGET_INSTANCE_ID:-}" || { echo "TARGET_INSTANCE_ID is required" >&2; exit 1; }
|
||||
for id in $(./scripts/compose.sh ps -q api); do
|
||||
instance="$(docker exec "$id" curl -fsS -D - -o /dev/null http://127.0.0.1:8080/health/live | tr -d '\r' | awk 'tolower($1) == "x-instance-id:" { print $2 }')"
|
||||
if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker stop --time 8 "$id" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
echo "Requested API instance was not found" >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,4 @@
|
||||
namespace Messenger.Api.Contracts;
|
||||
public sealed record AttachmentDto(Guid Id, Guid ChatId, Guid OwnerId, string Filename,
|
||||
string ContentType, long SizeBytes, string Sha256, string Status, object[] Variants,
|
||||
string? ErrorCode, string CreatedAt, string UpdatedAt);
|
||||
@@ -0,0 +1,7 @@
|
||||
namespace Messenger.Api.Contracts;
|
||||
|
||||
public sealed record ChatCursor(
|
||||
string Scope,
|
||||
string CreatedAt,
|
||||
Guid Id
|
||||
);
|
||||
@@ -0,0 +1,3 @@
|
||||
namespace Messenger.Api.Contracts;
|
||||
|
||||
public sealed record ChatDto(Guid Id, string Title, List<Guid> MemberIds, string CreatedAt);
|
||||
@@ -0,0 +1,4 @@
|
||||
namespace Messenger.Api.Contracts;
|
||||
|
||||
public sealed record ChatPreviewDto(
|
||||
Guid Id, string Title, List<Guid> MemberIds, string CreatedAt, MessageDto? LastMessage);
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace Messenger.Api.Contracts;
|
||||
|
||||
public sealed record CreateChatRequest(
|
||||
string? Title,
|
||||
List<Guid>? MemberIds
|
||||
);
|
||||
@@ -0,0 +1,8 @@
|
||||
using System.Text.Json;
|
||||
|
||||
namespace Messenger.Api.Contracts;
|
||||
|
||||
public sealed record CreateMessageRequest(
|
||||
string? Text,
|
||||
JsonElement AttachmentIds = default
|
||||
);
|
||||
@@ -0,0 +1,7 @@
|
||||
namespace Messenger.Api.Contracts;
|
||||
|
||||
public sealed record CreateUserRequest(
|
||||
string? Username,
|
||||
string? DisplayName,
|
||||
string? Password
|
||||
);
|
||||
@@ -0,0 +1,3 @@
|
||||
namespace Messenger.Api.Contracts;
|
||||
|
||||
public sealed record MessageDto(Guid Id, Guid ChatId, Guid SenderId, string Text, string CreatedAt, Guid[] AttachmentIds);
|
||||
@@ -0,0 +1,3 @@
|
||||
namespace Messenger.Api.Contracts;
|
||||
|
||||
public sealed record SessionDto(string Token, string TokenType, string ExpiresAt, UserDto User);
|
||||
@@ -0,0 +1,3 @@
|
||||
namespace Messenger.Api.Contracts;
|
||||
|
||||
public sealed record UserDto(Guid Id, string Username, string DisplayName, string CreatedAt);
|
||||
@@ -0,0 +1,2 @@
|
||||
namespace Messenger.Api.Contracts;
|
||||
public sealed record VariantDto(string Name, string ContentType, long SizeBytes, int Width, int Height, string Sha256);
|
||||
@@ -0,0 +1,206 @@
|
||||
using Messenger.Api.Data.Entities;
|
||||
using System.Net.Sockets;
|
||||
using Npgsql;
|
||||
|
||||
namespace Messenger.Api.Data;
|
||||
|
||||
public sealed partial class AppDatabase(NpgsqlDataSource dataSource, IConfiguration configuration)
|
||||
{
|
||||
public async Task<bool> IsReadyAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var command = dataSource.CreateCommand("SELECT 1");
|
||||
return (int)(await command.ExecuteScalarAsync())! == 1;
|
||||
}
|
||||
catch (Exception exception) when (exception is NpgsqlException or SocketException or TimeoutException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<UserEntity?> FindUserByUsernameAsync(string username)
|
||||
{
|
||||
await using var command = dataSource.CreateCommand(
|
||||
"SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = @username");
|
||||
command.Parameters.AddWithValue("username", username);
|
||||
await using var reader = await command.ExecuteReaderAsync();
|
||||
return await reader.ReadAsync() ? ReadUser(reader) : null;
|
||||
}
|
||||
|
||||
public async Task<UserEntity?> FindUserByIdAsync(Guid id)
|
||||
{
|
||||
await using var command = dataSource.CreateCommand(
|
||||
"SELECT id, username, display_name, password_hash, created_at FROM users WHERE id = @id");
|
||||
command.Parameters.AddWithValue("id", id);
|
||||
await using var reader = await command.ExecuteReaderAsync();
|
||||
return await reader.ReadAsync() ? ReadUser(reader) : null;
|
||||
}
|
||||
|
||||
public async Task<List<UserEntity>> ListChatUsersAsync(Guid chatId)
|
||||
{
|
||||
await using var command = dataSource.CreateCommand("""
|
||||
SELECT u.id, u.username, u.display_name, u.password_hash, u.created_at
|
||||
FROM users u JOIN chat_members m ON m.user_id = u.id
|
||||
WHERE m.chat_id = @chat_id ORDER BY u.display_name, u.id
|
||||
""");
|
||||
command.Parameters.AddWithValue("chat_id", chatId);
|
||||
await using var reader = await command.ExecuteReaderAsync();
|
||||
var result = new List<UserEntity>();
|
||||
while (await reader.ReadAsync()) result.Add(ReadUser(reader));
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<UserEntity> CreateUserAsync(string username, string displayName, string passwordHash)
|
||||
{
|
||||
var user = new UserEntity(Guid.NewGuid(), username, displayName, passwordHash, UtcClock.Now());
|
||||
await using var command = dataSource.CreateCommand(
|
||||
"INSERT INTO users(id, username, display_name, password_hash, created_at) VALUES (@id, @username, @display_name, @password_hash, @created_at)");
|
||||
command.Parameters.AddWithValue("id", user.Id);
|
||||
command.Parameters.AddWithValue("username", user.Username);
|
||||
command.Parameters.AddWithValue("display_name", user.DisplayName);
|
||||
command.Parameters.AddWithValue("password_hash", user.PasswordHash);
|
||||
command.Parameters.AddWithValue("created_at", user.CreatedAt);
|
||||
await command.ExecuteNonQueryAsync();
|
||||
return user;
|
||||
}
|
||||
|
||||
public async Task CreateSessionAsync(byte[] tokenHash, Guid userId, DateTime expiresAt)
|
||||
{
|
||||
await using var command = dataSource.CreateCommand(
|
||||
"INSERT INTO sessions(token_hash, user_id, expires_at, created_at) VALUES (@token_hash, @user_id, @expires_at, @created_at)");
|
||||
command.Parameters.AddWithValue("token_hash", tokenHash);
|
||||
command.Parameters.AddWithValue("user_id", userId);
|
||||
command.Parameters.AddWithValue("expires_at", expiresAt);
|
||||
command.Parameters.AddWithValue("created_at", UtcClock.Now());
|
||||
await command.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
public async Task<SessionEntity?> FindActiveSessionAsync(byte[] tokenHash)
|
||||
{
|
||||
await using var command = dataSource.CreateCommand("""
|
||||
SELECT user_id, expires_at FROM sessions
|
||||
WHERE token_hash = @token_hash AND revoked_at IS NULL AND expires_at > now()
|
||||
""");
|
||||
command.Parameters.AddWithValue("token_hash", tokenHash);
|
||||
await using var reader = await command.ExecuteReaderAsync();
|
||||
return await reader.ReadAsync() ? new SessionEntity(reader.GetGuid(0), reader.GetDateTime(1)) : null;
|
||||
}
|
||||
|
||||
public async Task<bool> RevokeSessionAsync(byte[] tokenHash)
|
||||
{
|
||||
await using var command = dataSource.CreateCommand("""
|
||||
UPDATE sessions SET revoked_at = now()
|
||||
WHERE token_hash = @token_hash AND revoked_at IS NULL AND expires_at > now()
|
||||
""");
|
||||
command.Parameters.AddWithValue("token_hash", tokenHash);
|
||||
return await command.ExecuteNonQueryAsync() == 1;
|
||||
}
|
||||
|
||||
public async Task<ChatEntity?> CreateChatAsync(string title, List<Guid> memberIds)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync();
|
||||
await using var transaction = await connection.BeginTransactionAsync();
|
||||
|
||||
await using (var membersCommand = new NpgsqlCommand(
|
||||
"SELECT count(*) FROM users WHERE id = ANY(@member_ids)", connection, transaction))
|
||||
{
|
||||
membersCommand.Parameters.AddWithValue("member_ids", memberIds.ToArray());
|
||||
if ((long)(await membersCommand.ExecuteScalarAsync())! != memberIds.Count)
|
||||
{
|
||||
await transaction.RollbackAsync();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
var chat = new ChatEntity(Guid.NewGuid(), title, memberIds, UtcClock.Now());
|
||||
await using (var chatCommand = new NpgsqlCommand(
|
||||
"INSERT INTO chats(id, title, created_at) VALUES (@id, @title, @created_at)", connection, transaction))
|
||||
{
|
||||
chatCommand.Parameters.AddWithValue("id", chat.Id);
|
||||
chatCommand.Parameters.AddWithValue("title", chat.Title);
|
||||
chatCommand.Parameters.AddWithValue("created_at", chat.CreatedAt);
|
||||
await chatCommand.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
await using (var membershipCommand = new NpgsqlCommand(
|
||||
"INSERT INTO chat_members(chat_id, user_id) SELECT @chat_id, unnest(CAST(@member_ids AS uuid[]))", connection, transaction))
|
||||
{
|
||||
membershipCommand.Parameters.AddWithValue("chat_id", chat.Id);
|
||||
membershipCommand.Parameters.AddWithValue("member_ids", memberIds.ToArray());
|
||||
await membershipCommand.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
await transaction.CommitAsync();
|
||||
return chat;
|
||||
}
|
||||
|
||||
public async Task<ChatEntity?> FindChatForMemberAsync(Guid chatId, Guid userId)
|
||||
{
|
||||
await using var command = dataSource.CreateCommand("""
|
||||
SELECT c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id)
|
||||
FROM chats c
|
||||
WHERE c.id = @chat_id AND EXISTS (
|
||||
SELECT 1 FROM chat_members m WHERE m.chat_id = c.id AND m.user_id = @user_id)
|
||||
""");
|
||||
command.Parameters.AddWithValue("chat_id", chatId);
|
||||
command.Parameters.AddWithValue("user_id", userId);
|
||||
await using var reader = await command.ExecuteReaderAsync();
|
||||
return await reader.ReadAsync() ? ReadChat(reader) : null;
|
||||
}
|
||||
|
||||
public async Task<List<ChatEntity>> ListChatsAsync(Guid userId, DateTime? cursorTime, Guid? cursorId, int count)
|
||||
{
|
||||
var sql = """
|
||||
SELECT c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id)
|
||||
FROM chats c JOIN chat_members m ON m.chat_id = c.id
|
||||
WHERE m.user_id = @user_id
|
||||
""";
|
||||
if (cursorTime is not null)
|
||||
sql += " AND (c.created_at, c.id) > (@cursor_time, @cursor_id)";
|
||||
sql += " ORDER BY c.created_at, c.id LIMIT @count";
|
||||
|
||||
await using var command = dataSource.CreateCommand(sql);
|
||||
command.Parameters.AddWithValue("user_id", userId);
|
||||
if (cursorTime is not null)
|
||||
{
|
||||
command.Parameters.AddWithValue("cursor_time", cursorTime.Value);
|
||||
command.Parameters.AddWithValue("cursor_id", cursorId!.Value);
|
||||
}
|
||||
command.Parameters.AddWithValue("count", count);
|
||||
await using var reader = await command.ExecuteReaderAsync();
|
||||
var result = new List<ChatEntity>();
|
||||
while (await reader.ReadAsync()) result.Add(ReadChat(reader));
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<List<MessageEntity>> ListMessagesAsync(Guid chatId, DateTime? cursorTime, Guid? cursorId, int count, bool newestFirst = false)
|
||||
{
|
||||
var sql = "SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages WHERE chat_id = @chat_id";
|
||||
if (cursorTime is not null)
|
||||
sql += newestFirst ? " AND (created_at, id) < (@cursor_time, @cursor_id)" : " AND (created_at, id) > (@cursor_time, @cursor_id)";
|
||||
sql += newestFirst ? " ORDER BY created_at DESC, id DESC LIMIT @count" : " ORDER BY created_at, id LIMIT @count";
|
||||
|
||||
await using var command = dataSource.CreateCommand(sql);
|
||||
command.Parameters.AddWithValue("chat_id", chatId);
|
||||
if (cursorTime is not null)
|
||||
{
|
||||
command.Parameters.AddWithValue("cursor_time", cursorTime.Value);
|
||||
command.Parameters.AddWithValue("cursor_id", cursorId!.Value);
|
||||
}
|
||||
command.Parameters.AddWithValue("count", count);
|
||||
await using var reader = await command.ExecuteReaderAsync();
|
||||
var result = new List<MessageEntity>();
|
||||
while (await reader.ReadAsync()) result.Add(new MessageEntity(
|
||||
reader.GetGuid(0), reader.GetGuid(1), reader.GetGuid(2), reader.GetString(3), reader.GetDateTime(4), reader.GetFieldValue<Guid[]>(5)));
|
||||
return result;
|
||||
}
|
||||
|
||||
private static UserEntity ReadUser(NpgsqlDataReader reader) => new(
|
||||
reader.GetGuid(0), reader.GetString(1), reader.GetString(2), reader.GetString(3), reader.GetDateTime(4));
|
||||
|
||||
private static ChatEntity ReadChat(NpgsqlDataReader reader) => new(
|
||||
reader.GetGuid(0), reader.GetString(1), reader.GetFieldValue<Guid[]>(3).ToList(), reader.GetDateTime(2));
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
using Messenger.Api.Contracts;
|
||||
using Npgsql;
|
||||
namespace Messenger.Api.Data;
|
||||
public sealed partial class AppDatabase
|
||||
{
|
||||
public const long AttachmentGate = 48260104;
|
||||
public async Task<(AttachmentDto Dto, string Key)?> FindAttachmentAsync(Guid id, Guid actor)
|
||||
{
|
||||
await using var cmd = dataSource.CreateCommand("""
|
||||
SELECT a.id,a.chat_id,a.owner_id,a.filename,a.content_type,a.size_bytes,a.sha256,
|
||||
a.status,a.created_at,a.updated_at,a.object_key,a.variants::text,a.error_code
|
||||
FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id
|
||||
WHERE a.id=@id AND m.user_id=@actor
|
||||
""");
|
||||
cmd.Parameters.AddWithValue("id",id); cmd.Parameters.AddWithValue("actor",actor);
|
||||
await using var r=await cmd.ExecuteReaderAsync();
|
||||
return await r.ReadAsync() ? (ReadAttachment(r),r.GetString(10)) : null;
|
||||
}
|
||||
public static AttachmentDto ReadAttachment(NpgsqlDataReader r) => new(r.GetGuid(0),r.GetGuid(1),
|
||||
r.GetGuid(2),r.GetString(3),r.GetString(4),r.GetInt64(5),r.GetString(6),r.GetString(7),
|
||||
System.Text.Json.JsonSerializer.Deserialize<VariantDto[]>(r.GetString(11), Pipeline.JobJson.Options) ?? [],
|
||||
r.IsDBNull(12)?null:r.GetString(12),r.GetDateTime(8).ToString("O"),r.GetDateTime(9).ToString("O"));
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
using Messenger.Api.Contracts;
|
||||
using Npgsql;
|
||||
|
||||
namespace Messenger.Api.Data;
|
||||
|
||||
public sealed partial class AppDatabase
|
||||
{
|
||||
public async Task<List<ChatPreviewDto>> ListChatPreviewsAsync(Guid userId, DateTime? cursorTime, Guid? cursorId, int count)
|
||||
{
|
||||
var query = """
|
||||
WITH page AS (
|
||||
SELECT c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids
|
||||
FROM chats c JOIN chat_members membership ON membership.chat_id = c.id
|
||||
WHERE membership.user_id = @user_id
|
||||
""";
|
||||
if (cursorTime is not null) query += " AND (c.created_at, c.id) > (@cursor_time, @cursor_id)";
|
||||
query += """
|
||||
ORDER BY c.created_at, c.id LIMIT @count
|
||||
)
|
||||
SELECT p.id, p.title, p.created_at, p.member_ids,
|
||||
latest.id, latest.sender_id, latest.text, latest.created_at, latest.attachment_ids
|
||||
FROM page p
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT id, sender_id, text, created_at, attachment_ids
|
||||
FROM messages WHERE chat_id = p.id
|
||||
ORDER BY created_at DESC, id DESC LIMIT 1
|
||||
) latest ON true
|
||||
ORDER BY p.created_at, p.id
|
||||
""";
|
||||
await using var command = dataSource.CreateCommand(query);
|
||||
command.Parameters.AddWithValue("user_id", userId);
|
||||
command.Parameters.AddWithValue("count", count);
|
||||
if (cursorTime is not null)
|
||||
{
|
||||
command.Parameters.AddWithValue("cursor_time", cursorTime.Value);
|
||||
command.Parameters.AddWithValue("cursor_id", cursorId!.Value);
|
||||
}
|
||||
await using var reader = await command.ExecuteReaderAsync();
|
||||
var result = new List<ChatPreviewDto>();
|
||||
while (await reader.ReadAsync())
|
||||
{
|
||||
var id = reader.GetGuid(0);
|
||||
var message = reader.IsDBNull(4) ? null : new MessageDto(reader.GetGuid(4), id,
|
||||
reader.GetGuid(5), reader.GetString(6), reader.GetDateTime(7).ToString("O"), reader.GetFieldValue<Guid[]>(8));
|
||||
result.Add(new ChatPreviewDto(id, reader.GetString(1), reader.GetFieldValue<Guid[]>(3).ToList(),
|
||||
reader.GetDateTime(2).ToString("O"), message));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
using Npgsql;
|
||||
|
||||
namespace Messenger.Api.Data;
|
||||
|
||||
public static class DatabaseMigrator
|
||||
{
|
||||
public static async Task ApplyAsync(NpgsqlDataSource dataSource)
|
||||
{
|
||||
await using var connection = await dataSource.OpenConnectionAsync();
|
||||
await using var transaction = await connection.BeginTransactionAsync();
|
||||
await using (var command = new NpgsqlCommand("SELECT pg_advisory_xact_lock(202602)", connection, transaction))
|
||||
await command.ExecuteNonQueryAsync();
|
||||
await using (var command = new NpgsqlCommand(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations (version integer PRIMARY KEY, applied_at timestamptz NOT NULL)",
|
||||
connection, transaction))
|
||||
await command.ExecuteNonQueryAsync();
|
||||
|
||||
var assembly = typeof(DatabaseMigrator).Assembly;
|
||||
var migrations = assembly.GetManifestResourceNames()
|
||||
.Where(name => name.Contains(".Data.Migrations.", StringComparison.Ordinal) && name.EndsWith(".sql", StringComparison.Ordinal))
|
||||
.Order(StringComparer.Ordinal);
|
||||
foreach (var name in migrations)
|
||||
{
|
||||
var version = int.Parse(name.Split('.')[^2].Split('_')[0]);
|
||||
await using var check = new NpgsqlCommand("SELECT EXISTS (SELECT 1 FROM schema_migrations WHERE version = @version)", connection, transaction);
|
||||
check.Parameters.AddWithValue("version", version);
|
||||
if ((bool)(await check.ExecuteScalarAsync())!) continue;
|
||||
|
||||
await using var stream = assembly.GetManifestResourceStream(name)!;
|
||||
using var reader = new StreamReader(stream);
|
||||
await using (var migration = new NpgsqlCommand(await reader.ReadToEndAsync(), connection, transaction))
|
||||
await migration.ExecuteNonQueryAsync();
|
||||
await using var record = new NpgsqlCommand(
|
||||
"INSERT INTO schema_migrations(version, applied_at) VALUES (@version, now())", connection, transaction);
|
||||
record.Parameters.AddWithValue("version", version);
|
||||
await record.ExecuteNonQueryAsync();
|
||||
}
|
||||
await transaction.CommitAsync();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace Messenger.Api.Data.Entities;
|
||||
|
||||
public sealed record ChatEntity(Guid Id, string Title, List<Guid> MemberIds, DateTime CreatedAt)
|
||||
{
|
||||
public Contracts.ChatDto ToDto() => new(Id, Title, MemberIds, CreatedAt.ToString("O"));
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace Messenger.Api.Data.Entities;
|
||||
|
||||
public sealed record MessageEntity(Guid Id, Guid ChatId, Guid SenderId, string Text, DateTime CreatedAt, Guid[]? AttachmentIds = null)
|
||||
{
|
||||
public Contracts.MessageDto ToDto() => new(Id, ChatId, SenderId, Text, CreatedAt.ToString("O"), AttachmentIds ?? []);
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
namespace Messenger.Api.Data.Entities;
|
||||
|
||||
public sealed record SessionEntity(Guid UserId, DateTime ExpiresAt);
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace Messenger.Api.Data.Entities;
|
||||
|
||||
public sealed record UserEntity(Guid Id, string Username, string DisplayName, string PasswordHash, DateTime CreatedAt)
|
||||
{
|
||||
public Contracts.UserDto ToDto() => new(Id, Username, DisplayName, CreatedAt.ToString("O"));
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Messenger.Api.Data.Entities;
|
||||
using Npgsql;
|
||||
|
||||
namespace Messenger.Api.Data;
|
||||
|
||||
public sealed partial class AppDatabase
|
||||
{
|
||||
private bool BatchCommands => !string.Equals(configuration["MESSAGE_BATCHING"], "false", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
public async Task<MessageEntity?> CreateMessageOnceAsync(Guid chatId, Guid senderId, string text, string key, Guid[] attachmentIds)
|
||||
{
|
||||
var hash = SHA256.HashData(System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(new { text, attachment_ids = attachmentIds }));
|
||||
await using var connection = await dataSource.OpenConnectionAsync();
|
||||
await using var transaction = await connection.BeginTransactionAsync();
|
||||
|
||||
await using (var gate = new NpgsqlCommand($"SELECT pg_advisory_xact_lock_shared({AttachmentGate})",connection,transaction))
|
||||
await gate.ExecuteNonQueryAsync();
|
||||
if (attachmentIds.Length>0)
|
||||
{
|
||||
await using var attachments = new NpgsqlCommand("SELECT id,status FROM attachments WHERE chat_id=@chat AND id=ANY(@ids)",connection,transaction);
|
||||
attachments.Parameters.AddWithValue("chat",chatId); attachments.Parameters.AddWithValue("ids",attachmentIds);
|
||||
await using var found = await attachments.ExecuteReaderAsync();
|
||||
var count=0; var ready=true;
|
||||
while(await found.ReadAsync()) { count++; ready &= found.GetString(1)!="failed"; }
|
||||
if(count!=attachmentIds.Length) throw new Services.AttachmentException(404,"not_found","Вложение не найдено");
|
||||
if(!ready) throw new Services.AttachmentException(409,"conflict","Вложение недоступно");
|
||||
}
|
||||
// A transaction-scoped lock coordinates the same request across all API processes.
|
||||
if (!BatchCommands)
|
||||
await using (var gate = new NpgsqlCommand("SELECT pg_advisory_xact_lock(hashtextextended(@scope, 0))", connection, transaction))
|
||||
{
|
||||
gate.Parameters.AddWithValue("scope", $"{senderId}:POST:/api/v1/chats/{chatId}/messages:{key}");
|
||||
await gate.ExecuteNonQueryAsync();
|
||||
}
|
||||
await using (var previous = new NpgsqlCommand((BatchCommands ? "SELECT pg_advisory_xact_lock(hashtextextended(@scope, 0));" : "") + """
|
||||
SELECT r.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at, m.attachment_ids
|
||||
FROM message_requests r JOIN messages m ON m.id = r.message_id
|
||||
WHERE r.user_id = @user_id AND r.chat_id = @chat_id AND r.request_key = @key
|
||||
AND r.expires_at > clock_timestamp()
|
||||
""", connection, transaction))
|
||||
{
|
||||
previous.Parameters.AddWithValue("scope", $"{senderId}:POST:/api/v1/chats/{chatId}/messages:{key}");
|
||||
previous.Parameters.AddWithValue("user_id", senderId);
|
||||
previous.Parameters.AddWithValue("chat_id", chatId);
|
||||
previous.Parameters.AddWithValue("key", key);
|
||||
await using var reader = await previous.ExecuteReaderAsync();
|
||||
if (BatchCommands) await reader.NextResultAsync();
|
||||
if (await reader.ReadAsync())
|
||||
{
|
||||
var same = reader.GetFieldValue<byte[]>(0).AsSpan().SequenceEqual(hash);
|
||||
var result = same ? new MessageEntity(reader.GetGuid(1), reader.GetGuid(2), reader.GetGuid(3), reader.GetString(4), reader.GetDateTime(5), reader.GetFieldValue<Guid[]>(6)) : null;
|
||||
await reader.DisposeAsync();
|
||||
await transaction.CommitAsync();
|
||||
return result;
|
||||
}
|
||||
}
|
||||
if (!BatchCommands)
|
||||
await using (var cleanup = new NpgsqlCommand("""
|
||||
DELETE FROM message_requests
|
||||
WHERE user_id = @user_id AND chat_id = @chat_id AND request_key = @key
|
||||
""", connection, transaction))
|
||||
{
|
||||
cleanup.Parameters.AddWithValue("user_id", senderId);
|
||||
cleanup.Parameters.AddWithValue("chat_id", chatId);
|
||||
cleanup.Parameters.AddWithValue("key", key);
|
||||
await cleanup.ExecuteNonQueryAsync();
|
||||
}
|
||||
var message = new MessageEntity(Guid.NewGuid(), chatId, senderId, text, UtcClock.Now(), attachmentIds);
|
||||
await using (var insert = new NpgsqlCommand((BatchCommands ? "DELETE FROM message_requests WHERE user_id = @sender_id AND chat_id = @chat_id AND request_key = @key;" : "") + """
|
||||
INSERT INTO messages(id, chat_id, sender_id, text, created_at, attachment_ids)
|
||||
VALUES (@id, @chat_id, @sender_id, @text, @created_at, @attachments);
|
||||
INSERT INTO message_requests(user_id, chat_id, request_key, request_hash, message_id, expires_at)
|
||||
VALUES (@sender_id, @chat_id, @key, @hash, @id, clock_timestamp() + interval '24 hours');
|
||||
""", connection, transaction))
|
||||
{
|
||||
insert.Parameters.AddWithValue("attachments", attachmentIds);
|
||||
insert.Parameters.AddWithValue("id", message.Id);
|
||||
insert.Parameters.AddWithValue("chat_id", chatId);
|
||||
insert.Parameters.AddWithValue("sender_id", senderId);
|
||||
insert.Parameters.AddWithValue("text", text);
|
||||
insert.Parameters.AddWithValue("created_at", message.CreatedAt);
|
||||
insert.Parameters.AddWithValue("key", key);
|
||||
insert.Parameters.AddWithValue("hash", hash);
|
||||
await insert.ExecuteNonQueryAsync();
|
||||
}
|
||||
await transaction.CommitAsync();
|
||||
return message;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id);
|
||||
CREATE INDEX chats_created_id_idx ON chats (created_at, id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX sessions_user_id_idx ON sessions (user_id);
|
||||
CREATE INDEX sessions_expires_at_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,11 @@
|
||||
CREATE TABLE message_requests (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
request_key text NOT NULL,
|
||||
request_hash bytea NOT NULL,
|
||||
message_id uuid NOT NULL REFERENCES messages(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, chat_id, request_key)
|
||||
);
|
||||
|
||||
CREATE INDEX message_requests_expires_idx ON message_requests (expires_at);
|
||||
@@ -0,0 +1,6 @@
|
||||
CREATE TABLE message_reactions (
|
||||
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
reaction text NOT NULL CHECK (reaction IN ('like', 'love', 'laugh', 'mushroom')),
|
||||
PRIMARY KEY (message_id, user_id, reaction)
|
||||
);
|
||||
@@ -0,0 +1,11 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY, chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL REFERENCES users(id), filename text NOT NULL,
|
||||
content_type text NOT NULL, size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL, object_key text NOT NULL UNIQUE,
|
||||
status text NOT NULL CHECK (status IN ('ready','missing')),
|
||||
created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX attachments_created_idx ON attachments(created_at, id);
|
||||
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
|
||||
CREATE INDEX messages_attachments_idx ON messages USING gin(attachment_ids);
|
||||
@@ -0,0 +1,25 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
|
||||
UPDATE attachments SET status='failed' WHERE status='missing';
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK(status IN ('queued','processing','ready','failed'));
|
||||
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]';
|
||||
ALTER TABLE attachments ADD COLUMN error_code text;
|
||||
UPDATE attachments SET error_code='original_missing' WHERE status='failed';
|
||||
CREATE TABLE attachment_jobs (
|
||||
id uuid PRIMARY KEY, attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id),
|
||||
pipeline_version int NOT NULL DEFAULT 1, staging_key text NOT NULL UNIQUE,
|
||||
state text NOT NULL CHECK(state IN ('queued','processing','ready','failed')),
|
||||
attempts int NOT NULL DEFAULT 0, next_at timestamptz NOT NULL DEFAULT now(),
|
||||
lease_token uuid, lease_until timestamptz, error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(), completed_at timestamptz,
|
||||
staging_deleted_at timestamptz, replay_count int NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX attachment_jobs_active_idx ON attachment_jobs(state,next_at);
|
||||
CREATE TABLE attachment_outbox (
|
||||
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id),
|
||||
available_at timestamptz NOT NULL DEFAULT now(), published_at timestamptz
|
||||
);
|
||||
CREATE INDEX attachment_outbox_pending_idx ON attachment_outbox(available_at) WHERE published_at IS NULL;
|
||||
CREATE TABLE attachment_job_events (
|
||||
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id),
|
||||
event text NOT NULL, attempt int NOT NULL, created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
@@ -0,0 +1 @@
|
||||
CREATE TABLE pipeline_presence (role text PRIMARY KEY, seen_at timestamptz NOT NULL);
|
||||
@@ -0,0 +1,48 @@
|
||||
using Npgsql;
|
||||
|
||||
namespace Messenger.Api.Data;
|
||||
|
||||
public sealed record ReactionDto(Guid MessageId, string Reaction, int Count, bool Mine);
|
||||
|
||||
public sealed partial class AppDatabase
|
||||
{
|
||||
public async Task<bool> SetReactionAsync(Guid chatId, Guid messageId, Guid userId, string reaction, bool present)
|
||||
{
|
||||
await using var command = dataSource.CreateCommand(present ? """
|
||||
WITH target AS (SELECT id FROM messages WHERE id = @message_id AND chat_id = @chat_id),
|
||||
written AS (
|
||||
INSERT INTO message_reactions(message_id, user_id, reaction)
|
||||
SELECT id, @user_id, @reaction FROM target ON CONFLICT DO NOTHING
|
||||
) SELECT EXISTS(SELECT 1 FROM target)
|
||||
""" : """
|
||||
WITH target AS (SELECT id FROM messages WHERE id = @message_id AND chat_id = @chat_id),
|
||||
removed AS (
|
||||
DELETE FROM message_reactions WHERE message_id IN (SELECT id FROM target)
|
||||
AND user_id = @user_id AND reaction = @reaction
|
||||
) SELECT EXISTS(SELECT 1 FROM target)
|
||||
""");
|
||||
command.Parameters.AddWithValue("chat_id", chatId);
|
||||
command.Parameters.AddWithValue("message_id", messageId);
|
||||
command.Parameters.AddWithValue("user_id", userId);
|
||||
command.Parameters.AddWithValue("reaction", reaction);
|
||||
return (bool)(await command.ExecuteScalarAsync())!;
|
||||
}
|
||||
|
||||
public async Task<List<ReactionDto>> ListReactionsAsync(Guid chatId, Guid[] messageIds, Guid userId)
|
||||
{
|
||||
await using var command = dataSource.CreateCommand("""
|
||||
SELECT r.message_id, r.reaction, count(*)::integer, bool_or(r.user_id = @user_id)
|
||||
FROM message_reactions r JOIN messages m ON m.id = r.message_id
|
||||
WHERE m.chat_id = @chat_id AND m.id = ANY(@message_ids)
|
||||
GROUP BY r.message_id, r.reaction ORDER BY r.message_id, r.reaction
|
||||
""");
|
||||
command.Parameters.AddWithValue("chat_id", chatId);
|
||||
command.Parameters.AddWithValue("message_ids", messageIds);
|
||||
command.Parameters.AddWithValue("user_id", userId);
|
||||
await using var reader = await command.ExecuteReaderAsync();
|
||||
var items = new List<ReactionDto>();
|
||||
while (await reader.ReadAsync())
|
||||
items.Add(new(reader.GetGuid(0), reader.GetString(1), reader.GetInt32(2), reader.GetBoolean(3)));
|
||||
return items;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace Messenger.Api.Data;
|
||||
|
||||
public static class UtcClock
|
||||
{
|
||||
// PostgreSQL timestamptz stores microseconds; keep the response identical to the saved value.
|
||||
public static DateTime Now()
|
||||
{
|
||||
var ticks = DateTime.UtcNow.Ticks;
|
||||
return new DateTime(ticks - ticks % 10, DateTimeKind.Utc);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
using Messenger.Api.Data;
|
||||
using Messenger.Api.Infrastructure;
|
||||
using Messenger.Api.Services;
|
||||
using Amazon.S3;
|
||||
using Amazon.S3.Model;
|
||||
using static Messenger.Api.Services.ApiSupport;
|
||||
namespace Messenger.Api.Endpoints;
|
||||
public static class AttachmentEndpoints
|
||||
{
|
||||
public static IEndpointRouteBuilder MapAttachmentEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
app.MapPost("/api/v1/chats/{chat_id}/attachments",async(string chat_id,HttpContext ctx,
|
||||
SessionService sessions,ChatService chats,AttachmentService files)=> {
|
||||
var(actor,_)=await sessions.AuthenticateAsync(ctx); if(actor is null) return Unauthorized(ctx);
|
||||
if(!Guid.TryParse(chat_id,out var chat)) return ApiError(ctx,400,"invalid_request","Некорректный chat_id");
|
||||
if(await chats.FindForMemberAsync(chat,actor.Id) is null) return ApiError(ctx,404,"not_found","Чат не найден");
|
||||
var names=ctx.Request.Query["filename"]; var name=names.ToString();
|
||||
if(names.Count!=1 || string.IsNullOrWhiteSpace(name) || name.EnumerateRunes().Count()>255 || name.Any(c=>char.IsControl(c)||c is '/' or '\\'))
|
||||
return ApiError(ctx,400,"invalid_request","Некорректное имя файла");
|
||||
if(ctx.Request.ContentType?.Split(';')[0].Trim() is not "image/png" and not "image/jpeg")
|
||||
return ApiError(ctx,415,"unsupported_media_type","Требуется PNG или JPEG");
|
||||
return Results.Json(await files.UploadAsync(ctx.Request,chat,actor.Id,name),statusCode:202);
|
||||
});
|
||||
app.MapGet("/api/v1/attachments/{attachment_id}",async(string attachment_id,HttpContext ctx,
|
||||
SessionService sessions,AppDatabase db)=> {
|
||||
var(actor,_)=await sessions.AuthenticateAsync(ctx); if(actor is null) return Unauthorized(ctx);
|
||||
if(!Guid.TryParse(attachment_id,out var id)) return ApiError(ctx,400,"invalid_request","Некорректный ID");
|
||||
var file=await db.FindAttachmentAsync(id,actor.Id);
|
||||
if(file is null) return ApiError(ctx,404,"not_found","Вложение не найдено");
|
||||
return Results.Ok(file.Value.Dto);
|
||||
});
|
||||
app.MapGet("/api/v1/attachments/{attachment_id}/processing",async(string attachment_id,HttpContext ctx,
|
||||
SessionService sessions,AppDatabase db,Npgsql.NpgsqlDataSource source)=> {
|
||||
var(actor,_)=await sessions.AuthenticateAsync(ctx); if(actor is null) return Unauthorized(ctx);
|
||||
if(!Guid.TryParse(attachment_id,out var id)) return ApiError(ctx,400,"invalid_request","Некорректный ID");
|
||||
if(await db.FindAttachmentAsync(id,actor.Id) is null) return ApiError(ctx,404,"not_found","Вложение не найдено");
|
||||
await using var cmd=source.CreateCommand("""
|
||||
SELECT j.state,j.attempts,j.error_code,j.created_at,j.completed_at,
|
||||
e.event,e.attempt,e.created_at
|
||||
FROM attachment_jobs j LEFT JOIN attachment_job_events e ON e.job_id=j.id
|
||||
WHERE j.attachment_id=@id ORDER BY e.id
|
||||
""");
|
||||
cmd.Parameters.AddWithValue("id",id);
|
||||
await using var reader=await cmd.ExecuteReaderAsync(ctx.RequestAborted);
|
||||
string? state=null,error=null;int attempts=0;DateTime? created=null,completed=null;
|
||||
var events=new List<object>();
|
||||
while(await reader.ReadAsync(ctx.RequestAborted)) {
|
||||
state=reader.GetString(0);attempts=reader.GetInt32(1);
|
||||
error=reader.IsDBNull(2)?null:reader.GetString(2);
|
||||
created=reader.GetDateTime(3);completed=reader.IsDBNull(4)?null:reader.GetDateTime(4);
|
||||
if(!reader.IsDBNull(5)) events.Add(new {name=reader.GetString(5),attempt=reader.GetInt32(6),at=reader.GetDateTime(7)});
|
||||
}
|
||||
ctx.Response.Headers.CacheControl="no-store";
|
||||
return Results.Ok(new {state,attempts,error_code=error,created_at=created,completed_at=completed,events});
|
||||
});
|
||||
app.MapGet("/api/v1/attachments/{attachment_id}/content",async(string attachment_id,HttpContext ctx,
|
||||
SessionService sessions,AppDatabase db,S3Storage storage)=> {
|
||||
var(actor,_)=await sessions.AuthenticateAsync(ctx); if(actor is null) return Unauthorized(ctx);
|
||||
if(!Guid.TryParse(attachment_id,out var id)) return ApiError(ctx,400,"invalid_request","Некорректный ID");
|
||||
var file=await db.FindAttachmentAsync(id,actor.Id);
|
||||
if(file is null) return ApiError(ctx,404,"not_found","Вложение не найдено");
|
||||
if(file.Value.Dto.Status!="ready") return ApiError(ctx,409,"conflict","Оригинал недоступен");
|
||||
var variant=ctx.Request.Query.TryGetValue("variant",out var value)?value.ToString():"original";
|
||||
var selected=file.Value.Dto.Variants.OfType<Messenger.Api.Contracts.VariantDto>().FirstOrDefault(v=>v.Name==variant);
|
||||
if(variant!="original" && selected is null) return ApiError(ctx,404,"not_found","Вариант не найден");
|
||||
var key=variant=="original"?file.Value.Key:$"variants/{id:N}/v1/{variant}";
|
||||
var mime=selected?.ContentType ?? file.Value.Dto.ContentType;
|
||||
var filename=variant=="original"?file.Value.Dto.Filename:variant+(mime=="image/webp"?".webp":".jpg");
|
||||
try {
|
||||
var original=await storage.Client.GetObjectAsync(storage.Bucket,key,ctx.RequestAborted);
|
||||
return Results.Stream(async stream=> {using(original) {await original.ResponseStream.CopyToAsync(stream,ctx.RequestAborted);}},
|
||||
mime,filename);
|
||||
} catch(AmazonS3Exception ex) when(ex.StatusCode==System.Net.HttpStatusCode.NotFound) {
|
||||
return ApiError(ctx,503,"unavailable","Оригинал временно недоступен");
|
||||
}
|
||||
});
|
||||
return app;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using Messenger.Api.Services;
|
||||
using static Messenger.Api.Services.ApiSupport;
|
||||
|
||||
namespace Messenger.Api.Endpoints;
|
||||
|
||||
public static class AuthEndpoints
|
||||
{
|
||||
private static readonly UTF8Encoding StrictUtf8 = new(false, true);
|
||||
|
||||
public static IEndpointRouteBuilder MapAuthEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
app.MapPost("/api/v1/auth/sessions", async (HttpContext context, SessionService sessions) =>
|
||||
{
|
||||
if (!context.Request.Headers.TryGetValue("Authorization", out var header))
|
||||
return Unauthorized(context, basic: true);
|
||||
|
||||
var authorization = header.ToString();
|
||||
if (!authorization.StartsWith("Basic ", StringComparison.OrdinalIgnoreCase))
|
||||
return Unauthorized(context, basic: true);
|
||||
|
||||
string credentials;
|
||||
try
|
||||
{
|
||||
credentials = StrictUtf8.GetString(Convert.FromBase64String(authorization[6..]));
|
||||
}
|
||||
catch (Exception exception) when (exception is FormatException or DecoderFallbackException)
|
||||
{
|
||||
return Unauthorized(context, basic: true);
|
||||
}
|
||||
|
||||
var delimiter = credentials.IndexOf(':');
|
||||
if (delimiter < 0) return Unauthorized(context, basic: true);
|
||||
var username = credentials[..delimiter];
|
||||
var password = credentials[(delimiter + 1)..];
|
||||
if (!Regex.IsMatch(username, "\\A[a-z0-9_]{3,32}\\z"))
|
||||
return Unauthorized(context, basic: true);
|
||||
|
||||
if (context.Request.ContentLength > 0)
|
||||
return ApiError(context, 400, "invalid_request", "Вход принимает Basic без тела запроса");
|
||||
|
||||
if (context.Request.ContentLength is null && context.Request.Headers.ContainsKey("Transfer-Encoding"))
|
||||
{
|
||||
using var deadline=CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted);
|
||||
deadline.CancelAfter(TimeSpan.FromSeconds(3));
|
||||
try {
|
||||
if(await context.Request.Body.ReadAsync(new byte[1],deadline.Token)>0)
|
||||
return ApiError(context,400,"invalid_request","Вход принимает Basic без тела запроса");
|
||||
} catch(OperationCanceledException) when(!context.RequestAborted.IsCancellationRequested) {
|
||||
return ApiError(context,400,"invalid_request","Тело запроса не завершено");
|
||||
}
|
||||
}
|
||||
|
||||
var session = await sessions.LoginAsync(username, password);
|
||||
return session is null
|
||||
? Unauthorized(context, basic: true)
|
||||
: Results.Json(session, statusCode: StatusCodes.Status201Created);
|
||||
});
|
||||
|
||||
app.MapDelete("/api/v1/auth/sessions/current", async (HttpContext context, SessionService sessions) =>
|
||||
{
|
||||
var (user, tokenHash) = await sessions.AuthenticateAsync(context);
|
||||
if (user is null || tokenHash is null) return Unauthorized(context);
|
||||
return await sessions.RevokeAsync(tokenHash)
|
||||
? Results.NoContent()
|
||||
: Unauthorized(context);
|
||||
});
|
||||
|
||||
return app;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
using Messenger.Api.Contracts;
|
||||
using Messenger.Api.Services;
|
||||
using static Messenger.Api.Services.ApiSupport;
|
||||
|
||||
namespace Messenger.Api.Endpoints;
|
||||
|
||||
public static class ChatEndpoints
|
||||
{
|
||||
public static IEndpointRouteBuilder MapChatEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
app.MapPost("/api/v1/chats", async (HttpContext context, SessionService sessions, ChatService chats) =>
|
||||
{
|
||||
var (actor, _) = await sessions.AuthenticateAsync(context);
|
||||
if (actor is null) return Unauthorized(context);
|
||||
|
||||
var (request, error) = await ReadJsonAsync<CreateChatRequest>(context);
|
||||
if (error is not null) return error;
|
||||
if (string.IsNullOrWhiteSpace(request!.Title) || request.Title.EnumerateRunes().Count() > 100)
|
||||
return ApiError(context, 400, "invalid_request", "Некорректное название чата");
|
||||
if (request.MemberIds is null || request.MemberIds.Count is < 1 or > 100 ||
|
||||
request.MemberIds.Distinct().Count() != request.MemberIds.Count ||
|
||||
!request.MemberIds.Contains(actor.Id))
|
||||
return ApiError(context, 400, "invalid_request", "Некорректный список участников");
|
||||
|
||||
var chat = await chats.CreateAsync(request.Title, request.MemberIds);
|
||||
return chat is null
|
||||
? ApiError(context, 400, "invalid_request", "Один из участников не существует")
|
||||
: Results.Json(chat, statusCode: StatusCodes.Status201Created);
|
||||
});
|
||||
|
||||
app.MapGet("/api/v1/chats", async (HttpContext context, SessionService sessions, ChatService chats) =>
|
||||
{
|
||||
var (actor, _) = await sessions.AuthenticateAsync(context);
|
||||
if (actor is null) return Unauthorized(context);
|
||||
var (limit, limitError) = ParseLimit(context);
|
||||
if (limitError is not null) return limitError;
|
||||
var (cursor, cursorTime, cursorError) = ParseCursor(context, $"user:{actor.Id}");
|
||||
if (cursorError is not null) return cursorError;
|
||||
|
||||
var page = await chats.ListAsync(actor.Id, cursorTime, cursor?.Id, limit + 1);
|
||||
string? nextCursor = null;
|
||||
if (page.Count > limit)
|
||||
{
|
||||
var last = page[limit - 1];
|
||||
nextCursor = CursorService.EncodeChatCursor(new ChatCursor($"user:{actor.Id}", last.CreatedAt, last.Id));
|
||||
page = page.Take(limit).ToList();
|
||||
}
|
||||
return Results.Ok(new { items = page, next_cursor = nextCursor });
|
||||
});
|
||||
|
||||
app.MapGet("/api/v1/chats/{chat_id}", async (
|
||||
string chat_id, HttpContext context, SessionService sessions, ChatService chats) =>
|
||||
{
|
||||
var (actor, _) = await sessions.AuthenticateAsync(context);
|
||||
if (actor is null) return Unauthorized(context);
|
||||
if (!Guid.TryParse(chat_id, out var chatId))
|
||||
return ApiError(context, 400, "invalid_request", "Некорректный chat_id");
|
||||
|
||||
var chat = await chats.FindForMemberAsync(chatId, actor.Id);
|
||||
return chat is null
|
||||
? ApiError(context, 404, "not_found", "Чат не найден")
|
||||
: Results.Ok(chat);
|
||||
});
|
||||
|
||||
return app;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
using Messenger.Api.Data;
|
||||
using Messenger.Api.Services;
|
||||
using Messenger.Api.Infrastructure;
|
||||
|
||||
namespace Messenger.Api.Endpoints;
|
||||
|
||||
public static class HealthEndpoints
|
||||
{
|
||||
public static IEndpointRouteBuilder MapHealthEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
app.MapGet("/health/live", () => Results.Ok(new { status = "ok" }));
|
||||
app.MapGet("/health/ready", async (HttpContext context, AppDatabase database, ReadinessState readiness, S3Storage storage) =>
|
||||
!readiness.IsStopping && await database.IsReadyAsync() && await storage.ReadyAsync()
|
||||
? Results.Ok(new { status = "ok" })
|
||||
: ApiSupport.ApiError(context, 503, "unavailable", "Хранилище недоступно"));
|
||||
return app;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
using Messenger.Api.Contracts;
|
||||
using Messenger.Api.Services;
|
||||
using System.Text.Json;
|
||||
using static Messenger.Api.Services.ApiSupport;
|
||||
|
||||
namespace Messenger.Api.Endpoints;
|
||||
|
||||
public static class MessageEndpoints
|
||||
{
|
||||
public static IEndpointRouteBuilder MapMessageEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
app.MapPost("/api/v1/chats/{chat_id}/messages", async (
|
||||
string chat_id, HttpContext context, SessionService sessions, ChatService chats, MessageService messages) =>
|
||||
{
|
||||
var (actor, _) = await sessions.AuthenticateAsync(context);
|
||||
if (actor is null) return Unauthorized(context);
|
||||
if (!Guid.TryParse(chat_id, out var chatId))
|
||||
return ApiError(context, 400, "invalid_request", "Некорректный chat_id");
|
||||
if (await chats.FindForMemberAsync(chatId, actor.Id) is null)
|
||||
return ApiError(context, 404, "not_found", "Чат не найден");
|
||||
|
||||
var keys = context.Request.Headers["Idempotency-Key"];
|
||||
var key = keys.ToString();
|
||||
if (keys.Count != 1 || key.Length is < 1 or > 128 ||
|
||||
key.Any(c => !char.IsAsciiLetterOrDigit(c) && c is not '.' and not '_' and not ':' and not '-'))
|
||||
return ApiError(context, 400, "invalid_request", "Требуется корректный Idempotency-Key");
|
||||
|
||||
var (request, error) = await ReadJsonAsync<CreateMessageRequest>(context);
|
||||
if (error is not null) return error;
|
||||
if (request!.Text is null || request.Text.EnumerateRunes().Count() > 4000)
|
||||
return ApiError(context, 400, "invalid_request", "Некорректный текст сообщения");
|
||||
var ids = new List<Guid>();
|
||||
if (request.AttachmentIds.ValueKind != JsonValueKind.Undefined)
|
||||
{
|
||||
if (request.AttachmentIds.ValueKind != JsonValueKind.Array || request.AttachmentIds.GetArrayLength()>10)
|
||||
return ApiError(context,400,"invalid_request","Некорректный список вложений");
|
||||
foreach(var item in request.AttachmentIds.EnumerateArray())
|
||||
{
|
||||
if(item.ValueKind!=JsonValueKind.String || !item.TryGetGuid(out var id) || ids.Contains(id))
|
||||
return ApiError(context,400,"invalid_request","Некорректный ID вложения");
|
||||
ids.Add(id);
|
||||
}
|
||||
}
|
||||
if(string.IsNullOrWhiteSpace(request.Text) && ids.Count==0)
|
||||
return ApiError(context,400,"invalid_request","Нужен текст или вложение");
|
||||
|
||||
var message = await messages.CreateAsync(chatId, actor.Id, request.Text, key, ids.ToArray());
|
||||
if (message is null)
|
||||
return ApiError(context, 409, "conflict", "Этот ключ уже использован для другого сообщения");
|
||||
return Results.Json(message, statusCode: StatusCodes.Status201Created);
|
||||
});
|
||||
|
||||
app.MapGet("/api/v1/chats/{chat_id}/messages", async (
|
||||
string chat_id, HttpContext context, SessionService sessions, ChatService chats, MessageService messages) =>
|
||||
{
|
||||
var (actor, _) = await sessions.AuthenticateAsync(context);
|
||||
if (actor is null) return Unauthorized(context);
|
||||
if (!Guid.TryParse(chat_id, out var chatId))
|
||||
return ApiError(context, 400, "invalid_request", "Некорректный chat_id");
|
||||
if (await chats.FindForMemberAsync(chatId, actor.Id) is null)
|
||||
return ApiError(context, 404, "not_found", "Чат не найден");
|
||||
var (limit, limitError) = ParseLimit(context);
|
||||
if (limitError is not null) return limitError;
|
||||
var (cursor, cursorTime, cursorError) = ParseCursor(context, $"chat:{chatId}");
|
||||
if (cursorError is not null) return cursorError;
|
||||
|
||||
var page = await messages.ListAsync(chatId, cursorTime, cursor?.Id, limit + 1);
|
||||
string? nextCursor = null;
|
||||
if (page.Count > limit)
|
||||
{
|
||||
var last = page[limit - 1];
|
||||
nextCursor = CursorService.EncodeChatCursor(new ChatCursor($"chat:{chatId}", last.CreatedAt, last.Id));
|
||||
page = page.Take(limit).ToList();
|
||||
}
|
||||
return Results.Ok(new { items = page, next_cursor = nextCursor });
|
||||
});
|
||||
|
||||
return app;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
using Messenger.Api.Data;
|
||||
using Messenger.Api.Infrastructure;
|
||||
using Messenger.Api.Pipeline;
|
||||
using Messenger.Api.Services;
|
||||
using Npgsql;
|
||||
using static Messenger.Api.Services.ApiSupport;
|
||||
namespace Messenger.Api.Endpoints;
|
||||
public static class OperationsEndpoints
|
||||
{
|
||||
public static bool IsOperator(Guid id,IConfiguration config)=>Guid.TryParse(config["OPS_USER_ID"],out var owner)&&owner==id;
|
||||
public static IEndpointRouteBuilder MapOperationsEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
app.MapGet("/api/ui/operator",async(HttpContext ctx,SessionService sessions,IConfiguration config)=> {
|
||||
var(actor,_)=await sessions.AuthenticateAsync(ctx);if(actor is null)return Unauthorized(ctx);
|
||||
ctx.Response.Headers.CacheControl="no-store";
|
||||
return Results.Ok(new {enabled=IsOperator(actor.Id,config)});
|
||||
});
|
||||
app.MapGet("/api/ui/operations",async(HttpContext ctx,SessionService sessions,IConfiguration config,
|
||||
NpgsqlDataSource source,ClusterPresence cluster,S3Storage storage)=> {
|
||||
var(actor,_)=await sessions.AuthenticateAsync(ctx);if(actor is null)return Unauthorized(ctx);
|
||||
if(!IsOperator(actor.Id,config))return ApiError(ctx,404,"not_found","Доступ только владельцу системы");
|
||||
ctx.Response.Headers.CacheControl="no-store";
|
||||
using var deadline=CancellationTokenSource.CreateLinkedTokenSource(ctx.RequestAborted);deadline.CancelAfter(TimeSpan.FromSeconds(7));
|
||||
var ct=deadline.Token;
|
||||
var counts=new Dictionary<string,long>{{"queued",0},{"processing",0},{"ready",0},{"failed",0}};
|
||||
var recent=new List<object>();var history=new List<object>();var heartbeats=new Dictionary<string,DateTime>();
|
||||
long pending=0;double? latency=null;
|
||||
await using(var cmd=source.CreateCommand("SELECT state,count(*) FROM attachment_jobs GROUP BY state"))
|
||||
await using(var rd=await cmd.ExecuteReaderAsync(ct))while(await rd.ReadAsync(ct))counts[rd.GetString(0)]=rd.GetInt64(1);
|
||||
await using(var cmd=source.CreateCommand("SELECT count(*) FROM attachment_outbox WHERE published_at IS NULL"))pending=(long)(await cmd.ExecuteScalarAsync(ct))!;
|
||||
await using(var cmd=source.CreateCommand("SELECT avg(extract(epoch FROM (completed_at-created_at)))::double precision FROM attachment_jobs WHERE state='ready' AND completed_at>now()-interval '15 minutes'")) {
|
||||
var value=await cmd.ExecuteScalarAsync(ct);if(value is double seconds)latency=seconds;
|
||||
}
|
||||
await using(var cmd=source.CreateCommand("SELECT role,seen_at FROM pipeline_presence"))
|
||||
await using(var rd=await cmd.ExecuteReaderAsync(ct))while(await rd.ReadAsync(ct))heartbeats[rd.GetString(0)]=rd.GetDateTime(1);
|
||||
await using(var cmd=source.CreateCommand("""
|
||||
SELECT a.id,a.filename,j.state,j.attempts,j.error_code,j.created_at,j.completed_at,
|
||||
j.state='failed' AND j.staging_deleted_at IS NULL
|
||||
FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id ORDER BY j.created_at DESC LIMIT 30
|
||||
"""))
|
||||
await using(var rd=await cmd.ExecuteReaderAsync(ct))while(await rd.ReadAsync(ct))recent.Add(new {
|
||||
attachment_id=rd.GetGuid(0),filename=rd.GetString(1),state=rd.GetString(2),attempts=rd.GetInt32(3),
|
||||
error_code=rd.IsDBNull(4)?null:rd.GetString(4),created_at=rd.GetDateTime(5),
|
||||
completed_at=rd.IsDBNull(6)?(DateTime?)null:rd.GetDateTime(6),can_replay=rd.GetBoolean(7)});
|
||||
await using(var cmd=source.CreateCommand("""
|
||||
SELECT e.event,e.attempt,e.created_at,a.filename FROM attachment_job_events e
|
||||
JOIN attachment_jobs j ON j.id=e.job_id JOIN attachments a ON a.id=j.attachment_id ORDER BY e.id DESC LIMIT 30
|
||||
"""))
|
||||
await using(var rd=await cmd.ExecuteReaderAsync(ct))while(await rd.ReadAsync(ct))history.Add(new {
|
||||
name=rd.GetString(0),attempt=rd.GetInt32(1),at=rd.GetDateTime(2),filename=rd.GetString(3)});
|
||||
bool? s3=null,broker=null;uint? messages=null,consumers=null;
|
||||
async Task ProbeStorage() {try {await storage.Client.GetBucketLocationAsync(storage.Bucket,ct);s3=true;}catch {s3=false;}}
|
||||
async Task ProbeBroker() {try {
|
||||
await using var connection=await BrokerConnection.OpenAsync(config,ct);
|
||||
await using var channel=await connection.CreateChannelAsync(cancellationToken:ct);
|
||||
var result=await channel.QueueDeclarePassiveAsync(BrokerConnection.Queue,ct);
|
||||
broker=true;messages=result.MessageCount;consumers=result.ConsumerCount;
|
||||
}catch {broker=false;}}
|
||||
var snapshotTask=cluster.ReadAsync();await Task.WhenAll(ProbeStorage(),ProbeBroker());var snapshot=await snapshotTask;
|
||||
var now=DateTime.UtcNow;
|
||||
return Results.Ok(new {checked_at=now,counts,pending_outbox=pending,average_cycle_seconds=latency,
|
||||
services=new {postgres=true,redis=snapshot.RedisAvailable,s3,broker,
|
||||
worker=heartbeats.TryGetValue("worker",out var worker)&&(now-worker).TotalSeconds<10,
|
||||
dispatcher=heartbeats.TryGetValue("dispatcher",out var dispatcher)&&(now-dispatcher).TotalSeconds<10},
|
||||
instances=snapshot.Instances,broker_messages=messages,broker_consumers=consumers,jobs=recent,events=history});
|
||||
});
|
||||
app.MapPost("/api/ui/operations/attachments/{id:guid}/replay",async(Guid id,HttpContext ctx,
|
||||
SessionService sessions,IConfiguration config,NpgsqlDataSource source)=> {
|
||||
var(actor,_)=await sessions.AuthenticateAsync(ctx);if(actor is null)return Unauthorized(ctx);
|
||||
if(!IsOperator(actor.Id,config))return ApiError(ctx,404,"not_found","Доступ только владельцу системы");
|
||||
await using var cmd=source.CreateCommand("SELECT id FROM attachment_jobs WHERE attachment_id=@id");cmd.Parameters.AddWithValue("id",id);
|
||||
var job=await cmd.ExecuteScalarAsync(ctx.RequestAborted);
|
||||
if(job is not Guid jobId)return ApiError(ctx,404,"not_found","Задание не найдено");
|
||||
var changed=await new JobStore(source,config).ReplayAsync(jobId,ctx.RequestAborted);
|
||||
return changed?Results.Accepted(value:new {status="queued"}):ApiError(ctx,409,"conflict","Повтор недоступен: задание не завершилось ошибкой или исходник уже удалён");
|
||||
});
|
||||
return app;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
using Messenger.Api.Data;
|
||||
using Messenger.Api.Services;
|
||||
using static Messenger.Api.Services.ApiSupport;
|
||||
|
||||
namespace Messenger.Api.Endpoints;
|
||||
|
||||
public static class ReactionEndpoints
|
||||
{
|
||||
public static IEndpointRouteBuilder MapReactionEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
app.MapGet("/api/ui/chats/{chat_id}/reactions", async (
|
||||
string chat_id, HttpContext context, SessionService sessions, ChatService chats, AppDatabase database) =>
|
||||
{
|
||||
var (actor, _) = await sessions.AuthenticateAsync(context);
|
||||
if (actor is null) return Unauthorized(context);
|
||||
if (!Guid.TryParse(chat_id, out var chatId)) return ApiError(context, 400, "invalid_request", "Некорректный chat_id");
|
||||
if (await chats.FindForMemberAsync(chatId, actor.Id) is null) return ApiError(context, 404, "not_found", "Чат не найден");
|
||||
var values = context.Request.Query["message_ids"].ToString().Split(',');
|
||||
if (values.Length is < 1 or > 100 || values.Any(value => !Guid.TryParse(value, out _)))
|
||||
return ApiError(context, 400, "invalid_request", "Укажите от 1 до 100 ID сообщений");
|
||||
var ids = values.Select(Guid.Parse).Distinct().ToArray();
|
||||
return Results.Ok(new { items = await database.ListReactionsAsync(chatId, ids, actor.Id) });
|
||||
});
|
||||
app.MapMethods("/api/ui/chats/{chat_id}/messages/{message_id}/reactions/{reaction}", ["PUT", "DELETE"], async (
|
||||
string chat_id, string message_id, string reaction, HttpContext context,
|
||||
SessionService sessions, ChatService chats, AppDatabase database) =>
|
||||
{
|
||||
var (actor, _) = await sessions.AuthenticateAsync(context);
|
||||
if (actor is null) return Unauthorized(context);
|
||||
if (!Guid.TryParse(chat_id, out var chatId) || !Guid.TryParse(message_id, out var messageId)
|
||||
|| reaction is not ("like" or "love" or "laugh" or "mushroom"))
|
||||
return ApiError(context, 400, "invalid_request", "Некорректная реакция или ID");
|
||||
if (await chats.FindForMemberAsync(chatId, actor.Id) is null) return ApiError(context, 404, "not_found", "Чат не найден");
|
||||
return await database.SetReactionAsync(chatId, messageId, actor.Id, reaction, context.Request.Method == "PUT")
|
||||
? Results.NoContent() : ApiError(context, 404, "not_found", "Сообщение не найдено");
|
||||
});
|
||||
return app;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using Messenger.Api.Contracts;
|
||||
using Messenger.Api.Services;
|
||||
using Messenger.Api.Data;
|
||||
using Messenger.Api.Infrastructure;
|
||||
using static Messenger.Api.Services.ApiSupport;
|
||||
|
||||
namespace Messenger.Api.Endpoints;
|
||||
|
||||
public static class UiEndpoints
|
||||
{
|
||||
// The required /api/v1 history stays ascending. The web client has a separate,
|
||||
// optional route to open a large chat at its newest page and page backwards.
|
||||
public static IEndpointRouteBuilder MapUiEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
app.MapGet("/api/ui/cluster", async (HttpContext context, SessionService sessions, ClusterPresence presence) =>
|
||||
{
|
||||
var (actor, _) = await sessions.AuthenticateAsync(context);
|
||||
if (actor is null) return Unauthorized(context);
|
||||
context.Response.Headers.CacheControl = "no-store";
|
||||
return Results.Ok(await presence.ReadAsync());
|
||||
});
|
||||
|
||||
app.MapGet("/api/ui/users/{username}",async(string username,HttpContext ctx,SessionService sessions,AppDatabase db)=> {
|
||||
var(actor,_)=await sessions.AuthenticateAsync(ctx);if(actor is null)return Unauthorized(ctx);
|
||||
if(!System.Text.RegularExpressions.Regex.IsMatch(username,"\\A[a-z0-9_]{3,32}\\z"))return ApiError(ctx,400,"invalid_request","Некорректный логин");
|
||||
var user=await db.FindUserByUsernameAsync(username);
|
||||
return user is null?ApiError(ctx,404,"not_found","Пользователь не найден"):Results.Ok(user.ToDto());
|
||||
});
|
||||
|
||||
app.MapGet("/api/ui/chats", async (HttpContext context, SessionService sessions, AppDatabase database) =>
|
||||
{
|
||||
var (actor, _) = await sessions.AuthenticateAsync(context);
|
||||
if (actor is null) return Unauthorized(context);
|
||||
var (limit, limitError) = ParseLimit(context);
|
||||
if (limitError is not null) return limitError;
|
||||
var (cursor, cursorTime, cursorError) = ParseCursor(context, $"ui-chats:{actor.Id}");
|
||||
if (cursorError is not null) return cursorError;
|
||||
var page = await database.ListChatPreviewsAsync(actor.Id, cursorTime, cursor?.Id, limit + 1);
|
||||
string? nextCursor = null;
|
||||
if (page.Count > limit)
|
||||
{
|
||||
var last = page[limit - 1];
|
||||
nextCursor = CursorService.EncodeChatCursor(new ChatCursor($"ui-chats:{actor.Id}", last.CreatedAt, last.Id));
|
||||
page = page.Take(limit).ToList();
|
||||
}
|
||||
return Results.Ok(new { items = page, next_cursor = nextCursor });
|
||||
});
|
||||
|
||||
app.MapGet("/api/ui/chats/{chat_id}/members", async (
|
||||
string chat_id, HttpContext context, SessionService sessions, ChatService chats, UserService users) =>
|
||||
{
|
||||
var (actor, _) = await sessions.AuthenticateAsync(context);
|
||||
if (actor is null) return Unauthorized(context);
|
||||
if (!Guid.TryParse(chat_id, out var chatId))
|
||||
return ApiError(context, 400, "invalid_request", "Некорректный chat_id");
|
||||
if (await chats.FindForMemberAsync(chatId, actor.Id) is null)
|
||||
return ApiError(context, 404, "not_found", "Чат не найден");
|
||||
return Results.Ok(new { items = await users.ListForChatAsync(chatId) });
|
||||
});
|
||||
|
||||
app.MapGet("/api/ui/chats/{chat_id}/messages", async (
|
||||
string chat_id, HttpContext context, SessionService sessions, ChatService chats, MessageService messages) =>
|
||||
{
|
||||
var (actor, _) = await sessions.AuthenticateAsync(context);
|
||||
if (actor is null) return Unauthorized(context);
|
||||
if (!Guid.TryParse(chat_id, out var chatId))
|
||||
return ApiError(context, 400, "invalid_request", "Некорректный chat_id");
|
||||
if (await chats.FindForMemberAsync(chatId, actor.Id) is null)
|
||||
return ApiError(context, 404, "not_found", "Чат не найден");
|
||||
var (limit, limitError) = ParseLimit(context);
|
||||
if (limitError is not null) return limitError;
|
||||
var (cursor, cursorTime, cursorError) = ParseCursor(context, $"recent-chat:{chatId}");
|
||||
if (cursorError is not null) return cursorError;
|
||||
|
||||
var page = await messages.ListAsync(chatId, cursorTime, cursor?.Id, limit + 1, newestFirst: true);
|
||||
string? nextCursor = null;
|
||||
if (page.Count > limit)
|
||||
{
|
||||
var last = page[limit - 1];
|
||||
nextCursor = CursorService.EncodeChatCursor(new ChatCursor($"recent-chat:{chatId}", last.CreatedAt, last.Id));
|
||||
page = page.Take(limit).ToList();
|
||||
}
|
||||
return Results.Ok(new { items = page, next_cursor = nextCursor });
|
||||
});
|
||||
return app;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using Messenger.Api.Contracts;
|
||||
using Messenger.Api.Services;
|
||||
using static Messenger.Api.Services.ApiSupport;
|
||||
|
||||
namespace Messenger.Api.Endpoints;
|
||||
|
||||
public static class UserEndpoints
|
||||
{
|
||||
public static IEndpointRouteBuilder MapUserEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
app.MapPost("/api/v1/users", async (HttpContext context, UserService users) =>
|
||||
{
|
||||
var (request, error) = await ReadJsonAsync<CreateUserRequest>(context);
|
||||
if (error is not null) return error;
|
||||
|
||||
if (request!.Username is null || !Regex.IsMatch(request.Username, "\\A[a-z0-9_]{3,32}\\z"))
|
||||
return ApiError(context, 400, "invalid_request", "Некорректный username");
|
||||
if (string.IsNullOrWhiteSpace(request.DisplayName) ||
|
||||
request.DisplayName.EnumerateRunes().Count() is < 1 or > 100)
|
||||
return ApiError(context, 400, "invalid_request", "Некорректный display_name");
|
||||
if (string.IsNullOrWhiteSpace(request.Password) ||
|
||||
request.Password.EnumerateRunes().Count() is < 12 or > 128)
|
||||
return ApiError(context, 400, "invalid_request", "Пароль должен содержать 12-128 символов");
|
||||
|
||||
var user = await users.RegisterAsync(request.Username, request.DisplayName, request.Password);
|
||||
return user is null
|
||||
? ApiError(context, 409, "conflict", "Username уже занят")
|
||||
: Results.Json(user, statusCode: StatusCodes.Status201Created);
|
||||
});
|
||||
|
||||
app.MapGet("/api/v1/users/me", async (HttpContext context, SessionService sessions) =>
|
||||
{
|
||||
var (user, _) = await sessions.AuthenticateAsync(context);
|
||||
return user is null ? Unauthorized(context) : Results.Ok(user);
|
||||
});
|
||||
|
||||
return app;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
using Messenger.Api.Data;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace Messenger.Api.Infrastructure;
|
||||
|
||||
// Registry entries live ten minutes; an eight-second heartbeat gap means unavailable.
|
||||
// A Redis outage is reported as unknown, never as proof that an API has stopped.
|
||||
public sealed class ClusterPresence(IConnectionMultiplexer redis, InstanceIdentity identity,
|
||||
ReadinessState readiness, AppDatabase database, ILogger<ClusterPresence> logger) : BackgroundService
|
||||
{
|
||||
private const string Registry = "messenger:ui:instances";
|
||||
public sealed record Node(string Id, bool Ready, long LastSeen);
|
||||
public sealed record Snapshot(bool RedisAvailable, bool PostgresAvailable, long CheckedAt, List<Node> Instances);
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(2));
|
||||
try
|
||||
{
|
||||
do
|
||||
{
|
||||
try
|
||||
{
|
||||
if (redis.IsConnected && !readiness.IsStopping && await database.IsReadyAsync())
|
||||
{
|
||||
var db = redis.GetDatabase();
|
||||
var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||
await db.SortedSetAddAsync(Registry, identity.Id, now).WaitAsync(TimeSpan.FromSeconds(1), stoppingToken);
|
||||
await db.SortedSetRemoveRangeByScoreAsync(Registry, double.NegativeInfinity, now - 600_000);
|
||||
await db.KeyExpireAsync(Registry, TimeSpan.FromMinutes(15));
|
||||
}
|
||||
}
|
||||
catch (Exception error) when (error is RedisException or TimeoutException)
|
||||
{ logger.LogDebug("Cluster heartbeat unavailable"); }
|
||||
} while (await timer.WaitForNextTickAsync(stoppingToken));
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
|
||||
}
|
||||
|
||||
public async Task<Snapshot> ReadAsync()
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||
var postgres = await database.IsReadyAsync();
|
||||
try
|
||||
{
|
||||
if (!redis.IsConnected) return new(false, postgres, now, []);
|
||||
var entries = await redis.GetDatabase().SortedSetRangeByScoreWithScoresAsync(Registry,
|
||||
now - 600_000, double.PositiveInfinity).WaitAsync(TimeSpan.FromSeconds(1));
|
||||
return new(true, postgres, now, entries.Select(entry => new Node(entry.Element.ToString(),
|
||||
now - entry.Score < 8000, (long)entry.Score)).ToList());
|
||||
}
|
||||
catch (Exception error) when (error is RedisException or TimeoutException)
|
||||
{ return new(false, postgres, now, []); }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace Messenger.Api.Infrastructure;
|
||||
|
||||
public sealed class InstanceIdentity
|
||||
{
|
||||
public string Id { get; } = Guid.NewGuid().ToString("N");
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace Messenger.Api.Infrastructure;
|
||||
|
||||
public sealed class ReadinessState
|
||||
{
|
||||
private int _stopping;
|
||||
public bool IsStopping => Volatile.Read(ref _stopping) != 0;
|
||||
public void Stop() => Interlocked.Exchange(ref _stopping, 1);
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using System.Text.Json;
|
||||
using Messenger.Api.Contracts;
|
||||
using StackExchange.Redis;
|
||||
|
||||
namespace Messenger.Api.Infrastructure;
|
||||
|
||||
// PostgreSQL validates the session on every request. This cache saves the user lookup
|
||||
// and never grants access independently of the durable session row.
|
||||
public sealed class RedisSessionCache(IConnectionMultiplexer connection, ILogger<RedisSessionCache> logger)
|
||||
{
|
||||
private static readonly TimeSpan OperationBudget = TimeSpan.FromMilliseconds(750);
|
||||
private static string Key(byte[] hash) => "messenger:session:" + Convert.ToHexString(hash).ToLowerInvariant();
|
||||
|
||||
public sealed record Entry(UserDto User, DateTime ExpiresAt);
|
||||
|
||||
public async Task<Entry?> GetAsync(byte[] hash)
|
||||
{
|
||||
if (!connection.IsConnected) return null;
|
||||
try
|
||||
{
|
||||
var value = await connection.GetDatabase().StringGetAsync(Key(hash)).WaitAsync(OperationBudget);
|
||||
return value.IsNullOrEmpty ? null : JsonSerializer.Deserialize<Entry>(value.ToString());
|
||||
}
|
||||
catch (Exception exception) when (exception is RedisException or TimeoutException or OperationCanceledException or JsonException)
|
||||
{
|
||||
logger.LogWarning("Session cache read unavailable; using PostgreSQL");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task SetAsync(byte[] hash, UserDto user, DateTime expiresAt)
|
||||
{
|
||||
var remaining = expiresAt - DateTime.UtcNow;
|
||||
if (!connection.IsConnected || remaining <= TimeSpan.Zero) return;
|
||||
try
|
||||
{
|
||||
await connection.GetDatabase().StringSetAsync(Key(hash),
|
||||
JsonSerializer.Serialize(new Entry(user, expiresAt)), remaining).WaitAsync(OperationBudget);
|
||||
}
|
||||
catch (Exception exception) when (exception is RedisException or TimeoutException or OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning("Session cache write unavailable; using PostgreSQL");
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RemoveAsync(byte[] hash)
|
||||
{
|
||||
if (!connection.IsConnected) return;
|
||||
try
|
||||
{
|
||||
await connection.GetDatabase().KeyDeleteAsync(Key(hash)).WaitAsync(OperationBudget);
|
||||
}
|
||||
catch (Exception exception) when (exception is RedisException or TimeoutException or OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning("Session cache invalidation unavailable; PostgreSQL still enforces revocation");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
using Amazon.S3;
|
||||
using Amazon.S3.Model;
|
||||
using Amazon.Runtime;
|
||||
namespace Messenger.Api.Infrastructure;
|
||||
public sealed class S3Storage : IDisposable
|
||||
{
|
||||
public AmazonS3Client Client { get; }
|
||||
public string Bucket { get; }
|
||||
public S3Storage(IConfiguration config)
|
||||
{
|
||||
Bucket=config["S3_BUCKET"] ?? "messenger";
|
||||
Client=new AmazonS3Client(new BasicAWSCredentials(
|
||||
File.ReadAllText(config["S3_ACCESS_KEY_FILE"]!).Trim(),
|
||||
File.ReadAllText(config["S3_SECRET_KEY_FILE"]!).Trim()),new AmazonS3Config {
|
||||
ServiceURL=config["S3_ENDPOINT"] ?? "http://s3:9000", ForcePathStyle=true,
|
||||
AuthenticationRegion="us-east-1", MaxErrorRetry=0, Timeout=TimeSpan.FromSeconds(20)
|
||||
});
|
||||
}
|
||||
public async Task<bool> ReadyAsync()
|
||||
{
|
||||
using var timeout=new CancellationTokenSource(TimeSpan.FromSeconds(1));
|
||||
try { await Client.ListObjectsV2Async(new ListObjectsV2Request {BucketName=Bucket,MaxKeys=1},timeout.Token); return true; }
|
||||
catch { return false; }
|
||||
}
|
||||
public void Dispose() => Client.Dispose();
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net9.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<EmbeddedResource Include="Data/Migrations/*.sql" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="AWSSDK.S3" Version="4.0.0" />
|
||||
<PackageReference Include="Isopoh.Cryptography.Argon2" Version="2.0.0" />
|
||||
<PackageReference Include="Npgsql" Version="9.0.5" />
|
||||
<PackageReference Include="RabbitMQ.Client" Version="7.1.2" />
|
||||
<PackageReference Include="SkiaSharp" Version="3.119.1" />
|
||||
<PackageReference Include="SkiaSharp.NativeAssets.Linux.NoDependencies" Version="3.119.1" />
|
||||
<PackageReference Include="StackExchange.Redis" Version="2.8.58" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,42 @@
|
||||
using Messenger.Api.Services;
|
||||
using System.Net.Sockets;
|
||||
using Npgsql;
|
||||
|
||||
namespace Messenger.Api.Middleware;
|
||||
|
||||
public sealed class ExceptionHandlingMiddleware(RequestDelegate next, ILogger<ExceptionHandlingMiddleware> logger)
|
||||
{
|
||||
public async Task InvokeAsync(HttpContext context)
|
||||
{
|
||||
try
|
||||
{
|
||||
await next(context);
|
||||
}
|
||||
catch (AttachmentException exception)
|
||||
{
|
||||
if(!context.Response.HasStarted)
|
||||
await ApiSupport.ApiError(context,exception.Status,exception.Code,exception.Message).ExecuteAsync(context);
|
||||
}
|
||||
catch (OperationCanceledException) when (!context.RequestAborted.IsCancellationRequested)
|
||||
{
|
||||
if(!context.Response.HasStarted)
|
||||
await ApiSupport.ApiError(context,503,"unavailable","Время ожидания истекло").ExecuteAsync(context);
|
||||
}
|
||||
catch (BadHttpRequestException exception) when(exception.StatusCode==413)
|
||||
{
|
||||
await ApiSupport.ApiError(context,413,"payload_too_large","Слишком большой запрос").ExecuteAsync(context);
|
||||
}
|
||||
catch (Exception exception) when (exception is NpgsqlException or SocketException or TimeoutException or Amazon.S3.AmazonS3Exception or HttpRequestException)
|
||||
{
|
||||
logger.LogError(exception, "Database request failed");
|
||||
if (!context.Response.HasStarted)
|
||||
await ApiSupport.ApiError(context, 503, "unavailable", "Сервис временно недоступен").ExecuteAsync(context);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(exception, "Request failed");
|
||||
if (!context.Response.HasStarted)
|
||||
await ApiSupport.ApiError(context, 500, "internal_error", "Внутренняя ошибка").ExecuteAsync(context);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
using Messenger.Api.Infrastructure;
|
||||
using Messenger.Api.Services;
|
||||
|
||||
namespace Messenger.Api.Middleware;
|
||||
|
||||
public sealed class InstanceMiddleware(RequestDelegate next)
|
||||
{
|
||||
public async Task InvokeAsync(HttpContext context, InstanceIdentity identity, ReadinessState readiness)
|
||||
{
|
||||
context.Response.Headers["X-Instance-Id"] = identity.Id;
|
||||
if (readiness.IsStopping && context.Request.Path != "/health/live")
|
||||
{
|
||||
await ApiSupport.ApiError(context, 503, "unavailable", "Экземпляр API останавливается").ExecuteAsync(context);
|
||||
return;
|
||||
}
|
||||
await next(context);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
namespace Messenger.Api.Middleware;
|
||||
|
||||
public sealed class RequestIdMiddleware
|
||||
{
|
||||
private readonly RequestDelegate _next;
|
||||
|
||||
public RequestIdMiddleware(RequestDelegate next) => _next = next;
|
||||
|
||||
public async Task InvokeAsync(HttpContext context)
|
||||
{
|
||||
var requestId = Guid.NewGuid().ToString();
|
||||
context.Items["RequestId"] = requestId;
|
||||
context.Response.Headers["X-Request-Id"] = requestId;
|
||||
await _next(context);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
namespace Messenger.Api.Middleware;
|
||||
|
||||
public sealed class RequestLoggingMiddleware
|
||||
{
|
||||
private readonly RequestDelegate _next;
|
||||
|
||||
public RequestLoggingMiddleware(RequestDelegate next) => _next = next;
|
||||
|
||||
public async Task InvokeAsync(HttpContext context)
|
||||
{
|
||||
var logger = context.RequestServices
|
||||
.GetRequiredService<ILoggerFactory>()
|
||||
.CreateLogger("Http");
|
||||
|
||||
await _next(context);
|
||||
|
||||
logger.LogInformation(
|
||||
"{Method} {Path} -> {StatusCode} request_id={RequestId}",
|
||||
context.Request.Method,
|
||||
context.Request.Path,
|
||||
context.Response.StatusCode,
|
||||
context.Items["RequestId"]
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using RabbitMQ.Client;
|
||||
namespace Messenger.Api.Pipeline;
|
||||
public static class BrokerConnection
|
||||
{
|
||||
public const string Queue="attachments.v1";
|
||||
public static Task<IConnection> OpenAsync(IConfiguration config,CancellationToken ct)
|
||||
{
|
||||
var factory=new ConnectionFactory {
|
||||
HostName=config["BROKER_HOST"]??"broker", UserName="messenger",
|
||||
Password=File.ReadAllText(config["BROKER_PASSWORD_FILE"]!).Trim(),
|
||||
VirtualHost="messenger",AutomaticRecoveryEnabled=true,
|
||||
NetworkRecoveryInterval=TimeSpan.FromSeconds(2),RequestedHeartbeat=TimeSpan.FromSeconds(10),
|
||||
RequestedConnectionTimeout=TimeSpan.FromSeconds(5)
|
||||
};
|
||||
return factory.CreateConnectionAsync(ct);
|
||||
}
|
||||
public static Task DeclareAsync(IChannel channel,CancellationToken ct)=>channel.QueueDeclareAsync(
|
||||
Queue,durable:true,exclusive:false,autoDelete:false,arguments:new Dictionary<string,object?> {
|
||||
["x-queue-type"]="quorum",["x-max-length"]=256,["x-overflow"]="reject-publish"
|
||||
},cancellationToken:ct);
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
using System.Text.Json;
|
||||
namespace Messenger.Api.Pipeline;
|
||||
public static class FaultInjection
|
||||
{
|
||||
// Explicit local operator instrumentation; disabled in the normal Compose configuration.
|
||||
public static async Task HitAsync(IConfiguration config,string point,Guid job,CancellationToken ct)
|
||||
{
|
||||
var path=config["PIPELINE_FAULT_FILE"];
|
||||
if(string.IsNullOrEmpty(path)||!File.Exists(path))return;
|
||||
try {
|
||||
using var json=JsonDocument.Parse(await File.ReadAllTextAsync(path,ct));
|
||||
if(json.RootElement.GetProperty("point").GetString()!=point || json.RootElement.GetProperty("job_id").GetGuid()!=job)return;
|
||||
File.Move(path,path+".used",true);
|
||||
await File.WriteAllTextAsync(path+".hit",point,ct);
|
||||
}catch(IOException){return;}
|
||||
Console.WriteLine($"Operator fault pause: {point}, job {job}");
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan,ct);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
using System.Text.Json;
|
||||
using System.Security.Cryptography;
|
||||
using Messenger.Api.Contracts;
|
||||
using SkiaSharp;
|
||||
namespace Messenger.Api.Pipeline;
|
||||
public static class ImageRenderer
|
||||
{
|
||||
public static void Render(string input,string output)
|
||||
{
|
||||
using var stream=File.OpenRead(input);using var managed=new SKManagedStream(stream,false);
|
||||
using var codec=SKCodec.Create(managed);
|
||||
if(codec is null || codec.EncodedFormat is not SKEncodedImageFormat.Png and not SKEncodedImageFormat.Jpeg)
|
||||
throw new InvalidDataException("invalid_image");
|
||||
var info=codec.Info;
|
||||
if((long)info.Width*info.Height>40_000_000 || info.Width<1 || info.Height<1)throw new InvalidDataException("invalid_image");
|
||||
using var bitmap=new SKBitmap(new SKImageInfo(info.Width,info.Height,SKColorType.Rgba8888,SKAlphaType.Premul));
|
||||
if(codec.GetPixels(bitmap.Info,bitmap.GetPixels())!=SKCodecResult.Success)throw new InvalidDataException("invalid_image");
|
||||
var rotated=(int)codec.EncodedOrigin>=5;
|
||||
int width=rotated?info.Height:info.Width,height=rotated?info.Width:info.Height;
|
||||
var matrix=codec.EncodedOrigin switch {
|
||||
SKEncodedOrigin.TopRight=>new SKMatrix(-1,0,info.Width,0,1,0,0,0,1),
|
||||
SKEncodedOrigin.BottomRight=>new SKMatrix(-1,0,info.Width,0,-1,info.Height,0,0,1),
|
||||
SKEncodedOrigin.BottomLeft=>new SKMatrix(1,0,0,0,-1,info.Height,0,0,1),
|
||||
SKEncodedOrigin.LeftTop=>new SKMatrix(0,1,0,1,0,0,0,0,1),
|
||||
SKEncodedOrigin.RightTop=>new SKMatrix(0,-1,info.Height,1,0,0,0,0,1),
|
||||
SKEncodedOrigin.RightBottom=>new SKMatrix(0,-1,info.Height,-1,0,info.Width,0,0,1),
|
||||
SKEncodedOrigin.LeftBottom=>new SKMatrix(0,1,0,-1,0,info.Width,0,0,1),
|
||||
_=>SKMatrix.Identity
|
||||
};
|
||||
Directory.CreateDirectory(output);var variants=new List<VariantDto>();
|
||||
foreach(var name in new[]{"thumbnail","crop","optimized"})
|
||||
{
|
||||
var crop=name=="crop";var limit=name=="thumbnail"?256:1280;
|
||||
var scale=Math.Min(1,(double)limit/Math.Max(width,height));
|
||||
int targetWidth=crop?128:Math.Max(1,(int)Math.Floor(width*scale));
|
||||
int targetHeight=crop?128:Math.Max(1,(int)Math.Floor(height*scale));
|
||||
float side=Math.Min(width,height);
|
||||
var rect=crop?new SKRect((width-side)/2,(height-side)/2,(width+side)/2,(height+side)/2):new SKRect(0,0,width,height);
|
||||
using var target=new SKBitmap(new SKImageInfo(targetWidth,targetHeight,SKColorType.Rgba8888,SKAlphaType.Premul));
|
||||
using(var canvas=new SKCanvas(target))
|
||||
{
|
||||
canvas.Clear(SKColors.White);
|
||||
canvas.Scale(targetWidth/rect.Width,targetHeight/rect.Height);canvas.Translate(-rect.Left,-rect.Top);
|
||||
canvas.Concat(matrix);
|
||||
using var paint=new SKPaint { IsAntialias=true };
|
||||
canvas.DrawBitmap(bitmap,0,0,paint);
|
||||
}
|
||||
using var image=SKImage.FromBitmap(target);
|
||||
var webp=name=="optimized";
|
||||
using var data=image.Encode(webp?SKEncodedImageFormat.Webp:SKEncodedImageFormat.Jpeg,webp?80:85);
|
||||
var bytes=data.ToArray();File.WriteAllBytes(Path.Combine(output,name),bytes);
|
||||
variants.Add(new(name,webp?"image/webp":"image/jpeg",bytes.Length,targetWidth,targetHeight,
|
||||
Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant()));
|
||||
}
|
||||
File.WriteAllText(Path.Combine(output,"variants.json"),JsonSerializer.Serialize(variants,JobJson.Options));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
using System.Text.Json;
|
||||
namespace Messenger.Api.Pipeline;
|
||||
public sealed record JobEnvelope(int SchemaVersion, Guid JobId, string Type, Guid AttachmentId,
|
||||
int PipelineVersion, int Attempt, string CreatedAt);
|
||||
public static class JobJson
|
||||
{
|
||||
public static readonly JsonSerializerOptions Options=new() {PropertyNamingPolicy=JsonNamingPolicy.SnakeCaseLower};
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
using System.Text.Json;
|
||||
using Npgsql;
|
||||
using RabbitMQ.Client;
|
||||
namespace Messenger.Api.Pipeline;
|
||||
public static class JobOperations
|
||||
{
|
||||
public static async Task RedeliverAsync(NpgsqlDataSource db,IConfiguration config,Guid id,CancellationToken ct)
|
||||
{
|
||||
await using var connection=await db.OpenConnectionAsync(ct);
|
||||
await using var cmd=JobStore.Command(connection,"SELECT attachment_id,pipeline_version,attempts,state,created_at FROM attachment_jobs WHERE id=@id",("id",id));
|
||||
JobEnvelope job;
|
||||
await using(var r=await cmd.ExecuteReaderAsync(ct)) {
|
||||
if(!await r.ReadAsync(ct))throw new InvalidOperationException("Job not found");
|
||||
job=new(1,id,"attachment.process.v1",r.GetGuid(0),r.GetInt32(1),Math.Max(1,r.GetInt32(2)+(r.GetString(3)=="queued"?1:0)),r.GetDateTime(4).ToString("O"));
|
||||
}
|
||||
await using var broker=await BrokerConnection.OpenAsync(config,ct);
|
||||
await using var channel=await broker.CreateChannelAsync(new CreateChannelOptions(true,true),ct);
|
||||
await BrokerConnection.DeclareAsync(channel,ct);
|
||||
await channel.BasicPublishAsync("",BrokerConnection.Queue,true,new BasicProperties {Persistent=true,ContentType="application/json",MessageId=id.ToString()},JsonSerializer.SerializeToUtf8Bytes(job,JobJson.Options),ct);
|
||||
Console.WriteLine(JsonSerializer.Serialize(job,JobJson.Options));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
using Npgsql;
|
||||
namespace Messenger.Api.Pipeline;
|
||||
public sealed record ClaimedJob(Guid Id,Guid AttachmentId,string StagingKey,string Sha256,long Size,string ContentType,int Attempt,Guid Lease);
|
||||
public sealed class JobStore(NpgsqlDataSource db,IConfiguration config)
|
||||
{
|
||||
public int MaxAttempts=>Math.Clamp(config.GetValue("JOB_MAX_ATTEMPTS",5),1,20);
|
||||
public int TimeoutSeconds=>Math.Clamp(config.GetValue("JOB_TIMEOUT_SECONDS",30),5,180);
|
||||
public static NpgsqlCommand Command(NpgsqlConnection connection,string sql,params (string,object)[] values)
|
||||
{
|
||||
var cmd=new NpgsqlCommand(sql,connection);
|
||||
foreach(var(name,value) in values) cmd.Parameters.AddWithValue(name,value);
|
||||
return cmd;
|
||||
}
|
||||
public async Task<ClaimedJob?> ClaimAsync(NpgsqlConnection connection,JobEnvelope envelope,CancellationToken ct)
|
||||
{
|
||||
await using var tx=await connection.BeginTransactionAsync(ct);
|
||||
var lease=Guid.NewGuid();
|
||||
await using var cmd=Command(connection,"""
|
||||
UPDATE attachment_jobs SET state='processing',attempts=attempts+1,lease_token=@lease,
|
||||
lease_until=clock_timestamp()+make_interval(secs=>@seconds),error_code=NULL
|
||||
WHERE id=@id AND attachment_id=@attachment AND pipeline_version=@version AND state='queued'
|
||||
AND attempts+1=@attempt AND next_at<=clock_timestamp()
|
||||
RETURNING staging_key,attempts
|
||||
""",("id",envelope.JobId),("attachment",envelope.AttachmentId),("version",envelope.PipelineVersion),
|
||||
("attempt",envelope.Attempt),("lease",lease),("seconds",TimeoutSeconds+5));
|
||||
string staging;int attempt;
|
||||
await using(var r=await cmd.ExecuteReaderAsync(ct))
|
||||
{ if(!await r.ReadAsync(ct))return null;staging=r.GetString(0);attempt=r.GetInt32(1); }
|
||||
await using var attachment=Command(connection,"""
|
||||
UPDATE attachments SET status='processing',error_code=NULL,updated_at=clock_timestamp() WHERE id=@id
|
||||
RETURNING sha256,size_bytes,content_type
|
||||
""",("id",envelope.AttachmentId));
|
||||
string hash,mime;long size;
|
||||
await using(var r=await attachment.ExecuteReaderAsync(ct))
|
||||
{await r.ReadAsync(ct);hash=r.GetString(0);size=r.GetInt64(1);mime=r.GetString(2);}
|
||||
await EventAsync(connection,envelope.JobId,"processing",attempt,ct);
|
||||
await tx.CommitAsync(ct);
|
||||
return new(envelope.JobId,envelope.AttachmentId,staging,hash,size,mime,attempt,lease);
|
||||
}
|
||||
public static async Task EventAsync(NpgsqlConnection connection,Guid id,string name,int attempt,CancellationToken ct)
|
||||
{
|
||||
await using var cmd=Command(connection,"INSERT INTO attachment_job_events(job_id,event,attempt) VALUES (@id,@event,@attempt)",
|
||||
("id",id),("event",name),("attempt",attempt));await cmd.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
public async Task FailAsync(NpgsqlConnection connection,ClaimedJob job,bool permanent,string code,CancellationToken ct)
|
||||
{
|
||||
var final=permanent||job.Attempt>=MaxAttempts;
|
||||
var status=final?"failed":"queued";
|
||||
var error=final && !permanent?"retry_exhausted":code;
|
||||
var delay=Math.Min(30,Math.Pow(2,job.Attempt))+Random.Shared.NextDouble();
|
||||
await using var tx=await connection.BeginTransactionAsync(ct);
|
||||
await using var cmd=Command(connection,"""
|
||||
UPDATE attachment_jobs SET state=@status,error_code=@code,lease_token=NULL,lease_until=NULL,
|
||||
next_at=clock_timestamp()+make_interval(secs=>@delay)
|
||||
WHERE id=@id AND state='processing' AND lease_token=@lease
|
||||
""",("id",job.Id),("status",status),("code",error),("delay",delay),("lease",job.Lease));
|
||||
if(await cmd.ExecuteNonQueryAsync(ct)==1)
|
||||
{
|
||||
await using var attachment=Command(connection,"UPDATE attachments SET status=@status,error_code=@code,updated_at=clock_timestamp() WHERE id=@id",
|
||||
("id",job.AttachmentId),("status",status),("code",final?(object)error:DBNull.Value));
|
||||
await attachment.ExecuteNonQueryAsync(ct);
|
||||
if(!final)
|
||||
{
|
||||
await using var outbox=Command(connection,"""
|
||||
INSERT INTO attachment_outbox(job_id,available_at) SELECT id,next_at FROM attachment_jobs WHERE id=@id
|
||||
ON CONFLICT(job_id) DO UPDATE SET published_at=NULL,available_at=EXCLUDED.available_at
|
||||
""",("id",job.Id));await outbox.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
await EventAsync(connection,job.Id,final?"failed":"retry_scheduled",job.Attempt,ct);
|
||||
}
|
||||
await tx.CommitAsync(ct);
|
||||
}
|
||||
public async Task RecoverAsync(CancellationToken ct)
|
||||
{
|
||||
await using var connection=await db.OpenConnectionAsync(ct);
|
||||
var ids=new List<Guid>();
|
||||
await using(var query=Command(connection,"SELECT id FROM attachment_jobs WHERE state='processing' AND lease_until<clock_timestamp() LIMIT 50"))
|
||||
await using(var r=await query.ExecuteReaderAsync(ct))while(await r.ReadAsync(ct))ids.Add(r.GetGuid(0));
|
||||
foreach(var id in ids)
|
||||
{
|
||||
await using var tx=await connection.BeginTransactionAsync(ct);
|
||||
await using var gate=Command(connection,"SELECT pg_try_advisory_xact_lock(hashtextextended(@scope,0))",("scope","job:"+id));
|
||||
if(!(bool)(await gate.ExecuteScalarAsync(ct))!)continue;
|
||||
await using var cmd=Command(connection,"""
|
||||
UPDATE attachment_jobs SET state=CASE WHEN attempts>=@max THEN 'failed' ELSE 'queued' END,
|
||||
error_code=CASE WHEN attempts>=@max THEN 'retry_exhausted' ELSE 'worker_lost' END,
|
||||
lease_token=NULL,lease_until=NULL,next_at=clock_timestamp()
|
||||
WHERE id=@id AND state='processing' AND lease_until<clock_timestamp()
|
||||
RETURNING attachment_id,state,attempts
|
||||
""",("id",id),("max",MaxAttempts));
|
||||
Guid attachment;string status;int attempt;
|
||||
await using(var r=await cmd.ExecuteReaderAsync(ct))
|
||||
{ if(!await r.ReadAsync(ct))continue;attachment=r.GetGuid(0);status=r.GetString(1);attempt=r.GetInt32(2); }
|
||||
await using(var update=Command(connection,"UPDATE attachments SET status=@state,error_code=@code,updated_at=clock_timestamp() WHERE id=@id",
|
||||
("id",attachment),("state",status),("code",status=="failed"?(object)"retry_exhausted":DBNull.Value)))await update.ExecuteNonQueryAsync(ct);
|
||||
if(status=="queued")
|
||||
{
|
||||
await using var outbox=Command(connection,"UPDATE attachment_outbox SET published_at=NULL,available_at=clock_timestamp() WHERE job_id=@id",("id",id));
|
||||
await outbox.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
await EventAsync(connection,id,"lease_recovered",attempt,ct);await tx.CommitAsync(ct);
|
||||
}
|
||||
// A durable intention also recovers a dropped delivery, without changing job identity.
|
||||
await using var rescue=Command(connection,"""
|
||||
UPDATE attachment_outbox o SET published_at=NULL
|
||||
FROM attachment_jobs j WHERE o.job_id=j.id AND j.state='queued'
|
||||
AND o.published_at<clock_timestamp()-interval '30 seconds' AND j.next_at<=clock_timestamp()
|
||||
""");await rescue.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
public async Task<bool> ReplayAsync(Guid id,CancellationToken ct)
|
||||
{
|
||||
await using var connection=await db.OpenConnectionAsync(ct);
|
||||
await using var tx=await connection.BeginTransactionAsync(ct);
|
||||
await using var gate=Command(connection,"SELECT pg_advisory_xact_lock(hashtextextended(@scope,0))",("scope","job:"+id));await gate.ExecuteNonQueryAsync(ct);
|
||||
await using var cmd=Command(connection,"""
|
||||
UPDATE attachment_jobs SET state='queued',attempts=0,error_code=NULL,next_at=clock_timestamp(),replay_count=replay_count+1,
|
||||
lease_token=NULL,lease_until=NULL WHERE id=@id AND state='failed' AND staging_deleted_at IS NULL
|
||||
RETURNING attachment_id
|
||||
""",("id",id));
|
||||
var attachment=await cmd.ExecuteScalarAsync(ct);
|
||||
if(attachment is Guid attachmentId)
|
||||
{
|
||||
await using(var update=Command(connection,"UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=@id",("id",attachmentId)))await update.ExecuteNonQueryAsync(ct);
|
||||
await using(var outbox=Command(connection,"UPDATE attachment_outbox SET published_at=NULL,available_at=clock_timestamp() WHERE job_id=@id",("id",id)))await outbox.ExecuteNonQueryAsync(ct);
|
||||
await EventAsync(connection,id,"operator_replay",0,ct);
|
||||
}
|
||||
await tx.CommitAsync(ct);Console.WriteLine(attachment is Guid?"Replay scheduled":"No-op: job is not failed or staging has expired");
|
||||
return attachment is Guid;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
using System.Text.Json;
|
||||
using Messenger.Api.Infrastructure;
|
||||
using Npgsql;
|
||||
using RabbitMQ.Client;
|
||||
namespace Messenger.Api.Pipeline;
|
||||
public sealed class OutboxDispatcher(NpgsqlDataSource db,S3Storage storage,IConfiguration config)
|
||||
{
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
var jobs=new JobStore(db,config);var maintenance=new PipelineMaintenance(db,storage,config);
|
||||
var nextMaintenance=DateTime.MinValue;
|
||||
while(!ct.IsCancellationRequested)
|
||||
{
|
||||
try {
|
||||
await using var broker=await BrokerConnection.OpenAsync(config,ct);
|
||||
await using var channel=await broker.CreateChannelAsync(new CreateChannelOptions(publisherConfirmationsEnabled:true,publisherConfirmationTrackingEnabled:true),ct);
|
||||
await BrokerConnection.DeclareAsync(channel,ct);
|
||||
while(!ct.IsCancellationRequested && channel.IsOpen)
|
||||
{
|
||||
if(DateTime.UtcNow>=nextMaintenance)
|
||||
{
|
||||
await jobs.RecoverAsync(ct);await maintenance.CleanupAsync(ct);
|
||||
nextMaintenance=DateTime.UtcNow.AddSeconds(5);
|
||||
}
|
||||
await using var connection=await db.OpenConnectionAsync(ct);
|
||||
await using var tx=await connection.BeginTransactionAsync(ct);
|
||||
JobEnvelope? job=null;
|
||||
await using(var select=JobStore.Command(connection,"""
|
||||
SELECT j.id,j.attachment_id,j.pipeline_version,j.attempts+1,j.created_at
|
||||
FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id
|
||||
WHERE o.published_at IS NULL AND o.available_at<=clock_timestamp() AND j.state='queued'
|
||||
ORDER BY o.available_at,o.job_id LIMIT 1 FOR UPDATE OF o SKIP LOCKED
|
||||
"""))
|
||||
await using(var r=await select.ExecuteReaderAsync(ct))
|
||||
if(await r.ReadAsync(ct))job=new(1,r.GetGuid(0),"attachment.process.v1",r.GetGuid(1),r.GetInt32(2),r.GetInt32(3),r.GetDateTime(4).ToString("O"));
|
||||
if(job is null) {await tx.CommitAsync(ct);await Task.Delay(100,ct);continue;}
|
||||
using var confirmation=CancellationTokenSource.CreateLinkedTokenSource(ct);confirmation.CancelAfter(TimeSpan.FromSeconds(5));
|
||||
await channel.BasicPublishAsync("",BrokerConnection.Queue,true,
|
||||
new BasicProperties {Persistent=true,ContentType="application/json",MessageId=job.JobId.ToString()},
|
||||
JsonSerializer.SerializeToUtf8Bytes(job,JobJson.Options),confirmation.Token);
|
||||
await FaultInjection.HitAsync(config,"after_publish",job.JobId,ct);
|
||||
await using var mark=JobStore.Command(connection,"UPDATE attachment_outbox SET published_at=clock_timestamp() WHERE job_id=@id",("id",job.JobId));
|
||||
await mark.ExecuteNonQueryAsync(ct);await JobStore.EventAsync(connection,job.JobId,"published",job.Attempt,ct);await tx.CommitAsync(ct);
|
||||
}
|
||||
} catch(OperationCanceledException) when(ct.IsCancellationRequested) {break;}
|
||||
catch(Exception ex) {Console.WriteLine("Outbox deferred: "+ex.GetType().Name);await Task.Delay(2000,ct);}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using Amazon.S3.Model;
|
||||
using Messenger.Api.Data;
|
||||
using Messenger.Api.Infrastructure;
|
||||
using Npgsql;
|
||||
namespace Messenger.Api.Pipeline;
|
||||
public sealed class PipelineMaintenance(NpgsqlDataSource db,S3Storage storage,IConfiguration config)
|
||||
{
|
||||
public async Task CleanupAsync(CancellationToken ct)
|
||||
{
|
||||
await using var connection=await db.OpenConnectionAsync(ct);
|
||||
var rows=new List<(Guid Id,string Key)>();
|
||||
var retention=Math.Max(3600,config.GetValue("STAGING_RETENTION_SECONDS",86400));
|
||||
await using(var cmd=JobStore.Command(connection,"""
|
||||
SELECT id,staging_key FROM attachment_jobs WHERE staging_deleted_at IS NULL
|
||||
AND (state='ready' OR state='failed' AND created_at<clock_timestamp()-make_interval(secs=>@retention)) LIMIT 20
|
||||
""",("retention",retention)))
|
||||
await using(var r=await cmd.ExecuteReaderAsync(ct))while(await r.ReadAsync(ct))rows.Add((r.GetGuid(0),r.GetString(1)));
|
||||
foreach(var row in rows)
|
||||
{
|
||||
await using var tx=await connection.BeginTransactionAsync(ct);
|
||||
await using var gate=JobStore.Command(connection,"SELECT pg_try_advisory_xact_lock(hashtextextended(@scope,0))",("scope","job:"+row.Id));
|
||||
if(!(bool)(await gate.ExecuteScalarAsync(ct))!)continue;
|
||||
await using var check=JobStore.Command(connection,"""
|
||||
SELECT state FROM attachment_jobs WHERE id=@id AND staging_deleted_at IS NULL
|
||||
AND (state='ready' OR state='failed' AND created_at<clock_timestamp()-make_interval(secs=>@retention)) FOR UPDATE
|
||||
""",("id",row.Id),("retention",retention));
|
||||
if(await check.ExecuteScalarAsync(ct) is not string)continue;
|
||||
await storage.Client.DeleteObjectAsync(storage.Bucket,row.Key,ct);
|
||||
await using var mark=JobStore.Command(connection,"UPDATE attachment_jobs SET staging_deleted_at=clock_timestamp() WHERE id=@id",("id",row.Id));
|
||||
await mark.ExecuteNonQueryAsync(ct);await tx.CommitAsync(ct);
|
||||
}
|
||||
}
|
||||
public async Task ReconcileStagingAsync(bool apply,CancellationToken ct)
|
||||
{
|
||||
var actions=new List<string>();
|
||||
await using var connection=await db.OpenConnectionAsync(ct);
|
||||
await using var tx=await connection.BeginTransactionAsync(ct);
|
||||
await using(var gate=JobStore.Command(connection,$"SELECT pg_advisory_xact_lock({AppDatabase.AttachmentGate})"))await gate.ExecuteNonQueryAsync(ct);
|
||||
string? continuation=null;
|
||||
do {
|
||||
var page=await storage.Client.ListObjectsV2Async(new ListObjectsV2Request {BucketName=storage.Bucket,Prefix="staging/",MaxKeys=1000,ContinuationToken=continuation},ct);
|
||||
foreach(var obj in page.S3Objects ?? [])
|
||||
{
|
||||
if(obj.LastModified is null || obj.LastModified.Value.ToUniversalTime()>DateTime.UtcNow.AddHours(-1))continue;
|
||||
await using var check=JobStore.Command(connection,"SELECT EXISTS(SELECT 1 FROM attachment_jobs WHERE staging_key=@key)",("key",obj.Key));
|
||||
if(!(bool)(await check.ExecuteScalarAsync(ct))!) {
|
||||
actions.Add(obj.Key);if(apply)await storage.Client.DeleteObjectAsync(storage.Bucket,obj.Key,ct);
|
||||
}
|
||||
}
|
||||
continuation=page.IsTruncated==true?page.NextContinuationToken:null;
|
||||
}while(continuation is not null);
|
||||
await tx.CommitAsync(ct);Console.WriteLine(System.Text.Json.JsonSerializer.Serialize(new {mode=apply?"apply":"dry-run",old_untracked_staging=actions}));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
using System.Security.Cryptography;
|
||||
using Amazon.S3.Model;
|
||||
using Messenger.Api.Contracts;
|
||||
using Messenger.Api.Infrastructure;
|
||||
using Npgsql;
|
||||
using RabbitMQ.Client;
|
||||
using RabbitMQ.Client.Events;
|
||||
namespace Messenger.Api.Pipeline;
|
||||
public sealed class PipelineWorker(NpgsqlDataSource db,S3Storage storage,IConfiguration config)
|
||||
{
|
||||
private readonly JobStore jobs=new(db,config);
|
||||
public async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
var concurrency=Math.Clamp(config.GetValue("WORKER_CONCURRENCY",2),1,8);
|
||||
var prefetch=(ushort)Math.Clamp(config.GetValue("WORKER_PREFETCH",1),1,16);
|
||||
while(!ct.IsCancellationRequested)
|
||||
{
|
||||
try {
|
||||
await using var connection=await BrokerConnection.OpenAsync(config,ct);
|
||||
var channels=new List<IChannel>();
|
||||
try {
|
||||
for(var i=0;i<concurrency;i++)
|
||||
{
|
||||
var channel=await connection.CreateChannelAsync(cancellationToken:ct);channels.Add(channel);
|
||||
await BrokerConnection.DeclareAsync(channel,ct);
|
||||
await channel.BasicQosAsync(0,prefetch,false,ct);
|
||||
var consumer=new AsyncEventingBasicConsumer(channel);
|
||||
consumer.ReceivedAsync+=async(_,delivery)=> {
|
||||
try {
|
||||
var envelope=JsonSerializer.Deserialize<JobEnvelope>(delivery.Body.Span,JobJson.Options);
|
||||
if(envelope is not null && envelope.SchemaVersion==1 && envelope.Type=="attachment.process.v1" && envelope.PipelineVersion==1)
|
||||
await ProcessAsync(envelope,ct);
|
||||
await channel.BasicAckAsync(delivery.DeliveryTag,false,ct);
|
||||
} catch(JsonException) { await channel.BasicRejectAsync(delivery.DeliveryTag,false,ct); }
|
||||
catch(OperationCanceledException) when(ct.IsCancellationRequested) { }
|
||||
catch(Exception ex) {
|
||||
Console.WriteLine("Delivery deferred: "+ex.GetType().Name);
|
||||
try {await channel.BasicNackAsync(delivery.DeliveryTag,false,true,ct);}catch { }
|
||||
await Task.Delay(500,ct);
|
||||
}
|
||||
};
|
||||
await channel.BasicConsumeAsync(BrokerConnection.Queue,false,consumer,ct);
|
||||
}
|
||||
Console.WriteLine($"Worker ready: concurrency={concurrency}, prefetch={prefetch}");
|
||||
while(!ct.IsCancellationRequested && connection.IsOpen && channels.All(c=>c.IsOpen))await Task.Delay(500,ct);
|
||||
} finally { foreach(var channel in channels)await channel.DisposeAsync(); }
|
||||
} catch(OperationCanceledException) when(ct.IsCancellationRequested) {break;}
|
||||
catch(Exception ex) {Console.WriteLine("Worker connection unavailable: "+ex.GetType().Name);await Task.Delay(2000,ct);}
|
||||
}
|
||||
}
|
||||
private async Task ProcessAsync(JobEnvelope envelope,CancellationToken stop)
|
||||
{
|
||||
await using var connection=await db.OpenConnectionAsync(stop);
|
||||
await using var gate=JobStore.Command(connection,"SELECT pg_try_advisory_lock(hashtextextended(@scope,0))",("scope","job:"+envelope.JobId));
|
||||
if(!(bool)(await gate.ExecuteScalarAsync(stop))!)return;
|
||||
try {
|
||||
var job=await jobs.ClaimAsync(connection,envelope,stop);if(job is null)return;
|
||||
Console.WriteLine($"Processing job {job.Id}, attempt {job.Attempt}");
|
||||
var directory=Path.Combine(Path.GetTempPath(),"job-"+Guid.NewGuid().ToString("N"));Directory.CreateDirectory(directory);
|
||||
using var timeout=CancellationTokenSource.CreateLinkedTokenSource(stop);timeout.CancelAfter(TimeSpan.FromSeconds(jobs.TimeoutSeconds));
|
||||
var ct=timeout.Token;
|
||||
try {
|
||||
var input=Path.Combine(directory,"original");
|
||||
using(var source=await storage.Client.GetObjectAsync(storage.Bucket,job.StagingKey,ct))
|
||||
{
|
||||
if(source.ContentLength>10485760)throw new InvalidDataException("invalid_image");
|
||||
await using var file=File.Create(input);var buffer=new byte[65536];long count=0;
|
||||
while(true) {
|
||||
var n=await source.ResponseStream.ReadAsync(buffer,ct);if(n==0)break;
|
||||
count+=n;if(count>10485760)throw new InvalidDataException("invalid_image");
|
||||
await file.WriteAsync(buffer.AsMemory(0,n),ct);
|
||||
}
|
||||
}
|
||||
await using(var source=File.OpenRead(input))
|
||||
if(source.Length!=job.Size || Convert.ToHexString(await SHA256.HashDataAsync(source,ct)).ToLowerInvariant()!=job.Sha256)
|
||||
throw new InvalidDataException("invalid_image");
|
||||
var render=new ProcessStartInfo("dotnet") {UseShellExecute=false,RedirectStandardOutput=true,RedirectStandardError=true};
|
||||
render.ArgumentList.Add(typeof(PipelineWorker).Assembly.Location);render.ArgumentList.Add("--render-image");
|
||||
render.ArgumentList.Add(input);render.ArgumentList.Add(directory);
|
||||
using(var process=Process.Start(render)!)
|
||||
{
|
||||
var stdout=process.StandardOutput.ReadToEndAsync(ct);var stderr=process.StandardError.ReadToEndAsync(ct);
|
||||
try { await process.WaitForExitAsync(ct);await Task.WhenAll(stdout,stderr); }
|
||||
catch { if(!process.HasExited)process.Kill(true);await process.WaitForExitAsync(CancellationToken.None);throw; }
|
||||
if(process.ExitCode==2)throw new InvalidDataException("invalid_image");
|
||||
if(process.ExitCode!=0)throw new IOException("renderer_unavailable");
|
||||
}
|
||||
var variants=JsonSerializer.Deserialize<VariantDto[]>(await File.ReadAllTextAsync(Path.Combine(directory,"variants.json"),ct),JobJson.Options)!;
|
||||
await PutAsync($"originals/{job.AttachmentId:N}/v1",input,job.ContentType,ct);
|
||||
foreach(var variant in variants)await PutAsync($"variants/{job.AttachmentId:N}/v1/{variant.Name}",Path.Combine(directory,variant.Name),variant.ContentType,ct);
|
||||
await FaultInjection.HitAsync(config,"after_objects",job.Id,ct);
|
||||
await using(var tx=await connection.BeginTransactionAsync(ct))
|
||||
{
|
||||
await using var done=JobStore.Command(connection,"""
|
||||
UPDATE attachment_jobs SET state='ready',completed_at=clock_timestamp(),error_code=NULL,lease_token=NULL,lease_until=NULL
|
||||
WHERE id=@id AND state='processing' AND lease_token=@lease AND lease_until>clock_timestamp()
|
||||
""",("id",job.Id),("lease",job.Lease));
|
||||
if(await done.ExecuteNonQueryAsync(ct)!=1)throw new OperationCanceledException("Lease expired");
|
||||
await using var result=JobStore.Command(connection,"UPDATE attachments SET status='ready',variants=CAST(@variants AS jsonb),error_code=NULL,updated_at=clock_timestamp() WHERE id=@id",
|
||||
("id",job.AttachmentId),("variants",JsonSerializer.Serialize(variants,JobJson.Options)));await result.ExecuteNonQueryAsync(ct);
|
||||
await JobStore.EventAsync(connection,job.Id,"ready",job.Attempt,ct);await tx.CommitAsync(ct);
|
||||
}
|
||||
await FaultInjection.HitAsync(config,"after_commit",job.Id,stop);
|
||||
Console.WriteLine($"Ready job {job.Id}");
|
||||
} catch(Exception ex) when(!stop.IsCancellationRequested) {
|
||||
var permanent=ex is InvalidDataException || ex is Amazon.S3.AmazonS3Exception s3 && s3.StatusCode==System.Net.HttpStatusCode.NotFound;
|
||||
var code=permanent?"invalid_image":ex is OperationCanceledException?"processing_timeout":"dependency_unavailable";
|
||||
await jobs.FailAsync(connection,job,permanent,code,stop);
|
||||
Console.WriteLine($"Job {job.Id}: {code} ({ex.GetType().Name})");
|
||||
}
|
||||
finally {Directory.Delete(directory,true);}
|
||||
}
|
||||
finally {
|
||||
await using var unlock=JobStore.Command(connection,"SELECT pg_advisory_unlock(hashtextextended(@scope,0))",("scope","job:"+envelope.JobId));
|
||||
await unlock.ExecuteNonQueryAsync(CancellationToken.None);
|
||||
}
|
||||
}
|
||||
private async Task PutAsync(string key,string path,string mime,CancellationToken ct)
|
||||
{
|
||||
await using var file=File.OpenRead(path);
|
||||
await storage.Client.PutObjectAsync(new PutObjectRequest {BucketName=storage.Bucket,Key=key,InputStream=file,ContentType=mime,UseChunkEncoding=false,AutoCloseStream=false},ct);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
using Npgsql;
|
||||
namespace Messenger.Api.Pipeline;
|
||||
public static class ProcessPresence
|
||||
{
|
||||
public static async Task RunAsync(NpgsqlDataSource source,string role,CancellationToken ct)
|
||||
{
|
||||
try {
|
||||
while(!ct.IsCancellationRequested) {
|
||||
try {
|
||||
await using var cmd=source.CreateCommand("INSERT INTO pipeline_presence(role,seen_at) VALUES(@role,clock_timestamp()) ON CONFLICT(role) DO UPDATE SET seen_at=EXCLUDED.seen_at");
|
||||
cmd.Parameters.AddWithValue("role",role);await cmd.ExecuteNonQueryAsync(ct);
|
||||
} catch(Exception) when(!ct.IsCancellationRequested) { }
|
||||
await Task.Delay(2000,ct);
|
||||
}
|
||||
} catch(OperationCanceledException) when(ct.IsCancellationRequested) { }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
using Messenger.Api.Pipeline;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Messenger.Api.Data;
|
||||
using Messenger.Api.Endpoints;
|
||||
using Messenger.Api.Infrastructure;
|
||||
using Messenger.Api.Middleware;
|
||||
using Messenger.Api.Services;
|
||||
using Npgsql;
|
||||
using StackExchange.Redis;
|
||||
|
||||
ThreadPool.GetMinThreads(out var workerThreads, out var completionThreads);
|
||||
ThreadPool.SetMinThreads(Math.Max(8, workerThreads), completionThreads);
|
||||
|
||||
if(args.Length==3 && args[0]=="--render-image")
|
||||
{
|
||||
try {ImageRenderer.Render(args[1],args[2]);}
|
||||
catch(InvalidDataException) {Environment.ExitCode=2;}
|
||||
catch {Environment.ExitCode=3;}
|
||||
return;
|
||||
}
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
if(args.Contains("--bootstrap"))
|
||||
{
|
||||
using var storage=new S3Storage(builder.Configuration);
|
||||
try { await storage.Client.GetBucketLocationAsync(storage.Bucket); }
|
||||
catch(Amazon.S3.AmazonS3Exception ex) when(ex.StatusCode==System.Net.HttpStatusCode.NotFound)
|
||||
{ await storage.Client.PutBucketAsync(storage.Bucket); }
|
||||
Console.WriteLine("Private bucket ready");
|
||||
return;
|
||||
}
|
||||
|
||||
var host = Environment.GetEnvironmentVariable("APP_HOST") ?? "0.0.0.0";
|
||||
var port = Environment.GetEnvironmentVariable("APP_PORT") ?? "8080";
|
||||
builder.WebHost.UseUrls($"http://{host}:{port}");
|
||||
|
||||
builder.Services.ConfigureHttpJsonOptions(options =>
|
||||
{
|
||||
options.SerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower;
|
||||
options.SerializerOptions.UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow;
|
||||
});
|
||||
|
||||
var databaseUrlFile = Environment.GetEnvironmentVariable("DATABASE_URL_FILE");
|
||||
var databaseUrl = databaseUrlFile is not null
|
||||
? File.ReadAllText(databaseUrlFile).Trim()
|
||||
: Environment.GetEnvironmentVariable("DATABASE_URL");
|
||||
if (string.IsNullOrWhiteSpace(databaseUrl))
|
||||
throw new InvalidOperationException("Set DATABASE_URL_FILE or DATABASE_URL before starting the API");
|
||||
|
||||
var connectionString = new NpgsqlConnectionStringBuilder(databaseUrl)
|
||||
{
|
||||
MaxPoolSize = int.TryParse(builder.Configuration["DB_POOL_SIZE"], out var poolSize) && poolSize > 0 ? poolSize : 20,
|
||||
Timeout = 5,
|
||||
ApplicationName = "messenger-api",
|
||||
CommandTimeout = 10
|
||||
};
|
||||
var dataSource = NpgsqlDataSource.Create(connectionString.ConnectionString);
|
||||
if (args.Contains("--migrate", StringComparer.Ordinal))
|
||||
{
|
||||
await DatabaseMigrator.ApplyAsync(dataSource);
|
||||
await dataSource.DisposeAsync();
|
||||
return;
|
||||
}
|
||||
if(args.Contains("--worker") || args.Contains("--dispatcher") || args.Contains("--replay") || args.Contains("--redeliver") || args.Contains("--reconcile"))
|
||||
{
|
||||
using var storage=new S3Storage(builder.Configuration);
|
||||
using var cancellation=new CancellationTokenSource();
|
||||
Console.CancelKeyPress+=(_,e)=>{e.Cancel=true;cancellation.Cancel();};
|
||||
using var signal=System.Runtime.InteropServices.PosixSignalRegistration.Create(System.Runtime.InteropServices.PosixSignal.SIGTERM,ctx=>{ctx.Cancel=true;cancellation.Cancel();});
|
||||
var heartbeat=(args.Contains("--worker") || args.Contains("--dispatcher"))
|
||||
? ProcessPresence.RunAsync(dataSource,args.Contains("--worker")?"worker":"dispatcher",cancellation.Token) : Task.CompletedTask;
|
||||
try {
|
||||
if(args.Contains("--worker"))await new PipelineWorker(dataSource,storage,builder.Configuration).RunAsync(cancellation.Token);
|
||||
else if(args.Contains("--dispatcher"))await new OutboxDispatcher(dataSource,storage,builder.Configuration).RunAsync(cancellation.Token);
|
||||
else if(args.Contains("--redeliver"))await JobOperations.RedeliverAsync(dataSource,builder.Configuration,Guid.Parse(args[Array.IndexOf(args,"--redeliver")+1]),cancellation.Token);
|
||||
else if(args.Contains("--replay"))await new JobStore(dataSource,builder.Configuration).ReplayAsync(Guid.Parse(args[Array.IndexOf(args,"--replay")+1]),cancellation.Token);
|
||||
else await new PipelineMaintenance(dataSource,storage,builder.Configuration).ReconcileStagingAsync(args.Contains("--apply"),cancellation.Token);
|
||||
}catch(OperationCanceledException) when(cancellation.IsCancellationRequested) { }
|
||||
cancellation.Cancel();await heartbeat;
|
||||
await dataSource.DisposeAsync();return;
|
||||
}
|
||||
builder.Services.AddSingleton(dataSource);
|
||||
builder.Services.AddSingleton<InstanceIdentity>();
|
||||
builder.Services.AddSingleton<ReadinessState>();
|
||||
builder.Services.Configure<HostOptions>(options => options.ShutdownTimeout = TimeSpan.FromSeconds(8));
|
||||
builder.Services.AddSingleton<AppDatabase>();
|
||||
builder.Services.AddSingleton<S3Storage>();
|
||||
builder.Services.AddSingleton<AttachmentService>();
|
||||
|
||||
var redisOptions = ConfigurationOptions.Parse(builder.Configuration["REDIS_URL"] ?? "localhost:6379");
|
||||
redisOptions.AbortOnConnectFail = false;
|
||||
redisOptions.ConnectTimeout = 500;
|
||||
redisOptions.ConnectRetry = 0;
|
||||
redisOptions.AsyncTimeout = 500;
|
||||
redisOptions.SyncTimeout = 500;
|
||||
redisOptions.BacklogPolicy = BacklogPolicy.FailFast;
|
||||
builder.Services.AddSingleton<IConnectionMultiplexer>(await ConnectionMultiplexer.ConnectAsync(redisOptions));
|
||||
builder.Services.AddSingleton<RedisSessionCache>();
|
||||
builder.Services.AddSingleton<PasswordService>();
|
||||
builder.Services.AddSingleton<UserService>();
|
||||
builder.Services.AddSingleton<SessionService>();
|
||||
builder.Services.AddSingleton<ChatService>();
|
||||
builder.Services.AddSingleton<MessageService>();
|
||||
builder.Services.AddSingleton<ClusterPresence>();
|
||||
builder.Services.AddHostedService(services => services.GetRequiredService<ClusterPresence>());
|
||||
|
||||
var app = builder.Build();
|
||||
app.Lifetime.ApplicationStopping.Register(app.Services.GetRequiredService<ReadinessState>().Stop);
|
||||
|
||||
app.UseMiddleware<RequestIdMiddleware>();
|
||||
app.UseMiddleware<InstanceMiddleware>();
|
||||
app.UseMiddleware<RequestLoggingMiddleware>();
|
||||
app.UseMiddleware<ExceptionHandlingMiddleware>();
|
||||
app.UseDefaultFiles();
|
||||
app.UseStaticFiles(new StaticFileOptions
|
||||
{
|
||||
OnPrepareResponse = context => context.Context.Response.Headers.CacheControl = "no-cache"
|
||||
});
|
||||
|
||||
app.MapHealthEndpoints();
|
||||
app.MapUserEndpoints();
|
||||
app.MapAuthEndpoints();
|
||||
app.MapChatEndpoints();
|
||||
app.MapMessageEndpoints();
|
||||
app.MapAttachmentEndpoints();
|
||||
app.MapUiEndpoints();
|
||||
app.MapOperationsEndpoints();
|
||||
app.MapReactionEndpoints();
|
||||
|
||||
app.Run();
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/launchsettings.json",
|
||||
"profiles": {
|
||||
"http": {
|
||||
"commandName": "Project",
|
||||
"dotnetRunMessages": true,
|
||||
"launchBrowser": true,
|
||||
"applicationUrl": "http://localhost:5052",
|
||||
"environmentVariables": {
|
||||
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||
}
|
||||
},
|
||||
"https": {
|
||||
"commandName": "Project",
|
||||
"dotnetRunMessages": true,
|
||||
"launchBrowser": true,
|
||||
"applicationUrl": "https://localhost:7253;http://localhost:5052",
|
||||
"environmentVariables": {
|
||||
"ASPNETCORE_ENVIRONMENT": "Development"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using Messenger.Api.Contracts;
|
||||
|
||||
namespace Messenger.Api.Services;
|
||||
|
||||
public static class ApiSupport
|
||||
{
|
||||
public static async Task<(T? Value, IResult? Error)> ReadJsonAsync<T>(HttpContext context)
|
||||
{
|
||||
if (!context.Request.HasJsonContentType())
|
||||
return (default, ApiError(context, 400, "invalid_request", "Требуется application/json"));
|
||||
|
||||
try
|
||||
{
|
||||
var value = await context.Request.ReadFromJsonAsync<T>();
|
||||
return value is null
|
||||
? (default, ApiError(context, 400, "invalid_request", "Тело запроса обязательно"))
|
||||
: (value, null);
|
||||
}
|
||||
catch (Exception exception) when (exception is JsonException or NotSupportedException or BadHttpRequestException)
|
||||
{
|
||||
return (default, ApiError(context, 400, "invalid_request", "Некорректный JSON"));
|
||||
}
|
||||
}
|
||||
|
||||
public static IResult Unauthorized(HttpContext context, bool basic = false)
|
||||
{
|
||||
context.Response.Headers["WWW-Authenticate"] = basic
|
||||
? "Basic realm=\"messenger\", charset=\"UTF-8\""
|
||||
: "Bearer realm=\"messenger\"";
|
||||
return ApiError(context, 401, "unauthorized", "Требуется действительная сессия");
|
||||
}
|
||||
|
||||
public static (ChatCursor? Cursor, DateTime? Time, IResult? Error) ParseCursor(
|
||||
HttpContext context, string scope)
|
||||
{
|
||||
if (!context.Request.Query.TryGetValue("cursor", out var value))
|
||||
return (null, null, null);
|
||||
|
||||
var cursor = CursorService.DecodeChatCursor(value.ToString());
|
||||
if (cursor is null || cursor.Scope != scope || cursor.Id == Guid.Empty ||
|
||||
!DateTime.TryParse(cursor.CreatedAt, CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.RoundtripKind, out var time) || time.Kind != DateTimeKind.Utc)
|
||||
return (null, null, ApiError(context, 400, "invalid_request", "Некорректный cursor"));
|
||||
|
||||
return (cursor, time, null);
|
||||
}
|
||||
|
||||
public static (int Limit, IResult? Error) ParseLimit(HttpContext context)
|
||||
{
|
||||
const int defaultLimit = 50;
|
||||
|
||||
if (!context.Request.Query.TryGetValue("limit", out var value))
|
||||
return (defaultLimit, null);
|
||||
|
||||
if (!int.TryParse(value.ToString(), out var limit) ||
|
||||
limit < 1 ||
|
||||
limit > 100)
|
||||
{
|
||||
return (
|
||||
0,
|
||||
ApiError(context, 400, "invalid_request", "limit должен быть от 1 до 100")
|
||||
);
|
||||
}
|
||||
|
||||
return (limit, null);
|
||||
}
|
||||
|
||||
public static IResult ApiError(
|
||||
HttpContext context,
|
||||
int status,
|
||||
string code,
|
||||
string message)
|
||||
{
|
||||
var requestId =
|
||||
context.Items["RequestId"]?.ToString()
|
||||
?? Guid.NewGuid().ToString();
|
||||
|
||||
return Results.Json(
|
||||
new
|
||||
{
|
||||
error = new
|
||||
{
|
||||
code,
|
||||
message,
|
||||
request_id = requestId
|
||||
}
|
||||
},
|
||||
statusCode: status
|
||||
);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user