Compare commits
1
Commits
main
...
lab6-final
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
258013b2d6 |
+17
-10
@@ -3,12 +3,12 @@
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SERVICE_VERSION=6.0.0
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
# Redis/Valkey is not used: sessions live in PostgreSQL.
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
@@ -18,13 +18,20 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
# Durable broker + worker (lab5)
|
||||
# BROKER_USER / BROKER_PASSWORD are read by compose for the broker container;
|
||||
# keep real values only in your local .env.
|
||||
BROKER_USER=change-me
|
||||
BROKER_PASSWORD=change-me
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
JOB_QUEUE=attachment_jobs
|
||||
STAGING_PREFIX=staging
|
||||
MAX_JOB_ATTEMPTS=5
|
||||
THUMBNAIL_SIZE=256
|
||||
THUMBNAIL_QUALITY=85
|
||||
# Observability (lab6): OTLP over gRPC to the collector.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=grpc
|
||||
OTEL_METRIC_INTERVAL_MS=10000
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
GRAFANA_ADMIN_PASSWORD=change-me
|
||||
|
||||
@@ -1,18 +1,27 @@
|
||||
## Лабораторная 6
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Лесковский Сергей БПИ2304
|
||||
Целевая оценка (3/4/5): 3
|
||||
Трек 5, если заявлен: Не заявлен
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено:
|
||||
Весь минимум lab5 сохранён. Добавлено: OTel Collector contrib 0.115.1, Prometheus 2.55.1 (метрики), Loki 3.2.1 (логи), Tempo 2.6.1 (трассы), Grafana 11.3.1 с provisioning datasources и импортируемым dashboard; OTLP-инструментация API (server spans, RPS/errors/duration) и worker (span attachment.process, attachment.jobs, attachment.job.duration); структурные JSON-логи с trace_id/span_id/request_id; W3C traceparent API → broker → worker; memory_limiter и batch в Collector.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Команды запуска из чистого clone:
|
||||
См. REPORT.md → «Запуск из чистого clone». Кратко: сгенерировать secrets и .env; docker compose --env-file .env build api1 && docker compose --env-file .env up -d; curl /health/ready; python tests\smoke.py; Grafana http://localhost:3000 (папка Messenger → dashboard messenger-lab6), Prometheus http://localhost:9090.
|
||||
|
||||
Результаты проверок и ссылка на REPORT.md:
|
||||
Smoke PASS при включённой телеметрии; persistence PASS; failover PASS (104 записи, 0 ошибок, recovery 1.639 s). В Prometheus видны http_server_requests_total и attachment_jobs_total по outcome; в Tempo — реальные API- и worker-трассы, связанные traceparent; в Loki — логи с trace_id/span_id. Доказательства — evidence/smoke.txt, metric-*.json, tr-traces.json, logs-trace.json, grafana-ds-*.json, docker-compose-ps.txt.
|
||||
|
||||
Ограничения / известные ошибки:
|
||||
Уровень 4 не заявлен (queue depth/oldest job age, DB pool/worker saturation, alert rules, p50/p95/p99 везде). Один Docker-host, HA хранилищ телеметрии нет; Grafana анонимна для учебного стенда. При остановке telemetry backend API продолжает работу, телеметрия за это окно теряется.
|
||||
|
||||
Источники и использование ИИ:
|
||||
OpenTelemetry SDK 1.28.2, instrumentation-fastapi 0.49b2, OTel Collector contrib 0.115.1, Prometheus 2.55.1, Loki 3.2.1, Tempo 2.6.1, Grafana 11.3.1, FastAPI 0.115, RabbitMQ 3.13, PostgreSQL 16.4, MinIO. ИИ использовался для консультаций по OTel SDK и provisioning Grafana.
|
||||
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] make check и make test выполнены; приложен реальный вывод.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
FROM python:3.12-slim AS runtime
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_DEFAULT_TIMEOUT=60 \
|
||||
PIP_RETRIES=2
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
|
||||
# Пробуем несколько зеркал по очереди. Первое, которое ответит, используется.
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
set -e; \
|
||||
for idx in \
|
||||
"https://pypi.depkit.ru/simple/" \
|
||||
"https://pypi-mirror.gitverse.ru/simple" \
|
||||
"https://pypi.org/simple/" \
|
||||
; do \
|
||||
echo "==> Trying index: $idx"; \
|
||||
if pip install --no-cache-dir --index-url "$idx" -r requirements.txt; then \
|
||||
echo "==> OK: $idx"; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
echo "==> FAILED: $idx"; \
|
||||
done; \
|
||||
echo "==> All indexes failed"; \
|
||||
exit 1
|
||||
|
||||
COPY src ./src
|
||||
|
||||
RUN useradd -r -u 10001 -s /usr/sbin/nologin appuser
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1"]
|
||||
@@ -1,37 +1,65 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- **Автор, группа:** Лесковский Сергей БПИ2304
|
||||
- **Целевая оценка; выбранный трек 5, если нужен:** 3; трек 5 не заявлен
|
||||
- **Commit/PR:** ветка `lab6-final` форка `Leskovskiy/mtusi2026lab6-Leskovskiy`, PR в `mtusi2026/mtusi2026lab6`
|
||||
- **Использованные источники, библиотеки и помощь ИИ:** OpenTelemetry SDK/экспортёры 1.28.2, instrumentation-fastapi 0.49b2, OTel Collector contrib 0.115.1, Prometheus 2.55.1, Loki 3.2.1, Tempo 2.6.1, Grafana 11.3.1, FastAPI 0.115, SQLAlchemy 2.0, pika 1.3.2, RabbitMQ 3.13, PostgreSQL 16.4, MinIO. ИИ использовался для консультаций по OTel SDK и provisioning Grafana.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требования: Docker с Compose v2, Python 3 (только для тестов/нагрузки), Git Bash на Windows (для resilience-обёрток).
|
||||
|
||||
1. Сгенерировать локальные секреты `secrets/*.txt` (postgres, database_url, s3, broker) и `.env` по блоку из `REPORT.md` репозитория lab5; добавить `BROKER_USER`, `BROKER_PASSWORD`, `OTEL_*`, `GRAFANA_ADMIN_PASSWORD`. Секреты в Git не попадают.
|
||||
2. `docker compose --env-file .env build api1 && docker compose --env-file .env up -d`
|
||||
3. Дождаться `curl.exe -fsS http://localhost:8080/health/ready`.
|
||||
4. `$env:PYTHONUTF8="1"; python tests\smoke.py --base-url http://localhost:8080`
|
||||
5. Открыть Grafana: `http://localhost:3000` (анонимный admin в учебном стенде), папка Messenger → dashboard «Messenger — lab6 observability». Prometheus — `http://localhost:9090`.
|
||||
6. Остановка: `docker compose --env-file .env down`. Удаление учебных данных: `docker compose --env-file .env down -v`.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
api1/api2/worker ── OTLP/gRPC ──→ otel-collector ──→ Tempo (traces)
|
||||
├──────────→ Loki (logs)
|
||||
└──────────→ Prometheus scrape :8889 (metrics)
|
||||
Grafana ──→ Prometheus + Loki + Tempo (provisioned datasources и dashboard)
|
||||
```
|
||||
|
||||
- **Роли хранилищ:** Prometheus хранит метрики (histogram/counter), Tempo — трассы, Loki — структурные логи; Grafana только читает их и не хранит сигналы.
|
||||
- **Collector:** `otlp` receiver (gRPC 4317/HTTP 4318), `memory_limiter` (256 MiB, spike 64) и `batch`; экспортёры prometheus/tempo/loki. Collector не является долговременным хранилищем.
|
||||
- **API:** FastAPIInstrumentor (server spans), middleware считает `http.server.requests` и `http.server.request.duration` с атрибутами `http.request.method`, `http.route` (шаблон, не UUID), `http.response.status_code`. JSON-лог на запрос содержит `request_id` и `trace_id`/`span_id`.
|
||||
- **Worker:** span `attachment.process` (CONSUMER), счётчик `attachment.jobs` и гистограмма `attachment.job.duration` с `job.type` и `job.outcome` (ready/failed/retry/duplicate). Логи `job.ready`/`job.failed` содержат `attachment_id` (технический ID синтетических данных) и `trace_id`.
|
||||
- **Связь API → broker → worker:** W3C `traceparent` инжектится в сообщение при публикации и извлекается worker-ом; worker-span — продолжение trace upload-запроса.
|
||||
- **Приватность:** в телеметрию не попадают пароли, токены, Authorization, тела сообщений, байты изображений, signed URL. `user_id`/`chat_id`/`message_id`/`attachment_id` не используются как labels метрик. Health исключён из метрик (`excluded_urls` и фильтр в dashboard).
|
||||
- **Ресурсы:** `service.name` = messenger-api / messenger-worker, `service.version` = 6.0.0, `service.instance.id` = api-1/api-2/worker, `deployment.environment.name` = lab.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Весь минимум lab5 работает | `python tests\smoke.py` | `PASS: lab6 public API scenario` | `evidence/smoke.txt` |
|
||||
| Collector + 3 хранилища + Grafana в стенде | `docker compose ps` | postgres/s3/broker healthy, otel-collector, prometheus, loki, tempo, grafana Up | `evidence/docker-compose-ps.txt` |
|
||||
| RPS/errors/duration API | Prometheus query | `http_server_requests_total`, `http_server_request_duration_seconds` | `evidence/metric-http-rps.json` |
|
||||
| Успешные/ошибочные jobs и duration worker | Prometheus query | `attachment_jobs_total{job_outcome=…}`, `attachment_job_duration_seconds` | `evidence/metric-jobs.json` |
|
||||
| Реальные трассы API и worker | Tempo search | API spans и worker `attachment.process` | `evidence/tr-traces.json` |
|
||||
| Связь API → worker | Tempo search по trace | один trace: upload → worker (W3C traceparent) | `evidence/tr-traces.json` |
|
||||
| Структурные логи с trace_id/span_id | Loki query_range | JSON-лог с `trace_id`, `span_id`, `request_id` по найденному trace | `evidence/logs-trace.json` |
|
||||
| Импортируемый dashboard | provisioning | dashboard uid `messenger-lab6` в папке Messenger, 6 панелей | `infra/grafana/dashboards/messenger.json` |
|
||||
| Datasources доступны | Grafana health API | Tempo/Prometheus — OK, Loki отвечает на запросы | `evidence/grafana-ds-*.json` |
|
||||
| Сохранность и failover при включённой телеметрии | resilience-тесты | persistence PASS; failover 104 записи, 0 ошибок | `evidence/persistence.txt`, `evidence/failover-1.txt` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Рестарт всего стека (persistence hook) | Данные/сессии/вложения сохранены, телеметрия продолжается | `PASS: persistence preserved...` | потерь нет |
|
||||
| Остановка одной API-реплики (failover hook) | LB переключает трафик, метрики обеих реплик видны | 104 записи, 0 ошибок, recovery 1.639 s | потерь нет |
|
||||
| Остановка телеметрии (Prometheus/Loki/Tempo) | API/worker продолжают работу; экспортёры ограничены memory_limiter/батчем | API отвечает 200, ошибок в логах приложения нет | потерь данных нет; телеметрия за окно недоступности теряется |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Уровень 4 не заявлен: нет queue depth/oldest job age, p50/p95/p99 по всем панелям с alert rules, DB pool и worker saturation, alert rules и их firing/resolved. Сквозная связь traceparent реализована как бонус к уровню 3.
|
||||
- Метрики отдаются pull-моделью через Prometheus-экспортёр Collector; период scrape 5 s, retention 15 d.
|
||||
- Один Docker-host; HA хранилищ телеметрии не заявляется. Grafana открыта анонимно для учебного стенда — для реального контура нужен SSO/пароль.
|
||||
- В логах сохраняется `attachment_id` (синтетические данные) для отладки; в продакшене это требует обоснования и политики retention.
|
||||
- `make test` требует запущенного стенда; при отключённой телеметрии API продолжает отвечать (проверено остановкой backend).
|
||||
|
||||
+213
@@ -0,0 +1,213 @@
|
||||
name: lab6
|
||||
|
||||
x-api-env: &api_env
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: lab
|
||||
SERVICE_VERSION: "6.0.0"
|
||||
SESSION_TTL_SECONDS: "3600"
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_QUEUE: attachment_jobs
|
||||
STAGING_PREFIX: staging
|
||||
MAX_JOB_ATTEMPTS: "5"
|
||||
THUMBNAIL_SIZE: "256"
|
||||
MAX_UPLOAD_BYTES: "10485760"
|
||||
MAX_IMAGE_PIXELS: "40000000"
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317
|
||||
OTEL_METRIC_INTERVAL_MS: "10000"
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.4-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
POSTGRES_DB: messenger
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
secrets:
|
||||
- postgres_password
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
|
||||
s3:
|
||||
image: milvusdb/minio:RELEASE.2024-12-18T13-15-44Z
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
|
||||
volumes:
|
||||
- s3data:/data
|
||||
secrets:
|
||||
- s3_root_user
|
||||
- s3_root_password
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS http://localhost:9000/minio/health/live || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
|
||||
broker:
|
||||
image: rabbitmq:3.13-management-alpine
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: ${BROKER_USER:?set BROKER_USER in .env}
|
||||
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?set BROKER_PASSWORD in .env}
|
||||
volumes:
|
||||
- rabbitdata:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
|
||||
api1:
|
||||
build: .
|
||||
image: lab6-api:local
|
||||
environment:
|
||||
<<: *api_env
|
||||
INSTANCE_ID: api-1
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- broker_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
|
||||
api2:
|
||||
image: lab6-api:local
|
||||
environment:
|
||||
<<: *api_env
|
||||
INSTANCE_ID: api-2
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- broker_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
|
||||
worker:
|
||||
image: lab6-api:local
|
||||
command: ["python", "-m", "src.worker"]
|
||||
environment:
|
||||
<<: *api_env
|
||||
INSTANCE_ID: worker
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- broker_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
|
||||
lb:
|
||||
image: nginx:1.27.1-alpine
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
depends_on:
|
||||
- api1
|
||||
- api2
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.115.1
|
||||
command: ["--config=/etc/otelcol/config.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.55.1
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=15d
|
||||
volumes:
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- promdata:/prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
depends_on:
|
||||
- otel-collector
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.2.1
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- lokidata:/loki
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.6.1
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempodata:/var/tempo
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.3.1
|
||||
environment:
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me}
|
||||
GF_USERS_DEFAULT_THEME: dark
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafanadata:/var/lib/grafana
|
||||
ports:
|
||||
- "3000:3000"
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
|
||||
volumes:
|
||||
pgdata: {}
|
||||
s3data: {}
|
||||
rabbitdata: {}
|
||||
promdata: {}
|
||||
lokidata: {}
|
||||
tempodata: {}
|
||||
grafanadata: {}
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password.txt
|
||||
database_url:
|
||||
file: ./secrets/database_url.txt
|
||||
s3_root_user:
|
||||
file: ./secrets/s3_root_user.txt
|
||||
s3_root_password:
|
||||
file: ./secrets/s3_root_password.txt
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id.txt
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key.txt
|
||||
broker_url:
|
||||
file: ./secrets/broker_url.txt
|
||||
@@ -0,0 +1,13 @@
|
||||
SERVICE STATUS
|
||||
api1 Up 46 seconds
|
||||
api2 Up 46 seconds
|
||||
broker Up 4 minutes (healthy)
|
||||
grafana Up 4 minutes
|
||||
lb Up 3 minutes
|
||||
loki Up 4 minutes
|
||||
otel-collector Up 4 minutes
|
||||
postgres Up 4 minutes (healthy)
|
||||
prometheus Up 4 minutes
|
||||
s3 Up 4 minutes (healthy)
|
||||
tempo Up 4 minutes
|
||||
worker Up 46 seconds
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
{"message":"Data source successfully connected.","status":"OK"}
|
||||
@@ -0,0 +1 @@
|
||||
{"details":{"application":"Prometheus","features":{"rulerApiEnabled":false}},"message":"Successfully queried the Prometheus API.","status":"OK"}
|
||||
@@ -0,0 +1 @@
|
||||
{"message":"Method not implemented","messageId":"plugin.notImplemented","statusCode":404,"traceID":""}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1 @@
|
||||
{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1795108961.821,"0.027683377390749272"]}]}}
|
||||
@@ -0,0 +1 @@
|
||||
{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"attachment_jobs_total","exported_instance":"worker","exported_job":"messenger-worker","instance":"otel-collector:8889","job":"otel-collector","job_outcome":"duplicate","job_type":"attachment.process.v1"},"value":[1795108961.844,"1"]},{"metric":{"__name__":"attachment_jobs_total","exported_instance":"worker","exported_job":"messenger-worker","instance":"otel-collector:8889","job":"otel-collector","job_outcome":"ready","job_type":"attachment.process.v1"},"value":[1795108961.844,"1"]}]}}
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1 @@
|
||||
{"traces":[{"traceID":"cea4fbd4ccc714db4cffa35e3c608139","rootServiceName":"messenger-api","rootTraceName":"POST /api/v1/chats/{chat_id}/attachments","startTimeUnixNano":"1795108925352572507","durationMs":275},{"traceID":"ff2253c00f06cd5fd5e5f4f47fbbadab","rootServiceName":"messenger-worker","rootTraceName":"attachment.process","startTimeUnixNano":"1795108812348163525","durationMs":3},{"traceID":"bf6eef1581cd68bf80af8a689a2f4b7e","rootServiceName":"messenger-api","rootTraceName":"POST /api/v1/chats/{chat_id}/attachments","startTimeUnixNano":"1795108812081229408","durationMs":266}],"metrics":{"inspectedTraces":77,"inspectedBytes":"39870","completedJobs":1,"totalJobs":1}}
|
||||
@@ -0,0 +1,108 @@
|
||||
{
|
||||
"uid": "messenger-lab6",
|
||||
"title": "Messenger — lab6 observability",
|
||||
"tags": ["messenger", "lab6"],
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "10s",
|
||||
"editable": true,
|
||||
"time": { "from": "now-30m", "to": "now" },
|
||||
"timezone": "utc",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "API RPS (excluding health)",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(http_server_requests_total{http_route!~\"/health/.*\"}[1m]))",
|
||||
"legendFormat": "RPS"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "API 5xx ratio",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0 }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(http_server_requests_total{http_response_status_code=~\"5..\"}[1m])) / clamp_min(sum(rate(http_server_requests_total{http_route!~\"/health/.*\"}[1m])), 0.001)",
|
||||
"legendFormat": "5xx / business"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "HTTP duration p95",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "p50"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Attachment jobs by outcome",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (job_outcome) (rate(attachment_jobs_total[1m]))",
|
||||
"legendFormat": "{{job_outcome}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Worker job duration p95 by outcome",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le, job_outcome) (rate(attachment_job_duration_seconds_bucket[5m])))",
|
||||
"legendFormat": "{{job_outcome}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "API request rate by route (top 10)",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "topk(10, sum by (http_route, http_request_method) (rate(http_server_requests_total{http_route!~\"/health/.*\"}[1m])))",
|
||||
"legendFormat": "{{http_request_method}} {{http_route}}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: messenger
|
||||
orgId: 1
|
||||
folder: Messenger
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
allowUiUpdates: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,28 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
jsonData:
|
||||
timeInterval: 5s
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: "-1m"
|
||||
spanEndTimeShift: "1m"
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
@@ -0,0 +1,26 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2020-10-24
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
@@ -0,0 +1,18 @@
|
||||
events {}
|
||||
http {
|
||||
upstream api {
|
||||
server api1:8080 max_fails=1 fail_timeout=5s;
|
||||
server api2:8080 max_fails=1 fail_timeout=5s;
|
||||
}
|
||||
server {
|
||||
listen 8080;
|
||||
client_max_body_size 11m;
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_read_timeout 30s;
|
||||
proxy_send_timeout 30s;
|
||||
location / {
|
||||
proxy_pass http://api;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 256
|
||||
spike_limit_mib: 64
|
||||
batch:
|
||||
timeout: 5s
|
||||
send_batch_size: 1024
|
||||
send_batch_max_size: 2048
|
||||
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: false
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true
|
||||
loki:
|
||||
endpoint: http://loki:3100/loki/api/v1/push
|
||||
|
||||
service:
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [loki]
|
||||
@@ -0,0 +1,8 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8889"]
|
||||
@@ -0,0 +1,19 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
local:
|
||||
path: /var/tempo/traces
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
@@ -0,0 +1,12 @@
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.30.6
|
||||
sqlalchemy==2.0.35
|
||||
psycopg[binary]==3.2.3
|
||||
boto3==1.35.36
|
||||
argon2-cffi==23.1.0
|
||||
pillow==10.4.0
|
||||
pydantic==2.9.2
|
||||
pika==1.3.2
|
||||
opentelemetry-sdk==1.28.2
|
||||
opentelemetry-exporter-otlp==1.28.2
|
||||
opentelemetry-instrumentation-fastapi==0.49b2
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lab5 failure drills: broker restart durability and bounded retry.
|
||||
|
||||
A) Stop the worker, accept an upload, restart the broker (persistent volume),
|
||||
start the worker: the accepted job still becomes ready.
|
||||
B) Accept an upload while the worker is down, stop S3, start the worker:
|
||||
transient failures exhaust MAX_JOB_ATTEMPTS and the attachment ends as
|
||||
failed with a safe error_code. S3 is restored afterwards.
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, arguments, check, run # noqa: E402
|
||||
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(COMPOSE + list(args), cwd=ROOT, check=True,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def wait_status(c, actor, att, wanted, seconds=120):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
row = c.call("GET", "/api/v1/attachments/" + att["id"], actor=actor, schema="Attachment")["json"]
|
||||
if row["status"] in wanted:
|
||||
return row
|
||||
time.sleep(0.5)
|
||||
raise AssertionError("attachment did not reach " + repr(wanted))
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("lab5 failure drills")
|
||||
c = Client(args.base_url, args.ca_file, timeout=10)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
|
||||
print("A. broker restart keeps accepted work")
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
att, _content = c.attachment(alice, chat)
|
||||
compose("restart", "broker")
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except Exception:
|
||||
check(time.monotonic() < deadline, "API did not recover after broker restart")
|
||||
time.sleep(0.5)
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
started = time.monotonic()
|
||||
ready = c.wait_ready(bob, att, wait_seconds=90)
|
||||
print(json.dumps({
|
||||
"scenario": "broker_restart",
|
||||
"attachment_id": att["id"],
|
||||
"status": ready["status"],
|
||||
"seconds_to_ready": round(time.monotonic() - started, 3),
|
||||
}, ensure_ascii=False))
|
||||
check(ready["status"] == "ready", "job lost after broker restart")
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
|
||||
print("B. bounded retry ends in a safe failed state")
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
att, _content = c.attachment(alice, chat)
|
||||
compose("stop", "s3")
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
failed = wait_status(c, bob, att, {"failed"}, seconds=180)
|
||||
print(json.dumps({
|
||||
"scenario": "retry_exhausted",
|
||||
"attachment_id": att["id"],
|
||||
"status": failed["status"],
|
||||
"error_code": failed["error_code"],
|
||||
}, ensure_ascii=False))
|
||||
check(failed["error_code"] in ("processing_failed", "invalid_image"), "unsafe error_code")
|
||||
c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 409, actor=bob)
|
||||
finally:
|
||||
compose("up", "-d", "s3")
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except Exception:
|
||||
check(time.monotonic() < deadline, "S3 did not recover")
|
||||
time.sleep(0.5)
|
||||
print("PASS: broker durability and bounded retry demonstrated.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Demonstrate the lab5 acceptance scenario without hiding infrastructure steps.
|
||||
|
||||
Worker stopped -> upload returns 202 queued -> message references the chat
|
||||
attachment -> API replicas restarted -> worker started -> the same job
|
||||
reaches ready with a contract-compliant JPEG thumbnail.
|
||||
|
||||
The script prints only synthetic IDs and timings. It never prints tokens.
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, arguments, check, run # noqa: E402
|
||||
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(COMPOSE + list(args), cwd=ROOT, check=True,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("lab5 worker-outage demonstration")
|
||||
c = Client(args.base_url, args.ca_file, timeout=10)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
|
||||
print("1. stop worker")
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
print("2. upload with worker down")
|
||||
att, content = c.attachment(alice, chat)
|
||||
check(att["status"] == "queued", "queued expected after upload")
|
||||
meta = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"]
|
||||
check(meta["status"] in ("queued", "processing"), "still unprocessed while worker is down")
|
||||
pending = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 409, actor=bob)
|
||||
check(pending["status"] == 409, "content must be 409 while queued")
|
||||
|
||||
print("3. attach the queued attachment to a message")
|
||||
msg = c.message(alice, chat, "", attachments=[att["id"]])["json"]
|
||||
check(msg["attachment_ids"] == [att["id"]], "message lost the attachment")
|
||||
|
||||
print("4. restart API replicas (accepted work must survive)")
|
||||
compose("restart", "api1", "api2")
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except Exception:
|
||||
check(time.monotonic() < deadline, "API did not recover")
|
||||
time.sleep(0.5)
|
||||
|
||||
print("5. start worker")
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
started = time.monotonic()
|
||||
ready = c.wait_ready(bob, att, wait_seconds=90)
|
||||
elapsed = round(time.monotonic() - started, 3)
|
||||
check(ready["status"] == "ready", "job did not reach ready")
|
||||
variants = [v for v in ready["variants"] if v["name"] == "thumbnail"]
|
||||
check(len(variants) == 1, "expected exactly one thumbnail")
|
||||
v = variants[0]
|
||||
check((v["width"], v["height"]) == (256, 160), "thumbnail dimensions differ")
|
||||
thumb = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob)
|
||||
check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail must be JPEG")
|
||||
from image_probe import jpeg_size
|
||||
check(jpeg_size(thumb["bytes"]) == (256, 160), "actual JPEG size differs")
|
||||
|
||||
print(json.dumps({
|
||||
"attachment_id": att["id"],
|
||||
"message_id": msg["id"],
|
||||
"status_after_worker_restart": ready["status"],
|
||||
"thumbnail": {"width": v["width"], "height": v["height"],
|
||||
"size_bytes": v["size_bytes"], "sha256": v["sha256"]},
|
||||
"seconds_from_worker_start_to_ready": elapsed,
|
||||
}, ensure_ascii=False))
|
||||
print("PASS: queued job completed after API and worker restart.")
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Executable → Regular
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
docker compose --env-file .env restart postgres s3 broker api1 api2 worker lb
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
"C:\Program Files\Git\bin\bash.exe" "%~dp0restart.sh"
|
||||
Executable → Regular
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
api-1) SVC=api1 ;;
|
||||
api-2) SVC=api2 ;;
|
||||
*) echo "unknown instance alias: $TARGET_INSTANCE_ID" >&2; exit 1 ;;
|
||||
esac
|
||||
docker compose --env-file .env stop "$SVC"
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
"C:\Program Files\Git\bin\bash.exe" "%~dp0stop-one.sh"
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
import hashlib
|
||||
import secrets
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from fastapi import Depends, Request
|
||||
from sqlalchemy.orm import Session
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerifyMismatchError
|
||||
from . import config
|
||||
from .db import get_db
|
||||
from .models import User, Session as DbSession
|
||||
from .errors import ApiError
|
||||
|
||||
_ph = PasswordHasher()
|
||||
_BEARER = 'Bearer realm="messenger"'
|
||||
|
||||
def hash_password(p: str) -> str:
|
||||
return _ph.hash(p)
|
||||
|
||||
def verify_password(h: str, p: str) -> bool:
|
||||
try:
|
||||
return _ph.verify(h, p)
|
||||
except VerifyMismatchError:
|
||||
return False
|
||||
|
||||
def new_session(db: Session, user: User) -> tuple[str, datetime]:
|
||||
token = secrets.token_urlsafe(48)
|
||||
token_hash = hashlib.sha256(token.encode()).hexdigest()
|
||||
expires = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
|
||||
db.add(DbSession(token_hash=token_hash, user_id=user.id, expires_at=expires))
|
||||
db.commit()
|
||||
return token, expires
|
||||
|
||||
def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
||||
auth = request.headers.get("authorization", "")
|
||||
if not auth.lower().startswith("bearer "):
|
||||
raise ApiError(401, "unauthorized", "missing bearer token", challenge=_BEARER)
|
||||
token = auth[7:].strip()
|
||||
token_hash = hashlib.sha256(token.encode()).hexdigest()
|
||||
s = db.get(DbSession, token_hash)
|
||||
if not s:
|
||||
raise ApiError(401, "unauthorized", "invalid session", challenge=_BEARER)
|
||||
if s.expires_at <= datetime.now(timezone.utc):
|
||||
raise ApiError(401, "unauthorized", "expired session", challenge=_BEARER)
|
||||
u = db.get(User, s.user_id)
|
||||
if not u:
|
||||
raise ApiError(401, "unauthorized", "user gone", challenge=_BEARER)
|
||||
return u
|
||||
@@ -0,0 +1,36 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
def _read_secret(env_name: str, file_env: str, default=None):
|
||||
path = os.environ.get(file_env)
|
||||
if path:
|
||||
return Path(path).read_text().strip()
|
||||
return os.environ.get(env_name, default)
|
||||
|
||||
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = int(os.environ.get("APP_PORT", "8080"))
|
||||
APP_ENV = os.environ.get("APP_ENV", "lab")
|
||||
SESSION_TTL_SECONDS = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
|
||||
|
||||
DATABASE_URL = _read_secret("DATABASE_URL", "DATABASE_URL_FILE")
|
||||
|
||||
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000")
|
||||
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
|
||||
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY = _read_secret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
|
||||
S3_SECRET_KEY = _read_secret("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
|
||||
|
||||
MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", str(10 * 1024 * 1024)))
|
||||
MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
|
||||
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID", "api-unknown")
|
||||
|
||||
# --- lab5: durable broker + worker ---
|
||||
BROKER_URL = _read_secret("BROKER_URL", "BROKER_URL_FILE", "amqp://messenger:messenger@broker:5672/")
|
||||
QUEUE_NAME = os.environ.get("JOB_QUEUE", "attachment_jobs")
|
||||
STAGING_PREFIX = os.environ.get("STAGING_PREFIX", "staging")
|
||||
THUMBNAIL_SIZE = int(os.environ.get("THUMBNAIL_SIZE", "256"))
|
||||
THUMBNAIL_QUALITY = int(os.environ.get("THUMBNAIL_QUALITY", "85"))
|
||||
MAX_JOB_ATTEMPTS = int(os.environ.get("MAX_JOB_ATTEMPTS", "5"))
|
||||
OUTBOX_INTERVAL_SECONDS = float(os.environ.get("OUTBOX_INTERVAL_SECONDS", "2"))
|
||||
OUTBOX_GRACE_SECONDS = float(os.environ.get("OUTBOX_GRACE_SECONDS", "0"))
|
||||
@@ -0,0 +1,16 @@
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker, DeclarativeBase
|
||||
from . import config
|
||||
|
||||
engine = create_engine(config.DATABASE_URL, pool_pre_ping=True, pool_size=5, max_overflow=10)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,29 @@
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, challenge: str | None = None):
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.challenge = challenge
|
||||
|
||||
def _rid(request: Request) -> str:
|
||||
return getattr(request.state, "request_id", None) or "unknown"
|
||||
|
||||
async def api_error_handler(request: Request, exc: ApiError):
|
||||
headers = {"X-Request-Id": _rid(request)}
|
||||
if exc.challenge:
|
||||
headers["WWW-Authenticate"] = exc.challenge
|
||||
return JSONResponse(
|
||||
status_code=exc.status,
|
||||
content={"error": {"code": exc.code, "message": exc.message, "request_id": _rid(request)}},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
async def unhandled_handler(request: Request, exc: Exception):
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": {"code": "internal_error", "message": "internal error", "request_id": _rid(request)}},
|
||||
headers={"X-Request-Id": _rid(request)},
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
import io
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from . import config
|
||||
from .errors import ApiError
|
||||
|
||||
def validate_image(data: bytes) -> tuple[str, int, int, int]:
|
||||
"""Возвращает (content_type, width, height, size)."""
|
||||
if not data:
|
||||
raise ApiError(400, "invalid_request", "empty body")
|
||||
if len(data) > config.MAX_UPLOAD_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "file too large")
|
||||
head = data[:8]
|
||||
if head.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
ctype = "image/png"
|
||||
elif head.startswith(b"\xff\xd8\xff"):
|
||||
ctype = "image/jpeg"
|
||||
else:
|
||||
raise ApiError(415, "unsupported_media_type", "only PNG/JPEG")
|
||||
try:
|
||||
img = Image.open(io.BytesIO(data))
|
||||
w, h = img.size
|
||||
except UnidentifiedImageError:
|
||||
raise ApiError(415, "unsupported_media_type", "invalid image")
|
||||
if w * h > config.MAX_IMAGE_PIXELS:
|
||||
raise ApiError(413, "payload_too_large", "too many pixels")
|
||||
return ctype, w, h, len(data)
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
"""Broker access: durable job publication and the transactional-outbox pump.
|
||||
|
||||
The broker message carries only identifiers and metadata (never bytes or
|
||||
credentials). The job row in PostgreSQL is the source of truth; `published_at`
|
||||
turns the row into a small transactional outbox so a crash between COMMIT and
|
||||
publish is repaired by the background pump.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
import pika
|
||||
from sqlalchemy import select
|
||||
|
||||
from . import config
|
||||
from .db import SessionLocal
|
||||
from .models import AttachmentJob
|
||||
|
||||
log = logging.getLogger("jobs")
|
||||
|
||||
TYPE_V1 = "attachment.process.v1"
|
||||
|
||||
|
||||
def build_payload(job: AttachmentJob, attempt: int, carrier: dict | None = None) -> dict:
|
||||
from opentelemetry.propagate import inject
|
||||
if carrier is None:
|
||||
carrier = {}
|
||||
inject(carrier) # W3C traceparent when a request span is active
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"job_id": str(job.id),
|
||||
"type": TYPE_V1,
|
||||
"attachment_id": str(job.attachment_id),
|
||||
"pipeline_version": job.pipeline_version,
|
||||
"attempt": attempt,
|
||||
"created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
if carrier.get("traceparent"):
|
||||
payload["traceparent"] = carrier["traceparent"]
|
||||
if carrier.get("tracestate"):
|
||||
payload["tracestate"] = carrier["tracestate"]
|
||||
return payload
|
||||
|
||||
|
||||
def publish(payload: dict) -> None:
|
||||
params = pika.URLParameters(config.BROKER_URL)
|
||||
params.heartbeat = 30
|
||||
params.blocked_connection_timeout = 30
|
||||
connection = pika.BlockingConnection(params)
|
||||
try:
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=config.QUEUE_NAME, durable=True)
|
||||
channel.confirm_delivery()
|
||||
channel.basic_publish(
|
||||
exchange="",
|
||||
routing_key=config.QUEUE_NAME,
|
||||
body=json.dumps(payload).encode("utf-8"),
|
||||
properties=pika.BasicProperties(
|
||||
delivery_mode=2, # persistent
|
||||
content_type="application/json",
|
||||
message_id=payload["job_id"],
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def publish_job_row(job_id: uuid.UUID, attempt: int, carrier: dict | None = None) -> None:
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
if not job or job.state in ("done", "failed"):
|
||||
return
|
||||
payload = build_payload(job, attempt, carrier)
|
||||
publish(payload)
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
if job:
|
||||
job.published_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
|
||||
|
||||
def pump_once(limit: int = 50) -> int:
|
||||
"""Publish queued rows whose outbox timestamp is still empty."""
|
||||
now = datetime.now(timezone.utc)
|
||||
cutoff = now - timedelta(seconds=config.OUTBOX_GRACE_SECONDS)
|
||||
published = 0
|
||||
with SessionLocal() as db:
|
||||
rows = (
|
||||
db.execute(
|
||||
select(AttachmentJob)
|
||||
.where(AttachmentJob.published_at.is_(None))
|
||||
.where(AttachmentJob.state == "queued")
|
||||
.where(AttachmentJob.created_at <= cutoff)
|
||||
.order_by(AttachmentJob.created_at)
|
||||
.limit(limit)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
pending = [(job.id, job.attempt + 1) for job in rows]
|
||||
for job_id, attempt in pending:
|
||||
try:
|
||||
publish_job_row(job_id, attempt)
|
||||
published += 1
|
||||
except Exception as exc: # broker may still be starting
|
||||
log.warning("outbox publish failed for %s: %s", job_id, exc)
|
||||
return published
|
||||
|
||||
|
||||
def start_outbox_thread() -> threading.Thread:
|
||||
def loop():
|
||||
while True:
|
||||
try:
|
||||
count = pump_once()
|
||||
if count:
|
||||
log.info("outbox published %d job(s)", count)
|
||||
except Exception as exc:
|
||||
log.warning("outbox pump error: %s", exc)
|
||||
time.sleep(config.OUTBOX_INTERVAL_SECONDS)
|
||||
|
||||
thread = threading.Thread(target=loop, name="outbox", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI, Request
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
|
||||
from . import config, jobs, telemetry
|
||||
from .db import engine, Base
|
||||
from .errors import ApiError, api_error_handler, unhandled_handler
|
||||
from .s3 import ensure_bucket
|
||||
from .routers import health, users, sessions, chats, messages, attachments
|
||||
|
||||
tracer_provider, meter_provider = telemetry.setup_telemetry("messenger-api", config.INSTANCE_ID)
|
||||
request_counter, request_duration = telemetry.http_instruments()
|
||||
logger = logging.getLogger("messenger.api")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# create_all is not idempotent under parallel startup: two replicas
|
||||
# may race and one hits UniqueViolation on pg_type. Catch and re-run
|
||||
# once after a short delay; by then the other replica has finished.
|
||||
import asyncio
|
||||
from sqlalchemy.exc import IntegrityError, OperationalError
|
||||
|
||||
for attempt in range(5):
|
||||
try:
|
||||
Base.metadata.create_all(engine)
|
||||
break
|
||||
except (IntegrityError, OperationalError) as exc:
|
||||
print(f"WARN: create_all attempt {attempt+1} failed: {exc}")
|
||||
await asyncio.sleep(1.0)
|
||||
else:
|
||||
print("WARN: create_all failed after 5 attempts; continuing")
|
||||
|
||||
try:
|
||||
ensure_bucket()
|
||||
except Exception as exc:
|
||||
print(f"WARN: ensure_bucket failed: {exc}")
|
||||
jobs.start_outbox_thread()
|
||||
logger.info("api.started", extra={"event_fields": json.dumps({"instance_id": config.INSTANCE_ID})})
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
app.add_exception_handler(ApiError, api_error_handler)
|
||||
app.add_exception_handler(Exception, unhandled_handler)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def observe_request(request: Request, call_next):
|
||||
started = time.perf_counter()
|
||||
response = await call_next(request)
|
||||
route = request.scope.get("route")
|
||||
template = getattr(route, "path", request.url.path)
|
||||
attributes = {
|
||||
"http.request.method": request.method,
|
||||
"http.route": template,
|
||||
"http.response.status_code": response.status_code,
|
||||
}
|
||||
request_counter.add(1, attributes)
|
||||
request_duration.record(time.perf_counter() - started, attributes)
|
||||
logger.info("http.request", extra={"event_fields": json.dumps({
|
||||
"http.request.method": request.method,
|
||||
"http.route": template,
|
||||
"http.response.status_code": response.status_code,
|
||||
"request_id": getattr(request.state, "request_id", None)
|
||||
or request.headers.get("X-Request-Id"),
|
||||
})})
|
||||
return response
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def add_request_id(request: Request, call_next):
|
||||
rid = request.headers.get("X-Request-Id") or uuid.uuid4().hex
|
||||
request.state.request_id = rid
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-Id"] = rid
|
||||
response.headers["X-Instance-Id"] = config.INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
for r in (health, users, sessions, chats, messages, attachments):
|
||||
app.include_router(r.router)
|
||||
|
||||
FastAPIInstrumentor.instrument_app(app, excluded_urls="health/live,health/ready")
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import String, DateTime, ForeignKey, Integer, UniqueConstraint, Index
|
||||
from sqlalchemy.dialects.postgresql import UUID as PGUUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .db import Base
|
||||
|
||||
|
||||
def _uuid() -> uuid.UUID:
|
||||
return uuid.uuid4()
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
username: Mapped[str] = mapped_column(String(32), unique=True, index=True)
|
||||
display_name: Mapped[str] = mapped_column(String(100))
|
||||
password_hash: Mapped[str] = mapped_column(String(255))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
|
||||
|
||||
class Session(Base):
|
||||
__tablename__ = "sessions"
|
||||
token_hash: Mapped[str] = mapped_column(String(64), primary_key=True) # sha256(token) hex
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
|
||||
|
||||
class Chat(Base):
|
||||
__tablename__ = "chats"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
title: Mapped[str] = mapped_column(String(100))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
|
||||
|
||||
|
||||
class ChatMember(Base):
|
||||
__tablename__ = "chat_members"
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
|
||||
|
||||
class Message(Base):
|
||||
__tablename__ = "messages"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
sender_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id")
|
||||
)
|
||||
text: Mapped[str] = mapped_column(String(4000), default="")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
|
||||
|
||||
|
||||
class MessageAttachment(Base):
|
||||
__tablename__ = "message_attachments"
|
||||
message_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("attachments.id"), primary_key=True
|
||||
)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
|
||||
class IdempotencyKey(Base):
|
||||
__tablename__ = "idempotency_keys"
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
method: Mapped[str] = mapped_column(String(8), primary_key=True)
|
||||
path: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
key: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
request_hash: Mapped[str] = mapped_column(String(64))
|
||||
response_json: Mapped[str] = mapped_column(String)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
|
||||
__table_args__ = (Index("ix_idem_created", "created_at"),)
|
||||
|
||||
|
||||
class Attachment(Base):
|
||||
__tablename__ = "attachments"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
owner_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id")
|
||||
)
|
||||
filename: Mapped[str] = mapped_column(String(255))
|
||||
content_type: Mapped[str] = mapped_column(String(32))
|
||||
size_bytes: Mapped[int] = mapped_column(Integer)
|
||||
sha256: Mapped[str] = mapped_column(String(64))
|
||||
s3_key: Mapped[str] = mapped_column(String(255))
|
||||
staging_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(16), default="ready")
|
||||
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AttachmentJob(Base):
|
||||
"""Durable intent to process one attachment (transactional outbox row)."""
|
||||
__tablename__ = "attachment_jobs"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), unique=True
|
||||
)
|
||||
pipeline_version: Mapped[int] = mapped_column(Integer, default=1)
|
||||
state: Mapped[str] = mapped_column(String(16), default="queued") # queued|processing|done|failed
|
||||
attempt: Mapped[int] = mapped_column(Integer, default=0)
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AttachmentVariant(Base):
|
||||
__tablename__ = "attachment_variants"
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
name: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
content_type: Mapped[str] = mapped_column(String(32))
|
||||
size_bytes: Mapped[int] = mapped_column(Integer)
|
||||
width: Mapped[int] = mapped_column(Integer)
|
||||
height: Mapped[int] = mapped_column(Integer)
|
||||
sha256: Mapped[str] = mapped_column(String(64))
|
||||
s3_key: Mapped[str] = mapped_column(String(255))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
@@ -0,0 +1,122 @@
|
||||
import hashlib
|
||||
import uuid
|
||||
from fastapi import APIRouter, Depends, Request, Query
|
||||
from fastapi.responses import Response
|
||||
from sqlalchemy.orm import Session
|
||||
from ..db import get_db
|
||||
from ..models import Attachment, ChatMember, AttachmentJob, AttachmentVariant
|
||||
from ..auth import current_user
|
||||
from ..errors import ApiError
|
||||
from .. import s3, config, jobs
|
||||
from ..images import validate_image
|
||||
from ..schemas import AttachmentOut, VariantOut
|
||||
|
||||
router = APIRouter(prefix="/api/v1", tags=["attachments"])
|
||||
|
||||
def _bad_filename(name: str) -> bool:
|
||||
return (not name) or len(name) > 255 or any(c in name for c in ("/", "\\", "\x00")) or any(ord(c) < 32 for c in name)
|
||||
|
||||
def _member_chat_or_404(db, chat_id, user_id):
|
||||
if not db.get(ChatMember, (chat_id, user_id)):
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
|
||||
def _attachment_for_member(db, att_id, user_id) -> Attachment:
|
||||
att = db.get(Attachment, att_id)
|
||||
if not att:
|
||||
raise ApiError(404, "not_found", "attachment not found")
|
||||
_member_chat_or_404(db, att.chat_id, user_id)
|
||||
return att
|
||||
|
||||
def _attachment_out(db, att: Attachment) -> AttachmentOut:
|
||||
variants = db.query(AttachmentVariant).filter(
|
||||
AttachmentVariant.attachment_id == att.id
|
||||
).all()
|
||||
return AttachmentOut(
|
||||
id=att.id, chat_id=att.chat_id, owner_id=att.owner_id,
|
||||
filename=att.filename, content_type=att.content_type,
|
||||
size_bytes=att.size_bytes, sha256=att.sha256,
|
||||
status=att.status, error_code=att.error_code,
|
||||
variants=[VariantOut.model_validate(v) for v in variants],
|
||||
created_at=att.created_at, updated_at=att.updated_at,
|
||||
)
|
||||
|
||||
@router.post("/chats/{chat_id}/attachments", status_code=202, response_model=AttachmentOut)
|
||||
async def upload(chat_id: uuid.UUID,
|
||||
filename: str = Query(..., min_length=1, max_length=255),
|
||||
request: Request = None,
|
||||
db: Session = Depends(get_db),
|
||||
user=Depends(current_user)):
|
||||
_member_chat_or_404(db, chat_id, user.id)
|
||||
if _bad_filename(filename):
|
||||
raise ApiError(400, "invalid_request", "invalid filename")
|
||||
ctype_header = request.headers.get("content-type", "").split(";")[0].strip()
|
||||
if ctype_header not in ("image/png", "image/jpeg"):
|
||||
raise ApiError(415, "unsupported_media_type", "expected image/png or image/jpeg")
|
||||
data = await request.body()
|
||||
ctype, w, h, size = validate_image(data)
|
||||
if ctype != ctype_header:
|
||||
raise ApiError(415, "unsupported_media_type", "content-type mismatch with file")
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
|
||||
att_id = uuid.uuid4()
|
||||
final_key = f"chats/{chat_id}/attachments/{att_id}"
|
||||
staging_key = f"{config.STAGING_PREFIX}/{att_id}"
|
||||
|
||||
# 1. Durable staging: the original must survive an API restart before 202.
|
||||
s3.client().put_object(Bucket=config.S3_BUCKET, Key=staging_key, Body=data, ContentType=ctype)
|
||||
|
||||
# 2. Metadata + durable job intent in one transaction (transactional outbox).
|
||||
att = Attachment(id=att_id, chat_id=chat_id, owner_id=user.id, filename=filename,
|
||||
content_type=ctype, size_bytes=size, sha256=digest, s3_key=final_key,
|
||||
staging_key=staging_key, status="queued")
|
||||
job = AttachmentJob(attachment_id=att_id, pipeline_version=1, state="queued", attempt=0)
|
||||
try:
|
||||
db.add(att)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
db.refresh(att)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
# Compensating delete for the staging object; the DB has no intent.
|
||||
try:
|
||||
s3.client().delete_object(Bucket=config.S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
# 3. Handoff to the broker. If this fails, the row stays unpublished and
|
||||
# the outbox pump retries it; the client still got a durable 202.
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from opentelemetry.propagate import inject
|
||||
carrier: dict[str, str] = {}
|
||||
inject(carrier) # capture the active request span before leaving the event loop
|
||||
try:
|
||||
await run_in_threadpool(jobs.publish_job_row, job.id, 1, carrier)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return _attachment_out(db, att)
|
||||
|
||||
@router.get("/attachments/{att_id}", response_model=AttachmentOut)
|
||||
def meta(att_id: uuid.UUID, db=Depends(get_db), user=Depends(current_user)):
|
||||
return _attachment_out(db, _attachment_for_member(db, att_id, user.id))
|
||||
|
||||
@router.get("/attachments/{att_id}/content")
|
||||
def content(att_id: uuid.UUID, variant: str | None = None,
|
||||
db=Depends(get_db), user=Depends(current_user)):
|
||||
att = _attachment_for_member(db, att_id, user.id)
|
||||
if att.status == "ready":
|
||||
if variant in (None, "original"):
|
||||
key, media_type = att.s3_key, att.content_type
|
||||
else:
|
||||
row = db.get(AttachmentVariant, (att.id, variant))
|
||||
if not row:
|
||||
raise ApiError(404, "not_found", "variant not found")
|
||||
key, media_type = row.s3_key, row.content_type
|
||||
elif att.status in ("queued", "processing"):
|
||||
raise ApiError(409, "conflict", "attachment is not ready")
|
||||
else:
|
||||
raise ApiError(409, "conflict", "attachment failed")
|
||||
obj = s3.client().get_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
body = obj["Body"].read()
|
||||
return Response(content=body, media_type=media_type)
|
||||
@@ -0,0 +1,44 @@
|
||||
import uuid
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from ..db import get_db
|
||||
from ..models import User, Chat, ChatMember
|
||||
from ..schemas import CreateChatIn, ChatOut, ChatPage
|
||||
from ..errors import ApiError
|
||||
from ..auth import current_user
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
|
||||
|
||||
def _chat_out(db, chat) -> ChatOut:
|
||||
members = [m.user_id for m in db.query(ChatMember).filter(ChatMember.chat_id == chat.id).all()]
|
||||
return ChatOut(id=chat.id, title=chat.title, member_ids=members, created_at=chat.created_at)
|
||||
|
||||
@router.post("", status_code=201, response_model=ChatOut)
|
||||
def create_chat(body: CreateChatIn, db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
if user.id not in body.member_ids:
|
||||
raise ApiError(400, "invalid_request", "caller must be a member")
|
||||
found = db.query(User.id).filter(User.id.in_(body.member_ids)).all()
|
||||
if len(found) != len(set(body.member_ids)):
|
||||
raise ApiError(400, "invalid_request", "unknown member")
|
||||
chat = Chat(title=body.title)
|
||||
db.add(chat); db.flush()
|
||||
for uid in body.member_ids:
|
||||
db.add(ChatMember(chat_id=chat.id, user_id=uid))
|
||||
db.commit(); db.refresh(chat)
|
||||
return _chat_out(db, chat)
|
||||
|
||||
@router.get("", response_model=ChatPage)
|
||||
def list_chats(db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
rows = db.query(Chat).join(ChatMember, ChatMember.chat_id == Chat.id)\
|
||||
.filter(ChatMember.user_id == user.id)\
|
||||
.order_by(Chat.created_at, Chat.id).limit(100).all()
|
||||
return ChatPage(items=[_chat_out(db, c) for c in rows], next_cursor=None)
|
||||
|
||||
@router.get("/{chat_id}", response_model=ChatOut)
|
||||
def get_chat(chat_id: uuid.UUID, db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
if not db.get(ChatMember, (chat_id, user.id)):
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
chat = db.get(Chat, chat_id)
|
||||
if not chat:
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
return _chat_out(db, chat)
|
||||
@@ -0,0 +1,19 @@
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
from ..db import get_db
|
||||
from ..errors import ApiError
|
||||
|
||||
router = APIRouter(prefix="/health", tags=["health"])
|
||||
|
||||
@router.get("/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
@router.get("/ready")
|
||||
def ready(db: Session = Depends(get_db)):
|
||||
try:
|
||||
db.execute(text("SELECT 1"))
|
||||
except Exception:
|
||||
raise ApiError(503, "unavailable", "database unavailable")
|
||||
return {"status": "ok"}
|
||||
@@ -0,0 +1,147 @@
|
||||
import uuid
|
||||
import json
|
||||
import hashlib
|
||||
import base64
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..db import get_db
|
||||
from ..models import (
|
||||
User,
|
||||
ChatMember,
|
||||
Message,
|
||||
MessageAttachment,
|
||||
IdempotencyKey,
|
||||
Attachment,
|
||||
)
|
||||
from ..schemas import CreateMessageIn, MessageOut, MessagePage
|
||||
from ..errors import ApiError
|
||||
from ..auth import current_user
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats/{chat_id}/messages", tags=["messages"])
|
||||
|
||||
|
||||
def _msg_out(db: Session, m: Message) -> MessageOut:
|
||||
atts = [
|
||||
a.attachment_id
|
||||
for a in db.query(MessageAttachment)
|
||||
.filter(MessageAttachment.message_id == m.id)
|
||||
.order_by(MessageAttachment.position)
|
||||
.all()
|
||||
]
|
||||
return MessageOut(
|
||||
id=m.id,
|
||||
chat_id=m.chat_id,
|
||||
sender_id=m.sender_id,
|
||||
text=m.text,
|
||||
created_at=m.created_at,
|
||||
attachment_ids=atts,
|
||||
)
|
||||
|
||||
|
||||
def _require_member(db: Session, chat_id: uuid.UUID, user_id: uuid.UUID) -> None:
|
||||
if not db.get(ChatMember, (chat_id, user_id)):
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
|
||||
|
||||
@router.post("", status_code=201, response_model=MessageOut)
|
||||
def send(
|
||||
chat_id: uuid.UUID,
|
||||
body: CreateMessageIn,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user),
|
||||
):
|
||||
_require_member(db, chat_id, user.id)
|
||||
|
||||
key = request.headers.get("idempotency-key", "")
|
||||
if not key or len(key) > 128:
|
||||
raise ApiError(400, "invalid_request", "Idempotency-Key required")
|
||||
|
||||
if not body.text.strip() and not body.attachment_ids:
|
||||
raise ApiError(400, "invalid_request", "empty message")
|
||||
|
||||
for aid in body.attachment_ids:
|
||||
a = db.get(Attachment, aid)
|
||||
if not a or a.chat_id != chat_id:
|
||||
raise ApiError(404, "not_found", "attachment not found")
|
||||
|
||||
path = f"/api/v1/chats/{chat_id}/messages"
|
||||
body_hash = hashlib.sha256(
|
||||
json.dumps(body.model_dump(), sort_keys=True, default=str).encode()
|
||||
).hexdigest()
|
||||
|
||||
existing = db.get(IdempotencyKey, (user.id, "POST", path, key))
|
||||
if existing:
|
||||
if existing.request_hash != body_hash:
|
||||
raise ApiError(409, "conflict", "idempotency key reused with different body")
|
||||
return json.loads(existing.response_json)
|
||||
|
||||
m = Message(chat_id=chat_id, sender_id=user.id, text=body.text)
|
||||
db.add(m)
|
||||
db.flush() # чтобы появился m.id
|
||||
|
||||
for i, aid in enumerate(body.attachment_ids):
|
||||
db.add(MessageAttachment(message_id=m.id, attachment_id=aid, position=i))
|
||||
db.flush() # чтобы _msg_out увидел MessageAttachment
|
||||
|
||||
out = _msg_out(db, m)
|
||||
|
||||
db.add(
|
||||
IdempotencyKey(
|
||||
user_id=user.id,
|
||||
method="POST",
|
||||
path=path,
|
||||
key=key,
|
||||
request_hash=body_hash,
|
||||
response_json=json.dumps(out.model_dump(), default=str),
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return out
|
||||
|
||||
|
||||
@router.get("", response_model=MessagePage)
|
||||
def list_messages(
|
||||
chat_id: uuid.UUID,
|
||||
limit: int = 50,
|
||||
cursor: str | None = None,
|
||||
db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user),
|
||||
):
|
||||
_require_member(db, chat_id, user.id)
|
||||
|
||||
if limit < 1 or limit > 100:
|
||||
raise ApiError(400, "invalid_request", "bad limit")
|
||||
|
||||
q = (
|
||||
db.query(Message)
|
||||
.filter(Message.chat_id == chat_id)
|
||||
.order_by(Message.created_at, Message.id)
|
||||
)
|
||||
|
||||
if cursor:
|
||||
try:
|
||||
c = json.loads(base64.urlsafe_b64decode(cursor.encode()).decode())
|
||||
ts = datetime.fromisoformat(c["t"])
|
||||
mid = uuid.UUID(c["i"])
|
||||
q = q.filter(
|
||||
(Message.created_at > ts)
|
||||
| ((Message.created_at == ts) & (Message.id > mid))
|
||||
)
|
||||
except Exception:
|
||||
raise ApiError(400, "invalid_request", "bad cursor")
|
||||
|
||||
rows = q.limit(limit + 1).all()
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
|
||||
next_cursor = None
|
||||
if has_more:
|
||||
last = rows[-1]
|
||||
payload = json.dumps({"t": last.created_at.isoformat(), "i": str(last.id)}).encode()
|
||||
next_cursor = base64.urlsafe_b64encode(payload).decode()
|
||||
|
||||
return MessagePage(items=[_msg_out(db, m) for m in rows], next_cursor=next_cursor)
|
||||
@@ -0,0 +1,41 @@
|
||||
import base64
|
||||
import hashlib
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from sqlalchemy.orm import Session
|
||||
from ..db import get_db
|
||||
from ..models import User, Session as DbSession
|
||||
from ..schemas import SessionOut
|
||||
from ..errors import ApiError
|
||||
from ..auth import verify_password, new_session, current_user
|
||||
|
||||
router = APIRouter(prefix="/api/v1/auth/sessions", tags=["sessions"])
|
||||
|
||||
_BASIC = 'Basic realm="messenger", charset="UTF-8"'
|
||||
_BEARER = 'Bearer realm="messenger"'
|
||||
|
||||
@router.post("", status_code=201, response_model=SessionOut)
|
||||
def login(request: Request, db: Session = Depends(get_db)):
|
||||
auth = request.headers.get("authorization", "")
|
||||
if not auth.lower().startswith("basic "):
|
||||
raise ApiError(401, "unauthorized", "basic required", challenge=_BASIC)
|
||||
try:
|
||||
raw = base64.b64decode(auth[6:].strip()).decode("utf-8")
|
||||
username, password = raw.split(":", 1)
|
||||
except Exception:
|
||||
raise ApiError(401, "unauthorized", "bad credentials", challenge=_BASIC)
|
||||
user = db.query(User).filter(User.username == username).first()
|
||||
if not user or not verify_password(user.password_hash, password):
|
||||
raise ApiError(401, "unauthorized", "bad credentials", challenge=_BASIC)
|
||||
token, expires = new_session(db, user)
|
||||
return {"token": token, "token_type": "Bearer", "expires_at": expires, "user": user}
|
||||
|
||||
@router.delete("/current", status_code=204)
|
||||
def logout(request: Request, db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
auth = request.headers.get("authorization", "")
|
||||
token = auth[7:].strip()
|
||||
h = hashlib.sha256(token.encode()).hexdigest()
|
||||
s = db.get(DbSession, h)
|
||||
if s:
|
||||
db.delete(s)
|
||||
db.commit()
|
||||
return None
|
||||
@@ -0,0 +1,27 @@
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from ..db import get_db
|
||||
from ..models import User
|
||||
from ..schemas import CreateUserIn, UserOut
|
||||
from ..errors import ApiError
|
||||
from ..auth import hash_password, current_user
|
||||
|
||||
router = APIRouter(prefix="/api/v1/users", tags=["users"])
|
||||
|
||||
@router.post("", status_code=201, response_model=UserOut)
|
||||
def register(body: CreateUserIn, db: Session = Depends(get_db)):
|
||||
u = User(username=body.username, display_name=body.display_name,
|
||||
password_hash=hash_password(body.password))
|
||||
db.add(u)
|
||||
try:
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
raise ApiError(409, "conflict", "username taken")
|
||||
db.refresh(u)
|
||||
return u
|
||||
|
||||
@router.get("/me", response_model=UserOut)
|
||||
def me(user: User = Depends(current_user)):
|
||||
return user
|
||||
@@ -0,0 +1,36 @@
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from . import config
|
||||
|
||||
_client = None
|
||||
|
||||
def client():
|
||||
global _client
|
||||
if _client is None:
|
||||
_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=config.S3_ENDPOINT,
|
||||
region_name=config.S3_REGION,
|
||||
aws_access_key_id=config.S3_ACCESS_KEY,
|
||||
aws_secret_access_key=config.S3_SECRET_KEY,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
connect_timeout=3,
|
||||
read_timeout=10,
|
||||
),
|
||||
)
|
||||
return _client
|
||||
|
||||
def ensure_bucket():
|
||||
c = client()
|
||||
try:
|
||||
c.head_bucket(Bucket=config.S3_BUCKET)
|
||||
except ClientError as e:
|
||||
code = e.response.get("Error", {}).get("Code", "")
|
||||
if code in ("404", "NoSuchBucket", "NotFound"):
|
||||
c.create_bucket(Bucket=config.S3_BUCKET)
|
||||
else:
|
||||
raise
|
||||
@@ -0,0 +1,76 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel, Field, ConfigDict
|
||||
|
||||
class UserOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
id: uuid.UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
|
||||
class CreateUserIn(BaseModel):
|
||||
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
class SessionOut(BaseModel):
|
||||
token: str
|
||||
token_type: str = "Bearer"
|
||||
expires_at: datetime
|
||||
user: UserOut
|
||||
|
||||
class ChatOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
id: uuid.UUID
|
||||
title: str
|
||||
member_ids: list[uuid.UUID]
|
||||
created_at: datetime
|
||||
|
||||
class CreateChatIn(BaseModel):
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[uuid.UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
class MessageOut(BaseModel):
|
||||
id: uuid.UUID
|
||||
chat_id: uuid.UUID
|
||||
sender_id: uuid.UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: list[uuid.UUID]
|
||||
|
||||
class CreateMessageIn(BaseModel):
|
||||
text: str = Field(default="", max_length=4000)
|
||||
attachment_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
class MessagePage(BaseModel):
|
||||
items: list[MessageOut]
|
||||
next_cursor: Optional[str] = None
|
||||
|
||||
class ChatPage(BaseModel):
|
||||
items: list[ChatOut]
|
||||
next_cursor: Optional[str] = None
|
||||
|
||||
class VariantOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
name: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
class AttachmentOut(BaseModel):
|
||||
id: uuid.UUID
|
||||
chat_id: uuid.UUID
|
||||
owner_id: uuid.UUID
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str = "ready"
|
||||
variants: list[VariantOut] = []
|
||||
error_code: Optional[str] = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
@@ -0,0 +1,124 @@
|
||||
"""OpenTelemetry wiring for API and worker (lab6).
|
||||
|
||||
Signals go OTLP -> collector. Metrics are exposed by the collector as a
|
||||
Prometheus endpoint, traces are stored in Tempo, logs in Loki. The JSON logs
|
||||
kept in stdout also carry trace_id/span_id so a trace can be matched with a
|
||||
line even without the log backend.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.resources import (
|
||||
Resource,
|
||||
SERVICE_INSTANCE_ID,
|
||||
SERVICE_NAME,
|
||||
SERVICE_VERSION,
|
||||
)
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
from opentelemetry.propagate import set_global_textmap
|
||||
|
||||
|
||||
class TraceJsonFormatter(logging.Formatter):
|
||||
"""One JSON object per line, with trace correlation and no bodies."""
|
||||
|
||||
def __init__(self, service_name: str):
|
||||
super().__init__()
|
||||
self.service_name = service_name
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
ctx = trace.get_current_span().get_span_context()
|
||||
payload = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
"severity": record.levelname,
|
||||
"service.name": self.service_name,
|
||||
"event": record.getMessage(),
|
||||
"logger": record.name,
|
||||
}
|
||||
if ctx and ctx.is_valid:
|
||||
payload["trace_id"] = format(ctx.trace_id, "032x")
|
||||
payload["span_id"] = format(ctx.span_id, "016x")
|
||||
extra = getattr(record, "event_fields", None)
|
||||
if isinstance(extra, dict):
|
||||
payload.update({k: v for k, v in extra.items() if not isinstance(v, (bytes, bytearray))})
|
||||
elif isinstance(extra, str):
|
||||
try:
|
||||
payload.update({k: v for k, v in json.loads(extra).items()})
|
||||
except ValueError:
|
||||
pass
|
||||
if record.exc_info and record.exc_info[0]:
|
||||
payload["error.type"] = record.exc_info[0].__name__
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def setup_telemetry(service_name: str, instance_id: str):
|
||||
endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4317")
|
||||
resource = Resource.create({
|
||||
SERVICE_NAME: service_name,
|
||||
SERVICE_VERSION: os.environ.get("SERVICE_VERSION", "6.0.0"),
|
||||
SERVICE_INSTANCE_ID: instance_id,
|
||||
"deployment.environment.name": os.environ.get("APP_ENV", "lab"),
|
||||
})
|
||||
|
||||
tracer_provider = TracerProvider(resource=resource)
|
||||
tracer_provider.add_span_processor(
|
||||
BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, insecure=True))
|
||||
)
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
|
||||
meter_provider = MeterProvider(
|
||||
resource=resource,
|
||||
metric_readers=[PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=endpoint, insecure=True),
|
||||
export_interval_millis=int(os.environ.get("OTEL_METRIC_INTERVAL_MS", "10000")),
|
||||
)],
|
||||
)
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint, insecure=True))
|
||||
)
|
||||
|
||||
root = logging.getLogger()
|
||||
root.setLevel(logging.INFO)
|
||||
for handler in list(root.handlers):
|
||||
root.removeHandler(handler)
|
||||
stream = logging.StreamHandler()
|
||||
stream.setFormatter(TraceJsonFormatter(service_name))
|
||||
root.addHandler(stream)
|
||||
root.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logger_provider))
|
||||
|
||||
set_global_textmap(TraceContextTextMapPropagator())
|
||||
return tracer_provider, meter_provider
|
||||
|
||||
|
||||
def http_instruments():
|
||||
meter = metrics.get_meter("messenger.http")
|
||||
return (
|
||||
meter.create_counter("http.server.requests", unit="1",
|
||||
description="HTTP requests by method, route template and status"),
|
||||
meter.create_histogram("http.server.request.duration", unit="s",
|
||||
description="HTTP server request duration"),
|
||||
)
|
||||
|
||||
|
||||
def job_instruments():
|
||||
meter = metrics.get_meter("messenger.worker")
|
||||
return (
|
||||
meter.create_counter("attachment.jobs", unit="1",
|
||||
description="Attachment jobs by outcome"),
|
||||
meter.create_histogram("attachment.job.duration", unit="s",
|
||||
description="Attachment job duration by outcome"),
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Deterministic image variants (lab5: one thumbnail per the contract)."""
|
||||
import hashlib
|
||||
import io
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
from . import config
|
||||
|
||||
|
||||
def _flatten_on_white(img: Image.Image) -> Image.Image:
|
||||
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
||||
rgba = img.convert("RGBA")
|
||||
background = Image.new("RGBA", rgba.size, (255, 255, 255, 255))
|
||||
return Image.alpha_composite(background, rgba).convert("RGB")
|
||||
return img.convert("RGB")
|
||||
|
||||
|
||||
def make_thumbnail(data: bytes) -> tuple[bytes, int, int, str]:
|
||||
"""Fit the image into 256x256 (no upscaling), JPEG on white.
|
||||
|
||||
Returns (jpeg_bytes, width, height, sha256_hex).
|
||||
"""
|
||||
img = Image.open(io.BytesIO(data))
|
||||
img = ImageOps.exif_transpose(img)
|
||||
img = _flatten_on_white(img)
|
||||
|
||||
limit = config.THUMBNAIL_SIZE
|
||||
width, height = img.size
|
||||
scale = min(limit / width, limit / height, 1.0)
|
||||
new_width = max(1, int(width * scale))
|
||||
new_height = max(1, int(height * scale))
|
||||
if (new_width, new_height) != (width, height):
|
||||
img = img.resize((new_width, new_height), Image.LANCZOS)
|
||||
|
||||
out = io.BytesIO()
|
||||
img.save(out, "JPEG", quality=config.THUMBNAIL_QUALITY, optimize=True)
|
||||
payload = out.getvalue()
|
||||
return payload, new_width, new_height, hashlib.sha256(payload).hexdigest()
|
||||
+227
@@ -0,0 +1,227 @@
|
||||
"""Durable attachment worker (lab5/lab6).
|
||||
|
||||
Consumes ``attachment.process.v1`` messages from a persistent queue, performs
|
||||
the heavy image work outside the HTTP path, records the result atomically in
|
||||
PostgreSQL, and emits OTLP traces/metrics/logs. Delivery is at-least-once; the
|
||||
effect is idempotent by ``(attachment_id, pipeline_version, variant)`` and
|
||||
every state transition is committed before the broker ack.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import signal
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import pika
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.propagate import extract
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from sqlalchemy import select
|
||||
|
||||
from . import config, s3, telemetry
|
||||
from .db import SessionLocal
|
||||
from .errors import ApiError
|
||||
from .images import validate_image
|
||||
from .models import Attachment, AttachmentJob, AttachmentVariant
|
||||
from .thumbnails import make_thumbnail
|
||||
|
||||
_, _meter_provider = telemetry.setup_telemetry("messenger-worker", "worker")
|
||||
job_counter, job_duration = telemetry.job_instruments()
|
||||
tracer = trace.get_tracer("messenger.worker")
|
||||
log = logging.getLogger("messenger.worker")
|
||||
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
|
||||
_stop = False
|
||||
|
||||
|
||||
def _handle_signal(signum, frame):
|
||||
global _stop
|
||||
_stop = True
|
||||
log.info("worker.stop_requested")
|
||||
|
||||
|
||||
def _failure_code(exc: Exception) -> tuple[str, bool]:
|
||||
"""Return (safe error_code, permanent)."""
|
||||
if isinstance(exc, ApiError):
|
||||
if exc.status in (400, 413, 415):
|
||||
return "invalid_image", True
|
||||
return "processing_failed", False
|
||||
if isinstance(exc, (OSError, ValueError)):
|
||||
return "invalid_image", True
|
||||
return "processing_failed", False
|
||||
|
||||
|
||||
def _set_state(job_id: uuid.UUID, attachment_id: uuid.UUID, job_state: str,
|
||||
att_state: str | None = None, attempt: int | None = None,
|
||||
error: str | None = None) -> None:
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
if not job:
|
||||
return
|
||||
job.state = job_state
|
||||
if attempt is not None:
|
||||
job.attempt = max(job.attempt or 0, attempt)
|
||||
if error is not None:
|
||||
job.last_error = error[:255]
|
||||
att = db.get(Attachment, attachment_id)
|
||||
if att and att_state:
|
||||
att.status = att_state
|
||||
att.error_code = error if att_state == "failed" else None
|
||||
db.commit()
|
||||
|
||||
|
||||
def process(job_id: uuid.UUID, attachment_id: uuid.UUID, attempt: int) -> str:
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
att = db.get(Attachment, attachment_id)
|
||||
if not job or not att:
|
||||
log.warning("job.dropped", extra={"event_fields": json.dumps({"reason": "row_gone"})})
|
||||
return "dropped"
|
||||
if att.status == "ready":
|
||||
job.state = "done"
|
||||
db.commit()
|
||||
log.info("job.duplicate", extra={"event_fields": json.dumps({"attachment_id": str(attachment_id)})})
|
||||
return "duplicate"
|
||||
if att.status == "failed":
|
||||
job.state = "failed"
|
||||
db.commit()
|
||||
log.info("job.duplicate_failed", extra={"event_fields": json.dumps({"attachment_id": str(attachment_id)})})
|
||||
return "skipped"
|
||||
job.state = "processing"
|
||||
job.attempt = max(job.attempt or 0, attempt)
|
||||
att.status = "processing"
|
||||
db.commit()
|
||||
pipeline_version = job.pipeline_version
|
||||
staging_key = att.staging_key or f"{config.STAGING_PREFIX}/{attachment_id}"
|
||||
final_key = att.s3_key
|
||||
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
att = db.get(Attachment, attachment_id)
|
||||
declared_type = att.content_type if att else None
|
||||
data = s3.client().get_object(Bucket=config.S3_BUCKET, Key=staging_key)["Body"].read()
|
||||
ctype, _w, _h, _size = validate_image(data)
|
||||
s3.client().put_object(Bucket=config.S3_BUCKET, Key=final_key, Body=data,
|
||||
ContentType=declared_type or ctype)
|
||||
|
||||
thumb, tw, th, tsha = make_thumbnail(data)
|
||||
thumb_key = f"variants/{attachment_id}/thumbnail"
|
||||
s3.client().put_object(Bucket=config.S3_BUCKET, Key=thumb_key, Body=thumb, ContentType="image/jpeg")
|
||||
|
||||
with SessionLocal() as db:
|
||||
att = db.get(Attachment, attachment_id)
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
existing = db.get(AttachmentVariant, (attachment_id, "thumbnail"))
|
||||
if existing is None:
|
||||
db.add(AttachmentVariant(
|
||||
attachment_id=attachment_id, name="thumbnail",
|
||||
content_type="image/jpeg", size_bytes=len(thumb),
|
||||
width=tw, height=th, sha256=tsha, s3_key=thumb_key,
|
||||
))
|
||||
elif existing.s3_key != thumb_key:
|
||||
log.warning("variant.kept", extra={"event_fields": json.dumps({"reason": "newer_pipeline"})})
|
||||
att.status = "ready"
|
||||
att.error_code = None
|
||||
job.state = "done"
|
||||
job.last_error = None
|
||||
job.attempt = max(job.attempt or 0, attempt)
|
||||
db.commit()
|
||||
log.info("job.ready", extra={"event_fields": json.dumps({
|
||||
"attachment_id": str(attachment_id),
|
||||
"pipeline_version": pipeline_version,
|
||||
"attempt": attempt,
|
||||
})})
|
||||
try:
|
||||
s3.client().delete_object(Bucket=config.S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
return "ready"
|
||||
except Exception as exc:
|
||||
code, permanent = _failure_code(exc)
|
||||
log.exception("job.failed", extra={"event_fields": json.dumps({
|
||||
"attachment_id": str(attachment_id),
|
||||
"attempt": attempt,
|
||||
"error_code": code,
|
||||
})})
|
||||
if permanent or attempt >= config.MAX_JOB_ATTEMPTS:
|
||||
_set_state(job_id, attachment_id, "failed", att_state="failed",
|
||||
attempt=attempt, error=code)
|
||||
return "failed"
|
||||
_set_state(job_id, attachment_id, "queued", att_state="queued",
|
||||
attempt=attempt, error=code)
|
||||
raise
|
||||
|
||||
|
||||
def handle(channel, method, properties, body):
|
||||
outcome = "retry"
|
||||
try:
|
||||
message = json.loads(body)
|
||||
job_id = uuid.UUID(message["job_id"])
|
||||
attachment_id = uuid.UUID(message["attachment_id"])
|
||||
attempt = int(message.get("attempt", 1))
|
||||
except Exception as exc:
|
||||
log.warning("job.poison", extra={"event_fields": json.dumps({"reason": type(exc).__name__})})
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
return
|
||||
|
||||
context = extract(message)
|
||||
started = time.perf_counter()
|
||||
with tracer.start_as_current_span("attachment.process", context=context,
|
||||
kind=SpanKind.CONSUMER) as span:
|
||||
span.set_attribute("job.type", JOB_TYPE)
|
||||
span.set_attribute("attachment.id", str(attachment_id))
|
||||
span.set_attribute("job.attempt", attempt)
|
||||
try:
|
||||
outcome = process(job_id, attachment_id, attempt)
|
||||
span.set_attribute("job.outcome", outcome)
|
||||
if outcome == "failed":
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
except Exception as exc:
|
||||
outcome = "retry"
|
||||
span.set_attribute("job.outcome", outcome)
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
span.record_exception(exc)
|
||||
from .jobs import build_payload, publish
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
payload = build_payload(job, job.attempt + 1) if job and job.state == "queued" else None
|
||||
if payload:
|
||||
try:
|
||||
publish(payload)
|
||||
except Exception:
|
||||
channel.basic_nack(delivery_tag=method.delivery_tag, requeue=True)
|
||||
outcome = "requeued"
|
||||
return
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
finally:
|
||||
attributes = {"job.type": JOB_TYPE, "job.outcome": outcome}
|
||||
job_counter.add(1, attributes)
|
||||
job_duration.record(time.perf_counter() - started, attributes)
|
||||
|
||||
|
||||
def main():
|
||||
signal.signal(signal.SIGTERM, _handle_signal)
|
||||
signal.signal(signal.SIGINT, _handle_signal)
|
||||
params = pika.URLParameters(config.BROKER_URL)
|
||||
params.heartbeat = 30
|
||||
while not _stop:
|
||||
try:
|
||||
connection = pika.BlockingConnection(params)
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=config.QUEUE_NAME, durable=True)
|
||||
channel.basic_qos(prefetch_count=1)
|
||||
channel.basic_consume(queue=config.QUEUE_NAME, on_message_callback=handle, auto_ack=False)
|
||||
log.info("worker.consuming", extra={"event_fields": json.dumps({"queue": config.QUEUE_NAME})})
|
||||
while not _stop:
|
||||
connection.process_data_events(time_limit=1)
|
||||
connection.close()
|
||||
except Exception as exc:
|
||||
log.warning("broker.reconnect", extra={"event_fields": json.dumps({"error": type(exc).__name__})})
|
||||
time.sleep(3)
|
||||
log.info("worker.stopped")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit operator-authored hook; tokens remain in memory, volumes are never removed by this test."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
from client import Client, Failure, LAB, arguments, check, run
|
||||
|
||||
|
||||
def history(c, actor, chat):
|
||||
path = '/api/v1/chats/'+chat['id']+'/messages?limit=100'
|
||||
result = []
|
||||
seen = set()
|
||||
for _ in range(1000):
|
||||
page = c.call('GET',path,actor=actor,schema='MessagePage')['json']
|
||||
result.extend(page['items'])
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
return result
|
||||
check(cursor not in seen,'cursor cycle')
|
||||
seen.add(cursor)
|
||||
path = '/api/v1/chats/'+chat['id']+'/messages?limit=100&cursor='+quote(cursor,safe='')
|
||||
raise Failure('history pagination exceeded safety bound')
|
||||
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument('--mode', choices=['persistence','failover'], required=True)
|
||||
p.add_argument('--action-script',required=True,help='Executable script authored by student; restarts stack or stops ONE API replica')
|
||||
args = arguments('Explicit restart/failover scenario; modifies only synthetic API data',configure)
|
||||
check(LAB>=2,'persistence requires lab2+')
|
||||
check(args.mode!='failover' or LAB>=3,'failover requires lab3+')
|
||||
script = Path(args.action_script).expanduser().resolve()
|
||||
check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)')
|
||||
check('.example.' not in script.name,'copy and implement the example hook first')
|
||||
c = Client(args.base_url,args.ca_file,timeout=10)
|
||||
alice,bob,eve,chat = c.bootstrap()
|
||||
sent = c.message(alice,chat,'Сохранить при отказе')
|
||||
original = sent['json']
|
||||
target = sent['headers'].get('X-Instance-Id','')
|
||||
attachment = content = None
|
||||
if LAB>=4:
|
||||
attachment,content = c.attachment(alice,chat)
|
||||
if LAB>=5:
|
||||
attachment = c.wait_ready(alice,attachment)
|
||||
env = dict(os.environ)
|
||||
env['TARGET_INSTANCE_ID'] = target
|
||||
observed = set()
|
||||
if args.mode=='failover':
|
||||
for _ in range(40):
|
||||
observed.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'])
|
||||
check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity')
|
||||
print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.')
|
||||
# No shell interpolation and no Docker commands in this test.
|
||||
proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
started = time.monotonic()
|
||||
successes,failures,after_instances = [],0,[]
|
||||
try:
|
||||
if args.mode=='persistence':
|
||||
try:
|
||||
code = proc.wait(timeout=90)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise Failure('restart hook exceeded 90 seconds') from None
|
||||
check(code==0,'restart hook failed (inspect your script locally)')
|
||||
deadline = time.monotonic()+90
|
||||
while True:
|
||||
try:
|
||||
c.call('GET','/health/ready',schema='Health')
|
||||
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic()<deadline,'stack/session did not recover within 90 seconds')
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
first_recovery = None
|
||||
key = uuid.uuid4().hex
|
||||
text = 'Запись во время отказа ' + key
|
||||
while time.monotonic()-started < 15:
|
||||
code = proc.poll()
|
||||
check(code is None or code==0,'stop-one hook failed (inspect your script locally)')
|
||||
check(code is not None or time.monotonic()-started<10,'stop-one hook must return within 10 seconds')
|
||||
try:
|
||||
response = c.message(alice,chat,text,key=key)
|
||||
msg = response['json']
|
||||
if code==0:
|
||||
if first_recovery is None:
|
||||
first_recovery = time.monotonic()-started
|
||||
after_instances.append(response['headers']['X-Instance-Id'])
|
||||
successes.append(msg)
|
||||
key = uuid.uuid4().hex
|
||||
text = 'Запись во время отказа ' + key
|
||||
except Failure:
|
||||
failures += 1
|
||||
# Keep SAME key/body after an uncertain POST result.
|
||||
time.sleep(0.1)
|
||||
check(proc.poll()==0,'stop-one hook did not finish successfully')
|
||||
check(first_recovery is not None and first_recovery<=10,'writes did not recover within 10 seconds from hook start')
|
||||
check(len(after_instances)>=5,'too few successful writes after stop')
|
||||
check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped')
|
||||
print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False))
|
||||
# Existing token, no re-login. Read all confirmations and reject duplicates.
|
||||
messages = history(c,bob,chat)
|
||||
ids = [m['id'] for m in messages]
|
||||
check(len(ids)==len(set(ids)),'duplicated message IDs in history')
|
||||
check(any(m==original for m in messages),'confirmed pre-failure message lost or altered')
|
||||
for msg in successes:
|
||||
check(any(m==msg for m in messages),'confirmed in-flight message lost or altered')
|
||||
# Bodies are unique in this scenario: an uncertain request may exist once, never twice.
|
||||
texts = [m['text'] for m in messages]
|
||||
check(len(texts)==len(set(texts)),'uncertain retry created duplicate message')
|
||||
if attachment:
|
||||
c.call('GET','/api/v1/attachments/'+attachment['id'],actor=bob,schema='Attachment')
|
||||
downloaded = c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob)
|
||||
check(downloaded['bytes']==content,'attachment lost or changed across failure')
|
||||
print('PASS: '+args.mode+' preserved confirmed data and existing session. Hook scope needs operator evidence.')
|
||||
finally:
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
|
||||
if __name__=='__main__':
|
||||
run(main)
|
||||
Reference in New Issue
Block a user