ЛР 6 - БПИ2301 - Булацев - на 5 #37
+34
-11
@@ -1,30 +1,53 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# Copy to .env for a LOCAL synthetic stand. Never commit .env.
|
||||
# Compose interpolates the variables below. The API also accepts *_FILE secrets.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
LOG_LEVEL=info
|
||||
SHUTDOWN_GRACE_SECONDS=10
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
CACHE_TTL_SECONDS=60
|
||||
# Sessions stay in Redis, carried from lab3/lab4.
|
||||
SESSION_STORE=redis
|
||||
API_REPLICAS=2
|
||||
API_CPUS=0.50
|
||||
API_MEM=256m
|
||||
POSTGRES_CPUS=1.0
|
||||
POSTGRES_MEM=512m
|
||||
DB_POOL_MAX=4
|
||||
LB_DRAIN_SECONDS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
|
||||
POSTGRES_PASSWORD=lab6-dev-only-change-me
|
||||
DATABASE_URL=postgresql://messenger:lab6-dev-only-change-me@postgres:5432/messenger
|
||||
REDIS_URL=redis://redis:6379/0
|
||||
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_ACCESS_KEY_ID=lab6appkey00000001
|
||||
S3_SECRET_ACCESS_KEY=lab6-dev-only-change-me
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
ORPHAN_GRACE_SECONDS=120
|
||||
S3_PUBLIC_ENDPOINT=http://127.0.0.1:9000
|
||||
PRESIGN_TTL_SECONDS=60
|
||||
|
||||
# RabbitMQ is the durable broker. The URL is built in compose.yaml from this password.
|
||||
RABBITMQ_PASSWORD=lab6-dev-only-change-me
|
||||
BROKER_URL=amqp://messenger:lab6-dev-only-change-me@rabbitmq:5672/
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_BACKOFF_BASE_SECONDS=0.5
|
||||
JOB_BACKOFF_CAP_SECONDS=30
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
|
||||
# OpenTelemetry. API and worker override OTEL_SERVICE_NAME in compose.
|
||||
TELEMETRY_ENABLED=1
|
||||
TRACE_SAMPLE_RATIO=1
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
GRAFANA_ADMIN_PASSWORD=lab6-dev-only-change-me
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
*.sh text eol=lf
|
||||
infra/s3/* text eol=lf
|
||||
*.yml text eol=lf
|
||||
*.yaml text eol=lf
|
||||
@@ -1,18 +1,18 @@
|
||||
## Лабораторная 6
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Булацев, БПИ2301
|
||||
Целевая оценка (3/4/5): 5
|
||||
Трек 5, если заявлен: 2, SLO и обнаружение инцидентов
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Что реализовано и что перенесено: база lab5; OTLP у API и worker, Collector, Prometheus, Loki, Tempo, Grafana provisioning. Инциденты и измерения burn rate ещё не сняты.
|
||||
Команды запуска из чистого clone: см. REPORT.md (`cp .env.example .env`, `docker compose --env-file .env up -d --build --scale api=2`).
|
||||
Результаты проверок и ссылка на REPORT.md: make check PASS, make test PASS, persistence PASS, failover PASS, lab5 worker-down PASS, lab6 incidents PASS. Вывод в evidence/ и REPORT.md.
|
||||
Ограничения / известные ошибки: полный замер времени обнаружения короткого всплеска против длинной деградации ещё не снят; на этом прогоне firing был у TimeToReadyFastBurn.
|
||||
Источники и использование ИИ: TODO
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] make check и make test выполнены; приложен реальный вывод.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
FROM python:3.12.8-slim-bookworm AS runtime
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN groupadd --system app && useradd --system --gid app --home /app --shell /usr/sbin/nologin app \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY app ./app
|
||||
COPY migrations ./migrations
|
||||
COPY contracts/openapi.json ./contracts/openapi.json
|
||||
|
||||
USER app
|
||||
|
||||
ENV APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=12 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"
|
||||
|
||||
CMD ["python", "-m", "app.main"]
|
||||
@@ -22,9 +22,13 @@ load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
ifeq ($(ACTION_SCRIPT),)
|
||||
$(error Set ACTION_SCRIPT to your implemented executable restart hook)
|
||||
endif
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
ifeq ($(ACTION_SCRIPT),)
|
||||
$(error Set ACTION_SCRIPT to your implemented executable stop-one hook)
|
||||
endif
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
@@ -1,37 +1,72 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Булацев, БПИ2301
|
||||
- Целевая оценка; выбранный трек 5: 5, трек 2 «SLO и обнаружение инцидентов». На стенде воспроизведены отказ worker, Postgres и collector; отдельный замер короткого всплеска против длинной деградации ещё не снят.
|
||||
- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/pulls/37
|
||||
- Использованные источники, библиотеки и помощь ИИ: контракт lab6, OpenTelemetry Python SDK 1.29.0, Collector Contrib 0.128.0, Prometheus 2.55.1, Loki 3.3.2, Tempo 2.7.1, Grafana 11.4.0. Реализация перенесена из lab5 и дополнена с помощью Cursor.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Compose v2 и Python 3.9+ на хосте для `make check` / `make test`. Образы закреплены: `python:3.12.8-slim-bookworm`, `postgres:16.6-alpine`, `redis:7.4.2-alpine`, `haproxy:2.8.14-alpine`, `chrislusf/seaweedfs:3.85`, `rabbitmq:3.13.7-alpine`, `otel/opentelemetry-collector-contrib:0.128.0`, `prom/prometheus:v2.55.1`, `grafana/loki:3.3.2`, `grafana/tempo:2.7.1`, `grafana/grafana:11.4.0`.
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
# заменить синтетические пароли в локальном .env; файл не коммитить
|
||||
docker compose --env-file .env up -d --build --scale api=2
|
||||
```
|
||||
|
||||
`migrate` применяет SQL, включая `007_job_trace.sql`. Снаружи: HAProxy `127.0.0.1:8080`, Grafana `127.0.0.1:3000` (анонимный Admin только на localhost), Prometheus `127.0.0.1:9090`, S3 `127.0.0.1:9000` для presigned PUT. OTLP наружу не публикуется.
|
||||
|
||||
Готовность API: `GET http://127.0.0.1:8080/health/ready`. Дашборд «Messenger observability» поднимается provisioning-ом, руками его импортировать не нужно. Остановка без удаления данных: `docker compose --env-file .env stop`. Удаление учебных volumes: `docker compose --env-file .env down -v`.
|
||||
|
||||
Проверки после готовности:
|
||||
|
||||
```sh
|
||||
python -X utf8 scripts/check_contract.py
|
||||
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Из lab5 перенесены две реплики API за HAProxy, Postgres, Redis, SeaweedFS, RabbitMQ и отдельный worker. Бизнес-контракт не менялся: upload по-прежнему `202`, обработка в worker.
|
||||
|
||||
Новое в lab6: API и worker шлют OTLP/HTTP в Collector. Collector раскладывает сигналы в Prometheus (метрики), Loki (логи) и Tempo (трассы). Grafana только читает эти хранилища.
|
||||
|
||||
`traceparent` запроса, который создал job, пишется в `attachment_jobs` и при публикации становится родителем producer span. Worker читает заголовок AMQP и открывает consumer span. Внутри обработки есть spans `s3.get`, `s3.put`, `image.render`, `db.complete`. Повторная публикация использует тот же сохранённый `traceparent`, поэтому retry остаётся в той же трассе.
|
||||
|
||||
Метрики не используют `user_id`, `chat_id`, `message_id`, `attachment_id` как labels. Маршрут — шаблон. Доля 5xx считается только по бизнес-запросам: health, docs и openapi в неё не входят. Процентили на дашборде — `histogram_quantile` по bucket. Глубина очереди и возраст старейшей queued-job читаются из Postgres раз в 5 секунд; на дашборде берётся `max`, чтобы две реплики API не удваивали gauge.
|
||||
|
||||
SLO и burn-rate: `infra/slo.md`, правила `infra/prometheus/alerts.yml`. Состояние firing/resolved смотрит Prometheus (`ALERTS`) и нижняя панель дашборда. Доставка наружу не настроена.
|
||||
|
||||
Если Collector или хранилище телеметрии недоступны, SDK держит очередь не длиннее 2048 и отбрасывает лишнее. Запросы и обработка job от этого не останавливаются.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| make check | `make PYTHON=python check` | PASS, offline skeleton | evidence/make-check.txt |
|
||||
| Публичный smoke | `make PYTHON=python BASE_URL=http://127.0.0.1:8082 test` | PASS lab6 public API scenario | evidence/make-test.txt |
|
||||
| Сохранность | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | PASS, сессия и данные живы после restart | evidence/persistence.txt |
|
||||
| Отказ одной реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd` | 2 реплики до отказа, 104 успешных записи, 1 ошибка, восстановление за 3.316 с | evidence/failover.txt |
|
||||
| Worker остановлен | `python -X utf8 scripts/lab5_checks.py` | upload 202, content 409, после старта worker ready и thumbnail | evidence/lab5-worker-down.txt |
|
||||
| Очередь, БД, collector | `python -X utf8 scripts/lab6_incidents.py` | возраст очереди вырос до 23 с, `TimeToReadyFastBurn` был firing; live 200 и ready/бизнес 503 без Postgres; регистрация 201 без collector | evidence/lab6-incidents.txt |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Гипотеза трека 2: пара окон 5m+1h ловит короткий всплеск 5xx быстрее, чем пара 30m+6h, а длинная деградация остаётся видна после того, как короткое окно уже чистое. Ложные срабатывания на одном шумном интервале режет требование совпадения обоих окон.
|
||||
|
||||
На этой машине lab4 занимает `127.0.0.1:8080` и `:9000`, lab5 — `:8081` и `:9001`, поэтому прогон шёл с `compose.altports.yaml`: API `127.0.0.1:8082`, S3 `127.0.0.1:9002`. `PYTHONUTF8=1`, потому что публичные проверки читают `lab.json` в локальной кодировке Windows.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Остановка worker | upload остаётся 202, content 409, растёт возраст очереди | 202 и 409, `messenger_queue_oldest_age_seconds` дошёл до 23.4; `TimeToReadyFastBurn` был firing | worker поднят, тот же attachment стал ready; после сценария alert уже не firing |
|
||||
| Postgres остановлен | `/health/live` жив, ready и запись дают 503 | live 200, ready 503, 8 из 8 регистраций 503 | ready снова 200 после `compose start postgres`; данные не удалялись |
|
||||
| Collector остановлен | API принимает запрос | регистрация 201 | collector запущен снова |
|
||||
| Restart хранилищ и API | тот же токен и вложение | PASS persistence | volumes сохранены |
|
||||
| Остановка одной API-реплики | вторая продолжает запись | 2 instance id до отказа, восстановление записи за 3.316 с | остановленная реплика поднята отдельно |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Алерты вычисляет Prometheus; отдельной доставки нет. Анонимный Admin Grafana слушает только `127.0.0.1`. Возраст очереди — возраст строки `queued` в Postgres, а не lag consumer RabbitMQ. Две реплики публикуют один и тот же gauge глубины, поэтому на графике `max`.
|
||||
|
||||
Дальше: снять одну трассу upload до S3 вместе со структурным логом и отдельно измерить время обнаружения короткого всплеска и длинной деградации. `QueueLagHigh` в этом прогоне не дождались: worker вернули, когда возраст очереди был около 23 с, порог правила — 30 с.
|
||||
@@ -0,0 +1 @@
|
||||
"""Messenger API (lab 2): PostgreSQL source of truth, Redis sessions/cache."""
|
||||
@@ -0,0 +1,310 @@
|
||||
"""Upload path: validate bytes, PUT, then commit metadata. See migration 003."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from app import config, db, jobs, s3
|
||||
from app.errors import AppError
|
||||
from app.store import format_ts
|
||||
|
||||
logger = logging.getLogger("messenger.attachments")
|
||||
_sweep_task: asyncio.Task | None = None
|
||||
|
||||
|
||||
def _public(row, variants=None) -> dict:
|
||||
error_code = row["error_code"] if "error_code" in row.keys() else None
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]),
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": row["status"],
|
||||
"variants": variants or [],
|
||||
"error_code": error_code if row["status"] == "failed" else None,
|
||||
"created_at": format_ts(row["created_at"]),
|
||||
"updated_at": format_ts(row["updated_at"]),
|
||||
}
|
||||
|
||||
|
||||
async def _variants(attachment_id, pipeline_version: int) -> list:
|
||||
async with db.require().acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT name, content_type, size_bytes, width, height, sha256
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = $1 AND pipeline_version = $2
|
||||
ORDER BY CASE name
|
||||
WHEN 'thumbnail' THEN 1
|
||||
WHEN 'crop' THEN 2
|
||||
WHEN 'optimized' THEN 3
|
||||
ELSE 4
|
||||
END
|
||||
""",
|
||||
attachment_id,
|
||||
pipeline_version,
|
||||
)
|
||||
return [
|
||||
{
|
||||
"name": row["name"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"width": row["width"],
|
||||
"height": row["height"],
|
||||
"sha256": row["sha256"],
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
async def present(row) -> dict:
|
||||
version = int(row["applied_pipeline_version"]) if "applied_pipeline_version" in row.keys() else 1
|
||||
variants = await _variants(row["id"], version) if row["status"] == "ready" else []
|
||||
return _public(row, variants)
|
||||
|
||||
|
||||
async def _drop_intent(key: str) -> None:
|
||||
async with db.require().acquire() as conn:
|
||||
await conn.execute(
|
||||
"DELETE FROM blob_writes WHERE object_key = $1 AND NOT committed",
|
||||
key,
|
||||
)
|
||||
|
||||
|
||||
async def _compensate(key: str) -> None:
|
||||
try:
|
||||
await s3.delete_key(key)
|
||||
except AppError:
|
||||
logger.warning("orphan object left for sweeper key=%s", key)
|
||||
return
|
||||
try:
|
||||
await _drop_intent(key)
|
||||
except Exception:
|
||||
logger.warning("orphan intent left for sweeper key=%s", key)
|
||||
|
||||
|
||||
async def create_attachment(
|
||||
chat_id: str,
|
||||
owner_id: str,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
data: bytes,
|
||||
) -> dict:
|
||||
attachment_id = uuid4()
|
||||
key = f"staging/{attachment_id}"
|
||||
created = datetime.now(timezone.utc)
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
async with db.require().acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO blob_writes (object_key, committed, created_at)
|
||||
VALUES ($1, FALSE, $2)
|
||||
""",
|
||||
key,
|
||||
created,
|
||||
)
|
||||
try:
|
||||
await s3.put_bytes(key, data, content_type)
|
||||
except Exception:
|
||||
await _compensate(key)
|
||||
raise
|
||||
try:
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
intent = await conn.fetchrow(
|
||||
"SELECT committed FROM blob_writes WHERE object_key = $1 FOR UPDATE",
|
||||
key,
|
||||
)
|
||||
if intent is None or intent["committed"]:
|
||||
raise AppError(503, "service unavailable")
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO attachments (
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
sha256, object_key, status, error_code, created_at, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'queued', NULL, $9, $9)
|
||||
""",
|
||||
attachment_id,
|
||||
UUID(chat_id),
|
||||
UUID(owner_id),
|
||||
filename,
|
||||
content_type,
|
||||
len(data),
|
||||
digest,
|
||||
key,
|
||||
created,
|
||||
)
|
||||
await jobs.insert_job(conn, attachment_id, created)
|
||||
await conn.execute(
|
||||
"UPDATE blob_writes SET committed = TRUE WHERE object_key = $1",
|
||||
key,
|
||||
)
|
||||
except AppError:
|
||||
await _compensate(key)
|
||||
raise
|
||||
except Exception:
|
||||
await _compensate(key)
|
||||
raise AppError(503, "service unavailable") from None
|
||||
jobs.kick_publish()
|
||||
return _public(
|
||||
{
|
||||
"id": attachment_id,
|
||||
"chat_id": UUID(chat_id),
|
||||
"owner_id": UUID(owner_id),
|
||||
"filename": filename,
|
||||
"content_type": content_type,
|
||||
"size_bytes": len(data),
|
||||
"sha256": digest,
|
||||
"status": "queued",
|
||||
"error_code": None,
|
||||
"created_at": created,
|
||||
"updated_at": created,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def get_for_member(attachment_id: str, user_id: str):
|
||||
async with db.require().acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
|
||||
a.size_bytes, a.sha256, a.object_key, a.status, a.error_code,
|
||||
a.applied_pipeline_version, a.created_at, a.updated_at
|
||||
FROM attachments a
|
||||
JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2
|
||||
WHERE a.id = $1
|
||||
""",
|
||||
UUID(attachment_id),
|
||||
UUID(user_id),
|
||||
)
|
||||
return row
|
||||
|
||||
|
||||
async def sweep_orphans() -> int:
|
||||
"""Delete uncommitted objects older than the grace period. In-use rows stay."""
|
||||
deleted = 0
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT object_key
|
||||
FROM blob_writes
|
||||
WHERE NOT committed
|
||||
AND created_at < now() - ($1::int * interval '1 second')
|
||||
ORDER BY created_at
|
||||
LIMIT 50
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""",
|
||||
config.ORPHAN_GRACE_SECONDS,
|
||||
)
|
||||
for row in rows:
|
||||
key = row["object_key"]
|
||||
linked = await conn.fetchval(
|
||||
"""
|
||||
SELECT 1 FROM attachments WHERE object_key = $1
|
||||
UNION ALL
|
||||
SELECT 1 FROM attachment_variants WHERE object_key = $1
|
||||
LIMIT 1
|
||||
""",
|
||||
key,
|
||||
)
|
||||
busy = False
|
||||
parts = key.split("/")
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
owner = UUID(parts[1])
|
||||
except ValueError:
|
||||
owner = None
|
||||
if owner is not None:
|
||||
busy = await conn.fetchval(
|
||||
"""
|
||||
SELECT 1 FROM attachment_jobs
|
||||
WHERE attachment_id = $1 AND status IN ('queued', 'processing')
|
||||
""",
|
||||
owner,
|
||||
)
|
||||
if busy:
|
||||
continue
|
||||
if linked:
|
||||
await conn.execute(
|
||||
"UPDATE blob_writes SET committed = TRUE WHERE object_key = $1",
|
||||
key,
|
||||
)
|
||||
continue
|
||||
await s3.delete_key(key)
|
||||
await conn.execute("DELETE FROM blob_writes WHERE object_key = $1", key)
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE direct_uploads
|
||||
SET status = 'aborted'
|
||||
WHERE object_key = $1 AND status = 'pending'
|
||||
""",
|
||||
key,
|
||||
)
|
||||
deleted += 1
|
||||
if deleted:
|
||||
logger.info("removed %s uncommitted objects past grace", deleted)
|
||||
return deleted
|
||||
|
||||
|
||||
async def sweep_failed_staging() -> int:
|
||||
"""Drop staging bytes of failed jobs after the retention window. Metadata stays."""
|
||||
deleted = 0
|
||||
async with db.require().acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT id, object_key
|
||||
FROM attachments
|
||||
WHERE status = 'failed'
|
||||
AND object_key LIKE 'staging/%'
|
||||
AND updated_at < now() - ($1::int * interval '1 second')
|
||||
ORDER BY updated_at
|
||||
LIMIT 20
|
||||
""",
|
||||
config.STAGING_RETENTION_SECONDS,
|
||||
)
|
||||
for row in rows:
|
||||
await s3.delete_key(row["object_key"])
|
||||
async with db.require().acquire() as conn:
|
||||
await conn.execute("DELETE FROM blob_writes WHERE object_key = $1", row["object_key"])
|
||||
deleted += 1
|
||||
return deleted
|
||||
|
||||
|
||||
async def _sweep_loop() -> None:
|
||||
while True:
|
||||
try:
|
||||
await sweep_orphans()
|
||||
await sweep_failed_staging()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.warning("orphan sweep failed")
|
||||
await asyncio.sleep(config.S3_SWEEP_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
def start_sweeper() -> None:
|
||||
global _sweep_task
|
||||
if _sweep_task is None or _sweep_task.done():
|
||||
_sweep_task = asyncio.create_task(_sweep_loop())
|
||||
|
||||
|
||||
async def stop_sweeper() -> None:
|
||||
global _sweep_task
|
||||
task = _sweep_task
|
||||
_sweep_task = None
|
||||
if task is None:
|
||||
return
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
"""Durable RabbitMQ topology. Messages contain job ids, never file bytes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
import aio_pika
|
||||
|
||||
from app import config, telemetry
|
||||
|
||||
logger = logging.getLogger("messenger.broker")
|
||||
|
||||
EXCHANGE = "attachment.jobs"
|
||||
QUEUE = "attachment.jobs"
|
||||
RETRY_QUEUE = "attachment.jobs.retry"
|
||||
DLQ = "attachment.jobs.dlq"
|
||||
ROUTING_KEY = "process"
|
||||
|
||||
_lock = asyncio.Lock()
|
||||
_connection: aio_pika.RobustConnection | None = None
|
||||
_channel: aio_pika.RobustChannel | None = None
|
||||
_exchange: aio_pika.Exchange | None = None
|
||||
|
||||
|
||||
async def _ensure() -> tuple[aio_pika.RobustChannel, aio_pika.Exchange]:
|
||||
global _connection, _channel, _exchange
|
||||
async with _lock:
|
||||
if _channel is not None and not _channel.is_closed:
|
||||
assert _exchange is not None
|
||||
return _channel, _exchange
|
||||
if not config.BROKER_URL:
|
||||
raise RuntimeError("BROKER_URL is not set")
|
||||
_connection = await aio_pika.connect_robust(config.BROKER_URL)
|
||||
_channel = await _connection.channel(publisher_confirms=True)
|
||||
_exchange = await _channel.declare_exchange(EXCHANGE, aio_pika.ExchangeType.DIRECT, durable=True)
|
||||
await _channel.declare_queue(
|
||||
RETRY_QUEUE,
|
||||
durable=True,
|
||||
arguments={
|
||||
"x-dead-letter-exchange": EXCHANGE,
|
||||
"x-dead-letter-routing-key": ROUTING_KEY,
|
||||
},
|
||||
)
|
||||
await _channel.declare_queue(DLQ, durable=True)
|
||||
queue = await _channel.declare_queue(QUEUE, durable=True)
|
||||
await queue.bind(_exchange, routing_key=ROUTING_KEY)
|
||||
return _channel, _exchange
|
||||
|
||||
|
||||
async def publish_job(payload: dict, traceparent: str | None = None, tracestate: str | None = None) -> None:
|
||||
channel, exchange = await _ensure()
|
||||
parent = telemetry.remote_context(traceparent, tracestate)
|
||||
with telemetry.span(
|
||||
"attachment.job.publish",
|
||||
context=parent,
|
||||
kind="producer",
|
||||
attributes={
|
||||
"messaging.system": "rabbitmq",
|
||||
"messaging.destination.name": QUEUE,
|
||||
"messaging.operation": "publish",
|
||||
"job.id": payload.get("job_id"),
|
||||
},
|
||||
):
|
||||
headers = telemetry.current_carrier() or None
|
||||
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
await exchange.publish(
|
||||
aio_pika.Message(
|
||||
body=body,
|
||||
content_type="application/json",
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
message_id=payload["job_id"],
|
||||
headers=headers,
|
||||
),
|
||||
routing_key=ROUTING_KEY,
|
||||
mandatory=True,
|
||||
)
|
||||
|
||||
|
||||
async def publish_delay(payload: dict, delay_seconds: float) -> None:
|
||||
channel, _exchange = await _ensure()
|
||||
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
milliseconds = max(1, int(delay_seconds * 1000))
|
||||
await channel.default_exchange.publish(
|
||||
aio_pika.Message(
|
||||
body=body,
|
||||
content_type="application/json",
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
expiration=str(milliseconds),
|
||||
message_id=payload["job_id"],
|
||||
),
|
||||
routing_key=RETRY_QUEUE,
|
||||
mandatory=True,
|
||||
)
|
||||
|
||||
|
||||
async def publish_dlq(payload: dict) -> None:
|
||||
channel, _exchange = await _ensure()
|
||||
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
|
||||
await channel.default_exchange.publish(
|
||||
aio_pika.Message(
|
||||
body=body,
|
||||
content_type="application/json",
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
message_id=payload["job_id"],
|
||||
),
|
||||
routing_key=DLQ,
|
||||
mandatory=True,
|
||||
)
|
||||
|
||||
|
||||
async def consume(handler, prefetch: int):
|
||||
channel, _exchange = await _ensure()
|
||||
await channel.set_qos(prefetch_count=max(1, prefetch))
|
||||
queue = await channel.declare_queue(QUEUE, durable=True)
|
||||
tag = await queue.consume(handler)
|
||||
return queue, tag
|
||||
|
||||
|
||||
async def close() -> None:
|
||||
global _connection, _channel, _exchange
|
||||
async with _lock:
|
||||
if _connection is not None:
|
||||
await _connection.close()
|
||||
_connection = None
|
||||
_channel = None
|
||||
_exchange = None
|
||||
|
||||
|
||||
def payload_from(raw: bytes) -> dict:
|
||||
parsed = json.loads(raw.decode("utf-8"))
|
||||
required = ("schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at")
|
||||
if not isinstance(parsed, dict) or any(key not in parsed for key in required):
|
||||
raise ValueError("job payload is incomplete")
|
||||
if parsed["schema_version"] != 1 or parsed["type"] != "attachment.process.v1":
|
||||
raise ValueError("job payload has the wrong type")
|
||||
return parsed
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import TypeVar
|
||||
|
||||
from app import config, rediscli
|
||||
from app.errors import AppError
|
||||
|
||||
T = TypeVar("T")
|
||||
CFG_SINGLEFLIGHT = "cfg:singleflight"
|
||||
CFG_INVALIDATE = "cfg:invalidate"
|
||||
CFG_DELAY_MS = "cfg:delay_ms"
|
||||
SQL_HISTORY = "metrics:sql:messages_select"
|
||||
|
||||
|
||||
class SingleFlight:
|
||||
def __init__(self) -> None:
|
||||
self._lock = asyncio.Lock()
|
||||
self._inflight: dict[str, asyncio.Task] = {}
|
||||
|
||||
async def do(self, key: str, factory: Callable[[], Awaitable[T]]) -> T:
|
||||
async with self._lock:
|
||||
task = self._inflight.get(key)
|
||||
if task is None:
|
||||
task = asyncio.create_task(factory())
|
||||
self._inflight[key] = task
|
||||
owner = True
|
||||
else:
|
||||
owner = False
|
||||
try:
|
||||
return await asyncio.shield(task)
|
||||
finally:
|
||||
if owner:
|
||||
async with self._lock:
|
||||
self._inflight.pop(key, None)
|
||||
|
||||
|
||||
_sf = SingleFlight()
|
||||
|
||||
|
||||
async def _flag(name: str, default: bool = True) -> bool:
|
||||
if not config.lab_mode():
|
||||
return default
|
||||
try:
|
||||
raw = await rediscli.require().get(name)
|
||||
except Exception:
|
||||
return default
|
||||
if raw is None:
|
||||
return default
|
||||
return raw not in {"0", "false", "off"}
|
||||
|
||||
|
||||
async def singleflight_enabled() -> bool:
|
||||
return await _flag(CFG_SINGLEFLIGHT, True)
|
||||
|
||||
|
||||
async def invalidate_enabled() -> bool:
|
||||
return await _flag(CFG_INVALIDATE, True)
|
||||
|
||||
|
||||
async def incr_history_sql() -> None:
|
||||
try:
|
||||
await rediscli.require().incr(SQL_HISTORY)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def history_sql_count() -> int:
|
||||
try:
|
||||
return int(await rediscli.require().get(SQL_HISTORY) or 0)
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
async def reset_history_sql_count() -> None:
|
||||
try:
|
||||
await rediscli.require().set(SQL_HISTORY, 0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def load_delay_seconds() -> float:
|
||||
if not config.lab_mode():
|
||||
return 0.0
|
||||
try:
|
||||
raw = await rediscli.require().get(CFG_DELAY_MS)
|
||||
except Exception:
|
||||
return 0.0
|
||||
try:
|
||||
return max(0.0, float(raw or 0) / 1000.0)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
async def set_flags(
|
||||
*,
|
||||
singleflight: bool | None = None,
|
||||
invalidate: bool | None = None,
|
||||
delay_ms: int | None = None,
|
||||
) -> dict:
|
||||
r = rediscli.require()
|
||||
if singleflight is not None:
|
||||
await r.set(CFG_SINGLEFLIGHT, "1" if singleflight else "0")
|
||||
if invalidate is not None:
|
||||
await r.set(CFG_INVALIDATE, "1" if invalidate else "0")
|
||||
if delay_ms is not None:
|
||||
await r.set(CFG_DELAY_MS, str(int(delay_ms)))
|
||||
return {
|
||||
"singleflight": await singleflight_enabled(),
|
||||
"invalidate": await invalidate_enabled(),
|
||||
"history_sql": await history_sql_count(),
|
||||
"delay_ms": int((await load_delay_seconds()) * 1000),
|
||||
}
|
||||
|
||||
|
||||
async def get_history_page(chat_id: str, cursor: str, limit: int, loader: Callable[[], Awaitable[dict]]) -> dict:
|
||||
r = rediscli.require()
|
||||
try:
|
||||
version = await r.get("histver:" + chat_id) or "0"
|
||||
key = f"hist:{chat_id}:{version}:{cursor}:{limit}"
|
||||
cached = await r.get(key)
|
||||
if cached:
|
||||
return json.loads(cached)
|
||||
|
||||
async def load_and_store() -> dict:
|
||||
delay = await load_delay_seconds()
|
||||
if delay:
|
||||
await asyncio.sleep(delay)
|
||||
page = await loader()
|
||||
try:
|
||||
await r.set(key, json.dumps(page, ensure_ascii=False), ex=config.CACHE_TTL_SECONDS)
|
||||
except Exception:
|
||||
pass
|
||||
return page
|
||||
|
||||
if await singleflight_enabled():
|
||||
return await _sf.do(key, load_and_store)
|
||||
return await load_and_store()
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
return await loader()
|
||||
|
||||
|
||||
async def invalidate_history(chat_id: str) -> None:
|
||||
if not await invalidate_enabled():
|
||||
return
|
||||
try:
|
||||
await rediscli.require().incr("histver:" + chat_id)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,67 @@
|
||||
import os
|
||||
import socket
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _env(name: str, default: str = "") -> str:
|
||||
value = os.environ.get(name)
|
||||
return default if value is None or value == "" else value
|
||||
|
||||
|
||||
def _file_or_env(name: str, default: str = "") -> str:
|
||||
path = os.environ.get(name + "_FILE")
|
||||
if path:
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
return _env(name, default)
|
||||
|
||||
|
||||
HOST = _env("APP_HOST", "0.0.0.0")
|
||||
PORT = int(_env("APP_PORT", "8080"))
|
||||
ENV = _env("APP_ENV", "lab")
|
||||
LOG_LEVEL = _env("LOG_LEVEL", "info").lower()
|
||||
SHUTDOWN_GRACE_SECONDS = float(_env("SHUTDOWN_GRACE_SECONDS", "10"))
|
||||
SESSION_TTL_SECONDS = int(_env("SESSION_TTL_SECONDS", "3600"))
|
||||
CACHE_TTL_SECONDS = int(_env("CACHE_TTL_SECONDS", "60"))
|
||||
DATABASE_URL = _file_or_env("DATABASE_URL")
|
||||
REDIS_URL = _file_or_env("REDIS_URL", "redis://redis:6379/0")
|
||||
SESSION_STORE = _env("SESSION_STORE", "redis")
|
||||
DB_POOL_MIN = int(_env("DB_POOL_MIN", "1"))
|
||||
DB_POOL_MAX = int(_env("DB_POOL_MAX", "10"))
|
||||
DB_COMMAND_TIMEOUT = float(_env("DB_COMMAND_TIMEOUT", "5"))
|
||||
DB_CONNECT_TIMEOUT = float(_env("DB_CONNECT_TIMEOUT", "5"))
|
||||
REDIS_SOCKET_TIMEOUT = float(_env("REDIS_SOCKET_TIMEOUT", "2"))
|
||||
IDEMPOTENCY_TTL_SECONDS = int(_env("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
MIGRATIONS_ON_START = _env("MIGRATIONS_ON_START", "1") == "1"
|
||||
LB_DRAIN_SECONDS = float(_env("LB_DRAIN_SECONDS", "2"))
|
||||
# Stable for the life of this process. Compose leaves it unset so each
|
||||
# container uses its own hostname; stop-one.sh maps that value to one replica.
|
||||
INSTANCE_ID = _env("INSTANCE_ID", socket.gethostname())
|
||||
S3_ENDPOINT = _env("S3_ENDPOINT", "http://s3:9000")
|
||||
S3_PUBLIC_ENDPOINT = _env("S3_PUBLIC_ENDPOINT", "http://127.0.0.1:9000")
|
||||
PRESIGN_TTL_SECONDS = min(max(int(_env("PRESIGN_TTL_SECONDS", "60")), 15), 300)
|
||||
S3_REGION = _env("S3_REGION", "us-east-1")
|
||||
S3_BUCKET = _env("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY_ID = _file_or_env("S3_ACCESS_KEY_ID")
|
||||
S3_SECRET_ACCESS_KEY = _file_or_env("S3_SECRET_ACCESS_KEY")
|
||||
MAX_UPLOAD_BYTES = min(int(_env("MAX_UPLOAD_BYTES", "10485760")), 10_485_760)
|
||||
MAX_IMAGE_PIXELS = min(int(_env("MAX_IMAGE_PIXELS", "40000000")), 40_000_000)
|
||||
ORPHAN_GRACE_SECONDS = int(_env("ORPHAN_GRACE_SECONDS", "120"))
|
||||
S3_SWEEP_INTERVAL_SECONDS = int(_env("S3_SWEEP_INTERVAL_SECONDS", "30"))
|
||||
BROKER_URL = _file_or_env("BROKER_URL")
|
||||
WORKER_CONCURRENCY = max(1, int(_env("WORKER_CONCURRENCY", "2")))
|
||||
JOB_MAX_ATTEMPTS = max(1, int(_env("JOB_MAX_ATTEMPTS", "5")))
|
||||
JOB_TIMEOUT_SECONDS = max(1, int(_env("JOB_TIMEOUT_SECONDS", "60")))
|
||||
JOB_LEASE_SECONDS = JOB_TIMEOUT_SECONDS + 15
|
||||
JOB_BACKOFF_BASE_SECONDS = float(_env("JOB_BACKOFF_BASE_SECONDS", "0.5"))
|
||||
JOB_BACKOFF_CAP_SECONDS = float(_env("JOB_BACKOFF_CAP_SECONDS", "30"))
|
||||
STAGING_RETENTION_SECONDS = int(_env("STAGING_RETENTION_SECONDS", "86400"))
|
||||
WORKER_CRASH_POINT = _env("WORKER_CRASH_POINT", "")
|
||||
TELEMETRY_ENABLED = _env("TELEMETRY_ENABLED", "1") == "1"
|
||||
OTEL_SERVICE_NAME = _env("OTEL_SERVICE_NAME", "messenger-api")
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = _env("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318")
|
||||
TRACE_SAMPLE_RATIO = float(_env("TRACE_SAMPLE_RATIO", "1"))
|
||||
SERVICE_VERSION = _env("SERVICE_VERSION", "lab6")
|
||||
|
||||
|
||||
def lab_mode() -> bool:
|
||||
return ENV == "lab"
|
||||
@@ -0,0 +1,112 @@
|
||||
import base64
|
||||
|
||||
import json
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from typing import Any
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def encode_cursor(chat_id: str, created_at: str, entity_id: str) -> str:
|
||||
|
||||
payload = json.dumps({"c": chat_id, "t": created_at, "i": entity_id}, separators=(",", ":"))
|
||||
|
||||
return base64.urlsafe_b64encode(payload.encode("utf-8")).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def _validate_cursor_timestamp(value: str) -> None:
|
||||
|
||||
if not value.endswith("Z"):
|
||||
|
||||
raise ValueError("invalid cursor timestamp")
|
||||
|
||||
try:
|
||||
|
||||
datetime.fromisoformat(value[:-1] + "+00:00")
|
||||
|
||||
except ValueError as exc:
|
||||
|
||||
raise ValueError("invalid cursor timestamp") from exc
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def _require_str_field(data: dict, key: str) -> str:
|
||||
|
||||
if key not in data:
|
||||
|
||||
raise ValueError("invalid cursor")
|
||||
|
||||
value = data[key]
|
||||
|
||||
if not isinstance(value, str):
|
||||
|
||||
raise ValueError("invalid cursor")
|
||||
|
||||
return value
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def decode_cursor(cursor: str, expected_chat_id: str) -> tuple[str, str]:
|
||||
|
||||
pad = "=" * (-len(cursor) % 4)
|
||||
|
||||
try:
|
||||
|
||||
raw = base64.urlsafe_b64decode((cursor + pad).encode("ascii"))
|
||||
|
||||
data: Any = json.loads(raw.decode("utf-8"))
|
||||
|
||||
except (ValueError, json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
|
||||
raise ValueError("invalid cursor") from exc
|
||||
|
||||
if not isinstance(data, dict):
|
||||
|
||||
raise ValueError("invalid cursor")
|
||||
|
||||
try:
|
||||
|
||||
chat_id = _require_str_field(data, "c")
|
||||
|
||||
created_at = _require_str_field(data, "t")
|
||||
|
||||
entity_id = _require_str_field(data, "i")
|
||||
|
||||
except ValueError as exc:
|
||||
|
||||
raise ValueError("invalid cursor") from exc
|
||||
|
||||
if chat_id != expected_chat_id:
|
||||
|
||||
raise ValueError("cursor chat mismatch")
|
||||
|
||||
try:
|
||||
|
||||
UUID(entity_id)
|
||||
|
||||
except ValueError as exc:
|
||||
|
||||
raise ValueError("invalid cursor") from exc
|
||||
|
||||
try:
|
||||
|
||||
_validate_cursor_timestamp(created_at)
|
||||
|
||||
except ValueError as exc:
|
||||
|
||||
raise ValueError("invalid cursor") from exc
|
||||
|
||||
return created_at, entity_id
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
from pathlib import Path
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app import config
|
||||
from app.errors import AppError
|
||||
|
||||
pool: asyncpg.Pool | None = None
|
||||
INTEGRITY = (
|
||||
asyncpg.UniqueViolationError,
|
||||
asyncpg.ForeignKeyViolationError,
|
||||
asyncpg.CheckViolationError,
|
||||
)
|
||||
|
||||
|
||||
def _migrations_dir() -> Path:
|
||||
here = Path(__file__).resolve().parent
|
||||
for candidate in (here.parent / "migrations", Path("/app/migrations")):
|
||||
if candidate.is_dir():
|
||||
return candidate
|
||||
raise RuntimeError("migrations directory not found")
|
||||
|
||||
|
||||
async def connect() -> asyncpg.Pool:
|
||||
global pool
|
||||
if not config.DATABASE_URL:
|
||||
raise RuntimeError("DATABASE_URL is not set")
|
||||
pool = await asyncpg.create_pool(
|
||||
dsn=config.DATABASE_URL,
|
||||
min_size=config.DB_POOL_MIN,
|
||||
max_size=config.DB_POOL_MAX,
|
||||
command_timeout=config.DB_COMMAND_TIMEOUT,
|
||||
timeout=config.DB_CONNECT_TIMEOUT,
|
||||
)
|
||||
return pool
|
||||
|
||||
|
||||
async def close() -> None:
|
||||
global pool
|
||||
if pool is not None:
|
||||
await pool.close()
|
||||
pool = None
|
||||
|
||||
|
||||
def require() -> asyncpg.Pool:
|
||||
if pool is None:
|
||||
raise AppError(503, "service unavailable")
|
||||
return pool
|
||||
|
||||
|
||||
async def ping() -> None:
|
||||
try:
|
||||
async with require().acquire() as conn:
|
||||
await conn.execute("SELECT 1")
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
|
||||
|
||||
_MIGRATION_LOCK = 87133103
|
||||
|
||||
|
||||
async def apply_migrations() -> None:
|
||||
"""Apply pending SQL files once. A session advisory lock stops replica races."""
|
||||
directory = _migrations_dir()
|
||||
async with require().acquire() as conn:
|
||||
await conn.execute("SELECT pg_advisory_lock($1)", _MIGRATION_LOCK)
|
||||
try:
|
||||
await conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
filename TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
)
|
||||
"""
|
||||
)
|
||||
applied = {row["filename"] for row in await conn.fetch("SELECT filename FROM schema_migrations")}
|
||||
for path in sorted(p for p in directory.glob("*.sql") if p.is_file()):
|
||||
if path.name in applied:
|
||||
continue
|
||||
sql = path.read_text(encoding="utf-8")
|
||||
async with conn.transaction():
|
||||
await conn.execute(sql)
|
||||
await conn.execute("INSERT INTO schema_migrations(filename) VALUES($1)", path.name)
|
||||
finally:
|
||||
await conn.execute("SELECT pg_advisory_unlock($1)", _MIGRATION_LOCK)
|
||||
+203
@@ -0,0 +1,203 @@
|
||||
"""Presigned PUT that does not replace the proxy upload endpoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from app import config, db, images, s3
|
||||
from app.attachments import _compensate, _public
|
||||
from app import jobs
|
||||
from app.errors import AppError
|
||||
from app.store import format_ts
|
||||
|
||||
|
||||
def _aware(value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value
|
||||
|
||||
|
||||
async def begin_upload(
|
||||
chat_id: str,
|
||||
owner_id: str,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
size_bytes: int,
|
||||
) -> dict:
|
||||
upload_id = uuid4()
|
||||
key = f"originals/{upload_id}"
|
||||
created = datetime.now(timezone.utc)
|
||||
expires_at = created + timedelta(seconds=config.PRESIGN_TTL_SECONDS)
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO blob_writes (object_key, committed, created_at)
|
||||
VALUES ($1, FALSE, $2)
|
||||
""",
|
||||
key,
|
||||
created,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO direct_uploads (
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
object_key, status, expires_at, created_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, 'pending', $8, $9)
|
||||
""",
|
||||
upload_id,
|
||||
UUID(chat_id),
|
||||
UUID(owner_id),
|
||||
filename,
|
||||
content_type,
|
||||
size_bytes,
|
||||
key,
|
||||
expires_at,
|
||||
created,
|
||||
)
|
||||
try:
|
||||
url = await asyncio.to_thread(s3.presign_put, key, content_type, size_bytes)
|
||||
except Exception:
|
||||
await _compensate(key)
|
||||
async with db.require().acquire() as conn:
|
||||
await conn.execute(
|
||||
"UPDATE direct_uploads SET status = 'aborted' WHERE id = $1 AND status = 'pending'",
|
||||
upload_id,
|
||||
)
|
||||
raise
|
||||
return {
|
||||
"upload_id": str(upload_id),
|
||||
"method": "PUT",
|
||||
"url": url,
|
||||
"headers": {"Content-Type": content_type},
|
||||
"expires_at": format_ts(expires_at),
|
||||
"size_bytes": size_bytes,
|
||||
}
|
||||
|
||||
|
||||
async def _abort_pending(upload_id: UUID, key: str) -> None:
|
||||
async with db.require().acquire() as conn:
|
||||
changed = await conn.fetchval(
|
||||
"""
|
||||
UPDATE direct_uploads
|
||||
SET status = 'aborted'
|
||||
WHERE id = $1 AND status = 'pending'
|
||||
RETURNING id
|
||||
""",
|
||||
upload_id,
|
||||
)
|
||||
if changed is not None:
|
||||
await _compensate(key)
|
||||
|
||||
|
||||
async def complete_upload(upload_id: str, owner_id: str, claimed_sha: str) -> dict:
|
||||
async with db.require().acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
object_key, status, expires_at
|
||||
FROM direct_uploads
|
||||
WHERE id = $1 AND owner_id = $2
|
||||
""",
|
||||
UUID(upload_id),
|
||||
UUID(owner_id),
|
||||
)
|
||||
if row is None:
|
||||
raise AppError(404, "upload not found")
|
||||
if row["status"] != "pending":
|
||||
raise AppError(409, "upload is already finished")
|
||||
if _aware(row["expires_at"]) <= datetime.now(timezone.utc):
|
||||
await _abort_pending(row["id"], row["object_key"])
|
||||
raise AppError(409, "upload signature expired")
|
||||
|
||||
actual_size = await s3.head_size(row["object_key"])
|
||||
if actual_size is None:
|
||||
raise AppError(409, "upload is incomplete")
|
||||
if actual_size != row["size_bytes"] or actual_size > config.MAX_UPLOAD_BYTES:
|
||||
await _abort_pending(row["id"], row["object_key"])
|
||||
raise AppError(400, "uploaded size does not match")
|
||||
|
||||
data = await s3.get_bytes(row["object_key"])
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
if len(data) != row["size_bytes"] or digest != claimed_sha:
|
||||
await _abort_pending(row["id"], row["object_key"])
|
||||
raise AppError(400, "checksum does not match")
|
||||
try:
|
||||
images.inspect_image(data, row["content_type"])
|
||||
except images.ImageRejected as exc:
|
||||
await _abort_pending(row["id"], row["object_key"])
|
||||
raise AppError(exc.status, exc.message) from None
|
||||
|
||||
created = datetime.now(timezone.utc)
|
||||
try:
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
locked = await conn.fetchrow(
|
||||
"""
|
||||
SELECT status, object_key
|
||||
FROM direct_uploads
|
||||
WHERE id = $1 AND owner_id = $2
|
||||
FOR UPDATE
|
||||
""",
|
||||
row["id"],
|
||||
UUID(owner_id),
|
||||
)
|
||||
if locked is None or locked["status"] != "pending":
|
||||
raise AppError(409, "upload is already finished")
|
||||
intent = await conn.fetchrow(
|
||||
"SELECT committed FROM blob_writes WHERE object_key = $1 FOR UPDATE",
|
||||
row["object_key"],
|
||||
)
|
||||
if intent is None or intent["committed"]:
|
||||
raise AppError(409, "upload is already finished")
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO attachments (
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
sha256, object_key, status, error_code, created_at, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'queued', NULL, $9, $9)
|
||||
""",
|
||||
row["id"],
|
||||
row["chat_id"],
|
||||
row["owner_id"],
|
||||
row["filename"],
|
||||
row["content_type"],
|
||||
len(data),
|
||||
digest,
|
||||
row["object_key"],
|
||||
created,
|
||||
)
|
||||
await jobs.insert_job(conn, row["id"], created)
|
||||
await conn.execute(
|
||||
"UPDATE blob_writes SET committed = TRUE WHERE object_key = $1",
|
||||
row["object_key"],
|
||||
)
|
||||
await conn.execute(
|
||||
"UPDATE direct_uploads SET status = 'completed' WHERE id = $1",
|
||||
row["id"],
|
||||
)
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
jobs.kick_publish()
|
||||
return _public(
|
||||
{
|
||||
"id": row["id"],
|
||||
"chat_id": row["chat_id"],
|
||||
"owner_id": row["owner_id"],
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": len(data),
|
||||
"sha256": digest,
|
||||
"status": "queued",
|
||||
"error_code": None,
|
||||
"created_at": created,
|
||||
"updated_at": created,
|
||||
}
|
||||
)
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
import json
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import HTMLResponse, JSONResponse, Response
|
||||
|
||||
_SPEC_CANDIDATES = (
|
||||
Path(__file__).resolve().parents[1] / "contracts" / "openapi.json",
|
||||
Path("/app/contracts/openapi.json"),
|
||||
)
|
||||
|
||||
_SWAGGER_HTML = """<!DOCTYPE html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<title>MTUSI Messenger — Swagger (lab2)</title>
|
||||
<link rel="stylesheet" href="https://unpkg.com/swagger-ui-dist@5.11.0/swagger-ui.css"/>
|
||||
</head>
|
||||
<body>
|
||||
<div id="swagger-ui"></div>
|
||||
<script src="https://unpkg.com/swagger-ui-dist@5.11.0/swagger-ui-bundle.js"></script>
|
||||
<script>
|
||||
window.onload = function () {
|
||||
SwaggerUIBundle({
|
||||
url: "/openapi.json",
|
||||
dom_id: "#swagger-ui",
|
||||
deepLinking: true,
|
||||
persistAuthorization: true,
|
||||
tryItOutEnabled: true
|
||||
});
|
||||
};
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _load_spec() -> dict:
|
||||
for path in _SPEC_CANDIDATES:
|
||||
if path.is_file():
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
return {"openapi": "3.1.0", "info": {"title": "MTUSI Messenger", "version": "1.2.0"}, "paths": {}}
|
||||
|
||||
|
||||
async def openapi_json(_: Request) -> Response:
|
||||
return JSONResponse(_load_spec())
|
||||
|
||||
|
||||
async def swagger_ui(_: Request) -> Response:
|
||||
return HTMLResponse(_SWAGGER_HTML)
|
||||
@@ -0,0 +1,49 @@
|
||||
from typing import Dict, Optional
|
||||
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
|
||||
ERROR_CODES = {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
503: "unavailable",
|
||||
500: "internal_error",
|
||||
}
|
||||
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
class AppError(Exception):
|
||||
def __init__(self, status: int, message: str, www_authenticate: str | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.message = message
|
||||
self.www_authenticate = www_authenticate
|
||||
|
||||
|
||||
def error_response(
|
||||
status: int,
|
||||
message: str,
|
||||
request_id: str,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
) -> JSONResponse:
|
||||
code = ERROR_CODES.get(status, "internal_error")
|
||||
headers = {"X-Request-Id": request_id, "Content-Type": "application/json"}
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
headers=headers,
|
||||
content={"error": {"code": code, "message": message, "request_id": request_id}},
|
||||
)
|
||||
|
||||
|
||||
def from_app_error(exc: AppError, request_id: str) -> JSONResponse:
|
||||
extra = {"WWW-Authenticate": exc.www_authenticate} if exc.www_authenticate else None
|
||||
return error_response(exc.status, exc.message, request_id, extra)
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
"""PNG/JPEG checks that reject bad files before a full pixel decode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from PIL import Image, ImageFile, UnidentifiedImageError
|
||||
|
||||
from app import config
|
||||
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = False
|
||||
_PNG = b"\x89PNG\r\n\x1a\n"
|
||||
_JPEG = b"\xff\xd8\xff"
|
||||
_SOF = {0xC0, 0xC1, 0xC2}
|
||||
|
||||
|
||||
class ImageRejected(Exception):
|
||||
def __init__(self, status: int, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.message = message
|
||||
|
||||
|
||||
def _reject(status: int, message: str) -> None:
|
||||
raise ImageRejected(status, message)
|
||||
|
||||
|
||||
def sniff(data: bytes) -> Optional[str]:
|
||||
if data.startswith(_PNG):
|
||||
return "image/png"
|
||||
if data.startswith(_JPEG):
|
||||
return "image/jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def _png_size(data: bytes) -> Tuple[int, int]:
|
||||
if len(data) < 24 or data[12:16] != b"IHDR":
|
||||
_reject(415, "unsupported or damaged image")
|
||||
length = int.from_bytes(data[8:12], "big")
|
||||
if length != 13 or len(data) < 33:
|
||||
_reject(415, "unsupported or damaged image")
|
||||
width = int.from_bytes(data[16:20], "big")
|
||||
height = int.from_bytes(data[20:24], "big")
|
||||
return width, height
|
||||
|
||||
|
||||
def _jpeg_size(data: bytes) -> Tuple[int, int]:
|
||||
index = 2
|
||||
size = len(data)
|
||||
while index < size:
|
||||
if data[index] != 0xFF:
|
||||
_reject(415, "unsupported or damaged image")
|
||||
while index < size and data[index] == 0xFF:
|
||||
index += 1
|
||||
if index >= size:
|
||||
break
|
||||
marker = data[index]
|
||||
index += 1
|
||||
if marker in (0xD8, 0xD9, 0x01) or 0xD0 <= marker <= 0xD7:
|
||||
continue
|
||||
if index + 1 >= size:
|
||||
break
|
||||
segment = int.from_bytes(data[index : index + 2], "big")
|
||||
if segment < 2 or index + segment > size:
|
||||
_reject(415, "unsupported or damaged image")
|
||||
if marker in _SOF:
|
||||
if segment < 7:
|
||||
_reject(415, "unsupported or damaged image")
|
||||
height = int.from_bytes(data[index + 3 : index + 5], "big")
|
||||
width = int.from_bytes(data[index + 5 : index + 7], "big")
|
||||
return width, height
|
||||
if marker == 0xDA:
|
||||
break
|
||||
index += segment
|
||||
_reject(415, "unsupported or damaged image")
|
||||
return 0, 0
|
||||
|
||||
|
||||
def _check_pixels(width: int, height: int) -> None:
|
||||
limit = config.MAX_IMAGE_PIXELS
|
||||
if width <= 0 or height <= 0:
|
||||
_reject(415, "unsupported or damaged image")
|
||||
if width > limit or height > limit or width * height > limit:
|
||||
_reject(413, "image dimensions exceed the limit")
|
||||
|
||||
|
||||
def _verify(data: bytes) -> None:
|
||||
Image.MAX_IMAGE_PIXELS = config.MAX_IMAGE_PIXELS
|
||||
try:
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
image.verify()
|
||||
except Image.DecompressionBombError:
|
||||
_reject(413, "image dimensions exceed the limit")
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
||||
_reject(415, "unsupported or damaged image")
|
||||
|
||||
|
||||
def inspect_image(data: bytes, declared: str) -> str:
|
||||
"""Return the sniffed content type or raise ImageRejected (413 or 415)."""
|
||||
found = sniff(data)
|
||||
if found is None or found != declared:
|
||||
_reject(415, "unsupported or damaged image")
|
||||
width, height = _png_size(data) if found == "image/png" else _jpeg_size(data)
|
||||
_check_pixels(width, height)
|
||||
_verify(data)
|
||||
return found
|
||||
+829
@@ -0,0 +1,829 @@
|
||||
"""Job rows are the transactional outbox. The broker never stores file bytes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from app import broker, config, db, pipeline, s3, telemetry
|
||||
from app.errors import AppError
|
||||
from app.pipeline import PermanentJobError, TransientJobError
|
||||
from app.store import format_ts
|
||||
|
||||
logger = logging.getLogger("messenger.jobs")
|
||||
_outbox_task: asyncio.Task | None = None
|
||||
_publish_lock = asyncio.Lock()
|
||||
|
||||
|
||||
def message_of(row) -> dict:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"job_id": str(row["job_id"]),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": str(row["attachment_id"]),
|
||||
"pipeline_version": int(row["pipeline_version"]),
|
||||
"attempt": int(row["attempt"]),
|
||||
"created_at": format_ts(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def backoff_seconds(attempt: int) -> float:
|
||||
base = min(config.JOB_BACKOFF_CAP_SECONDS, config.JOB_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1)))
|
||||
return base * random.uniform(0.5, 1.5)
|
||||
|
||||
|
||||
def _trace_columns() -> tuple[str | None, str | None]:
|
||||
carrier = telemetry.current_carrier()
|
||||
return carrier.get("traceparent") or None, carrier.get("tracestate") or None
|
||||
|
||||
|
||||
async def insert_job(conn, attachment_id: UUID, created: datetime) -> UUID:
|
||||
job_id = uuid4()
|
||||
traceparent, tracestate = _trace_columns()
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO attachment_jobs (
|
||||
job_id, attachment_id, pipeline_version, attempt, status,
|
||||
next_attempt_at, created_at, updated_at, traceparent, tracestate
|
||||
)
|
||||
VALUES ($1, $2, 1, 1, 'queued', $3, $3, $3, $4, $5)
|
||||
""",
|
||||
job_id,
|
||||
attachment_id,
|
||||
created,
|
||||
traceparent,
|
||||
tracestate,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail)
|
||||
VALUES ($1, $2, 'queued', 'upload committed')
|
||||
""",
|
||||
job_id,
|
||||
attachment_id,
|
||||
)
|
||||
return job_id
|
||||
|
||||
|
||||
def kick_publish() -> None:
|
||||
try:
|
||||
asyncio.get_running_loop().create_task(publish_pending())
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
|
||||
async def publish_pending(limit: int = 20) -> int:
|
||||
if not config.BROKER_URL:
|
||||
return 0
|
||||
async with _publish_lock:
|
||||
try:
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT job_id, attachment_id, pipeline_version, attempt, created_at,
|
||||
traceparent, tracestate
|
||||
FROM attachment_jobs
|
||||
WHERE published_at IS NULL
|
||||
AND status = 'queued'
|
||||
AND (next_attempt_at IS NULL OR next_attempt_at <= now())
|
||||
ORDER BY created_at
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""",
|
||||
limit,
|
||||
)
|
||||
for row in rows:
|
||||
await broker.publish_job(
|
||||
message_of(row),
|
||||
traceparent=row["traceparent"],
|
||||
tracestate=row["tracestate"],
|
||||
)
|
||||
await conn.execute(
|
||||
"UPDATE attachment_jobs SET published_at = now() WHERE job_id = $1",
|
||||
row["job_id"],
|
||||
)
|
||||
return len(rows)
|
||||
except Exception:
|
||||
logger.warning("outbox publish failed; the job row stays unpublished")
|
||||
return 0
|
||||
|
||||
|
||||
async def _outbox_loop() -> None:
|
||||
while True:
|
||||
try:
|
||||
await publish_pending()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.warning("outbox loop failed")
|
||||
await asyncio.sleep(0.25)
|
||||
|
||||
|
||||
def start_outbox() -> None:
|
||||
global _outbox_task
|
||||
if _outbox_task is None or _outbox_task.done():
|
||||
_outbox_task = asyncio.create_task(_outbox_loop())
|
||||
|
||||
|
||||
async def stop_outbox() -> None:
|
||||
global _outbox_task
|
||||
task = _outbox_task
|
||||
_outbox_task = None
|
||||
if task is None:
|
||||
return
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
await broker.close()
|
||||
|
||||
|
||||
def _aware(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value
|
||||
|
||||
|
||||
async def claim(job_id: UUID, pipeline_version: int):
|
||||
token = uuid4()
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
row = await conn.fetchrow(
|
||||
"SELECT * FROM attachment_jobs WHERE job_id = $1 FOR UPDATE",
|
||||
job_id,
|
||||
)
|
||||
if row is None:
|
||||
return "missing", None
|
||||
if int(row["pipeline_version"]) != pipeline_version:
|
||||
return "stale", row
|
||||
if row["status"] == "ready":
|
||||
return "ready", row
|
||||
attachment = await conn.fetchrow(
|
||||
"SELECT status, applied_pipeline_version FROM attachments WHERE id = $1",
|
||||
row["attachment_id"],
|
||||
)
|
||||
applied = int(attachment["applied_pipeline_version"]) if attachment is not None else 0
|
||||
if applied > int(row["pipeline_version"]):
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'ready', error_code = NULL, lease_token = NULL,
|
||||
lease_until = NULL, updated_at = now()
|
||||
WHERE job_id = $1 AND status <> 'ready'
|
||||
""",
|
||||
job_id,
|
||||
)
|
||||
return "ready", row
|
||||
if row["status"] == "failed":
|
||||
return "failed", row
|
||||
lease = _aware(row["lease_until"])
|
||||
if row["status"] == "processing" and lease is not None and lease > datetime.now(timezone.utc):
|
||||
return "busy", row
|
||||
if int(row["attempt"]) > config.JOB_MAX_ATTEMPTS:
|
||||
await _fail(conn, row, "retry_exhausted")
|
||||
return "failed", row
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'processing',
|
||||
lease_token = $2,
|
||||
lease_until = now() + ($3::int * interval '1 second'),
|
||||
error_code = NULL,
|
||||
updated_at = now()
|
||||
WHERE job_id = $1
|
||||
""",
|
||||
job_id,
|
||||
token,
|
||||
config.JOB_LEASE_SECONDS,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'processing', error_code = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status <> 'ready'
|
||||
""",
|
||||
row["attachment_id"],
|
||||
)
|
||||
claimed = dict(row)
|
||||
claimed["lease_token"] = token
|
||||
claimed["status"] = "processing"
|
||||
return "claimed", claimed
|
||||
|
||||
|
||||
async def _fail(conn, row, code: str) -> None:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', error_code = $2, lease_token = NULL, lease_until = NULL, updated_at = now()
|
||||
WHERE job_id = $1 AND status <> 'ready'
|
||||
""",
|
||||
row["job_id"],
|
||||
code,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = $2, updated_at = now()
|
||||
WHERE id = $1 AND status <> 'ready'
|
||||
""",
|
||||
row["attachment_id"],
|
||||
code,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail)
|
||||
VALUES ($1, $2, 'failed', $3)
|
||||
""",
|
||||
row["job_id"],
|
||||
row["attachment_id"],
|
||||
code,
|
||||
)
|
||||
|
||||
|
||||
async def fail_job(job_id: UUID, lease_token: UUID, code: str) -> bool:
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT * FROM attachment_jobs
|
||||
WHERE job_id = $1 AND lease_token = $2 AND status = 'processing'
|
||||
FOR UPDATE
|
||||
""",
|
||||
job_id,
|
||||
lease_token,
|
||||
)
|
||||
if row is None:
|
||||
return False
|
||||
await _fail(conn, row, code)
|
||||
failed = dict(row)
|
||||
try:
|
||||
await broker.publish_dlq(message_of(failed))
|
||||
except Exception:
|
||||
logger.warning("failed job is stored; DLQ publish will be retried by an operator replay")
|
||||
return True
|
||||
|
||||
|
||||
async def schedule_retry(job_id: UUID, lease_token: UUID, code: str) -> str:
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT * FROM attachment_jobs
|
||||
WHERE job_id = $1 AND lease_token = $2 AND status = 'processing'
|
||||
FOR UPDATE
|
||||
""",
|
||||
job_id,
|
||||
lease_token,
|
||||
)
|
||||
if row is None:
|
||||
return "lost"
|
||||
nxt = int(row["attempt"]) + 1
|
||||
if nxt > config.JOB_MAX_ATTEMPTS:
|
||||
await _fail(conn, row, "retry_exhausted")
|
||||
failed = True
|
||||
else:
|
||||
delay = backoff_seconds(int(row["attempt"]))
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued',
|
||||
attempt = $2,
|
||||
error_code = $3,
|
||||
lease_token = NULL,
|
||||
lease_until = NULL,
|
||||
published_at = NULL,
|
||||
next_attempt_at = now() + ($4::double precision * interval '1 second'),
|
||||
updated_at = now()
|
||||
WHERE job_id = $1
|
||||
""",
|
||||
job_id,
|
||||
nxt,
|
||||
code,
|
||||
delay,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'queued', error_code = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status <> 'ready'
|
||||
""",
|
||||
row["attachment_id"],
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail)
|
||||
VALUES ($1, $2, 'retry', $3)
|
||||
""",
|
||||
job_id,
|
||||
row["attachment_id"],
|
||||
code,
|
||||
)
|
||||
failed = False
|
||||
failed_row = None
|
||||
if failed:
|
||||
failed_row = row
|
||||
if failed:
|
||||
try:
|
||||
await broker.publish_dlq(message_of(failed_row))
|
||||
except Exception:
|
||||
logger.warning("retry exhausted; DLQ publish failed")
|
||||
return "failed"
|
||||
return "retry"
|
||||
|
||||
|
||||
async def _intent(key: str) -> None:
|
||||
async with db.require().acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO blob_writes (object_key, committed)
|
||||
VALUES ($1, FALSE)
|
||||
ON CONFLICT (object_key) DO NOTHING
|
||||
""",
|
||||
key,
|
||||
)
|
||||
|
||||
|
||||
async def complete_job(
|
||||
job_id: UUID,
|
||||
lease_token: UUID,
|
||||
attachment_id: UUID,
|
||||
final_key: str,
|
||||
variants: list,
|
||||
pipeline_version: int,
|
||||
) -> bool:
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
owned = await conn.fetchrow(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'ready', error_code = NULL, lease_token = NULL, lease_until = NULL, updated_at = now()
|
||||
WHERE job_id = $1 AND lease_token = $2 AND status = 'processing'
|
||||
RETURNING pipeline_version
|
||||
""",
|
||||
job_id,
|
||||
lease_token,
|
||||
)
|
||||
if owned is None:
|
||||
return False
|
||||
current = await conn.fetchrow(
|
||||
"""
|
||||
SELECT status, applied_pipeline_version
|
||||
FROM attachments
|
||||
WHERE id = $1
|
||||
FOR UPDATE
|
||||
""",
|
||||
attachment_id,
|
||||
)
|
||||
if current is None:
|
||||
raise PermanentJobError("invalid_image")
|
||||
version = int(owned["pipeline_version"])
|
||||
applied = int(current["applied_pipeline_version"])
|
||||
superseded = applied > version
|
||||
if not superseded:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'ready',
|
||||
object_key = CASE WHEN $2 = '' THEN object_key ELSE $2 END,
|
||||
error_code = NULL,
|
||||
applied_pipeline_version = $3,
|
||||
updated_at = now()
|
||||
WHERE id = $1 AND applied_pipeline_version <= $3
|
||||
""",
|
||||
attachment_id,
|
||||
final_key,
|
||||
version,
|
||||
)
|
||||
for variant in variants:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO attachment_variants (
|
||||
attachment_id, pipeline_version, name, content_type, size_bytes,
|
||||
width, height, sha256, object_key
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING
|
||||
""",
|
||||
attachment_id,
|
||||
version,
|
||||
variant["name"],
|
||||
variant["content_type"],
|
||||
variant["size_bytes"],
|
||||
variant["width"],
|
||||
variant["height"],
|
||||
variant["sha256"],
|
||||
variant["object_key"],
|
||||
)
|
||||
await conn.execute(
|
||||
"UPDATE blob_writes SET committed = TRUE WHERE object_key = $1",
|
||||
variant["object_key"],
|
||||
)
|
||||
if final_key:
|
||||
await conn.execute(
|
||||
"UPDATE blob_writes SET committed = TRUE WHERE object_key = $1",
|
||||
final_key,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail)
|
||||
VALUES ($1, $2, 'ready', $3)
|
||||
""",
|
||||
job_id,
|
||||
attachment_id,
|
||||
"superseded" if superseded else "variants committed",
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
async def load_attachment(attachment_id: UUID):
|
||||
async with db.require().acquire() as conn:
|
||||
return await conn.fetchrow(
|
||||
"""
|
||||
SELECT id, filename, content_type, size_bytes, sha256, object_key, status,
|
||||
applied_pipeline_version
|
||||
FROM attachments
|
||||
WHERE id = $1
|
||||
""",
|
||||
attachment_id,
|
||||
)
|
||||
|
||||
|
||||
async def process(job_id: UUID, pipeline_version: int) -> str:
|
||||
started = time.perf_counter()
|
||||
state, row = await claim(job_id, pipeline_version)
|
||||
if state != "claimed":
|
||||
return state
|
||||
assert row is not None
|
||||
lease = row["lease_token"]
|
||||
try:
|
||||
await asyncio.wait_for(_effect(row, lease), config.JOB_TIMEOUT_SECONDS)
|
||||
outcome = "ready"
|
||||
except PermanentJobError as exc:
|
||||
await fail_job(job_id, lease, exc.code)
|
||||
outcome = "failed"
|
||||
except (TransientJobError, asyncio.TimeoutError, AppError):
|
||||
outcome = await schedule_retry(job_id, lease, "processing_failed")
|
||||
except Exception:
|
||||
logger.warning("job processing failed job_id=%s", job_id)
|
||||
outcome = await schedule_retry(job_id, lease, "processing_failed")
|
||||
elapsed = time.perf_counter() - started
|
||||
if outcome in ("ready", "failed", "retry", "lost"):
|
||||
telemetry.record_job(outcome, elapsed)
|
||||
if outcome == "ready":
|
||||
created = _aware(row["created_at"])
|
||||
if created is not None:
|
||||
telemetry.record_time_to_ready((datetime.now(timezone.utc) - created).total_seconds())
|
||||
return outcome
|
||||
|
||||
|
||||
async def _effect(row, lease: UUID) -> None:
|
||||
attachment = await load_attachment(row["attachment_id"])
|
||||
if attachment is None:
|
||||
raise PermanentJobError("invalid_image")
|
||||
version = int(row["pipeline_version"])
|
||||
if int(attachment["applied_pipeline_version"]) > version:
|
||||
committed = await complete_job(row["job_id"], lease, attachment["id"], "", [], version)
|
||||
if not committed:
|
||||
raise TransientJobError("processing_failed")
|
||||
return
|
||||
fault = await consume_fault()
|
||||
if fault == "crash_before_s3":
|
||||
logger.warning("injected crash before side effect job_id=%s", row["job_id"])
|
||||
os._exit(1)
|
||||
if fault == "fail_once":
|
||||
raise PermanentJobError("processing_failed")
|
||||
with telemetry.span("s3.get", kind="client", attributes={"s3.operation": "get"}):
|
||||
data = await s3.get_bytes(attachment["object_key"])
|
||||
if len(data) != attachment["size_bytes"] or hashlib.sha256(data).hexdigest() != attachment["sha256"]:
|
||||
raise PermanentJobError("invalid_image")
|
||||
final_key = f"originals/{attachment['id']}"
|
||||
if attachment["object_key"] != final_key:
|
||||
await _intent(final_key)
|
||||
with telemetry.span("s3.put", kind="client", attributes={"s3.operation": "put", "variant": "original"}):
|
||||
await s3.put_bytes(final_key, data, attachment["content_type"])
|
||||
with telemetry.span("image.render"):
|
||||
rendered = await asyncio.to_thread(pipeline.render, data, attachment["content_type"], version)
|
||||
stored = []
|
||||
for name, (payload, content_type, width, height) in rendered.items():
|
||||
suffix = "webp" if content_type == "image/webp" else "jpg"
|
||||
key = f"variants/{attachment['id']}/v{version}/{name}.{suffix}"
|
||||
await _intent(key)
|
||||
with telemetry.span("s3.put", kind="client", attributes={"s3.operation": "put", "variant": name}):
|
||||
await s3.put_bytes(key, payload, content_type)
|
||||
stored.append(
|
||||
{
|
||||
"name": name,
|
||||
"content_type": content_type,
|
||||
"size_bytes": len(payload),
|
||||
"width": width,
|
||||
"height": height,
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
"object_key": key,
|
||||
}
|
||||
)
|
||||
if fault == "crash_after_s3" or config.WORKER_CRASH_POINT == "after_s3":
|
||||
logger.warning("injected crash after object write before result commit job_id=%s", row["job_id"])
|
||||
os._exit(1)
|
||||
fresh = await load_attachment(attachment["id"])
|
||||
if fresh is not None and int(fresh["applied_pipeline_version"]) > version:
|
||||
stored = []
|
||||
final_key = ""
|
||||
with telemetry.span("db.complete", kind="client"):
|
||||
committed = await complete_job(row["job_id"], lease, attachment["id"], final_key, stored, version)
|
||||
if not committed:
|
||||
raise TransientJobError("processing_failed")
|
||||
if stored and attachment["object_key"] != final_key and final_key:
|
||||
try:
|
||||
await s3.delete_key(attachment["object_key"])
|
||||
async with db.require().acquire() as conn:
|
||||
await conn.execute("DELETE FROM blob_writes WHERE object_key = $1", attachment["object_key"])
|
||||
except Exception:
|
||||
logger.warning("staging object left for retention attachment_id=%s", attachment["id"])
|
||||
|
||||
|
||||
async def consume_fault() -> str:
|
||||
async with db.require().acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
UPDATE pipeline_faults
|
||||
SET remaining = remaining - 1
|
||||
WHERE id = 1 AND remaining > 0
|
||||
RETURNING mode
|
||||
"""
|
||||
)
|
||||
if row is None:
|
||||
return ""
|
||||
return row["mode"] or ""
|
||||
|
||||
|
||||
async def arm_fault(mode: str, remaining: int) -> dict:
|
||||
if mode not in ("", "crash_before_s3", "crash_after_s3", "fail_once"):
|
||||
raise AppError(400, "invalid fault")
|
||||
if remaining < 0 or remaining > 5:
|
||||
raise AppError(400, "invalid fault")
|
||||
async with db.require().acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO pipeline_faults (id, mode, remaining)
|
||||
VALUES (1, $1, $2)
|
||||
ON CONFLICT (id) DO UPDATE SET mode = EXCLUDED.mode, remaining = EXCLUDED.remaining
|
||||
""",
|
||||
mode,
|
||||
remaining,
|
||||
)
|
||||
return {"mode": mode, "remaining": remaining}
|
||||
|
||||
|
||||
async def bump_pipeline(attachment_id: UUID) -> dict:
|
||||
created = datetime.now(timezone.utc)
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
current = await conn.fetchval(
|
||||
"SELECT COALESCE(MAX(pipeline_version), 0) FROM attachment_jobs WHERE attachment_id = $1",
|
||||
attachment_id,
|
||||
)
|
||||
if current is None:
|
||||
return {"result": "missing"}
|
||||
exists = await conn.fetchval("SELECT 1 FROM attachments WHERE id = $1", attachment_id)
|
||||
if exists is None:
|
||||
return {"result": "missing"}
|
||||
version = int(current) + 1
|
||||
job_id = uuid4()
|
||||
traceparent, tracestate = _trace_columns()
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO attachment_jobs (
|
||||
job_id, attachment_id, pipeline_version, attempt, status,
|
||||
next_attempt_at, created_at, updated_at, traceparent, tracestate
|
||||
)
|
||||
VALUES ($1, $2, $3, 1, 'queued', $4, $4, $4, $5, $6)
|
||||
""",
|
||||
job_id,
|
||||
attachment_id,
|
||||
version,
|
||||
created,
|
||||
traceparent,
|
||||
tracestate,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail)
|
||||
VALUES ($1, $2, 'queued', 'pipeline version bumped')
|
||||
""",
|
||||
job_id,
|
||||
attachment_id,
|
||||
)
|
||||
kick_publish()
|
||||
return {"result": "queued", "job_id": str(job_id), "pipeline_version": version}
|
||||
|
||||
|
||||
async def redeliver(job_id: UUID) -> dict:
|
||||
async with db.require().acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT job_id, attachment_id, pipeline_version, attempt, created_at, status,
|
||||
traceparent, tracestate
|
||||
FROM attachment_jobs
|
||||
WHERE job_id = $1
|
||||
""",
|
||||
job_id,
|
||||
)
|
||||
if row is None:
|
||||
return {"result": "missing"}
|
||||
await broker.publish_job(message_of(row), traceparent=row["traceparent"], tracestate=row["tracestate"])
|
||||
return {
|
||||
"result": "published",
|
||||
"job_id": str(row["job_id"]),
|
||||
"pipeline_version": int(row["pipeline_version"]),
|
||||
"status": row["status"],
|
||||
}
|
||||
|
||||
|
||||
async def list_jobs(attachment_id: UUID) -> list:
|
||||
async with db.require().acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT job_id, pipeline_version, status, attempt
|
||||
FROM attachment_jobs
|
||||
WHERE attachment_id = $1
|
||||
ORDER BY pipeline_version
|
||||
""",
|
||||
attachment_id,
|
||||
)
|
||||
variants = await conn.fetch(
|
||||
"""
|
||||
SELECT pipeline_version, name, sha256
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = $1
|
||||
ORDER BY pipeline_version, name
|
||||
""",
|
||||
attachment_id,
|
||||
)
|
||||
applied = await conn.fetchval(
|
||||
"SELECT applied_pipeline_version FROM attachments WHERE id = $1",
|
||||
attachment_id,
|
||||
)
|
||||
return {
|
||||
"applied_pipeline_version": applied,
|
||||
"jobs": [
|
||||
{
|
||||
"job_id": str(row["job_id"]),
|
||||
"pipeline_version": int(row["pipeline_version"]),
|
||||
"status": row["status"],
|
||||
"attempt": int(row["attempt"]),
|
||||
}
|
||||
for row in rows
|
||||
],
|
||||
"variants": [
|
||||
{
|
||||
"pipeline_version": int(row["pipeline_version"]),
|
||||
"name": row["name"],
|
||||
"sha256": row["sha256"],
|
||||
}
|
||||
for row in variants
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
async def replay_attachment(attachment_id: str) -> dict:
|
||||
async with db.require().acquire() as conn:
|
||||
job_id = await conn.fetchval(
|
||||
"""
|
||||
SELECT job_id FROM attachment_jobs
|
||||
WHERE attachment_id = $1 AND status = 'failed'
|
||||
ORDER BY pipeline_version DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
UUID(attachment_id),
|
||||
)
|
||||
if job_id is None:
|
||||
return {"result": "missing"}
|
||||
return await replay(job_id)
|
||||
|
||||
|
||||
async def replay(job_id: UUID) -> dict:
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
row = await conn.fetchrow(
|
||||
"SELECT * FROM attachment_jobs WHERE job_id = $1 FOR UPDATE",
|
||||
job_id,
|
||||
)
|
||||
if row is None:
|
||||
return {"result": "missing"}
|
||||
if row["status"] == "ready":
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail)
|
||||
VALUES ($1, $2, 'replay', 'noop ready')
|
||||
""",
|
||||
row["job_id"],
|
||||
row["attachment_id"],
|
||||
)
|
||||
return {"result": "noop", "job_id": str(row["job_id"])}
|
||||
lease = _aware(row["lease_until"])
|
||||
if row["status"] == "processing" and lease is not None and lease > datetime.now(timezone.utc):
|
||||
return {"result": "busy", "job_id": str(row["job_id"])}
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued',
|
||||
attempt = 1,
|
||||
error_code = NULL,
|
||||
lease_token = NULL,
|
||||
lease_until = NULL,
|
||||
published_at = NULL,
|
||||
next_attempt_at = now(),
|
||||
updated_at = now()
|
||||
WHERE job_id = $1 AND status <> 'ready'
|
||||
""",
|
||||
job_id,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'queued', error_code = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status <> 'ready'
|
||||
""",
|
||||
row["attachment_id"],
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail)
|
||||
VALUES ($1, $2, 'replay', 'operator replay')
|
||||
""",
|
||||
row["job_id"],
|
||||
row["attachment_id"],
|
||||
)
|
||||
await publish_pending()
|
||||
return {"result": "queued", "job_id": str(job_id)}
|
||||
|
||||
|
||||
async def reap_expired() -> int:
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT job_id, attachment_id, attempt
|
||||
FROM attachment_jobs
|
||||
WHERE status = 'processing' AND lease_until < now()
|
||||
FOR UPDATE SKIP LOCKED
|
||||
"""
|
||||
)
|
||||
changed = 0
|
||||
for row in rows:
|
||||
nxt = int(row["attempt"]) + 1
|
||||
if nxt > config.JOB_MAX_ATTEMPTS:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed', error_code = 'retry_exhausted',
|
||||
lease_token = NULL, lease_until = NULL, updated_at = now()
|
||||
WHERE job_id = $1 AND status = 'processing'
|
||||
""",
|
||||
row["job_id"],
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = 'retry_exhausted', updated_at = now()
|
||||
WHERE id = $1 AND status <> 'ready'
|
||||
""",
|
||||
row["attachment_id"],
|
||||
)
|
||||
else:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued',
|
||||
attempt = $2,
|
||||
error_code = 'lease_expired',
|
||||
lease_token = NULL,
|
||||
lease_until = NULL,
|
||||
published_at = NULL,
|
||||
next_attempt_at = now(),
|
||||
updated_at = now()
|
||||
WHERE job_id = $1 AND status = 'processing'
|
||||
""",
|
||||
row["job_id"],
|
||||
nxt,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'queued', error_code = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status <> 'ready'
|
||||
""",
|
||||
row["attachment_id"],
|
||||
)
|
||||
changed += 1
|
||||
return changed
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import signal
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import uvicorn
|
||||
from starlette.applications import Starlette
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
from starlette.routing import Route
|
||||
|
||||
from app import attachments, config, db, jobs, rediscli, telemetry
|
||||
from app.docs import openapi_json, swagger_ui
|
||||
from app.errors import AppError, error_response, from_app_error
|
||||
from app.middleware import RequestContextMiddleware
|
||||
from app.routes import (
|
||||
begin_direct_upload,
|
||||
complete_direct_upload,
|
||||
create_chat,
|
||||
create_message,
|
||||
create_session,
|
||||
current_user,
|
||||
download_attachment,
|
||||
get_attachment,
|
||||
get_chat,
|
||||
health_live,
|
||||
health_ready,
|
||||
lab_bump,
|
||||
lab_fault,
|
||||
lab_flags,
|
||||
lab_jobs,
|
||||
lab_metrics,
|
||||
lab_redeliver,
|
||||
lab_replay,
|
||||
lab_sweep,
|
||||
list_chats,
|
||||
list_messages,
|
||||
register_user,
|
||||
revoke_session,
|
||||
upload_attachment,
|
||||
)
|
||||
|
||||
|
||||
def _request_id(request: Request) -> str:
|
||||
existing = getattr(request.state, "request_id", None)
|
||||
if existing:
|
||||
return existing
|
||||
return request.headers.get("X-Request-Id") or str(uuid.uuid4())
|
||||
|
||||
|
||||
async def not_found_handler(request: Request, exc: HTTPException) -> Response:
|
||||
return error_response(404, "not found", _request_id(request))
|
||||
|
||||
|
||||
async def app_error_handler(request: Request, exc: AppError) -> Response:
|
||||
return from_app_error(exc, _request_id(request))
|
||||
|
||||
|
||||
def publish_instance_id() -> None:
|
||||
path = Path(config._env("INSTANCE_ID_FILE", "/tmp/instance-id"))
|
||||
try:
|
||||
path.write_text(config.INSTANCE_ID, encoding="utf-8")
|
||||
except OSError:
|
||||
logging.getLogger("messenger").warning("instance id file was not written")
|
||||
|
||||
|
||||
async def startup() -> None:
|
||||
publish_instance_id()
|
||||
logging.getLogger("messenger").info("instance_id=%s", config.INSTANCE_ID)
|
||||
await db.connect()
|
||||
if config.MIGRATIONS_ON_START:
|
||||
await db.apply_migrations()
|
||||
await rediscli.connect()
|
||||
attachments.start_sweeper()
|
||||
jobs.start_outbox()
|
||||
telemetry.start_probes()
|
||||
|
||||
|
||||
async def shutdown() -> None:
|
||||
await telemetry.stop_probes()
|
||||
await attachments.stop_sweeper()
|
||||
await jobs.stop_outbox()
|
||||
await rediscli.close()
|
||||
await db.close()
|
||||
telemetry.shutdown()
|
||||
|
||||
|
||||
def build_app() -> Starlette:
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Route("/docs", swagger_ui, methods=["GET"]),
|
||||
Route("/openapi.json", openapi_json, methods=["GET"]),
|
||||
Route("/health/live", health_live, methods=["GET"]),
|
||||
Route("/health/ready", health_ready, methods=["GET"]),
|
||||
Route("/api/v1/users", register_user, methods=["POST"]),
|
||||
Route("/api/v1/users/me", current_user, methods=["GET"]),
|
||||
Route("/api/v1/auth/sessions", create_session, methods=["POST"]),
|
||||
Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]),
|
||||
Route("/api/v1/chats", create_chat, methods=["POST"]),
|
||||
Route("/api/v1/chats", list_chats, methods=["GET"]),
|
||||
Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]),
|
||||
Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]),
|
||||
Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]),
|
||||
Route("/api/v1/chats/{chat_id}/attachments", upload_attachment, methods=["POST"]),
|
||||
Route("/api/v1/chats/{chat_id}/uploads", begin_direct_upload, methods=["POST"]),
|
||||
Route("/api/v1/uploads/{upload_id}/complete", complete_direct_upload, methods=["POST"]),
|
||||
Route("/api/v1/attachments/{attachment_id}", get_attachment, methods=["GET"]),
|
||||
Route("/api/v1/attachments/{attachment_id}/content", download_attachment, methods=["GET"]),
|
||||
Route("/api/v1/_lab/metrics", lab_metrics, methods=["GET"]),
|
||||
Route("/api/v1/_lab/flags", lab_flags, methods=["POST"]),
|
||||
Route("/api/v1/_lab/sweep", lab_sweep, methods=["POST"]),
|
||||
Route("/api/v1/_lab/faults", lab_fault, methods=["POST"]),
|
||||
Route("/api/v1/_lab/attachments/{attachment_id}/jobs", lab_jobs, methods=["GET"]),
|
||||
Route("/api/v1/_lab/attachments/{attachment_id}/pipelines", lab_bump, methods=["POST"]),
|
||||
Route("/api/v1/_lab/jobs/{job_id}/redeliver", lab_redeliver, methods=["POST"]),
|
||||
Route("/api/v1/_lab/attachments/{attachment_id}/replay", lab_replay, methods=["POST"]),
|
||||
],
|
||||
exception_handlers={404: not_found_handler, AppError: app_error_handler},
|
||||
on_startup=[startup],
|
||||
on_shutdown=[shutdown],
|
||||
)
|
||||
app.state.shutting_down = False
|
||||
app.add_middleware(RequestContextMiddleware)
|
||||
return app
|
||||
|
||||
|
||||
def configure_logging() -> None:
|
||||
telemetry.configure_logging()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
configure_logging()
|
||||
app = build_app()
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
server = uvicorn.Server(
|
||||
uvicorn.Config(
|
||||
app,
|
||||
host=config.HOST,
|
||||
port=config.PORT,
|
||||
log_level=config.LOG_LEVEL,
|
||||
access_log=False,
|
||||
timeout_graceful_shutdown=int(config.SHUTDOWN_GRACE_SECONDS),
|
||||
)
|
||||
)
|
||||
|
||||
def _begin_shutdown() -> None:
|
||||
if app.state.shutting_down:
|
||||
return
|
||||
logging.getLogger("messenger").info(
|
||||
"shutdown signal received; readiness fails now, exit in %.1fs",
|
||||
config.LB_DRAIN_SECONDS,
|
||||
)
|
||||
app.state.shutting_down = True
|
||||
loop.call_later(config.LB_DRAIN_SECONDS, lambda: setattr(server, "should_exit", True))
|
||||
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
try:
|
||||
loop.add_signal_handler(sig, _begin_shutdown)
|
||||
except NotImplementedError:
|
||||
signal.signal(sig, lambda *_: _begin_shutdown())
|
||||
|
||||
loop.run_until_complete(server.serve())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,93 @@
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import nullcontext
|
||||
|
||||
from starlette.datastructures import MutableHeaders
|
||||
from starlette.responses import Response
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from app import telemetry
|
||||
from app.config import INSTANCE_ID
|
||||
from app.errors import AppError, error_response, from_app_error
|
||||
|
||||
logger = logging.getLogger("messenger.access")
|
||||
|
||||
|
||||
class RequestContextMiddleware:
|
||||
"""Pure ASGI middleware so upload handlers can read the body stream themselves."""
|
||||
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
if scope["type"] != "http":
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
headers = {key.decode("latin1").lower(): value.decode("latin1") for key, value in scope.get("headers", [])}
|
||||
request_id = headers.get("x-request-id") or str(uuid.uuid4())
|
||||
raw_state = scope.get("state")
|
||||
if not isinstance(raw_state, dict):
|
||||
raw_state = {}
|
||||
scope["state"] = raw_state
|
||||
raw_state["request_id"] = request_id
|
||||
path = scope.get("path", "")
|
||||
method = scope.get("method", "")
|
||||
started = time.perf_counter()
|
||||
status_code = 500
|
||||
record = telemetry.observe_http(path)
|
||||
route = telemetry.route_template(path)
|
||||
|
||||
async def send_wrapper(message: Message) -> None:
|
||||
nonlocal status_code
|
||||
if message["type"] == "http.response.start":
|
||||
status_code = int(message["status"])
|
||||
response_headers = MutableHeaders(scope=message)
|
||||
response_headers["X-Request-Id"] = request_id
|
||||
response_headers["X-Instance-Id"] = INSTANCE_ID
|
||||
await send(message)
|
||||
|
||||
parent = telemetry.remote_context(headers.get("traceparent"), headers.get("tracestate"))
|
||||
span_cm = (
|
||||
telemetry.span(
|
||||
"%s %s" % (method, route),
|
||||
context=parent,
|
||||
kind="server",
|
||||
attributes={
|
||||
"http.request.method": method,
|
||||
"http.route": route,
|
||||
"http.request_id": request_id,
|
||||
},
|
||||
)
|
||||
if record
|
||||
else nullcontext(None)
|
||||
)
|
||||
with span_cm as current:
|
||||
app = scope.get("app")
|
||||
shutting_down = bool(app and getattr(app.state, "shutting_down", False))
|
||||
try:
|
||||
if shutting_down and path not in ("/health/live", "/health/ready"):
|
||||
response: Response = error_response(503, "service is shutting down", request_id)
|
||||
await response(scope, receive, send_wrapper)
|
||||
else:
|
||||
await self.app(scope, receive, send_wrapper)
|
||||
except AppError as exc:
|
||||
status_code = exc.status
|
||||
await from_app_error(exc, request_id)(scope, receive, send_wrapper)
|
||||
except Exception:
|
||||
status_code = 500
|
||||
logger.exception("request failed request_id=%s route=%s", request_id, route)
|
||||
await error_response(500, "internal error", request_id)(scope, receive, send_wrapper)
|
||||
finally:
|
||||
if record:
|
||||
telemetry.finish_http_span(current, status_code)
|
||||
telemetry.record_http(method, route, status_code, time.perf_counter() - started)
|
||||
if record:
|
||||
logger.info(
|
||||
"request completed request_id=%s method=%s route=%s status=%s duration_ms=%.2f",
|
||||
request_id,
|
||||
method,
|
||||
route,
|
||||
status_code,
|
||||
(time.perf_counter() - started) * 1000,
|
||||
)
|
||||
@@ -0,0 +1,29 @@
|
||||
"""One-shot schema apply. API replicas do not run this on startup."""
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from app import config, db
|
||||
|
||||
|
||||
async def _main() -> None:
|
||||
if not config.DATABASE_URL:
|
||||
raise SystemExit("DATABASE_URL is not set")
|
||||
await db.connect()
|
||||
try:
|
||||
await db.apply_migrations()
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO),
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
stream=sys.stdout,
|
||||
)
|
||||
asyncio.run(_main())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,23 @@
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerifyMismatchError
|
||||
|
||||
hasher = PasswordHasher(time_cost=2, memory_cost=8192, parallelism=1)
|
||||
_dummy = None
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return hasher.hash(password)
|
||||
|
||||
|
||||
def dummy_hash() -> str:
|
||||
global _dummy
|
||||
if _dummy is None:
|
||||
_dummy = hash_password("timing-dummy-password")
|
||||
return _dummy
|
||||
|
||||
|
||||
def verify_password(password_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return hasher.verify(password_hash, password)
|
||||
except (VerifyMismatchError, InvalidHashError, TypeError, ValueError):
|
||||
return False
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
"""Image variants. The worker calls this; the API process does not resize."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import math
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
from app import config
|
||||
from app.images import ImageRejected, inspect_image
|
||||
|
||||
|
||||
class PermanentJobError(Exception):
|
||||
def __init__(self, code: str) -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
class TransientJobError(Exception):
|
||||
def __init__(self, code: str = "processing_failed") -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def _flatten(image: Image.Image) -> Image.Image:
|
||||
if image.mode == "RGB":
|
||||
return image
|
||||
rgba = image.convert("RGBA")
|
||||
background = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
background.paste(rgba, mask=rgba.getchannel("A"))
|
||||
return background
|
||||
|
||||
|
||||
def _open_rgb(data: bytes, content_type: str) -> Image.Image:
|
||||
try:
|
||||
inspect_image(data, content_type)
|
||||
except ImageRejected as exc:
|
||||
raise PermanentJobError("invalid_image") from exc
|
||||
Image.MAX_IMAGE_PIXELS = config.MAX_IMAGE_PIXELS
|
||||
try:
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
image.load()
|
||||
oriented = ImageOps.exif_transpose(image) or image
|
||||
return _flatten(oriented).copy()
|
||||
except Image.DecompressionBombError as exc:
|
||||
raise PermanentJobError("invalid_image") from exc
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc:
|
||||
raise PermanentJobError("invalid_image") from exc
|
||||
|
||||
|
||||
def _fit_down(image: Image.Image, box: int) -> Image.Image:
|
||||
width, height = image.size
|
||||
scale = min(box / width, box / height, 1.0)
|
||||
target = (max(1, math.floor(width * scale)), max(1, math.floor(height * scale)))
|
||||
if target == (width, height):
|
||||
return image
|
||||
return image.resize(target, Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def _crop_square(image: Image.Image, side: int) -> Image.Image:
|
||||
width, height = image.size
|
||||
edge = min(width, height)
|
||||
left = (width - edge) // 2
|
||||
top = (height - edge) // 2
|
||||
cropped = image.crop((left, top, left + edge, top + edge))
|
||||
if cropped.size != (side, side):
|
||||
cropped = cropped.resize((side, side), Image.Resampling.LANCZOS)
|
||||
return cropped
|
||||
|
||||
|
||||
def _jpeg(image: Image.Image, quality: int) -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, format="JPEG", quality=quality, optimize=True)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def _webp(image: Image.Image, quality: int) -> bytes:
|
||||
# method=6. EXIF is omitted because the saved image is a new RGB buffer.
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, format="WEBP", quality=quality, method=6)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def render(data: bytes, content_type: str, pipeline_version: int = 1) -> dict:
|
||||
"""Return thumbnail, crop and optimized payloads. Does not touch storage.
|
||||
|
||||
Version 1 is the contract profile (JPEG 90, WebP 80). A later version lowers
|
||||
quality so a stale writer cannot be confused with the newer bytes.
|
||||
"""
|
||||
jpeg_quality = 90 if pipeline_version <= 1 else 60
|
||||
webp_quality = 80 if pipeline_version <= 1 else 50
|
||||
image = _open_rgb(data, content_type)
|
||||
thumb = _fit_down(image, 256)
|
||||
crop = _crop_square(image, 128)
|
||||
optimized = _fit_down(image, 1280)
|
||||
return {
|
||||
"thumbnail": (_jpeg(thumb, jpeg_quality), "image/jpeg", thumb.size[0], thumb.size[1]),
|
||||
"crop": (_jpeg(crop, jpeg_quality), "image/jpeg", crop.size[0], crop.size[1]),
|
||||
"optimized": (_webp(optimized, webp_quality), "image/webp", optimized.size[0], optimized.size[1]),
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import redis.asyncio as redis
|
||||
|
||||
from app import config
|
||||
from app.errors import AppError
|
||||
|
||||
client: redis.Redis | None = None
|
||||
|
||||
|
||||
async def connect() -> redis.Redis:
|
||||
global client
|
||||
client = redis.from_url(
|
||||
config.REDIS_URL,
|
||||
socket_connect_timeout=config.REDIS_SOCKET_TIMEOUT,
|
||||
socket_timeout=config.REDIS_SOCKET_TIMEOUT,
|
||||
decode_responses=True,
|
||||
)
|
||||
await client.ping()
|
||||
return client
|
||||
|
||||
|
||||
async def close() -> None:
|
||||
global client
|
||||
if client is not None:
|
||||
await client.close()
|
||||
client = None
|
||||
|
||||
|
||||
def require() -> redis.Redis:
|
||||
if client is None:
|
||||
raise AppError(503, "service unavailable")
|
||||
return client
|
||||
|
||||
|
||||
async def ping() -> None:
|
||||
try:
|
||||
await require().ping()
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Operator replay of one failed job. A ready job is left unchanged."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from uuid import UUID
|
||||
|
||||
from app import db, jobs
|
||||
|
||||
|
||||
async def _run(job_id: UUID) -> int:
|
||||
await db.connect()
|
||||
try:
|
||||
result = await jobs.replay(job_id)
|
||||
finally:
|
||||
await db.close()
|
||||
print(result["result"])
|
||||
return 0 if result["result"] in {"noop", "queued"} else 1
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: python -m app.replay JOB_ID", file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
raise SystemExit(asyncio.run(_run(UUID(sys.argv[1]))))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+523
@@ -0,0 +1,523 @@
|
||||
import base64
|
||||
import binascii
|
||||
import re
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse, Response
|
||||
|
||||
from app import attachments, cache, config, db, direct, images, jobs, rediscli, s3, sessions, store
|
||||
from app.cursor_codec import decode_cursor, encode_cursor
|
||||
from app.errors import BASIC_CHALLENGE, BEARER_CHALLENGE, AppError, error_response
|
||||
from app.passwords import dummy_hash, hash_password, verify_password
|
||||
from app.validation import (
|
||||
parse_json_body,
|
||||
require_exact_keys,
|
||||
parse_attachment_ids,
|
||||
validate_display_name,
|
||||
validate_filename,
|
||||
validate_limit,
|
||||
validate_idempotency_key,
|
||||
validate_message_text,
|
||||
validate_password,
|
||||
validate_title,
|
||||
validate_username,
|
||||
validate_uuid,
|
||||
)
|
||||
|
||||
|
||||
def _json(data: dict, request_id: str, status: int = 200) -> JSONResponse:
|
||||
return JSONResponse(status_code=status, headers={"X-Request-Id": request_id}, content=data)
|
||||
|
||||
|
||||
def _empty(request_id: str) -> Response:
|
||||
return Response(status_code=204, headers={"X-Request-Id": request_id})
|
||||
|
||||
|
||||
async def _bearer_user(request: Request) -> dict:
|
||||
header = request.headers.get("Authorization") or ""
|
||||
scheme, _, rest = header.partition(" ")
|
||||
if scheme.lower() != "bearer" or not rest.strip():
|
||||
raise AppError(401, "authentication required", BEARER_CHALLENGE)
|
||||
user_id = str(await sessions.resolve(rest.strip()))
|
||||
user = await store.get_user(user_id)
|
||||
if user is None:
|
||||
raise AppError(401, "authentication required", BEARER_CHALLENGE)
|
||||
return user
|
||||
|
||||
|
||||
async def health_live(request: Request) -> Response:
|
||||
return _json({"status": "ok"}, request.state.request_id)
|
||||
|
||||
|
||||
async def health_ready(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
if request.app.state.shutting_down:
|
||||
return error_response(503, "service is shutting down", rid)
|
||||
await db.ping()
|
||||
await rediscli.ping()
|
||||
await s3.head_bucket()
|
||||
return _json({"status": "ok"}, rid)
|
||||
|
||||
|
||||
async def register_user(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
if request.headers.get("content-type", "").split(";")[0].strip().lower() not in ("application/json", ""):
|
||||
if request.headers.get("content-type"):
|
||||
return error_response(415, "unsupported media type", rid)
|
||||
raw = await request.body()
|
||||
body, err = parse_json_body(raw)
|
||||
if err:
|
||||
return error_response(400, err, rid)
|
||||
assert body is not None
|
||||
if require_exact_keys(body, {"username", "display_name", "password"}, {"username", "display_name", "password"}):
|
||||
return error_response(400, "invalid user payload", rid)
|
||||
if validate_username(body["username"]) or validate_display_name(body["display_name"]) or validate_password(body["password"]):
|
||||
return error_response(400, "invalid user fields", rid)
|
||||
user, conflict = await store.create_user(body["username"], body["display_name"], hash_password(body["password"]))
|
||||
if conflict:
|
||||
return error_response(409, "username already taken", rid)
|
||||
assert user is not None
|
||||
return _json(user, rid, 201)
|
||||
|
||||
|
||||
async def current_user(request: Request) -> Response:
|
||||
user = await _bearer_user(request)
|
||||
return _json(user, request.state.request_id)
|
||||
|
||||
|
||||
async def create_session(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
header = request.headers.get("Authorization") or ""
|
||||
scheme, _, rest = header.partition(" ")
|
||||
if scheme.lower() != "basic" or not rest.strip():
|
||||
return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
|
||||
try:
|
||||
decoded = base64.b64decode(rest.strip(), validate=True).decode("utf-8")
|
||||
except (binascii.Error, UnicodeError, ValueError):
|
||||
return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
|
||||
username, sep, password = decoded.partition(":")
|
||||
if not sep:
|
||||
return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
|
||||
row = await store.get_user_by_username(username)
|
||||
password_hash = row["password_hash"] if row else dummy_hash()
|
||||
if row is None or not verify_password(password_hash, password):
|
||||
return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
|
||||
token, expires_at = await sessions.create(row["id"])
|
||||
return _json(
|
||||
{
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": expires_at,
|
||||
"user": store.user_dict(row),
|
||||
},
|
||||
rid,
|
||||
201,
|
||||
)
|
||||
|
||||
|
||||
async def revoke_session(request: Request) -> Response:
|
||||
header = request.headers.get("Authorization") or ""
|
||||
scheme, _, rest = header.partition(" ")
|
||||
if scheme.lower() != "bearer" or not rest.strip():
|
||||
raise AppError(401, "authentication required", BEARER_CHALLENGE)
|
||||
await sessions.revoke(rest.strip())
|
||||
return _empty(request.state.request_id)
|
||||
|
||||
|
||||
async def create_chat(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if ct and ct != "application/json":
|
||||
return error_response(415, "unsupported media type", rid)
|
||||
raw = await request.body()
|
||||
body, err = parse_json_body(raw)
|
||||
if err:
|
||||
return error_response(400, err, rid)
|
||||
assert body is not None
|
||||
if require_exact_keys(body, {"title", "member_ids"}, {"title", "member_ids"}):
|
||||
return error_response(400, "invalid chat payload", rid)
|
||||
if validate_title(body["title"]):
|
||||
return error_response(400, "invalid title", rid)
|
||||
members = body["member_ids"]
|
||||
if not isinstance(members, list) or not members:
|
||||
return error_response(400, "invalid member_ids", rid)
|
||||
if len(members) > 100:
|
||||
return error_response(400, "too many members", rid)
|
||||
for mid in members:
|
||||
if validate_uuid(mid):
|
||||
return error_response(400, "invalid member id", rid)
|
||||
chat, create_err = await store.create_chat(body["title"], members, user["id"])
|
||||
if create_err == "caller not in members":
|
||||
return error_response(400, "caller must be a member", rid)
|
||||
if create_err in ("duplicate members", "unknown member"):
|
||||
return error_response(400, "invalid members", rid)
|
||||
assert chat is not None
|
||||
return _json(chat, rid, 201)
|
||||
|
||||
|
||||
async def list_chats(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
limit, limit_err = validate_limit(request.query_params.get("limit"))
|
||||
if limit_err:
|
||||
return error_response(400, limit_err, rid)
|
||||
assert limit is not None
|
||||
cursor_raw = request.query_params.get("cursor")
|
||||
after = None
|
||||
if cursor_raw is not None:
|
||||
if not cursor_raw.strip():
|
||||
return error_response(400, "invalid cursor", rid)
|
||||
try:
|
||||
created_at, entity_id = decode_cursor(cursor_raw, user["id"])
|
||||
except ValueError:
|
||||
return error_response(400, "invalid cursor", rid)
|
||||
after = (created_at, entity_id)
|
||||
items, next_key = await store.list_chats(user["id"], limit, after)
|
||||
next_cursor = None
|
||||
if next_key is not None:
|
||||
next_cursor = encode_cursor(user["id"], next_key[0], next_key[1])
|
||||
return _json({"items": items, "next_cursor": next_cursor}, rid)
|
||||
|
||||
|
||||
async def get_chat(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
chat_id = request.path_params["chat_id"]
|
||||
if validate_uuid(chat_id):
|
||||
return error_response(400, "invalid chat id", rid)
|
||||
chat = await store.get_chat(chat_id, user["id"])
|
||||
if chat is None:
|
||||
return error_response(404, "chat not found", rid)
|
||||
return _json(chat, rid)
|
||||
|
||||
|
||||
async def create_message(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
chat_id = request.path_params["chat_id"]
|
||||
if validate_uuid(chat_id):
|
||||
return error_response(400, "invalid chat id", rid)
|
||||
ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if ct and ct != "application/json":
|
||||
return error_response(415, "unsupported media type", rid)
|
||||
raw = await request.body()
|
||||
body, err = parse_json_body(raw)
|
||||
if err:
|
||||
return error_response(400, err, rid)
|
||||
assert body is not None
|
||||
if require_exact_keys(body, {"text", "attachment_ids"}, {"text"}):
|
||||
return error_response(400, "invalid message payload", rid)
|
||||
attachment_ids, attachment_err = parse_attachment_ids(body.get("attachment_ids"))
|
||||
if attachment_err:
|
||||
return error_response(400, attachment_err, rid)
|
||||
assert attachment_ids is not None
|
||||
if validate_message_text(body["text"], allow_blank=bool(attachment_ids)):
|
||||
return error_response(400, "invalid message text", rid)
|
||||
if validate_idempotency_key(request.headers.get("Idempotency-Key")):
|
||||
return error_response(400, "invalid idempotency key", rid)
|
||||
message, msg_err = await store.add_message(
|
||||
chat_id,
|
||||
user["id"],
|
||||
body["text"],
|
||||
request.headers.get("Idempotency-Key") or "",
|
||||
request.url.path,
|
||||
attachment_ids,
|
||||
)
|
||||
if msg_err == "not_found":
|
||||
return error_response(404, "chat not found", rid)
|
||||
if msg_err == "conflict":
|
||||
return error_response(409, "idempotency key reused with a different body", rid)
|
||||
if msg_err == "not_ready":
|
||||
return error_response(409, "attachment is not ready", rid)
|
||||
assert message is not None
|
||||
return _json(message, rid, 201)
|
||||
|
||||
|
||||
async def list_messages(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
chat_id = request.path_params["chat_id"]
|
||||
if validate_uuid(chat_id):
|
||||
return error_response(400, "invalid chat id", rid)
|
||||
limit, limit_err = validate_limit(request.query_params.get("limit"))
|
||||
if limit_err:
|
||||
return error_response(400, limit_err, rid)
|
||||
assert limit is not None
|
||||
cursor_raw = request.query_params.get("cursor")
|
||||
after = None
|
||||
if cursor_raw is not None:
|
||||
if not cursor_raw.strip():
|
||||
return error_response(400, "invalid cursor", rid)
|
||||
try:
|
||||
created_at, entity_id = decode_cursor(cursor_raw, chat_id)
|
||||
except ValueError:
|
||||
return error_response(400, "invalid cursor", rid)
|
||||
after = (created_at, entity_id)
|
||||
page, next_key, list_err = await store.list_messages(chat_id, user["id"], limit, after)
|
||||
if list_err == "not_found":
|
||||
return error_response(404, "chat not found", rid)
|
||||
assert page is not None
|
||||
next_cursor = None
|
||||
if next_key is not None:
|
||||
next_cursor = encode_cursor(chat_id, next_key[0], next_key[1])
|
||||
return _json({"items": page, "next_cursor": next_cursor}, rid)
|
||||
|
||||
|
||||
def _media_type(value: str) -> str:
|
||||
return value.split(";", 1)[0].strip().lower()
|
||||
|
||||
|
||||
async def _read_limited(request: Request, limit: int) -> bytes:
|
||||
received = bytearray()
|
||||
async for chunk in request.stream():
|
||||
if len(received) + len(chunk) > limit:
|
||||
raise AppError(413, "file exceeds upload limit")
|
||||
received.extend(chunk)
|
||||
return bytes(received)
|
||||
|
||||
|
||||
async def upload_attachment(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
chat_id = request.path_params["chat_id"]
|
||||
if validate_uuid(chat_id):
|
||||
return error_response(400, "invalid chat id", rid)
|
||||
if validate_filename(request.query_params.get("filename")):
|
||||
return error_response(400, "invalid filename", rid)
|
||||
filename = request.query_params.get("filename") or ""
|
||||
declared = _media_type(request.headers.get("content-type", ""))
|
||||
if declared not in ("image/png", "image/jpeg"):
|
||||
return error_response(415, "unsupported media type", rid)
|
||||
content_length = request.headers.get("content-length")
|
||||
if content_length is not None:
|
||||
try:
|
||||
announced = int(content_length)
|
||||
except ValueError:
|
||||
return error_response(400, "invalid content length", rid)
|
||||
if announced > config.MAX_UPLOAD_BYTES:
|
||||
return error_response(413, "file exceeds upload limit", rid)
|
||||
if announced <= 0:
|
||||
return error_response(415, "unsupported or damaged image", rid)
|
||||
if await store.load_chat_for_member(chat_id, user["id"]) is None:
|
||||
return error_response(404, "chat not found", rid)
|
||||
data = await _read_limited(request, config.MAX_UPLOAD_BYTES)
|
||||
if not data:
|
||||
return error_response(415, "unsupported or damaged image", rid)
|
||||
try:
|
||||
content_type = images.inspect_image(data, declared)
|
||||
except images.ImageRejected as exc:
|
||||
return error_response(exc.status, exc.message, rid)
|
||||
created = await attachments.create_attachment(chat_id, user["id"], filename, content_type, data)
|
||||
return _json(created, rid, 202)
|
||||
|
||||
|
||||
async def begin_direct_upload(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
chat_id = request.path_params["chat_id"]
|
||||
if validate_uuid(chat_id):
|
||||
return error_response(400, "invalid chat id", rid)
|
||||
raw = await request.body()
|
||||
body, err = parse_json_body(raw)
|
||||
if err:
|
||||
return error_response(400, err, rid)
|
||||
assert body is not None
|
||||
if require_exact_keys(body, {"filename", "content_type", "size_bytes"}, {"filename", "content_type", "size_bytes"}):
|
||||
return error_response(400, "invalid upload payload", rid)
|
||||
if validate_filename(body["filename"]):
|
||||
return error_response(400, "invalid filename", rid)
|
||||
content_type = body["content_type"]
|
||||
if content_type not in ("image/png", "image/jpeg"):
|
||||
return error_response(415, "unsupported media type", rid)
|
||||
size = body["size_bytes"]
|
||||
if isinstance(size, bool) or not isinstance(size, int):
|
||||
return error_response(400, "invalid size", rid)
|
||||
if size > config.MAX_UPLOAD_BYTES:
|
||||
return error_response(413, "file exceeds upload limit", rid)
|
||||
if size < 1:
|
||||
return error_response(400, "invalid size", rid)
|
||||
if await store.load_chat_for_member(chat_id, user["id"]) is None:
|
||||
return error_response(404, "chat not found", rid)
|
||||
started = await direct.begin_upload(chat_id, user["id"], body["filename"], content_type, size)
|
||||
return _json(started, rid, 201)
|
||||
|
||||
|
||||
async def complete_direct_upload(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
upload_id = request.path_params["upload_id"]
|
||||
if validate_uuid(upload_id):
|
||||
return error_response(400, "invalid upload id", rid)
|
||||
raw = await request.body()
|
||||
body, err = parse_json_body(raw)
|
||||
if err:
|
||||
return error_response(400, err, rid)
|
||||
assert body is not None
|
||||
if require_exact_keys(body, {"sha256"}, {"sha256"}):
|
||||
return error_response(400, "invalid upload payload", rid)
|
||||
digest = body["sha256"]
|
||||
if not isinstance(digest, str) or re.fullmatch(r"[a-f0-9]{64}", digest) is None:
|
||||
return error_response(400, "invalid checksum", rid)
|
||||
created = await direct.complete_upload(upload_id, user["id"], digest)
|
||||
return _json(created, rid, 202)
|
||||
|
||||
|
||||
async def get_attachment(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
attachment_id = request.path_params["attachment_id"]
|
||||
if validate_uuid(attachment_id):
|
||||
return error_response(400, "invalid attachment id", rid)
|
||||
row = await attachments.get_for_member(attachment_id, user["id"])
|
||||
if row is None:
|
||||
return error_response(404, "attachment not found", rid)
|
||||
return _json(await attachments.present(row), rid)
|
||||
|
||||
|
||||
async def download_attachment(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
attachment_id = request.path_params["attachment_id"]
|
||||
if validate_uuid(attachment_id):
|
||||
return error_response(400, "invalid attachment id", rid)
|
||||
variant = request.query_params.get("variant") or "original"
|
||||
if variant not in ("original", "thumbnail", "crop", "optimized"):
|
||||
return error_response(400, "invalid variant", rid)
|
||||
row = await attachments.get_for_member(attachment_id, user["id"])
|
||||
if row is None:
|
||||
return error_response(404, "attachment not found", rid)
|
||||
if row["status"] != "ready":
|
||||
return error_response(409, "attachment is not ready", rid)
|
||||
if variant == "original":
|
||||
payload = await s3.get_bytes(row["object_key"])
|
||||
media_type = row["content_type"]
|
||||
else:
|
||||
async with db.require().acquire() as conn:
|
||||
found = await conn.fetchrow(
|
||||
"""
|
||||
SELECT content_type, object_key
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = $1 AND pipeline_version = $2 AND name = $3
|
||||
""",
|
||||
row["id"],
|
||||
int(row["applied_pipeline_version"]),
|
||||
variant,
|
||||
)
|
||||
if found is None:
|
||||
return error_response(404, "attachment not found", rid)
|
||||
payload = await s3.get_bytes(found["object_key"])
|
||||
media_type = found["content_type"]
|
||||
return Response(payload, status_code=200, media_type=media_type, headers={"X-Request-Id": rid})
|
||||
|
||||
|
||||
async def lab_fault(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
if not config.lab_mode():
|
||||
return error_response(404, "not found", rid)
|
||||
raw = await request.body()
|
||||
body, err = parse_json_body(raw)
|
||||
if err:
|
||||
return error_response(400, err, rid)
|
||||
assert body is not None
|
||||
mode = body.get("mode", "")
|
||||
remaining = body.get("remaining", 1)
|
||||
if isinstance(remaining, bool) or not isinstance(remaining, int):
|
||||
return error_response(400, "invalid fault", rid)
|
||||
try:
|
||||
armed = await jobs.arm_fault(mode, remaining)
|
||||
except AppError as exc:
|
||||
return error_response(exc.status, exc.message, rid)
|
||||
return _json(armed, rid)
|
||||
|
||||
|
||||
async def lab_jobs(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
if not config.lab_mode():
|
||||
return error_response(404, "not found", rid)
|
||||
attachment_id = request.path_params["attachment_id"]
|
||||
if validate_uuid(attachment_id):
|
||||
return error_response(400, "invalid attachment id", rid)
|
||||
listed = await jobs.list_jobs(UUID(attachment_id))
|
||||
if listed["applied_pipeline_version"] is None:
|
||||
return error_response(404, "attachment not found", rid)
|
||||
return _json(listed, rid)
|
||||
|
||||
|
||||
async def lab_bump(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
if not config.lab_mode():
|
||||
return error_response(404, "not found", rid)
|
||||
attachment_id = request.path_params["attachment_id"]
|
||||
if validate_uuid(attachment_id):
|
||||
return error_response(400, "invalid attachment id", rid)
|
||||
result = await jobs.bump_pipeline(UUID(attachment_id))
|
||||
if result["result"] == "missing":
|
||||
return error_response(404, "attachment not found", rid)
|
||||
return _json(result, rid, 202)
|
||||
|
||||
|
||||
async def lab_redeliver(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
if not config.lab_mode():
|
||||
return error_response(404, "not found", rid)
|
||||
job_id = request.path_params["job_id"]
|
||||
if validate_uuid(job_id):
|
||||
return error_response(400, "invalid job id", rid)
|
||||
result = await jobs.redeliver(UUID(job_id))
|
||||
if result["result"] == "missing":
|
||||
return error_response(404, "attachment not found", rid)
|
||||
return _json(result, rid, 202)
|
||||
|
||||
|
||||
async def lab_replay(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
if not config.lab_mode():
|
||||
return error_response(404, "not found", rid)
|
||||
attachment_id = request.path_params["attachment_id"]
|
||||
if validate_uuid(attachment_id):
|
||||
return error_response(400, "invalid attachment id", rid)
|
||||
result = await jobs.replay_attachment(attachment_id)
|
||||
if result["result"] == "missing":
|
||||
return error_response(404, "attachment not found", rid)
|
||||
return _json(result, rid)
|
||||
|
||||
|
||||
async def lab_sweep(request: Request) -> Response:
|
||||
if not config.lab_mode():
|
||||
return error_response(404, "not found", request.state.request_id)
|
||||
deleted = await attachments.sweep_orphans()
|
||||
return _json({"deleted": deleted}, request.state.request_id)
|
||||
|
||||
|
||||
async def lab_metrics(request: Request) -> Response:
|
||||
if not config.lab_mode():
|
||||
return error_response(404, "not found", request.state.request_id)
|
||||
return _json(
|
||||
{
|
||||
"singleflight": await cache.singleflight_enabled(),
|
||||
"invalidate": await cache.invalidate_enabled(),
|
||||
"history_sql": await cache.history_sql_count(),
|
||||
},
|
||||
request.state.request_id,
|
||||
)
|
||||
|
||||
|
||||
async def lab_flags(request: Request) -> Response:
|
||||
if not config.lab_mode():
|
||||
return error_response(404, "not found", request.state.request_id)
|
||||
raw = await request.body()
|
||||
body, err = parse_json_body(raw)
|
||||
if err:
|
||||
return error_response(400, err, request.state.request_id)
|
||||
assert body is not None
|
||||
if body.get("reset_sql"):
|
||||
await cache.reset_history_sql_count()
|
||||
flags = await cache.set_flags(
|
||||
singleflight=body.get("singleflight"),
|
||||
invalidate=body.get("invalidate"),
|
||||
delay_ms=body.get("delay_ms"),
|
||||
)
|
||||
return _json(flags, request.state.request_id)
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Private bucket access. The API never redirects a client to this endpoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from app import config
|
||||
from app.errors import AppError
|
||||
|
||||
logger = logging.getLogger("messenger.s3")
|
||||
_client = None
|
||||
_signer = None
|
||||
|
||||
|
||||
def _client_or_create():
|
||||
global _client
|
||||
if _client is None:
|
||||
if not config.S3_ACCESS_KEY_ID or not config.S3_SECRET_ACCESS_KEY:
|
||||
raise AppError(503, "service unavailable")
|
||||
_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=config.S3_ENDPOINT,
|
||||
aws_access_key_id=config.S3_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=config.S3_SECRET_ACCESS_KEY,
|
||||
region_name=config.S3_REGION,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=2,
|
||||
read_timeout=20,
|
||||
retries={"max_attempts": 2},
|
||||
),
|
||||
)
|
||||
return _client
|
||||
|
||||
|
||||
def _signer_or_create():
|
||||
"""Signs URLs for the host the browser/client can reach. It does not send traffic."""
|
||||
global _signer
|
||||
if _signer is None:
|
||||
if not config.S3_ACCESS_KEY_ID or not config.S3_SECRET_ACCESS_KEY:
|
||||
raise AppError(503, "service unavailable")
|
||||
_signer = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=config.S3_PUBLIC_ENDPOINT,
|
||||
aws_access_key_id=config.S3_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=config.S3_SECRET_ACCESS_KEY,
|
||||
region_name=config.S3_REGION,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
),
|
||||
)
|
||||
return _signer
|
||||
|
||||
|
||||
def _unavailable(exc: Exception) -> AppError:
|
||||
code = ""
|
||||
if isinstance(exc, ClientError):
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
logger.warning("object storage call failed code=%s", code or type(exc).__name__)
|
||||
return AppError(503, "service unavailable")
|
||||
|
||||
|
||||
async def head_bucket() -> None:
|
||||
def _head() -> None:
|
||||
_client_or_create().head_bucket(Bucket=config.S3_BUCKET)
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(_head)
|
||||
except AppError:
|
||||
raise
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise _unavailable(exc) from None
|
||||
|
||||
|
||||
async def put_bytes(key: str, data: bytes, content_type: str) -> None:
|
||||
def _put() -> None:
|
||||
_client_or_create().put_object(
|
||||
Bucket=config.S3_BUCKET,
|
||||
Key=key,
|
||||
Body=data,
|
||||
ContentType=content_type,
|
||||
ContentLength=len(data),
|
||||
)
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(_put)
|
||||
except AppError:
|
||||
raise
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise _unavailable(exc) from None
|
||||
|
||||
|
||||
async def get_bytes(key: str) -> bytes:
|
||||
def _get() -> bytes:
|
||||
response = _client_or_create().get_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
body = response["Body"].read()
|
||||
response["Body"].close()
|
||||
return body
|
||||
|
||||
try:
|
||||
return await asyncio.to_thread(_get)
|
||||
except AppError:
|
||||
raise
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise _unavailable(exc) from None
|
||||
|
||||
|
||||
def presign_put(key: str, content_type: str, size: int) -> str:
|
||||
return _signer_or_create().generate_presigned_url(
|
||||
"put_object",
|
||||
Params={
|
||||
"Bucket": config.S3_BUCKET,
|
||||
"Key": key,
|
||||
"ContentType": content_type,
|
||||
"ContentLength": size,
|
||||
},
|
||||
ExpiresIn=config.PRESIGN_TTL_SECONDS,
|
||||
HttpMethod="PUT",
|
||||
)
|
||||
|
||||
|
||||
async def head_size(key: str) -> int | None:
|
||||
def _head() -> int | None:
|
||||
try:
|
||||
response = _client_or_create().head_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
if code in {"404", "NoSuchKey", "NotFound"}:
|
||||
return None
|
||||
raise
|
||||
return int(response["ContentLength"])
|
||||
|
||||
try:
|
||||
return await asyncio.to_thread(_head)
|
||||
except AppError:
|
||||
raise
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise _unavailable(exc) from None
|
||||
|
||||
|
||||
async def delete_key(key: str) -> None:
|
||||
def _delete() -> None:
|
||||
try:
|
||||
_client_or_create().delete_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
if code in {"NoSuchKey", "404", "NoSuchBucket"}:
|
||||
return
|
||||
raise
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(_delete)
|
||||
except AppError:
|
||||
raise
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise _unavailable(exc) from None
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Create the private bucket if it is missing. Safe to run again."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
|
||||
def main() -> None:
|
||||
endpoint = os.environ["S3_ENDPOINT"]
|
||||
bucket = os.environ["S3_BUCKET"]
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],
|
||||
aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
|
||||
region_name=os.environ.get("S3_REGION", "us-east-1"),
|
||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, retries={"max_attempts": 2}),
|
||||
)
|
||||
last_error = "not attempted"
|
||||
for _ in range(20):
|
||||
try:
|
||||
client.head_bucket(Bucket=bucket)
|
||||
print("bucket ready")
|
||||
return
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
if code not in {"404", "NoSuchBucket", "NotFound"}:
|
||||
last_error = code or "ClientError"
|
||||
else:
|
||||
try:
|
||||
client.create_bucket(Bucket=bucket)
|
||||
print("bucket created")
|
||||
return
|
||||
except ClientError as create_exc:
|
||||
last_error = create_exc.response.get("Error", {}).get("Code", "create failed")
|
||||
except BotoCoreError:
|
||||
last_error = "connection failed"
|
||||
time.sleep(1)
|
||||
print("bucket bootstrap failed:", last_error, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID
|
||||
|
||||
from app import config, rediscli
|
||||
from app.errors import AppError, BEARER_CHALLENGE
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def _key(token: str) -> str:
|
||||
return "sess:" + hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _fmt(dt: datetime) -> str:
|
||||
return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
async def create(user_id: UUID) -> tuple[str, str]:
|
||||
token = new_token()
|
||||
expires = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
|
||||
payload = json.dumps({"user_id": str(user_id), "expires_at": _fmt(expires)})
|
||||
try:
|
||||
await rediscli.require().set(_key(token), payload, ex=config.SESSION_TTL_SECONDS)
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
return token, _fmt(expires)
|
||||
|
||||
|
||||
async def resolve(token: str) -> UUID:
|
||||
try:
|
||||
raw = await rediscli.require().get(_key(token))
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
if not raw:
|
||||
raise AppError(401, "authentication required", BEARER_CHALLENGE)
|
||||
return UUID(json.loads(raw)["user_id"])
|
||||
|
||||
|
||||
async def revoke(token: str) -> None:
|
||||
try:
|
||||
deleted = await rediscli.require().delete(_key(token))
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
if not deleted:
|
||||
raise AppError(401, "authentication required", BEARER_CHALLENGE)
|
||||
+408
@@ -0,0 +1,408 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import List, Optional, Tuple
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app import cache, config, db
|
||||
from app.errors import AppError
|
||||
|
||||
|
||||
def format_ts(dt: datetime) -> str:
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def user_dict(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": row["display_name"],
|
||||
"created_at": format_ts(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def chat_dict(row, member_ids: List[str]) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": row["title"],
|
||||
"member_ids": member_ids,
|
||||
"created_at": format_ts(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def message_dict(row) -> dict:
|
||||
raw_ids = row["attachment_ids"] if "attachment_ids" in row.keys() else []
|
||||
if raw_ids is None:
|
||||
raw_ids = []
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": row["text"],
|
||||
"created_at": format_ts(row["created_at"]),
|
||||
"attachment_ids": [str(item) for item in raw_ids],
|
||||
}
|
||||
|
||||
|
||||
async def _run(coro):
|
||||
try:
|
||||
return await coro
|
||||
except AppError:
|
||||
raise
|
||||
except db.INTEGRITY:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
|
||||
|
||||
async def create_user(username: str, display_name: str, password_hash: str) -> Tuple[Optional[dict], Optional[str]]:
|
||||
user_id = uuid4()
|
||||
created = datetime.now(timezone.utc)
|
||||
try:
|
||||
async with db.require().acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
""",
|
||||
user_id,
|
||||
username,
|
||||
display_name,
|
||||
password_hash,
|
||||
created,
|
||||
)
|
||||
except asyncpg.UniqueViolationError:
|
||||
return None, "conflict"
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
return user_dict({"id": user_id, "username": username, "display_name": display_name, "created_at": created}), None
|
||||
|
||||
|
||||
async def get_user(user_id: str) -> Optional[dict]:
|
||||
row = await _run(
|
||||
_fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", UUID(user_id))
|
||||
)
|
||||
return user_dict(row) if row else None
|
||||
|
||||
|
||||
async def get_user_by_username(username: str):
|
||||
return await _run(
|
||||
_fetchrow(
|
||||
"SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1",
|
||||
username,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _fetchrow(sql: str, *args):
|
||||
async with db.require().acquire() as conn:
|
||||
return await conn.fetchrow(sql, *args)
|
||||
|
||||
|
||||
async def _fetch(sql: str, *args):
|
||||
async with db.require().acquire() as conn:
|
||||
return await conn.fetch(sql, *args)
|
||||
|
||||
|
||||
async def member_ids_of(chat_id: UUID) -> List[str]:
|
||||
rows = await _run(_fetch("SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id", chat_id))
|
||||
return [str(r["user_id"]) for r in rows]
|
||||
|
||||
|
||||
async def create_chat(title: str, member_ids: List[str], caller_id: str) -> Tuple[Optional[dict], Optional[str]]:
|
||||
if caller_id not in member_ids:
|
||||
return None, "caller not in members"
|
||||
if len(set(member_ids)) != len(member_ids):
|
||||
return None, "duplicate members"
|
||||
ids = [UUID(x) for x in member_ids]
|
||||
chat_id = uuid4()
|
||||
created = datetime.now(timezone.utc)
|
||||
try:
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
existing = await conn.fetch("SELECT id FROM users WHERE id = ANY($1::uuid[])", ids)
|
||||
if len(existing) != len(ids):
|
||||
return None, "unknown member"
|
||||
await conn.execute(
|
||||
"INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)",
|
||||
chat_id,
|
||||
title,
|
||||
created,
|
||||
)
|
||||
await conn.executemany(
|
||||
"INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)",
|
||||
[(chat_id, mid) for mid in ids],
|
||||
)
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
return chat_dict({"id": chat_id, "title": title, "created_at": created}, member_ids), None
|
||||
|
||||
|
||||
async def get_chat(chat_id: str, user_id: str) -> Optional[dict]:
|
||||
row = await load_chat_for_member(chat_id, user_id)
|
||||
if row is None:
|
||||
return None
|
||||
return chat_dict(row, await member_ids_of(UUID(chat_id)))
|
||||
|
||||
|
||||
async def load_chat_for_member(chat_id: str, user_id: str):
|
||||
return await _run(
|
||||
_fetchrow(
|
||||
"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2
|
||||
WHERE c.id = $1
|
||||
""",
|
||||
UUID(chat_id),
|
||||
UUID(user_id),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def list_chats(user_id: str, limit: int, after: Optional[Tuple[str, str]] = None):
|
||||
args: list = [UUID(user_id)]
|
||||
clause = ""
|
||||
if after is not None:
|
||||
clause = "AND (c.created_at, c.id) > ($2, $3)"
|
||||
args.extend([datetime.fromisoformat(after[0].replace("Z", "+00:00")), UUID(after[1])])
|
||||
rows = await _run(
|
||||
_fetch(
|
||||
f"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = $1
|
||||
WHERE 1=1 {clause}
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT {int(limit) + 1}
|
||||
""",
|
||||
*args,
|
||||
)
|
||||
)
|
||||
page = rows[:limit]
|
||||
items = [chat_dict(row, await member_ids_of(row["id"])) for row in page]
|
||||
next_key = None
|
||||
if len(rows) > limit:
|
||||
last = page[-1]
|
||||
next_key = (format_ts(last["created_at"]), str(last["id"]))
|
||||
return items, next_key
|
||||
|
||||
|
||||
def _canonical_message(text: str, attachment_ids: List[str]) -> str:
|
||||
# Missing attachment_ids and [] are the same message. Key order does not matter.
|
||||
return json.dumps(
|
||||
{"attachment_ids": attachment_ids, "text": text},
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
|
||||
|
||||
async def add_message(
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
idem_key: str,
|
||||
path: str,
|
||||
attachment_ids: Optional[List[str]] = None,
|
||||
) -> Tuple[Optional[dict], Optional[str]]:
|
||||
"""Insert one message or replay the stored 201. The row and the result commit together.
|
||||
|
||||
Same user+method+path+key is serialized with a Postgres advisory lock so two
|
||||
replicas cannot insert two messages. A failed transaction stores nothing.
|
||||
"""
|
||||
attachment_ids = list(attachment_ids or [])
|
||||
digest = hashlib.sha256(_canonical_message(text, attachment_ids).encode("utf-8")).hexdigest()
|
||||
lock_key = f"{sender_id}|POST|{path}|{idem_key}"
|
||||
payload: Optional[dict] = None
|
||||
created_new = False
|
||||
try:
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
await conn.execute("SELECT pg_advisory_xact_lock(hashtextextended($1::text, 0))", lock_key)
|
||||
member = await conn.fetchrow(
|
||||
"""
|
||||
SELECT c.id
|
||||
FROM chats c
|
||||
JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2
|
||||
WHERE c.id = $1
|
||||
""",
|
||||
UUID(chat_id),
|
||||
UUID(sender_id),
|
||||
)
|
||||
if member is None:
|
||||
return None, "not_found"
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT request_hash, response_json, created_at
|
||||
FROM idempotency_keys
|
||||
WHERE user_id = $1 AND http_method = 'POST' AND path = $2 AND idem_key = $3
|
||||
""",
|
||||
UUID(sender_id),
|
||||
path,
|
||||
idem_key,
|
||||
)
|
||||
if row is not None:
|
||||
created_at = row["created_at"]
|
||||
if created_at.tzinfo is None:
|
||||
created_at = created_at.replace(tzinfo=timezone.utc)
|
||||
age = (datetime.now(timezone.utc) - created_at).total_seconds()
|
||||
if age <= config.IDEMPOTENCY_TTL_SECONDS:
|
||||
if row["request_hash"] != digest:
|
||||
return None, "conflict"
|
||||
return json.loads(row["response_json"]), None
|
||||
await conn.execute(
|
||||
"""
|
||||
DELETE FROM idempotency_keys
|
||||
WHERE user_id = $1 AND http_method = 'POST' AND path = $2 AND idem_key = $3
|
||||
""",
|
||||
UUID(sender_id),
|
||||
path,
|
||||
idem_key,
|
||||
)
|
||||
resolved: List[str] = []
|
||||
if attachment_ids:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT id, chat_id, status
|
||||
FROM attachments
|
||||
WHERE id = ANY($1::uuid[])
|
||||
""",
|
||||
[UUID(item) for item in attachment_ids],
|
||||
)
|
||||
found = {str(row["id"]): row for row in rows}
|
||||
if len(found) != len(attachment_ids) or any(
|
||||
str(found[item]["chat_id"]) != chat_id for item in attachment_ids
|
||||
):
|
||||
return None, "not_found"
|
||||
if any(found[item]["status"] == "failed" for item in attachment_ids):
|
||||
return None, "not_ready"
|
||||
resolved = attachment_ids
|
||||
message_id = uuid4()
|
||||
created = datetime.now(timezone.utc)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
""",
|
||||
message_id,
|
||||
UUID(chat_id),
|
||||
UUID(sender_id),
|
||||
text,
|
||||
created,
|
||||
)
|
||||
if resolved:
|
||||
await conn.executemany(
|
||||
"""
|
||||
INSERT INTO message_attachments (message_id, attachment_id, position)
|
||||
VALUES ($1, $2, $3)
|
||||
""",
|
||||
[(message_id, UUID(item), index) for index, item in enumerate(resolved)],
|
||||
)
|
||||
payload = message_dict(
|
||||
{
|
||||
"id": message_id,
|
||||
"chat_id": UUID(chat_id),
|
||||
"sender_id": UUID(sender_id),
|
||||
"text": text,
|
||||
"created_at": created,
|
||||
"attachment_ids": resolved,
|
||||
}
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO idempotency_keys
|
||||
(user_id, http_method, path, idem_key, request_hash, response_json, created_at)
|
||||
VALUES ($1, 'POST', $2, $3, $4, $5, $6)
|
||||
""",
|
||||
UUID(sender_id),
|
||||
path,
|
||||
idem_key,
|
||||
digest,
|
||||
json.dumps(payload, ensure_ascii=False, separators=(",", ":")),
|
||||
created,
|
||||
)
|
||||
created_new = True
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
if created_new:
|
||||
await cache.invalidate_history(chat_id)
|
||||
return payload, None
|
||||
|
||||
|
||||
async def list_messages(chat_id: str, user_id: str, limit: int, after: Optional[Tuple[str, str]] = None):
|
||||
if await load_chat_for_member(chat_id, user_id) is None:
|
||||
return None, None, "not_found"
|
||||
cursor = ""
|
||||
after_ts = after_id = None
|
||||
if after is not None:
|
||||
cursor = after[0] + ":" + after[1]
|
||||
after_ts = datetime.fromisoformat(after[0].replace("Z", "+00:00"))
|
||||
after_id = UUID(after[1])
|
||||
|
||||
async def loader() -> dict:
|
||||
await cache.incr_history_sql()
|
||||
if after_ts is None:
|
||||
rows = await _run(
|
||||
_fetch(
|
||||
"""
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
COALESCE(
|
||||
array_agg(ma.attachment_id ORDER BY ma.position)
|
||||
FILTER (WHERE ma.attachment_id IS NOT NULL),
|
||||
ARRAY[]::uuid[]
|
||||
) AS attachment_ids
|
||||
FROM messages m
|
||||
LEFT JOIN message_attachments ma ON ma.message_id = m.id
|
||||
WHERE m.chat_id = $1
|
||||
GROUP BY m.id
|
||||
ORDER BY m.created_at, m.id
|
||||
LIMIT $2
|
||||
""",
|
||||
UUID(chat_id),
|
||||
limit + 1,
|
||||
)
|
||||
)
|
||||
else:
|
||||
rows = await _run(
|
||||
_fetch(
|
||||
"""
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
COALESCE(
|
||||
array_agg(ma.attachment_id ORDER BY ma.position)
|
||||
FILTER (WHERE ma.attachment_id IS NOT NULL),
|
||||
ARRAY[]::uuid[]
|
||||
) AS attachment_ids
|
||||
FROM messages m
|
||||
LEFT JOIN message_attachments ma ON ma.message_id = m.id
|
||||
WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3)
|
||||
GROUP BY m.id
|
||||
ORDER BY m.created_at, m.id
|
||||
LIMIT $4
|
||||
""",
|
||||
UUID(chat_id),
|
||||
after_ts,
|
||||
after_id,
|
||||
limit + 1,
|
||||
)
|
||||
)
|
||||
page = rows[:limit]
|
||||
items = [message_dict(r) for r in page]
|
||||
next_key = None
|
||||
if len(rows) > limit:
|
||||
last = page[-1]
|
||||
next_key = (format_ts(last["created_at"]), str(last["id"]))
|
||||
return {"items": items, "next_key": next_key}
|
||||
|
||||
page = await cache.get_history_page(chat_id, cursor, limit, loader)
|
||||
return page["items"], page.get("next_key"), None
|
||||
@@ -0,0 +1,430 @@
|
||||
"""OTLP telemetry for the API and the worker.
|
||||
|
||||
Metric labels stay low-cardinality: method, route template, status class,
|
||||
job type, outcome, queue name. User, chat, message and attachment ids are
|
||||
never metric labels. Trace and log records may carry a job id so one upload
|
||||
can be found; passwords, tokens, message text and image bytes are not exported.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
|
||||
import psutil
|
||||
from opentelemetry import metrics, propagate, trace
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
|
||||
from app import config
|
||||
|
||||
logger = logging.getLogger("messenger.telemetry")
|
||||
|
||||
_HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
|
||||
_JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60)
|
||||
_READY_BUCKETS = (0.5, 1, 2, 5, 10, 30, 60, 120, 300)
|
||||
_JOB_TYPE = "attachment.process.v1"
|
||||
_QUEUE = "attachment.jobs"
|
||||
|
||||
_ROUTES = (
|
||||
(re.compile(r"^/api/v1/chats/[^/]+/messages$"), "/api/v1/chats/{chat_id}/messages"),
|
||||
(re.compile(r"^/api/v1/chats/[^/]+/attachments$"), "/api/v1/chats/{chat_id}/attachments"),
|
||||
(re.compile(r"^/api/v1/chats/[^/]+/uploads$"), "/api/v1/chats/{chat_id}/uploads"),
|
||||
(re.compile(r"^/api/v1/chats/[^/]+$"), "/api/v1/chats/{chat_id}"),
|
||||
(re.compile(r"^/api/v1/uploads/[^/]+/complete$"), "/api/v1/uploads/{upload_id}/complete"),
|
||||
(re.compile(r"^/api/v1/attachments/[^/]+/content$"), "/api/v1/attachments/{attachment_id}/content"),
|
||||
(re.compile(r"^/api/v1/attachments/[^/]+$"), "/api/v1/attachments/{attachment_id}"),
|
||||
(re.compile(r"^/api/v1/_lab/attachments/[^/]+/jobs$"), "/api/v1/_lab/attachments/{attachment_id}/jobs"),
|
||||
(re.compile(r"^/api/v1/_lab/attachments/[^/]+/pipelines$"), "/api/v1/_lab/attachments/{attachment_id}/pipelines"),
|
||||
(re.compile(r"^/api/v1/_lab/attachments/[^/]+/replay$"), "/api/v1/_lab/attachments/{attachment_id}/replay"),
|
||||
(re.compile(r"^/api/v1/_lab/jobs/[^/]+/redeliver$"), "/api/v1/_lab/jobs/{job_id}/redeliver"),
|
||||
)
|
||||
_SECRET = re.compile(r"(?i)(authorization|password|token|secret|session)([\"']?\s*[:=]\s*[\"']?)(\S+)")
|
||||
|
||||
_ready = False
|
||||
_tracer = None
|
||||
_http_requests = None
|
||||
_http_duration = None
|
||||
_jobs = None
|
||||
_job_duration = None
|
||||
_time_to_ready = None
|
||||
_log_handler = None
|
||||
_providers = []
|
||||
_process = psutil.Process()
|
||||
_process.cpu_percent(interval=None)
|
||||
_inflight = 0
|
||||
_queue_depth = 0
|
||||
_queue_oldest = 0.0
|
||||
_probe_task: asyncio.Task | None = None
|
||||
|
||||
_KIND = {
|
||||
"internal": SpanKind.INTERNAL,
|
||||
"server": SpanKind.SERVER,
|
||||
"client": SpanKind.CLIENT,
|
||||
"producer": SpanKind.PRODUCER,
|
||||
"consumer": SpanKind.CONSUMER,
|
||||
}
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def __init__(self, service_name: str) -> None:
|
||||
super().__init__()
|
||||
self.service_name = service_name
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
span = trace.get_current_span()
|
||||
ctx = span.get_span_context()
|
||||
payload = {
|
||||
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z", time.gmtime(record.created)),
|
||||
"severity": record.levelname,
|
||||
"service.name": self.service_name,
|
||||
"event": _SECRET.sub(r"\1\2[redacted]", record.getMessage()),
|
||||
"logger": record.name,
|
||||
}
|
||||
if ctx.is_valid:
|
||||
payload["trace_id"] = format(ctx.trace_id, "032x")
|
||||
payload["span_id"] = format(ctx.span_id, "016x")
|
||||
if record.exc_info:
|
||||
payload["exception"] = _SECRET.sub(r"\1\2[redacted]", self.formatException(record.exc_info))
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def route_template(path: str) -> str:
|
||||
for pattern, template in _ROUTES:
|
||||
if pattern.match(path):
|
||||
return template
|
||||
return path
|
||||
|
||||
|
||||
def observe_http(path: str) -> bool:
|
||||
return path not in ("/health/live", "/health/ready", "/docs", "/openapi.json")
|
||||
|
||||
|
||||
def status_class(code: int) -> str:
|
||||
return "%sxx" % (int(code) // 100)
|
||||
|
||||
|
||||
def configure_logging() -> None:
|
||||
setup()
|
||||
root = logging.getLogger()
|
||||
root.handlers.clear()
|
||||
root.setLevel(getattr(logging, config.LOG_LEVEL.upper(), logging.INFO))
|
||||
stream = logging.StreamHandler(sys.stdout)
|
||||
stream.setFormatter(JsonFormatter(config.OTEL_SERVICE_NAME))
|
||||
root.addHandler(stream)
|
||||
if _log_handler is not None:
|
||||
root.addHandler(_log_handler)
|
||||
logging.getLogger("opentelemetry").setLevel(logging.ERROR)
|
||||
|
||||
|
||||
def setup() -> None:
|
||||
global _ready, _tracer, _http_requests, _http_duration, _jobs, _job_duration, _time_to_ready, _log_handler
|
||||
if _ready:
|
||||
return
|
||||
_ready = True
|
||||
if not config.TELEMETRY_ENABLED:
|
||||
logger.info("telemetry disabled")
|
||||
return
|
||||
resource = Resource.create(
|
||||
{
|
||||
"service.name": config.OTEL_SERVICE_NAME,
|
||||
"service.version": config.SERVICE_VERSION,
|
||||
"service.instance.id": config.INSTANCE_ID,
|
||||
"deployment.environment.name": "lab",
|
||||
}
|
||||
)
|
||||
ratio = min(1.0, max(0.0, config.TRACE_SAMPLE_RATIO))
|
||||
tracer_provider = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(ratio)))
|
||||
tracer_provider.add_span_processor(
|
||||
BatchSpanProcessor(
|
||||
OTLPSpanExporter(endpoint=_signal_endpoint("traces"), timeout=5),
|
||||
max_queue_size=2048,
|
||||
schedule_delay_millis=5000,
|
||||
max_export_batch_size=512,
|
||||
export_timeout_millis=5000,
|
||||
)
|
||||
)
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
_tracer = trace.get_tracer("messenger", config.SERVICE_VERSION)
|
||||
reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=_signal_endpoint("metrics"), timeout=5),
|
||||
export_interval_millis=10000,
|
||||
export_timeout_millis=5000,
|
||||
)
|
||||
meter_provider = MeterProvider(
|
||||
resource=resource,
|
||||
metric_readers=[reader],
|
||||
views=[
|
||||
View(
|
||||
instrument_name="messenger.http.request.duration",
|
||||
aggregation=ExplicitBucketHistogramAggregation(_HTTP_BUCKETS),
|
||||
),
|
||||
View(
|
||||
instrument_name="messenger.job.duration",
|
||||
aggregation=ExplicitBucketHistogramAggregation(_JOB_BUCKETS),
|
||||
),
|
||||
View(
|
||||
instrument_name="messenger.time_to_ready",
|
||||
aggregation=ExplicitBucketHistogramAggregation(_READY_BUCKETS),
|
||||
),
|
||||
],
|
||||
)
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
meter = metrics.get_meter("messenger", config.SERVICE_VERSION)
|
||||
_http_requests = meter.create_counter("messenger.http.requests", unit="1")
|
||||
_http_duration = meter.create_histogram("messenger.http.request.duration", unit="s")
|
||||
_jobs = meter.create_counter("messenger.jobs", unit="1")
|
||||
_job_duration = meter.create_histogram("messenger.job.duration", unit="s")
|
||||
_time_to_ready = meter.create_histogram("messenger.time_to_ready", unit="s")
|
||||
meter.create_observable_gauge("messenger.db.pool.used", callbacks=[_pool_used])
|
||||
meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[_pool_waiting])
|
||||
meter.create_observable_gauge("process.cpu.percent", callbacks=[_cpu])
|
||||
meter.create_observable_gauge("process.memory.rss", unit="By", callbacks=[_rss])
|
||||
if config.OTEL_SERVICE_NAME == "messenger-api":
|
||||
meter.create_observable_gauge("messenger.queue.depth", callbacks=[_depth])
|
||||
meter.create_observable_gauge("messenger.queue.oldest_age", unit="s", callbacks=[_oldest])
|
||||
if config.OTEL_SERVICE_NAME == "messenger-worker":
|
||||
meter.create_observable_gauge("messenger.worker.busy", callbacks=[_busy])
|
||||
meter.create_observable_gauge("messenger.worker.capacity", callbacks=[_capacity])
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(
|
||||
OTLPLogExporter(endpoint=_signal_endpoint("logs"), timeout=5),
|
||||
max_queue_size=2048,
|
||||
schedule_delay_millis=5000,
|
||||
max_export_batch_size=512,
|
||||
export_timeout_millis=5000,
|
||||
)
|
||||
)
|
||||
set_logger_provider(logger_provider)
|
||||
_log_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||
_providers.extend([tracer_provider, meter_provider, logger_provider])
|
||||
|
||||
|
||||
def shutdown() -> None:
|
||||
for provider in _providers:
|
||||
try:
|
||||
provider.shutdown()
|
||||
except Exception:
|
||||
logger.warning("telemetry shutdown failed")
|
||||
_providers.clear()
|
||||
|
||||
|
||||
def _signal_endpoint(signal: str) -> str:
|
||||
base = config.OTEL_EXPORTER_OTLP_ENDPOINT.rstrip("/")
|
||||
return "%s/v1/%s" % (base, signal)
|
||||
|
||||
|
||||
def current_carrier() -> dict:
|
||||
if _tracer is None:
|
||||
return {}
|
||||
carrier: dict = {}
|
||||
propagate.inject(carrier)
|
||||
return carrier
|
||||
|
||||
|
||||
def remote_context(traceparent: str | None, tracestate: str | None = None):
|
||||
carrier = {}
|
||||
if traceparent:
|
||||
carrier["traceparent"] = traceparent
|
||||
if tracestate:
|
||||
carrier["tracestate"] = tracestate
|
||||
if not carrier:
|
||||
return None
|
||||
return propagate.extract(carrier)
|
||||
|
||||
|
||||
def carrier_from_headers(headers) -> dict:
|
||||
carrier = {}
|
||||
if not headers:
|
||||
return carrier
|
||||
for key in ("traceparent", "tracestate"):
|
||||
value = headers.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("latin1")
|
||||
carrier[key] = str(value)
|
||||
return carrier
|
||||
|
||||
|
||||
@contextmanager
|
||||
def span(name: str, context=None, kind: str = "internal", attributes: dict | None = None):
|
||||
if _tracer is None:
|
||||
yield None
|
||||
return
|
||||
kwargs = {"kind": _KIND.get(kind, SpanKind.INTERNAL)}
|
||||
if context is not None:
|
||||
kwargs["context"] = context
|
||||
with _tracer.start_as_current_span(name, **kwargs) as current:
|
||||
if attributes:
|
||||
for key, value in attributes.items():
|
||||
if value is not None:
|
||||
current.set_attribute(key, value)
|
||||
yield current
|
||||
|
||||
|
||||
def finish_http_span(current, status_code: int) -> None:
|
||||
if current is None:
|
||||
return
|
||||
current.set_attribute("http.response.status_code", int(status_code))
|
||||
if status_code >= 500:
|
||||
current.set_status(Status(StatusCode.ERROR))
|
||||
else:
|
||||
current.set_status(Status(StatusCode.OK))
|
||||
|
||||
|
||||
def record_http(method: str, route: str, status_code: int, duration_seconds: float) -> None:
|
||||
if _http_requests is None:
|
||||
return
|
||||
attributes = {
|
||||
"method": method,
|
||||
"route": route,
|
||||
"status_class": status_class(status_code),
|
||||
}
|
||||
_http_requests.add(1, attributes)
|
||||
_http_duration.record(max(0.0, duration_seconds), attributes)
|
||||
|
||||
|
||||
def record_job(outcome: str, duration_seconds: float) -> None:
|
||||
if _jobs is None:
|
||||
return
|
||||
attributes = {"job_type": _JOB_TYPE, "outcome": outcome}
|
||||
_jobs.add(1, attributes)
|
||||
_job_duration.record(max(0.0, duration_seconds), attributes)
|
||||
|
||||
|
||||
def record_time_to_ready(duration_seconds: float) -> None:
|
||||
if _time_to_ready is None:
|
||||
return
|
||||
_time_to_ready.record(max(0.0, duration_seconds), {"job_type": _JOB_TYPE})
|
||||
|
||||
|
||||
def worker_enter() -> None:
|
||||
global _inflight
|
||||
_inflight += 1
|
||||
|
||||
|
||||
def worker_leave() -> None:
|
||||
global _inflight
|
||||
_inflight = max(0, _inflight - 1)
|
||||
|
||||
|
||||
def start_probes() -> None:
|
||||
global _probe_task
|
||||
if _tracer is None or config.OTEL_SERVICE_NAME != "messenger-api":
|
||||
return
|
||||
if _probe_task is None or _probe_task.done():
|
||||
_probe_task = asyncio.create_task(_probe_loop())
|
||||
|
||||
|
||||
async def stop_probes() -> None:
|
||||
global _probe_task
|
||||
task = _probe_task
|
||||
_probe_task = None
|
||||
if task is None:
|
||||
return
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
async def _probe_loop() -> None:
|
||||
from app import db
|
||||
|
||||
global _queue_depth, _queue_oldest
|
||||
while True:
|
||||
try:
|
||||
async with db.require().acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT
|
||||
count(*) FILTER (WHERE status IN ('queued', 'processing')) AS depth,
|
||||
COALESCE(
|
||||
EXTRACT(EPOCH FROM (now() - min(created_at) FILTER (WHERE status = 'queued'))),
|
||||
0
|
||||
) AS oldest
|
||||
FROM attachment_jobs
|
||||
"""
|
||||
)
|
||||
_queue_depth = int(row["depth"] or 0)
|
||||
_queue_oldest = float(row["oldest"] or 0)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
logger.warning("queue probe failed")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
|
||||
def _observation(value, attributes=None):
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
return [Observation(value, attributes or {})]
|
||||
|
||||
|
||||
def _pool_used(_options):
|
||||
from app import db
|
||||
|
||||
pool = db.pool
|
||||
if pool is None:
|
||||
return _observation(0)
|
||||
return _observation(max(0, pool.get_size() - pool.get_idle_size()))
|
||||
|
||||
|
||||
def _pool_waiting(_options):
|
||||
from app import db
|
||||
|
||||
pool = db.pool
|
||||
if pool is None:
|
||||
return _observation(0)
|
||||
waiters = getattr(pool, "_waiters", None)
|
||||
if waiters is not None:
|
||||
return _observation(len(waiters))
|
||||
queue = getattr(pool, "_queue", None)
|
||||
getters = getattr(queue, "_getters", None)
|
||||
if getters is not None:
|
||||
return _observation(len(getters))
|
||||
return _observation(0)
|
||||
|
||||
|
||||
def _depth(_options):
|
||||
return _observation(_queue_depth, {"queue": _QUEUE})
|
||||
|
||||
|
||||
def _oldest(_options):
|
||||
return _observation(_queue_oldest, {"queue": _QUEUE})
|
||||
|
||||
|
||||
def _busy(_options):
|
||||
return _observation(_inflight)
|
||||
|
||||
|
||||
def _capacity(_options):
|
||||
return _observation(config.WORKER_CONCURRENCY)
|
||||
|
||||
|
||||
def _cpu(_options):
|
||||
return _observation(_process.cpu_percent(interval=None))
|
||||
|
||||
|
||||
def _rss(_options):
|
||||
return _observation(_process.memory_info().rss)
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Seed ≥10 000 messages and print EXPLAIN (ANALYZE, BUFFERS) for keyset pagination."""
|
||||
import asyncio
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app.passwords import hash_password
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
conn = await asyncpg.connect(dsn=os.environ["DATABASE_URL"], timeout=10, command_timeout=120)
|
||||
try:
|
||||
alice = uuid4()
|
||||
bob = uuid4()
|
||||
chat = uuid4()
|
||||
pwd = hash_password("Example-only-not-a-real-secret-123")
|
||||
await conn.execute(
|
||||
"INSERT INTO users (id, username, display_name, password_hash) VALUES ($1,$2,$3,$4),($5,$6,$7,$4)",
|
||||
alice,
|
||||
"explain_a_" + alice.hex[:8],
|
||||
"Alice",
|
||||
pwd,
|
||||
bob,
|
||||
"explain_b_" + bob.hex[:8],
|
||||
"Bob",
|
||||
)
|
||||
await conn.execute("INSERT INTO chats (id, title) VALUES ($1,$2)", chat, "Explain chat")
|
||||
await conn.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id) VALUES ($1,$2),($1,$3)",
|
||||
chat,
|
||||
alice,
|
||||
bob,
|
||||
)
|
||||
n = int(os.environ.get("EXPLAIN_MESSAGES", "10000"))
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), $1, $2, 'seed-' || g, now() - (g || ' milliseconds')::interval
|
||||
FROM generate_series(1, $3) AS g
|
||||
""",
|
||||
chat,
|
||||
alice,
|
||||
n,
|
||||
)
|
||||
count = await conn.fetchval("SELECT count(*) FROM messages WHERE chat_id = $1", chat)
|
||||
print("seeded_messages", count, "chat_id", chat)
|
||||
first = await conn.fetchrow(
|
||||
"SELECT created_at, id FROM messages WHERE chat_id = $1 ORDER BY created_at, id LIMIT 1",
|
||||
chat,
|
||||
)
|
||||
query = """
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
|
||||
ORDER BY created_at, id
|
||||
LIMIT 50
|
||||
"""
|
||||
print("\n=== WITH index messages_chat_created_id_idx ===")
|
||||
plan = await conn.fetch("EXPLAIN (ANALYZE, BUFFERS) " + query, chat, first["created_at"], first["id"])
|
||||
print("\n".join(row[0] for row in plan))
|
||||
await conn.execute("SET enable_indexscan = off")
|
||||
await conn.execute("SET enable_bitmapscan = off")
|
||||
await conn.execute("SET enable_indexonlyscan = off")
|
||||
print("\n=== WITHOUT index scans (seq scan baseline) ===")
|
||||
plan2 = await conn.fetch("EXPLAIN (ANALYZE, BUFFERS) " + query, chat, first["created_at"], first["id"])
|
||||
print("\n".join(row[0] for row in plan2))
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,131 @@
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
from uuid import UUID
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
|
||||
|
||||
def parse_json_body(raw: bytes) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
if not raw:
|
||||
return None, "empty body"
|
||||
try:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
return None, "malformed JSON"
|
||||
if not isinstance(data, dict):
|
||||
return None, "JSON body must be an object"
|
||||
return data, None
|
||||
|
||||
|
||||
def require_exact_keys(data: Dict[str, Any], allowed: set, required: set) -> Optional[str]:
|
||||
keys = set(data.keys())
|
||||
if keys - allowed:
|
||||
return "unexpected fields"
|
||||
if required - keys:
|
||||
return "missing required fields"
|
||||
return None
|
||||
|
||||
|
||||
def validate_username(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str) or not USERNAME_RE.fullmatch(value):
|
||||
return "invalid username"
|
||||
return None
|
||||
|
||||
|
||||
def validate_display_name(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return "invalid display_name"
|
||||
if not value.strip():
|
||||
return "display_name must not be blank"
|
||||
if len(value) > 100:
|
||||
return "display_name too long"
|
||||
return None
|
||||
|
||||
|
||||
def validate_password(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return "invalid password"
|
||||
if not value.strip():
|
||||
return "password must not be blank"
|
||||
if len(value) < 12 or len(value) > 128:
|
||||
return "password length"
|
||||
return None
|
||||
|
||||
|
||||
def validate_title(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return "invalid title"
|
||||
if not value.strip():
|
||||
return "title must not be blank"
|
||||
if len(value) > 100:
|
||||
return "title too long"
|
||||
return None
|
||||
|
||||
|
||||
def validate_uuid(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return "invalid uuid"
|
||||
try:
|
||||
UUID(value)
|
||||
except ValueError:
|
||||
return "invalid uuid"
|
||||
return None
|
||||
|
||||
|
||||
def validate_idempotency_key(value: Optional[str]) -> Optional[str]:
|
||||
if value is None or IDEMPOTENCY_RE.fullmatch(value) is None:
|
||||
return "invalid idempotency key"
|
||||
return None
|
||||
|
||||
|
||||
def validate_message_text(value: Any, *, allow_blank: bool = False) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return "invalid text"
|
||||
if len(value) > 4000:
|
||||
return "text too long"
|
||||
if not allow_blank and not value.strip():
|
||||
return "text must not be blank"
|
||||
return None
|
||||
|
||||
|
||||
def validate_filename(value: Optional[str]) -> Optional[str]:
|
||||
if not isinstance(value, str) or not value.strip() or len(value) > 255:
|
||||
return "invalid filename"
|
||||
for char in value:
|
||||
code = ord(char)
|
||||
if code < 32 or code == 127 or char in "/\\":
|
||||
return "invalid filename"
|
||||
return None
|
||||
|
||||
|
||||
def parse_attachment_ids(value: Any) -> Tuple[Optional[list], Optional[str]]:
|
||||
if value is None:
|
||||
return [], None
|
||||
if not isinstance(value, list):
|
||||
return None, "invalid attachment_ids"
|
||||
if len(value) > 10:
|
||||
return None, "too many attachments"
|
||||
seen = set()
|
||||
ids = []
|
||||
for item in value:
|
||||
if validate_uuid(item):
|
||||
return None, "invalid attachment id"
|
||||
if item in seen:
|
||||
return None, "duplicate attachment_ids"
|
||||
seen.add(item)
|
||||
ids.append(item)
|
||||
return ids, None
|
||||
|
||||
|
||||
def validate_limit(value: Optional[str], default: int = 50) -> Tuple[Optional[int], Optional[str]]:
|
||||
if value is None:
|
||||
return default, None
|
||||
try:
|
||||
limit = int(value)
|
||||
except ValueError:
|
||||
return None, "invalid limit"
|
||||
if limit < 1 or limit > 100:
|
||||
return None, "limit out of range"
|
||||
return limit, None
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
"""Separate process: consume durable jobs and publish image variants."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import signal
|
||||
from uuid import UUID
|
||||
|
||||
import aio_pika
|
||||
|
||||
from app import broker, config, db, jobs, telemetry
|
||||
from app.broker import payload_from
|
||||
|
||||
logger = logging.getLogger("messenger.worker")
|
||||
|
||||
|
||||
def configure_logging() -> None:
|
||||
telemetry.configure_logging()
|
||||
|
||||
|
||||
async def _settle(message: aio_pika.IncomingMessage) -> None:
|
||||
carrier = telemetry.carrier_from_headers(message.headers)
|
||||
parent = telemetry.remote_context(carrier.get("traceparent"), carrier.get("tracestate"))
|
||||
with telemetry.span(
|
||||
"attachment.job.consume",
|
||||
context=parent,
|
||||
kind="consumer",
|
||||
attributes={
|
||||
"messaging.system": "rabbitmq",
|
||||
"messaging.destination.name": broker.QUEUE,
|
||||
"messaging.operation": "consume",
|
||||
},
|
||||
) as current:
|
||||
telemetry.worker_enter()
|
||||
try:
|
||||
try:
|
||||
payload = payload_from(message.body)
|
||||
job_id = UUID(payload["job_id"])
|
||||
pipeline_version = int(payload["pipeline_version"])
|
||||
except (ValueError, KeyError, TypeError):
|
||||
logger.warning("dropping a job message that does not match the schema")
|
||||
await message.ack()
|
||||
return
|
||||
if current is not None:
|
||||
current.set_attribute("job.id", str(job_id))
|
||||
try:
|
||||
waits = 0
|
||||
while True:
|
||||
state = await jobs.process(job_id, pipeline_version)
|
||||
if state == "busy":
|
||||
waits += 1
|
||||
if waits > config.JOB_LEASE_SECONDS + 5:
|
||||
await message.reject(requeue=True)
|
||||
return
|
||||
await asyncio.sleep(1)
|
||||
continue
|
||||
logger.info("job settled job_id=%s state=%s", job_id, state)
|
||||
await message.ack()
|
||||
return
|
||||
except Exception:
|
||||
logger.warning("job settle failed job_id=%s", payload.get("job_id"))
|
||||
await message.reject(requeue=True)
|
||||
finally:
|
||||
telemetry.worker_leave()
|
||||
|
||||
|
||||
async def _reap_loop(stop: asyncio.Event) -> None:
|
||||
while not stop.is_set():
|
||||
try:
|
||||
count = await jobs.reap_expired()
|
||||
if count:
|
||||
logger.info("requeued %s expired leases", count)
|
||||
except Exception:
|
||||
logger.warning("lease reaper failed")
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), timeout=2)
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
|
||||
|
||||
async def serve() -> None:
|
||||
await db.connect()
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
try:
|
||||
loop.add_signal_handler(sig, stop.set)
|
||||
except NotImplementedError:
|
||||
signal.signal(sig, lambda *_: stop.set())
|
||||
reaper = asyncio.create_task(_reap_loop(stop))
|
||||
queue, consumer = await broker.consume(_settle, config.WORKER_CONCURRENCY)
|
||||
logger.info("worker consuming prefetch=%s", config.WORKER_CONCURRENCY)
|
||||
await stop.wait()
|
||||
await queue.cancel(consumer)
|
||||
reaper.cancel()
|
||||
try:
|
||||
await reaper
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
await broker.close()
|
||||
await db.close()
|
||||
telemetry.shutdown()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
configure_logging()
|
||||
asyncio.run(serve())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,9 @@
|
||||
# Local-only overlay so this stand can run beside lab4 (8080/9000) and lab5 (8081/9001).
|
||||
# Not part of the submission. Delete it and use compose.yaml when those ports are free.
|
||||
services:
|
||||
lb:
|
||||
ports: !override
|
||||
- "127.0.0.1:8082:8080"
|
||||
s3:
|
||||
ports: !override
|
||||
- "127.0.0.1:9002:9000"
|
||||
+341
@@ -0,0 +1,341 @@
|
||||
# Client -> lb:8080 -> api x2 -> Postgres + Redis + private SeaweedFS S3 + RabbitMQ.
|
||||
# Worker is a separate process and is the only place that resizes images.
|
||||
# API and worker export OTLP to the collector. Prometheus, Loki, Tempo and Grafana
|
||||
# keep metrics, logs and traces. The collector is not required for the API to start.
|
||||
# API replicas do not publish host ports. MinIO console is not published.
|
||||
# API replicas do not publish host ports. Migrations run once in `migrate`.
|
||||
# Check period: HAProxy inter 1s, fall 2, rise 1. Worst detection of a dead
|
||||
# or not-ready replica is about 2s, plus the 1s DNS hold. API on SIGTERM
|
||||
# fails /health/ready immediately and exits after LB_DRAIN_SECONDS (default 2).
|
||||
|
||||
services:
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: mtusi-messenger:lab6
|
||||
command: ["python", "-m", "app.migrate"]
|
||||
environment:
|
||||
APP_ENV: lab
|
||||
LOG_LEVEL: info
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
|
||||
DB_POOL_MIN: "1"
|
||||
DB_POOL_MAX: "2"
|
||||
MIGRATIONS_ON_START: "1"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
mem_limit: 256m
|
||||
cpus: "0.50"
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: mtusi-messenger:lab6
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: lab
|
||||
LOG_LEVEL: info
|
||||
TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1}
|
||||
TRACE_SAMPLE_RATIO: ${TRACE_SAMPLE_RATIO:-1}
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
SERVICE_VERSION: lab6
|
||||
SHUTDOWN_GRACE_SECONDS: "3"
|
||||
LB_DRAIN_SECONDS: ${LB_DRAIN_SECONDS:-2}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-60}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
SESSION_STORE: redis
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-120}
|
||||
BROKER_URL: amqp://messenger:${RABBITMQ_PASSWORD}@rabbitmq:5672/
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
S3_SWEEP_INTERVAL_SECONDS: ${S3_SWEEP_INTERVAL_SECONDS:-30}
|
||||
S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://127.0.0.1:9000}
|
||||
PRESIGN_TTL_SECONDS: ${PRESIGN_TTL_SECONDS:-60}
|
||||
DB_POOL_MIN: "1"
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-4}
|
||||
DB_COMMAND_TIMEOUT: "5"
|
||||
DB_CONNECT_TIMEOUT: "5"
|
||||
MIGRATIONS_ON_START: "0"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
expose:
|
||||
- "8080"
|
||||
restart: unless-stopped
|
||||
mem_limit: ${API_MEM:-256m}
|
||||
cpus: ${API_CPUS:-0.50}
|
||||
stop_grace_period: 8s
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD",
|
||||
"python",
|
||||
"-c",
|
||||
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)",
|
||||
]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 6
|
||||
start_period: 5s
|
||||
|
||||
lb:
|
||||
image: haproxy:2.8.14-alpine
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
mem_limit: 128m
|
||||
cpus: "0.25"
|
||||
|
||||
postgres:
|
||||
image: postgres:16.6-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_DB: messenger
|
||||
volumes:
|
||||
- pg_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
mem_limit: ${POSTGRES_MEM:-512m}
|
||||
cpus: ${POSTGRES_CPUS:-1.0}
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:3.85
|
||||
entrypoint: ["/bin/sh", "/seaweed-entrypoint.sh"]
|
||||
environment:
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
|
||||
ports:
|
||||
- "127.0.0.1:9000:9000"
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
- ./infra/s3/seaweed-entrypoint.sh:/seaweed-entrypoint.sh:ro
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD-SHELL",
|
||||
"wget -q -O /dev/null http://127.0.0.1:9333/cluster/status || exit 1; wget -q -O /dev/null http://127.0.0.1:9000/ || test $$? -ne 4",
|
||||
]
|
||||
interval: 3s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 8s
|
||||
restart: unless-stopped
|
||||
mem_limit: 256m
|
||||
cpus: "0.50"
|
||||
|
||||
s3-init:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: mtusi-messenger:lab6
|
||||
command: ["python", "-m", "app.s3bootstrap"]
|
||||
environment:
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:3.13.7-alpine
|
||||
hostname: rabbitmq
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: messenger
|
||||
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD}
|
||||
RABBITMQ_DEFAULT_VHOST: /
|
||||
volumes:
|
||||
- rabbitmq_data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 15s
|
||||
restart: unless-stopped
|
||||
mem_limit: 256m
|
||||
cpus: "0.50"
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: mtusi-messenger:lab6
|
||||
command: ["python", "-m", "app.worker"]
|
||||
environment:
|
||||
APP_ENV: lab
|
||||
LOG_LEVEL: info
|
||||
TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-1}
|
||||
TRACE_SAMPLE_RATIO: ${TRACE_SAMPLE_RATIO:-1}
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
SERVICE_VERSION: lab6
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
BROKER_URL: amqp://messenger:${RABBITMQ_PASSWORD}@rabbitmq:5672/
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
DB_POOL_MIN: "1"
|
||||
DB_POOL_MAX: "4"
|
||||
MIGRATIONS_ON_START: "0"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
mem_limit: 768m
|
||||
cpus: "1.0"
|
||||
stop_grace_period: 20s
|
||||
healthcheck:
|
||||
disable: true
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
command:
|
||||
- redis-server
|
||||
- --appendonly
|
||||
- "yes"
|
||||
- --appendfsync
|
||||
- everysec
|
||||
- --save
|
||||
- "60"
|
||||
- "1"
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
mem_limit: 128m
|
||||
cpus: "0.50"
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.128.0
|
||||
command: ["--config=/etc/otel/collector.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otel/collector.yaml:ro
|
||||
restart: unless-stopped
|
||||
mem_limit: 320m
|
||||
cpus: "0.50"
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.55.1
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --web.enable-remote-write-receiver
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
volumes:
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
|
||||
- prometheus_data:/prometheus
|
||||
ports:
|
||||
- "127.0.0.1:9090:9090"
|
||||
restart: unless-stopped
|
||||
mem_limit: 512m
|
||||
cpus: "0.50"
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.3.2
|
||||
command: ["-config.file=/etc/loki/loki.yml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yml:/etc/loki/loki.yml:ro
|
||||
- loki_data:/loki
|
||||
restart: unless-stopped
|
||||
mem_limit: 512m
|
||||
cpus: "0.50"
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.7.1
|
||||
command: ["-config.file=/etc/tempo/tempo.yml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yml:/etc/tempo/tempo.yml:ro
|
||||
- tempo_data:/var/tempo
|
||||
restart: unless-stopped
|
||||
mem_limit: 512m
|
||||
cpus: "0.50"
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.4.0
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-lab6-dev-only-change-me}
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
|
||||
GF_USERS_DEFAULT_THEME: light
|
||||
ports:
|
||||
- "127.0.0.1:3000:3000"
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
restart: unless-stopped
|
||||
mem_limit: 256m
|
||||
cpus: "0.50"
|
||||
|
||||
volumes:
|
||||
pg_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
rabbitmq_data:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
grafana_data:
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"openapi": "3.1.0",
|
||||
"info": {
|
||||
"title": "Messenger lab4 optional direct upload",
|
||||
"version": "1.4.0-track1"
|
||||
},
|
||||
"paths": {
|
||||
"/api/v1/chats/{chat_id}/uploads": {
|
||||
"post": {
|
||||
"operationId": "beginDirectUpload",
|
||||
"summary": "Issue a 60s PUT signature for one new object key. Does not replace POST /attachments."
|
||||
}
|
||||
},
|
||||
"/api/v1/uploads/{upload_id}/complete": {
|
||||
"post": {
|
||||
"operationId": "completeDirectUpload",
|
||||
"summary": "Check owner, size and sha256, store the original, and enqueue processing. Returns 202 queued."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,156 @@
|
||||
{
|
||||
"uid": "messenger-lab6",
|
||||
"title": "Messenger observability",
|
||||
"tags": ["messenger", "lab6"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "10s",
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"description": "RPS and errors use messenger_http_requests_total. Durations are histogram_quantile over buckets, in seconds. Queue age is seconds. API error ratio excludes health.",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "timeseries",
|
||||
"title": "API RPS",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
|
||||
"fieldConfig": {"defaults": {"unit": "reqps", "custom": {"drawStyle": "line"}}, "overrides": []},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (method, route) (rate(messenger_http_requests_total[1m]))",
|
||||
"legendFormat": "{{method}} {{route}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "API error ratio (5xx / business)",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
|
||||
"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total[1m])), 1e-9)",
|
||||
"legendFormat": "5xx ratio"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "API duration p50 / p95 / p99",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_http_request_duration_seconds_bucket[1m])))", "legendFormat": "p50"},
|
||||
{"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_request_duration_seconds_bucket[1m])))", "legendFormat": "p95"},
|
||||
{"refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_request_duration_seconds_bucket[1m])))", "legendFormat": "p99"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Jobs by outcome",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
|
||||
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))",
|
||||
"legendFormat": "{{outcome}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Worker duration p50 / p95 / p99",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 16},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))", "legendFormat": "p50"},
|
||||
{"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))", "legendFormat": "p95"},
|
||||
{"refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket[1m])))", "legendFormat": "p99"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Time to ready p50 / p95 / p99",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 16},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_time_to_ready_seconds_bucket[1m])))", "legendFormat": "p50"},
|
||||
{"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_time_to_ready_seconds_bucket[1m])))", "legendFormat": "p95"},
|
||||
{"refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_time_to_ready_seconds_bucket[1m])))", "legendFormat": "p99"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Queue depth and oldest job age",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 24},
|
||||
"fieldConfig": {"defaults": {}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "max(messenger_queue_depth)", "legendFormat": "depth (jobs)"},
|
||||
{"refId": "B", "expr": "max(messenger_queue_oldest_age_seconds)", "legendFormat": "oldest age (seconds)"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "DB pool used and waiting",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 24},
|
||||
"fieldConfig": {"defaults": {"unit": "none"}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "max by (service_name, service_instance_id) (messenger_db_pool_used)", "legendFormat": "used {{service_name}} {{service_instance_id}}"},
|
||||
{"refId": "B", "expr": "max by (service_name, service_instance_id) (messenger_db_pool_waiting)", "legendFormat": "waiting {{service_name}} {{service_instance_id}}"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "timeseries",
|
||||
"title": "Worker busy / capacity",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 32},
|
||||
"fieldConfig": {"defaults": {"unit": "none"}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "max(messenger_worker_busy)", "legendFormat": "busy"},
|
||||
{"refId": "B", "expr": "max(messenger_worker_capacity)", "legendFormat": "capacity"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "Process CPU and RSS",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 32},
|
||||
"fieldConfig": {"defaults": {}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "max by (service_name, service_instance_id) (process_cpu_percent)", "legendFormat": "cpu % {{service_name}} {{service_instance_id}}"},
|
||||
{"refId": "B", "expr": "max by (service_name, service_instance_id) (process_memory_rss_bytes)", "legendFormat": "rss {{service_name}} {{service_instance_id}}"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "Firing alerts",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 40},
|
||||
"fieldConfig": {"defaults": {"unit": "none"}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "ALERTS{alertstate=\"firing\"}", "legendFormat": "{{alertname}}"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: messenger
|
||||
orgId: 1
|
||||
folder: Messenger
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,38 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
timeInterval: 15s
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- datasourceUid: tempo
|
||||
matcherRegex: '"trace_id":"([0-9a-f]+)"'
|
||||
name: trace_id
|
||||
url: "$${__value.raw}"
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
filterByTraceID: true
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
@@ -0,0 +1,9 @@
|
||||
# High API error rate
|
||||
|
||||
Signal: `messenger_http_requests_total`. Numerator is `status_class="5xx"`. Denominator is every instrumented business request. `/health/*`, `/docs` and `/openapi.json` are not in this ratio, so a bad password (401) is not an incident.
|
||||
|
||||
1. Open the Messenger dashboard and compare the 5xx ratio with p95 duration.
|
||||
2. In Tempo, open a server span with `http.response.status_code >= 500`. The span attribute `http.route` is the template, not a user id.
|
||||
3. In Loki, query `{service_name="messenger-api"}` and match `trace_id` from that span. The log line has `event`, `severity`, `trace_id` and `span_id`.
|
||||
4. If the span stops inside the database client or the pool-used gauge is at `DB_POOL_MAX` with waiters, the cause is Postgres, not the HTTP handler itself.
|
||||
5. After the dependency recovers, the 5m ratio falls and `HighApiErrorRate` leaves `firing`.
|
||||
@@ -0,0 +1,8 @@
|
||||
# Queue lag
|
||||
|
||||
Signal: `messenger_queue_oldest_age_seconds` is `now() - min(created_at)` for rows still `queued`. `messenger_queue_depth` counts `queued` and `processing`. Age is not the same number as worker duration.
|
||||
|
||||
1. If age grows and `messenger_worker_busy` is 0, the worker process is stopped or not consuming. Messages stay in RabbitMQ and in `attachment_jobs`.
|
||||
2. If busy equals capacity and age still grows, the worker is saturated. Look at `messenger_job_duration_seconds` and the consumer span.
|
||||
3. A consumer span's parent is the producer span, whose parent is the API server span that accepted the upload (`traceparent` stored on the job row).
|
||||
4. Start the worker again. Depth and age return toward zero, then `QueueLagHigh` resolves. Do not delete the RabbitMQ volume.
|
||||
@@ -0,0 +1,13 @@
|
||||
# SLO burn
|
||||
|
||||
Two SLOs, each with a 1% error budget over 30 days.
|
||||
|
||||
- API availability: business requests that are not 5xx. Fast burn pages when both the 5 minute and 1 hour windows exceed 14.4% errors (14.4x budget). Slow burn warns when both 30 minutes and 6 hours exceed 6% (6x budget).
|
||||
- Time to ready: jobs whose ready time is at most 30 seconds, read from the `le="30"` histogram bucket, not from an average. The same 14.4x and 6x windows apply.
|
||||
|
||||
A short spike can trip only the fast pair. A long degradation keeps the slow pair firing after the spike is gone. Both windows must agree, so one noisy minute does not page by itself.
|
||||
|
||||
1. Read which alert is firing and which SLO label it has.
|
||||
2. For `api-availability`, follow `high-error-rate.md`. Separate 4xx from 5xx before calling it an incident.
|
||||
3. For `time-to-ready`, follow `queue-lag.md`, then open the worker trace and see whether time is in `s3.get`, `image.render`, `s3.put` or `db.complete`.
|
||||
4. Record the first firing timestamp and the first resolved timestamp. That pair is time-to-detect and time-to-recover for the experiment.
|
||||
@@ -0,0 +1,35 @@
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 200
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
timeout connect 2s
|
||||
timeout client 120s
|
||||
timeout server 120s
|
||||
timeout http-request 15s
|
||||
timeout check 2s
|
||||
timeout queue 2s
|
||||
retries 1
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
accepted_payload_size 8192
|
||||
|
||||
frontend fe_http
|
||||
bind *:8080
|
||||
default_backend be_api
|
||||
|
||||
backend be_api
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr host api
|
||||
http-check expect status 200
|
||||
default-server inter 1s fall 2 rise 1
|
||||
server-template srv 8 api:8080 check resolvers docker init-addr none resolve-prefer ipv4
|
||||
@@ -0,0 +1,30 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
@@ -1,7 +1,11 @@
|
||||
# Конфигурация наблюдаемости — реализовать
|
||||
# Collector
|
||||
|
||||
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
|
||||
`collector.yaml` — OTel Collector Contrib 0.128.0. Принимает OTLP/HTTP `:4318` и OTLP/gRPC `:4317` только внутри сети Compose.
|
||||
|
||||
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
|
||||
- traces → Tempo `http://tempo:4318/v1/traces`
|
||||
- metrics → Prometheus remote write `http://prometheus:9090/api/v1/write`
|
||||
- logs → Loki OTLP `http://loki:3100/otlp/v1/logs`
|
||||
|
||||
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
|
||||
`memory_limiter` (192 MiB) и очереди exporter ограничены. Если backend недоступен, collector сбрасывает телеметрию после повторных попыток и не хранит её вечно. API от этого не зависит: SDK тоже пишет в ограниченную очередь и продолжает отвечать на запросы.
|
||||
|
||||
Grafana provisioning и dashboard лежат в `infra/grafana`. Правила Prometheus — в `infra/prometheus/alerts.yml`. SLO — в `infra/slo.md`.
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# Collector 0.128.0. It forwards signals and is not a database.
|
||||
# memory_limiter refuses data before the process grows without bound.
|
||||
# Exporter queues are capped; a dead backend drops telemetry after retry, the API keeps serving.
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 48
|
||||
batch:
|
||||
timeout: 5s
|
||||
send_batch_size: 256
|
||||
send_batch_max_size: 512
|
||||
|
||||
exporters:
|
||||
otlphttp/tempo:
|
||||
traces_endpoint: http://tempo:4318/v1/traces
|
||||
tls:
|
||||
insecure: true
|
||||
timeout: 5s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 1000
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 15s
|
||||
max_elapsed_time: 30s
|
||||
prometheusremotewrite:
|
||||
endpoint: http://prometheus:9090/api/v1/write
|
||||
tls:
|
||||
insecure: true
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
remote_write_queue:
|
||||
enabled: true
|
||||
queue_size: 1000
|
||||
num_consumers: 2
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 15s
|
||||
max_elapsed_time: 30s
|
||||
otlphttp/loki:
|
||||
logs_endpoint: http://loki:3100/otlp/v1/logs
|
||||
tls:
|
||||
insecure: true
|
||||
timeout: 5s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 1000
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 15s
|
||||
max_elapsed_time: 30s
|
||||
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
telemetry:
|
||||
logs:
|
||||
level: info
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheusremotewrite]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
@@ -0,0 +1,112 @@
|
||||
groups:
|
||||
- name: messenger
|
||||
rules:
|
||||
- alert: HighApiErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(messenger_http_requests_total{status_class="5xx"}[5m]))
|
||||
/
|
||||
clamp_min(sum(rate(messenger_http_requests_total[5m])), 1e-9)
|
||||
) > 0.05
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
slo: api-availability
|
||||
annotations:
|
||||
summary: API 5xx ratio is above 5% over 5 minutes
|
||||
description: Numerator is status_class=5xx. Denominator is instrumented business requests. Health, docs and openapi are excluded.
|
||||
runbook: infra/grafana/runbooks/high-error-rate.md
|
||||
- alert: QueueLagHigh
|
||||
expr: max(messenger_queue_oldest_age_seconds) > 30
|
||||
for: 30s
|
||||
labels:
|
||||
severity: warning
|
||||
slo: time-to-ready
|
||||
annotations:
|
||||
summary: Oldest queued job is older than 30 seconds
|
||||
description: Age is now minus created_at of jobs still in status queued. Depth is a separate gauge.
|
||||
runbook: infra/grafana/runbooks/queue-lag.md
|
||||
- alert: ApiAvailabilityFastBurn
|
||||
expr: |
|
||||
(
|
||||
sum(rate(messenger_http_requests_total{status_class="5xx"}[5m]))
|
||||
/ clamp_min(sum(rate(messenger_http_requests_total[5m])), 1e-9)
|
||||
) > 0.144
|
||||
and
|
||||
(
|
||||
sum(rate(messenger_http_requests_total{status_class="5xx"}[1h]))
|
||||
/ clamp_min(sum(rate(messenger_http_requests_total[1h])), 1e-9)
|
||||
) > 0.144
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
slo: api-availability
|
||||
window: fast
|
||||
annotations:
|
||||
summary: API error budget burn rate is above 14.4x on both 5m and 1h
|
||||
runbook: infra/grafana/runbooks/slo-burn.md
|
||||
- alert: ApiAvailabilitySlowBurn
|
||||
expr: |
|
||||
(
|
||||
sum(rate(messenger_http_requests_total{status_class="5xx"}[30m]))
|
||||
/ clamp_min(sum(rate(messenger_http_requests_total[30m])), 1e-9)
|
||||
) > 0.06
|
||||
and
|
||||
(
|
||||
sum(rate(messenger_http_requests_total{status_class="5xx"}[6h]))
|
||||
/ clamp_min(sum(rate(messenger_http_requests_total[6h])), 1e-9)
|
||||
) > 0.06
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
slo: api-availability
|
||||
window: slow
|
||||
annotations:
|
||||
summary: API error budget burn rate is above 6x on both 30m and 6h
|
||||
runbook: infra/grafana/runbooks/slo-burn.md
|
||||
- alert: TimeToReadyFastBurn
|
||||
expr: |
|
||||
(
|
||||
1 - (
|
||||
sum(rate(messenger_time_to_ready_seconds_bucket{le=~"30|30\\.0"}[5m]))
|
||||
/ clamp_min(sum(rate(messenger_time_to_ready_seconds_count[5m])), 1e-9)
|
||||
)
|
||||
) > 0.144
|
||||
and
|
||||
(
|
||||
1 - (
|
||||
sum(rate(messenger_time_to_ready_seconds_bucket{le=~"30|30\\.0"}[1h]))
|
||||
/ clamp_min(sum(rate(messenger_time_to_ready_seconds_count[1h])), 1e-9)
|
||||
)
|
||||
) > 0.144
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
slo: time-to-ready
|
||||
window: fast
|
||||
annotations:
|
||||
summary: Share of jobs slower than 30s is burning the ready-time budget (5m and 1h)
|
||||
runbook: infra/grafana/runbooks/slo-burn.md
|
||||
- alert: TimeToReadySlowBurn
|
||||
expr: |
|
||||
(
|
||||
1 - (
|
||||
sum(rate(messenger_time_to_ready_seconds_bucket{le=~"30|30\\.0"}[30m]))
|
||||
/ clamp_min(sum(rate(messenger_time_to_ready_seconds_count[30m])), 1e-9)
|
||||
)
|
||||
) > 0.06
|
||||
and
|
||||
(
|
||||
1 - (
|
||||
sum(rate(messenger_time_to_ready_seconds_bucket{le=~"30|30\\.0"}[6h]))
|
||||
/ clamp_min(sum(rate(messenger_time_to_ready_seconds_count[6h])), 1e-9)
|
||||
)
|
||||
) > 0.06
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
slo: time-to-ready
|
||||
window: slow
|
||||
annotations:
|
||||
summary: Share of jobs slower than 30s is burning the ready-time budget (30m and 6h)
|
||||
runbook: infra/grafana/runbooks/slo-burn.md
|
||||
@@ -0,0 +1,6 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yml
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/bin/sh
|
||||
# One SeaweedFS process: master, volume, filer, S3. The filer UI is off.
|
||||
# Anonymous identity has no actions. The API user is limited to one bucket.
|
||||
set -eu
|
||||
: "${S3_BUCKET:?}"
|
||||
: "${S3_ACCESS_KEY_ID:?}"
|
||||
: "${S3_SECRET_ACCESS_KEY:?}"
|
||||
|
||||
cat > /tmp/s3.json <<EOF
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "anonymous",
|
||||
"actions": []
|
||||
},
|
||||
{
|
||||
"name": "messenger-app",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "${S3_ACCESS_KEY_ID}",
|
||||
"secretKey": "${S3_SECRET_ACCESS_KEY}"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read:${S3_BUCKET}",
|
||||
"Read:${S3_BUCKET}/*",
|
||||
"Write:${S3_BUCKET}",
|
||||
"Write:${S3_BUCKET}/*",
|
||||
"List:${S3_BUCKET}",
|
||||
"Tagging:${S3_BUCKET}/*",
|
||||
"Admin:${S3_BUCKET}"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
exec weed server \
|
||||
-dir=/data \
|
||||
-ip.bind=0.0.0.0 \
|
||||
-master.port=9333 \
|
||||
-volume.port=8080 \
|
||||
-filer \
|
||||
-filer.port=8888 \
|
||||
-filer.exposeDirectoryData=false \
|
||||
-s3 \
|
||||
-s3.port=9000 \
|
||||
-s3.config=/tmp/s3.json
|
||||
@@ -0,0 +1,12 @@
|
||||
# SLI и SLO
|
||||
|
||||
Окно цели: 30 суток. Бюджет ошибок: 1% у каждого SLO. Пороги burn rate — 14.4x (быстрое выгорание, окна 5 минут и 1 час) и 6x (медленное, окна 30 минут и 6 часов). Оба окна пары должны выполниться одновременно.
|
||||
|
||||
| SLI | Хорошее событие | SLO | Единица |
|
||||
| --- | --- | --- | --- |
|
||||
| Доступность API | бизнес-запрос со `status_class` не `5xx` | 99% | доля запросов |
|
||||
| Время до ready | job, у которой `messenger_time_to_ready_seconds` попала в bucket `le=30` | 99% готовы не дольше 30 секунд | секунды от durable acceptance до `ready` |
|
||||
|
||||
Health, `/docs` и `/openapi.json` в SLI API не входят. 401 и 404 остаются в знаменателе, но не в числителе ошибок: это не отказ сервиса. Процентили дашборда считаются `histogram_quantile` по bucket, не средним значений и не средним p99 реплик.
|
||||
|
||||
Правила лежат в `infra/prometheus/alerts.yml`. Runbook: `infra/grafana/runbooks/`.
|
||||
@@ -0,0 +1,28 @@
|
||||
stream_over_http_enabled: true
|
||||
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
@@ -0,0 +1,49 @@
|
||||
-- Lab2 schema (written before application queries).
|
||||
-- Source of truth: PostgreSQL for users, chats, membership, messages.
|
||||
-- Transaction boundary: INSERT chat + INSERT members in one transaction (rollback together).
|
||||
-- Sessions are stored in Redis with AOF, not in this database.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
filename TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE users (
|
||||
id UUID PRIMARY KEY,
|
||||
username TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX users_username_key ON users (username);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users (id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_id_idx ON chat_members (user_id);
|
||||
CREATE INDEX chats_created_at_id_idx ON chats (created_at, id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL REFERENCES users (id),
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000)
|
||||
);
|
||||
|
||||
-- Keyset history: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n
|
||||
CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id);
|
||||
@@ -0,0 +1,14 @@
|
||||
-- Shared idempotency for POST /messages. Scope: user + method + path + key.
|
||||
-- The message row and this result are committed in one transaction.
|
||||
-- A rolled-back request leaves no row, so a later retry with the same key can insert.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS idempotency_keys (
|
||||
user_id UUID NOT NULL REFERENCES users (id),
|
||||
http_method TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
idem_key TEXT NOT NULL,
|
||||
request_hash TEXT NOT NULL,
|
||||
response_json TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, http_method, path, idem_key)
|
||||
);
|
||||
@@ -0,0 +1,50 @@
|
||||
-- Attachment metadata stays in Postgres. Bytes stay in the private bucket.
|
||||
-- A blob_writes row is inserted before the S3 PUT and marked committed in the
|
||||
-- same transaction as the attachment row. If that transaction rolls back, the
|
||||
-- API deletes the object. Rows still uncommitted after ORPHAN_GRACE_SECONDS
|
||||
-- are removed by the sweeper, so a crash between PUT and COMMIT cannot leave
|
||||
-- a permanent untracked object. Fresh in-flight uploads are inside the grace
|
||||
-- window and are not deleted.
|
||||
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_len;
|
||||
ALTER TABLE messages
|
||||
ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
owner_id UUID NOT NULL REFERENCES users (id),
|
||||
filename TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
status TEXT NOT NULL,
|
||||
error_code TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachments_type CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_status CHECK (status = 'ready'),
|
||||
CONSTRAINT attachments_ready_error CHECK (status <> 'ready' OR error_code IS NULL)
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_idx ON attachments (chat_id);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id UUID NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments (id),
|
||||
position INTEGER NOT NULL CHECK (position >= 0 AND position < 10),
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
UNIQUE (message_id, position)
|
||||
);
|
||||
|
||||
CREATE TABLE blob_writes (
|
||||
object_key TEXT PRIMARY KEY,
|
||||
committed BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX blob_writes_pending_idx ON blob_writes (created_at) WHERE NOT committed;
|
||||
@@ -0,0 +1,22 @@
|
||||
-- Optional presigned upload. The signed URL dies quickly. The row stays pending
|
||||
-- until complete checks the object that actually landed in the bucket.
|
||||
-- Incomplete rows are aborted by the same grace sweeper as other uncommitted keys.
|
||||
|
||||
CREATE TABLE direct_uploads (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
owner_id UUID NOT NULL REFERENCES users (id),
|
||||
filename TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
status TEXT NOT NULL,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT direct_uploads_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
CONSTRAINT direct_uploads_size CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT direct_uploads_type CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT direct_uploads_status CHECK (status IN ('pending', 'completed', 'aborted'))
|
||||
);
|
||||
|
||||
CREATE INDEX direct_uploads_pending_idx ON direct_uploads (expires_at) WHERE status = 'pending';
|
||||
@@ -0,0 +1,67 @@
|
||||
-- Async attachment processing. Bytes stay in S3. The broker message carries only ids.
|
||||
-- 202 is returned after the staging object and this job row commit. Publish is a
|
||||
-- later step: published_at stays null until the broker confirms, so a crash
|
||||
-- between COMMIT and publish is recovered by the outbox loop.
|
||||
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_status CHECK (status IN ('queued', 'processing', 'ready', 'failed'));
|
||||
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_ready_error;
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_error CHECK (
|
||||
(status = 'failed' AND error_code IS NOT NULL AND error_code ~ '^[a-z0-9_]{1,64}$')
|
||||
OR (status <> 'failed' AND error_code IS NULL)
|
||||
);
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version INTEGER NOT NULL CHECK (pipeline_version >= 1),
|
||||
name TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL CHECK (size_bytes >= 1),
|
||||
width INTEGER NOT NULL CHECK (width >= 1),
|
||||
height INTEGER NOT NULL CHECK (height >= 1),
|
||||
sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
PRIMARY KEY (attachment_id, pipeline_version, name),
|
||||
CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized'))
|
||||
);
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
job_id UUID PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE,
|
||||
pipeline_version INTEGER NOT NULL CHECK (pipeline_version >= 1),
|
||||
attempt INTEGER NOT NULL CHECK (attempt >= 1),
|
||||
status TEXT NOT NULL,
|
||||
error_code TEXT,
|
||||
lease_until TIMESTAMPTZ,
|
||||
lease_token UUID,
|
||||
next_attempt_at TIMESTAMPTZ,
|
||||
published_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE (attachment_id, pipeline_version),
|
||||
CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
CONSTRAINT attachment_jobs_error CHECK (
|
||||
error_code IS NULL OR error_code ~ '^[a-z0-9_]{1,64}$'
|
||||
),
|
||||
CONSTRAINT attachment_jobs_failed_error CHECK (status <> 'failed' OR error_code IS NOT NULL)
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_jobs_outbox_idx
|
||||
ON attachment_jobs (created_at)
|
||||
WHERE published_at IS NULL AND status = 'queued';
|
||||
|
||||
CREATE INDEX attachment_jobs_lease_idx
|
||||
ON attachment_jobs (lease_until)
|
||||
WHERE status = 'processing';
|
||||
|
||||
CREATE TABLE attachment_job_events (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
job_id UUID NOT NULL,
|
||||
attachment_id UUID NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
detail TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
@@ -0,0 +1,14 @@
|
||||
-- The served result is one pipeline version. An older job must not replace it.
|
||||
-- Variant rows stay addressable per version; the attachment points at the winner.
|
||||
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN applied_pipeline_version INTEGER NOT NULL DEFAULT 1
|
||||
CHECK (applied_pipeline_version >= 1);
|
||||
|
||||
CREATE TABLE pipeline_faults (
|
||||
id INTEGER PRIMARY KEY,
|
||||
mode TEXT NOT NULL DEFAULT '',
|
||||
remaining INTEGER NOT NULL DEFAULT 0 CHECK (remaining >= 0)
|
||||
);
|
||||
|
||||
INSERT INTO pipeline_faults (id, mode, remaining) VALUES (1, '', 0);
|
||||
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE attachment_jobs
|
||||
ADD COLUMN traceparent TEXT,
|
||||
ADD COLUMN tracestate TEXT;
|
||||
@@ -0,0 +1,12 @@
|
||||
starlette==0.41.3
|
||||
uvicorn[standard]==0.32.1
|
||||
asyncpg==0.30.0
|
||||
redis==5.2.1
|
||||
argon2-cffi==23.1.0
|
||||
boto3==1.35.99
|
||||
pillow==11.1.0
|
||||
aio-pika==9.5.5
|
||||
opentelemetry-api==1.29.0
|
||||
opentelemetry-sdk==1.29.0
|
||||
opentelemetry-exporter-otlp-proto-http==1.29.0
|
||||
psutil==6.1.1
|
||||
@@ -0,0 +1,323 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Negative attachment cases. Prints statuses only, never tokens."""
|
||||
|
||||
import os
|
||||
import base64
|
||||
from urllib.parse import urlparse
|
||||
import hashlib
|
||||
import json
|
||||
import socket
|
||||
import struct
|
||||
import time
|
||||
import uuid
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
JPEG = base64.b64decode(
|
||||
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAARCAAIAAgDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD5soooroMD/9k="
|
||||
)
|
||||
|
||||
|
||||
def header_only_png(width: int, height: int) -> bytes:
|
||||
signature = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
def chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return signature + chunk(b"IHDR", ihdr) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def png(width: int, height: int) -> bytes:
|
||||
signature = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
def chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
raw = b"".join(b"\x00" + (b"\x10\x20\x30" * width) for _ in range(height))
|
||||
return signature + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self) -> None:
|
||||
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
def call(self, method, path, body=None, token=None, headers=None, raw=None, content_type=None):
|
||||
hdrs = {}
|
||||
data = raw
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
if content_type:
|
||||
hdrs["Content-Type"] = content_type
|
||||
if token:
|
||||
hdrs["Authorization"] = "Bearer " + token
|
||||
if headers:
|
||||
hdrs.update(headers)
|
||||
request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method)
|
||||
try:
|
||||
response = self.opener.open(request, timeout=30)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
payload = response.read()
|
||||
parsed = None
|
||||
if payload and response.headers.get_content_type() == "application/json":
|
||||
parsed = json.loads(payload)
|
||||
return response.status, response.headers, parsed, payload
|
||||
|
||||
|
||||
def wait_ready(api: Api, attachment_id: str, token: str):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
status, _, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
|
||||
expect(200, status, "poll attachment")
|
||||
if body["status"] == "failed":
|
||||
raise SystemExit("FAIL attachment failed")
|
||||
if body["status"] == "ready":
|
||||
return body
|
||||
time.sleep(0.3)
|
||||
raise SystemExit("FAIL attachment did not become ready")
|
||||
|
||||
|
||||
def expect(status, got, label):
|
||||
if got == 500:
|
||||
raise SystemExit("FAIL %s returned 500" % label)
|
||||
if got != status:
|
||||
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
|
||||
print("PASS", label, got)
|
||||
|
||||
|
||||
def user(api: Api, label: str):
|
||||
username = "g4_" + label + "_" + uuid.uuid4().hex[:10]
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
status, _, body, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
{"username": username, "display_name": "Grade " + label, "password": password},
|
||||
)
|
||||
expect(201, status, "register " + label)
|
||||
basic = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
status, _, session, _ = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
||||
expect(201, status, "login " + label)
|
||||
return body["id"], session["token"]
|
||||
|
||||
|
||||
def declared_length(token, chat_id):
|
||||
parsed = urlparse(BASE)
|
||||
host = parsed.netloc or "127.0.0.1:8080"
|
||||
request = (
|
||||
"POST /api/v1/chats/%s/attachments?filename=big.png HTTP/1.1\r\n"
|
||||
"Host: %s\r\n"
|
||||
"Authorization: Bearer %s\r\n"
|
||||
"Content-Type: image/png\r\n"
|
||||
"Content-Length: 10485761\r\n"
|
||||
"Connection: close\r\n\r\n" % (chat_id, host, token)
|
||||
).encode() + b"\x89PNG"
|
||||
sock = socket.create_connection((parsed.hostname or "127.0.0.1", parsed.port or 80), timeout=5)
|
||||
try:
|
||||
sock.sendall(request)
|
||||
data = b""
|
||||
while b"\r\n" not in data:
|
||||
chunk = sock.recv(256)
|
||||
if not chunk:
|
||||
break
|
||||
data += chunk
|
||||
finally:
|
||||
sock.close()
|
||||
line = data.split(b"\r\n", 1)[0].decode("latin1", "replace")
|
||||
code = int(line.split()[1])
|
||||
expect(413, code, "content-length above 10MiB before body is read")
|
||||
|
||||
|
||||
def main():
|
||||
api = Api()
|
||||
alice, alice_token = user(api, "alice")
|
||||
bob, bob_token = user(api, "bob")
|
||||
eve, eve_token = user(api, "eve")
|
||||
status, _, chat, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
{"title": "grade4", "member_ids": [alice, bob]},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(201, status, "create chat")
|
||||
chat_id = chat["id"]
|
||||
status, _, other, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
{"title": "other", "member_ids": [alice, eve]},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(201, status, "create other chat")
|
||||
|
||||
status, _, message, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/messages" % chat_id,
|
||||
{"text": "только текст"},
|
||||
token=alice_token,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
expect(201, status, "text message")
|
||||
if message["attachment_ids"] != []:
|
||||
raise SystemExit("FAIL text message lost empty attachment_ids")
|
||||
|
||||
good = png(32, 32)
|
||||
status, _, attachment, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=ok.png" % chat_id,
|
||||
token=alice_token,
|
||||
raw=good,
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(202, status, "png upload")
|
||||
if attachment["sha256"] != hashlib.sha256(good).hexdigest() or attachment["status"] != "queued":
|
||||
raise SystemExit("FAIL png metadata")
|
||||
attachment = wait_ready(api, attachment["id"], bob_token)
|
||||
status, headers, _, downloaded = api.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/%s/content" % attachment["id"],
|
||||
token=bob_token,
|
||||
)
|
||||
expect(200, status, "member download")
|
||||
if downloaded != good or headers.get_content_type() != "image/png":
|
||||
raise SystemExit("FAIL download bytes")
|
||||
if headers.get("Location"):
|
||||
raise SystemExit("FAIL download redirected")
|
||||
|
||||
status, _, jpeg_meta, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=ok.jpg" % chat_id,
|
||||
token=alice_token,
|
||||
raw=JPEG,
|
||||
content_type="image/jpeg",
|
||||
)
|
||||
expect(202, status, "jpeg upload")
|
||||
if jpeg_meta["content_type"] != "image/jpeg" or jpeg_meta["sha256"] != hashlib.sha256(JPEG).hexdigest():
|
||||
raise SystemExit("FAIL jpeg metadata")
|
||||
|
||||
status, _, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=a/b.png" % chat_id,
|
||||
token=alice_token,
|
||||
raw=good,
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(400, status, "filename with slash")
|
||||
status, _, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=%s" % (chat_id, "bad%0Aname.png"),
|
||||
token=alice_token,
|
||||
raw=good,
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(400, status, "filename with control character")
|
||||
|
||||
status, _, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=lie.png" % chat_id,
|
||||
token=alice_token,
|
||||
raw=good,
|
||||
content_type="image/jpeg",
|
||||
)
|
||||
expect(415, status, "png bytes declared as jpeg")
|
||||
status, _, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=cut.png" % chat_id,
|
||||
token=alice_token,
|
||||
raw=b"\x89PNG\r\n\x1a\ntruncated",
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(415, status, "truncated png")
|
||||
status, _, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=cut.jpg" % chat_id,
|
||||
token=alice_token,
|
||||
raw=JPEG[:-40],
|
||||
content_type="image/jpeg",
|
||||
)
|
||||
expect(415, status, "truncated jpeg")
|
||||
status, _, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=huge.png" % chat_id,
|
||||
token=alice_token,
|
||||
raw=header_only_png(10000, 5000),
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(413, status, "pixel limit")
|
||||
declared_length(alice_token, chat_id)
|
||||
status, _, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=big.png" % chat_id,
|
||||
token=alice_token,
|
||||
raw=b"\x89PNG" + b"\x00" * (10_485_760),
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(413, status, "body above 10MiB")
|
||||
|
||||
status, _, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=nope.png" % chat_id,
|
||||
token=eve_token,
|
||||
raw=good,
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(404, status, "outsider upload")
|
||||
for suffix in ("", "/content"):
|
||||
status, _, _, _ = api.call("GET", "/api/v1/attachments/%s%s" % (attachment["id"], suffix), token=eve_token)
|
||||
expect(404, status, "outsider get " + (suffix or "metadata"))
|
||||
|
||||
missing = str(uuid.uuid4())
|
||||
status, _, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/messages" % chat_id,
|
||||
{"text": "", "attachment_ids": [missing]},
|
||||
token=alice_token,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
expect(404, status, "unknown attachment id")
|
||||
status, _, foreign, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=other.png" % other["id"],
|
||||
token=alice_token,
|
||||
raw=good,
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(202, status, "upload into other chat")
|
||||
status, _, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/messages" % chat_id,
|
||||
{"text": "", "attachment_ids": [foreign["id"]]},
|
||||
token=alice_token,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
expect(404, status, "attachment from another chat")
|
||||
ids = [str(uuid.uuid4()) for _ in range(11)]
|
||||
status, _, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/messages" % chat_id,
|
||||
{"text": "x", "attachment_ids": ids},
|
||||
token=alice_token,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
expect(400, status, "more than 10 attachment ids")
|
||||
status, _, linked, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/messages" % chat_id,
|
||||
{"text": "", "attachment_ids": [attachment["id"]]},
|
||||
token=bob_token,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
expect(201, status, "message with attachment")
|
||||
if linked["attachment_ids"] != [attachment["id"]] or linked["text"] != "":
|
||||
raise SystemExit("FAIL linked message")
|
||||
print("PASS grade4 api checks")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Private bucket, orphan grace, and a bounded concurrent-upload memory sample."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "evidence" / "grade4-infra.json"
|
||||
|
||||
|
||||
def compose(*args, input_text=None):
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", *args],
|
||||
cwd=ROOT,
|
||||
input=input_text,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(result.stderr[-500:] or result.stdout[-500:] or "compose failed")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def anonymous_status():
|
||||
script = r"""
|
||||
code=$(wget -S -O /dev/null http://127.0.0.1:9000/messenger-attachments/ 2>&1 | awk 'BEGIN{c=""} /^[[:space:]]*HTTP\//{c=$2} END{print c}')
|
||||
printf '%s' "$code"
|
||||
"""
|
||||
output = compose("exec", "-T", "s3", "sh", "-c", script).strip()
|
||||
if output != "403":
|
||||
raise SystemExit("anonymous bucket status %s" % output)
|
||||
return output
|
||||
|
||||
|
||||
def other_bucket_denied():
|
||||
code = r"""
|
||||
import os
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
from botocore.exceptions import ClientError
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.environ["S3_ENDPOINT"],
|
||||
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],
|
||||
aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
|
||||
region_name=os.environ.get("S3_REGION", "us-east-1"),
|
||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
||||
)
|
||||
try:
|
||||
client.create_bucket(Bucket="not-the-messenger-bucket")
|
||||
print("allowed")
|
||||
except ClientError as exc:
|
||||
print(exc.response.get("Error", {}).get("Code", "denied"))
|
||||
"""
|
||||
output = compose("exec", "-T", "api", "python", "-c", code).strip()
|
||||
if output == "allowed":
|
||||
raise SystemExit("app credentials created a second bucket")
|
||||
return output
|
||||
|
||||
|
||||
def plant_and_sweep():
|
||||
code = r"""
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from app import config, db, s3
|
||||
|
||||
async def main():
|
||||
await db.connect()
|
||||
old_key = "originals/orphan-evidence"
|
||||
fresh_key = "originals/fresh-evidence"
|
||||
payload = b"not-a-real-image"
|
||||
await s3.put_bytes(old_key, payload, "application/octet-stream")
|
||||
await s3.put_bytes(fresh_key, payload, "application/octet-stream")
|
||||
old = datetime.now(timezone.utc) - timedelta(seconds=config.ORPHAN_GRACE_SECONDS + 60)
|
||||
fresh = datetime.now(timezone.utc)
|
||||
async with db.require().acquire() as conn:
|
||||
await conn.execute("DELETE FROM blob_writes WHERE object_key = ANY($1::text[])", [old_key, fresh_key])
|
||||
await conn.execute(
|
||||
"INSERT INTO blob_writes (object_key, committed, created_at) VALUES ($1, FALSE, $2), ($3, FALSE, $4)",
|
||||
old_key, old, fresh_key, fresh,
|
||||
)
|
||||
from app.attachments import sweep_orphans
|
||||
deleted = await sweep_orphans()
|
||||
async def exists(key):
|
||||
try:
|
||||
await s3.get_bytes(key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
print("%s %s %s" % (deleted, int(await exists(old_key)), int(await exists(fresh_key))))
|
||||
await s3.delete_key(fresh_key)
|
||||
async with db.require().acquire() as conn:
|
||||
await conn.execute("DELETE FROM blob_writes WHERE object_key = $1 AND NOT committed", fresh_key)
|
||||
await db.close()
|
||||
|
||||
asyncio.run(main())
|
||||
"""
|
||||
output = compose("exec", "-T", "api", "python", "-c", code).strip().split()
|
||||
deleted, old_exists, fresh_exists = int(output[0]), int(output[1]), int(output[2])
|
||||
if deleted < 1 or old_exists != 0 or fresh_exists != 1:
|
||||
raise SystemExit("sweep result deleted=%s old=%s fresh=%s" % (deleted, old_exists, fresh_exists))
|
||||
return {"deleted": deleted, "old_removed": True, "fresh_kept": True}
|
||||
|
||||
|
||||
def memory_sample():
|
||||
import base64
|
||||
|
||||
username = "mem_" + uuid.uuid4().hex[:10]
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
body = json.dumps({"username": username, "display_name": "Mem", "password": password}).encode()
|
||||
|
||||
def call(method, path, data=None, headers=None):
|
||||
request = urllib.request.Request("http://127.0.0.1:8080" + path, data=data, headers=headers or {}, method=method)
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=60)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
return response.status, response.read()
|
||||
|
||||
status, raw = call("POST", "/api/v1/users", body, {"Content-Type": "application/json"})
|
||||
if status != 201:
|
||||
raise SystemExit("memory user %s" % status)
|
||||
user = json.loads(raw)
|
||||
basic = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
status, raw = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
||||
if status != 201:
|
||||
raise SystemExit("memory login %s" % status)
|
||||
token = json.loads(raw)["token"]
|
||||
status, raw = call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
json.dumps({"title": "mem", "member_ids": [user["id"]]}).encode(),
|
||||
{"Content-Type": "application/json", "Authorization": "Bearer " + token},
|
||||
)
|
||||
if status != 201:
|
||||
raise SystemExit("memory chat %s" % status)
|
||||
chat_id = json.loads(raw)["id"]
|
||||
import os
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
def png():
|
||||
signature = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
|
||||
width, height = 320, 240
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
raw_rows = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
|
||||
return signature + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw_rows, 1)) + chunk(b"IEND", b"")
|
||||
|
||||
blob = png()
|
||||
codes = []
|
||||
|
||||
def upload():
|
||||
status, _ = call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=m.png" % chat_id,
|
||||
blob,
|
||||
{"Content-Type": "image/png", "Authorization": "Bearer " + token},
|
||||
)
|
||||
codes.append(status)
|
||||
|
||||
samples = []
|
||||
|
||||
def watch():
|
||||
end = time.monotonic() + 8
|
||||
while time.monotonic() < end:
|
||||
stats = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
for line in stats.stdout.splitlines():
|
||||
if "mtusi2026lab6-api-" in line:
|
||||
samples.append(line.strip())
|
||||
time.sleep(0.4)
|
||||
|
||||
watcher = threading.Thread(target=watch)
|
||||
watcher.start()
|
||||
threads = [threading.Thread(target=upload) for _ in range(10)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
watcher.join()
|
||||
if codes.count(202) != 10:
|
||||
raise SystemExit("concurrent uploads %s" % codes)
|
||||
|
||||
def mib(line):
|
||||
value = line.split()[1]
|
||||
number = float(value[:-3]) if value.lower().endswith("mib") else float(value[:-3]) / 1024
|
||||
return number
|
||||
|
||||
peak = max(mib(line) for line in samples) if samples else 0
|
||||
return {"uploads": 10, "bytes_each": len(blob), "success": 10, "peak_api_mib": round(peak, 1), "samples": len(samples)}
|
||||
|
||||
|
||||
def main():
|
||||
published = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "port", "s3", "9000"],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
binding = published.stdout.strip()
|
||||
if published.returncode == 0 and binding and not binding.startswith("127.0.0.1:"):
|
||||
raise SystemExit("s3 port is published outside loopback: " + binding)
|
||||
summary = {
|
||||
"anonymous_bucket_status": anonymous_status(),
|
||||
"second_bucket": other_bucket_denied(),
|
||||
"sweep": plant_and_sweep(),
|
||||
"s3_port_published": binding or False,
|
||||
"memory": memory_sample(),
|
||||
}
|
||||
OUT.parent.mkdir(exist_ok=True)
|
||||
OUT.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(summary, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import urllib.error
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab5 fault checks: worker down, content 409, then the same upload reaches ready.
|
||||
|
||||
Prints statuses and ids only. Does not print tokens.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
|
||||
|
||||
|
||||
def compose(*args: str) -> None:
|
||||
command = ["docker", "compose", "--env-file", ".env"]
|
||||
extra = os.environ.get("LAB5_COMPOSE_FILES", "")
|
||||
if extra:
|
||||
for name in extra.split(os.pathsep):
|
||||
if name:
|
||||
command.extend(["-f", name])
|
||||
subprocess.run([*command, *args], cwd=ROOT, check=True)
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self) -> None:
|
||||
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
def call(self, method, path, body=None, token=None, raw=None, content_type=None, headers=None):
|
||||
hdrs = dict(headers or {})
|
||||
data = raw
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
if content_type:
|
||||
hdrs["Content-Type"] = content_type
|
||||
if token:
|
||||
hdrs["Authorization"] = "Bearer " + token
|
||||
request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method)
|
||||
try:
|
||||
response = self.opener.open(request, timeout=30)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
payload = response.read()
|
||||
parsed = json.loads(payload) if payload and response.headers.get_content_type() == "application/json" else None
|
||||
return response.status, parsed, payload
|
||||
|
||||
|
||||
def expect(status, got, label):
|
||||
if got != status:
|
||||
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
|
||||
print("PASS", label, got)
|
||||
|
||||
|
||||
def user(api: Api):
|
||||
username = "l5_" + uuid.uuid4().hex[:10]
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
status, body, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
{"username": username, "display_name": "Lab5", "password": password},
|
||||
)
|
||||
expect(201, status, "register")
|
||||
basic = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
status, session, _ = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
||||
expect(201, status, "login")
|
||||
return body["id"], session["token"]
|
||||
|
||||
|
||||
def wait_ready(api: Api, attachment_id: str, token: str):
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
|
||||
expect(200, status, "poll")
|
||||
if body["status"] == "failed":
|
||||
raise SystemExit("FAIL attachment failed " + str(body.get("error_code")))
|
||||
if body["status"] == "ready":
|
||||
return body
|
||||
time.sleep(0.4)
|
||||
raise SystemExit("FAIL not ready")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
api = Api()
|
||||
alice, alice_token = user(api)
|
||||
bob, bob_token = user(api)
|
||||
status, chat, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
{"title": "lab5", "member_ids": [alice, bob]},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(201, status, "chat")
|
||||
chat_id = chat["id"]
|
||||
print("chat", chat_id)
|
||||
|
||||
compose("stop", "worker")
|
||||
content = FIXTURE.read_bytes()
|
||||
status, attachment, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=sample.png" % chat_id,
|
||||
token=alice_token,
|
||||
raw=content,
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(202, status, "upload while worker is stopped")
|
||||
if attachment["status"] != "queued" or attachment["sha256"] != hashlib.sha256(content).hexdigest():
|
||||
raise SystemExit("FAIL queued metadata")
|
||||
print("attachment", attachment["id"])
|
||||
status, _, _ = api.call("GET", "/api/v1/attachments/%s/content" % attachment["id"], token=bob_token)
|
||||
expect(409, status, "content before ready")
|
||||
status, message, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/messages" % chat_id,
|
||||
{"text": "очередь не блокирует чат", "attachment_ids": [attachment["id"]]},
|
||||
token=alice_token,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
expect(201, status, "message while queued")
|
||||
if message["attachment_ids"] != [attachment["id"]]:
|
||||
raise SystemExit("FAIL queued attachment was not linkable")
|
||||
|
||||
compose("start", "worker")
|
||||
ready = wait_ready(api, attachment["id"], bob_token)
|
||||
thumbs = [item for item in ready["variants"] if item["name"] == "thumbnail"]
|
||||
if len(thumbs) != 1 or (thumbs[0]["width"], thumbs[0]["height"]) != (256, 160):
|
||||
raise SystemExit("FAIL thumbnail metadata %s" % thumbs)
|
||||
names = sorted(item["name"] for item in ready["variants"])
|
||||
if names != ["crop", "optimized", "thumbnail"]:
|
||||
raise SystemExit("FAIL variants %s" % names)
|
||||
status, _, payload = api.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/%s/content?variant=thumbnail" % attachment["id"],
|
||||
token=bob_token,
|
||||
)
|
||||
expect(200, status, "thumbnail bytes")
|
||||
if hashlib.sha256(payload).hexdigest() != thumbs[0]["sha256"]:
|
||||
raise SystemExit("FAIL thumbnail checksum")
|
||||
print("PASS lab5 worker-down upload became ready", attachment["id"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,222 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab6 failure scenarios. Prints statuses, ids and metric values. Never tokens."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
PROM = os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090")
|
||||
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
|
||||
|
||||
|
||||
def compose(*args: str) -> None:
|
||||
command = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml"]
|
||||
if os.environ.get("COMPOSE_ALTPORTS") == "1":
|
||||
command.extend(["-f", "compose.altports.yaml"])
|
||||
subprocess.run([*command, *args], cwd=ROOT, check=True)
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self) -> None:
|
||||
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
def call(self, method, path, body=None, token=None, raw=None, content_type=None, headers=None, timeout=15):
|
||||
hdrs = dict(headers or {})
|
||||
data = raw
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
if content_type:
|
||||
hdrs["Content-Type"] = content_type
|
||||
if token:
|
||||
hdrs["Authorization"] = "Bearer " + token
|
||||
request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method)
|
||||
try:
|
||||
response = self.opener.open(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
payload = response.read()
|
||||
return response.status, payload
|
||||
|
||||
|
||||
def expect(status, got, label):
|
||||
if got != status:
|
||||
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
|
||||
print("PASS", label, got)
|
||||
|
||||
|
||||
def prom_query(expr: str):
|
||||
url = PROM + "/api/v1/query?" + urllib.parse.urlencode({"query": expr})
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
body = json.loads(response.read().decode())
|
||||
if body.get("status") != "success":
|
||||
raise SystemExit("FAIL prometheus query")
|
||||
result = body["data"]["result"]
|
||||
if not result:
|
||||
return None
|
||||
return float(result[0]["value"][1])
|
||||
|
||||
|
||||
def alerts():
|
||||
with urllib.request.urlopen(PROM + "/api/v1/alerts", timeout=10) as response:
|
||||
body = json.loads(response.read().decode())
|
||||
names = []
|
||||
for item in body["data"]["alerts"]:
|
||||
if item.get("state") == "firing":
|
||||
names.append(item["labels"].get("alertname", ""))
|
||||
return sorted(names)
|
||||
|
||||
|
||||
def wait_ready_http():
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
status, _ = Api().call("GET", "/health/ready", timeout=3)
|
||||
if status == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise SystemExit("FAIL API did not become ready")
|
||||
|
||||
|
||||
def user(api: Api):
|
||||
username = "l6_" + uuid.uuid4().hex[:10]
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
status, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
{"username": username, "display_name": "Lab6", "password": password},
|
||||
)
|
||||
expect(201, status, "register")
|
||||
basic = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
status, payload = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
||||
expect(201, status, "login")
|
||||
return json.loads(payload)["token"]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
api = Api()
|
||||
token = user(api)
|
||||
status, chat_raw = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
{"title": "lab6-incidents", "member_ids": []},
|
||||
token=token,
|
||||
)
|
||||
# member_ids must include someone? empty may 400. Create with only self via API rules.
|
||||
if status != 201:
|
||||
me_status, me_raw = api.call("GET", "/api/v1/users/me", token=token)
|
||||
expect(200, me_status, "me")
|
||||
me = json.loads(me_raw)["id"]
|
||||
status, chat_raw = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
{"title": "lab6-incidents", "member_ids": [me]},
|
||||
token=token,
|
||||
)
|
||||
expect(201, status, "chat")
|
||||
chat_id = json.loads(chat_raw)["id"]
|
||||
print("chat", chat_id)
|
||||
|
||||
print("-- worker stopped: queue age grows, content stays 409 --")
|
||||
compose("stop", "worker")
|
||||
content = FIXTURE.read_bytes()
|
||||
status, attachment_raw = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=sample.png" % chat_id,
|
||||
token=token,
|
||||
raw=content,
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(202, status, "upload while worker stopped")
|
||||
attachment_id = json.loads(attachment_raw)["id"]
|
||||
print("attachment", attachment_id)
|
||||
status, _ = api.call("GET", "/api/v1/attachments/%s/content" % attachment_id, token=token)
|
||||
expect(409, status, "content before ready")
|
||||
deadline = time.monotonic() + 80
|
||||
age = None
|
||||
while time.monotonic() < deadline:
|
||||
age = prom_query("max(messenger_queue_oldest_age_seconds)")
|
||||
print("queue_oldest_age_seconds", age, "firing", ",".join(alerts()) or "-")
|
||||
if age is not None and age >= 15:
|
||||
break
|
||||
time.sleep(5)
|
||||
if age is None or age < 5:
|
||||
raise SystemExit("FAIL queue age did not grow")
|
||||
compose("start", "worker")
|
||||
ready_deadline = time.monotonic() + 90
|
||||
while time.monotonic() < ready_deadline:
|
||||
status, body = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
|
||||
expect(200, status, "poll")
|
||||
parsed = json.loads(body)
|
||||
if parsed["status"] == "ready":
|
||||
print("PASS attachment ready after worker start", attachment_id)
|
||||
break
|
||||
if parsed["status"] == "failed":
|
||||
raise SystemExit("FAIL attachment failed")
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
raise SystemExit("FAIL attachment did not become ready")
|
||||
time.sleep(12)
|
||||
print("queue_oldest_age_seconds_after", prom_query("max(messenger_queue_oldest_age_seconds)"))
|
||||
|
||||
print("-- postgres stopped: live stays 200, ready becomes 503 --")
|
||||
compose("stop", "postgres")
|
||||
time.sleep(2)
|
||||
status, _ = api.call("GET", "/health/live")
|
||||
expect(200, status, "live while postgres stopped")
|
||||
status, _ = api.call("GET", "/health/ready")
|
||||
expect(503, status, "ready while postgres stopped")
|
||||
burst_503 = 0
|
||||
for _ in range(8):
|
||||
status, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
{
|
||||
"username": "l6d_" + uuid.uuid4().hex[:8],
|
||||
"display_name": "DbDown",
|
||||
"password": "T3st-" + uuid.uuid4().hex,
|
||||
},
|
||||
)
|
||||
if status == 503:
|
||||
burst_503 += 1
|
||||
else:
|
||||
print("unexpected status during postgres stop", status)
|
||||
if burst_503 < 8:
|
||||
raise SystemExit("FAIL expected every business write to be 503")
|
||||
print("PASS business 503 while postgres stopped", burst_503)
|
||||
compose("start", "postgres")
|
||||
wait_ready_http()
|
||||
print("PASS ready after postgres start")
|
||||
|
||||
print("-- collector stopped: API still accepts a registration --")
|
||||
compose("stop", "otel-collector")
|
||||
time.sleep(1)
|
||||
status, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
{
|
||||
"username": "l6c_" + uuid.uuid4().hex[:10],
|
||||
"display_name": "CollectorDown",
|
||||
"password": "T3st-" + uuid.uuid4().hex,
|
||||
},
|
||||
)
|
||||
expect(201, status, "register while collector is down")
|
||||
compose("start", "otel-collector")
|
||||
print("PASS collector restarted")
|
||||
print("firing_at_end", ",".join(alerts()) or "-")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
cd /d "%~dp0\.."
|
||||
docker compose --env-file .env restart postgres redis s3 rabbitmq api worker
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Restart API replicas and every store they use. Volumes are kept.
|
||||
set -eu
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
if [ -f .env ]; then
|
||||
docker compose --env-file .env restart postgres redis s3 rabbitmq api worker
|
||||
else
|
||||
docker compose restart postgres redis s3 rabbitmq api worker
|
||||
fi
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/bin/sh
|
||||
# Replace API replicas one at a time. The other replica keeps serving.
|
||||
set -eu
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
|
||||
compose() {
|
||||
if [ -f .env ]; then
|
||||
docker compose --env-file .env "$@"
|
||||
else
|
||||
docker compose "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
PY=python3
|
||||
else
|
||||
PY=python
|
||||
fi
|
||||
|
||||
ids=$(compose ps -q api)
|
||||
count=0
|
||||
for id in $ids; do
|
||||
count=$((count + 1))
|
||||
done
|
||||
if [ "$count" -lt 2 ]; then
|
||||
echo "rolling update needs at least 2 running API replicas" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for id in $ids; do
|
||||
echo "replacing $(docker inspect -f '{{.Config.Hostname}}' "$id")"
|
||||
docker stop -t 8 "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
compose up -d --scale "api=$count" --no-deps --no-recreate api >/dev/null
|
||||
ready=0
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
running=0
|
||||
for cid in $(compose ps -q api); do
|
||||
running=$((running + 1))
|
||||
done
|
||||
if [ "$running" -ge "$count" ] && "$PY" -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" >/dev/null 2>&1; then
|
||||
ready=1
|
||||
break
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 1
|
||||
done
|
||||
if [ "$ready" -ne 1 ]; then
|
||||
echo "replacement did not become ready" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "rolling update finished; replicas=$count"
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0stop-one.ps1"
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,23 @@
|
||||
$ErrorActionPreference = "Stop"
|
||||
if (-not $env:TARGET_INSTANCE_ID) {
|
||||
Write-Error "TARGET_INSTANCE_ID is required"
|
||||
exit 1
|
||||
}
|
||||
Set-Location (Join-Path $PSScriptRoot "..")
|
||||
$ids = docker compose --env-file .env ps -q api
|
||||
$found = $false
|
||||
foreach ($id in $ids) {
|
||||
if (-not $id) { continue }
|
||||
$iid = ""
|
||||
try { $iid = (docker exec $id cat /tmp/instance-id).Trim() } catch { $iid = "" }
|
||||
$hostName = (docker inspect -f "{{.Config.Hostname}}" $id).Trim()
|
||||
if ($iid -eq $env:TARGET_INSTANCE_ID -or $hostName -eq $env:TARGET_INSTANCE_ID) {
|
||||
docker stop -t 5 $id | Out-Null
|
||||
$found = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if (-not $found) {
|
||||
Write-Error "no running API replica matched TARGET_INSTANCE_ID"
|
||||
exit 1
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/bin/sh
|
||||
# Stop the single API replica whose X-Instance-Id is TARGET_INSTANCE_ID.
|
||||
# Leaves it stopped. Does not touch the database, Redis, the load balancer, or volumes.
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
|
||||
compose() {
|
||||
if [ -f .env ]; then
|
||||
docker compose --env-file .env "$@"
|
||||
else
|
||||
docker compose "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
found=0
|
||||
for id in $(compose ps -q api); do
|
||||
iid=$(docker exec "$id" cat /tmp/instance-id 2>/dev/null | tr -d '\r\n' || true)
|
||||
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
|
||||
if [ "$iid" = "$TARGET_INSTANCE_ID" ] || [ "$host" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker stop -t 5 "$id" >/dev/null
|
||||
found=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$found" -ne 1 ]; then
|
||||
echo "no running API replica matched TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Stop the worker and leave it stopped. The broker, API and volumes keep running.
|
||||
set -eu
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
if [ -f .env ]; then
|
||||
docker compose --env-file .env stop worker
|
||||
else
|
||||
docker compose stop worker
|
||||
fi
|
||||
@@ -0,0 +1,394 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 1: presigned upload, negative cases, API traffic and RAM at two sizes.
|
||||
|
||||
The signed URL is never written to the evidence file.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "evidence" / "track1.json"
|
||||
BASE = "http://127.0.0.1:8080"
|
||||
|
||||
|
||||
def expect(status, got, label):
|
||||
if got == 500:
|
||||
raise SystemExit("FAIL %s returned 500" % label)
|
||||
if got != status:
|
||||
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
|
||||
|
||||
|
||||
def wait_ready(api, attachment_id, token):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
|
||||
expect(200, status, "poll attachment")
|
||||
if body["status"] == "failed":
|
||||
raise SystemExit("FAIL attachment failed")
|
||||
if body["status"] == "ready":
|
||||
return body
|
||||
time.sleep(0.3)
|
||||
raise SystemExit("FAIL attachment did not become ready")
|
||||
print("PASS", label, got)
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self) -> None:
|
||||
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
def call(self, method, path, body=None, token=None, raw=None, content_type=None):
|
||||
headers = {}
|
||||
data = raw
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
if content_type:
|
||||
headers["Content-Type"] = content_type
|
||||
if token:
|
||||
headers["Authorization"] = "Bearer " + token
|
||||
request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method)
|
||||
try:
|
||||
response = self.opener.open(request, timeout=60)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
payload = response.read()
|
||||
parsed = None
|
||||
if payload and response.headers.get_content_type() == "application/json":
|
||||
parsed = json.loads(payload)
|
||||
return response.status, parsed, payload
|
||||
|
||||
|
||||
def png(width: int, height: int) -> bytes:
|
||||
signature = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
def chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
|
||||
return signature + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def put_signed(url: str, data: bytes, content_type: str) -> int:
|
||||
request = urllib.request.Request(url, data=data, method="PUT", headers={"Content-Type": content_type})
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=60)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
response.read()
|
||||
return response.status
|
||||
|
||||
|
||||
def api_rx() -> int:
|
||||
ids = subprocess.check_output(
|
||||
["docker", "compose", "--env-file", ".env", "ps", "-q", "api"],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
).split()
|
||||
total = 0
|
||||
for container_id in ids:
|
||||
raw = subprocess.check_output(["docker", "exec", container_id, "cat", "/proc/net/dev"], text=True)
|
||||
for line in raw.splitlines():
|
||||
if "eth0" not in line:
|
||||
continue
|
||||
total += int(line.split()[1])
|
||||
return total
|
||||
|
||||
|
||||
def peak_during(action):
|
||||
peaks = []
|
||||
stop = threading.Event()
|
||||
|
||||
def watch():
|
||||
while not stop.is_set():
|
||||
stats = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
for line in stats.stdout.splitlines():
|
||||
if "mtusi2026lab6-api-" not in line:
|
||||
continue
|
||||
value = line.split()[1]
|
||||
number = float(value[:-3])
|
||||
if not value.lower().endswith("mib"):
|
||||
number = number / 1024
|
||||
peaks.append(number)
|
||||
time.sleep(0.25)
|
||||
|
||||
thread = threading.Thread(target=watch)
|
||||
thread.start()
|
||||
before = api_rx()
|
||||
result = action()
|
||||
after = api_rx()
|
||||
time.sleep(0.3)
|
||||
stop.set()
|
||||
thread.join()
|
||||
return {"rx_bytes": after - before, "peak_api_mib": round(max(peaks) if peaks else 0, 1), "result": result}
|
||||
|
||||
|
||||
def compose_psql(sql: str) -> None:
|
||||
subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c", sql],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def negatives(api, alice_token, bob_token, eve_token, chat_id):
|
||||
small = png(16, 16)
|
||||
status, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/uploads" % chat_id,
|
||||
{"filename": "a/b.png", "content_type": "image/png", "size_bytes": len(small)},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(400, status, "direct filename")
|
||||
status, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/uploads" % chat_id,
|
||||
{"filename": "big.png", "content_type": "image/png", "size_bytes": 10485761},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(413, status, "direct declared size")
|
||||
status, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/uploads" % chat_id,
|
||||
{"filename": "x.png", "content_type": "image/png", "size_bytes": len(small)},
|
||||
token=eve_token,
|
||||
)
|
||||
expect(404, status, "outsider presign")
|
||||
|
||||
status, started, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/uploads" % chat_id,
|
||||
{"filename": "wait.png", "content_type": "image/png", "size_bytes": len(small)},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(201, status, "presign for incomplete")
|
||||
status, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/uploads/%s/complete" % started["upload_id"],
|
||||
{"sha256": hashlib.sha256(small).hexdigest()},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(409, status, "complete before PUT")
|
||||
status, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/uploads/%s/complete" % started["upload_id"],
|
||||
{"sha256": hashlib.sha256(small).hexdigest()},
|
||||
token=bob_token,
|
||||
)
|
||||
expect(404, status, "complete by another member")
|
||||
|
||||
tampered = put_signed(started["url"] + "x", small, "image/png")
|
||||
if tampered == 200:
|
||||
raise SystemExit("FAIL tampered signature was accepted")
|
||||
print("PASS tampered signature", tampered)
|
||||
|
||||
status, started, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/uploads" % chat_id,
|
||||
{"filename": "badsum.png", "content_type": "image/png", "size_bytes": len(small)},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(201, status, "presign for bad checksum")
|
||||
if put_signed(started["url"], small, "image/png") != 200:
|
||||
raise SystemExit("FAIL signed PUT")
|
||||
status, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/uploads/%s/complete" % started["upload_id"],
|
||||
{"sha256": "0" * 64},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(400, status, "wrong checksum")
|
||||
status, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/uploads/%s/complete" % started["upload_id"],
|
||||
{"sha256": hashlib.sha256(small).hexdigest()},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(409, status, "complete after rejected checksum")
|
||||
|
||||
status, started, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/uploads" % chat_id,
|
||||
{"filename": "late.png", "content_type": "image/png", "size_bytes": len(small)},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(201, status, "presign for expiry")
|
||||
compose_psql(
|
||||
"UPDATE direct_uploads SET expires_at = now() - interval '5 minutes' WHERE id = '%s'"
|
||||
% started["upload_id"]
|
||||
)
|
||||
status, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/uploads/%s/complete" % started["upload_id"],
|
||||
{"sha256": hashlib.sha256(small).hexdigest()},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(409, status, "expired upload")
|
||||
|
||||
digest = hashlib.sha256(small).hexdigest()
|
||||
status, started, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/uploads" % chat_id,
|
||||
{"filename": "ok.png", "content_type": "image/png", "size_bytes": len(small)},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(201, status, "presign happy path")
|
||||
if "originals/" in json.dumps({k: v for k, v in started.items() if k != "url"}):
|
||||
raise SystemExit("FAIL object key leaked outside the signed URL")
|
||||
if put_signed(started["url"], small, "image/png") != 200:
|
||||
raise SystemExit("FAIL happy PUT")
|
||||
status, attachment, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/uploads/%s/complete" % started["upload_id"],
|
||||
{"sha256": digest},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(202, status, "complete happy path")
|
||||
if attachment["sha256"] != digest or attachment["status"] != "queued":
|
||||
raise SystemExit("FAIL complete metadata")
|
||||
attachment = wait_ready(api, attachment["id"], bob_token)
|
||||
status, _, downloaded = api.call("GET", "/api/v1/attachments/%s/content" % attachment["id"], token=bob_token)
|
||||
expect(200, status, "member downloads direct upload")
|
||||
if downloaded != small:
|
||||
raise SystemExit("FAIL direct bytes")
|
||||
status, _, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/uploads/%s/complete" % started["upload_id"],
|
||||
{"sha256": digest},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(409, status, "second complete")
|
||||
status, proxy, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=proxy.png" % chat_id,
|
||||
token=alice_token,
|
||||
raw=small,
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(202, status, "proxy endpoint still works")
|
||||
return proxy["id"]
|
||||
|
||||
|
||||
def compare(api, token, chat_id):
|
||||
rows = []
|
||||
for label, image in (("small", png(200, 150)), ("large", png(900, 700))):
|
||||
digest = hashlib.sha256(image).hexdigest()
|
||||
proxy = peak_during(
|
||||
lambda image=image: api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=%s.png" % (chat_id, label),
|
||||
token=token,
|
||||
raw=image,
|
||||
content_type="image/png",
|
||||
)
|
||||
)
|
||||
if proxy["result"][0] != 202:
|
||||
raise SystemExit("FAIL proxy %s %s" % (label, proxy["result"][0]))
|
||||
status, started, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/uploads" % chat_id,
|
||||
{"filename": label + "-direct.png", "content_type": "image/png", "size_bytes": len(image)},
|
||||
token=token,
|
||||
)
|
||||
expect(201, status, "presign " + label)
|
||||
direct_put = peak_during(lambda image=image, url=started["url"]: put_signed(url, image, "image/png"))
|
||||
if direct_put["result"] != 200:
|
||||
raise SystemExit("FAIL direct PUT %s %s" % (label, direct_put["result"]))
|
||||
direct_complete = peak_during(
|
||||
lambda digest=digest, upload_id=started["upload_id"]: api.call(
|
||||
"POST",
|
||||
"/api/v1/uploads/%s/complete" % upload_id,
|
||||
{"sha256": digest},
|
||||
token=token,
|
||||
)
|
||||
)
|
||||
if direct_complete["result"][0] != 202:
|
||||
raise SystemExit("FAIL complete %s %s" % (label, direct_complete["result"][0]))
|
||||
rows.append(
|
||||
{
|
||||
"label": label,
|
||||
"bytes": len(image),
|
||||
"proxy_rx_bytes": proxy["rx_bytes"],
|
||||
"proxy_peak_api_mib": proxy["peak_api_mib"],
|
||||
"direct_put_rx_bytes": direct_put["rx_bytes"],
|
||||
"direct_put_peak_api_mib": direct_put["peak_api_mib"],
|
||||
"complete_rx_bytes": direct_complete["rx_bytes"],
|
||||
"complete_peak_api_mib": direct_complete["peak_api_mib"],
|
||||
}
|
||||
)
|
||||
print("MEASURED", rows[-1])
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
api = Api()
|
||||
|
||||
def register(label):
|
||||
username = "t1_" + label + "_" + uuid.uuid4().hex[:10]
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
status, body, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
{"username": username, "display_name": "Track " + label, "password": password},
|
||||
)
|
||||
expect(201, status, "register " + label)
|
||||
return username, password, body
|
||||
|
||||
def login(username, password):
|
||||
basic = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
request = urllib.request.Request(
|
||||
BASE + "/api/v1/auth/sessions",
|
||||
method="POST",
|
||||
headers={"Authorization": "Basic " + basic},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
return json.loads(response.read())["token"]
|
||||
|
||||
alice_name, alice_password, alice = register("alice")
|
||||
bob_name, bob_password, bob = register("bob")
|
||||
eve_name, eve_password, eve = register("eve")
|
||||
alice_token = login(alice_name, alice_password)
|
||||
bob_token = login(bob_name, bob_password)
|
||||
eve_token = login(eve_name, eve_password)
|
||||
status, chat, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
{"title": "track1", "member_ids": [alice["id"], bob["id"]]},
|
||||
token=alice_token,
|
||||
)
|
||||
expect(201, status, "chat")
|
||||
negatives(api, alice_token, bob_token, eve_token, chat["id"])
|
||||
rows = compare(api, alice_token, chat["id"])
|
||||
summary = {
|
||||
"hypothesis": "A presigned PUT does not enter the API, so API RX stays near zero at both sizes. complete reads the object once to check the checksum, and the proxy endpoint still receives the whole body.",
|
||||
"presign_ttl_seconds": 60,
|
||||
"sizes": rows,
|
||||
}
|
||||
OUT.parent.mkdir(exist_ok=True)
|
||||
OUT.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(summary, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 2: versioned pipeline under duplicate, reorder, crash and a stale job.
|
||||
|
||||
Prints statuses and attachment ids only. Tokens stay in memory.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "evidence" / "track2.json"
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
|
||||
|
||||
|
||||
def compose(*args: str) -> None:
|
||||
command = ["docker", "compose", "--env-file", ".env"]
|
||||
extra = os.environ.get("LAB5_COMPOSE_FILES", "")
|
||||
for name in extra.split(os.pathsep):
|
||||
if name:
|
||||
command.extend(["-f", name])
|
||||
subprocess.run([*command, *args], cwd=ROOT, check=True)
|
||||
|
||||
|
||||
class Api:
|
||||
def __init__(self) -> None:
|
||||
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
|
||||
def call(self, method, path, body=None, token=None, raw=None, content_type=None, headers=None):
|
||||
hdrs = dict(headers or {})
|
||||
data = raw
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
if content_type:
|
||||
hdrs["Content-Type"] = content_type
|
||||
if token:
|
||||
hdrs["Authorization"] = "Bearer " + token
|
||||
request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method)
|
||||
try:
|
||||
response = self.opener.open(request, timeout=30)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
payload = response.read()
|
||||
parsed = json.loads(payload) if payload and response.headers.get_content_type() == "application/json" else None
|
||||
return response.status, parsed, payload
|
||||
|
||||
|
||||
def expect(status, got, label):
|
||||
if got != status:
|
||||
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
|
||||
|
||||
|
||||
def wait_applied(api: Api, attachment_id: str, token: str, version: int, seconds: int = 150, allow_failed: bool = False):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
|
||||
expect(200, status, "poll")
|
||||
state = jobs(api, attachment_id)
|
||||
if body["status"] == "failed" and allow_failed:
|
||||
return body
|
||||
if (
|
||||
body["status"] == "ready"
|
||||
and state["applied_pipeline_version"] == version
|
||||
and any(item["name"] == "thumbnail" for item in body["variants"])
|
||||
):
|
||||
return body
|
||||
time.sleep(0.4)
|
||||
raise SystemExit("FAIL timeout waiting for pipeline %s" % version)
|
||||
deadline = time.monotonic() + seconds
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
|
||||
expect(200, status, "poll")
|
||||
last = body
|
||||
if body["status"] == "ready" and any(item["name"] == "thumbnail" for item in body["variants"]):
|
||||
return body
|
||||
if body["status"] == "failed":
|
||||
return body
|
||||
time.sleep(0.4)
|
||||
raise SystemExit("FAIL timeout last=%s" % (last or {}).get("status"))
|
||||
|
||||
|
||||
def jobs(api: Api, attachment_id: str):
|
||||
status, body, _ = api.call("GET", "/api/v1/_lab/attachments/%s/jobs" % attachment_id)
|
||||
expect(200, status, "jobs")
|
||||
return body
|
||||
|
||||
|
||||
def thumb(body):
|
||||
found = [item for item in body["variants"] if item["name"] == "thumbnail"]
|
||||
if len(found) != 1:
|
||||
raise SystemExit("FAIL thumbnail count %s" % len(found))
|
||||
return found[0]["sha256"]
|
||||
|
||||
|
||||
def user(api: Api):
|
||||
username = "t2_" + uuid.uuid4().hex[:10]
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
status, body, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
{"username": username, "display_name": "Track2", "password": password},
|
||||
)
|
||||
expect(201, status, "register")
|
||||
basic = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
status, session, _ = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
||||
expect(201, status, "login")
|
||||
return body["id"], session["token"]
|
||||
|
||||
|
||||
def upload(api: Api, chat_id: str, token: str, filename: str):
|
||||
status, body, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats/%s/attachments?filename=%s" % (chat_id, filename),
|
||||
token=token,
|
||||
raw=FIXTURE.read_bytes(),
|
||||
content_type="image/png",
|
||||
)
|
||||
expect(202, status, "upload " + filename)
|
||||
return body
|
||||
|
||||
|
||||
def main() -> None:
|
||||
api = Api()
|
||||
content = FIXTURE.read_bytes()
|
||||
original = hashlib.sha256(content).hexdigest()
|
||||
alice, token = user(api)
|
||||
status, chat, _ = api.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
{"title": "track2", "member_ids": [alice]},
|
||||
token=token,
|
||||
)
|
||||
expect(201, status, "chat")
|
||||
chat_id = chat["id"]
|
||||
report = {"hypothesis": "at-least-once plus a versioned effect key keeps the newest committed pipeline", "cases": []}
|
||||
|
||||
first = upload(api, chat_id, token, "v1.png")
|
||||
ready = wait_applied(api, first["id"], token, 1)
|
||||
sha_v1 = thumb(ready)
|
||||
state = jobs(api, first["id"])
|
||||
v1_job = state["jobs"][0]["job_id"]
|
||||
expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "duplicate 1")
|
||||
expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "duplicate 2")
|
||||
time.sleep(2)
|
||||
again = wait_applied(api, first["id"], token, 1)
|
||||
state = jobs(api, first["id"])
|
||||
v1_rows = [item for item in state["variants"] if item["pipeline_version"] == 1]
|
||||
if thumb(again) != sha_v1 or len(v1_rows) != 3:
|
||||
raise SystemExit("FAIL duplicate changed the effect")
|
||||
report["cases"].append({"name": "duplicate", "attachment_id": first["id"], "thumbnail_sha256": sha_v1, "variant_rows": len(v1_rows)})
|
||||
|
||||
expect(202, api.call("POST", "/api/v1/_lab/attachments/%s/pipelines" % first["id"])[0], "bump")
|
||||
bumped = wait_applied(api, first["id"], token, 2)
|
||||
sha_v2 = thumb(bumped)
|
||||
state = jobs(api, first["id"])
|
||||
if sha_v2 == sha_v1 or state["applied_pipeline_version"] != 2:
|
||||
raise SystemExit("FAIL version 2 did not replace the served bytes")
|
||||
v1_job = [item["job_id"] for item in state["jobs"] if item["pipeline_version"] == 1][0]
|
||||
expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "stale redelivery")
|
||||
time.sleep(2)
|
||||
after_stale = wait_applied(api, first["id"], token, 2)
|
||||
state = jobs(api, first["id"])
|
||||
if thumb(after_stale) != sha_v2 or state["applied_pipeline_version"] != 2:
|
||||
raise SystemExit("FAIL stale job overwrote version 2")
|
||||
report["cases"].append(
|
||||
{
|
||||
"name": "reorder_and_stale_version",
|
||||
"attachment_id": first["id"],
|
||||
"version1_sha256": sha_v1,
|
||||
"version2_sha256": sha_v2,
|
||||
"applied_pipeline_version": state["applied_pipeline_version"],
|
||||
"variant_rows": len(state["variants"]),
|
||||
}
|
||||
)
|
||||
|
||||
def crashed(mode: str):
|
||||
expect(200, api.call("POST", "/api/v1/_lab/faults", {"mode": mode, "remaining": 1})[0], mode)
|
||||
created = upload(api, chat_id, token, mode + ".png")
|
||||
# The worker process exits. Docker restarts it; the lease must expire before retry.
|
||||
body = wait_applied(api, created["id"], token, 1, seconds=180)
|
||||
if body["status"] != "ready":
|
||||
raise SystemExit("FAIL %s ended %s" % (mode, body["status"]))
|
||||
status_code, _, payload = api.call("GET", "/api/v1/attachments/%s/content" % created["id"], token=token)
|
||||
expect(200, status_code, mode + " original")
|
||||
if hashlib.sha256(payload).hexdigest() != original:
|
||||
raise SystemExit("FAIL %s lost the original" % mode)
|
||||
state_now = jobs(api, created["id"])
|
||||
rows = [item for item in state_now["variants"] if item["pipeline_version"] == 1]
|
||||
if len(rows) != 3:
|
||||
raise SystemExit("FAIL %s variant rows %s" % (mode, len(rows)))
|
||||
report["cases"].append(
|
||||
{
|
||||
"name": mode,
|
||||
"attachment_id": created["id"],
|
||||
"original_sha256": original,
|
||||
"variant_rows": len(rows),
|
||||
"thumbnail_sha256": thumb(body),
|
||||
}
|
||||
)
|
||||
|
||||
crashed("crash_before_s3")
|
||||
crashed("crash_after_s3")
|
||||
|
||||
expect(200, api.call("POST", "/api/v1/_lab/faults", {"mode": "fail_once", "remaining": 1})[0], "fail_once")
|
||||
failed = upload(api, chat_id, token, "fail.png")
|
||||
failed_body = wait_applied(api, failed["id"], token, 1, allow_failed=True)
|
||||
if failed_body["status"] != "failed" or failed_body["error_code"] != "processing_failed":
|
||||
raise SystemExit("FAIL expected a terminal processing_failed")
|
||||
expect(200, api.call("POST", "/api/v1/_lab/attachments/%s/replay" % failed["id"])[0], "replay")
|
||||
restored = wait_applied(api, failed["id"], token, 1)
|
||||
if restored["status"] != "ready":
|
||||
raise SystemExit("FAIL replay did not reach ready")
|
||||
state = jobs(api, failed["id"])
|
||||
replay_job = state["jobs"][0]["job_id"]
|
||||
expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % replay_job)[0], "replay duplicate")
|
||||
time.sleep(2)
|
||||
state = jobs(api, failed["id"])
|
||||
if len(state["variants"]) != 3 or state["jobs"][0]["status"] != "ready":
|
||||
raise SystemExit("FAIL replay produced another effect")
|
||||
report["cases"].append(
|
||||
{
|
||||
"name": "replay_then_duplicate",
|
||||
"attachment_id": failed["id"],
|
||||
"variant_rows": len(state["variants"]),
|
||||
"job_status": state["jobs"][0]["status"],
|
||||
}
|
||||
)
|
||||
|
||||
report["not_exactly_once"] = (
|
||||
"The broker delivers a message more than once. A crash after the object PUT and before ack "
|
||||
"runs the S3 write again. The database commit of a pipeline version happens once, and an older "
|
||||
"version cannot replace the applied one. That is idempotent effect, not exactly-once delivery."
|
||||
)
|
||||
OUT.parent.mkdir(exist_ok=True)
|
||||
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print("PASS track2", OUT)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user