diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..db3469e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.11.7-slim-bookworm AS builder + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir --prefix=/install -r requirements.txt + +FROM python:3.11.7-slim-bookworm AS runner + +RUN useradd -u 10001 -m appuser +WORKDIR /home/appuser + +COPY --from=builder /install /usr/local +COPY app.py worker.py ./ + +USER appuser +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')" + +ENV PYTHONUNBUFFERED=1 +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "10"] \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index b97ae42..2827d03 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,108 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Бондаренко Мария, БПИ2302 +- Целевая оценка: 4 -## Запуск из чистого clone +## 1. Архитектура и стек наблюдаемости -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +В рамках лабораторной работы для распределённого сервиса мессенджера развёрнут единый контур наблюдаемости на базе OpenTelemetry и стека LGTM/Prometheus: -## Архитектура и решения +* Сбор и маршрутизация телеметрии — OpenTelemetry Collector (otel-collector:4317/4318): + * принимает сигналы по протоколу OTLP (gRPC и HTTP/protobuf) от messenger-api и messenger-worker; + * применяет процессоры memory_limiter (ограничение памяти 256 MiB) и batch (пакетная отправка по 256 записей или каждые 2 секунды); + * маршрутизирует данные в три независимых хранилища с использованием очередей отправки: + * Метрики -> prometheusremotewrite; + * Трейсы -> otlphttp/tempo; + * Логи -> otlphttp/loki. +* Хранилище метрик и правила алертинга — Prometheus: + * запущен с флагом --web.enable-remote-write-receiver и ретеншеном 7 дней; + * оценивает правила оповещений из /etc/prometheus/alerts.yml с интервалом 15s. +* Хранилище структурированных логов — Grafana Loki: + * использует схему хранения tsdb и локальное файловое хранилище; + * индексирует атрибуты ресурса service.name, service.namespace, deployment.environment, а также поддерживает allow_structured_metadata: true. +* Хранилище распределённых трассировок — Grafana Tempo: + * принимает OTLP HTTP трейсы на порту 4318, хранит блоки локально с периодом хранения 168 часов. +* Визуализация и корреляция — Grafana: + * настроена через provisioning (datasources.yml и dashboards.yml); + * реализована двусторонняя связка Logs <-Traces: + * в источнике Loki настроено derivedFields по регулярному выражению "trace_id":"([0-9a-f]{32})" для перехода из строки лога в соответствующий трейс в Tempo; + * в источнике Tempo настроено tracesToLogsV2 с фильтрацией по TraceID и тегу service.name. -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +--- -## Доказательства критериев +## 2. Инструментирование сервисов -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +### 2.1. Сервис messenger-api (app.py) +* Инициализированы TracerProvider, MeterProvider (экспорт каждые 5000 ms) и LoggerProvider через OTLP HTTP экспортеры. +* Подключено автоматическое инструментирование FastAPI и кастомный HTTP middleware telemetry_middleware, фиксирующий низкокардинальные атрибуты. +* Реализовано структурированное JSON-логирование, автоматически извлекающее активный контекст OpenTelemetry и добавляющее в каждую запись поля trace_id (32 hex-символа) и span_id (16 hex-символов), а также контекст запроса (method, path, status_code). +* Реализован потокобезопасный пул соединений к PostgreSQL с корректным возвратом соединений в блоках finally. +* При постановке задачи обработки вложения в таблицу outbox вызывается inject(job_payload), что сохраняет traceparent в JSON-полезной нагрузке задачи. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +### 2.2. Асинхронный обработчик messenger-worker (worker.py) +* Настроен экспорт метрик, логов и спанов с именем сервиса messenger-worker. +* При чтении задачи из Redis Stream (attachment_jobs) воркер извлекает контекст трассировки с помощью extract(job) и открывает дочерний спан worker.process_attachment с атрибутами job.pipeline_version, job.attempt, job.outcome. +* Реализован конвейер генерации трёх вариантов изображений (thumbnail 256px JPEG, optimized 1280px WebP, crop 128x128 JPEG), контроль таймаута (JOB_TIMEOUT_SECONDS=30), экспоненциальный backoff при временных сбоях и перенос исчерпавших попытки задач в Dead Letter Queue (attachment_jobs_dlq). -## Эксперименты и отказы +--- -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## 3. Реализованные метрики, дашборды и алерты -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +### 3.1. Набор метрик +1. Метрики API (RED и зависимости): + * http_server_requests_total (Counter) — общее число HTTP-запросов по методу, маршруту и коду ответа; + * http_server_errors_total (Counter) — количество ответов со статусом 4xx и 5xx; + * http_server_request_duration_seconds (Histogram) — распределение длительности обработки HTTP-запросов; + * http_server_dependency_errors_total (Counter) — счётчик ошибок внешних зависимостей (postgres, redis, s3, outbox, readiness); + * db_pool_connections (ObservableGauge) — состояние соединений пула PostgreSQL в разрезе state="used" и state="idle". +2. Метрики Worker (очередь, насыщение, длительность): + * worker_jobs_total (Counter) — количество завершённых задач по статусу (success, failed); + * worker_job_duration_seconds (Histogram) — время обработки задачи воркером; + * worker_jobs_in_progress (UpDownCounter) — число задач, обрабатываемых в текущий момент (saturation); + * worker_job_retries_total (Counter) — число запланированных повторных попыток по типу исключения (reason); + * worker_queue_depth (ObservableGauge) — текущая глубина очереди Redis Stream attachment_jobs; + * worker_oldest_job_age_seconds (ObservableGauge) — возраст самой старой необработанной задачи в очереди. -## Ограничения и следующие шаги +### 3.2. Дашборды Grafana +Подготовлены и автоматически загружены два дашборда в папку Messenger: +* MTUSI Messenger — Lab 6 (Grade 3) (mtusi-messenger-lab6-g3): базовый мониторинг RPS, частоты ошибок, p95 задержки API, скорости и p95 длительности задач воркера, а также поток структурированных логов из Loki. +* MTUSI Messenger — Lab 6 (Grade 4) (mtusi-messenger-lab6-g4): расширенный операционный дашборд, включающий: + * API RPS (req/s) и API Error Ratio (%); + * API Latency p50 / p95 / p99 (s); + * DB Connection Pool (used / idle); + * Queue Depth & Oldest Job Age; + * Worker Saturation & Retries; + * Worker Job Duration p50 / p95 / p99 (s); + * панель Logs with TraceID (Loki) для быстрой навигации в Tempo. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +### 3.3. Правила алертинга Prometheus (alerts.yml) +Настроены две группы правил с интервалом оценки 15s: +* MessengerApiHighErrorRate (warning) — доля ошибок API превышает 5% в течение 30s при нагрузке > 0.2 req/s. +* MessengerApiDependencyErrors (critical) — зафиксированы ошибки обращения к PostgreSQL, Redis, S3 или в работе outbox в течение 30s. +* MessengerApiHighLatencyP95 (warning) — 95-й перцентиль времени ответа API превышает 1.0s в течение 1m. +* MessengerWorkerJobFailures (warning) — появление задач воркера со статусом failed в течение 30s. +* MessengerWorkerQueueLag (critical) — возраст самой старой задачи превышает 30s или глубина очереди превышает 20 сообщений в течение 30s. + +--- + +## 4. Проверка работоспособности и сквозной корреляции + +1. Проверка статуса компонентов и приёма телеметрии: + * Все контейнеры инфраструктуры (postgres, redis, s3, prometheus, loki, tempo, otel-collector, grafana) и приложения (api-1, api-2, worker, nginx) запущены и проходят проверки healthcheck. + * В Prometheus подтверждено поступление временных рядов от otel-collector через Remote Write и активный статус правил в разделе /alerts. +2. Сквозной пользовательский сценарий с вложением: + * Выполнена регистрация пользователя (POST /api/v1/users), создание сессии (POST /api/v1/auth/sessions), создание чата (POST /api/v1/chats) и загрузка изображения (POST /api/v1/chats/{chat_id}/attachments). + * Воркер забрал задачу из потока attachment_jobs, сгенерировал варианты thumbnail, optimized, crop, сохранил их в бакет messenger-attachments в S3 и перевёл статус вложения в ready. +3. Проверка связки Metrics -> Logs -> Traces: + * По всплеску на графиках Grafana открыта панель логов Loki, где для HTTP-запроса загрузки вложения зафиксирован JSON-лог с полем trace_id. + * Переход по ссылке TraceID в Tempo отобразил единый распределённый трейс, начинающийся со спана POST /api/v1/chats/{chat_id}/attachments в сервисе messenger-api и продолжающийся дочерним спаном worker.process_attachment в сервисе messenger-worker. + +--- + +## 5. Диагностика типовых инцидентов по данным телеметрии + +| Сценарий инцидента | Проявление на дашборде и в алертах | Локализация в Loki и Tempo | +| :--- | :--- | :--- | +| Остановка или деградация worker | Рост worker_queue_depth и линейный рост worker_oldest_job_age_seconds; срабатывание критического алерта MessengerWorkerQueueLag. | В логах messenger-api задачи успешно записываются в outbox и Redis Stream, однако спаны worker.process_attachment в Tempo отсутствуют до момента перезапуска воркера, после чего накопившаяся очередь дренируется. | +| Недоступность S3 (SeaweedFS) при загрузке | Всплеск API Error Ratio (%) (ответы 503), рост http_server_dependency_errors_total{component="s3"}, срабатывание MessengerApiDependencyErrors и MessengerApiHighErrorRate. | Поиск в Loki по status_code=503 выявляет логи с trace_id; в Tempo трейс обрывается на спане загрузки в S3 без постановки задачи в outbox. | +| Загрузка повреждённого файла изображения | Воркер фиксирует неудачную задачу (worker_jobs_total{status="failed"}), срабатывает алерт MessengerWorkerJobFailures без повторных ретраев (перманентная ошибка). | В логах messenger-worker фиксируется запись "Permanent image error" с error_code="invalid_image" и trace_id, позволяющая отследить исходный запрос загрузки в messenger-api. | \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000..9c09d2c --- /dev/null +++ b/app.py @@ -0,0 +1,1331 @@ +import os +import time +import base64 +import secrets +import uuid +import json +import re +import struct +import hashlib +import hmac +import logging +import threading +from datetime import datetime, timezone, timedelta +from typing import List, Optional +from uuid import UUID + +import psycopg2 +import psycopg2.extras +from psycopg2.pool import ThreadedConnectionPool +psycopg2.extras.register_uuid() +import bcrypt +from fastapi import FastAPI, Header, HTTPException, Query, Response, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from pydantic import BaseModel, Field, field_validator, model_validator, ConfigDict + +import redis +import boto3 +from botocore.client import Config + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.propagate import inject + + +class JsonLogFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + span = trace.get_current_span() + ctx = span.get_span_context() if span else None + trace_id = format(ctx.trace_id, "032x") if ctx and ctx.is_valid else None + span_id = format(ctx.span_id, "016x") if ctx and ctx.is_valid else None + + payload = { + "ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "level": record.levelname, + "logger": record.name, + "service": os.getenv("OTEL_SERVICE_NAME", "messenger-api"), + "msg": record.getMessage(), + } + if trace_id and span_id: + payload["trace_id"] = trace_id + payload["span_id"] = span_id + + for attr in ("method", "path", "status_code", "job_id", "attachment_id", "job_status", "attempt", "error_code"): + if hasattr(record, attr): + payload[attr] = getattr(record, attr) + + if record.exc_info: + payload["exception"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception" + + return json.dumps(payload, ensure_ascii=False) + + +def setup_api_telemetry(): + service_name = os.getenv("OTEL_SERVICE_NAME", "messenger-api") + endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/") + env_attrs = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=lab") + + attrs = {"service.name": service_name} + for part in env_attrs.split(","): + if "=" in part: + k, v = part.split("=", 1) + attrs[k.strip()] = v.strip() + + resource = Resource.create(attrs) + + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")) + ) + trace.set_tracer_provider(tracer_provider) + + metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"), + export_interval_millis=5000, + ) + meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) + metrics.set_meter_provider(meter_provider) + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs")) + ) + set_logger_provider(logger_provider) + + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + root_logger.handlers.clear() + + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(JsonLogFormatter()) + root_logger.addHandler(stream_handler) + + otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + otel_handler.setFormatter(JsonLogFormatter()) + root_logger.addHandler(otel_handler) + + return tracer_provider, trace.get_tracer(service_name), metrics.get_meter(service_name) + + +app = FastAPI(title="MTUSI Messenger — lab6", version="1.6.0") + +_tracer_provider, tracer, meter = setup_api_telemetry() +FastAPIInstrumentor.instrument_app(app, tracer_provider=_tracer_provider) +api_logger = logging.getLogger("messenger-api") + +api_requests_counter = meter.create_counter( + name="http_server_requests_total", + description="Total HTTP requests served by API", + unit="1", +) +api_errors_counter = meter.create_counter( + name="http_server_errors_total", + description="Total HTTP 4xx/5xx responses served by API", + unit="1", +) +api_dependency_errors_counter = meter.create_counter( + name="http_server_dependency_errors_total", + description="Dependency and internal errors in API (DB/Redis/S3)", + unit="1", +) +api_duration_histogram = meter.create_histogram( + name="http_server_request_duration_seconds", + description="HTTP request duration in seconds", + unit="s", +) + +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +INSTANCE_ID = os.getenv("INSTANCE_ID", "instance-1") + +REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") +redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True) + +SECRET_HMAC_KEY = os.environ["SECRET_HMAC_KEY"] + +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser") +S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword") + +STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs") + +s3_client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + config=Config(signature_version="s3v4"), + region_name="us-east-1" +) + +db_pool_lock = threading.Lock() +db_pool: Optional[ThreadedConnectionPool] = None +_active_db_conns = 0 + + +def init_db_pool(minconn: int = 1, maxconn: int = 15): + global db_pool + if db_pool is None: + db_pool = ThreadedConnectionPool( + minconn=minconn, + maxconn=maxconn, + dsn=DATABASE_URL, + cursor_factory=psycopg2.extras.RealDictCursor, + ) + + +def acquire_db_conn(): + global _active_db_conns + if db_pool is None: + init_db_pool() + conn = db_pool.getconn() + with db_pool_lock: + _active_db_conns += 1 + return conn + + +def release_db_conn(conn): + global _active_db_conns + if conn is None or db_pool is None: + return + try: + close_broken = bool(getattr(conn, "closed", 0)) + db_pool.putconn(conn, close=close_broken) + finally: + with db_pool_lock: + _active_db_conns = max(0, _active_db_conns - 1) + + +def observe_db_pool(options): + max_c = db_pool.maxconn if db_pool else 0 + with db_pool_lock: + used = min(_active_db_conns, max_c) + idle = max(0, max_c - used) + yield metrics.Observation(value=used, attributes={"state": "used", "instance": INSTANCE_ID}) + yield metrics.Observation(value=idle, attributes={"state": "idle", "instance": INSTANCE_ID}) + + +meter.create_observable_gauge( + name="db_pool_connections", + callbacks=[observe_db_pool], + description="PostgreSQL connection pool connections by state", + unit="1", +) + + +def outbox_publisher(): + while True: + conn = None + try: + conn = acquire_db_conn() + with conn.cursor() as cur: + cur.execute(""" + SELECT id, stream, payload FROM outbox + WHERE published_at IS NULL + ORDER BY created_at + LIMIT 100 + FOR UPDATE SKIP LOCKED + """) + for row in cur.fetchall(): + redis_client.xadd(row["stream"], {"payload": json.dumps(row["payload"])}) + cur.execute("UPDATE outbox SET published_at=NOW() WHERE id=%s", (row["id"],)) + conn.commit() + except Exception as e: + if conn and not conn.closed: + try: + conn.rollback() + except Exception: + pass + api_dependency_errors_counter.add(1, {"component": "outbox"}) + api_logger.error("Outbox publisher error: %s", type(e).__name__) + finally: + release_db_conn(conn) + time.sleep(1) + + +@app.on_event("startup") +def _start_services(): + try: + init_db_pool(minconn=1, maxconn=15) + except Exception as e: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + api_logger.error("DB pool init error: %s", type(e).__name__) + threading.Thread(target=outbox_publisher, daemon=True).start() + + +@app.on_event("shutdown") +def _shutdown_services(): + if db_pool is not None: + try: + db_pool.closeall() + except Exception: + pass + + +@app.middleware("http") +async def telemetry_middleware(request: Request, call_next): + start = time.monotonic() + route_obj = request.scope.get("route") + route_path = getattr(route_obj, "path", request.url.path) + method = request.method + status_code = 500 + try: + response = await call_next(request) + status_code = response.status_code + return response + finally: + elapsed = time.monotonic() - start + low_card_attrs = { + "http.method": method, + "http.route": route_path, + "http.status_code": status_code, + } + api_requests_counter.add(1, low_card_attrs) + api_duration_histogram.record(elapsed, low_card_attrs) + if status_code >= 400: + api_errors_counter.add(1, low_card_attrs) + api_logger.info( + "HTTP request completed", + extra={ + "method": method, + "path": route_path, + "status_code": status_code, + }, + ) + + +def generate_request_id() -> str: + return uuid.uuid4().hex + + +def make_error_response( + status_code: int, + code: str, + message: str, + req_id: str, + extra_headers: Optional[dict] = None, +) -> JSONResponse: + headers = {"X-Request-Id": req_id, "X-Instance-Id": INSTANCE_ID} + if extra_headers: + headers.update(extra_headers) + return JSONResponse( + status_code=status_code, + headers=headers, + content={ + "error": { + "code": code, + "message": message, + "request_id": req_id, + } + }, + ) + + +def encode_cursor(chat_id: str, created_at: str, msg_id: str) -> str: + payload = f"{chat_id}|{created_at}|{msg_id}" + signature = hmac.new(SECRET_HMAC_KEY.encode(), payload.encode(), hashlib.sha256).hexdigest()[:16] + raw = f"{payload}|{signature}" + return base64.urlsafe_b64encode(raw.encode()).decode() + + +def decode_cursor(cursor: str, expected_chat_id: str) -> tuple: + try: + decoded = base64.urlsafe_b64decode(cursor.encode()).decode() + parts = decoded.split("|") + if len(parts) != 4: + raise ValueError() + chat_id, created_at, msg_id, signature = parts + + if chat_id != expected_chat_id: + raise HTTPException(status_code=400, detail="Cursor belongs to another chat") + + expected_sig = hmac.new( + SECRET_HMAC_KEY.encode(), + f"{chat_id}|{created_at}|{msg_id}".encode(), + hashlib.sha256 + ).hexdigest()[:16] + if not hmac.compare_digest(signature, expected_sig): + raise ValueError() + + return created_at, msg_id + except HTTPException: + raise + except Exception: + raise HTTPException(status_code=400, detail="Invalid cursor format") + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + req_id = request.headers.get("X-Request-Id") or generate_request_id() + return make_error_response( + status_code=400, + code="invalid_request", + message="Validation error in request body or parameters", + req_id=req_id + ) + + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + req_id = request.headers.get("X-Request-Id") or generate_request_id() + headers = {"X-Request-Id": req_id, "X-Instance-Id": INSTANCE_ID} + if exc.headers: + headers.update(exc.headers) + + detail = exc.detail + if isinstance(detail, dict) and "error" in detail: + detail["error"]["request_id"] = req_id + return JSONResponse(status_code=exc.status_code, headers=headers, content=detail) + + code_map = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + 500: "internal_error", + } + return JSONResponse( + status_code=exc.status_code, + headers=headers, + content={ + "error": { + "code": code_map.get(exc.status_code, "internal_error"), + "message": str(detail), + "request_id": req_id, + } + }, + ) + + +@app.exception_handler(Exception) +async def unhandled_exception_handler(request: Request, exc: Exception): + req_id = request.headers.get("X-Request-Id") or generate_request_id() + api_dependency_errors_counter.add(1, {"component": "unhandled"}) + return make_error_response( + status_code=500, + code="internal_error", + message="Internal server error", + req_id=req_id, + ) + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra='forbid') + username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(..., min_length=1, max_length=100) + password: str = Field(..., min_length=12, max_length=128) + + @field_validator("display_name", "password") + def not_blank(cls, v): + if not v.strip(): + raise ValueError("Must not be blank") + return v + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra='forbid') + title: str = Field(..., min_length=1, max_length=100) + member_ids: List[uuid.UUID] = Field(..., min_length=1, max_length=100) + + @field_validator("title") + def not_blank(cls, v): + if not v.strip(): + raise ValueError("Must not be blank") + return v + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra='forbid') + text: str = Field(default="", max_length=4000) + attachment_ids: List[uuid.UUID] = Field(default=[], max_length=10) + + @model_validator(mode="after") + def check_content(self) -> 'CreateMessage': + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Message must contain either text or attachments") + return self + + +@app.get("/health/live") +def health_live(response: Response): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return {"status": "ok"} + + +@app.get("/health/ready") +def health_ready(response: Response): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + conn = None + try: + conn = acquire_db_conn() + with conn.cursor() as cur: + cur.execute("SELECT 1") + cur.fetchone() + redis_client.ping() + s3_client.head_bucket(Bucket=S3_BUCKET) + except Exception: + api_dependency_errors_counter.add(1, {"component": "readiness"}) + raise HTTPException(status_code=503, detail="Database, Redis or S3 unavailable") + finally: + release_db_conn(conn) + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +def register_user(body: CreateUser, response: Response): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + + password_hash = bcrypt.hashpw(body.password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8') + user_id = str(uuid.uuid4()) + created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO users (id, username, display_name, password_hash, created_at) VALUES (%s, %s, %s, %s, %s)", + (user_id, body.username, body.display_name, password_hash, created_at) + ) + conn.commit() + except psycopg2.errors.UniqueViolation: + conn.rollback() + raise HTTPException( + status_code=409, + detail={"error": {"code": "conflict", "message": "Username already taken", "request_id": req_id}} + ) + except Exception: + conn.rollback() + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) + + return { + "id": user_id, + "username": body.username, + "display_name": body.display_name, + "created_at": created_at + } + + +@app.post("/api/v1/auth/sessions", status_code=201) +def create_session(authorization: Optional[str] = Header(None), response: Response = None): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + + if not authorization or not authorization.startswith("Basic "): + raise HTTPException( + status_code=401, + detail="Missing or invalid basic auth", + headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} + ) + + try: + encoded = authorization.split(" ")[1] + decoded = base64.b64decode(encoded).decode("utf-8") + username, password = decoded.split(":", 1) + except Exception: + raise HTTPException( + status_code=401, + detail="Invalid credentials format", + headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} + ) + + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute("SELECT * FROM users WHERE username = %s", (username,)) + user = cur.fetchone() + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) + + if not user or not bcrypt.checkpw(password.encode('utf-8'), user['password_hash'].encode('utf-8')): + raise HTTPException( + status_code=401, + detail="Invalid credentials", + headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} + ) + + token = secrets.token_urlsafe(48) + try: + redis_client.setex(f"session:{token}", SESSION_TTL_SECONDS, str(user['id'])) + except Exception: + api_dependency_errors_counter.add(1, {"component": "redis"}) + raise HTTPException(status_code=503, detail="Session store unavailable") + + expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS) + expires_at_str = expires_at.isoformat().replace("+00:00", "Z") + + return { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at_str, + "user": { + "id": str(user['id']), + "username": user['username'], + "display_name": user['display_name'], + "created_at": user['created_at'].isoformat().replace("+00:00", "Z") if isinstance(user['created_at'], datetime) else user['created_at'] + } + } + + +def get_current_user_from_token(authorization: Optional[str] = Header(None)) -> dict: + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException( + status_code=401, + detail="Missing or invalid bearer token", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'} + ) + token = authorization.split(" ")[1] + + try: + user_id = redis_client.get(f"session:{token}") + except Exception: + api_dependency_errors_counter.add(1, {"component": "redis"}) + raise HTTPException(status_code=503, detail="Session store unavailable") + + if not user_id: + raise HTTPException( + status_code=401, + detail="Invalid or expired session token", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'} + ) + + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute("SELECT * FROM users WHERE id = %s", (user_id,)) + row = cur.fetchone() + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) + + if not row: + raise HTTPException( + status_code=401, + detail="User not found", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'} + ) + + return { + "id": str(row['id']), + "username": row['username'], + "display_name": row['display_name'], + "created_at": row['created_at'].isoformat().replace("+00:00", "Z") if isinstance(row['created_at'], datetime) else row['created_at'] + } + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def revoke_session(authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException( + status_code=401, + detail="Unauthorized", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'} + ) + + token = authorization.split(" ")[1] + try: + deleted = redis_client.delete(f"session:{token}") + except Exception: + api_dependency_errors_counter.add(1, {"component": "redis"}) + raise HTTPException(status_code=503, detail="Session store unavailable") + + if not deleted: + raise HTTPException( + status_code=401, + detail="Invalid session token", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'} + ) + + return Response( + status_code=204, + headers={ + "X-Request-Id": req_id, + "X-Instance-Id": INSTANCE_ID + }, + ) + + +@app.get("/api/v1/users/me") +def current_user(response: Response, authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return get_current_user_from_token(authorization) + + +@app.post("/api/v1/chats", status_code=201) +def create_chat(body: CreateChat, response: Response, authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + + member_ids_str = [str(mid) for mid in body.member_ids] + + if user["id"] not in member_ids_str: + raise HTTPException(status_code=400, detail="Caller must be in member_ids") + if len(member_ids_str) != len(set(member_ids_str)): + raise HTTPException(status_code=400, detail="Duplicate member IDs") + + chat_id = str(uuid.uuid4()) + created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + for mid_str in member_ids_str: + cur.execute("SELECT id FROM users WHERE id = %s", (mid_str,)) + if not cur.fetchone(): + raise HTTPException(status_code=400, detail=f"User {mid_str} not found") + + cur.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", + (chat_id, body.title, created_at) + ) + for mid_str in member_ids_str: + cur.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + (chat_id, mid_str) + ) + conn.commit() + except HTTPException: + conn.rollback() + raise + except Exception: + conn.rollback() + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=500, detail="Internal error creating chat") + finally: + release_db_conn(conn) + + return { + "id": chat_id, + "title": body.title, + "member_ids": member_ids_str, + "created_at": created_at + } + + +@app.get("/api/v1/chats") +def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response = None, authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + + if limit < 1 or limit > 100: + raise HTTPException(status_code=400, detail="Invalid limit") + + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute(""" + SELECT c.id, c.title, c.created_at FROM chats c + JOIN chat_members cm ON c.id = cm.chat_id + WHERE cm.user_id = %s + ORDER BY c.created_at ASC, c.id ASC + """, (user["id"],)) + rows = cur.fetchall() + + user_chats = [] + for r in rows: + cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (r['id'],)) + m_rows = cur.fetchall() + user_chats.append({ + "id": str(r['id']), + "title": r['title'], + "member_ids": [str(m['user_id']) for m in m_rows], + "created_at": r['created_at'].isoformat().replace("+00:00", "Z") if isinstance(r['created_at'], datetime) else r['created_at'] + }) + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) + + start_idx = 0 + if cursor: + try: + parts = cursor.split("_") + c_time, c_id = parts[0], parts[1] + found = False + for idx, c in enumerate(user_chats): + if (c["created_at"], c["id"]) >= (c_time, c_id): + start_idx = idx if (c["created_at"], c["id"]) > (c_time, c_id) else idx + 1 + found = True + break + if not found: + start_idx = len(user_chats) + except Exception: + raise HTTPException(status_code=400, detail="Invalid cursor format") + + page_items = user_chats[start_idx:start_idx + limit] + next_cursor = None + if start_idx + limit < len(user_chats): + last = page_items[-1] + next_cursor = f"{last['created_at']}_{last['id']}" + + return {"items": page_items, "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + + chat_id_str = str(chat_id) + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute(""" + SELECT c.* FROM chats c + JOIN chat_members cm ON c.id = cm.chat_id + WHERE c.id = %s AND cm.user_id = %s + """, (chat_id_str, user["id"])) + chat = cur.fetchone() + members = [] + if chat: + cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (chat_id_str,)) + members = cur.fetchall() + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) + + if not chat: + raise HTTPException(status_code=404, detail="Chat not found") + + return { + "id": str(chat['id']), + "title": chat['title'], + "member_ids": [str(m['user_id']) for m in members], + "created_at": chat['created_at'].isoformat().replace("+00:00", "Z") if isinstance(chat['created_at'], datetime) else chat['created_at'] + } + + +def validate_image_and_get_dimensions(data: bytes, content_type: str): + if len(data) > 10 * 1024 * 1024: + raise HTTPException(status_code=413, detail="Payload too large (>10 MiB)") + + width, height = 0, 0 + if content_type == "image/png": + if not data.startswith(b'\x89PNG\r\n\x1a\n'): + raise HTTPException(status_code=415, detail="Invalid PNG signature") + if len(data) < 33: + raise HTTPException(status_code=415, detail="Truncated PNG") + if data[12:16] != b'IHDR': + raise HTTPException(status_code=415, detail="PNG missing IHDR chunk") + ihdr_len = struct.unpack('>I', data[8:12])[0] + if ihdr_len != 13: + raise HTTPException(status_code=415, detail="Invalid IHDR length") + width, height = struct.unpack('>II', data[16:24]) + elif content_type == "image/jpeg": + if not data.startswith(b'\xff\xd8'): + raise HTTPException(status_code=415, detail="Invalid JPEG signature") + i = 2 + while i < len(data) - 9: + marker = data[i] + if marker == 0xff: + marker_type = data[i + 1] + if 0xc0 <= marker_type <= 0xc3: + height, width = struct.unpack('>HH', data[i + 5:i + 9]) + break + if marker_type in {0xd8, 0xd9} or (0xd0 <= marker_type <= 0xd7): + i += 2 + continue + seg_len = struct.unpack('>H', data[i + 2:i + 4])[0] + if seg_len < 2: + raise HTTPException(status_code=415, detail="Invalid JPEG segment") + i += 2 + seg_len + else: + i += 1 + else: + raise HTTPException(status_code=415, detail="Unsupported media type") + + if width == 0 or height == 0: + raise HTTPException(status_code=415, detail="Could not read image dimensions") + + if width * height > 40_000_000: + raise HTTPException(status_code=413, detail="Image exceeds 40 million pixels limit") + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment( + chat_id: UUID, + filename: str = Query(..., min_length=1, max_length=255), + request: Request = None, + response: Response = None, + authorization: Optional[str] = Header(None) +): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + chat_id_str = str(chat_id) + + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id_str, user["id"]) + ) + if not cur.fetchone(): + raise HTTPException(status_code=404, detail="Chat not found or inaccessible") + except HTTPException: + raise + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) + + if any(ord(c) < 0x20 or ord(c) == 0x7f for c in filename) or '/' in filename or '\\' in filename: + raise HTTPException(status_code=400, detail="Invalid filename") + + content_type = request.headers.get("content-type") + if content_type not in ["image/png", "image/jpeg"]: + raise HTTPException(status_code=415, detail="Unsupported media type") + + MAX_UPLOAD = 10 * 1024 * 1024 + cl = request.headers.get("content-length") + if cl is not None: + try: + if int(cl) > MAX_UPLOAD: + raise HTTPException(status_code=413, detail="Payload too large (>10 MiB)") + except ValueError: + raise HTTPException(status_code=400, detail="Invalid Content-Length") + + chunks = [] + total = 0 + async for chunk in request.stream(): + total += len(chunk) + if total > MAX_UPLOAD: + raise HTTPException(status_code=413, detail="Payload too large (>10 MiB)") + chunks.append(chunk) + body_bytes = b"".join(chunks) + + validate_image_and_get_dimensions(body_bytes, content_type) + + size_bytes = len(body_bytes) + sha256_hash = hashlib.sha256(body_bytes).hexdigest() + attachment_id = str(uuid.uuid4()) + job_id = str(uuid.uuid4()) + now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + staging_key = f"staging/{chat_id_str}/{attachment_id}/original" + try: + s3_client.put_object( + Bucket=S3_BUCKET, + Key=staging_key, + Body=body_bytes, + ContentType=content_type + ) + except Exception: + api_dependency_errors_counter.add(1, {"component": "s3"}) + raise HTTPException(status_code=503, detail="S3 storage unavailable") + + job_payload = { + "schema_version": 1, + "job_id": job_id, + "type": "attachment.process.v1", + "attachment_id": attachment_id, + "pipeline_version": 1, + "attempt": 1, + "created_at": now + } + + inject(job_payload) + + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute(""" + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, status, variants, error_code, + pipeline_version, attempt, created_at, updated_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', '[]', NULL, 1, 0, %s, %s) + """, (attachment_id, chat_id_str, user["id"], filename, + content_type, size_bytes, sha256_hash, now, now)) + + cur.execute( + "INSERT INTO outbox (id, stream, payload, created_at) " + "VALUES (%s, %s, %s::jsonb, %s)", + (str(uuid.uuid4()), STREAM_NAME, json.dumps(job_payload), now) + ) + conn.commit() + except Exception: + conn.rollback() + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=500, detail="Internal error saving attachment") + finally: + release_db_conn(conn) + + return { + "id": attachment_id, + "chat_id": chat_id_str, + "owner_id": user["id"], + "filename": filename, + "content_type": content_type, + "size_bytes": size_bytes, + "sha256": sha256_hash, + "status": "queued", + "variants": [], + "error_code": None, + "created_at": now, + "updated_at": now + } + + +@app.get("/api/v1/attachments/{attachment_id}") +def get_attachment(attachment_id: UUID, response: Response, authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + att_id_str = str(attachment_id) + + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute(""" + SELECT a.* FROM attachments a + JOIN chat_members cm ON a.chat_id = cm.chat_id + WHERE a.id = %s AND cm.user_id = %s + """, (att_id_str, user["id"])) + row = cur.fetchone() + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) + + if not row: + raise HTTPException(status_code=404, detail="Attachment not found or inaccessible") + + return { + "id": str(row['id']), + "chat_id": str(row['chat_id']), + "owner_id": str(row['owner_id']), + "filename": row['filename'], + "content_type": row['content_type'], + "size_bytes": row['size_bytes'], + "sha256": row['sha256'], + "status": row['status'], + "variants": row['variants'], + "error_code": row['error_code'], + "created_at": row['created_at'].isoformat().replace("+00:00", "Z") if isinstance(row['created_at'], datetime) else row['created_at'], + "updated_at": row['updated_at'].isoformat().replace("+00:00", "Z") if isinstance(row['updated_at'], datetime) else row['updated_at'] + } + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + attachment_id: UUID, + variant: str = Query("original", pattern="^(original|thumbnail|crop|optimized)$"), + response: Response = None, + authorization: Optional[str] = Header(None) +): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + att_id_str = str(attachment_id) + + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute(""" + SELECT a.* FROM attachments a + JOIN chat_members cm ON a.chat_id = cm.chat_id + WHERE a.id = %s AND cm.user_id = %s + """, (att_id_str, user["id"])) + row = cur.fetchone() + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) + + if not row: + raise HTTPException(status_code=404, detail="Attachment not found or inaccessible") + + if row['status'] != 'ready': + raise HTTPException(status_code=409, detail=f"Attachment is in status '{row['status']}'") + + if variant == "original": + s3_key = f"attachments/{row['chat_id']}/{row['id']}/original" + media_type = row['content_type'] + else: + variants_list = row['variants'] or [] + matched = [v for v in variants_list if v.get("name") == variant] + if not matched: + raise HTTPException(status_code=404, detail=f"Variant '{variant}' not found") + s3_key = f"attachments/{row['chat_id']}/{row['id']}/{variant}" + media_type = matched[0]["content_type"] + + try: + obj = s3_client.get_object(Bucket=S3_BUCKET, Key=s3_key) + data = obj['Body'].read() + except Exception: + api_dependency_errors_counter.add(1, {"component": "s3"}) + raise HTTPException(status_code=503, detail="S3 storage unavailable or object missing") + + return Response( + content=data, + media_type=media_type, + headers={ + "X-Request-Id": req_id, + "X-Instance-Id": INSTANCE_ID, + }, + ) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message( + chat_id: UUID, + body: CreateMessage, + response: Response, + authorization: Optional[str] = Header(None), + idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key") +): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + + if not idempotency_key or not re.match(r"^[A-Za-z0-9._:-]{1,128}$", idempotency_key): + raise HTTPException(status_code=400, detail="Missing or invalid Idempotency-Key") + + chat_id_str = str(chat_id) + method = "POST" + path = f"/api/v1/chats/{chat_id_str}/messages" + + body_json_str = json.dumps(body.model_dump(mode='json'), sort_keys=True) + request_hash = hashlib.sha256(body_json_str.encode("utf-8")).hexdigest() + + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id_str, user["id"]) + ) + if not cur.fetchone(): + raise HTTPException(status_code=404, detail="Chat not found") + + attachment_ids_str = [str(aid) for aid in body.attachment_ids] + if attachment_ids_str: + cur.execute(""" + SELECT id, status FROM attachments + WHERE id = ANY(%s::uuid[]) AND chat_id = %s + """, (attachment_ids_str, chat_id_str)) + found_atts = cur.fetchall() + if len(found_atts) != len(attachment_ids_str): + raise HTTPException(status_code=404, detail="One or more attachments not found or in another chat") + for att in found_atts: + if att['status'] == 'failed': + raise HTTPException(status_code=409, detail="Cannot attach failed attachment") + + msg_id = str(uuid.uuid4()) + created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + msg_response = { + "id": msg_id, + "chat_id": chat_id_str, + "sender_id": user["id"], + "text": body.text, + "created_at": created_at, + "attachment_ids": attachment_ids_str + } + + cur.execute(""" + INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, response_status, response_body) + VALUES (%s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (user_id, method, path, key) DO NOTHING + """, (user["id"], method, path, idempotency_key, request_hash, 201, json.dumps(msg_response))) + + if cur.rowcount == 0: + cur.execute(""" + SELECT request_hash, response_status, response_body + FROM idempotency_keys + WHERE user_id = %s AND method = %s AND path = %s AND key = %s + """, (user["id"], method, path, idempotency_key)) + row = cur.fetchone() + conn.commit() + + if row["request_hash"] != request_hash: + raise HTTPException(status_code=409, detail="Idempotency key conflict with different payload") + + return row["response_body"] + + cur.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) VALUES (%s, %s, %s, %s, %s, %s::uuid[])", + (msg_id, chat_id_str, user["id"], body.text, created_at, attachment_ids_str) + ) + conn.commit() + except HTTPException: + conn.rollback() + raise + except Exception: + conn.rollback() + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=500, detail="Internal error processing message") + finally: + release_db_conn(conn) + + return msg_response + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None, + response: Response = None, authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + chat_id_str = str(chat_id) + + if cursor: + c_time, c_id = decode_cursor(cursor, chat_id_str) + else: + c_time, c_id = None, None + + if limit < 1 or limit > 100: + raise HTTPException(status_code=400, detail="Invalid limit") + + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", + (chat_id_str, user["id"])) + if not cur.fetchone(): + raise HTTPException(status_code=404, detail="Chat not found") + + query = "SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages WHERE chat_id=%s" + params = [chat_id_str] + if cursor: + query += " AND (created_at, id) > (%s, %s)" + params.extend([c_time, c_id]) + query += " ORDER BY created_at ASC, id ASC LIMIT %s" + params.append(limit + 1) + + cur.execute(query, tuple(params)) + rows = cur.fetchall() + except HTTPException: + raise + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) + + has_next = len(rows) > limit + page_items = rows[:limit] + + formatted = [] + for m in page_items: + raw = m.get('attachment_ids') + atts = [str(a) for a in raw if a] if raw else [] + formatted.append({ + "id": str(m['id']), "chat_id": str(m['chat_id']), + "sender_id": str(m['sender_id']), "text": m['text'], + "created_at": m['created_at'].isoformat().replace("+00:00", "Z") + if isinstance(m['created_at'], datetime) else m['created_at'], + "attachment_ids": atts + }) + + next_cursor = None + if has_next and formatted: + last = formatted[-1] + next_cursor = encode_cursor(chat_id_str, last['created_at'], last['id']) + + return {"items": formatted, "next_cursor": next_cursor} + + +@app.post("/api/v1/attachments/{attachment_id}/replay", status_code=202) +def replay_attachment(attachment_id: UUID, response: Response, + authorization: Optional[str] = Header(None)): + req_id = generate_request_id() + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = INSTANCE_ID + user = get_current_user_from_token(authorization) + att_id = str(attachment_id) + + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute(""" + SELECT a.* FROM attachments a + JOIN chat_members cm ON a.chat_id = cm.chat_id + WHERE a.id=%s AND cm.user_id=%s + """, (att_id, user["id"])) + row = cur.fetchone() + if not row: + raise HTTPException(status_code=404, detail="Attachment not found") + if row["status"] != "failed": + raise HTTPException(status_code=409, detail="Only failed attachments can be replayed") + + now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + cur.execute(""" + UPDATE attachments SET status='queued', attempt=0, error_code=NULL, updated_at=%s + WHERE id=%s + """, (now, att_id)) + + job = {"schema_version": 1, "job_id": str(uuid.uuid4()), + "type": "attachment.process.v1", "attachment_id": att_id, + "pipeline_version": 1, "attempt": 1, "created_at": now} + inject(job) + cur.execute("INSERT INTO outbox (id, stream, payload, created_at) " + "VALUES (%s, %s, %s::jsonb, %s)", + (str(uuid.uuid4()), STREAM_NAME, json.dumps(job), now)) + conn.commit() + except HTTPException: + conn.rollback() + raise + except Exception: + conn.rollback() + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) + + return {"id": att_id, "status": "queued"} \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..32958ee --- /dev/null +++ b/compose.yaml @@ -0,0 +1,311 @@ +services: + postgres: + image: postgres:15.5-alpine3.19 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: postgres + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} + volumes: + - pgdata:/var/lib/postgresql/data + deploy: + resources: + limits: + cpus: '0.50' + memory: 512M + reservations: + cpus: '0.10' + memory: 128M + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d messenger"] + interval: 3s + timeout: 3s + retries: 5 + + redis: + image: redis:7.2.4-alpine + command: redis-server --appendonly yes --appendfsync everysec --loglevel warning + volumes: + - redisdata:/data + deploy: + resources: + limits: + cpus: '0.25' + memory: 256M + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 5 + + migrate: + image: postgres:15.5-alpine3.19 + environment: + PGPASSWORD: ${POSTGRES_PASSWORD} + volumes: + - ./migrations:/migrations + command: > + sh -c "until pg_isready -h postgres -U postgres; do sleep 1; done; + psql -h postgres -U postgres -d messenger -f /migrations/init.sql" + depends_on: + postgres: + condition: service_healthy + restart: "no" + + s3: + image: chrislusf/seaweedfs:3.67 + command: ["server", "-s3", "-dir=/data", "-s3.port=8333"] + environment: + - AWS_ACCESS_KEY_ID=appuser + - AWS_SECRET_ACCESS_KEY=appsecretpassword + volumes: + - s3data:/data + deploy: + resources: + limits: + cpus: '0.50' + memory: 512M + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/ || test $$? -eq 8 || exit 1"] + interval: 5s + timeout: 3s + retries: 5 + start_period: 30s + + s3-init: + image: python:3.11.7-slim-bookworm + volumes: + - ./scripts:/scripts + - ./requirements.txt:/requirements.txt + environment: + - S3_ENDPOINT=http://s3:8333 + - S3_BUCKET=messenger-attachments + - S3_ROOT_ACCESS_KEY=appuser + - S3_ROOT_SECRET_KEY=appsecretpassword + command: > + sh -c "pip install --no-cache-dir -r /requirements.txt && + python /scripts/bootstrap_s3.py" + depends_on: + s3: + condition: service_healthy + restart: "no" + + prometheus: + image: prom/prometheus:v2.51.0 + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=7d" + - "--web.enable-remote-write-receiver" + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro + - promdata:/prometheus + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9090/-/healthy || exit 1"] + interval: 5s + timeout: 3s + retries: 5 + + loki: + image: grafana/loki:3.0.0 + command: ["-config.file=/etc/loki/local-config.yaml"] + volumes: + - ./infra/loki/loki-config.yaml:/etc/loki/local-config.yaml:ro + - lokidata:/loki + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3100/ready || exit 1"] + interval: 5s + timeout: 3s + retries: 5 + + tempo: + image: grafana/tempo:2.4.1 + command: ["-config.file=/etc/tempo.yaml"] + volumes: + - ./infra/tempo/tempo-config.yaml:/etc/tempo.yaml:ro + - tempodata:/var/tempo + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3200/ready || exit 1"] + interval: 5s + timeout: 3s + retries: 5 + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.97.0 + command: ["--config=/etc/otelcol-contrib/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro + depends_on: + - prometheus + - loki + - tempo + + grafana: + image: grafana/grafana:10.4.2 + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_USER=${GF_ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${GF_ADMIN_PASSWORD:-admin} + - GF_AUTH_ANONYMOUS_ENABLED=false + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafanadata:/var/lib/grafana + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_healthy + tempo: + condition: service_healthy + + api-1: + build: . + stop_grace_period: 15s + environment: + - DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/messenger + - REDIS_URL=redis://redis:6379/0 + - SESSION_TTL_SECONDS=3600 + - INSTANCE_ID=api-replica-1 + - SECRET_HMAC_KEY=${SECRET_HMAC_KEY} + - S3_ENDPOINT=http://s3:8333 + - S3_BUCKET=messenger-attachments + - S3_ACCESS_KEY_ID=appuser + - S3_SECRET_ACCESS_KEY=appsecretpassword + - BROKER_STREAM=attachment_jobs + - OTEL_SERVICE_NAME=messenger-api + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf + - OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab + deploy: + resources: + limits: + cpus: '0.50' + memory: 256M + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + otel-collector: + condition: service_started + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] + interval: 5s + timeout: 3s + retries: 3 + + api-2: + build: . + stop_grace_period: 15s + environment: + - DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/messenger + - REDIS_URL=redis://redis:6379/0 + - SESSION_TTL_SECONDS=3600 + - INSTANCE_ID=api-replica-2 + - SECRET_HMAC_KEY=${SECRET_HMAC_KEY} + - S3_ENDPOINT=http://s3:8333 + - S3_BUCKET=messenger-attachments + - S3_ACCESS_KEY_ID=appuser + - S3_SECRET_ACCESS_KEY=appsecretpassword + - BROKER_STREAM=attachment_jobs + - OTEL_SERVICE_NAME=messenger-api + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf + - OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab + deploy: + resources: + limits: + cpus: '0.50' + memory: 256M + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + otel-collector: + condition: service_started + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] + interval: 5s + timeout: 3s + retries: 3 + + worker: + build: . + command: ["python", "worker.py"] + stop_grace_period: 15s + environment: + - DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/messenger + - REDIS_URL=redis://redis:6379/0 + - S3_ENDPOINT=http://s3:8333 + - S3_BUCKET=messenger-attachments + - S3_ACCESS_KEY_ID=appuser + - S3_SECRET_ACCESS_KEY=appsecretpassword + - BROKER_STREAM=attachment_jobs + - BROKER_GROUP=attachment_workers + - JOB_MAX_ATTEMPTS=5 + - MAX_IMAGE_PIXELS=40000000 + - OTEL_SERVICE_NAME=messenger-worker + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf + - OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab + deploy: + resources: + limits: + cpus: '0.50' + memory: 256M + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + otel-collector: + condition: service_started + healthcheck: + test: ["CMD", "python", "-c", "import os, redis; redis.Redis.from_url(os.environ['REDIS_URL']).ping()"] + interval: 5s + timeout: 3s + retries: 3 + + nginx: + image: nginx:1.25.4-alpine + ports: + - "8080:8080" + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + deploy: + resources: + limits: + cpus: '0.25' + memory: 128M + depends_on: + - api-1 + - api-2 + healthcheck: + test: ["CMD", "nginx", "-t"] + interval: 5s + timeout: 3s + retries: 3 + +volumes: + pgdata: + redisdata: + s3data: + promdata: + lokidata: + tempodata: + grafanadata: \ No newline at end of file diff --git a/infra/grafana/dashboards/messenger-grade3.json b/infra/grafana/dashboards/messenger-grade3.json new file mode 100644 index 0000000..ef8b211 --- /dev/null +++ b/infra/grafana/dashboards/messenger-grade3.json @@ -0,0 +1,86 @@ +{ + "title": "MTUSI Messenger — Lab 6 (Grade 3)", + "uid": "mtusi-messenger-lab6-g3", + "schemaVersion": 39, + "version": 2, + "refresh": "5s", + "panels": [ + { + "id": 1, + "title": "API RPS", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 0 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(http_server_requests_total[1m]))", + "legendFormat": "req/s" + } + ] + }, + { + "id": 2, + "title": "API Errors (4xx/5xx/s)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 0 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(http_server_errors_total[1m]))", + "legendFormat": "errors/s" + } + ] + }, + { + "id": 3, + "title": "API Duration p95 (s)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p95" + } + ] + }, + { + "id": 4, + "title": "Worker Jobs (success / failed)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(worker_jobs_total[1m])) by (status)", + "legendFormat": "{{status}}" + } + ] + }, + { + "id": 5, + "title": "Worker Job Duration p95 (s)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p95" + } + ] + }, + { + "id": 6, + "title": "Structured Logs (API & Worker)", + "type": "logs", + "gridPos": { "h": 9, "w": 24, "x": 0, "y": 14 }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "{service_name=~\".+\"}" + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/grafana/dashboards/messenger-grade4.json b/infra/grafana/dashboards/messenger-grade4.json new file mode 100644 index 0000000..e979547 --- /dev/null +++ b/infra/grafana/dashboards/messenger-grade4.json @@ -0,0 +1,147 @@ +{ + "title": "MTUSI Messenger — Lab 6 (Grade 4)", + "uid": "mtusi-messenger-lab6-g4", + "schemaVersion": 39, + "version": 3, + "refresh": "5s", + "panels": [ + { + "id": 1, + "title": "API RPS (req/s)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "reqps" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(http_server_requests_total[1m]))", + "legendFormat": "req/s" + } + ] + }, + { + "id": 2, + "title": "API Error Ratio (%)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "percentunit" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(http_server_errors_total[1m])) / clamp_min(sum(rate(http_server_requests_total[1m])), 0.0001)", + "legendFormat": "error_ratio" + } + ] + }, + { + "id": 3, + "title": "API Latency p50 / p95 / p99 (s)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 6, "x": 12, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.50, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p50" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p95" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p99" + } + ] + }, + { + "id": 4, + "title": "DB Connection Pool (used / idle)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 6, "x": 18, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "short" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(db_pool_connections) by (state)", + "legendFormat": "{{state}}" + } + ] + }, + { + "id": 5, + "title": "Queue Depth & Oldest Job Age", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 7 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(worker_queue_depth)", + "legendFormat": "queue_depth (jobs)" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "max(worker_oldest_job_age_seconds)", + "legendFormat": "oldest_job_age (s)" + } + ] + }, + { + "id": 6, + "title": "Worker Saturation & Retries", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 7 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(worker_jobs_in_progress)", + "legendFormat": "in_progress (jobs)" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(worker_job_retries_total[1m])) by (reason)", + "legendFormat": "retries/s: {{reason}}" + } + ] + }, + { + "id": 7, + "title": "Worker Job Duration p50 / p95 / p99 (s)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 7 }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.50, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p50" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p95" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p99" + } + ] + }, + { + "id": 8, + "title": "Logs with TraceID (Loki)", + "type": "logs", + "gridPos": { "h": 9, "w": 24, "x": 0, "y": 14 }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "{service_name=~\".+\"}" + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/grafana/provisioning/dashboards/dashboards.yml b/infra/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..606870f --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: MTUSI Messenger Lab6 + orgId: 1 + folder: Messenger + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/infra/grafana/provisioning/datasources/datasources.yml b/infra/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..64cedaf --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,37 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + + - name: Loki + type: loki + uid: loki + access: proxy + url: http://loki:3100 + editable: true + jsonData: + derivedFields: + - datasourceUid: tempo + matcherRegex: '"trace_id":"([0-9a-f]{32})"' + name: TraceID + url: '$${__value.raw}' + + - name: Tempo + type: tempo + uid: tempo + access: proxy + url: http://tempo:3200 + editable: true + jsonData: + tracesToLogsV2: + datasourceUid: loki + filterByTraceID: true + tags: + - key: service.name + value: service \ No newline at end of file diff --git a/infra/loki/loki-config.yaml b/infra/loki/loki-config.yaml new file mode 100644 index 0000000..7617c4b --- /dev/null +++ b/infra/loki/loki-config.yaml @@ -0,0 +1,38 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +ruler: + alertmanager_url: http://localhost:9093 + +limits_config: + allow_structured_metadata: true + +distributor: + otlp_config: + default_resource_attributes_as_index_labels: + - service.name + - service.namespace + - deployment.environment \ No newline at end of file diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..09c1046 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,61 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: + send_batch_size: 256 + timeout: 2s + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +exporters: + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + sending_queue: + enabled: true + queue_size: 1000 + otlphttp/tempo: + endpoint: http://tempo:4318 + tls: + insecure: true + sending_queue: + enabled: true + queue_size: 1000 + otlphttp/loki: + endpoint: http://loki:3100/otlp + headers: + X-Scope-OrgID: fake + tls: + insecure: true + sending_queue: + enabled: true + queue_size: 1000 + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki, debug] \ No newline at end of file diff --git a/infra/prometheus/alerts.yml b/infra/prometheus/alerts.yml new file mode 100644 index 0000000..cb4d25b --- /dev/null +++ b/infra/prometheus/alerts.yml @@ -0,0 +1,62 @@ +groups: + - name: messenger_api_alerts + interval: 15s + rules: + - alert: MessengerApiHighErrorRate + expr: | + ( + sum(rate(http_server_errors_total[1m])) + / + clamp_min(sum(rate(http_server_requests_total[1m])), 0.001) + ) > 0.05 + and sum(rate(http_server_requests_total[1m])) > 0.2 + for: 30s + labels: + severity: warning + service: messenger-api + annotations: + summary: "High HTTP error ratio (>5%) in messenger-api" + description: "Error ratio exceeded 5% over 30s; inspect Loki logs and Tempo traces." + + - alert: MessengerApiDependencyErrors + expr: sum(rate(http_server_dependency_errors_total[1m])) > 0 + for: 30s + labels: + severity: critical + service: messenger-api + annotations: + summary: "Dependency errors in messenger-api" + description: "PostgreSQL, Redis, S3, or outbox errors occurred." + + - alert: MessengerApiHighLatencyP95 + expr: histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le)) > 1.0 + for: 1m + labels: + severity: warning + service: messenger-api + annotations: + summary: "High API p95 latency (>1.0s)" + description: "API p95 duration is above 1.0s." + + - name: messenger_worker_alerts + interval: 15s + rules: + - alert: MessengerWorkerJobFailures + expr: sum(rate(worker_jobs_total{status="failed"}[1m])) > 0 + for: 30s + labels: + severity: warning + service: messenger-worker + annotations: + summary: "Attachment worker jobs failing" + description: "Failed worker jobs detected; check error_code and trace_id in Loki/Tempo." + + - alert: MessengerWorkerQueueLag + expr: max(worker_oldest_job_age_seconds) > 30 or sum(worker_queue_depth) > 20 + for: 30s + labels: + severity: critical + service: messenger-worker + annotations: + summary: "Growing attachment queue lag or backlog" + description: "Oldest job age > 30s or queue depth > 20; check worker saturation and dependencies." \ No newline at end of file diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..cf87d7b --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,11 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/alerts.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] \ No newline at end of file diff --git a/infra/tempo/tempo-config.yaml b/infra/tempo/tempo-config.yaml new file mode 100644 index 0000000..72a9e64 --- /dev/null +++ b/infra/tempo/tempo-config.yaml @@ -0,0 +1,25 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +ingester: + trace_idle_period: 10s + max_block_duration: 5m + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + local: + path: /var/tempo/blocks + wal: + path: /var/tempo/wal \ No newline at end of file diff --git a/migrations/init.sql b/migrations/init.sql new file mode 100644 index 0000000..444f86c --- /dev/null +++ b/migrations/init.sql @@ -0,0 +1,74 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username VARCHAR(32) UNIQUE NOT NULL, + display_name VARCHAR(100) NOT NULL, + password_hash VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID REFERENCES users(id) ON DELETE CASCADE, + text VARCHAR(4000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + attachment_ids UUID[] DEFAULT '{}' +); + +CREATE INDEX IF NOT EXISTS idx_messages_chat_pagination ON messages(chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL, + method VARCHAR(16) NOT NULL, + path TEXT NOT NULL, + key VARCHAR(128) NOT NULL, + request_hash VARCHAR(64) NOT NULL, + response_status INT NOT NULL, + response_body JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (user_id, method, path, key) +); + +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID REFERENCES users(id) ON DELETE CASCADE, + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(50) NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 VARCHAR(64) NOT NULL, + status VARCHAR(32) NOT NULL DEFAULT 'queued', + variants JSONB NOT NULL DEFAULT '[]', + error_code VARCHAR(64), + pipeline_version INT NOT NULL DEFAULT 1, + attempt INT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE IF NOT EXISTS outbox ( + id UUID PRIMARY KEY, + stream VARCHAR(64) NOT NULL, + payload JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + published_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments(chat_id); + +ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] DEFAULT '{}'; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INT NOT NULL DEFAULT 1; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS attempt INT NOT NULL DEFAULT 0; +CREATE INDEX IF NOT EXISTS idx_outbox_unpublished ON outbox(created_at) WHERE published_at IS NULL; \ No newline at end of file diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000..bf62681 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,28 @@ +events { worker_connections 1024; } + +http { + upstream messenger_api { + server api-1:8080 max_fails=1 fail_timeout=5s; + server api-2:8080 max_fails=1 fail_timeout=5s; + } + + server { + listen 8080; + + location / { + proxy_pass http://messenger_api; + + proxy_connect_timeout 2s; + proxy_send_timeout 5s; + proxy_read_timeout 5s; + + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 5s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + } +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a4b3e9f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,20 @@ +fastapi==0.110.0 +uvicorn==0.28.0 +pydantic==2.6.2 +starlette==0.36.3 +idna==3.6 +anyio==4.3.0 +sniffio==1.3.1 +click==8.1.7 +h11==0.14.0 +annotated-types==0.6.0 +psycopg2-binary==2.9.9 +bcrypt==4.1.2 +redis==5.0.1 +boto3==1.34.0 +Pillow==10.2.0 +opentelemetry-api==1.24.0 +opentelemetry-sdk==1.24.0 +opentelemetry-exporter-otlp-proto-http==1.24.0 +opentelemetry-instrumentation-fastapi==0.45b0 +opentelemetry-instrumentation-requests==0.45b0 \ No newline at end of file diff --git a/scripts/bootstrap_s3.py b/scripts/bootstrap_s3.py new file mode 100644 index 0000000..72fc848 --- /dev/null +++ b/scripts/bootstrap_s3.py @@ -0,0 +1,29 @@ +import os +import boto3 +from botocore.client import Config + +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +S3_ROOT_ACCESS_KEY = os.getenv("S3_ROOT_ACCESS_KEY", "minioadmin") +S3_ROOT_SECRET_KEY = os.getenv("S3_ROOT_SECRET_KEY", "minioadminpassword") + +s3_client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ROOT_ACCESS_KEY, + aws_secret_access_key=S3_ROOT_SECRET_KEY, + config=Config(signature_version="s3v4"), + region_name="us-east-1" +) + +def init_bucket(): + print(f"Checking/Creating bucket: {S3_BUCKET}") + try: + s3_client.head_bucket(Bucket=S3_BUCKET) + print("Bucket already exists.") + except Exception: + s3_client.create_bucket(Bucket=S3_BUCKET) + print("Bucket successfully created.") + +if __name__ == "__main__": + init_bucket() \ No newline at end of file diff --git a/scripts/check_bucket_private.py b/scripts/check_bucket_private.py new file mode 100644 index 0000000..dc66599 --- /dev/null +++ b/scripts/check_bucket_private.py @@ -0,0 +1,11 @@ +import os, sys, urllib.request, urllib.error +BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +KEY = os.getenv("S3_KEY") +URL = f"http://s3:8333/{BUCKET}/{KEY}" +try: + urllib.request.urlopen(URL) + print("FAIL: object accessible without auth"); sys.exit(1) +except urllib.error.HTTPError as e: + if e.code in (401, 403): + print(f"PASS: anonymous GET -> {e.code}"); sys.exit(0) + print(f"FAIL: unexpected {e.code}"); sys.exit(1) \ No newline at end of file diff --git a/scripts/check_contract.py b/scripts/check_contract.py index dcb34df..614ddc1 100644 --- a/scripts/check_contract.py +++ b/scripts/check_contract.py @@ -8,8 +8,8 @@ import re ROOT = Path(__file__).resolve().parents[1] def main(): - lab = json.loads((ROOT/'lab.json').read_text())['number'] - spec = json.loads((ROOT/'contracts/openapi.json').read_text()) + lab = json.loads((ROOT/'lab.json').read_text(encoding='utf-8'))['number'] + spec = json.loads((ROOT/'contracts/openapi.json').read_text(encoding='utf-8')) assert spec['openapi']=='3.1.0' assert spec['info']['version']=='1.%d.0'%lab operation_ids = set() @@ -41,11 +41,11 @@ def main(): assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2) assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4) for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')): - ast.parse(file.read_text(),filename=str(file)) + ast.parse(file.read_text(encoding='utf-8'),filename=str(file)) for file in ROOT.rglob('*.json'): if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts): continue - json.loads(file.read_text()) + json.loads(file.read_text(encoding='utf-8-sig')) for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'): assert (ROOT/required).is_file(), 'missing '+required print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.') diff --git a/scripts/reconcile.py b/scripts/reconcile.py new file mode 100644 index 0000000..64e3fba --- /dev/null +++ b/scripts/reconcile.py @@ -0,0 +1,81 @@ +import os +import sys +import boto3 +from botocore.client import Config +import psycopg2 +import psycopg2.extras +from datetime import datetime, timezone, timedelta + +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser") +S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword") + +s3_client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + config=Config(signature_version="s3v4"), + region_name="us-east-1" +) + +def run_reconciliation(dry_run: bool = False): + print(f"Starting reconciliation (dry_run={dry_run})...") + conn = psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor) + + threshold = datetime.now(timezone.utc) - timedelta(hours=1) + + with conn.cursor() as cur: + cur.execute(""" + SELECT id, chat_id, filename, created_at FROM attachments + WHERE status = 'pending' AND created_at < %s + """, (threshold,)) + stale_pending = cur.fetchall() + + for row in stale_pending: + att_id = str(row['id']) + chat_id = str(row['chat_id']) + filename = row['filename'] + s3_key = f"attachments/{chat_id}/{att_id}/{filename}" + print(f"[DB Stale Pending] Found stale attachment {att_id}, marking as failed and cleaning S3.") + + if not dry_run: + try: + s3_client.delete_object(Bucket=S3_BUCKET, Key=s3_key) + except Exception as e: + print(f"Failed to delete S3 key {s3_key}: {e}") + with conn.cursor() as cur: + cur.execute("UPDATE attachments SET status = 'failed', error_code = 'timeout_stale' WHERE id = %s", (att_id,)) + conn.commit() + + with conn.cursor() as cur: + cur.execute("SELECT id, chat_id, filename FROM attachments") + all_db_attachments = cur.fetchall() + + valid_s3_keys = set() + for row in all_db_attachments: + valid_s3_keys.add(f"attachments/{str(row['chat_id'])}/{str(row['id'])}/{row['filename']}") + + paginator = s3_client.get_paginator('list_objects_v2') + for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="attachments/"): + for obj in page.get('Contents', []): + key = obj['Key'] + last_modified = obj['LastModified'] + + if key not in valid_s3_keys and last_modified < threshold: + print(f"[S3 Orphan] Found orphan S3 object {key} (Modified: {last_modified})") + if not dry_run: + try: + s3_client.delete_object(Bucket=S3_BUCKET, Key=key) + print(f"Deleted orphan object {key} from S3.") + except Exception as e: + print(f"Failed to delete orphan {key}: {e}") + + conn.close() + print("Reconciliation completed successfully.") + +if __name__ == "__main__": + dry_run_mode = "--dry-run" in sys.argv + run_reconciliation(dry_run=dry_run_mode) \ No newline at end of file diff --git a/scripts/restart.bat b/scripts/restart.bat new file mode 100644 index 0000000..ce7fdbe --- /dev/null +++ b/scripts/restart.bat @@ -0,0 +1,3 @@ +@echo off +docker compose restart +exit /b %ERRORLEVEL% \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..e6cfbc5 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +docker compose restart \ No newline at end of file diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100644 index 0000000..173f1db --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,35 @@ +#!/bin/sh +set -eu + +wait_ready() { + svc=$1 + timeout=60 + echo "Waiting for $svc readiness (up to ${timeout}s)..." + while [ "$timeout" -gt 0 ]; do + if docker compose exec -T "$svc" python -c \ + "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')" 2>/dev/null; then + echo "$svc is ready!" + return 0 + fi + sleep 1 + timeout=$((timeout - 1)) + done + echo "$svc failed to become ready" >&2 + return 1 +} + +echo "Starting Rolling Update" + +echo "1. Rebuilding and replacing api-1..." +docker compose up -d --no-deps --build --force-recreate api-1 +wait_ready api-1 +echo "Reloading nginx to pick up new api-1 IP..." +docker compose exec -T nginx nginx -s reload + +echo "2. Rebuilding and replacing api-2..." +docker compose up -d --no-deps --build --force-recreate api-2 +wait_ready api-2 +echo "Reloading nginx to pick up new api-2 IP..." +docker compose exec -T nginx nginx -s reload + +echo "Rolling Update Completed Successfully" \ No newline at end of file diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..d35a3c0 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +if [ "$TARGET_INSTANCE_ID" = "api-replica-1" ]; then + docker compose stop -t 3 api-1 +elif [ "$TARGET_INSTANCE_ID" = "api-replica-2" ]; then + docker compose stop -t 3 api-2 +else + echo "Unknown target instance: $TARGET_INSTANCE_ID" >&2 + exit 1 +fi \ No newline at end of file diff --git a/scripts/upload_load_test.py b/scripts/upload_load_test.py new file mode 100644 index 0000000..cdd2e2e --- /dev/null +++ b/scripts/upload_load_test.py @@ -0,0 +1,54 @@ +import base64, concurrent.futures, json, pathlib, time, urllib.request, uuid + +BASE = "http://localhost:8080" + +def _req(method, path, body=None, headers=None, token=None): + h = {"Accept": "application/json"} + if token: h["Authorization"] = f"Bearer {token}" + h.update(headers or {}) + req = urllib.request.Request(BASE + path, data=body, headers=h, method=method) + with urllib.request.urlopen(req) as r: + return r.status, r.read() + +def register_and_login(): + u = f"u_{uuid.uuid4().hex[:10]}" + p = "T3st-" + uuid.uuid4().hex + _req("POST", "/api/v1/users", + json.dumps({"username": u, "display_name": "Load", "password": p}).encode(), + {"Content-Type": "application/json"}) + basic = base64.b64encode(f"{u}:{p}".encode()).decode() + _, body = _req("POST", "/api/v1/auth/sessions", + headers={"Authorization": f"Basic {basic}"}) + return json.loads(body)["token"] + +def make_chat(token): + _, me = _req("GET", "/api/v1/users/me", token=token) + uid = json.loads(me)["id"] + _, chat = _req("POST", "/api/v1/chats", + json.dumps({"title": "load", "member_ids": [uid]}).encode(), + {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}, + token=token) + return json.loads(chat)["id"] + +def main(): + token = register_and_login() + chat = make_chat(token) + png = pathlib.Path("tests/fixtures/sample.png").read_bytes() + + def one(_): + try: + status, _ = _req("POST", + f"/api/v1/chats/{chat}/attachments?filename=f.png", + png, {"Content-Type": "image/png"}, token) + return status + except Exception as e: + return repr(e) + + t0 = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as ex: + results = list(ex.map(one, range(10))) + print(json.dumps({"results": results, + "elapsed_s": round(time.monotonic() - t0, 3)})) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/telemetry.py b/telemetry.py new file mode 100644 index 0000000..9f9b976 --- /dev/null +++ b/telemetry.py @@ -0,0 +1,92 @@ +import json +import logging +import os +from datetime import datetime, timezone + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + + +class JsonLogFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + span = trace.get_current_span() + ctx = span.get_span_context() if span else None + trace_id = format(ctx.trace_id, "032x") if ctx and ctx.is_valid else None + span_id = format(ctx.span_id, "016x") if ctx and ctx.is_valid else None + + payload = { + "ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "level": record.levelname, + "logger": record.name, + "service": os.getenv("OTEL_SERVICE_NAME", "messenger-app"), + "msg": record.getMessage(), + } + if trace_id and span_id: + payload["trace_id"] = trace_id + payload["span_id"] = span_id + + for attr in ("method", "path", "status_code", "job_id", "attachment_id", "job_status", "attempt", "error_code"): + if hasattr(record, attr): + payload[attr] = getattr(record, attr) + + if record.exc_info: + payload["exception"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception" + + return json.dumps(payload, ensure_ascii=False) + + +def setup_telemetry(default_service_name: str): + service_name = os.getenv("OTEL_SERVICE_NAME", default_service_name) + endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/") + env_name = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=lab") + + attrs = {"service.name": service_name} + for part in env_name.split(","): + if "=" in part: + k, v = part.split("=", 1) + attrs[k.strip()] = v.strip() + + resource = Resource.create(attrs) + + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")) + ) + trace.set_tracer_provider(tracer_provider) + + metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"), + export_interval_millis=5000, + ) + meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) + metrics.set_meter_provider(meter_provider) + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs")) + ) + set_logger_provider(logger_provider) + + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + root_logger.handlers.clear() + + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(JsonLogFormatter()) + root_logger.addHandler(stream_handler) + + otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + otel_handler.setFormatter(JsonLogFormatter()) + root_logger.addHandler(otel_handler) + + return trace.get_tracer(service_name), metrics.get_meter(service_name) \ No newline at end of file diff --git a/tests/client.py b/tests/client.py index f0499a2..38159a0 100644 --- a/tests/client.py +++ b/tests/client.py @@ -14,7 +14,7 @@ import uuid ROOT = Path(__file__).resolve().parents[1] LAB = json.loads((ROOT / 'lab.json').read_text())['number'] -SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text()) +SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text(encoding='utf-8')) class Failure(AssertionError): pass diff --git a/tests/custom_tests.py b/tests/custom_tests.py new file mode 100644 index 0000000..f5e46e6 --- /dev/null +++ b/tests/custom_tests.py @@ -0,0 +1,109 @@ +import concurrent.futures +import urllib.request +import urllib.error +import json +import unittest +from client import Client + +class TestMessengerLab4(unittest.TestCase): + def setUp(self): + self.client = Client() + self.alice = self.client.user("alice") + self.bob = self.client.user("bob") + self.eve = self.client.user("eve") + + res = self.client.call( + 'POST', '/api/v1/chats', 201, + actor=self.alice, + body={"title": "Test Chat", "member_ids": [self.alice["id"], self.bob["id"]]}, + schema='Chat' + ) + self.chat = res['json'] + + def test_unauthorized_access_to_chat(self): + self.client.call('GET', f"/api/v1/chats/{self.chat['id']}", 404, actor=self.eve) + + def test_empty_message(self): + self.client.call('POST', f"/api/v1/chats/{self.chat['id']}/messages", 400, actor=self.alice, body={"text": ""}) + + def test_malformed_json(self): + url = self.client.base + f"/api/v1/chats/{self.chat['id']}/messages" + req = urllib.request.Request( + url, + data=b"{broken json content without quotes", + headers={ + "Content-Type": "application/json", + "X-User-Id": self.alice["id"] + }, + method="POST" + ) + try: + with urllib.request.urlopen(req) as resp: + self.fail("Expected HTTP 400 for malformed JSON") + except urllib.error.HTTPError as e: + self.assertEqual(e.code, 400) + body = json.loads(e.read().decode()) + self.assertEqual(body["error"]["code"], "invalid_request") + + def test_duplicate_username(self): + self.client.call('POST', '/api/v1/users', 409, body=self.alice["registration"]) + + def test_pagination_multiple_pages(self): + import time + msg_texts = ["msg-page-1", "msg-page-2", "msg-page-3"] + for txt in msg_texts: + self.client.message(self.alice, self.chat, txt) + time.sleep(0.01) + + path = f"/api/v1/chats/{self.chat['id']}/messages" + collected_items = [] + cursor = None + + for _ in range(5): + url_suffix = "?limit=1" + if cursor: + url_suffix += f"&cursor={cursor}" + + page_res = self.client.call('GET', path + url_suffix, actor=self.alice, schema='MessagePage') + page = page_res['json'] + + for item in page['items']: + self.assertNotIn(item['id'], [x['id'] for x in collected_items]) + collected_items.append(item) + + cursor = page.get('next_keyword') or page.get('next_cursor') + if not cursor: + break + + self.assertEqual(len(collected_items), 3) + self.assertEqual(sorted([x['text'] for x in collected_items]), sorted(msg_texts)) + + def test_concurrent_messages_strict(self): + def worker_send(worker_idx): + local_client = Client() + ids = [] + for i in range(10): + res = local_client.message(self.alice, self.chat, f"w-{worker_idx}-msg-{i}") + ids.append(res['json']['id']) + return ids + + all_ids = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + futures = [executor.submit(worker_send, w) for w in range(10)] + for f in concurrent.futures.as_completed(futures): + all_ids.extend(f.result()) + + self.assertEqual(len(all_ids), 100) + self.assertEqual(len(set(all_ids)), 100) + + page = self.client.call( + 'GET', f"/api/v1/chats/{self.chat['id']}/messages?limit=100", + actor=self.alice, + schema='MessagePage' + ) + server_ids = [m['id'] for m in page['json']['items']] + self.assertEqual(len(server_ids), 100) + self.assertEqual(len(set(server_ids)), 100) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/level4_tests.py b/tests/level4_tests.py new file mode 100644 index 0000000..38489a7 --- /dev/null +++ b/tests/level4_tests.py @@ -0,0 +1,139 @@ +import urllib.request +import urllib.error +import json +import unittest +from client import Client + +class TestMessengerLevel4Complete(unittest.TestCase): + def setUp(self): + self.client = Client() + self.alice = self.client.user("alice") + self.bob = self.client.user("bob") + self.eve = self.client.user("eve") + + res_chat1 = self.client.call( + 'POST', '/api/v1/chats', 201, + actor=self.alice, + body={"title": "Team Chat", "member_ids": [self.alice["id"], self.bob["id"]]}, + schema='Chat' + ) + self.chat1 = res_chat1['json'] + + res_chat2 = self.client.call( + 'POST', '/api/v1/chats', 201, + actor=self.eve, + body={"title": "Private Chat", "member_ids": [self.eve["id"], self.bob["id"]]}, + schema='Chat' + ) + self.chat2 = res_chat2['json'] + + def test_unsupported_media_type(self): + """Проверка 415: передача текстового файла под видом картинки.""" + fake_bytes = b"Just a simple plain text file content." + url = self.client.base + f"/api/v1/chats/{self.chat1['id']}/attachments?filename=notes.txt" + req = urllib.request.Request( + url, data=fake_bytes, + headers={"Authorization": f"Bearer {self.alice['token']}", "Content-Type": "image/png"}, + method="POST" + ) + try: + with urllib.request.urlopen(req) as resp: + self.fail("Expected HTTP 415") + except urllib.error.HTTPError as e: + self.assertEqual(e.code, 415) + + def test_payload_too_large(self): + """Проверка 413: файл размером больше 10 MiB.""" + large_data = b'\x89PNG\r\n\x1a\n' + b'\x00' * (10 * 1024 * 1024 + 100) + url = self.client.base + f"/api/v1/chats/{self.chat1['id']}/attachments?filename=huge.png" + req = urllib.request.Request( + url, data=large_data, + headers={"Authorization": f"Bearer {self.alice['token']}", "Content-Type": "image/png"}, + method="POST" + ) + try: + with urllib.request.urlopen(req) as resp: + self.fail("Expected HTTP 413") + except urllib.error.HTTPError as e: + self.assertEqual(e.code, 413) + + def test_corrupted_image_signature(self): + """Проверка 415: битая структура / неверные размеры PNG.""" + corrupted_png = b'\x89PNG\r\n\x1a\n' + b'\xFF' * 50 + url = self.client.base + f"/api/v1/chats/{self.chat1['id']}/attachments?filename=corrupt.png" + req = urllib.request.Request( + url, data=corrupted_png, + headers={"Authorization": f"Bearer {self.alice['token']}", "Content-Type": "image/png"}, + method="POST" + ) + try: + with urllib.request.urlopen(req) as resp: + self.fail("Expected HTTP 415") + except urllib.error.HTTPError as e: + self.assertEqual(e.code, 415) + + def test_invalid_filename_characters(self): + """Проверка 400: недопустимые символы (path traversal) в имени файла.""" + valid_png = ( + b'\x89PNG\r\n\x1a\n' + b'\x00\x00\x00\rIHDR' + b'\x00\x00\x00\n\x00\x00\x00\n' + b'\x08\x02\x00\x00\x00' + b'\x02p\xdd\xb5' + b'\x00\x00\x00\x0fcIDATx\x9cc`\x00\x00\x00\x02\x00\x01H\xaf\xa4q' + b'\x00\x00\x00\x00IEND\xaeB`\x82' + ) + url = self.client.base + f"/api/v1/chats/{self.chat1['id']}/attachments?filename=../../etc/passwd" + req = urllib.request.Request( + url, data=valid_png, + headers={"Authorization": f"Bearer {self.alice['token']}", "Content-Type": "image/png"}, + method="POST" + ) + try: + with urllib.request.urlopen(req) as resp: + self.fail("Expected HTTP 400") + except urllib.error.HTTPError as e: + self.assertEqual(e.code, 400) + + def test_nonexistent_attachment_id(self): + """Проверка 404: передача несуществующего ID вложения при отправке сообщения.""" + fake_uuid = "12345678-1234-4123-8123-123456789abc" + res = self.client.call( + 'POST', f"/api/v1/chats/{self.chat1['id']}/messages", 404, + actor=self.alice, + body={"text": "Check this out", "attachment_ids": [fake_uuid]}, + headers={"Idempotency-Key": "test-key-lvl4-1"} + ) + self.assertEqual(res['status'], 404) + + def test_cross_chat_attachment_isolation(self): + """Проверка 404: попытка прикрепить вложение из чужого чата.""" + valid_png = ( + b'\x89PNG\r\n\x1a\n' + b'\x00\x00\x00\rIHDR' + b'\x00\x00\x00\n\x00\x00\x00\n' + b'\x08\x02\x00\x00\x00' + b'\x02p\xdd\xb5' + b'\x00\x00\x00\x0fcID1Tx\x9cc`\x00\x00\x00\x02\x00\x01H\xaf\xa4q' + b'\x00\x00\x00\x00IEND\xaeB`\x82' + ) + url = self.client.base + f"/api/v1/chats/{self.chat2['id']}/attachments?filename=secret.png" + req = urllib.request.Request( + url, data=valid_png, + headers={"Authorization": f"Bearer {self.eve['token']}", "Content-Type": "image/png"}, + method="POST" + ) + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read().decode()) + eve_att_id = data["id"] + + res = self.client.call( + 'POST', f"/api/v1/chats/{self.chat1['id']}/messages", 404, + actor=self.alice, + body={"text": "Stealing file", "attachment_ids": [eve_att_id]}, + headers={"Idempotency-Key": "test-key-lvl4-2"} + ) + self.assertEqual(res['status'], 404) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/resilience.py b/tests/resilience.py index f3fa700..d5d67fe 100644 --- a/tests/resilience.py +++ b/tests/resilience.py @@ -55,7 +55,7 @@ def main(): check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity') print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.') # No shell interpolation and no Docker commands in this test. - proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + proc = subprocess.Popen(['sh', str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) started = time.monotonic() successes,failures,after_instances = [],0,[] try: diff --git a/worker.py b/worker.py new file mode 100644 index 0000000..90da41b --- /dev/null +++ b/worker.py @@ -0,0 +1,628 @@ +import os +import io +import json +import time +import signal +import random +import hashlib +import logging +from datetime import datetime, timezone +from concurrent.futures import ThreadPoolExecutor + +import psycopg2 +import psycopg2.extras +import redis +import boto3 +import botocore.exceptions +from botocore.client import Config +from PIL import Image, ImageOps, UnidentifiedImageError + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.propagate import extract + + +class JsonLogFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + span = trace.get_current_span() + ctx = span.get_span_context() if span else None + trace_id = format(ctx.trace_id, "032x") if ctx and ctx.is_valid else None + span_id = format(ctx.span_id, "016x") if ctx and ctx.is_valid else None + + payload = { + "ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "level": record.levelname, + "logger": record.name, + "service": os.getenv("OTEL_SERVICE_NAME", "messenger-worker"), + "msg": record.getMessage(), + } + if trace_id and span_id: + payload["trace_id"] = trace_id + payload["span_id"] = span_id + + for attr in ("method", "path", "status_code", "job_id", "attachment_id", "job_status", "attempt", "error_code"): + if hasattr(record, attr): + payload[attr] = getattr(record, attr) + + if record.exc_info: + payload["exception"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception" + + return json.dumps(payload, ensure_ascii=False) + + +def setup_worker_telemetry(): + service_name = os.getenv("OTEL_SERVICE_NAME", "messenger-worker") + endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/") + env_attrs = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=lab") + + attrs = {"service.name": service_name} + for part in env_attrs.split(","): + if "=" in part: + k, v = part.split("=", 1) + attrs[k.strip()] = v.strip() + + resource = Resource.create(attrs) + + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")) + ) + trace.set_tracer_provider(tracer_provider) + + metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"), + export_interval_millis=5000, + ) + meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) + metrics.set_meter_provider(meter_provider) + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs")) + ) + set_logger_provider(logger_provider) + + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + root_logger.handlers.clear() + + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(JsonLogFormatter()) + root_logger.addHandler(stream_handler) + + otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + otel_handler.setFormatter(JsonLogFormatter()) + root_logger.addHandler(otel_handler) + + return trace.get_tracer(service_name), metrics.get_meter(service_name) + + +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") +REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser") +S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword") + +STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs") +DLQ_STREAM_NAME = f"{STREAM_NAME}_dlq" +GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers") +CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1") +JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) +MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) +JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "30")) +WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "4")) + +CLAIM_MIN_IDLE_MS = (JOB_TIMEOUT_SECONDS + 20) * 1000 +Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS + +redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True) +s3_client = boto3.client( + "s3", endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY, + config=Config(signature_version="s3v4"), region_name="us-east-1" +) + + +def observe_queue_metrics(options): + try: + q_len = redis_client.xlen(STREAM_NAME) + entries = redis_client.xrange(STREAM_NAME, count=1) + oldest_age = 0.0 + if entries: + msg_id, fields = entries[0] + try: + payload = json.loads(fields.get("payload", "{}")) + created_str = payload.get("created_at") + if created_str: + created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00")) + oldest_age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds()) + except Exception: + ms = int(msg_id.split("-")[0]) + oldest_age = max(0.0, time.time() - (ms / 1000.0)) + yield metrics.Observation(value=q_len, attributes={"stream": STREAM_NAME}) + except Exception: + yield metrics.Observation(value=0, attributes={"stream": STREAM_NAME}) + + +def observe_oldest_job_age(options): + try: + entries = redis_client.xrange(STREAM_NAME, count=1) + if not entries: + yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME}) + return + msg_id, fields = entries[0] + try: + payload = json.loads(fields.get("payload", "{}")) + created_str = payload.get("created_at") + if created_str: + created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00")) + age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds()) + yield metrics.Observation(value=age, attributes={"stream": STREAM_NAME}) + return + except Exception: + pass + ms = int(msg_id.split("-")[0]) + yield metrics.Observation(value=max(0.0, time.time() - (ms / 1000.0)), attributes={"stream": STREAM_NAME}) + except Exception: + yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME}) + + +tracer, meter = setup_worker_telemetry() +logger = logging.getLogger("attachment-worker") + +worker_jobs_counter = meter.create_counter( + name="worker_jobs_total", + description="Completed and failed attachment jobs", + unit="1", +) +worker_retries_counter = meter.create_counter( + name="worker_job_retries_total", + description="Transient retry attempts scheduled for attachment jobs", + unit="1", +) +worker_duration_histogram = meter.create_histogram( + name="worker_job_duration_seconds", + description="Attachment job processing duration in seconds", + unit="s", +) +worker_jobs_in_progress = meter.create_up_down_counter( + name="worker_jobs_in_progress", + description="Jobs currently being processed", + unit="1", +) +meter.create_observable_gauge( + name="worker_queue_depth", + callbacks=[observe_queue_metrics], + description="Number of messages currently in attachment jobs stream", + unit="1", +) +meter.create_observable_gauge( + name="worker_oldest_job_age_seconds", + callbacks=[observe_oldest_job_age], + description="Age of the oldest message in attachment jobs stream in seconds", + unit="s", +) + +running = True + + +def handle_sigterm(signum, frame): + global running + logger.info("Received shutdown signal, finishing in-flight jobs...") + running = False + + +signal.signal(signal.SIGTERM, handle_sigterm) +signal.signal(signal.SIGINT, handle_sigterm) + + +def get_db_connection(): + return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor) + + +def ensure_consumer_groups(): + while running: + try: + redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True) + redis_client.xgroup_create(DLQ_STREAM_NAME, GROUP_NAME, id="0", mkstream=True) + logger.info("Consumer groups ensured on '%s' and '%s'", STREAM_NAME, DLQ_STREAM_NAME) + return + except redis.exceptions.ResponseError as e: + if "BUSYGROUP" in str(e): + return + raise + except Exception as e: + logger.warning("Waiting for Redis: %s", type(e).__name__) + time.sleep(1) + + +def prepare_base_image(raw_bytes): + img = Image.open(io.BytesIO(raw_bytes)) + img = ImageOps.exif_transpose(img) + if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info): + rgba = img.convert("RGBA") + bg = Image.new("RGB", rgba.size, (255, 255, 255)) + bg.paste(rgba, mask=rgba.split()[3]) + img = bg + elif img.mode != "RGB": + img = img.convert("RGB") + return img + + +def generate_grade4_variants(raw_bytes): + base_img = prepare_base_image(raw_bytes) + orig_w, orig_h = base_img.size + meta = [] + blobs = {} + + thumb = base_img.copy() + if orig_w <= 256 and orig_h <= 256: + t_w, t_h = orig_w, orig_h + else: + s = min(256.0 / orig_w, 256.0 / orig_h) + t_w, t_h = max(1, int(orig_w * s)), max(1, int(orig_h * s)) + if (t_w, t_h) != (orig_w, orig_h): + thumb = thumb.resize((t_w, t_h), Image.Resampling.LANCZOS) + buf = io.BytesIO() + thumb.save(buf, format="JPEG", quality=85, optimize=True) + tb = buf.getvalue() + blobs["thumbnail"] = tb + meta.append({ + "name": "thumbnail", "content_type": "image/jpeg", + "size_bytes": len(tb), "width": t_w, "height": t_h, + "sha256": hashlib.sha256(tb).hexdigest() + }) + + opt = base_img.copy() + if orig_w <= 1280 and orig_h <= 1280: + o_w, o_h = orig_w, orig_h + else: + s = min(1280.0 / orig_w, 1280.0 / orig_h) + o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s)) + if (o_w, o_h) != (orig_w, orig_h): + opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS) + buf = io.BytesIO() + opt.save(buf, format="WEBP", quality=85) + ob = buf.getvalue() + blobs["optimized"] = ob + meta.append({ + "name": "optimized", "content_type": "image/webp", + "size_bytes": len(ob), "width": o_w, "height": o_h, + "sha256": hashlib.sha256(ob).hexdigest() + }) + + c = min(orig_w, orig_h) + left, top = (orig_w - c) // 2, (orig_h - c) // 2 + crop = base_img.crop((left, top, left + c, top + c)) + if c != 128: + crop = crop.resize((128, 128), Image.Resampling.LANCZOS) + buf = io.BytesIO() + crop.save(buf, format="JPEG", quality=85, optimize=True) + cb = buf.getvalue() + blobs["crop"] = cb + meta.append({ + "name": "crop", "content_type": "image/jpeg", + "size_bytes": len(cb), "width": 128, "height": 128, + "sha256": hashlib.sha256(cb).hexdigest() + }) + + return meta, blobs + + +def mark_failed(attachment_id, error_code): + now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + conn = get_db_connection() + try: + with conn.cursor() as cur: + cur.execute( + "UPDATE attachments SET status='failed', error_code=%s, updated_at=%s " + "WHERE id=%s AND status != 'ready'", + (error_code, now, attachment_id) + ) + conn.commit() + finally: + conn.close() + + +TRANSIENT_ERRORS = ( + TimeoutError, + botocore.exceptions.EndpointConnectionError, + botocore.exceptions.ReadTimeoutError, + psycopg2.OperationalError, + redis.exceptions.ConnectionError, +) + + +def _record_job_outcome(span, status: str, elapsed: float): + span.set_attribute("job.outcome", status) + worker_jobs_counter.add(1, {"status": status}) + worker_duration_histogram.record(elapsed, {"status": status}) + + +def process_job(message_id, fields): + job_start = time.monotonic() + try: + job = json.loads(fields.get("payload", "{}")) + except Exception: + logger.error("Malformed job %s, ack to drop", message_id) + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + return + + ctx = extract(job) + + with tracer.start_as_current_span("worker.process_attachment", context=ctx) as span: + worker_jobs_in_progress.add(1) + try: + attachment_id = job.get("attachment_id") + job_id = job.get("job_id") + pipeline_version = int(job.get("pipeline_version", 1)) + attempt = int(job.get("attempt", 1)) + + span.set_attribute("job.pipeline_version", pipeline_version) + span.set_attribute("job.attempt", attempt) + + if not attachment_id: + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + return + + conn = get_db_connection() + try: + with conn.cursor() as cur: + cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)) + att = cur.fetchone() + if not att: + conn.commit() + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + return + if att["status"] == "ready" or att["pipeline_version"] > pipeline_version: + conn.commit() + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "success", time.monotonic() - job_start) + return + if att["status"] == "failed": + conn.commit() + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + return + + if att["status"] == "processing": + updated = att["updated_at"] + if isinstance(updated, str): + updated = datetime.fromisoformat(updated.replace("Z", "+00:00")) + age_seconds = (datetime.now(timezone.utc) - updated).total_seconds() + if age_seconds < (JOB_TIMEOUT_SECONDS + 10): + conn.commit() + logger.info( + "Attachment already processing, ack duplicate", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt}, + ) + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + return + + now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + cur.execute( + "UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s", + (attempt, now, attachment_id) + ) + conn.commit() + finally: + conn.close() + + chat_id = str(att["chat_id"]) + staging_key = f"staging/{chat_id}/{attachment_id}/original" + final_original_key = f"attachments/{chat_id}/{attachment_id}/original" + + start_time = time.monotonic() + + def check_timeout(): + if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS: + raise TimeoutError("Job exceeded time limit") + + try: + try: + obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key) + except Exception: + obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key) + raw_bytes = obj["Body"].read() + check_timeout() + + if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]: + raise ValueError("Staging checksum mismatch") + + variants_meta, variants_bytes = generate_grade4_variants(raw_bytes) + check_timeout() + + s3_client.put_object( + Bucket=S3_BUCKET, Key=final_original_key, + Body=raw_bytes, ContentType=att["content_type"] + ) + for var_name, var_bytes in variants_bytes.items(): + ct = "image/webp" if var_name == "optimized" else "image/jpeg" + s3_client.put_object( + Bucket=S3_BUCKET, + Key=f"attachments/{chat_id}/{attachment_id}/{var_name}", + Body=var_bytes, ContentType=ct + ) + check_timeout() + + ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + conn = get_db_connection() + try: + with conn.cursor() as cur: + cur.execute(""" + UPDATE attachments SET status='ready', variants=%s::jsonb, + error_code=NULL, updated_at=%s + WHERE id=%s AND pipeline_version <= %s + """, (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version)) + conn.commit() + finally: + conn.close() + + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "success", time.monotonic() - job_start) + logger.info( + "Job completed: attachment ready", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "ready", "attempt": attempt}, + ) + + try: + s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key) + except Exception: + pass + + except (UnidentifiedImageError, Image.DecompressionBombError): + logger.error( + "Permanent image error", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_image"}, + ) + mark_failed(attachment_id, "invalid_image") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + + except ValueError as e: + logger.error( + "Checksum error", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_checksum"}, + ) + mark_failed(attachment_id, "invalid_checksum") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + + except psycopg2.errors.DataError: + logger.error( + "DB data error", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "data_error"}, + ) + mark_failed(attachment_id, "data_error") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + + except TRANSIENT_ERRORS as e: + logger.warning( + "Transient error during job processing", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt}, + ) + if attempt >= JOB_MAX_ATTEMPTS: + mark_failed(attachment_id, "retry_exhausted") + try: + dlq_job = dict(job) + dlq_job["attempt"] = 1 + redis_client.xadd( + DLQ_STREAM_NAME, + {"payload": json.dumps(dlq_job), "error": type(e).__name__} + ) + except Exception: + pass + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + else: + worker_retries_counter.add(1, {"reason": type(e).__name__}) + delay = min(2 ** attempt, 30) + random.uniform(0, 1) + time.sleep(delay) + next_job = dict(job) + next_job["attempt"] = attempt + 1 + redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)}) + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + + except Exception: + logger.error( + "Unknown error during job processing", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "processing_failed"}, + ) + mark_failed(attachment_id, "processing_failed") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + finally: + worker_jobs_in_progress.add(-1) + + +def run_staging_cleanup(): + try: + cutoff = datetime.now(timezone.utc).timestamp() - 3600 + paginator = s3_client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="staging/"): + for obj in page.get("Contents", []): + if obj["LastModified"].timestamp() < cutoff: + try: + s3_client.delete_object(Bucket=S3_BUCKET, Key=obj["Key"]) + except Exception: + pass + except Exception as e: + logger.debug("Staging cleanup: %s", type(e).__name__) + + +def safe_process(msg_id, fields): + try: + process_job(msg_id, fields) + except Exception as e: + logger.error("Unhandled in process_job %s: %s", msg_id, type(e).__name__) + + +def main(): + ensure_consumer_groups() + logger.info("Worker '%s' started, concurrency=%d", CONSUMER_NAME, WORKER_CONCURRENCY) + + executor = ThreadPoolExecutor(max_workers=WORKER_CONCURRENCY) + last_claim = 0 + last_cleanup = 0 + + try: + while running: + try: + now = time.time() + + if now - last_claim > 15: + last_claim = now + try: + claimed = redis_client.xautoclaim( + STREAM_NAME, GROUP_NAME, CONSUMER_NAME, + min_idle_time=CLAIM_MIN_IDLE_MS, + start_id="0-0", count=WORKER_CONCURRENCY + ) + if claimed and len(claimed) > 1 and claimed[1]: + for msg_id, fields in claimed[1]: + if not running: + break + executor.submit(safe_process, msg_id, fields) + except Exception: + pass + + if now - last_cleanup > 600: + last_cleanup = now + executor.submit(run_staging_cleanup) + + resp = redis_client.xreadgroup( + GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"}, + count=WORKER_CONCURRENCY, block=2000 + ) + if not resp: + continue + for _, messages in resp: + for msg_id, fields in messages: + if not running: + break + executor.submit(safe_process, msg_id, fields) + except Exception as e: + logger.warning("Worker loop: %s", type(e).__name__) + time.sleep(1) + finally: + logger.info("Shutting down, waiting for in-flight jobs...") + executor.shutdown(wait=True) + logger.info("Worker stopped cleanly") + + +if __name__ == "__main__": + main() \ No newline at end of file