forked from mtusi2026/mtusi2026lab6
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
38a00e1930 |
+27
@@ -0,0 +1,27 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080
|
||||
|
||||
RUN groupadd -g 10001 appuser && \
|
||||
useradd -u 10001 -g appuser -s /sbin/nologin -M appuser
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY app/ ./app/
|
||||
|
||||
RUN chown -R appuser:appuser /app
|
||||
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready')" || exit 1
|
||||
|
||||
CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "10"]
|
||||
@@ -4,10 +4,12 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-persistence test-failover test-student up down restart
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "test-student: run student tests (pytest); test-persistence: test stack restart with data preservation"
|
||||
@echo "test-failover: test replica failure handling; up: build and start stack; down: stop stack"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -28,3 +30,15 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-student:
|
||||
BASE_URL="$(BASE_URL)" $(if $(wildcard .venv/bin/pytest),.venv/bin/pytest,pytest) tests_student/ -v
|
||||
|
||||
up:
|
||||
docker compose up -d --build
|
||||
|
||||
down:
|
||||
docker compose down
|
||||
|
||||
restart:
|
||||
docker compose restart
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Messenger application package."""
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerifyMismatchError
|
||||
|
||||
_hasher = PasswordHasher()
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
"""Хэширует пароль с использованием Argon2id."""
|
||||
return _hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(hash_str: str, password: str) -> bool:
|
||||
"""Проверяет пароль на соответствие хэшу Argon2id."""
|
||||
try:
|
||||
return _hasher.verify(hash_str, password)
|
||||
except VerifyMismatchError:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def generate_session_token() -> str:
|
||||
"""Генерирует криптографически стойкий непрозрачный токен сессии (256 бит энтропии)."""
|
||||
return secrets.token_urlsafe(32)
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import aio_pika
|
||||
from aio_pika import DeliveryMode, ExchangeType
|
||||
|
||||
from app.config import BROKER_URL
|
||||
|
||||
logger = logging.getLogger("messenger.broker")
|
||||
|
||||
EXCHANGE_NAME = "messenger.attachments"
|
||||
QUEUE_NAME = "messenger.attachments.jobs"
|
||||
ROUTING_KEY = "attachment.process"
|
||||
|
||||
DLX_NAME = "messenger.attachments.dlx"
|
||||
DLQ_NAME = "messenger.attachments.dlq"
|
||||
DL_ROUTING_KEY = "attachment.dead"
|
||||
|
||||
|
||||
async def setup_broker_topology(channel: aio_pika.abc.AbstractChannel) -> tuple[aio_pika.abc.AbstractExchange, aio_pika.abc.AbstractQueue]:
|
||||
"""Настраивает durable exchange, durable очереди и связку с Dead Letter Exchange."""
|
||||
# 1. Dead Letter Exchange и Dead Letter Queue
|
||||
dlx = await channel.declare_exchange(
|
||||
DLX_NAME,
|
||||
type=ExchangeType.DIRECT,
|
||||
durable=True,
|
||||
)
|
||||
dlq = await channel.declare_queue(
|
||||
DLQ_NAME,
|
||||
durable=True,
|
||||
)
|
||||
await dlq.bind(dlx, routing_key=DL_ROUTING_KEY)
|
||||
|
||||
# 2. Основной Exchange и Queue с dead-lettering
|
||||
exchange = await channel.declare_exchange(
|
||||
EXCHANGE_NAME,
|
||||
type=ExchangeType.DIRECT,
|
||||
durable=True,
|
||||
)
|
||||
queue = await channel.declare_queue(
|
||||
QUEUE_NAME,
|
||||
durable=True,
|
||||
arguments={
|
||||
"x-dead-letter-exchange": DLX_NAME,
|
||||
"x-dead-letter-routing-key": DL_ROUTING_KEY,
|
||||
},
|
||||
)
|
||||
await queue.bind(exchange, routing_key=ROUTING_KEY)
|
||||
return exchange, queue
|
||||
|
||||
|
||||
class BrokerPublisher:
|
||||
"""Отказоустойчивый издатель сообщений в RabbitMQ с поддержкой Publisher Confirms."""
|
||||
|
||||
def __init__(self, broker_url: str = BROKER_URL) -> None:
|
||||
self.broker_url = broker_url
|
||||
self._connection: aio_pika.abc.AbstractRobustConnection | None = None
|
||||
self._channel: aio_pika.abc.AbstractChannel | None = None
|
||||
self._exchange: aio_pika.abc.AbstractExchange | None = None
|
||||
|
||||
async def get_exchange(self) -> aio_pika.abc.AbstractExchange:
|
||||
if self._exchange is None or self._channel is None or self._channel.is_closed:
|
||||
if self._connection is None or self._connection.is_closed:
|
||||
self._connection = await aio_pika.connect_robust(self.broker_url)
|
||||
self._channel = await self._connection.channel(publisher_confirms=True)
|
||||
self._exchange, _ = await setup_broker_topology(self._channel)
|
||||
return self._exchange
|
||||
|
||||
async def publish_job(self, job_dict: dict[str, Any]) -> bool:
|
||||
try:
|
||||
exchange = await self.get_exchange()
|
||||
payload = json.dumps(job_dict, ensure_ascii=False).encode("utf-8")
|
||||
message = aio_pika.Message(
|
||||
body=payload,
|
||||
delivery_mode=DeliveryMode.PERSISTENT,
|
||||
content_type="application/json",
|
||||
message_id=str(job_dict.get("job_id", "")),
|
||||
)
|
||||
await exchange.publish(
|
||||
message,
|
||||
routing_key=ROUTING_KEY,
|
||||
mandatory=True,
|
||||
)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.error("Failed to publish job to broker: %s", exc)
|
||||
self._exchange = None
|
||||
self._channel = None
|
||||
return False
|
||||
|
||||
async def close(self) -> None:
|
||||
try:
|
||||
if self._channel and not self._channel.is_closed:
|
||||
await self._channel.close()
|
||||
if self._connection and not self._connection.is_closed:
|
||||
await self._connection.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._exchange = None
|
||||
self._channel = None
|
||||
self._connection = None
|
||||
|
||||
|
||||
publisher = BrokerPublisher()
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app.broker import publisher
|
||||
from app.config import DATABASE_URL
|
||||
from app.schemas import now_utc_iso
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
logger = logging.getLogger("messenger.cli")
|
||||
|
||||
|
||||
async def replay_attachment(attachment_id_str: str) -> None:
|
||||
"""Безопасный операторский replay задачи из failed в queued."""
|
||||
try:
|
||||
att_id = uuid.UUID(attachment_id_str)
|
||||
except ValueError:
|
||||
logger.error("Invalid UUID: %s", attachment_id_str)
|
||||
sys.exit(1)
|
||||
|
||||
pool = await asyncpg.create_pool(dsn=DATABASE_URL)
|
||||
assert pool is not None
|
||||
|
||||
async with pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"SELECT id, status, chat_id, filename FROM attachments WHERE id = $1",
|
||||
att_id,
|
||||
)
|
||||
if not row:
|
||||
logger.error("Attachment %s not found in DB", att_id)
|
||||
await pool.close()
|
||||
sys.exit(1)
|
||||
|
||||
if row["status"] == "ready":
|
||||
logger.warning(
|
||||
"Attachment %s is already 'ready'. Replay will not re-execute successful job.",
|
||||
att_id,
|
||||
)
|
||||
await pool.close()
|
||||
return
|
||||
|
||||
# Переводим в queued и создаем новое задание
|
||||
new_job_id = uuid.uuid4()
|
||||
now_iso = now_utc_iso()
|
||||
job_payload = {
|
||||
"schema_version": 1,
|
||||
"job_id": str(new_job_id),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": str(att_id),
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": now_iso,
|
||||
}
|
||||
|
||||
async with conn.transaction():
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'queued',
|
||||
error_code = NULL,
|
||||
processing_started_at = NULL,
|
||||
processing_worker_id = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
""",
|
||||
att_id,
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO outbox_jobs (
|
||||
job_id, attachment_id, event_type, payload, status, attempt, created_at, updated_at
|
||||
)
|
||||
VALUES ($1, $2, 'attachment.process.v1', $3::jsonb, 'pending', 1, NOW(), NOW())
|
||||
""",
|
||||
new_job_id,
|
||||
att_id,
|
||||
json.dumps(job_payload),
|
||||
)
|
||||
|
||||
# Публикуем напрямую в брокер
|
||||
published = await publisher.publish_job(job_payload)
|
||||
if published:
|
||||
await conn.execute(
|
||||
"UPDATE outbox_jobs SET status = 'published' WHERE job_id = $1",
|
||||
new_job_id,
|
||||
)
|
||||
logger.info("Attachment %s successfully replayed with new job %s", att_id, new_job_id)
|
||||
else:
|
||||
logger.warning(
|
||||
"Attachment %s queued in outbox_jobs, will be dispatched by outbox relay",
|
||||
att_id,
|
||||
)
|
||||
|
||||
await publisher.close()
|
||||
await pool.close()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Operator CLI for messenger")
|
||||
subparsers = parser.add_subparsers(dest="command")
|
||||
|
||||
replay_parser = subparsers.add_parser("replay", help="Replay a failed attachment")
|
||||
replay_parser.add_argument("attachment_id", help="UUID of the attachment to replay")
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.command == "replay":
|
||||
asyncio.run(replay_attachment(args.attachment_id))
|
||||
else:
|
||||
parser.print_help()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import socket
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _get_val(env_name: str, file_env_name: str | None = None, default: str = "") -> str:
|
||||
if file_env_name and file_env_name in os.environ:
|
||||
file_path = Path(os.environ[file_env_name])
|
||||
if file_path.is_file():
|
||||
return file_path.read_text(encoding="utf-8").strip()
|
||||
return os.getenv(env_name, default)
|
||||
|
||||
|
||||
APP_HOST: str = os.getenv("APP_HOST", "0.0.0.0")
|
||||
APP_PORT: int = int(os.getenv("APP_PORT", "8080"))
|
||||
APP_ENV: str = os.getenv("APP_ENV", "lab")
|
||||
LOG_LEVEL: str = os.getenv("LOG_LEVEL", "INFO")
|
||||
|
||||
SESSION_TTL_SECONDS: int = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
SESSION_STORE: str = os.getenv("SESSION_STORE", "redis").lower()
|
||||
INSTANCE_ID: str = os.getenv("INSTANCE_ID") or socket.gethostname() or "api-1"
|
||||
IDEMPOTENCY_TTL_SECONDS: int = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
|
||||
DATABASE_URL: str = _get_val(
|
||||
"DATABASE_URL",
|
||||
"DATABASE_URL_FILE",
|
||||
"postgresql://postgres:postgres@postgres:5432/messenger",
|
||||
)
|
||||
|
||||
CACHE_URL: str = _get_val(
|
||||
"CACHE_URL",
|
||||
"CACHE_URL_FILE",
|
||||
"redis://redis:6379/0",
|
||||
)
|
||||
|
||||
S3_ENDPOINT: str = os.getenv("S3_ENDPOINT", "http://minio:9000")
|
||||
S3_REGION: str = os.getenv("S3_REGION", "us-east-1")
|
||||
S3_BUCKET: str = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY_ID: str = _get_val("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE", "minioadmin")
|
||||
S3_SECRET_ACCESS_KEY: str = _get_val("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE", "minioadmin")
|
||||
MAX_UPLOAD_BYTES: int = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
|
||||
MAX_IMAGE_PIXELS: int = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
|
||||
BROKER_URL: str = _get_val(
|
||||
"BROKER_URL",
|
||||
"BROKER_URL_FILE",
|
||||
"amqp://messenger:messenger_broker_pass@broker:5672/",
|
||||
)
|
||||
WORKER_CONCURRENCY: int = int(os.getenv("WORKER_CONCURRENCY", "4"))
|
||||
WORKER_LEASE_SECONDS: int = int(os.getenv("WORKER_LEASE_SECONDS", "60"))
|
||||
MAX_JOB_RETRIES: int = int(os.getenv("MAX_JOB_RETRIES", "5"))
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import Depends, Header, Request
|
||||
|
||||
from app.errors import UnauthorizedError
|
||||
from app.schemas import UserResponse
|
||||
from app.storage import storage
|
||||
|
||||
|
||||
async def get_current_token(
|
||||
authorization: str | None = Header(default=None, alias="Authorization"),
|
||||
) -> str:
|
||||
if not authorization or not authorization.strip():
|
||||
raise UnauthorizedError("Отсутствует заголовок Authorization")
|
||||
|
||||
parts = authorization.strip().split(maxsplit=1)
|
||||
if len(parts) != 2 or parts[0].lower() != "bearer":
|
||||
raise UnauthorizedError("Неверная схема авторизации (ожидается Bearer)")
|
||||
|
||||
token = parts[1].strip()
|
||||
if not token:
|
||||
raise UnauthorizedError("Пустой Bearer токен")
|
||||
return token
|
||||
|
||||
|
||||
async def get_current_user(token: str = Depends(get_current_token)) -> UserResponse:
|
||||
user = await storage.get_session_user(token)
|
||||
if not user:
|
||||
raise UnauthorizedError("Недействительная или истекшая сессия")
|
||||
return user
|
||||
+260
@@ -0,0 +1,260 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
|
||||
import asyncpg
|
||||
import botocore.exceptions
|
||||
from fastapi import FastAPI, Request, status
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
import redis.exceptions
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from app.config import INSTANCE_ID
|
||||
from app.schemas import ErrorCode, ErrorDetail, ErrorResponse
|
||||
|
||||
|
||||
class StorageError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UnavailableError(StorageError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidRequestError(StorageError):
|
||||
pass
|
||||
|
||||
|
||||
class NotFoundError(StorageError):
|
||||
pass
|
||||
|
||||
|
||||
class ConflictError(StorageError):
|
||||
pass
|
||||
|
||||
|
||||
class PayloadTooLargeError(StorageError):
|
||||
pass
|
||||
|
||||
|
||||
class UnsupportedMediaTypeError(StorageError):
|
||||
pass
|
||||
|
||||
|
||||
class UnauthorizedError(StorageError):
|
||||
pass
|
||||
|
||||
|
||||
logger = logging.getLogger("app.errors")
|
||||
|
||||
|
||||
def get_request_id(request: Request) -> str:
|
||||
req_id = getattr(request.state, "request_id", None)
|
||||
if not req_id:
|
||||
req_id = request.headers.get("X-Request-Id") or str(uuid.uuid4())
|
||||
return req_id
|
||||
|
||||
|
||||
def make_error_response(
|
||||
status_code: int,
|
||||
code: ErrorCode,
|
||||
message: str,
|
||||
request_id: str,
|
||||
headers: dict[str, str] | None = None,
|
||||
) -> JSONResponse:
|
||||
payload = ErrorResponse(
|
||||
error=ErrorDetail(code=code, message=message, request_id=request_id)
|
||||
).model_dump()
|
||||
resp_headers = {
|
||||
"X-Request-Id": request_id,
|
||||
"X-Instance-Id": INSTANCE_ID,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
if headers:
|
||||
resp_headers.update(headers)
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content=payload,
|
||||
headers=resp_headers,
|
||||
)
|
||||
|
||||
|
||||
def register_error_handlers(app: FastAPI) -> None:
|
||||
@app.exception_handler(InvalidRequestError)
|
||||
async def invalid_request_handler(
|
||||
request: Request, exc: InvalidRequestError
|
||||
) -> JSONResponse:
|
||||
req_id = get_request_id(request)
|
||||
return make_error_response(
|
||||
status.HTTP_400_BAD_REQUEST, ErrorCode.INVALID_REQUEST, str(exc), req_id
|
||||
)
|
||||
|
||||
@app.exception_handler(UnauthorizedError)
|
||||
async def unauthorized_handler(
|
||||
request: Request, exc: UnauthorizedError
|
||||
) -> JSONResponse:
|
||||
req_id = get_request_id(request)
|
||||
scheme = (
|
||||
'Basic realm="messenger", charset="UTF-8"'
|
||||
if request.url.path == "/api/v1/auth/sessions"
|
||||
else 'Bearer realm="messenger"'
|
||||
)
|
||||
return make_error_response(
|
||||
status.HTTP_401_UNAUTHORIZED,
|
||||
ErrorCode.UNAUTHORIZED,
|
||||
str(exc),
|
||||
req_id,
|
||||
headers={"WWW-Authenticate": scheme},
|
||||
)
|
||||
|
||||
@app.exception_handler(NotFoundError)
|
||||
async def not_found_handler(request: Request, exc: NotFoundError) -> JSONResponse:
|
||||
req_id = get_request_id(request)
|
||||
return make_error_response(
|
||||
status.HTTP_404_NOT_FOUND, ErrorCode.NOT_FOUND, str(exc), req_id
|
||||
)
|
||||
|
||||
@app.exception_handler(ConflictError)
|
||||
async def conflict_handler(request: Request, exc: ConflictError) -> JSONResponse:
|
||||
req_id = get_request_id(request)
|
||||
return make_error_response(
|
||||
status.HTTP_409_CONFLICT, ErrorCode.CONFLICT, str(exc), req_id
|
||||
)
|
||||
|
||||
@app.exception_handler(PayloadTooLargeError)
|
||||
async def payload_too_large_handler(
|
||||
request: Request, exc: PayloadTooLargeError
|
||||
) -> JSONResponse:
|
||||
req_id = get_request_id(request)
|
||||
return make_error_response(
|
||||
status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
ErrorCode.PAYLOAD_TOO_LARGE,
|
||||
str(exc),
|
||||
req_id,
|
||||
)
|
||||
|
||||
@app.exception_handler(UnsupportedMediaTypeError)
|
||||
async def unsupported_media_type_handler(
|
||||
request: Request, exc: UnsupportedMediaTypeError
|
||||
) -> JSONResponse:
|
||||
req_id = get_request_id(request)
|
||||
return make_error_response(
|
||||
status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||
ErrorCode.UNSUPPORTED_MEDIA_TYPE,
|
||||
str(exc),
|
||||
req_id,
|
||||
)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(
|
||||
request: Request, exc: RequestValidationError
|
||||
) -> JSONResponse:
|
||||
req_id = get_request_id(request)
|
||||
errors = exc.errors()
|
||||
messages = []
|
||||
for err in errors:
|
||||
loc = ".".join(str(x) for x in err.get("loc", []) if x != "body")
|
||||
msg = err.get("msg", "invalid")
|
||||
messages.append(f"{loc}: {msg}" if loc else msg)
|
||||
detail_msg = "; ".join(messages) or "Некорректное тело запроса или параметры"
|
||||
return make_error_response(
|
||||
status.HTTP_400_BAD_REQUEST, ErrorCode.INVALID_REQUEST, detail_msg, req_id
|
||||
)
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_exception_handler(
|
||||
request: Request, exc: StarletteHTTPException
|
||||
) -> JSONResponse:
|
||||
req_id = get_request_id(request)
|
||||
code = ErrorCode.INTERNAL_ERROR
|
||||
extra_headers: dict[str, str] = {}
|
||||
if exc.status_code == 400:
|
||||
code = ErrorCode.INVALID_REQUEST
|
||||
elif exc.status_code == 401:
|
||||
code = ErrorCode.UNAUTHORIZED
|
||||
scheme = (
|
||||
'Basic realm="messenger", charset="UTF-8"'
|
||||
if request.url.path == "/api/v1/auth/sessions"
|
||||
else 'Bearer realm="messenger"'
|
||||
)
|
||||
extra_headers["WWW-Authenticate"] = scheme
|
||||
elif exc.status_code == 404:
|
||||
code = ErrorCode.NOT_FOUND
|
||||
elif exc.status_code == 409:
|
||||
code = ErrorCode.CONFLICT
|
||||
elif exc.status_code == 413:
|
||||
code = ErrorCode.PAYLOAD_TOO_LARGE
|
||||
elif exc.status_code == 415:
|
||||
code = ErrorCode.UNSUPPORTED_MEDIA_TYPE
|
||||
elif exc.status_code == 429:
|
||||
code = ErrorCode.RATE_LIMITED
|
||||
elif exc.status_code == 503:
|
||||
code = ErrorCode.UNAVAILABLE
|
||||
|
||||
msg = exc.detail if isinstance(exc.detail, str) else "HTTP ошибка"
|
||||
return make_error_response(
|
||||
exc.status_code, code, msg, req_id, headers=extra_headers
|
||||
)
|
||||
|
||||
async def dependency_unavailable_handler(
|
||||
request: Request, exc: Exception
|
||||
) -> JSONResponse:
|
||||
req_id = get_request_id(request)
|
||||
logger.warning(
|
||||
"Upstream storage dependency unavailable for request %s: %s",
|
||||
req_id,
|
||||
exc,
|
||||
)
|
||||
return make_error_response(
|
||||
status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
ErrorCode.UNAVAILABLE,
|
||||
"Зависимость недоступна",
|
||||
req_id,
|
||||
)
|
||||
|
||||
for exc_cls in (
|
||||
UnavailableError,
|
||||
asyncpg.PostgresError,
|
||||
asyncpg.InterfaceError,
|
||||
redis.exceptions.RedisError,
|
||||
botocore.exceptions.BotoCoreError,
|
||||
botocore.exceptions.ClientError,
|
||||
TimeoutError,
|
||||
asyncio.TimeoutError,
|
||||
ConnectionError,
|
||||
OSError,
|
||||
):
|
||||
app.add_exception_handler(exc_cls, dependency_unavailable_handler)
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unhandled_exception_handler(
|
||||
request: Request, exc: Exception
|
||||
) -> JSONResponse:
|
||||
req_id = get_request_id(request)
|
||||
if isinstance(
|
||||
exc,
|
||||
(
|
||||
UnavailableError,
|
||||
asyncpg.PostgresError,
|
||||
asyncpg.InterfaceError,
|
||||
redis.exceptions.RedisError,
|
||||
botocore.exceptions.BotoCoreError,
|
||||
botocore.exceptions.ClientError,
|
||||
TimeoutError,
|
||||
asyncio.TimeoutError,
|
||||
ConnectionError,
|
||||
OSError,
|
||||
),
|
||||
):
|
||||
return await dependency_unavailable_handler(request, exc)
|
||||
|
||||
logger.error(
|
||||
"Unhandled exception for request %s: %s", req_id, exc, exc_info=True
|
||||
)
|
||||
return make_error_response(
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
ErrorCode.INTERNAL_ERROR,
|
||||
"Внутренняя ошибка сервера",
|
||||
req_id,
|
||||
)
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import math
|
||||
from typing import TypedDict
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
MAX_IMAGE_PIXELS = 40_000_000
|
||||
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
|
||||
|
||||
|
||||
class GeneratedVariant(TypedDict):
|
||||
name: str
|
||||
content_type: str
|
||||
data: bytes
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
|
||||
def _calculate_contained_dimensions(orig_w: int, orig_h: int, max_w: int, max_h: int) -> tuple[int, int]:
|
||||
"""Вычисляет размеры, вписанные в max_w x max_h с сохранением пропорций,
|
||||
без увеличения маленьких исходников, с округлением вниз (floor) и минимумом 1px."""
|
||||
if orig_w <= max_w and orig_h <= max_h:
|
||||
return orig_w, orig_h
|
||||
scale = min(max_w / orig_w, max_h / orig_h)
|
||||
new_w = max(1, math.floor(orig_w * scale))
|
||||
new_h = max(1, math.floor(orig_h * scale))
|
||||
return new_w, new_h
|
||||
|
||||
|
||||
def _flatten_to_rgb(img: Image.Image) -> Image.Image:
|
||||
"""Накладывает альфа-канал на белый фон для форматов без поддержки прозрачности (JPEG)."""
|
||||
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
||||
rgba = img.convert("RGBA")
|
||||
background = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
background.paste(rgba, mask=rgba.split()[3])
|
||||
return background
|
||||
if img.mode != "RGB":
|
||||
return img.convert("RGB")
|
||||
return img
|
||||
|
||||
|
||||
def generate_thumbnail(img: Image.Image) -> tuple[bytes, int, int]:
|
||||
"""Генерирует thumbnail: вписан в 256x256 без апскейла, JPEG, alpha на белый фон.
|
||||
|
||||
Размеры рассчитываются строго с округлением вниз (floor) и минимумом 1px.
|
||||
Возвращает (jpeg_bytes, width, height).
|
||||
"""
|
||||
target_w, target_h = _calculate_contained_dimensions(img.width, img.height, 256, 256)
|
||||
if (img.width, img.height) != (target_w, target_h):
|
||||
thumb = img.resize((target_w, target_h), Image.Resampling.LANCZOS)
|
||||
else:
|
||||
thumb = img.copy()
|
||||
rgb = _flatten_to_rgb(thumb)
|
||||
|
||||
buf = io.BytesIO()
|
||||
rgb.save(buf, format="JPEG", quality=85, optimize=True)
|
||||
data = buf.getvalue()
|
||||
return data, rgb.width, rgb.height
|
||||
|
||||
|
||||
def generate_crop(img: Image.Image) -> tuple[bytes, int, int]:
|
||||
"""Генерирует crop: центрированный квадрат 128x128 (с апскейлом для меньших), JPEG.
|
||||
|
||||
Возвращает (jpeg_bytes, width, height).
|
||||
"""
|
||||
cropped = ImageOps.fit(img, (128, 128), method=Image.Resampling.LANCZOS)
|
||||
rgb = _flatten_to_rgb(cropped)
|
||||
|
||||
buf = io.BytesIO()
|
||||
rgb.save(buf, format="JPEG", quality=85, optimize=True)
|
||||
data = buf.getvalue()
|
||||
return data, rgb.width, rgb.height
|
||||
|
||||
|
||||
def generate_optimized(img: Image.Image) -> tuple[bytes, int, int]:
|
||||
"""Генерирует optimized: WebP, вписан в 1280x1280 без апскейла, удаление EXIF.
|
||||
|
||||
Размеры рассчитываются строго с округлением вниз (floor) и минимумом 1px.
|
||||
Возвращает (webp_bytes, width, height).
|
||||
"""
|
||||
target_w, target_h = _calculate_contained_dimensions(img.width, img.height, 1280, 1280)
|
||||
if (img.width, img.height) != (target_w, target_h):
|
||||
opt = img.resize((target_w, target_h), Image.Resampling.LANCZOS)
|
||||
else:
|
||||
opt = img.copy()
|
||||
|
||||
if opt.mode not in ("RGB", "RGBA"):
|
||||
opt = opt.convert("RGBA" if "transparency" in opt.info else "RGB")
|
||||
|
||||
buf = io.BytesIO()
|
||||
opt.save(buf, format="WEBP", quality=80, method=4)
|
||||
data = buf.getvalue()
|
||||
return data, opt.width, opt.height
|
||||
|
||||
|
||||
def process_image_variants(raw_bytes: bytes) -> list[GeneratedVariant]:
|
||||
"""Безопасно декодирует сырые байты оригинала и генерирует thumbnail, crop, optimized.
|
||||
|
||||
Применяет EXIF orientation к базовому изображению.
|
||||
"""
|
||||
with Image.open(io.BytesIO(raw_bytes)) as img:
|
||||
w, h = img.size
|
||||
if w * h > MAX_IMAGE_PIXELS:
|
||||
raise ValueError("decompression bomb: image exceeds pixel limit")
|
||||
img.load()
|
||||
transposed = ImageOps.exif_transpose(img) or img
|
||||
|
||||
thumb_bytes, thumb_w, thumb_h = generate_thumbnail(transposed)
|
||||
crop_bytes, crop_w, crop_h = generate_crop(transposed)
|
||||
opt_bytes, opt_w, opt_h = generate_optimized(transposed)
|
||||
|
||||
def make_entry(name: str, ctype: str, data: bytes, w: int, h: int) -> GeneratedVariant:
|
||||
return {
|
||||
"name": name,
|
||||
"content_type": ctype,
|
||||
"data": data,
|
||||
"size_bytes": len(data),
|
||||
"width": w,
|
||||
"height": h,
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
}
|
||||
|
||||
return [
|
||||
make_entry("thumbnail", "image/jpeg", thumb_bytes, thumb_w, thumb_h),
|
||||
make_entry("crop", "image/jpeg", crop_bytes, crop_w, crop_h),
|
||||
make_entry("optimized", "image/webp", opt_bytes, opt_w, opt_h),
|
||||
]
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI
|
||||
|
||||
from app.config import APP_HOST, APP_PORT, LOG_LEVEL
|
||||
from app.errors import register_error_handlers
|
||||
from app.middleware import RequestIdAndLoggingMiddleware
|
||||
from app.routes.attachments import router as attachments_router
|
||||
from app.routes.auth import router as auth_router
|
||||
from app.routes.chats import router as chats_router
|
||||
from app.routes.health import router as health_router
|
||||
from app.routes.messages import router as messages_router
|
||||
from app.routes.users import router as users_router
|
||||
from app.broker import publisher
|
||||
from app.outbox import outbox_relay
|
||||
from app.storage import storage
|
||||
|
||||
logger = logging.getLogger("app.lifecycle")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
logger.info(
|
||||
"Starting messenger server on %s:%d (env: %s)", APP_HOST, APP_PORT, LOG_LEVEL
|
||||
)
|
||||
await storage.connect()
|
||||
if storage.pg_pool:
|
||||
await outbox_relay.start(storage.pg_pool)
|
||||
yield
|
||||
logger.info("Graceful shutdown: closing connections...")
|
||||
await outbox_relay.stop()
|
||||
await publisher.close()
|
||||
await storage.disconnect()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="highload-messenger",
|
||||
version="1.6.0",
|
||||
docs_url="/docs",
|
||||
redoc_url=None,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
app.add_middleware(RequestIdAndLoggingMiddleware)
|
||||
|
||||
register_error_handlers(app)
|
||||
|
||||
app.include_router(health_router)
|
||||
app.include_router(auth_router)
|
||||
app.include_router(users_router)
|
||||
app.include_router(chats_router)
|
||||
app.include_router(messages_router)
|
||||
app.include_router(attachments_router)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run(
|
||||
"app.main:app",
|
||||
host=APP_HOST,
|
||||
port=APP_PORT,
|
||||
log_level=LOG_LEVEL.lower(),
|
||||
timeout_graceful_shutdown=10,
|
||||
)
|
||||
@@ -0,0 +1,93 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
from app.config import INSTANCE_ID
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
stream=sys.stdout,
|
||||
)
|
||||
logger = logging.getLogger("app.access")
|
||||
|
||||
|
||||
class RequestIdAndLoggingMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(
|
||||
self, request: Request, call_next: RequestResponseEndpoint
|
||||
) -> Response:
|
||||
# Получаем X-Request-Id или генерируем новый
|
||||
request_id = request.headers.get("X-Request-Id")
|
||||
if not request_id or not request_id.strip():
|
||||
request_id = str(uuid.uuid4())
|
||||
|
||||
request.state.request_id = request_id
|
||||
start_time = time.monotonic()
|
||||
|
||||
if request.method in ("POST", "PUT", "PATCH") and request.url.path != "/api/v1/auth/sessions":
|
||||
is_attachment_upload = (
|
||||
request.method == "POST"
|
||||
and "/attachments" in request.url.path
|
||||
and request.url.path.startswith("/api/v1/chats/")
|
||||
)
|
||||
ct = request.headers.get("content-type")
|
||||
ct_clean = ct.split(";")[0].strip().lower() if ct else ""
|
||||
|
||||
if is_attachment_upload:
|
||||
if not ct or ct_clean not in ("image/png", "image/jpeg"):
|
||||
from fastapi import status
|
||||
from app.errors import make_error_response
|
||||
from app.schemas import ErrorCode
|
||||
return make_error_response(
|
||||
status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||
ErrorCode.UNSUPPORTED_MEDIA_TYPE,
|
||||
f"Неподдерживаемый тип содержимого '{ct}'. Ожидается image/png или image/jpeg",
|
||||
request_id,
|
||||
)
|
||||
elif ct and ct_clean != "application/json":
|
||||
from fastapi import status
|
||||
from app.errors import make_error_response
|
||||
from app.schemas import ErrorCode
|
||||
return make_error_response(
|
||||
status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||
ErrorCode.UNSUPPORTED_MEDIA_TYPE,
|
||||
f"Неподдерживаемый тип содержимого '{ct}'. Ожидается application/json",
|
||||
request_id,
|
||||
)
|
||||
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
# Исключение перехватится exception_handler
|
||||
duration_ms = (time.monotonic() - start_time) * 1000
|
||||
logger.error(
|
||||
"%s %s -> 500 error | duration=%.2fms | request_id=%s",
|
||||
request.method,
|
||||
request.url.path,
|
||||
duration_ms,
|
||||
request_id,
|
||||
)
|
||||
raise
|
||||
|
||||
duration_ms = (time.monotonic() - start_time) * 1000
|
||||
response.headers["X-Request-Id"] = request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
|
||||
# лог без логирования тела запросов и текстов сообщений
|
||||
logger.info(
|
||||
"%s %s -> %d | duration=%.2fms | request_id=%s",
|
||||
request.method,
|
||||
request.url.path,
|
||||
response.status_code,
|
||||
duration_ms,
|
||||
request_id,
|
||||
)
|
||||
|
||||
return response
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app.broker import publisher
|
||||
|
||||
logger = logging.getLogger("messenger.outbox")
|
||||
|
||||
|
||||
class OutboxRelay:
|
||||
"""Фоновый диспетчер Transactional Outbox.
|
||||
|
||||
Вычитывает сохранённые задачи из PostgreSQL и надёжно публикует их в RabbitMQ.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._notify_event = asyncio.Event()
|
||||
self._running = False
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
|
||||
def notify(self) -> None:
|
||||
"""Сигнализирует о появлении новой записи для немедленной отправки без ожидания тика."""
|
||||
self._notify_event.set()
|
||||
|
||||
async def start(self, pool: asyncpg.Pool) -> None:
|
||||
if self._running:
|
||||
return
|
||||
self._running = True
|
||||
self._task = asyncio.create_task(self._loop(pool))
|
||||
logger.info("OutboxRelay started")
|
||||
|
||||
async def stop(self) -> None:
|
||||
self._running = False
|
||||
self._notify_event.set()
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._task = None
|
||||
logger.info("OutboxRelay stopped")
|
||||
|
||||
async def process_pending(self, pool: asyncpg.Pool, limit: int = 50) -> int:
|
||||
"""Выбирает и публикует порцию pending-задач с FOR UPDATE SKIP LOCKED."""
|
||||
dispatched_count = 0
|
||||
async with pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT job_id, payload
|
||||
FROM outbox_jobs
|
||||
WHERE status = 'pending'
|
||||
ORDER BY created_at ASC
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""",
|
||||
limit,
|
||||
)
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
for row in rows:
|
||||
job_id = row["job_id"]
|
||||
payload_raw = row["payload"]
|
||||
payload_dict = json.loads(payload_raw) if isinstance(payload_raw, str) else dict(payload_raw)
|
||||
|
||||
success = await publisher.publish_job(payload_dict)
|
||||
if success:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE outbox_jobs
|
||||
SET status = 'published', updated_at = NOW()
|
||||
WHERE job_id = $1
|
||||
""",
|
||||
job_id,
|
||||
)
|
||||
dispatched_count += 1
|
||||
else:
|
||||
logger.warning("Broker reject for job %s, will retry", job_id)
|
||||
break
|
||||
return dispatched_count
|
||||
|
||||
async def _loop(self, pool: asyncpg.Pool) -> None:
|
||||
while self._running:
|
||||
try:
|
||||
dispatched = await self.process_pending(pool)
|
||||
# Если были отправлены задачи, проверяем сразу следующую порцию
|
||||
if dispatched > 0:
|
||||
continue
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as exc:
|
||||
logger.error("Error in OutboxRelay loop: %s", exc)
|
||||
|
||||
# Ждём уведомления о новой задаче или таймаута 1.0 с
|
||||
try:
|
||||
await asyncio.wait_for(self._notify_event.wait(), timeout=1.0)
|
||||
self._notify_event.clear()
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
|
||||
outbox_relay = OutboxRelay()
|
||||
@@ -0,0 +1,85 @@
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Request, Response, status
|
||||
|
||||
from app.dependencies import get_current_user
|
||||
from app.errors import InvalidRequestError
|
||||
from app.schemas import AttachmentResponse, UserResponse
|
||||
from app.storage import storage
|
||||
|
||||
router = APIRouter(tags=["attachments"])
|
||||
|
||||
|
||||
def validate_uuid(value: str, field_name: str = "id") -> str:
|
||||
try:
|
||||
UUID(value)
|
||||
return value
|
||||
except ValueError as exc:
|
||||
raise InvalidRequestError(f"Поле '{field_name}' не является валидным UUID") from exc
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/chats/{chat_id}/attachments",
|
||||
response_model=AttachmentResponse,
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
)
|
||||
async def upload_attachment(
|
||||
chat_id: str,
|
||||
request: Request,
|
||||
filename: str = Query(..., min_length=1, max_length=255),
|
||||
current_user: UserResponse = Depends(get_current_user),
|
||||
) -> AttachmentResponse:
|
||||
"""Загрузка бинарного вложения в чат."""
|
||||
valid_chat_id = validate_uuid(chat_id, "chat_id")
|
||||
content_type = request.headers.get("content-type") or ""
|
||||
data = await request.body()
|
||||
|
||||
return await storage.upload_attachment(
|
||||
caller_id=current_user.id,
|
||||
chat_id=valid_chat_id,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
data=data,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/attachments/{attachment_id}",
|
||||
response_model=AttachmentResponse,
|
||||
status_code=status.HTTP_200_OK,
|
||||
)
|
||||
async def get_attachment_metadata(
|
||||
attachment_id: str,
|
||||
current_user: UserResponse = Depends(get_current_user),
|
||||
) -> AttachmentResponse:
|
||||
"""Получение метаданных вложения."""
|
||||
valid_att_id = validate_uuid(attachment_id, "attachment_id")
|
||||
return await storage.get_attachment(
|
||||
caller_id=current_user.id,
|
||||
attachment_id=valid_att_id,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/attachments/{attachment_id}/content",
|
||||
status_code=status.HTTP_200_OK,
|
||||
)
|
||||
async def download_attachment_content(
|
||||
attachment_id: str,
|
||||
variant: str = Query(default="original"),
|
||||
current_user: UserResponse = Depends(get_current_user),
|
||||
) -> Response:
|
||||
"""Скачивание бинарного содержимого вложения."""
|
||||
valid_att_id = validate_uuid(attachment_id, "attachment_id")
|
||||
data, content_type = await storage.get_attachment_content(
|
||||
caller_id=current_user.id,
|
||||
attachment_id=valid_att_id,
|
||||
variant=variant,
|
||||
)
|
||||
return Response(
|
||||
content=data,
|
||||
media_type=content_type,
|
||||
headers={
|
||||
"Content-Length": str(len(data)),
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,63 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from fastapi import APIRouter, Depends, Header, Request, Response, status
|
||||
|
||||
from app.dependencies import get_current_token
|
||||
from app.errors import InvalidRequestError, UnauthorizedError
|
||||
from app.schemas import SessionResponse
|
||||
from app.storage import storage
|
||||
|
||||
router = APIRouter(prefix="/api/v1/auth/sessions", tags=["auth"])
|
||||
|
||||
|
||||
@router.post("", response_model=SessionResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def create_session(
|
||||
request: Request,
|
||||
authorization: str | None = Header(default=None, alias="Authorization"),
|
||||
) -> SessionResponse:
|
||||
"""Аутентификация пользователя через Basic Auth и создание новой сессии."""
|
||||
body = await request.body()
|
||||
if body and body.strip():
|
||||
raise InvalidRequestError("Тело запроса для создания сессии должно быть пустым")
|
||||
|
||||
if not authorization or not authorization.strip():
|
||||
raise UnauthorizedError("Отсутствует заголовок Authorization")
|
||||
|
||||
parts = authorization.strip().split(maxsplit=1)
|
||||
if len(parts) != 2 or parts[0].lower() != "basic":
|
||||
raise UnauthorizedError("Неверная схема авторизации (ожидается Basic)")
|
||||
|
||||
try:
|
||||
raw_bytes = base64.b64decode(parts[1].strip())
|
||||
raw_str = raw_bytes.decode("utf-8")
|
||||
if ":" not in raw_str:
|
||||
raise UnauthorizedError("Некорректный формат Basic credentials")
|
||||
username, password = raw_str.split(":", 1)
|
||||
except Exception:
|
||||
raise UnauthorizedError("Некорректный заголовок Basic авторизации")
|
||||
|
||||
user = await storage.authenticate_user(username, password)
|
||||
if not user:
|
||||
raise UnauthorizedError("Неверное имя пользователя или пароль")
|
||||
|
||||
session = await storage.create_session(user)
|
||||
return session
|
||||
|
||||
|
||||
@router.delete("/current", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_current_session(
|
||||
request: Request,
|
||||
token: str = Depends(get_current_token),
|
||||
) -> Response:
|
||||
"""Отзыв текущей сессии."""
|
||||
body = await request.body()
|
||||
if body and body.strip():
|
||||
raise InvalidRequestError("Тело запроса для отзыва сессии должно быть пустым")
|
||||
|
||||
user = await storage.get_session_user(token)
|
||||
if not user:
|
||||
raise UnauthorizedError("Недействительная или уже отозванная сессия")
|
||||
|
||||
await storage.delete_session(token)
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
@@ -0,0 +1,64 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, status
|
||||
|
||||
from app.dependencies import get_current_user
|
||||
from app.schemas import (
|
||||
ChatPageResponse,
|
||||
ChatResponse,
|
||||
CreateChatRequest,
|
||||
UserResponse,
|
||||
)
|
||||
from app.storage import InvalidRequestError, storage
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
|
||||
|
||||
|
||||
def validate_uuid(value: str, field_name: str = "id") -> str:
|
||||
try:
|
||||
UUID(value)
|
||||
return value
|
||||
except ValueError as exc:
|
||||
raise InvalidRequestError(f"Поле '{field_name}' не является валидным UUID") from exc
|
||||
|
||||
|
||||
@router.post("", response_model=ChatResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def create_chat(
|
||||
body: CreateChatRequest,
|
||||
current_user: UserResponse = Depends(get_current_user),
|
||||
) -> ChatResponse:
|
||||
"""Создание нового чата."""
|
||||
chat = await storage.create_chat(
|
||||
caller_id=current_user.id,
|
||||
title=body.title,
|
||||
member_ids=body.member_ids,
|
||||
)
|
||||
return chat
|
||||
|
||||
|
||||
@router.get("", response_model=ChatPageResponse, status_code=status.HTTP_200_OK)
|
||||
async def list_chats(
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: str | None = Query(default=None),
|
||||
current_user: UserResponse = Depends(get_current_user),
|
||||
) -> ChatPageResponse:
|
||||
"""Список чатов, в которых состоит текущий пользователь."""
|
||||
items, next_cursor = await storage.list_chats(
|
||||
caller_id=current_user.id,
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
)
|
||||
return ChatPageResponse(items=items, next_cursor=next_cursor)
|
||||
|
||||
|
||||
@router.get("/{chat_id}", response_model=ChatResponse, status_code=status.HTTP_200_OK)
|
||||
async def get_chat(
|
||||
chat_id: str,
|
||||
current_user: UserResponse = Depends(get_current_user),
|
||||
) -> ChatResponse:
|
||||
"""Получение информации о чате по ID."""
|
||||
valid_id = validate_uuid(chat_id, "chat_id")
|
||||
chat = await storage.get_chat(caller_id=current_user.id, chat_id=valid_id)
|
||||
return chat
|
||||
@@ -0,0 +1,31 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, status
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from app.schemas import HealthResponse
|
||||
from app.storage import storage
|
||||
|
||||
router = APIRouter(tags=["health"])
|
||||
|
||||
|
||||
@router.get(
|
||||
"/health/live", response_model=HealthResponse, status_code=status.HTTP_200_OK
|
||||
)
|
||||
async def health_live() -> HealthResponse:
|
||||
"""Проверка жизни процесса (Liveness)."""
|
||||
return HealthResponse(status="ok")
|
||||
|
||||
|
||||
@router.get(
|
||||
"/health/ready", response_model=HealthResponse, status_code=status.HTTP_200_OK
|
||||
)
|
||||
async def health_ready() -> HealthResponse:
|
||||
"""Проверка готовности сервиса и его зависимостей (Readiness)."""
|
||||
is_ready = await storage.check_health()
|
||||
if not is_ready:
|
||||
raise StarletteHTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="Сервис или его хранилища недоступны",
|
||||
)
|
||||
return HealthResponse(status="ok")
|
||||
@@ -0,0 +1,74 @@
|
||||
import re
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, Query, status
|
||||
|
||||
from app.dependencies import get_current_user
|
||||
from app.schemas import (
|
||||
CreateMessageRequest,
|
||||
MessagePageResponse,
|
||||
MessageResponse,
|
||||
UserResponse,
|
||||
)
|
||||
from app.storage import InvalidRequestError, storage
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats", tags=["messages"])
|
||||
|
||||
IDEMPOTENCY_KEY_REGEX = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
|
||||
|
||||
def validate_uuid(value: str, field_name: str = "id") -> str:
|
||||
try:
|
||||
UUID(value)
|
||||
return value
|
||||
except ValueError as exc:
|
||||
raise InvalidRequestError(f"Поле '{field_name}' не является валидным UUID") from exc
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{chat_id}/messages",
|
||||
response_model=MessageResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
async def create_message(
|
||||
chat_id: str,
|
||||
body: CreateMessageRequest,
|
||||
idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"),
|
||||
current_user: UserResponse = Depends(get_current_user),
|
||||
) -> MessageResponse:
|
||||
"""Отправка сообщения в чат."""
|
||||
if not idempotency_key or not IDEMPOTENCY_KEY_REGEX.match(idempotency_key):
|
||||
raise InvalidRequestError(
|
||||
"Заголовок 'Idempotency-Key' обязателен и должен соответствовать шаблону ^[A-Za-z0-9._:-]{1,128}$"
|
||||
)
|
||||
valid_id = validate_uuid(chat_id, "chat_id")
|
||||
msg = await storage.send_message(
|
||||
caller_id=current_user.id,
|
||||
chat_id=valid_id,
|
||||
text=body.text,
|
||||
idempotency_key=idempotency_key,
|
||||
attachment_ids=body.attachment_ids,
|
||||
)
|
||||
return msg
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{chat_id}/messages",
|
||||
response_model=MessagePageResponse,
|
||||
status_code=status.HTTP_200_OK,
|
||||
)
|
||||
async def list_messages(
|
||||
chat_id: str,
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: str | None = Query(default=None),
|
||||
current_user: UserResponse = Depends(get_current_user),
|
||||
) -> MessagePageResponse:
|
||||
"""Получение истории сообщений чата с keyset-пагинацией."""
|
||||
valid_id = validate_uuid(chat_id, "chat_id")
|
||||
items, next_cursor = await storage.list_messages(
|
||||
caller_id=current_user.id,
|
||||
chat_id=valid_id,
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
)
|
||||
return MessagePageResponse(items=items, next_cursor=next_cursor)
|
||||
@@ -0,0 +1,28 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, status
|
||||
|
||||
from app.dependencies import get_current_user
|
||||
from app.schemas import CreateUserRequest, UserResponse
|
||||
from app.storage import storage
|
||||
|
||||
router = APIRouter(prefix="/api/v1/users", tags=["users"])
|
||||
|
||||
|
||||
@router.post("", response_model=UserResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def create_user(body: CreateUserRequest) -> UserResponse:
|
||||
"""Регистрация нового пользователя."""
|
||||
user = await storage.create_user(
|
||||
username=body.username,
|
||||
display_name=body.display_name,
|
||||
password=body.password,
|
||||
)
|
||||
return user
|
||||
|
||||
|
||||
@router.get("/me", response_model=UserResponse, status_code=status.HTTP_200_OK)
|
||||
async def get_me(
|
||||
current_user: UserResponse = Depends(get_current_user),
|
||||
) -> UserResponse:
|
||||
"""Получение профиля текущего пользователя."""
|
||||
return current_user
|
||||
+221
@@ -0,0 +1,221 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import re
|
||||
from enum import Enum
|
||||
from uuid import UUID
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
def now_utc_iso() -> str:
|
||||
"""Возвращает текущее время в формате RFC 3339 UTC с суффиксом Z."""
|
||||
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
|
||||
|
||||
class ErrorCode(str, Enum):
|
||||
INVALID_REQUEST = "invalid_request"
|
||||
UNAUTHORIZED = "unauthorized"
|
||||
NOT_FOUND = "not_found"
|
||||
CONFLICT = "conflict"
|
||||
PAYLOAD_TOO_LARGE = "payload_too_large"
|
||||
UNSUPPORTED_MEDIA_TYPE = "unsupported_media_type"
|
||||
RATE_LIMITED = "rate_limited"
|
||||
UNAVAILABLE = "unavailable"
|
||||
INTERNAL_ERROR = "internal_error"
|
||||
|
||||
|
||||
class ErrorDetail(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
code: ErrorCode
|
||||
message: str = Field(min_length=1)
|
||||
request_id: str = Field(min_length=1)
|
||||
|
||||
|
||||
class ErrorResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
error: ErrorDetail
|
||||
|
||||
|
||||
class HealthResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
status: str = Field(default="ok")
|
||||
|
||||
|
||||
class CreateUserRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
|
||||
username: str = Field(min_length=3, max_length=32)
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def validate_username(cls, v: str) -> str:
|
||||
if not re.match(r"^[a-z0-9_]{3,32}\Z", v):
|
||||
raise ValueError("Имя пользователя должно соответствовать шаблону ^[a-z0-9_]{3,32}$")
|
||||
return v
|
||||
|
||||
@field_validator("display_name")
|
||||
@classmethod
|
||||
def validate_display_name(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("Отображаемое имя не может состоять только из пробелов")
|
||||
return v
|
||||
|
||||
@field_validator("password")
|
||||
@classmethod
|
||||
def validate_password(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("Пароль не может состоять только из пробелов")
|
||||
return v
|
||||
|
||||
|
||||
class UserResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class SessionResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
token: str = Field(min_length=32)
|
||||
token_type: str = Field(default="Bearer")
|
||||
expires_at: str
|
||||
user: UserResponse
|
||||
|
||||
|
||||
class CreateChatRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[str] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def validate_title(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("Название не может состоять только из пробелов")
|
||||
return v
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def validate_member_ids(cls, v: list[str]) -> list[str]:
|
||||
if len(v) != len(set(v)):
|
||||
raise ValueError("Идентификаторы участников должны быть уникальными")
|
||||
for member_id in v:
|
||||
try:
|
||||
UUID(member_id)
|
||||
except ValueError:
|
||||
raise ValueError(f"Идентификатор участника '{member_id}' не является валидным UUID")
|
||||
return v
|
||||
|
||||
|
||||
class ChatResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
title: str
|
||||
member_ids: list[str]
|
||||
created_at: str
|
||||
|
||||
|
||||
class ChatPageResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
items: list[ChatResponse]
|
||||
next_cursor: str | None = None
|
||||
|
||||
|
||||
class VariantResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: Literal["thumbnail", "crop", "optimized"]
|
||||
content_type: Literal["image/jpeg", "image/png", "image/webp"]
|
||||
size_bytes: int = Field(ge=1)
|
||||
width: int = Field(ge=1)
|
||||
height: int = Field(ge=1)
|
||||
sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
|
||||
class AttachmentResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: Literal["queued", "processing", "ready", "failed"] = "queued"
|
||||
variants: list[VariantResponse] = Field(default_factory=list)
|
||||
error_code: str | None = None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
|
||||
class AttachmentJobMessage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
schema_version: Literal[1] = 1
|
||||
job_id: str
|
||||
type: Literal["attachment.process.v1"] = "attachment.process.v1"
|
||||
attachment_id: str
|
||||
pipeline_version: int = 1
|
||||
attempt: int = 1
|
||||
created_at: str
|
||||
|
||||
|
||||
class CreateMessageRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
|
||||
text: str = Field(default="", max_length=4000)
|
||||
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def validate_attachment_ids(cls, v: list[str]) -> list[str]:
|
||||
if len(v) != len(set(v)):
|
||||
raise ValueError("attachment_ids must contain unique items")
|
||||
for item in v:
|
||||
try:
|
||||
UUID(item)
|
||||
except ValueError:
|
||||
raise ValueError(f"Идентификатор вложения '{item}' не является валидным UUID")
|
||||
return v
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_text_or_attachment(self) -> CreateMessageRequest:
|
||||
has_text = bool(self.text and self.text.strip())
|
||||
has_attachments = bool(self.attachment_ids)
|
||||
if not has_text and not has_attachments:
|
||||
raise ValueError("Сообщение должно содержать непустой текст или хотя бы одно вложение")
|
||||
return self
|
||||
|
||||
|
||||
class MessageResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
attachment_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class MessagePageResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
items: list[MessageResponse]
|
||||
next_cursor: str | None = None
|
||||
+1001
File diff suppressed because it is too large
Load Diff
+436
@@ -0,0 +1,436 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import signal
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
import aio_pika
|
||||
import asyncpg
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from PIL import UnidentifiedImageError
|
||||
|
||||
from app.broker import (
|
||||
DL_ROUTING_KEY,
|
||||
DLX_NAME,
|
||||
EXCHANGE_NAME,
|
||||
QUEUE_NAME,
|
||||
ROUTING_KEY,
|
||||
setup_broker_topology,
|
||||
)
|
||||
from app.config import (
|
||||
BROKER_URL,
|
||||
DATABASE_URL,
|
||||
MAX_JOB_RETRIES,
|
||||
S3_ACCESS_KEY_ID,
|
||||
S3_BUCKET,
|
||||
S3_ENDPOINT,
|
||||
S3_REGION,
|
||||
S3_SECRET_ACCESS_KEY,
|
||||
WORKER_CONCURRENCY,
|
||||
WORKER_LEASE_SECONDS,
|
||||
)
|
||||
from app.imaging import process_image_variants
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
)
|
||||
logger = logging.getLogger("messenger.worker")
|
||||
|
||||
|
||||
class AttachmentWorker:
|
||||
def __init__(self) -> None:
|
||||
self.worker_id = f"worker-{uuid.uuid4().hex[:8]}"
|
||||
self.pool: asyncpg.Pool | None = None
|
||||
self.s3_client: Any = None
|
||||
self._connection: aio_pika.abc.AbstractRobustConnection | None = None
|
||||
self._channel: aio_pika.abc.AbstractChannel | None = None
|
||||
self._running = False
|
||||
self._watchdog_task: asyncio.Task[None] | None = None
|
||||
|
||||
async def init_resources(self) -> None:
|
||||
logger.info("Initializing worker resources (%s)...", self.worker_id)
|
||||
# 1. PostgreSQL pool
|
||||
self.pool = await asyncpg.create_pool(
|
||||
dsn=DATABASE_URL,
|
||||
min_size=2,
|
||||
max_size=WORKER_CONCURRENCY + 2,
|
||||
command_timeout=10.0,
|
||||
)
|
||||
|
||||
# 2. S3 client
|
||||
s3_cfg = Config(
|
||||
signature_version="s3v4",
|
||||
connect_timeout=3,
|
||||
read_timeout=10,
|
||||
retries={"max_attempts": 3, "mode": "standard"},
|
||||
)
|
||||
self.s3_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
region_name=S3_REGION,
|
||||
aws_access_key_id=S3_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
|
||||
config=s3_cfg,
|
||||
)
|
||||
|
||||
async def _lease_watchdog(self) -> None:
|
||||
while self._running:
|
||||
try:
|
||||
await asyncio.sleep(15.0)
|
||||
if not self.pool:
|
||||
continue
|
||||
async with self.pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'queued',
|
||||
processing_started_at = NULL,
|
||||
processing_worker_id = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE status = 'processing'
|
||||
AND processing_started_at < NOW() - ($1 || ' seconds')::interval
|
||||
RETURNING id
|
||||
""",
|
||||
str(WORKER_LEASE_SECONDS),
|
||||
)
|
||||
for r in rows:
|
||||
logger.warning(
|
||||
"Lease expired for attachment %s, reverting to queued and republishing", r["id"]
|
||||
)
|
||||
if self._channel and not self._channel.is_closed:
|
||||
try:
|
||||
exchange = await self._channel.get_exchange(EXCHANGE_NAME)
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"job_id": str(uuid.uuid4()),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": str(r["id"]),
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
}
|
||||
msg = aio_pika.Message(
|
||||
body=json.dumps(payload).encode("utf-8"),
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
content_type="application/json",
|
||||
)
|
||||
await exchange.publish(msg, routing_key=ROUTING_KEY)
|
||||
except Exception as pub_err:
|
||||
logger.error("Failed republish on lease expiration for %s: %s", r["id"], pub_err)
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as exc:
|
||||
logger.error("Error in lease watchdog: %s", exc)
|
||||
|
||||
async def process_job_message(self, message: aio_pika.abc.AbstractIncomingMessage) -> None:
|
||||
"""Обрабатывает одно задание на генерацию вариантов изображения."""
|
||||
async with message.process(requeue=False):
|
||||
try:
|
||||
payload = json.loads(message.body.decode("utf-8"))
|
||||
except Exception as parse_err:
|
||||
logger.error("Failed to parse job JSON: %s. Acking.", parse_err)
|
||||
return
|
||||
|
||||
att_id_str = payload.get("attachment_id")
|
||||
if not att_id_str:
|
||||
logger.error("Job payload missing attachment_id: %s", payload)
|
||||
return
|
||||
|
||||
try:
|
||||
att_id = uuid.UUID(att_id_str)
|
||||
except ValueError:
|
||||
logger.error("Invalid attachment_id UUID: %s", att_id_str)
|
||||
return
|
||||
|
||||
attempt = int(payload.get("attempt", 1))
|
||||
|
||||
if not self.pool:
|
||||
logger.error("DB pool not initialized")
|
||||
return
|
||||
|
||||
async with self.pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'processing',
|
||||
processing_started_at = NOW(),
|
||||
processing_worker_id = $1,
|
||||
updated_at = NOW()
|
||||
WHERE id = $2 AND status IN ('queued', 'failed')
|
||||
RETURNING id, chat_id, filename, content_type, size_bytes, sha256, s3_key, status
|
||||
""",
|
||||
self.worker_id,
|
||||
att_id,
|
||||
)
|
||||
if not row:
|
||||
current_status = await conn.fetchval(
|
||||
"SELECT status FROM attachments WHERE id = $1", att_id
|
||||
)
|
||||
logger.info(
|
||||
"Attachment %s is in state '%s', skipping processing (idempotent no-op).",
|
||||
att_id,
|
||||
current_status,
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
async with asyncio.timeout(30.0):
|
||||
staging_key = row["s3_key"]
|
||||
try:
|
||||
s3_resp = await asyncio.to_thread(
|
||||
self.s3_client.get_object,
|
||||
Bucket=S3_BUCKET,
|
||||
Key=staging_key,
|
||||
)
|
||||
raw_bytes = s3_resp["Body"].read()
|
||||
except Exception as s3_err:
|
||||
logger.error("Failed to read staging S3 key %s: %s", staging_key, s3_err)
|
||||
await self._handle_retryable_error(payload, attempt, str(s3_err))
|
||||
return
|
||||
|
||||
# 3. Генерируем варианты через Pillow
|
||||
try:
|
||||
variants = await asyncio.to_thread(process_image_variants, raw_bytes)
|
||||
except (UnidentifiedImageError, ValueError) as img_err:
|
||||
logger.warning(
|
||||
"Permanent image processing error for %s: %s", att_id, img_err
|
||||
)
|
||||
await self._fail_attachment(att_id, "invalid_image")
|
||||
await self._send_to_dlq(payload, f"permanent_image_error: {img_err}")
|
||||
return
|
||||
except Exception as unk_err:
|
||||
logger.error(
|
||||
"Unexpected processing error for %s: %s", att_id, unk_err
|
||||
)
|
||||
await self._handle_retryable_error(payload, attempt, str(unk_err))
|
||||
return
|
||||
|
||||
# 4. Сохраняем оригинал и варианты в постоянные S3-ключи
|
||||
chat_id = str(row["chat_id"])
|
||||
final_original_key = f"chats/{chat_id}/{att_id}/original"
|
||||
|
||||
try:
|
||||
# Оригинал
|
||||
await asyncio.to_thread(
|
||||
self.s3_client.put_object,
|
||||
Bucket=S3_BUCKET,
|
||||
Key=final_original_key,
|
||||
Body=raw_bytes,
|
||||
ContentType=row["content_type"],
|
||||
)
|
||||
# Варианты
|
||||
for var in variants:
|
||||
var_key = f"chats/{chat_id}/{att_id}/{var['name']}"
|
||||
await asyncio.to_thread(
|
||||
self.s3_client.put_object,
|
||||
Bucket=S3_BUCKET,
|
||||
Key=var_key,
|
||||
Body=var["data"],
|
||||
ContentType=var["content_type"],
|
||||
)
|
||||
except Exception as upload_err:
|
||||
logger.error("Failed uploading variants to S3 for %s: %s", att_id, upload_err)
|
||||
await self._handle_retryable_error(payload, attempt, str(upload_err))
|
||||
return
|
||||
|
||||
# 5. Атомарно обновляем PostgreSQL в 'ready'
|
||||
variants_metadata = [
|
||||
{
|
||||
"name": v["name"],
|
||||
"content_type": v["content_type"],
|
||||
"size_bytes": v["size_bytes"],
|
||||
"width": v["width"],
|
||||
"height": v["height"],
|
||||
"sha256": v["sha256"],
|
||||
}
|
||||
for v in variants
|
||||
]
|
||||
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'ready',
|
||||
s3_key = $1,
|
||||
variants = $2::jsonb,
|
||||
error_code = NULL,
|
||||
processing_started_at = NULL,
|
||||
processing_worker_id = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE id = $3
|
||||
""",
|
||||
final_original_key,
|
||||
json.dumps(variants_metadata),
|
||||
att_id,
|
||||
)
|
||||
|
||||
# 6. Очистка staging-файла в MinIO
|
||||
if staging_key != final_original_key:
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
self.s3_client.delete_object,
|
||||
Bucket=S3_BUCKET,
|
||||
Key=staging_key,
|
||||
)
|
||||
except Exception as del_err:
|
||||
logger.warning("Failed to delete staging key %s: %s", staging_key, del_err)
|
||||
|
||||
logger.info("Attachment %s successfully processed into ready with 3 variants.", att_id)
|
||||
except TimeoutError:
|
||||
logger.error("Job processing timed out (>30s) for %s. Aborting hung worker task.", att_id)
|
||||
await self._handle_retryable_error(payload, attempt, "job_deadline_exceeded_30s")
|
||||
return
|
||||
|
||||
async def _fail_attachment(self, att_id: uuid.UUID, error_code: str) -> None:
|
||||
"""Переводит вложение в статус failed с безопасным машинным кодом."""
|
||||
if not self.pool:
|
||||
return
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'failed',
|
||||
error_code = $1,
|
||||
processing_started_at = NULL,
|
||||
processing_worker_id = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE id = $2
|
||||
""",
|
||||
error_code,
|
||||
att_id,
|
||||
)
|
||||
|
||||
async def _send_to_dlq(self, payload: dict[str, Any], reason: str) -> None:
|
||||
try:
|
||||
if self._channel and not self._channel.is_closed:
|
||||
dlx = await self._channel.get_exchange(DLX_NAME)
|
||||
dlq_payload = dict(payload, error="permanent_or_exhausted", last_reason=reason)
|
||||
dlq_msg = aio_pika.Message(
|
||||
body=json.dumps(dlq_payload).encode("utf-8"),
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
content_type="application/json",
|
||||
)
|
||||
await dlx.publish(dlq_msg, routing_key=DL_ROUTING_KEY)
|
||||
except Exception as dlx_err:
|
||||
logger.error("Failed publishing to DLQ: %s", dlx_err)
|
||||
|
||||
async def _handle_retryable_error(self, payload: dict[str, Any], attempt: int, reason: str) -> None:
|
||||
att_id_str = payload.get("attachment_id")
|
||||
att_id = uuid.UUID(att_id_str)
|
||||
|
||||
if attempt < MAX_JOB_RETRIES:
|
||||
next_attempt = attempt + 1
|
||||
delay = 0.5 * (2 ** attempt) + random.uniform(0.1, 0.5)
|
||||
logger.warning(
|
||||
"Retryable error for %s (attempt %d/%d). Retrying in %.2fs: %s",
|
||||
att_id, attempt, MAX_JOB_RETRIES, delay, reason
|
||||
)
|
||||
if self.pool:
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'queued',
|
||||
processing_started_at = NULL,
|
||||
processing_worker_id = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
""",
|
||||
att_id,
|
||||
)
|
||||
|
||||
async def _delayed_republish() -> None:
|
||||
await asyncio.sleep(delay)
|
||||
try:
|
||||
if self._channel and not self._channel.is_closed:
|
||||
exchange = await self._channel.get_exchange(EXCHANGE_NAME)
|
||||
new_payload = dict(payload, attempt=next_attempt)
|
||||
msg = aio_pika.Message(
|
||||
body=json.dumps(new_payload).encode("utf-8"),
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
content_type="application/json",
|
||||
)
|
||||
await exchange.publish(msg, routing_key=ROUTING_KEY)
|
||||
except Exception as pub_err:
|
||||
logger.error("Failed delayed republish for %s: %s", att_id, pub_err)
|
||||
|
||||
asyncio.create_task(_delayed_republish())
|
||||
else:
|
||||
logger.error(
|
||||
"Max retries (%d) exhausted for attachment %s. Moving to DLQ.",
|
||||
MAX_JOB_RETRIES, att_id
|
||||
)
|
||||
await self._fail_attachment(att_id, "retry_exhausted")
|
||||
await self._send_to_dlq(payload, f"retry_exhausted: {reason}")
|
||||
|
||||
async def start(self) -> None:
|
||||
self._running = True
|
||||
await self.init_resources()
|
||||
|
||||
# Подключение к RabbitMQ с автоматическим reconnect
|
||||
logger.info("Connecting to RabbitMQ at %s...", BROKER_URL.split("@")[-1])
|
||||
while self._running:
|
||||
try:
|
||||
self._connection = await aio_pika.connect_robust(BROKER_URL)
|
||||
break
|
||||
except Exception as conn_err:
|
||||
logger.warning("RabbitMQ not ready yet (%s). Retrying in 2s...", conn_err)
|
||||
await asyncio.sleep(2.0)
|
||||
|
||||
assert self._connection is not None
|
||||
self._channel = await self._connection.channel()
|
||||
await self._channel.set_qos(prefetch_count=WORKER_CONCURRENCY)
|
||||
|
||||
_, queue = await setup_broker_topology(self._channel)
|
||||
|
||||
self._watchdog_task = asyncio.create_task(self._lease_watchdog())
|
||||
logger.info(
|
||||
"Worker %s started consuming queue '%s' (prefetch=%d)...",
|
||||
self.worker_id, QUEUE_NAME, WORKER_CONCURRENCY
|
||||
)
|
||||
|
||||
await queue.consume(self.process_job_message)
|
||||
|
||||
async def stop(self) -> None:
|
||||
logger.info("Stopping worker %s...", self.worker_id)
|
||||
self._running = False
|
||||
if self._watchdog_task:
|
||||
self._watchdog_task.cancel()
|
||||
if self._channel and not self._channel.is_closed:
|
||||
await self._channel.close()
|
||||
if self._connection and not self._connection.is_closed:
|
||||
await self._connection.close()
|
||||
if self.pool:
|
||||
await self.pool.close()
|
||||
logger.info("Worker stopped.")
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
worker = AttachmentWorker()
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
stop_event = asyncio.Event()
|
||||
|
||||
def _sig_handler() -> None:
|
||||
logger.info("Received termination signal")
|
||||
stop_event.set()
|
||||
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
try:
|
||||
loop.add_signal_handler(sig, _sig_handler)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
|
||||
await worker.start()
|
||||
await stop_event.wait()
|
||||
await worker.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
+259
@@ -0,0 +1,259 @@
|
||||
services:
|
||||
load_balancer:
|
||||
image: nginx:1.27-alpine
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
depends_on:
|
||||
api-1:
|
||||
condition: service_healthy
|
||||
api-2:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/live"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.25'
|
||||
memory: 128M
|
||||
|
||||
migration:
|
||||
image: postgres:16-alpine
|
||||
restart: "no"
|
||||
environment:
|
||||
PGPASSWORD: postgres
|
||||
command: >
|
||||
sh -c "until pg_isready -h postgres -U postgres -d messenger; do sleep 1; done;
|
||||
psql -h postgres -U postgres -d messenger -f /docker-entrypoint-initdb.d/init.sql"
|
||||
volumes:
|
||||
- ./infra/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.25'
|
||||
memory: 128M
|
||||
|
||||
api-1:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: highload-messenger:lab6
|
||||
stop_grace_period: 15s
|
||||
environment:
|
||||
APP_HOST: "0.0.0.0"
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: "lab"
|
||||
INSTANCE_ID: "api-1"
|
||||
SESSION_TTL_SECONDS: "3600"
|
||||
DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/messenger"
|
||||
SESSION_STORE: "redis"
|
||||
CACHE_URL: "redis://redis:6379/0"
|
||||
IDEMPOTENCY_TTL_SECONDS: "86400"
|
||||
BROKER_URL: "amqp://messenger:messenger_broker_pass@broker:5672/"
|
||||
S3_ENDPOINT: "http://minio:9000"
|
||||
S3_REGION: "us-east-1"
|
||||
S3_BUCKET: "messenger-attachments"
|
||||
S3_ACCESS_KEY_ID: "minioadmin"
|
||||
S3_SECRET_ACCESS_KEY: "minioadmin"
|
||||
MAX_UPLOAD_BYTES: "10485760"
|
||||
MAX_IMAGE_PIXELS: "40000000"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
migration:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.50'
|
||||
memory: 512M
|
||||
|
||||
api-2:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: highload-messenger:lab6
|
||||
stop_grace_period: 15s
|
||||
environment:
|
||||
APP_HOST: "0.0.0.0"
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: "lab"
|
||||
INSTANCE_ID: "api-2"
|
||||
SESSION_TTL_SECONDS: "3600"
|
||||
DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/messenger"
|
||||
SESSION_STORE: "redis"
|
||||
CACHE_URL: "redis://redis:6379/0"
|
||||
IDEMPOTENCY_TTL_SECONDS: "86400"
|
||||
BROKER_URL: "amqp://messenger:messenger_broker_pass@broker:5672/"
|
||||
S3_ENDPOINT: "http://minio:9000"
|
||||
S3_REGION: "us-east-1"
|
||||
S3_BUCKET: "messenger-attachments"
|
||||
S3_ACCESS_KEY_ID: "minioadmin"
|
||||
S3_SECRET_ACCESS_KEY: "minioadmin"
|
||||
MAX_UPLOAD_BYTES: "10485760"
|
||||
MAX_IMAGE_PIXELS: "40000000"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
migration:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.50'
|
||||
memory: 512M
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: highload-messenger:lab6
|
||||
command: python -m app.worker
|
||||
stop_grace_period: 15s
|
||||
environment:
|
||||
DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/messenger"
|
||||
BROKER_URL: "amqp://messenger:messenger_broker_pass@broker:5672/"
|
||||
S3_ENDPOINT: "http://minio:9000"
|
||||
S3_REGION: "us-east-1"
|
||||
S3_BUCKET: "messenger-attachments"
|
||||
S3_ACCESS_KEY_ID: "minioadmin"
|
||||
S3_SECRET_ACCESS_KEY: "minioadmin"
|
||||
WORKER_CONCURRENCY: "4"
|
||||
WORKER_LEASE_SECONDS: "60"
|
||||
MAX_JOB_RETRIES: "5"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c 'import pika, os; conn = pika.BlockingConnection(pika.URLParameters(os.environ[\"BROKER_URL\"])); conn.close()' || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
migration:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.50'
|
||||
memory: 512M
|
||||
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: messenger
|
||||
volumes:
|
||||
- pg_data:/var/lib/postgresql/data
|
||||
- ./infra/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.50'
|
||||
memory: 512M
|
||||
|
||||
broker:
|
||||
image: rabbitmq:3.13-alpine
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: messenger
|
||||
RABBITMQ_DEFAULT_PASS: messenger_broker_pass
|
||||
volumes:
|
||||
- rabbitmq_data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 3s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.50'
|
||||
memory: 512M
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
command: ["redis-server", "--appendonly", "yes"]
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.25'
|
||||
memory: 256M
|
||||
|
||||
minio:
|
||||
image: cleanstart/minio:0.20260330.001845
|
||||
user: "0:0"
|
||||
command: server /data --address ":9000" --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/live"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.50'
|
||||
memory: 512M
|
||||
|
||||
volumes:
|
||||
pg_data:
|
||||
redis_data:
|
||||
minio_data:
|
||||
rabbitmq_data:
|
||||
@@ -0,0 +1,91 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY,
|
||||
username VARCHAR(32) NOT NULL UNIQUE,
|
||||
display_name VARCHAR(100) NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_users_username ON users (username);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title VARCHAR(100) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_chats_cursor ON chats (created_at ASC, id ASC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_chat_members_user_id ON chat_members (user_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
text BYTEA NOT NULL,
|
||||
attachment_ids UUID[] NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] NOT NULL DEFAULT '{}';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_cursor ON messages (chat_id, created_at ASC, id ASC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
filename VARCHAR(255) NOT NULL,
|
||||
content_type VARCHAR(50) NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 VARCHAR(64) NOT NULL,
|
||||
s3_key VARCHAR(512) NOT NULL,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'queued',
|
||||
error_code VARCHAR(100),
|
||||
variants JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
processing_started_at TIMESTAMPTZ,
|
||||
processing_worker_id TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS variants JSONB NOT NULL DEFAULT '[]'::jsonb;
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS processing_started_at TIMESTAMPTZ;
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS processing_worker_id TEXT;
|
||||
ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_chat_id ON attachments (chat_id, created_at ASC);
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_owner_id ON attachments (owner_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_lease ON attachments (status, processing_started_at) WHERE status = 'processing';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS outbox_jobs (
|
||||
job_id UUID PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
event_type VARCHAR(64) NOT NULL DEFAULT 'attachment.process.v1',
|
||||
payload JSONB NOT NULL,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'pending',
|
||||
attempt INT NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_outbox_pending ON outbox_jobs (status, created_at) WHERE status = 'pending';
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS idempotency_keys (
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
idempotency_key VARCHAR(128) NOT NULL,
|
||||
message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
request_hash VARCHAR(64) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
PRIMARY KEY (user_id, chat_id, idempotency_key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_idempotency_created_at ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,72 @@
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" upstream: $upstream_addr '
|
||||
'upstream_status: $upstream_status '
|
||||
'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
client_max_body_size 15M;
|
||||
|
||||
# DNS-сервер Docker для динамического обновления IP-адресов контейнеров
|
||||
resolver 127.0.0.11 valid=2s ipv6=off;
|
||||
|
||||
upstream api_backend {
|
||||
zone api_backend 64k;
|
||||
server api-1:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
server api-2:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
server_name _;
|
||||
|
||||
# Быстрый таймаут обнаружения недоступной реплики
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 15s;
|
||||
proxy_send_timeout 15s;
|
||||
|
||||
# Базовая политика переключения: безопасно для идемпотентных запросов
|
||||
# Неидемпотентные POST не повторяем
|
||||
proxy_next_upstream error timeout http_502 http_503;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 15s;
|
||||
|
||||
location ~ ^/api/v1/chats/[^/]+/messages$ {
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Request-Id $http_x_request_id;
|
||||
|
||||
proxy_next_upstream error timeout http_502 http_503 non_idempotent;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 5s;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Request-Id $http_x_request_id;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
[pytest]
|
||||
pythonpath = .
|
||||
asyncio_mode = auto
|
||||
python_files = test_*.py *_test.py tests.py test.py
|
||||
@@ -0,0 +1,13 @@
|
||||
fastapi==0.141.1
|
||||
uvicorn[standard]==0.52.4
|
||||
pydantic==2.13.5
|
||||
asyncpg==0.30.0
|
||||
redis==5.2.1
|
||||
argon2-cffi==23.1.0
|
||||
pytest==9.1.1
|
||||
pytest-asyncio==1.4.0
|
||||
httpx==0.28.1
|
||||
boto3==1.43.123
|
||||
pillow==13.0.0
|
||||
aio-pika>=9.4.0
|
||||
pika>=1.3.2
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
# Перезапуск всех сервисов API и хранилищ с сохранением volumes
|
||||
docker compose -f "$PROJECT_DIR/compose.yaml" restart
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
if docker compose -f "$PROJECT_DIR/compose.yaml" stop "$TARGET_INSTANCE_ID" 2>/dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
docker stop "$TARGET_INSTANCE_ID" >/dev/null
|
||||
@@ -0,0 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
base_url = os.getenv("BASE_URL", "http://localhost:8080")
|
||||
if base_url.startswith("http://testserver"):
|
||||
from app.main import app
|
||||
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://testserver") as ac:
|
||||
yield ac
|
||||
else:
|
||||
async with AsyncClient(base_url=base_url, timeout=15.0) as ac:
|
||||
yield ac
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user