diff --git a/app.py b/app.py index fb41f90..fcf14b3 100644 --- a/app.py +++ b/app.py @@ -8,6 +8,7 @@ import re import struct import hashlib import hmac +import logging from datetime import datetime, timezone, timedelta from typing import List, Optional from uuid import UUID @@ -27,35 +28,117 @@ from botocore.client import Config import threading -app = FastAPI(title="MTUSI Messenger — lab5", version="1.5.0") +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor -def outbox_publisher(): - while True: - conn = None - try: - conn = get_db_connection() - with conn.cursor() as cur: - cur.execute(""" - SELECT id, stream, payload FROM outbox - WHERE published_at IS NULL - ORDER BY created_at - LIMIT 100 - FOR UPDATE SKIP LOCKED - """) - for row in cur.fetchall(): - redis_client.xadd(row["stream"], {"payload": json.dumps(row["payload"])}) - cur.execute("UPDATE outbox SET published_at=NOW() WHERE id=%s", (row["id"],)) - conn.commit() - except Exception as e: - print(f"OUTBOX PUBLISHER ERROR: {type(e).__name__} - {e}", flush=True) - finally: - if conn is not None and not conn.closed: - conn.close() - time.sleep(1) -@app.on_event("startup") -def _start_outbox_publisher(): - threading.Thread(target=outbox_publisher, daemon=True).start() +class JsonLogFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + span = trace.get_current_span() + ctx = span.get_span_context() if span else None + trace_id = format(ctx.trace_id, "032x") if ctx and ctx.is_valid else None + span_id = format(ctx.span_id, "016x") if ctx and ctx.is_valid else None + + payload = { + "ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "level": record.levelname, + "logger": record.name, + "service": os.getenv("OTEL_SERVICE_NAME", "messenger-api"), + "msg": record.getMessage(), + } + if trace_id and span_id: + payload["trace_id"] = trace_id + payload["span_id"] = span_id + + for attr in ("method", "path", "status_code", "job_id", "attachment_id", "job_status", "attempt", "error_code"): + if hasattr(record, attr): + payload[attr] = getattr(record, attr) + + if record.exc_info: + payload["exception"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception" + + return json.dumps(payload, ensure_ascii=False) + + +def setup_api_telemetry(): + service_name = os.getenv("OTEL_SERVICE_NAME", "messenger-api") + endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/") + env_attrs = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=lab") + + attrs = {"service.name": service_name} + for part in env_attrs.split(","): + if "=" in part: + k, v = part.split("=", 1) + attrs[k.strip()] = v.strip() + + resource = Resource.create(attrs) + + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")) + ) + trace.set_tracer_provider(tracer_provider) + + metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"), + export_interval_millis=5000, + ) + meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) + metrics.set_meter_provider(meter_provider) + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs")) + ) + set_logger_provider(logger_provider) + + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + root_logger.handlers.clear() + + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(JsonLogFormatter()) + root_logger.addHandler(stream_handler) + + otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + otel_handler.setFormatter(JsonLogFormatter()) + root_logger.addHandler(otel_handler) + + return tracer_provider, trace.get_tracer(service_name), metrics.get_meter(service_name) + + +app = FastAPI(title="MTUSI Messenger — lab6", version="1.6.0") + +_tracer_provider, tracer, meter = setup_api_telemetry() +FastAPIInstrumentor.instrument_app(app, tracer_provider=_tracer_provider) +api_logger = logging.getLogger("messenger-api") + +api_requests_counter = meter.create_counter( + name="http_server_requests_total", + description="Total HTTP requests served by API", + unit="1", +) +api_errors_counter = meter.create_counter( + name="http_server_errors_total", + description="Total HTTP 4xx/5xx responses served by API", + unit="1", +) +api_duration_histogram = meter.create_histogram( + name="http_server_request_duration_seconds", + description="HTTP request duration in seconds", + unit="s", +) DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) @@ -82,12 +165,77 @@ s3_client = boto3.client( region_name="us-east-1" ) + def get_db_connection(): return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor) + +def outbox_publisher(): + while True: + conn = None + try: + conn = get_db_connection() + with conn.cursor() as cur: + cur.execute(""" + SELECT id, stream, payload FROM outbox + WHERE published_at IS NULL + ORDER BY created_at + LIMIT 100 + FOR UPDATE SKIP LOCKED + """) + for row in cur.fetchall(): + redis_client.xadd(row["stream"], {"payload": json.dumps(row["payload"])}) + cur.execute("UPDATE outbox SET published_at=NOW() WHERE id=%s", (row["id"],)) + conn.commit() + except Exception as e: + api_logger.error("Outbox publisher error: %s", type(e).__name__) + finally: + if conn is not None and not conn.closed: + conn.close() + time.sleep(1) + + +@app.on_event("startup") +def _start_outbox_publisher(): + threading.Thread(target=outbox_publisher, daemon=True).start() + + +@app.middleware("http") +async def telemetry_middleware(request: Request, call_next): + start = time.monotonic() + route_obj = request.scope.get("route") + route_path = getattr(route_obj, "path", request.url.path) + method = request.method + status_code = 500 + try: + response = await call_next(request) + status_code = response.status_code + return response + finally: + elapsed = time.monotonic() - start + low_card_attrs = { + "http.method": method, + "http.route": route_path, + "http.status_code": status_code, + } + api_requests_counter.add(1, low_card_attrs) + api_duration_histogram.record(elapsed, low_card_attrs) + if status_code >= 400: + api_errors_counter.add(1, low_card_attrs) + api_logger.info( + "HTTP request completed", + extra={ + "method": method, + "path": route_path, + "status_code": status_code, + }, + ) + + def generate_request_id() -> str: return uuid.uuid4().hex + def make_error_response( status_code: int, code: str, @@ -110,12 +258,14 @@ def make_error_response( }, ) + def encode_cursor(chat_id: str, created_at: str, msg_id: str) -> str: payload = f"{chat_id}|{created_at}|{msg_id}" signature = hmac.new(SECRET_HMAC_KEY.encode(), payload.encode(), hashlib.sha256).hexdigest()[:16] raw = f"{payload}|{signature}" return base64.urlsafe_b64encode(raw.encode()).decode() + def decode_cursor(cursor: str, expected_chat_id: str) -> tuple: try: decoded = base64.urlsafe_b64decode(cursor.encode()).decode() @@ -141,6 +291,7 @@ def decode_cursor(cursor: str, expected_chat_id: str) -> tuple: except Exception: raise HTTPException(status_code=400, detail="Invalid cursor format") + @app.exception_handler(RequestValidationError) async def validation_exception_handler(request: Request, exc: RequestValidationError): req_id = request.headers.get("X-Request-Id") or generate_request_id() @@ -151,6 +302,7 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE req_id=req_id ) + @app.exception_handler(HTTPException) async def http_exception_handler(request: Request, exc: HTTPException): req_id = request.headers.get("X-Request-Id") or generate_request_id() @@ -186,6 +338,7 @@ async def http_exception_handler(request: Request, exc: HTTPException): }, ) + @app.exception_handler(Exception) async def unhandled_exception_handler(request: Request, exc: Exception): req_id = request.headers.get("X-Request-Id") or generate_request_id() @@ -196,6 +349,7 @@ async def unhandled_exception_handler(request: Request, exc: Exception): req_id=req_id, ) + class CreateUser(BaseModel): model_config = ConfigDict(extra='forbid') username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$") @@ -208,6 +362,7 @@ class CreateUser(BaseModel): raise ValueError("Must not be blank") return v + class CreateChat(BaseModel): model_config = ConfigDict(extra='forbid') title: str = Field(..., min_length=1, max_length=100) @@ -219,6 +374,7 @@ class CreateChat(BaseModel): raise ValueError("Must not be blank") return v + class CreateMessage(BaseModel): model_config = ConfigDict(extra='forbid') text: str = Field(default="", max_length=4000) @@ -230,6 +386,7 @@ class CreateMessage(BaseModel): raise ValueError("Message must contain either text or attachments") return self + @app.get("/health/live") def health_live(response: Response): req_id = generate_request_id() @@ -237,6 +394,7 @@ def health_live(response: Response): response.headers["X-Instance-Id"] = INSTANCE_ID return {"status": "ok"} + @app.get("/health/ready") def health_ready(response: Response): req_id = generate_request_id() @@ -251,6 +409,7 @@ def health_ready(response: Response): raise HTTPException(status_code=503, detail="Database, Redis or S3 unavailable") return {"status": "ok"} + @app.post("/api/v1/users", status_code=201) def register_user(body: CreateUser, response: Response): req_id = generate_request_id() @@ -285,6 +444,7 @@ def register_user(body: CreateUser, response: Response): "created_at": created_at } + @app.post("/api/v1/auth/sessions", status_code=201) def create_session(authorization: Optional[str] = Header(None), response: Response = None): req_id = generate_request_id() @@ -339,6 +499,7 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon } } + def get_current_user_from_token(authorization: Optional[str] = Header(None)) -> dict: if not authorization or not authorization.startswith("Bearer "): raise HTTPException( @@ -380,6 +541,7 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) -> "created_at": row['created_at'].isoformat().replace("+00:00", "Z") if isinstance(row['created_at'], datetime) else row['created_at'] } + @app.delete("/api/v1/auth/sessions/current", status_code=204) def revoke_session(authorization: Optional[str] = Header(None)): req_id = generate_request_id() @@ -409,6 +571,7 @@ def revoke_session(authorization: Optional[str] = Header(None)): }, ) + @app.get("/api/v1/users/me") def current_user(response: Response, authorization: Optional[str] = Header(None)): req_id = generate_request_id() @@ -416,6 +579,7 @@ def current_user(response: Response, authorization: Optional[str] = Header(None) response.headers["X-Instance-Id"] = INSTANCE_ID return get_current_user_from_token(authorization) + @app.post("/api/v1/chats", status_code=201) def create_chat(body: CreateChat, response: Response, authorization: Optional[str] = Header(None)): req_id = generate_request_id() @@ -466,6 +630,7 @@ def create_chat(body: CreateChat, response: Response, authorization: Optional[st "created_at": created_at } + @app.get("/api/v1/chats") def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response = None, authorization: Optional[str] = Header(None)): req_id = generate_request_id() @@ -522,6 +687,7 @@ def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response return {"items": page_items, "next_cursor": next_cursor} + @app.get("/api/v1/chats/{chat_id}") def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = Header(None)): req_id = generate_request_id() @@ -553,6 +719,7 @@ def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = H "created_at": chat['created_at'].isoformat().replace("+00:00", "Z") if isinstance(chat['created_at'], datetime) else chat['created_at'] } + def validate_image_and_get_dimensions(data: bytes, content_type: str): if len(data) > 10 * 1024 * 1024: raise HTTPException(status_code=413, detail="Payload too large (>10 MiB)") @@ -576,14 +743,14 @@ def validate_image_and_get_dimensions(data: bytes, content_type: str): while i < len(data) - 9: marker = data[i] if marker == 0xff: - marker_type = data[i+1] + marker_type = data[i + 1] if 0xc0 <= marker_type <= 0xc3: - height, width = struct.unpack('>HH', data[i+5:i+9]) + height, width = struct.unpack('>HH', data[i + 5:i + 9]) break if marker_type in {0xd8, 0xd9} or (0xd0 <= marker_type <= 0xd7): i += 2 continue - seg_len = struct.unpack('>H', data[i+2:i+4])[0] + seg_len = struct.unpack('>H', data[i + 2:i + 4])[0] if seg_len < 2: raise HTTPException(status_code=415, detail="Invalid JPEG segment") i += 2 + seg_len @@ -598,6 +765,7 @@ def validate_image_and_get_dimensions(data: bytes, content_type: str): if width * height > 40_000_000: raise HTTPException(status_code=413, detail="Image exceeds 40 million pixels limit") + @app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) async def upload_attachment( chat_id: UUID, @@ -718,6 +886,7 @@ async def upload_attachment( "updated_at": now } + @app.get("/api/v1/attachments/{attachment_id}") def get_attachment(attachment_id: UUID, response: Response, authorization: Optional[str] = Header(None)): req_id = generate_request_id() @@ -754,6 +923,7 @@ def get_attachment(attachment_id: UUID, response: Response, authorization: Optio "updated_at": row['updated_at'].isoformat().replace("+00:00", "Z") if isinstance(row['updated_at'], datetime) else row['updated_at'] } + @app.get("/api/v1/attachments/{attachment_id}/content") def download_attachment( attachment_id: UUID, @@ -809,6 +979,7 @@ def download_attachment( }, ) + @app.post("/api/v1/chats/{chat_id}/messages", status_code=201) def create_message( chat_id: UUID, @@ -908,6 +1079,7 @@ def create_message( return msg_response + @app.get("/api/v1/chats/{chat_id}/messages") def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None, response: Response = None, authorization: Optional[str] = Header(None)): @@ -965,6 +1137,7 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None, return {"items": formatted, "next_cursor": next_cursor} + @app.post("/api/v1/attachments/{attachment_id}/replay", status_code=202) def replay_attachment(attachment_id: UUID, response: Response, authorization: Optional[str] = Header(None)): diff --git a/compose.yaml b/compose.yaml index e4822ca..456ab94 100644 --- a/compose.yaml +++ b/compose.yaml @@ -52,7 +52,7 @@ services: restart: "no" s3: - image: chrislusf/seaweedfs:latest + image: chrislusf/seaweedfs:3.67 command: ["server", "-s3", "-dir=/data", "-s3.port=8333"] environment: - AWS_ACCESS_KEY_ID=appuser @@ -65,11 +65,11 @@ services: cpus: '0.50' memory: 512M healthcheck: - test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz || exit 1"] + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/ || test $$? -eq 8 || exit 1"] interval: 5s timeout: 3s retries: 5 - start_period: 10s + start_period: 30s s3-init: image: python:3.11.7-slim-bookworm @@ -89,6 +89,76 @@ services: condition: service_healthy restart: "no" + prometheus: + image: prom/prometheus:v2.51.0 + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=7d" + - "--web.enable-remote-write-receiver" + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - promdata:/prometheus + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9090/-/healthy || exit 1"] + interval: 5s + timeout: 3s + retries: 5 + + loki: + image: grafana/loki:3.0.0 + command: ["-config.file=/etc/loki/local-config.yaml"] + volumes: + - ./infra/loki/loki-config.yaml:/etc/loki/local-config.yaml:ro + - lokidata:/loki + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3100/ready || exit 1"] + interval: 5s + timeout: 3s + retries: 5 + + tempo: + image: grafana/tempo:2.4.1 + command: ["-config.file=/etc/tempo.yaml"] + volumes: + - ./infra/tempo/tempo-config.yaml:/etc/tempo.yaml:ro + - tempodata:/var/tempo + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3200/ready || exit 1"] + interval: 5s + timeout: 3s + retries: 5 + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.97.0 + command: ["--config=/etc/otelcol-contrib/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro + depends_on: + - prometheus + - loki + - tempo + + grafana: + image: grafana/grafana:10.4.2 + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_USER=${GF_ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${GF_ADMIN_PASSWORD:-admin} + - GF_AUTH_ANONYMOUS_ENABLED=false + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafanadata:/var/lib/grafana + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_healthy + tempo: + condition: service_healthy + api-1: build: . stop_grace_period: 15s @@ -103,6 +173,10 @@ services: - S3_ACCESS_KEY_ID=appuser - S3_SECRET_ACCESS_KEY=appsecretpassword - BROKER_STREAM=attachment_jobs + - OTEL_SERVICE_NAME=messenger-api + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf + - OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab deploy: resources: limits: @@ -117,6 +191,8 @@ services: condition: service_completed_successfully s3-init: condition: service_completed_successfully + otel-collector: + condition: service_started healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] interval: 5s @@ -137,6 +213,10 @@ services: - S3_ACCESS_KEY_ID=appuser - S3_SECRET_ACCESS_KEY=appsecretpassword - BROKER_STREAM=attachment_jobs + - OTEL_SERVICE_NAME=messenger-api + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf + - OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab deploy: resources: limits: @@ -151,6 +231,8 @@ services: condition: service_completed_successfully s3-init: condition: service_completed_successfully + otel-collector: + condition: service_started healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] interval: 5s @@ -172,6 +254,10 @@ services: - BROKER_GROUP=attachment_workers - JOB_MAX_ATTEMPTS=5 - MAX_IMAGE_PIXELS=40000000 + - OTEL_SERVICE_NAME=messenger-worker + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf + - OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab deploy: resources: limits: @@ -186,6 +272,8 @@ services: condition: service_completed_successfully s3-init: condition: service_completed_successfully + otel-collector: + condition: service_started healthcheck: test: ["CMD", "python", "-c", "import os, redis; redis.Redis.from_url(os.environ['REDIS_URL']).ping()"] interval: 5s @@ -215,4 +303,8 @@ services: volumes: pgdata: redisdata: - s3data: \ No newline at end of file + s3data: + promdata: + lokidata: + tempodata: + grafanadata: \ No newline at end of file diff --git a/infra/grafana/dashboards/messenger-grade3.json b/infra/grafana/dashboards/messenger-grade3.json new file mode 100644 index 0000000..6b16499 --- /dev/null +++ b/infra/grafana/dashboards/messenger-grade3.json @@ -0,0 +1,86 @@ +{ + "title": "MTUSI Messenger — Lab 6 (Grade 3)", + "uid": "mtusi-messenger-lab6-g3", + "schemaVersion": 39, + "version": 2, + "refresh": "5s", + "panels": [ + { + "id": 1, + "title": "API RPS", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 0 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(http_server_requests_total[1m]))", + "legendFormat": "req/s" + } + ] + }, + { + "id": 2, + "title": "API Errors (4xx/5xx/s)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 0 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(http_server_errors_total[1m]))", + "legendFormat": "errors/s" + } + ] + }, + { + "id": 3, + "title": "API Duration p95 (s)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p95" + } + ] + }, + { + "id": 4, + "title": "Worker Jobs (success / failed)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(worker_jobs_total[1m])) by (status)", + "legendFormat": "{{status}}" + } + ] + }, + { + "id": 5, + "title": "Worker Job Duration p95 (s)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p95" + } + ] + }, + { + "id": 6, + "title": "Structured Logs (API & Worker)", + "type": "logs", + "gridPos": { "h": 9, "w": 24, "x": 0, "y": 14 }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "{service_name=~\"messenger-api|messenger-worker\"}" + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/grafana/provisioning/dashboards/dashboards.yml b/infra/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..606870f --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: MTUSI Messenger Lab6 + orgId: 1 + folder: Messenger + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/infra/grafana/provisioning/datasources/datasources.yml b/infra/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..64cedaf --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,37 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + + - name: Loki + type: loki + uid: loki + access: proxy + url: http://loki:3100 + editable: true + jsonData: + derivedFields: + - datasourceUid: tempo + matcherRegex: '"trace_id":"([0-9a-f]{32})"' + name: TraceID + url: '$${__value.raw}' + + - name: Tempo + type: tempo + uid: tempo + access: proxy + url: http://tempo:3200 + editable: true + jsonData: + tracesToLogsV2: + datasourceUid: loki + filterByTraceID: true + tags: + - key: service.name + value: service \ No newline at end of file diff --git a/infra/loki/loki-config.yaml b/infra/loki/loki-config.yaml new file mode 100644 index 0000000..7617c4b --- /dev/null +++ b/infra/loki/loki-config.yaml @@ -0,0 +1,38 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +ruler: + alertmanager_url: http://localhost:9093 + +limits_config: + allow_structured_metadata: true + +distributor: + otlp_config: + default_resource_attributes_as_index_labels: + - service.name + - service.namespace + - deployment.environment \ No newline at end of file diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..e817167 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,54 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: + send_batch_size: 256 + timeout: 2s + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +exporters: + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + otlphttp/tempo: + endpoint: http://tempo:4318 + tls: + insecure: true + otlphttp/loki: + endpoint: http://loki:3100/otlp + headers: + X-Scope-OrgID: fake + tls: + insecure: true + debug: + verbosity: detailed + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki, debug] \ No newline at end of file diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..22240f6 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,8 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] \ No newline at end of file diff --git a/infra/tempo/tempo-config.yaml b/infra/tempo/tempo-config.yaml new file mode 100644 index 0000000..72a9e64 --- /dev/null +++ b/infra/tempo/tempo-config.yaml @@ -0,0 +1,25 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +ingester: + trace_idle_period: 10s + max_block_duration: 5m + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + local: + path: /var/tempo/blocks + wal: + path: /var/tempo/wal \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 00abd85..a4b3e9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,9 @@ psycopg2-binary==2.9.9 bcrypt==4.1.2 redis==5.0.1 boto3==1.34.0 -Pillow==10.2.0 \ No newline at end of file +Pillow==10.2.0 +opentelemetry-api==1.24.0 +opentelemetry-sdk==1.24.0 +opentelemetry-exporter-otlp-proto-http==1.24.0 +opentelemetry-instrumentation-fastapi==0.45b0 +opentelemetry-instrumentation-requests==0.45b0 \ No newline at end of file diff --git a/telemetry.py b/telemetry.py new file mode 100644 index 0000000..9f9b976 --- /dev/null +++ b/telemetry.py @@ -0,0 +1,92 @@ +import json +import logging +import os +from datetime import datetime, timezone + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + + +class JsonLogFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + span = trace.get_current_span() + ctx = span.get_span_context() if span else None + trace_id = format(ctx.trace_id, "032x") if ctx and ctx.is_valid else None + span_id = format(ctx.span_id, "016x") if ctx and ctx.is_valid else None + + payload = { + "ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "level": record.levelname, + "logger": record.name, + "service": os.getenv("OTEL_SERVICE_NAME", "messenger-app"), + "msg": record.getMessage(), + } + if trace_id and span_id: + payload["trace_id"] = trace_id + payload["span_id"] = span_id + + for attr in ("method", "path", "status_code", "job_id", "attachment_id", "job_status", "attempt", "error_code"): + if hasattr(record, attr): + payload[attr] = getattr(record, attr) + + if record.exc_info: + payload["exception"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception" + + return json.dumps(payload, ensure_ascii=False) + + +def setup_telemetry(default_service_name: str): + service_name = os.getenv("OTEL_SERVICE_NAME", default_service_name) + endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/") + env_name = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=lab") + + attrs = {"service.name": service_name} + for part in env_name.split(","): + if "=" in part: + k, v = part.split("=", 1) + attrs[k.strip()] = v.strip() + + resource = Resource.create(attrs) + + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")) + ) + trace.set_tracer_provider(tracer_provider) + + metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"), + export_interval_millis=5000, + ) + meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) + metrics.set_meter_provider(meter_provider) + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs")) + ) + set_logger_provider(logger_provider) + + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + root_logger.handlers.clear() + + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(JsonLogFormatter()) + root_logger.addHandler(stream_handler) + + otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + otel_handler.setFormatter(JsonLogFormatter()) + root_logger.addHandler(otel_handler) + + return trace.get_tracer(service_name), metrics.get_meter(service_name) \ No newline at end of file diff --git a/worker.py b/worker.py index 9f33be8..5f6a9fe 100644 --- a/worker.py +++ b/worker.py @@ -17,9 +17,109 @@ import botocore.exceptions from botocore.client import Config from PIL import Image, ImageOps, UnidentifiedImageError -logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + + +class JsonLogFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + span = trace.get_current_span() + ctx = span.get_span_context() if span else None + trace_id = format(ctx.trace_id, "032x") if ctx and ctx.is_valid else None + span_id = format(ctx.span_id, "016x") if ctx and ctx.is_valid else None + + payload = { + "ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "level": record.levelname, + "logger": record.name, + "service": os.getenv("OTEL_SERVICE_NAME", "messenger-worker"), + "msg": record.getMessage(), + } + if trace_id and span_id: + payload["trace_id"] = trace_id + payload["span_id"] = span_id + + for attr in ("method", "path", "status_code", "job_id", "attachment_id", "job_status", "attempt", "error_code"): + if hasattr(record, attr): + payload[attr] = getattr(record, attr) + + if record.exc_info: + payload["exception"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception" + + return json.dumps(payload, ensure_ascii=False) + + +def setup_worker_telemetry(): + service_name = os.getenv("OTEL_SERVICE_NAME", "messenger-worker") + endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/") + env_attrs = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=lab") + + attrs = {"service.name": service_name} + for part in env_attrs.split(","): + if "=" in part: + k, v = part.split("=", 1) + attrs[k.strip()] = v.strip() + + resource = Resource.create(attrs) + + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")) + ) + trace.set_tracer_provider(tracer_provider) + + metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"), + export_interval_millis=5000, + ) + meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) + metrics.set_meter_provider(meter_provider) + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs")) + ) + set_logger_provider(logger_provider) + + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + root_logger.handlers.clear() + + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(JsonLogFormatter()) + root_logger.addHandler(stream_handler) + + otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + otel_handler.setFormatter(JsonLogFormatter()) + root_logger.addHandler(otel_handler) + + return trace.get_tracer(service_name), metrics.get_meter(service_name) + + +tracer, meter = setup_worker_telemetry() logger = logging.getLogger("attachment-worker") +worker_jobs_counter = meter.create_counter( + name="worker_jobs_total", + description="Completed and failed attachment jobs", + unit="1", +) +worker_duration_histogram = meter.create_histogram( + name="worker_job_duration_seconds", + description="Attachment job processing duration in seconds", + unit="s", +) + DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333") @@ -49,11 +149,13 @@ s3_client = boto3.client( running = True + def handle_sigterm(signum, frame): global running logger.info("Received shutdown signal, finishing in-flight jobs...") running = False + signal.signal(signal.SIGTERM, handle_sigterm) signal.signal(signal.SIGINT, handle_sigterm) @@ -91,7 +193,7 @@ def prepare_base_image(raw_bytes): return img -def generate_grade4_variants(raw_bytes): +def generate_grade3_variants(raw_bytes): base_img = prepare_base_image(raw_bytes) orig_w, orig_h = base_img.size meta = [] @@ -105,36 +207,18 @@ def generate_grade4_variants(raw_bytes): t_w, t_h = max(1, int(orig_w * s)), max(1, int(orig_h * s)) if (t_w, t_h) != (orig_w, orig_h): thumb = thumb.resize((t_w, t_h), Image.Resampling.LANCZOS) - buf = io.BytesIO(); thumb.save(buf, format="JPEG", quality=85, optimize=True) - tb = buf.getvalue(); blobs["thumbnail"] = tb - meta.append({"name": "thumbnail", "content_type": "image/jpeg", - "size_bytes": len(tb), "width": t_w, "height": t_h, - "sha256": hashlib.sha256(tb).hexdigest()}) - - opt = base_img.copy() - if orig_w <= 1280 and orig_h <= 1280: - o_w, o_h = orig_w, orig_h - else: - s = min(1280.0 / orig_w, 1280.0 / orig_h) - o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s)) - if (o_w, o_h) != (orig_w, orig_h): - opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS) - buf = io.BytesIO(); opt.save(buf, format="WEBP", quality=85) - ob = buf.getvalue(); blobs["optimized"] = ob - meta.append({"name": "optimized", "content_type": "image/webp", - "size_bytes": len(ob), "width": o_w, "height": o_h, - "sha256": hashlib.sha256(ob).hexdigest()}) - - c = min(orig_w, orig_h) - left, top = (orig_w - c) // 2, (orig_h - c) // 2 - crop = base_img.crop((left, top, left + c, top + c)) - if c != 128: - crop = crop.resize((128, 128), Image.Resampling.LANCZOS) - buf = io.BytesIO(); crop.save(buf, format="JPEG", quality=85, optimize=True) - cb = buf.getvalue(); blobs["crop"] = cb - meta.append({"name": "crop", "content_type": "image/jpeg", - "size_bytes": len(cb), "width": 128, "height": 128, - "sha256": hashlib.sha256(cb).hexdigest()}) + buf = io.BytesIO() + thumb.save(buf, format="JPEG", quality=85, optimize=True) + tb = buf.getvalue() + blobs["thumbnail"] = tb + meta.append({ + "name": "thumbnail", + "content_type": "image/jpeg", + "size_bytes": len(tb), + "width": t_w, + "height": t_h, + "sha256": hashlib.sha256(tb).hexdigest(), + }) return meta, blobs @@ -144,18 +228,16 @@ def mark_failed(attachment_id, error_code): conn = get_db_connection() try: with conn.cursor() as cur: - cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=%s " - "WHERE id=%s AND status != 'ready'", - (error_code, now, attachment_id)) + cur.execute( + "UPDATE attachments SET status='failed', error_code=%s, updated_at=%s " + "WHERE id=%s AND status != 'ready'", + (error_code, now, attachment_id) + ) conn.commit() finally: conn.close() -PERMANENT_ERRORS = ( - UnidentifiedImageError, Image.DecompressionBombError, ValueError, - psycopg2.errors.DataError, -) TRANSIENT_ERRORS = ( TimeoutError, botocore.exceptions.EndpointConnectionError, @@ -165,165 +247,203 @@ TRANSIENT_ERRORS = ( ) +def _record_job_outcome(status: str, elapsed: float): + worker_jobs_counter.add(1, {"status": status}) + worker_duration_histogram.record(elapsed, {"status": status}) + + def process_job(message_id, fields): - try: - job = json.loads(fields.get("payload", "{}")) - except Exception: - logger.error("Malformed job %s, ack to drop", message_id) - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - return - - attachment_id = job.get("attachment_id") - job_id = job.get("job_id") - pipeline_version = int(job.get("pipeline_version", 1)) - attempt = int(job.get("attempt", 1)) - - if not attachment_id: - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - return - - conn = get_db_connection() - try: - with conn.cursor() as cur: - cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)) - att = cur.fetchone() - if not att: - conn.commit() - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - return - if att["status"] == "ready" or att["pipeline_version"] > pipeline_version: - conn.commit() - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - return - if att["status"] == "failed": - conn.commit() - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - return - - if att["status"] == "processing": - updated = att["updated_at"] - if isinstance(updated, str): - updated = datetime.fromisoformat(updated.replace("Z", "+00:00")) - age_seconds = (datetime.now(timezone.utc) - updated).total_seconds() - if age_seconds < (JOB_TIMEOUT_SECONDS + 10): - conn.commit() - logger.info("Attachment %s already processing (age=%.1fs), ack duplicate", - attachment_id, age_seconds) - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - return - - now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - cur.execute( - "UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s", - (attempt, now, attachment_id) - ) - conn.commit() - finally: - conn.close() - - chat_id = str(att["chat_id"]) - staging_key = f"staging/{chat_id}/{attachment_id}/original" - final_original_key = f"attachments/{chat_id}/{attachment_id}/original" - - start_time = time.monotonic() - - def check_timeout(): - if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS: - raise TimeoutError("Job exceeded time limit") - - try: + job_start = time.monotonic() + with tracer.start_as_current_span("worker.process_attachment") as span: try: - obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key) + job = json.loads(fields.get("payload", "{}")) except Exception: - obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key) - raw_bytes = obj["Body"].read() - check_timeout() + logger.error("Malformed job %s, ack to drop", message_id) + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome("failed", time.monotonic() - job_start) + return - if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]: - raise ValueError("Staging checksum mismatch") + attachment_id = job.get("attachment_id") + job_id = job.get("job_id") + pipeline_version = int(job.get("pipeline_version", 1)) + attempt = int(job.get("attempt", 1)) - variants_meta, variants_bytes = generate_grade4_variants(raw_bytes) - check_timeout() + span.set_attribute("job.pipeline_version", pipeline_version) + span.set_attribute("job.attempt", attempt) - s3_client.put_object( - Bucket=S3_BUCKET, Key=final_original_key, - Body=raw_bytes, ContentType=att["content_type"] - ) - for var_name, var_bytes in variants_bytes.items(): - ct = "image/webp" if var_name == "optimized" else "image/jpeg" - s3_client.put_object( - Bucket=S3_BUCKET, - Key=f"attachments/{chat_id}/{attachment_id}/{var_name}", - Body=var_bytes, ContentType=ct - ) - check_timeout() + if not attachment_id: + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome("failed", time.monotonic() - job_start) + return - ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") conn = get_db_connection() try: with conn.cursor() as cur: - cur.execute(""" - UPDATE attachments SET status='ready', variants=%s::jsonb, - error_code=NULL, updated_at=%s - WHERE id=%s AND pipeline_version <= %s - """, (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version)) + cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)) + att = cur.fetchone() + if not att: + conn.commit() + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome("failed", time.monotonic() - job_start) + return + if att["status"] == "ready" or att["pipeline_version"] > pipeline_version: + conn.commit() + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome("success", time.monotonic() - job_start) + return + if att["status"] == "failed": + conn.commit() + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome("failed", time.monotonic() - job_start) + return + + if att["status"] == "processing": + updated = att["updated_at"] + if isinstance(updated, str): + updated = datetime.fromisoformat(updated.replace("Z", "+00:00")) + age_seconds = (datetime.now(timezone.utc) - updated).total_seconds() + if age_seconds < (JOB_TIMEOUT_SECONDS + 10): + conn.commit() + logger.info( + "Attachment already processing, ack duplicate", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt}, + ) + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + return + + now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + cur.execute( + "UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s", + (attempt, now, attachment_id) + ) conn.commit() finally: conn.close() - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - logger.info("Job %s completed: attachment %s ready", job_id, attachment_id) + chat_id = str(att["chat_id"]) + staging_key = f"staging/{chat_id}/{attachment_id}/original" + final_original_key = f"attachments/{chat_id}/{attachment_id}/original" + + start_time = time.monotonic() + + def check_timeout(): + if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS: + raise TimeoutError("Job exceeded time limit") try: - s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key) - except Exception: - pass - - except (UnidentifiedImageError, Image.DecompressionBombError) as e: - logger.error("Permanent image error for %s: %s", attachment_id, type(e).__name__) - mark_failed(attachment_id, "invalid_image") - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - - except ValueError as e: - logger.error("Checksum error for %s: %s", attachment_id, e) - mark_failed(attachment_id, "invalid_checksum") - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - - except psycopg2.errors.DataError as e: - logger.error("DB data error for %s: %s", attachment_id, type(e).__name__) - mark_failed(attachment_id, "data_error") - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - - except TRANSIENT_ERRORS as e: - logger.warning( - "Transient error for %s (attempt %d/%d): %s", - attachment_id, attempt, JOB_MAX_ATTEMPTS, type(e).__name__ - ) - if attempt >= JOB_MAX_ATTEMPTS: - mark_failed(attachment_id, "retry_exhausted") try: - dlq_job = dict(job) - dlq_job["attempt"] = 1 - redis_client.xadd( - DLQ_STREAM_NAME, - {"payload": json.dumps(dlq_job), "error": str(e)} + obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key) + except Exception: + obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key) + raw_bytes = obj["Body"].read() + check_timeout() + + if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]: + raise ValueError("Staging checksum mismatch") + + variants_meta, variants_bytes = generate_grade3_variants(raw_bytes) + check_timeout() + + s3_client.put_object( + Bucket=S3_BUCKET, Key=final_original_key, + Body=raw_bytes, ContentType=att["content_type"] + ) + for var_name, var_bytes in variants_bytes.items(): + ct = "image/webp" if var_name == "optimized" else "image/jpeg" + s3_client.put_object( + Bucket=S3_BUCKET, + Key=f"attachments/{chat_id}/{attachment_id}/{var_name}", + Body=var_bytes, ContentType=ct ) + check_timeout() + + ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + conn = get_db_connection() + try: + with conn.cursor() as cur: + cur.execute(""" + UPDATE attachments SET status='ready', variants=%s::jsonb, + error_code=NULL, updated_at=%s + WHERE id=%s AND pipeline_version <= %s + """, (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version)) + conn.commit() + finally: + conn.close() + + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome("success", time.monotonic() - job_start) + logger.info( + "Job completed: attachment ready", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "ready", "attempt": attempt}, + ) + + try: + s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key) except Exception: pass - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - else: - delay = min(2 ** attempt, 30) + random.uniform(0, 1) - time.sleep(delay) - next_job = dict(job) - next_job["attempt"] = attempt + 1 - redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)}) - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - except Exception as e: - logger.error("Unknown error for %s: %s, marking failed", - attachment_id, type(e).__name__) - mark_failed(attachment_id, "processing_failed") - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + except (UnidentifiedImageError, Image.DecompressionBombError) as e: + logger.error( + "Permanent image error", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_image"}, + ) + mark_failed(attachment_id, "invalid_image") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome("failed", time.monotonic() - job_start) + + except ValueError as e: + logger.error( + "Checksum error", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_checksum"}, + ) + mark_failed(attachment_id, "invalid_checksum") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome("failed", time.monotonic() - job_start) + + except psycopg2.errors.DataError as e: + logger.error( + "DB data error", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "data_error"}, + ) + mark_failed(attachment_id, "data_error") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome("failed", time.monotonic() - job_start) + + except TRANSIENT_ERRORS as e: + logger.warning( + "Transient error during job processing", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt}, + ) + if attempt >= JOB_MAX_ATTEMPTS: + mark_failed(attachment_id, "retry_exhausted") + try: + dlq_job = dict(job) + dlq_job["attempt"] = 1 + redis_client.xadd( + DLQ_STREAM_NAME, + {"payload": json.dumps(dlq_job), "error": type(e).__name__} + ) + except Exception: + pass + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome("failed", time.monotonic() - job_start) + else: + delay = min(2 ** attempt, 30) + random.uniform(0, 1) + time.sleep(delay) + next_job = dict(job) + next_job["attempt"] = attempt + 1 + redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)}) + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + + except Exception as e: + logger.error( + "Unknown error during job processing", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "processing_failed"}, + ) + mark_failed(attachment_id, "processing_failed") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome("failed", time.monotonic() - job_start) + def run_staging_cleanup(): try: @@ -337,11 +457,10 @@ def run_staging_cleanup(): except Exception: pass except Exception as e: - logger.debug("Staging cleanup: %s", e) + logger.debug("Staging cleanup: %s", type(e).__name__) def safe_process(msg_id, fields): - """Wrapper - exceptions в threads иначе исчезают молча.""" try: process_job(msg_id, fields) except Exception as e: