forked from mtusi2026/mtusi2026lab6
grade3
This commit is contained in:
@@ -8,6 +8,7 @@ import re
|
||||
import struct
|
||||
import hashlib
|
||||
import hmac
|
||||
import logging
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import List, Optional
|
||||
from uuid import UUID
|
||||
@@ -27,35 +28,117 @@ from botocore.client import Config
|
||||
|
||||
import threading
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger — lab5", version="1.5.0")
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
|
||||
def outbox_publisher():
|
||||
while True:
|
||||
conn = None
|
||||
try:
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT id, stream, payload FROM outbox
|
||||
WHERE published_at IS NULL
|
||||
ORDER BY created_at
|
||||
LIMIT 100
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""")
|
||||
for row in cur.fetchall():
|
||||
redis_client.xadd(row["stream"], {"payload": json.dumps(row["payload"])})
|
||||
cur.execute("UPDATE outbox SET published_at=NOW() WHERE id=%s", (row["id"],))
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
print(f"OUTBOX PUBLISHER ERROR: {type(e).__name__} - {e}", flush=True)
|
||||
finally:
|
||||
if conn is not None and not conn.closed:
|
||||
conn.close()
|
||||
time.sleep(1)
|
||||
|
||||
@app.on_event("startup")
|
||||
def _start_outbox_publisher():
|
||||
threading.Thread(target=outbox_publisher, daemon=True).start()
|
||||
class JsonLogFormatter(logging.Formatter):
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
span = trace.get_current_span()
|
||||
ctx = span.get_span_context() if span else None
|
||||
trace_id = format(ctx.trace_id, "032x") if ctx and ctx.is_valid else None
|
||||
span_id = format(ctx.span_id, "016x") if ctx and ctx.is_valid else None
|
||||
|
||||
payload = {
|
||||
"ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"service": os.getenv("OTEL_SERVICE_NAME", "messenger-api"),
|
||||
"msg": record.getMessage(),
|
||||
}
|
||||
if trace_id and span_id:
|
||||
payload["trace_id"] = trace_id
|
||||
payload["span_id"] = span_id
|
||||
|
||||
for attr in ("method", "path", "status_code", "job_id", "attachment_id", "job_status", "attempt", "error_code"):
|
||||
if hasattr(record, attr):
|
||||
payload[attr] = getattr(record, attr)
|
||||
|
||||
if record.exc_info:
|
||||
payload["exception"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception"
|
||||
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def setup_api_telemetry():
|
||||
service_name = os.getenv("OTEL_SERVICE_NAME", "messenger-api")
|
||||
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
|
||||
env_attrs = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=lab")
|
||||
|
||||
attrs = {"service.name": service_name}
|
||||
for part in env_attrs.split(","):
|
||||
if "=" in part:
|
||||
k, v = part.split("=", 1)
|
||||
attrs[k.strip()] = v.strip()
|
||||
|
||||
resource = Resource.create(attrs)
|
||||
|
||||
tracer_provider = TracerProvider(resource=resource)
|
||||
tracer_provider.add_span_processor(
|
||||
BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces"))
|
||||
)
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
|
||||
metric_reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"),
|
||||
export_interval_millis=5000,
|
||||
)
|
||||
meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs"))
|
||||
)
|
||||
set_logger_provider(logger_provider)
|
||||
|
||||
root_logger = logging.getLogger()
|
||||
root_logger.setLevel(logging.INFO)
|
||||
root_logger.handlers.clear()
|
||||
|
||||
stream_handler = logging.StreamHandler()
|
||||
stream_handler.setFormatter(JsonLogFormatter())
|
||||
root_logger.addHandler(stream_handler)
|
||||
|
||||
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||
otel_handler.setFormatter(JsonLogFormatter())
|
||||
root_logger.addHandler(otel_handler)
|
||||
|
||||
return tracer_provider, trace.get_tracer(service_name), metrics.get_meter(service_name)
|
||||
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger — lab6", version="1.6.0")
|
||||
|
||||
_tracer_provider, tracer, meter = setup_api_telemetry()
|
||||
FastAPIInstrumentor.instrument_app(app, tracer_provider=_tracer_provider)
|
||||
api_logger = logging.getLogger("messenger-api")
|
||||
|
||||
api_requests_counter = meter.create_counter(
|
||||
name="http_server_requests_total",
|
||||
description="Total HTTP requests served by API",
|
||||
unit="1",
|
||||
)
|
||||
api_errors_counter = meter.create_counter(
|
||||
name="http_server_errors_total",
|
||||
description="Total HTTP 4xx/5xx responses served by API",
|
||||
unit="1",
|
||||
)
|
||||
api_duration_histogram = meter.create_histogram(
|
||||
name="http_server_request_duration_seconds",
|
||||
description="HTTP request duration in seconds",
|
||||
unit="s",
|
||||
)
|
||||
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
|
||||
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
@@ -82,12 +165,77 @@ s3_client = boto3.client(
|
||||
region_name="us-east-1"
|
||||
)
|
||||
|
||||
|
||||
def get_db_connection():
|
||||
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
|
||||
|
||||
|
||||
def outbox_publisher():
|
||||
while True:
|
||||
conn = None
|
||||
try:
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT id, stream, payload FROM outbox
|
||||
WHERE published_at IS NULL
|
||||
ORDER BY created_at
|
||||
LIMIT 100
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""")
|
||||
for row in cur.fetchall():
|
||||
redis_client.xadd(row["stream"], {"payload": json.dumps(row["payload"])})
|
||||
cur.execute("UPDATE outbox SET published_at=NOW() WHERE id=%s", (row["id"],))
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
api_logger.error("Outbox publisher error: %s", type(e).__name__)
|
||||
finally:
|
||||
if conn is not None and not conn.closed:
|
||||
conn.close()
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
def _start_outbox_publisher():
|
||||
threading.Thread(target=outbox_publisher, daemon=True).start()
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def telemetry_middleware(request: Request, call_next):
|
||||
start = time.monotonic()
|
||||
route_obj = request.scope.get("route")
|
||||
route_path = getattr(route_obj, "path", request.url.path)
|
||||
method = request.method
|
||||
status_code = 500
|
||||
try:
|
||||
response = await call_next(request)
|
||||
status_code = response.status_code
|
||||
return response
|
||||
finally:
|
||||
elapsed = time.monotonic() - start
|
||||
low_card_attrs = {
|
||||
"http.method": method,
|
||||
"http.route": route_path,
|
||||
"http.status_code": status_code,
|
||||
}
|
||||
api_requests_counter.add(1, low_card_attrs)
|
||||
api_duration_histogram.record(elapsed, low_card_attrs)
|
||||
if status_code >= 400:
|
||||
api_errors_counter.add(1, low_card_attrs)
|
||||
api_logger.info(
|
||||
"HTTP request completed",
|
||||
extra={
|
||||
"method": method,
|
||||
"path": route_path,
|
||||
"status_code": status_code,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def generate_request_id() -> str:
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
def make_error_response(
|
||||
status_code: int,
|
||||
code: str,
|
||||
@@ -110,12 +258,14 @@ def make_error_response(
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def encode_cursor(chat_id: str, created_at: str, msg_id: str) -> str:
|
||||
payload = f"{chat_id}|{created_at}|{msg_id}"
|
||||
signature = hmac.new(SECRET_HMAC_KEY.encode(), payload.encode(), hashlib.sha256).hexdigest()[:16]
|
||||
raw = f"{payload}|{signature}"
|
||||
return base64.urlsafe_b64encode(raw.encode()).decode()
|
||||
|
||||
|
||||
def decode_cursor(cursor: str, expected_chat_id: str) -> tuple:
|
||||
try:
|
||||
decoded = base64.urlsafe_b64decode(cursor.encode()).decode()
|
||||
@@ -141,6 +291,7 @@ def decode_cursor(cursor: str, expected_chat_id: str) -> tuple:
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="Invalid cursor format")
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_exception_handler(request: Request, exc: RequestValidationError):
|
||||
req_id = request.headers.get("X-Request-Id") or generate_request_id()
|
||||
@@ -151,6 +302,7 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
|
||||
req_id=req_id
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_exception_handler(request: Request, exc: HTTPException):
|
||||
req_id = request.headers.get("X-Request-Id") or generate_request_id()
|
||||
@@ -186,6 +338,7 @@ async def http_exception_handler(request: Request, exc: HTTPException):
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unhandled_exception_handler(request: Request, exc: Exception):
|
||||
req_id = request.headers.get("X-Request-Id") or generate_request_id()
|
||||
@@ -196,6 +349,7 @@ async def unhandled_exception_handler(request: Request, exc: Exception):
|
||||
req_id=req_id,
|
||||
)
|
||||
|
||||
|
||||
class CreateUser(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$")
|
||||
@@ -208,6 +362,7 @@ class CreateUser(BaseModel):
|
||||
raise ValueError("Must not be blank")
|
||||
return v
|
||||
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
title: str = Field(..., min_length=1, max_length=100)
|
||||
@@ -219,6 +374,7 @@ class CreateChat(BaseModel):
|
||||
raise ValueError("Must not be blank")
|
||||
return v
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
text: str = Field(default="", max_length=4000)
|
||||
@@ -230,6 +386,7 @@ class CreateMessage(BaseModel):
|
||||
raise ValueError("Message must contain either text or attachments")
|
||||
return self
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
def health_live(response: Response):
|
||||
req_id = generate_request_id()
|
||||
@@ -237,6 +394,7 @@ def health_live(response: Response):
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
def health_ready(response: Response):
|
||||
req_id = generate_request_id()
|
||||
@@ -251,6 +409,7 @@ def health_ready(response: Response):
|
||||
raise HTTPException(status_code=503, detail="Database, Redis or S3 unavailable")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
def register_user(body: CreateUser, response: Response):
|
||||
req_id = generate_request_id()
|
||||
@@ -285,6 +444,7 @@ def register_user(body: CreateUser, response: Response):
|
||||
"created_at": created_at
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
def create_session(authorization: Optional[str] = Header(None), response: Response = None):
|
||||
req_id = generate_request_id()
|
||||
@@ -339,6 +499,7 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def get_current_user_from_token(authorization: Optional[str] = Header(None)) -> dict:
|
||||
if not authorization or not authorization.startswith("Bearer "):
|
||||
raise HTTPException(
|
||||
@@ -380,6 +541,7 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) ->
|
||||
"created_at": row['created_at'].isoformat().replace("+00:00", "Z") if isinstance(row['created_at'], datetime) else row['created_at']
|
||||
}
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def revoke_session(authorization: Optional[str] = Header(None)):
|
||||
req_id = generate_request_id()
|
||||
@@ -409,6 +571,7 @@ def revoke_session(authorization: Optional[str] = Header(None)):
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
def current_user(response: Response, authorization: Optional[str] = Header(None)):
|
||||
req_id = generate_request_id()
|
||||
@@ -416,6 +579,7 @@ def current_user(response: Response, authorization: Optional[str] = Header(None)
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return get_current_user_from_token(authorization)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
def create_chat(body: CreateChat, response: Response, authorization: Optional[str] = Header(None)):
|
||||
req_id = generate_request_id()
|
||||
@@ -466,6 +630,7 @@ def create_chat(body: CreateChat, response: Response, authorization: Optional[st
|
||||
"created_at": created_at
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response = None, authorization: Optional[str] = Header(None)):
|
||||
req_id = generate_request_id()
|
||||
@@ -522,6 +687,7 @@ def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response
|
||||
|
||||
return {"items": page_items, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = Header(None)):
|
||||
req_id = generate_request_id()
|
||||
@@ -553,6 +719,7 @@ def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = H
|
||||
"created_at": chat['created_at'].isoformat().replace("+00:00", "Z") if isinstance(chat['created_at'], datetime) else chat['created_at']
|
||||
}
|
||||
|
||||
|
||||
def validate_image_and_get_dimensions(data: bytes, content_type: str):
|
||||
if len(data) > 10 * 1024 * 1024:
|
||||
raise HTTPException(status_code=413, detail="Payload too large (>10 MiB)")
|
||||
@@ -576,14 +743,14 @@ def validate_image_and_get_dimensions(data: bytes, content_type: str):
|
||||
while i < len(data) - 9:
|
||||
marker = data[i]
|
||||
if marker == 0xff:
|
||||
marker_type = data[i+1]
|
||||
marker_type = data[i + 1]
|
||||
if 0xc0 <= marker_type <= 0xc3:
|
||||
height, width = struct.unpack('>HH', data[i+5:i+9])
|
||||
height, width = struct.unpack('>HH', data[i + 5:i + 9])
|
||||
break
|
||||
if marker_type in {0xd8, 0xd9} or (0xd0 <= marker_type <= 0xd7):
|
||||
i += 2
|
||||
continue
|
||||
seg_len = struct.unpack('>H', data[i+2:i+4])[0]
|
||||
seg_len = struct.unpack('>H', data[i + 2:i + 4])[0]
|
||||
if seg_len < 2:
|
||||
raise HTTPException(status_code=415, detail="Invalid JPEG segment")
|
||||
i += 2 + seg_len
|
||||
@@ -598,6 +765,7 @@ def validate_image_and_get_dimensions(data: bytes, content_type: str):
|
||||
if width * height > 40_000_000:
|
||||
raise HTTPException(status_code=413, detail="Image exceeds 40 million pixels limit")
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
|
||||
async def upload_attachment(
|
||||
chat_id: UUID,
|
||||
@@ -718,6 +886,7 @@ async def upload_attachment(
|
||||
"updated_at": now
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
def get_attachment(attachment_id: UUID, response: Response, authorization: Optional[str] = Header(None)):
|
||||
req_id = generate_request_id()
|
||||
@@ -754,6 +923,7 @@ def get_attachment(attachment_id: UUID, response: Response, authorization: Optio
|
||||
"updated_at": row['updated_at'].isoformat().replace("+00:00", "Z") if isinstance(row['updated_at'], datetime) else row['updated_at']
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def download_attachment(
|
||||
attachment_id: UUID,
|
||||
@@ -809,6 +979,7 @@ def download_attachment(
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
def create_message(
|
||||
chat_id: UUID,
|
||||
@@ -908,6 +1079,7 @@ def create_message(
|
||||
|
||||
return msg_response
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None,
|
||||
response: Response = None, authorization: Optional[str] = Header(None)):
|
||||
@@ -965,6 +1137,7 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None,
|
||||
|
||||
return {"items": formatted, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
@app.post("/api/v1/attachments/{attachment_id}/replay", status_code=202)
|
||||
def replay_attachment(attachment_id: UUID, response: Response,
|
||||
authorization: Optional[str] = Header(None)):
|
||||
|
||||
+96
-4
@@ -52,7 +52,7 @@ services:
|
||||
restart: "no"
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:latest
|
||||
image: chrislusf/seaweedfs:3.67
|
||||
command: ["server", "-s3", "-dir=/data", "-s3.port=8333"]
|
||||
environment:
|
||||
- AWS_ACCESS_KEY_ID=appuser
|
||||
@@ -65,11 +65,11 @@ services:
|
||||
cpus: '0.50'
|
||||
memory: 512M
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz || exit 1"]
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/ || test $$? -eq 8 || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
start_period: 10s
|
||||
start_period: 30s
|
||||
|
||||
s3-init:
|
||||
image: python:3.11.7-slim-bookworm
|
||||
@@ -89,6 +89,76 @@ services:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.51.0
|
||||
command:
|
||||
- "--config.file=/etc/prometheus/prometheus.yml"
|
||||
- "--storage.tsdb.path=/prometheus"
|
||||
- "--storage.tsdb.retention.time=7d"
|
||||
- "--web.enable-remote-write-receiver"
|
||||
volumes:
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- promdata:/prometheus
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9090/-/healthy || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.0.0
|
||||
command: ["-config.file=/etc/loki/local-config.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki-config.yaml:/etc/loki/local-config.yaml:ro
|
||||
- lokidata:/loki
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3100/ready || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.4.1
|
||||
command: ["-config.file=/etc/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo-config.yaml:/etc/tempo.yaml:ro
|
||||
- tempodata:/var/tempo
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3200/ready || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.97.0
|
||||
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:10.4.2
|
||||
ports:
|
||||
- "3000:3000"
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=${GF_ADMIN_USER:-admin}
|
||||
- GF_SECURITY_ADMIN_PASSWORD=${GF_ADMIN_PASSWORD:-admin}
|
||||
- GF_AUTH_ANONYMOUS_ENABLED=false
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafanadata:/var/lib/grafana
|
||||
depends_on:
|
||||
prometheus:
|
||||
condition: service_healthy
|
||||
loki:
|
||||
condition: service_healthy
|
||||
tempo:
|
||||
condition: service_healthy
|
||||
|
||||
api-1:
|
||||
build: .
|
||||
stop_grace_period: 15s
|
||||
@@ -103,6 +173,10 @@ services:
|
||||
- S3_ACCESS_KEY_ID=appuser
|
||||
- S3_SECRET_ACCESS_KEY=appsecretpassword
|
||||
- BROKER_STREAM=attachment_jobs
|
||||
- OTEL_SERVICE_NAME=messenger-api
|
||||
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
- OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
- OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
@@ -117,6 +191,8 @@ services:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"]
|
||||
interval: 5s
|
||||
@@ -137,6 +213,10 @@ services:
|
||||
- S3_ACCESS_KEY_ID=appuser
|
||||
- S3_SECRET_ACCESS_KEY=appsecretpassword
|
||||
- BROKER_STREAM=attachment_jobs
|
||||
- OTEL_SERVICE_NAME=messenger-api
|
||||
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
- OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
- OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
@@ -151,6 +231,8 @@ services:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"]
|
||||
interval: 5s
|
||||
@@ -172,6 +254,10 @@ services:
|
||||
- BROKER_GROUP=attachment_workers
|
||||
- JOB_MAX_ATTEMPTS=5
|
||||
- MAX_IMAGE_PIXELS=40000000
|
||||
- OTEL_SERVICE_NAME=messenger-worker
|
||||
- OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
- OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
- OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
@@ -186,6 +272,8 @@ services:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os, redis; redis.Redis.from_url(os.environ['REDIS_URL']).ping()"]
|
||||
interval: 5s
|
||||
@@ -215,4 +303,8 @@ services:
|
||||
volumes:
|
||||
pgdata:
|
||||
redisdata:
|
||||
s3data:
|
||||
s3data:
|
||||
promdata:
|
||||
lokidata:
|
||||
tempodata:
|
||||
grafanadata:
|
||||
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"title": "MTUSI Messenger — Lab 6 (Grade 3)",
|
||||
"uid": "mtusi-messenger-lab6-g3",
|
||||
"schemaVersion": 39,
|
||||
"version": 2,
|
||||
"refresh": "5s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API RPS",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(http_server_requests_total[1m]))",
|
||||
"legendFormat": "req/s"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "API Errors (4xx/5xx/s)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(http_server_errors_total[1m]))",
|
||||
"legendFormat": "errors/s"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API Duration p95 (s)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Worker Jobs (success / failed)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(worker_jobs_total[1m])) by (status)",
|
||||
"legendFormat": "{{status}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Worker Job Duration p95 (s)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Structured Logs (API & Worker)",
|
||||
"type": "logs",
|
||||
"gridPos": { "h": 9, "w": 24, "x": 0, "y": 14 },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "{service_name=~\"messenger-api|messenger-worker\"}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: MTUSI Messenger Lab6
|
||||
orgId: 1
|
||||
folder: Messenger
|
||||
type: file
|
||||
disableDeletion: false
|
||||
editable: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,37 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: true
|
||||
|
||||
- name: Loki
|
||||
type: loki
|
||||
uid: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: true
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- datasourceUid: tempo
|
||||
matcherRegex: '"trace_id":"([0-9a-f]{32})"'
|
||||
name: TraceID
|
||||
url: '$${__value.raw}'
|
||||
|
||||
- name: Tempo
|
||||
type: tempo
|
||||
uid: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: true
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
filterByTraceID: true
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service
|
||||
@@ -0,0 +1,38 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
ruler:
|
||||
alertmanager_url: http://localhost:9093
|
||||
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
|
||||
distributor:
|
||||
otlp_config:
|
||||
default_resource_attributes_as_index_labels:
|
||||
- service.name
|
||||
- service.namespace
|
||||
- deployment.environment
|
||||
@@ -0,0 +1,54 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 256
|
||||
spike_limit_mib: 64
|
||||
batch:
|
||||
send_batch_size: 256
|
||||
timeout: 2s
|
||||
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
exporters:
|
||||
prometheusremotewrite:
|
||||
endpoint: http://prometheus:9090/api/v1/write
|
||||
tls:
|
||||
insecure: true
|
||||
otlphttp/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
tls:
|
||||
insecure: true
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
headers:
|
||||
X-Scope-OrgID: fake
|
||||
tls:
|
||||
insecure: true
|
||||
debug:
|
||||
verbosity: detailed
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheusremotewrite]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki, debug]
|
||||
@@ -0,0 +1,8 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
@@ -0,0 +1,25 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
ingester:
|
||||
trace_idle_period: 10s
|
||||
max_block_duration: 5m
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
+6
-1
@@ -12,4 +12,9 @@ psycopg2-binary==2.9.9
|
||||
bcrypt==4.1.2
|
||||
redis==5.0.1
|
||||
boto3==1.34.0
|
||||
Pillow==10.2.0
|
||||
Pillow==10.2.0
|
||||
opentelemetry-api==1.24.0
|
||||
opentelemetry-sdk==1.24.0
|
||||
opentelemetry-exporter-otlp-proto-http==1.24.0
|
||||
opentelemetry-instrumentation-fastapi==0.45b0
|
||||
opentelemetry-instrumentation-requests==0.45b0
|
||||
@@ -0,0 +1,92 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
|
||||
|
||||
class JsonLogFormatter(logging.Formatter):
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
span = trace.get_current_span()
|
||||
ctx = span.get_span_context() if span else None
|
||||
trace_id = format(ctx.trace_id, "032x") if ctx and ctx.is_valid else None
|
||||
span_id = format(ctx.span_id, "016x") if ctx and ctx.is_valid else None
|
||||
|
||||
payload = {
|
||||
"ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"service": os.getenv("OTEL_SERVICE_NAME", "messenger-app"),
|
||||
"msg": record.getMessage(),
|
||||
}
|
||||
if trace_id and span_id:
|
||||
payload["trace_id"] = trace_id
|
||||
payload["span_id"] = span_id
|
||||
|
||||
for attr in ("method", "path", "status_code", "job_id", "attachment_id", "job_status", "attempt", "error_code"):
|
||||
if hasattr(record, attr):
|
||||
payload[attr] = getattr(record, attr)
|
||||
|
||||
if record.exc_info:
|
||||
payload["exception"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception"
|
||||
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def setup_telemetry(default_service_name: str):
|
||||
service_name = os.getenv("OTEL_SERVICE_NAME", default_service_name)
|
||||
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
|
||||
env_name = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=lab")
|
||||
|
||||
attrs = {"service.name": service_name}
|
||||
for part in env_name.split(","):
|
||||
if "=" in part:
|
||||
k, v = part.split("=", 1)
|
||||
attrs[k.strip()] = v.strip()
|
||||
|
||||
resource = Resource.create(attrs)
|
||||
|
||||
tracer_provider = TracerProvider(resource=resource)
|
||||
tracer_provider.add_span_processor(
|
||||
BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces"))
|
||||
)
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
|
||||
metric_reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"),
|
||||
export_interval_millis=5000,
|
||||
)
|
||||
meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs"))
|
||||
)
|
||||
set_logger_provider(logger_provider)
|
||||
|
||||
root_logger = logging.getLogger()
|
||||
root_logger.setLevel(logging.INFO)
|
||||
root_logger.handlers.clear()
|
||||
|
||||
stream_handler = logging.StreamHandler()
|
||||
stream_handler.setFormatter(JsonLogFormatter())
|
||||
root_logger.addHandler(stream_handler)
|
||||
|
||||
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||
otel_handler.setFormatter(JsonLogFormatter())
|
||||
root_logger.addHandler(otel_handler)
|
||||
|
||||
return trace.get_tracer(service_name), metrics.get_meter(service_name)
|
||||
@@ -17,9 +17,109 @@ import botocore.exceptions
|
||||
from botocore.client import Config
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
|
||||
|
||||
class JsonLogFormatter(logging.Formatter):
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
span = trace.get_current_span()
|
||||
ctx = span.get_span_context() if span else None
|
||||
trace_id = format(ctx.trace_id, "032x") if ctx and ctx.is_valid else None
|
||||
span_id = format(ctx.span_id, "016x") if ctx and ctx.is_valid else None
|
||||
|
||||
payload = {
|
||||
"ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"service": os.getenv("OTEL_SERVICE_NAME", "messenger-worker"),
|
||||
"msg": record.getMessage(),
|
||||
}
|
||||
if trace_id and span_id:
|
||||
payload["trace_id"] = trace_id
|
||||
payload["span_id"] = span_id
|
||||
|
||||
for attr in ("method", "path", "status_code", "job_id", "attachment_id", "job_status", "attempt", "error_code"):
|
||||
if hasattr(record, attr):
|
||||
payload[attr] = getattr(record, attr)
|
||||
|
||||
if record.exc_info:
|
||||
payload["exception"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception"
|
||||
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def setup_worker_telemetry():
|
||||
service_name = os.getenv("OTEL_SERVICE_NAME", "messenger-worker")
|
||||
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
|
||||
env_attrs = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=lab")
|
||||
|
||||
attrs = {"service.name": service_name}
|
||||
for part in env_attrs.split(","):
|
||||
if "=" in part:
|
||||
k, v = part.split("=", 1)
|
||||
attrs[k.strip()] = v.strip()
|
||||
|
||||
resource = Resource.create(attrs)
|
||||
|
||||
tracer_provider = TracerProvider(resource=resource)
|
||||
tracer_provider.add_span_processor(
|
||||
BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces"))
|
||||
)
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
|
||||
metric_reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"),
|
||||
export_interval_millis=5000,
|
||||
)
|
||||
meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs"))
|
||||
)
|
||||
set_logger_provider(logger_provider)
|
||||
|
||||
root_logger = logging.getLogger()
|
||||
root_logger.setLevel(logging.INFO)
|
||||
root_logger.handlers.clear()
|
||||
|
||||
stream_handler = logging.StreamHandler()
|
||||
stream_handler.setFormatter(JsonLogFormatter())
|
||||
root_logger.addHandler(stream_handler)
|
||||
|
||||
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||
otel_handler.setFormatter(JsonLogFormatter())
|
||||
root_logger.addHandler(otel_handler)
|
||||
|
||||
return trace.get_tracer(service_name), metrics.get_meter(service_name)
|
||||
|
||||
|
||||
tracer, meter = setup_worker_telemetry()
|
||||
logger = logging.getLogger("attachment-worker")
|
||||
|
||||
worker_jobs_counter = meter.create_counter(
|
||||
name="worker_jobs_total",
|
||||
description="Completed and failed attachment jobs",
|
||||
unit="1",
|
||||
)
|
||||
worker_duration_histogram = meter.create_histogram(
|
||||
name="worker_job_duration_seconds",
|
||||
description="Attachment job processing duration in seconds",
|
||||
unit="s",
|
||||
)
|
||||
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
|
||||
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
|
||||
@@ -49,11 +149,13 @@ s3_client = boto3.client(
|
||||
|
||||
running = True
|
||||
|
||||
|
||||
def handle_sigterm(signum, frame):
|
||||
global running
|
||||
logger.info("Received shutdown signal, finishing in-flight jobs...")
|
||||
running = False
|
||||
|
||||
|
||||
signal.signal(signal.SIGTERM, handle_sigterm)
|
||||
signal.signal(signal.SIGINT, handle_sigterm)
|
||||
|
||||
@@ -91,7 +193,7 @@ def prepare_base_image(raw_bytes):
|
||||
return img
|
||||
|
||||
|
||||
def generate_grade4_variants(raw_bytes):
|
||||
def generate_grade3_variants(raw_bytes):
|
||||
base_img = prepare_base_image(raw_bytes)
|
||||
orig_w, orig_h = base_img.size
|
||||
meta = []
|
||||
@@ -105,36 +207,18 @@ def generate_grade4_variants(raw_bytes):
|
||||
t_w, t_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
|
||||
if (t_w, t_h) != (orig_w, orig_h):
|
||||
thumb = thumb.resize((t_w, t_h), Image.Resampling.LANCZOS)
|
||||
buf = io.BytesIO(); thumb.save(buf, format="JPEG", quality=85, optimize=True)
|
||||
tb = buf.getvalue(); blobs["thumbnail"] = tb
|
||||
meta.append({"name": "thumbnail", "content_type": "image/jpeg",
|
||||
"size_bytes": len(tb), "width": t_w, "height": t_h,
|
||||
"sha256": hashlib.sha256(tb).hexdigest()})
|
||||
|
||||
opt = base_img.copy()
|
||||
if orig_w <= 1280 and orig_h <= 1280:
|
||||
o_w, o_h = orig_w, orig_h
|
||||
else:
|
||||
s = min(1280.0 / orig_w, 1280.0 / orig_h)
|
||||
o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
|
||||
if (o_w, o_h) != (orig_w, orig_h):
|
||||
opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS)
|
||||
buf = io.BytesIO(); opt.save(buf, format="WEBP", quality=85)
|
||||
ob = buf.getvalue(); blobs["optimized"] = ob
|
||||
meta.append({"name": "optimized", "content_type": "image/webp",
|
||||
"size_bytes": len(ob), "width": o_w, "height": o_h,
|
||||
"sha256": hashlib.sha256(ob).hexdigest()})
|
||||
|
||||
c = min(orig_w, orig_h)
|
||||
left, top = (orig_w - c) // 2, (orig_h - c) // 2
|
||||
crop = base_img.crop((left, top, left + c, top + c))
|
||||
if c != 128:
|
||||
crop = crop.resize((128, 128), Image.Resampling.LANCZOS)
|
||||
buf = io.BytesIO(); crop.save(buf, format="JPEG", quality=85, optimize=True)
|
||||
cb = buf.getvalue(); blobs["crop"] = cb
|
||||
meta.append({"name": "crop", "content_type": "image/jpeg",
|
||||
"size_bytes": len(cb), "width": 128, "height": 128,
|
||||
"sha256": hashlib.sha256(cb).hexdigest()})
|
||||
buf = io.BytesIO()
|
||||
thumb.save(buf, format="JPEG", quality=85, optimize=True)
|
||||
tb = buf.getvalue()
|
||||
blobs["thumbnail"] = tb
|
||||
meta.append({
|
||||
"name": "thumbnail",
|
||||
"content_type": "image/jpeg",
|
||||
"size_bytes": len(tb),
|
||||
"width": t_w,
|
||||
"height": t_h,
|
||||
"sha256": hashlib.sha256(tb).hexdigest(),
|
||||
})
|
||||
|
||||
return meta, blobs
|
||||
|
||||
@@ -144,18 +228,16 @@ def mark_failed(attachment_id, error_code):
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=%s "
|
||||
"WHERE id=%s AND status != 'ready'",
|
||||
(error_code, now, attachment_id))
|
||||
cur.execute(
|
||||
"UPDATE attachments SET status='failed', error_code=%s, updated_at=%s "
|
||||
"WHERE id=%s AND status != 'ready'",
|
||||
(error_code, now, attachment_id)
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
PERMANENT_ERRORS = (
|
||||
UnidentifiedImageError, Image.DecompressionBombError, ValueError,
|
||||
psycopg2.errors.DataError,
|
||||
)
|
||||
TRANSIENT_ERRORS = (
|
||||
TimeoutError,
|
||||
botocore.exceptions.EndpointConnectionError,
|
||||
@@ -165,165 +247,203 @@ TRANSIENT_ERRORS = (
|
||||
)
|
||||
|
||||
|
||||
def _record_job_outcome(status: str, elapsed: float):
|
||||
worker_jobs_counter.add(1, {"status": status})
|
||||
worker_duration_histogram.record(elapsed, {"status": status})
|
||||
|
||||
|
||||
def process_job(message_id, fields):
|
||||
try:
|
||||
job = json.loads(fields.get("payload", "{}"))
|
||||
except Exception:
|
||||
logger.error("Malformed job %s, ack to drop", message_id)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
attachment_id = job.get("attachment_id")
|
||||
job_id = job.get("job_id")
|
||||
pipeline_version = int(job.get("pipeline_version", 1))
|
||||
attempt = int(job.get("attempt", 1))
|
||||
|
||||
if not attachment_id:
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,))
|
||||
att = cur.fetchone()
|
||||
if not att:
|
||||
conn.commit()
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
|
||||
conn.commit()
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
if att["status"] == "failed":
|
||||
conn.commit()
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
if att["status"] == "processing":
|
||||
updated = att["updated_at"]
|
||||
if isinstance(updated, str):
|
||||
updated = datetime.fromisoformat(updated.replace("Z", "+00:00"))
|
||||
age_seconds = (datetime.now(timezone.utc) - updated).total_seconds()
|
||||
if age_seconds < (JOB_TIMEOUT_SECONDS + 10):
|
||||
conn.commit()
|
||||
logger.info("Attachment %s already processing (age=%.1fs), ack duplicate",
|
||||
attachment_id, age_seconds)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
cur.execute(
|
||||
"UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s",
|
||||
(attempt, now, attachment_id)
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
chat_id = str(att["chat_id"])
|
||||
staging_key = f"staging/{chat_id}/{attachment_id}/original"
|
||||
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
def check_timeout():
|
||||
if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS:
|
||||
raise TimeoutError("Job exceeded time limit")
|
||||
|
||||
try:
|
||||
job_start = time.monotonic()
|
||||
with tracer.start_as_current_span("worker.process_attachment") as span:
|
||||
try:
|
||||
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key)
|
||||
job = json.loads(fields.get("payload", "{}"))
|
||||
except Exception:
|
||||
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
|
||||
raw_bytes = obj["Body"].read()
|
||||
check_timeout()
|
||||
logger.error("Malformed job %s, ack to drop", message_id)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
return
|
||||
|
||||
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
|
||||
raise ValueError("Staging checksum mismatch")
|
||||
attachment_id = job.get("attachment_id")
|
||||
job_id = job.get("job_id")
|
||||
pipeline_version = int(job.get("pipeline_version", 1))
|
||||
attempt = int(job.get("attempt", 1))
|
||||
|
||||
variants_meta, variants_bytes = generate_grade4_variants(raw_bytes)
|
||||
check_timeout()
|
||||
span.set_attribute("job.pipeline_version", pipeline_version)
|
||||
span.set_attribute("job.attempt", attempt)
|
||||
|
||||
s3_client.put_object(
|
||||
Bucket=S3_BUCKET, Key=final_original_key,
|
||||
Body=raw_bytes, ContentType=att["content_type"]
|
||||
)
|
||||
for var_name, var_bytes in variants_bytes.items():
|
||||
ct = "image/webp" if var_name == "optimized" else "image/jpeg"
|
||||
s3_client.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=f"attachments/{chat_id}/{attachment_id}/{var_name}",
|
||||
Body=var_bytes, ContentType=ct
|
||||
)
|
||||
check_timeout()
|
||||
if not attachment_id:
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
return
|
||||
|
||||
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
UPDATE attachments SET status='ready', variants=%s::jsonb,
|
||||
error_code=NULL, updated_at=%s
|
||||
WHERE id=%s AND pipeline_version <= %s
|
||||
""", (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version))
|
||||
cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,))
|
||||
att = cur.fetchone()
|
||||
if not att:
|
||||
conn.commit()
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
return
|
||||
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
|
||||
conn.commit()
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("success", time.monotonic() - job_start)
|
||||
return
|
||||
if att["status"] == "failed":
|
||||
conn.commit()
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
return
|
||||
|
||||
if att["status"] == "processing":
|
||||
updated = att["updated_at"]
|
||||
if isinstance(updated, str):
|
||||
updated = datetime.fromisoformat(updated.replace("Z", "+00:00"))
|
||||
age_seconds = (datetime.now(timezone.utc) - updated).total_seconds()
|
||||
if age_seconds < (JOB_TIMEOUT_SECONDS + 10):
|
||||
conn.commit()
|
||||
logger.info(
|
||||
"Attachment already processing, ack duplicate",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
|
||||
)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
cur.execute(
|
||||
"UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s",
|
||||
(attempt, now, attachment_id)
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
logger.info("Job %s completed: attachment %s ready", job_id, attachment_id)
|
||||
chat_id = str(att["chat_id"])
|
||||
staging_key = f"staging/{chat_id}/{attachment_id}/original"
|
||||
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
def check_timeout():
|
||||
if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS:
|
||||
raise TimeoutError("Job exceeded time limit")
|
||||
|
||||
try:
|
||||
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
except (UnidentifiedImageError, Image.DecompressionBombError) as e:
|
||||
logger.error("Permanent image error for %s: %s", attachment_id, type(e).__name__)
|
||||
mark_failed(attachment_id, "invalid_image")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
|
||||
except ValueError as e:
|
||||
logger.error("Checksum error for %s: %s", attachment_id, e)
|
||||
mark_failed(attachment_id, "invalid_checksum")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
|
||||
except psycopg2.errors.DataError as e:
|
||||
logger.error("DB data error for %s: %s", attachment_id, type(e).__name__)
|
||||
mark_failed(attachment_id, "data_error")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
|
||||
except TRANSIENT_ERRORS as e:
|
||||
logger.warning(
|
||||
"Transient error for %s (attempt %d/%d): %s",
|
||||
attachment_id, attempt, JOB_MAX_ATTEMPTS, type(e).__name__
|
||||
)
|
||||
if attempt >= JOB_MAX_ATTEMPTS:
|
||||
mark_failed(attachment_id, "retry_exhausted")
|
||||
try:
|
||||
dlq_job = dict(job)
|
||||
dlq_job["attempt"] = 1
|
||||
redis_client.xadd(
|
||||
DLQ_STREAM_NAME,
|
||||
{"payload": json.dumps(dlq_job), "error": str(e)}
|
||||
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
|
||||
raw_bytes = obj["Body"].read()
|
||||
check_timeout()
|
||||
|
||||
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
|
||||
raise ValueError("Staging checksum mismatch")
|
||||
|
||||
variants_meta, variants_bytes = generate_grade3_variants(raw_bytes)
|
||||
check_timeout()
|
||||
|
||||
s3_client.put_object(
|
||||
Bucket=S3_BUCKET, Key=final_original_key,
|
||||
Body=raw_bytes, ContentType=att["content_type"]
|
||||
)
|
||||
for var_name, var_bytes in variants_bytes.items():
|
||||
ct = "image/webp" if var_name == "optimized" else "image/jpeg"
|
||||
s3_client.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=f"attachments/{chat_id}/{attachment_id}/{var_name}",
|
||||
Body=var_bytes, ContentType=ct
|
||||
)
|
||||
check_timeout()
|
||||
|
||||
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
UPDATE attachments SET status='ready', variants=%s::jsonb,
|
||||
error_code=NULL, updated_at=%s
|
||||
WHERE id=%s AND pipeline_version <= %s
|
||||
""", (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("success", time.monotonic() - job_start)
|
||||
logger.info(
|
||||
"Job completed: attachment ready",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "ready", "attempt": attempt},
|
||||
)
|
||||
|
||||
try:
|
||||
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
else:
|
||||
delay = min(2 ** attempt, 30) + random.uniform(0, 1)
|
||||
time.sleep(delay)
|
||||
next_job = dict(job)
|
||||
next_job["attempt"] = attempt + 1
|
||||
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Unknown error for %s: %s, marking failed",
|
||||
attachment_id, type(e).__name__)
|
||||
mark_failed(attachment_id, "processing_failed")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
except (UnidentifiedImageError, Image.DecompressionBombError) as e:
|
||||
logger.error(
|
||||
"Permanent image error",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_image"},
|
||||
)
|
||||
mark_failed(attachment_id, "invalid_image")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
|
||||
except ValueError as e:
|
||||
logger.error(
|
||||
"Checksum error",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_checksum"},
|
||||
)
|
||||
mark_failed(attachment_id, "invalid_checksum")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
|
||||
except psycopg2.errors.DataError as e:
|
||||
logger.error(
|
||||
"DB data error",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "data_error"},
|
||||
)
|
||||
mark_failed(attachment_id, "data_error")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
|
||||
except TRANSIENT_ERRORS as e:
|
||||
logger.warning(
|
||||
"Transient error during job processing",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
|
||||
)
|
||||
if attempt >= JOB_MAX_ATTEMPTS:
|
||||
mark_failed(attachment_id, "retry_exhausted")
|
||||
try:
|
||||
dlq_job = dict(job)
|
||||
dlq_job["attempt"] = 1
|
||||
redis_client.xadd(
|
||||
DLQ_STREAM_NAME,
|
||||
{"payload": json.dumps(dlq_job), "error": type(e).__name__}
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
else:
|
||||
delay = min(2 ** attempt, 30) + random.uniform(0, 1)
|
||||
time.sleep(delay)
|
||||
next_job = dict(job)
|
||||
next_job["attempt"] = attempt + 1
|
||||
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Unknown error during job processing",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "processing_failed"},
|
||||
)
|
||||
mark_failed(attachment_id, "processing_failed")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
|
||||
|
||||
def run_staging_cleanup():
|
||||
try:
|
||||
@@ -337,11 +457,10 @@ def run_staging_cleanup():
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.debug("Staging cleanup: %s", e)
|
||||
logger.debug("Staging cleanup: %s", type(e).__name__)
|
||||
|
||||
|
||||
def safe_process(msg_id, fields):
|
||||
"""Wrapper - exceptions в threads иначе исчезают молча."""
|
||||
try:
|
||||
process_job(msg_id, fields)
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user