Собрать наблюдаемый стенд, чтобы причину деградации API и worker было видно по метрикам, логам и трассам.

This commit is contained in:
2026-11-07 01:36:19 +03:00
parent 470d9d6f1e
commit 7d4e1237a3
56 changed files with 4419 additions and 53 deletions
+1
View File
@@ -0,0 +1 @@
"""Messenger application for lab 6."""
+124
View File
@@ -0,0 +1,124 @@
import asyncio
import random
import time
import uuid
from opentelemetry.trace import SpanKind
from app import faults, http, telemetry
from app.errors import AppError
from app.settings import settings
class _TooLarge(Exception):
pass
async def _read_body(receive, limit):
chunks = []
size = 0
while True:
message = await receive()
if message["type"] == "http.disconnect":
return None
if message["type"] != "http.request":
continue
data = message.get("body", b"") or b""
size += len(data)
if size > limit:
raise _TooLarge()
chunks.append(data)
if not message.get("more_body"):
return b"".join(chunks)
def _too_large(scope, limit):
for name, value in scope.get("headers", []):
if name == b"content-length":
try:
return int(value) > limit
except ValueError:
return True
return False
async def app(scope, receive, send):
if scope["type"] == "lifespan":
while True:
message = await receive()
if message["type"] == "lifespan.startup":
telemetry.log_event("service.listening", port=8080)
await send({"type": "lifespan.startup.complete"})
elif message["type"] == "lifespan.shutdown":
await send({"type": "lifespan.shutdown.complete"})
return
return
if scope["type"] != "http":
return
method = scope.get("method", "GET")
path = scope.get("path", "/")
raw_headers = {}
for name, value in scope.get("headers", []):
raw_headers[name.decode("latin-1").lower()] = value.decode("latin-1")
request_id = http.request_id_from(raw_headers)
template, _name, _groups = http.match(method, path)
limit = http.body_limit(path)
started = time.perf_counter()
status = 500
response_headers = {}
body = b""
with telemetry.tracer().start_as_current_span(
f"{method} {template}",
kind=SpanKind.SERVER,
attributes={"http.request.method": method, "http.route": template},
) as span:
try:
if _too_large(scope, limit):
raise AppError(413, "payload_too_large", "Слишком большой объект")
try:
payload = await _read_body(receive, limit)
except _TooLarge as exc:
raise AppError(413, "payload_too_large", "Слишком большой объект") from exc
if payload is None:
return
if method in {"POST", "PUT"} and path.rstrip("/").endswith("/attachments") and len(payload) > settings.max_upload:
raise AppError(413, "payload_too_large", "Слишком большой объект")
query = http.query_from(scope.get("query_string", b"").decode("ascii", "replace"))
if path.startswith("/api/") and random.random() < faults.http_error_ratio():
raise AppError(500, "internal_error", "Внутренняя ошибка")
def run():
return http.handle(method, path, query, raw_headers, payload)
template, status, response_headers, body = await asyncio.to_thread(run)
span.set_attribute("http.route", template)
except Exception as exc:
status, response_headers, body = http.error_response(exc, request_id)
if not isinstance(exc, AppError):
telemetry.log_event("internal_error", severity="ERROR", error_type=type(exc).__name__, request_id=request_id)
if status >= 500:
telemetry.mark_error(span)
span.set_attribute("error.type", type(exc).__name__)
duration = time.perf_counter() - started
span.set_attribute("http.response.status_code", status)
telemetry.record_http(method, template, status, duration)
telemetry.log_event(
"http.request.completed",
request_id=request_id,
http_route=template,
http_method=method,
status=status,
duration_ms=int(duration * 1000),
)
header_list = [
(b"x-request-id", request_id.encode("ascii")),
(b"x-instance-id", settings.instance_id.encode("ascii")),
]
for key, value in response_headers.items():
header_list.append((key.lower().encode("latin-1"), value.encode("latin-1")))
await send({"type": "http.response.start", "status": status, "headers": header_list})
await send({"type": "http.response.body", "body": body})
def new_request_id():
return uuid.uuid4().hex
+153
View File
@@ -0,0 +1,153 @@
import base64
import json
import uuid
from urllib.parse import quote, urlparse, unquote
from urllib.request import Request, urlopen
import pika
from app import telemetry
from app.db import jsonb
from app.settings import settings
QUEUE = "attachment.process.v1"
def _params():
parsed = urlparse(settings.broker_url)
return pika.ConnectionParameters(
host=parsed.hostname or "rabbitmq",
port=parsed.port or 5672,
virtual_host=unquote(parsed.path[1:] or "/"),
credentials=pika.PlainCredentials(unquote(parsed.username or ""), unquote(parsed.password or "")),
heartbeat=60,
blocked_connection_timeout=2,
socket_timeout=2,
connection_attempts=1,
retry_delay=0,
)
def _credentials():
parsed = urlparse(settings.broker_url)
return unquote(parsed.username or ""), unquote(parsed.password or ""), unquote(parsed.path[1:] or "/")
def init():
connection = pika.BlockingConnection(_params())
try:
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True)
finally:
connection.close()
def ready():
try:
connection = pika.BlockingConnection(_params())
try:
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True, passive=True)
finally:
connection.close()
return True
except Exception:
return False
def insert_outbox(conn, payload, created_at):
conn.execute(
"""
INSERT INTO outbox (job_id, attachment_id, payload, created_at)
VALUES (%s, %s, %s, %s)
""",
(uuid.UUID(str(payload["job_id"])), uuid.UUID(str(payload["attachment_id"])), jsonb(payload), created_at),
)
def publish_pending(conn, job_id=None):
query = """
SELECT job_id, payload
FROM outbox
WHERE published_at IS NULL
"""
args = []
if job_id is not None:
query += " AND job_id = %s"
args.append(job_id)
query += " ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 20"
rows = conn.execute(query, args).fetchall()
for row in rows:
payload = row["payload"]
if isinstance(payload, str):
payload = json.loads(payload)
_publish(payload)
conn.execute("UPDATE outbox SET published_at = NOW() WHERE job_id = %s", (row["job_id"],))
return len(rows)
def _publish(payload):
carrier = {}
if payload.get("traceparent"):
carrier["traceparent"] = payload["traceparent"]
if payload.get("tracestate"):
carrier["tracestate"] = payload["tracestate"]
parent = telemetry.extract_context(carrier)
with telemetry.tracer().start_as_current_span(
"attachment.process publish",
context=parent,
kind=telemetry.trace.SpanKind.PRODUCER,
attributes={
"messaging.system": "rabbitmq",
"messaging.destination.name": QUEUE,
"messaging.operation.type": "publish",
},
):
outgoing = telemetry.inject_carrier()
body = dict(payload)
if outgoing.get("traceparent"):
body["traceparent"] = outgoing["traceparent"]
if outgoing.get("tracestate"):
body["tracestate"] = outgoing["tracestate"]
_send(body, outgoing)
telemetry.log_event(
"job.published",
job_id=str(payload.get("job_id")),
attempt=int(payload.get("attempt") or 1),
)
def _send(payload, headers):
connection = pika.BlockingConnection(_params())
try:
channel = connection.channel()
channel.confirm_delivery()
channel.basic_publish(
exchange="",
routing_key=QUEUE,
body=json.dumps(payload, separators=(",", ":")).encode("utf-8"),
properties=pika.BasicProperties(
delivery_mode=2,
content_type="application/json",
headers=headers,
),
mandatory=True,
)
finally:
connection.close()
def consumer_count():
user, password, vhost = _credentials()
url = (
settings.rabbit_management_url.rstrip("/")
+ "/api/queues/"
+ quote(vhost, safe="")
+ "/"
+ quote(QUEUE, safe="")
)
token = base64.b64encode(f"{user}:{password}".encode("utf-8")).decode("ascii")
request = Request(url, headers={"Authorization": "Basic " + token})
with urlopen(request, timeout=2) as response:
data = json.loads(response.read().decode("utf-8"))
return int(data.get("consumers") or 0)
+139
View File
@@ -0,0 +1,139 @@
import base64
import datetime as dt
import hashlib
import json
import re
import uuid
from app.errors import AppError, bad_request
USERNAME = re.compile(r"^[a-z0-9_]{3,32}$")
IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
def utcnow():
return dt.datetime.now(dt.timezone.utc)
def iso(value):
if value.tzinfo is None:
value = value.replace(tzinfo=dt.timezone.utc)
value = value.astimezone(dt.timezone.utc)
return value.strftime("%Y-%m-%dT%H:%M:%S.") + f"{value.microsecond:06d}Z"
def parse_iso(value):
if not isinstance(value, str) or not value.endswith("Z"):
raise bad_request()
try:
return dt.datetime.fromisoformat(value[:-1] + "+00:00")
except ValueError as exc:
raise bad_request() from exc
def parse_uuid(value):
if not isinstance(value, str):
raise bad_request()
try:
return str(uuid.UUID(value))
except ValueError as exc:
raise bad_request() from exc
def require_text(value, minimum, maximum, allow_blank=False):
if not isinstance(value, str):
raise bad_request()
if "\x00" in value or any(ord(ch) < 32 or ord(ch) == 127 for ch in value):
raise bad_request()
if not allow_blank and not value.strip():
raise bad_request()
if not (minimum <= len(value) <= maximum):
raise bad_request()
return value
def canonical_message(text, attachment_ids):
body = {"attachment_ids": list(attachment_ids), "text": text}
raw = json.dumps(body, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
return hashlib.sha256(raw.encode("utf-8")).digest()
def encode_cursor(kind, resource_id, created_at, row_id):
raw = f"{kind}|{resource_id}|{iso(created_at)}|{row_id}"
return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii").rstrip("=")
def decode_cursor(token, kind, resource_id):
if not isinstance(token, str) or not token:
raise bad_request()
padded = token + ("=" * (-len(token) % 4))
try:
raw = base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8")
cursor_kind, cursor_resource, stamp, row_id = raw.split("|", 3)
created_at = parse_iso(stamp)
row_uuid = str(uuid.UUID(row_id))
except (ValueError, UnicodeError, AppError) as exc:
raise bad_request() from exc
if cursor_kind != kind or cursor_resource != resource_id:
raise bad_request()
return created_at, row_uuid
def fitted_size(width, height, box=256):
if width < 1 or height < 1:
raise ValueError("dimensions")
if width <= box and height <= box:
return width, height
if width >= height:
return box, max(1, (height * box) // width)
return max(1, (width * box) // height), box
def detect_image(data, declared):
if data.startswith(b"\x89PNG\r\n\x1a\n"):
detected = "image/png"
elif data.startswith(b"\xff\xd8\xff"):
detected = "image/jpeg"
else:
return None
if declared != detected:
return None
return detected
def media_type(header):
if not header or not isinstance(header, str):
return ""
return header.split(";", 1)[0].strip().lower()
def loads_object(raw):
try:
text = raw.decode("utf-8")
except UnicodeError as exc:
raise bad_request() from exc
def pairs(items):
obj = {}
for key, value in items:
if key in obj:
raise ValueError("duplicate")
obj[key] = value
return obj
try:
value = json.loads(text, object_pairs_hook=pairs)
except (ValueError, UnicodeError) as exc:
raise bad_request() from exc
if not isinstance(value, dict):
raise bad_request()
return value
def expect_fields(data, allowed, required):
if set(data) - set(allowed):
raise bad_request()
for key in required:
if key not in data:
raise bad_request()
+98
View File
@@ -0,0 +1,98 @@
import time
from contextlib import contextmanager
from pathlib import Path
import psycopg
from psycopg.rows import dict_row
from psycopg.types.json import Jsonb
from psycopg_pool import ConnectionPool, PoolTimeout
from app import telemetry
from app.errors import AppError, unavailable
from app.faults import db_fail, db_slow_seconds
from app.settings import settings
pool = None
def init():
global pool
pool = ConnectionPool(
settings.database_url,
min_size=1,
max_size=settings.pool_max,
timeout=2,
max_waiting=16,
check=ConnectionPool.check_connection,
kwargs={
"row_factory": dict_row,
"autocommit": False,
"connect_timeout": 3,
"options": "-c timezone=UTC -c statement_timeout=4000",
},
open=True,
reconnect_timeout=2,
)
def migrate():
script = Path(__file__).with_name("schema.sql").read_text(encoding="utf-8")
statements = []
buffer = []
for line in script.splitlines():
stripped = line.strip()
if not stripped or stripped.startswith("--"):
continue
buffer.append(line)
if stripped.endswith(";"):
statements.append("\n".join(buffer))
buffer = []
with pool.connection() as conn:
conn.execute("SELECT pg_advisory_lock(861006)")
for statement in statements:
conn.execute(statement)
conn.execute("SELECT pg_advisory_unlock(861006)")
conn.commit()
def jsonb(value):
return Jsonb(value)
@contextmanager
def connection():
if db_fail():
raise unavailable()
delay = db_slow_seconds()
started = time.perf_counter()
span = telemetry.tracer().start_as_current_span(
"db.transaction",
attributes={"db.system": "postgresql"},
)
span.__enter__()
try:
if delay:
time.sleep(delay)
with pool.connection() as conn:
try:
yield conn
conn.commit()
except Exception:
conn.rollback()
raise
except AppError:
raise
except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError) as exc:
raise unavailable() from exc
finally:
span.__exit__(None, None, None)
telemetry.record_db(time.perf_counter() - started)
def stats():
if pool is None:
return 0, 0
data = pool.get_stats()
used = int(data.get("pool_size", 0)) - int(data.get("pool_available", 0))
waiting = int(data.get("requests_waiting", 0))
return max(used, 0), waiting
+584
View File
@@ -0,0 +1,584 @@
import datetime as dt
import hashlib
import uuid
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
from argon2.low_level import Type
from psycopg.errors import UniqueViolation
from app import broker, storage, telemetry
from app.codec import (
USERNAME,
IDEMPOTENCY,
canonical_message,
decode_cursor,
detect_image,
encode_cursor,
expect_fields,
iso,
parse_uuid,
require_text,
utcnow,
)
from app.db import connection, jsonb
from app.errors import bad_request, conflict, not_found, too_large, unauthorized, unsupported
from app.images import validate_image
from app.settings import settings
BASIC = 'Basic realm="messenger", charset="UTF-8"'
BEARER = 'Bearer realm="messenger"'
_hasher = PasswordHasher(time_cost=1, memory_cost=8192, parallelism=1, hash_len=32, salt_len=16, type=Type.ID)
_dummy = None
def init_hasher():
global _dummy
_dummy = _hasher.hash("not-a-real-password")
def _user_json(row):
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": iso(row["created_at"]),
}
def _token_hash(token):
return hashlib.sha256(token.encode("utf-8")).digest()
def _password(value):
if not isinstance(value, str) or "\x00" in value or not (12 <= len(value) <= 128):
raise bad_request()
return value
def as_uuid(value):
if isinstance(value, uuid.UUID):
return value
return uuid.UUID(str(value))
def _username(value):
if not isinstance(value, str) or USERNAME.fullmatch(value) is None:
raise bad_request()
return value
def register(data):
expect_fields(data, ["username", "display_name", "password"], ["username", "display_name", "password"])
username = _username(data["username"])
display_name = require_text(data["display_name"], 1, 100)
password = _password(data["password"])
password_hash = _hasher.hash(password)
user_id = uuid.uuid4()
created_at = utcnow()
try:
with connection() as conn:
conn.execute(
"""
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES (%s, %s, %s, %s, %s)
""",
(user_id, username, display_name, password_hash, created_at),
)
row = conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
(user_id,),
).fetchone()
except UniqueViolation as exc:
raise conflict("Имя пользователя занято") from exc
return 201, _user_json(row)
def login(username, password):
if not isinstance(username, str) or not isinstance(password, str) or "\x00" in username or "\x00" in password:
raise unauthorized(BASIC)
with connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = %s",
(username,),
).fetchone()
stored = row["password_hash"] if row else _dummy
try:
_hasher.verify(stored, password)
valid = row is not None
except (VerifyMismatchError, VerificationError, InvalidHashError):
valid = False
if not valid:
raise unauthorized(BASIC)
token = secrets_token()
expires_at = utcnow() + dt.timedelta(seconds=settings.session_ttl)
with connection() as conn:
conn.execute(
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
(_token_hash(token), row["id"], expires_at),
)
return 201, {
"token": token,
"token_type": "Bearer",
"expires_at": iso(expires_at),
"user": _user_json(row),
}
def secrets_token():
import secrets
return secrets.token_urlsafe(32)
def current_user(token):
if not token or len(token) < 32:
raise unauthorized(BEARER)
with connection() as conn:
row = conn.execute(
"""
SELECT u.id, u.username, u.display_name, u.created_at
FROM sessions s
JOIN users u ON u.id = s.user_id
WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > NOW()
""",
(_token_hash(token),),
).fetchone()
if not row:
raise unauthorized(BEARER)
return row
def ready_check():
with connection() as conn:
conn.execute("SELECT 1")
if not storage.ready() or not broker.ready():
raise unavailable()
def revoke(token):
user = current_user(token)
with connection() as conn:
conn.execute(
"UPDATE sessions SET revoked_at = NOW() WHERE token_hash = %s AND revoked_at IS NULL",
(_token_hash(token),),
)
return user
def create_chat(user, data):
expect_fields(data, ["title", "member_ids"], ["title", "member_ids"])
title = require_text(data["title"], 1, 100)
raw_members = data["member_ids"]
if not isinstance(raw_members, list) or not (1 <= len(raw_members) <= 100):
raise bad_request()
members = [parse_uuid(item) for item in raw_members]
if len(set(members)) != len(members):
raise bad_request()
if str(user["id"]) not in members:
raise bad_request()
chat_id = uuid.uuid4()
created_at = utcnow()
with connection() as conn:
found = conn.execute("SELECT id FROM users WHERE id = ANY(%s)", ([as_uuid(item) for item in members],)).fetchall()
if len(found) != len(members):
raise bad_request()
conn.execute(
"INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)",
(chat_id, title, created_at),
)
for position, member_id in enumerate(members):
conn.execute(
"INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)",
(chat_id, as_uuid(member_id), position),
)
return 201, {
"id": str(chat_id),
"title": title,
"member_ids": members,
"created_at": iso(created_at),
}
def _chat_json(conn, row):
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY position",
(row["id"],),
).fetchall()
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": [str(item["user_id"]) for item in members],
"created_at": iso(row["created_at"]),
}
def list_chats(user, limit, cursor):
created_at = None
row_id = None
if cursor is not None:
created_at, row_id = decode_cursor(cursor, "chats", str(user["id"]))
with connection() as conn:
if created_at is None:
rows = conn.execute(
"""
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members m ON m.chat_id = c.id
WHERE m.user_id = %s
ORDER BY c.created_at, c.id
LIMIT %s
""",
(user["id"], limit + 1),
).fetchall()
else:
rows = conn.execute(
"""
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members m ON m.chat_id = c.id
WHERE m.user_id = %s AND (c.created_at, c.id) > (%s, %s)
ORDER BY c.created_at, c.id
LIMIT %s
""",
(user["id"], created_at, as_uuid(row_id), limit + 1),
).fetchall()
extra = len(rows) > limit
rows = rows[:limit]
items = [_chat_json(conn, row) for row in rows]
next_cursor = None
if extra and rows:
last = rows[-1]
next_cursor = encode_cursor("chats", str(user["id"]), last["created_at"], str(last["id"]))
return {"items": items, "next_cursor": next_cursor}
def get_chat(user, chat_id):
with connection() as conn:
row = _visible_chat(conn, user["id"], chat_id)
return _chat_json(conn, row)
def _visible_chat(conn, user_id, chat_id):
row = conn.execute(
"""
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members m ON m.chat_id = c.id
WHERE c.id = %s AND m.user_id = %s
""",
(as_uuid(chat_id), as_uuid(user_id)),
).fetchone()
if not row:
raise not_found()
return row
def _message_json(row, attachment_ids):
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"created_at": iso(row["created_at"]),
"attachment_ids": attachment_ids,
}
def _attachment_ids(conn, message_id):
rows = conn.execute(
"SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position",
(message_id,),
).fetchall()
return [str(row["attachment_id"]) for row in rows]
def create_message(user, chat_id, key, data, path):
chat_uuid = as_uuid(chat_id)
if IDEMPOTENCY.fullmatch(key or "") is None:
raise bad_request()
expect_fields(data, ["text", "attachment_ids"], ["text"])
if not isinstance(data["text"], str) or "\x00" in data["text"] or len(data["text"]) > 4000:
raise bad_request()
text = data["text"]
if "attachment_ids" in data:
raw_ids = data["attachment_ids"]
if not isinstance(raw_ids, list) or len(raw_ids) > 10:
raise bad_request()
attachment_ids = [parse_uuid(item) for item in raw_ids]
if len(set(attachment_ids)) != len(attachment_ids):
raise bad_request()
else:
attachment_ids = []
if not text.strip() and not attachment_ids:
raise bad_request()
body_hash = canonical_message(text, attachment_ids)
lock = int.from_bytes(hashlib.sha256(f"{user['id']}|POST|{path}|{key}".encode("utf-8")).digest()[:8], "big", signed=True)
with connection() as conn:
_visible_chat(conn, user["id"], chat_id)
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
existing = conn.execute(
"""
SELECT body_hash, response_json, created_at
FROM idempotency
WHERE user_id = %s AND method = 'POST' AND path = %s AND idem_key = %s
""",
(user["id"], path, key),
).fetchone()
if existing and existing["created_at"] >= utcnow() - dt.timedelta(seconds=settings.idempotency_ttl):
if bytes(existing["body_hash"]) != body_hash:
raise conflict("Ключ идемпотентности уже использован с другим телом")
stored = existing["response_json"]
if isinstance(stored, str):
import json
stored = json.loads(stored)
return 201, stored
if existing:
conn.execute(
"DELETE FROM idempotency WHERE user_id = %s AND method = 'POST' AND path = %s AND idem_key = %s",
(user["id"], path, key),
)
if attachment_ids:
rows = conn.execute(
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s)",
([as_uuid(item) for item in attachment_ids],),
).fetchall()
found = {str(row["id"]): row for row in rows}
if len(found) != len(attachment_ids):
raise not_found()
for attachment_id in attachment_ids:
row = found[attachment_id]
if str(row["chat_id"]) != chat_id:
raise not_found()
if row["status"] == "failed":
raise conflict("Нельзя прикрепить неуспешное вложение")
message_id = uuid.uuid4()
created_at = utcnow()
conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)",
(message_id, chat_uuid, user["id"], text, created_at),
)
for position, attachment_id in enumerate(attachment_ids):
conn.execute(
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
(message_id, as_uuid(attachment_id), position),
)
stored = conn.execute(
"SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = %s",
(message_id,),
).fetchone()
payload = _message_json(stored, attachment_ids)
conn.execute(
"""
INSERT INTO idempotency (user_id, method, path, idem_key, body_hash, status_code, response_json, created_at)
VALUES (%s, 'POST', %s, %s, %s, 201, %s, %s)
""",
(user["id"], path, key, body_hash, jsonb(payload), created_at),
)
return 201, payload
def list_messages(user, chat_id, limit, cursor):
chat_uuid = as_uuid(chat_id)
created_at = None
row_id = None
if cursor is not None:
created_at, row_id = decode_cursor(cursor, "messages", chat_id)
with connection() as conn:
_visible_chat(conn, user["id"], chat_uuid)
if created_at is None:
rows = conn.execute(
"""
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = %s
ORDER BY created_at, id
LIMIT %s
""",
(chat_uuid, limit + 1),
).fetchall()
else:
rows = conn.execute(
"""
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = %s AND (created_at, id) > (%s, %s)
ORDER BY created_at, id
LIMIT %s
""",
(chat_uuid, created_at, as_uuid(row_id), limit + 1),
).fetchall()
extra = len(rows) > limit
rows = rows[:limit]
items = [_message_json(row, _attachment_ids(conn, row["id"])) for row in rows]
next_cursor = None
if extra and rows:
last = rows[-1]
next_cursor = encode_cursor("messages", chat_id, last["created_at"], str(last["id"]))
return {"items": items, "next_cursor": next_cursor}
def _variants(conn, attachment_id):
rows = conn.execute(
"""
SELECT name, content_type, size_bytes, width, height, sha256
FROM attachment_variants
WHERE attachment_id = %s AND pipeline_version = 1
ORDER BY name
""",
(attachment_id,),
).fetchall()
return [
{
"name": row["name"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"width": row["width"],
"height": row["height"],
"sha256": row["sha256"],
}
for row in rows
]
def _attachment_json(row, variants):
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": row["status"],
"variants": variants,
"error_code": row["error_code"],
"created_at": iso(row["created_at"]),
"updated_at": iso(row["updated_at"]),
}
def upload(user, chat_id, filename, content_type, data):
import hashlib as hl
if not data:
raise unsupported()
if len(data) > settings.max_upload:
raise too_large()
detected = detect_image(data, content_type)
if not detected:
raise unsupported()
validate_image(data)
with connection() as conn:
_visible_chat(conn, user["id"], chat_id)
attachment_id = uuid.uuid4()
created_at = utcnow()
digest = hl.sha256(data).hexdigest()
staging_key = f"staging/{attachment_id}"
storage.put(staging_key, data, detected, "staging")
carrier = telemetry.inject_carrier()
job_id = uuid.uuid4()
payload = {
"schema_version": 1,
"job_id": str(job_id),
"type": "attachment.process.v1",
"attachment_id": str(attachment_id),
"pipeline_version": 1,
"attempt": 1,
"created_at": iso(created_at),
}
if carrier.get("traceparent"):
payload["traceparent"] = carrier["traceparent"]
if carrier.get("tracestate"):
payload["tracestate"] = carrier["tracestate"]
row = {
"id": attachment_id,
"chat_id": chat_id,
"owner_id": user["id"],
"filename": filename,
"content_type": detected,
"size_bytes": len(data),
"sha256": digest,
"status": "queued",
"error_code": None,
"created_at": created_at,
"updated_at": created_at,
}
with connection() as conn:
_visible_chat(conn, user["id"], chat_id)
conn.execute(
"""
INSERT INTO attachments (
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status,
error_code, staging_key, pipeline_version, attempt, created_at, updated_at
) VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', NULL, %s, 1, 0, %s, %s)
""",
(
attachment_id,
as_uuid(chat_id),
user["id"],
filename,
detected,
len(data),
digest,
staging_key,
created_at,
created_at,
),
)
broker.insert_outbox(conn, payload, created_at)
try:
with connection() as conn:
broker.publish_pending(conn, job_id)
except Exception:
telemetry.log_event("job.publish_deferred", severity="WARNING", job_id=str(job_id))
telemetry.log_event("attachment.accepted", attachment_id=str(attachment_id), size_bytes=len(data), content_type=detected)
return 202, _attachment_json(row, [])
def get_attachment(user, attachment_id):
attachment_uuid = as_uuid(attachment_id)
with connection() as conn:
row = _visible_attachment(conn, user["id"], attachment_uuid)
return _attachment_json(row, _variants(conn, attachment_uuid))
def _visible_attachment(conn, user_id, attachment_id):
row = conn.execute("SELECT * FROM attachments WHERE id = %s", (as_uuid(attachment_id),)).fetchone()
if not row:
raise not_found()
member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(row["chat_id"], user_id),
).fetchone()
if not member:
raise not_found()
return row
def download(user, attachment_id, variant):
attachment_uuid = as_uuid(attachment_id)
if variant not in {"original", "thumbnail", "crop", "optimized"}:
raise bad_request()
with connection() as conn:
row = _visible_attachment(conn, user["id"], attachment_uuid)
if row["status"] != "ready":
raise conflict("Вложение ещё не готово")
if variant == "original":
key = row["original_key"]
content_type = row["content_type"]
else:
found = conn.execute(
"""
SELECT object_key, content_type
FROM attachment_variants
WHERE attachment_id = %s AND name = %s AND pipeline_version = 1
""",
(attachment_uuid, variant),
).fetchone()
if not found:
raise not_found()
key = found["object_key"]
content_type = found["content_type"]
return storage.get(key, variant), content_type
+40
View File
@@ -0,0 +1,40 @@
class AppError(Exception):
def __init__(self, status, code, message, headers=None):
super().__init__(message)
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def bad_request(message="Неверный запрос"):
return AppError(400, "invalid_request", message)
def unauthorized(challenge):
return AppError(
401,
"unauthorized",
"Требуется аутентификация",
{"WWW-Authenticate": challenge},
)
def not_found():
return AppError(404, "not_found", "Объект не найден")
def conflict(message="Конфликт"):
return AppError(409, "conflict", message)
def too_large(message="Слишком большой объект"):
return AppError(413, "payload_too_large", message)
def unsupported(message="Неподдерживаемый тип изображения"):
return AppError(415, "unsupported_media_type", message)
def unavailable(message="Сервис временно недоступен"):
return AppError(503, "unavailable", message)
+73
View File
@@ -0,0 +1,73 @@
from pathlib import Path
from app.settings import settings
def _text(name):
path = Path(settings.fault_dir) / name
try:
return path.read_text(encoding="utf-8").strip()
except (FileNotFoundError, OSError):
return ""
def db_slow_seconds():
raw = _text("db_slow_ms")
if not raw:
return 0.0
try:
value = float(raw)
except ValueError:
return 0.0
if value <= 0:
return 0.0
return min(value, 10000.0) / 1000.0
def db_fail():
return _text("db_fail") == "1"
def http_error_ratio():
raw = _text("http_error_ratio")
if not raw:
return 0.0
try:
value = float(raw)
except ValueError:
return 0.0
if value <= 0:
return 0.0
return min(value, 1.0)
def handler_slow_seconds():
raw = _text("handler_slow_ms")
if not raw:
return 0.0
try:
value = float(raw)
except ValueError:
return 0.0
if value <= 0:
return 0.0
return min(value, 10000.0) / 1000.0
def consume_process_failure():
path = Path(settings.fault_dir) / "process_fail_remaining"
try:
raw = path.read_text(encoding="utf-8").strip()
except (FileNotFoundError, OSError):
return False
try:
remaining = int(raw or "0")
except ValueError:
return False
if remaining <= 0:
return False
try:
path.write_text(str(remaining - 1), encoding="utf-8")
except OSError:
return False
return True
+221
View File
@@ -0,0 +1,221 @@
import json
import re
from urllib.parse import parse_qs
from app import domain, faults
from app.codec import REQUEST_ID, loads_object, media_type, require_text
from app.errors import AppError, bad_request, not_found, unauthorized
from app.settings import settings
_ROUTES = [
("GET", re.compile(r"^/health/live$"), "/health/live", "live"),
("GET", re.compile(r"^/health/ready$"), "/health/ready", "ready"),
("POST", re.compile(r"^/api/v1/users$"), "/api/v1/users", "register"),
("GET", re.compile(r"^/api/v1/users/me$"), "/api/v1/users/me", "me"),
("POST", re.compile(r"^/api/v1/auth/sessions$"), "/api/v1/auth/sessions", "login"),
("DELETE", re.compile(r"^/api/v1/auth/sessions/current$"), "/api/v1/auth/sessions/current", "logout"),
("POST", re.compile(r"^/api/v1/chats$"), "/api/v1/chats", "create_chat"),
("GET", re.compile(r"^/api/v1/chats$"), "/api/v1/chats", "list_chats"),
("GET", re.compile(r"^/api/v1/chats/([^/]+)$"), "/api/v1/chats/{chat_id}", "get_chat"),
("POST", re.compile(r"^/api/v1/chats/([^/]+)/messages$"), "/api/v1/chats/{chat_id}/messages", "create_message"),
("GET", re.compile(r"^/api/v1/chats/([^/]+)/messages$"), "/api/v1/chats/{chat_id}/messages", "list_messages"),
("POST", re.compile(r"^/api/v1/chats/([^/]+)/attachments$"), "/api/v1/chats/{chat_id}/attachments", "upload"),
("GET", re.compile(r"^/api/v1/attachments/([^/]+)$"), "/api/v1/attachments/{attachment_id}", "get_attachment"),
("GET", re.compile(r"^/api/v1/attachments/([^/]+)/content$"), "/api/v1/attachments/{attachment_id}/content", "download"),
]
def match(method, path):
for route_method, pattern, template, name in _ROUTES:
found = pattern.fullmatch(path)
if found and route_method == method:
return template, name, found.groups()
return "unmatched", "unmatched", ()
def _single(query, name):
values = query.get(name)
if not values:
return None
if len(values) != 1 or values[0] == "":
raise bad_request()
return values[0]
def _limit(query):
raw = _single(query, "limit")
if raw is None:
return 50
if re.fullmatch(r"[1-9][0-9]*", raw) is None:
raise bad_request()
value = int(raw)
if value > 100:
raise bad_request()
return value
def _json_body(headers, body):
if media_type(headers.get("content-type", "")) != "application/json":
raise bad_request()
if not body:
raise bad_request()
return loads_object(body)
def _bearer(headers):
value = headers.get("authorization", "")
if not value.lower().startswith("bearer "):
raise unauthorized(domain.BEARER)
token = value.split(" ", 1)[1].strip()
return token
def _basic(headers):
import base64
value = headers.get("authorization", "")
if not value.lower().startswith("basic "):
raise unauthorized(domain.BASIC)
try:
decoded = base64.b64decode(value.split(" ", 1)[1].strip(), validate=True).decode("utf-8")
except (ValueError, UnicodeError) as exc:
raise unauthorized(domain.BASIC) from exc
if ":" not in decoded:
raise unauthorized(domain.BASIC)
return decoded.split(":", 1)
def _filename(query):
raw = _single(query, "filename")
if raw is None:
raise bad_request()
if len(raw) < 1 or len(raw) > 255 or not raw.strip():
raise bad_request()
if any(ord(ch) < 32 or ord(ch) == 127 or ch in "/\\" for ch in raw):
raise bad_request()
return raw
def handle(method, path, query, headers, body):
template, name, groups = match(method, path)
if name == "unmatched":
raise not_found()
if name not in {"live", "ready"}:
delay = faults.handler_slow_seconds()
if delay:
import time
time.sleep(delay)
if name == "live":
return template, 200, {"Content-Type": "application/json; charset=utf-8"}, _dump({"status": "ok"})
if name == "ready":
domain.ready_check()
return template, 200, {"Content-Type": "application/json; charset=utf-8"}, _dump({"status": "ok"})
if name == "register":
status, payload = domain.register(_json_body(headers, body))
return template, status, _json_headers(), _dump(payload)
if name == "login":
if body:
raise bad_request()
username, password = _basic(headers)
status, payload = domain.login(username, password)
return template, status, _json_headers(), _dump(payload)
if name == "me":
user = domain.current_user(_bearer(headers))
return template, 200, _json_headers(), _dump(domain._user_json(user))
if name == "logout":
domain.revoke(_bearer(headers))
return template, 204, {}, b""
if name == "create_chat":
user = domain.current_user(_bearer(headers))
status, payload = domain.create_chat(user, _json_body(headers, body))
return template, status, _json_headers(), _dump(payload)
if name == "list_chats":
user = domain.current_user(_bearer(headers))
payload = domain.list_chats(user, _limit(query), _single(query, "cursor"))
return template, 200, _json_headers(), _dump(payload)
if name == "get_chat":
user = domain.current_user(_bearer(headers))
payload = domain.get_chat(user, _uuid_segment(groups[0]))
return template, 200, _json_headers(), _dump(payload)
if name == "create_message":
user = domain.current_user(_bearer(headers))
chat_id = _uuid_segment(groups[0])
status, payload = domain.create_message(
user,
chat_id,
headers.get("idempotency-key"),
_json_body(headers, body),
path,
)
return template, status, _json_headers(), _dump(payload)
if name == "list_messages":
user = domain.current_user(_bearer(headers))
payload = domain.list_messages(user, _uuid_segment(groups[0]), _limit(query), _single(query, "cursor"))
return template, 200, _json_headers(), _dump(payload)
if name == "upload":
user = domain.current_user(_bearer(headers))
content_type = media_type(headers.get("content-type", ""))
status, payload = domain.upload(user, _uuid_segment(groups[0]), _filename(query), content_type, body)
return template, status, _json_headers(), _dump(payload)
if name == "get_attachment":
user = domain.current_user(_bearer(headers))
payload = domain.get_attachment(user, _uuid_segment(groups[0]))
return template, 200, _json_headers(), _dump(payload)
if name == "download":
user = domain.current_user(_bearer(headers))
variant = _single(query, "variant") or "original"
data, content_type = domain.download(user, _uuid_segment(groups[0]), variant)
return template, 200, {"Content-Type": content_type}, data
raise not_found()
def _uuid_segment(value):
from app.codec import parse_uuid
return parse_uuid(value)
def _json_headers():
return {"Content-Type": "application/json; charset=utf-8"}
def _dump(payload):
return json.dumps(payload, ensure_ascii=False).encode("utf-8")
def error_response(exc, request_id):
if isinstance(exc, AppError):
status = exc.status
code = exc.code
message = exc.message
headers = dict(exc.headers)
else:
status = 500
code = "internal_error"
message = "Внутренняя ошибка"
headers = {}
headers["Content-Type"] = "application/json; charset=utf-8"
body = _dump({"error": {"code": code, "message": message, "request_id": request_id}})
return status, headers, body
def request_id_from(headers):
incoming = headers.get("x-request-id", "")
if REQUEST_ID.fullmatch(incoming):
return incoming
import uuid
return uuid.uuid4().hex
def query_from(raw):
if not raw:
return {}
return parse_qs(raw, keep_blank_values=True, strict_parsing=False)
def body_limit(path):
if "/attachments" in path and path.rstrip("/").endswith("attachments"):
return settings.max_upload + 1
return 1_000_000
+62
View File
@@ -0,0 +1,62 @@
import io
from PIL import Image, ImageOps, UnidentifiedImageError
from app.codec import fitted_size
from app.errors import AppError, too_large, unsupported
from app.settings import settings
Image.MAX_IMAGE_PIXELS = settings.max_pixels
def _open(data):
try:
with Image.open(io.BytesIO(data)) as image:
image.verify()
with Image.open(io.BytesIO(data)) as image:
width, height = image.size
if width < 1 or height < 1 or width * height > settings.max_pixels:
raise too_large()
image.load()
return image.copy()
except AppError:
raise
except Image.DecompressionBombError as exc:
raise too_large() from exc
except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombWarning) as exc:
raise unsupported() from exc
def validate_image(data):
_open(data).close()
def _rgb(image):
if image.mode == "RGB":
return image
if image.mode in ("RGBA", "LA") or (image.mode == "P" and "transparency" in image.info):
rgba = image.convert("RGBA")
background = Image.new("RGB", rgba.size, (255, 255, 255))
background.paste(rgba, mask=rgba.getchannel("A"))
return background
return image.convert("RGB")
def thumbnail_jpeg(data):
image = _open(data)
try:
image = ImageOps.exif_transpose(image)
image = _rgb(image)
target = fitted_size(image.width, image.height, 256)
if (image.width, image.height) != target:
image = image.resize(target, Image.Resampling.LANCZOS)
buffer = io.BytesIO()
image.save(buffer, format="JPEG", quality=85, optimize=True)
blob = buffer.getvalue()
return blob, image.width, image.height
except AppError:
raise
except (UnidentifiedImageError, OSError, ValueError) as exc:
raise unsupported() from exc
finally:
image.close()
+50
View File
@@ -0,0 +1,50 @@
import os
import time
import uvicorn
from app import broker, db, domain, storage, telemetry
from app.asgi import app
from app.poller import start
def _retry(label, action):
last = None
for _ in range(40):
try:
action()
return
except Exception as exc:
last = exc
telemetry.log_event("dependency.retry", severity="WARNING", dependency=label, error_type=type(exc).__name__)
time.sleep(1)
raise last
def main():
telemetry.init()
_retry("postgres", db.init)
db.migrate()
_retry("s3", storage.init)
_retry("broker", broker.init)
domain.init_hasher()
start()
uvicorn.run(
app,
host=os.environ.get("APP_HOST", "0.0.0.0"),
port=int(os.environ.get("APP_PORT", "8080")),
access_log=False,
log_config={
"version": 1,
"disable_existing_loggers": False,
"loggers": {
"uvicorn": {"level": "WARNING"},
"uvicorn.error": {"level": "WARNING"},
"uvicorn.access": {"level": "WARNING"},
},
},
)
if __name__ == "__main__":
main()
+44
View File
@@ -0,0 +1,44 @@
import threading
from app import broker, db, telemetry
def _once():
try:
with db.connection() as conn:
row = conn.execute(
"""
SELECT COUNT(*) AS depth,
EXTRACT(EPOCH FROM (NOW() - MIN(created_at))) AS age
FROM attachments
WHERE status IN ('queued', 'processing')
"""
).fetchone()
depth = int(row["depth"] or 0)
age = float(row["age"] or 0) if depth else 0.0
used, waiting = db.stats()
telemetry.update_runtime(queue_depth=depth, queue_age=age, db_up=1, pool_used=used, pool_waiting=waiting)
except Exception:
telemetry.update_runtime(db_up=0)
try:
telemetry.update_runtime(worker_consumers=broker.consumer_count())
except Exception:
telemetry.log_event("broker.stats_unavailable", severity="WARNING")
telemetry.sample_process()
try:
with db.connection() as conn:
broker.publish_pending(conn)
except Exception:
telemetry.log_event("job.publish_deferred", severity="WARNING")
def start():
stop = threading.Event()
def loop():
while not stop.wait(2):
_once()
_once()
threading.Thread(target=loop, name="poller", daemon=True).start()
return stop
+108
View File
@@ -0,0 +1,108 @@
CREATE TABLE IF NOT EXISTS users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE,
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE IF NOT EXISTS sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users (id),
expires_at timestamptz NOT NULL,
revoked_at timestamptz
);
CREATE INDEX IF NOT EXISTS sessions_user_idx ON sessions (user_id);
CREATE TABLE IF NOT EXISTS chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id uuid NOT NULL REFERENCES chats (id),
user_id uuid NOT NULL REFERENCES users (id),
position integer NOT NULL,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members (user_id, chat_id);
CREATE TABLE IF NOT EXISTS messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id),
sender_id uuid NOT NULL REFERENCES users (id),
text text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE INDEX IF NOT EXISTS messages_order_idx ON messages (chat_id, created_at, id);
CREATE TABLE IF NOT EXISTS attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id),
owner_id uuid NOT NULL REFERENCES users (id),
filename text NOT NULL,
content_type text NOT NULL,
size_bytes integer NOT NULL,
sha256 text NOT NULL,
status text NOT NULL CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
error_code text,
staging_key text NOT NULL,
original_key text,
pipeline_version integer NOT NULL DEFAULT 1,
attempt integer NOT NULL DEFAULT 0,
lease_until timestamptz,
worker_id text,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL,
ready_at timestamptz
);
CREATE INDEX IF NOT EXISTS attachments_open_idx ON attachments (created_at)
WHERE status IN ('queued', 'processing');
CREATE TABLE IF NOT EXISTS attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments (id),
name text NOT NULL,
content_type text NOT NULL,
size_bytes integer NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
sha256 text NOT NULL,
object_key text NOT NULL,
pipeline_version integer NOT NULL,
PRIMARY KEY (attachment_id, name, pipeline_version)
);
CREATE TABLE IF NOT EXISTS message_attachments (
message_id uuid NOT NULL REFERENCES messages (id),
attachment_id uuid NOT NULL REFERENCES attachments (id),
position integer NOT NULL,
PRIMARY KEY (message_id, attachment_id)
);
CREATE TABLE IF NOT EXISTS idempotency (
user_id uuid NOT NULL,
method text NOT NULL,
path text NOT NULL,
idem_key text NOT NULL,
body_hash bytea NOT NULL,
status_code integer NOT NULL,
response_json jsonb NOT NULL,
created_at timestamptz NOT NULL,
PRIMARY KEY (user_id, method, path, idem_key)
);
CREATE TABLE IF NOT EXISTS outbox (
job_id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments (id),
payload jsonb NOT NULL,
created_at timestamptz NOT NULL,
published_at timestamptz
);
CREATE INDEX IF NOT EXISTS outbox_unpublished_idx ON outbox (created_at)
WHERE published_at IS NULL;
+45
View File
@@ -0,0 +1,45 @@
import os
from pathlib import Path
def _read(name, file_name=None, default=None):
value = os.environ.get(name)
if value:
return value.strip()
file_env = os.environ.get(file_name) if file_name else None
if file_env:
return Path(file_env).read_text(encoding="utf-8").strip()
if default is not None:
return default
raise RuntimeError("missing configuration " + name)
class Settings:
def __init__(self):
self.service_name = os.environ.get("OTEL_SERVICE_NAME", "messenger-api")
self.service_version = os.environ.get("SERVICE_VERSION", "1.6.0")
self.instance_id = os.environ.get("INSTANCE_ID", "api-local")
self.environment = "lab"
self.database_url = _read("DATABASE_URL", "DATABASE_URL_FILE")
self.session_ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
self.idempotency_ttl = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400"))
self.s3_endpoint = os.environ.get("S3_ENDPOINT", "http://minio:9000")
self.s3_region = os.environ.get("S3_REGION", "us-east-1")
self.s3_bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
self.s3_access_key = _read("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
self.s3_secret_key = _read("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
self.broker_url = _read("BROKER_URL", "BROKER_URL_FILE")
self.worker_concurrency = int(os.environ.get("WORKER_CONCURRENCY", "2"))
self.job_max_attempts = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
self.job_timeout = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))
self.staging_retention = int(os.environ.get("STAGING_RETENTION_SECONDS", "86400"))
self.max_upload = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760"))
self.max_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
self.pool_max = int(os.environ.get("DB_POOL_MAX", "8"))
self.otlp_endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
self.fault_dir = os.environ.get("FAULT_DIR", "/faults")
self.rabbit_management_url = os.environ.get("RABBITMQ_MANAGEMENT_URL", "http://rabbitmq:15672")
self.role = os.environ.get("APP_ROLE", "api")
settings = Settings()
+92
View File
@@ -0,0 +1,92 @@
import threading
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from app import telemetry
from app.settings import settings
_client = None
_lock = threading.Lock()
def init():
global _client
_client = boto3.client(
"s3",
endpoint_url=settings.s3_endpoint,
aws_access_key_id=settings.s3_access_key,
aws_secret_access_key=settings.s3_secret_key,
region_name=settings.s3_region,
config=Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=2,
read_timeout=10,
retries={"max_attempts": 2, "mode": "standard"},
),
)
try:
_client.head_bucket(Bucket=settings.s3_bucket)
return
except ClientError as exc:
code = str(exc.response.get("Error", {}).get("Code", ""))
if code not in {"404", "NoSuchBucket", "NotFound"}:
raise
try:
_client.create_bucket(Bucket=settings.s3_bucket)
except ClientError as exc:
code = str(exc.response.get("Error", {}).get("Code", ""))
if code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
raise
_warm()
def _warm():
"""Read one existing object so the first user download is not a cold volume open."""
try:
listed = _client.list_objects_v2(Bucket=settings.s3_bucket, MaxKeys=1)
except (BotoCoreError, ClientError, OSError):
return
contents = listed.get("Contents") or []
if not contents:
return
try:
body = _client.get_object(Bucket=settings.s3_bucket, Key=contents[0]["Key"])["Body"]
body.read()
except (BotoCoreError, ClientError, OSError):
return
def ready():
try:
with _lock:
_client.head_bucket(Bucket=settings.s3_bucket)
return True
except (BotoCoreError, ClientError, OSError):
return False
def put(key, data, content_type, variant):
with telemetry.tracer().start_as_current_span(
"s3.put",
attributes={"s3.operation": "put", "variant": variant},
):
with _lock:
_client.put_object(Bucket=settings.s3_bucket, Key=key, Body=data, ContentType=content_type)
def get(key, variant):
with telemetry.tracer().start_as_current_span(
"s3.get",
attributes={"s3.operation": "get", "variant": variant},
):
with _lock:
response = _client.get_object(Bucket=settings.s3_bucket, Key=key)
return response["Body"].read()
def delete(key):
with _lock:
_client.delete_object(Bucket=settings.s3_bucket, Key=key)
+379
View File
@@ -0,0 +1,379 @@
import json
import logging
import threading
import time
from opentelemetry import context as otel_context
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.propagate import extract, inject
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk._logs.export import LogExportResult
from opentelemetry.sdk.metrics.export import MetricExportResult
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
from opentelemetry.trace import Status, StatusCode
from app.settings import settings
BANNED_FIELDS = {
"password",
"token",
"authorization",
"secret",
"filename",
"text",
"body",
"database_url",
"broker_url",
"image",
}
_dropped = None
_http_requests = None
_http_duration = None
_jobs = None
_job_duration = None
_ready_duration = None
_db_duration = None
_state = {
"queue_depth": 0,
"queue_age": 0.0,
"db_up": 0,
"pool_used": 0,
"pool_waiting": 0,
"worker_busy": 0,
"worker_capacity": 0,
"worker_consumers": 0,
"cpu": 0.0,
"memory": 0,
}
_cpu_last = None
_lock = threading.Lock()
class _GuardedExporter:
def __init__(self, inner, signal):
self.inner = inner
self.signal = signal
def __getattr__(self, name):
return getattr(self.inner, name)
def export(self, items, **kwargs):
try:
result = self.inner.export(items)
except Exception:
_count_drop(self.signal, len(items))
return self._failure()
if getattr(result, "name", None) != "SUCCESS":
_count_drop(self.signal, len(items))
return result
def _failure(self):
if self.signal == "metrics":
return MetricExportResult.FAILURE
if self.signal == "logs":
return LogExportResult.FAILURE
return SpanExportResult.FAILURE
def shutdown(self, **kwargs):
try:
return self.inner.shutdown(**kwargs)
except Exception:
return None
def force_flush(self, timeout_millis=3000):
try:
return self.inner.force_flush(timeout_millis)
except Exception:
return False
class _RawFormatter(logging.Formatter):
def format(self, record):
return record.getMessage()
def _count_drop(signal, count):
if _dropped is not None and count:
_dropped.add(count, {"signal": signal})
def _resource():
return Resource.create(
{
"service.name": settings.service_name,
"service.version": settings.service_version,
"service.instance.id": settings.instance_id,
"deployment.environment.name": settings.environment,
}
)
def _histogram(name, buckets):
return View(instrument_name=name, aggregation=ExplicitBucketHistogramAggregation(buckets))
def init():
global _dropped, _http_requests, _http_duration, _jobs, _job_duration, _ready_duration, _db_duration
resource = _resource()
endpoint = settings.otlp_endpoint
tracer_provider = TracerProvider(resource=resource)
tracer_provider.add_span_processor(
BatchSpanProcessor(
_GuardedExporter(OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=5), "spans"),
max_queue_size=256,
schedule_delay_millis=2000,
max_export_batch_size=64,
export_timeout_millis=5000,
)
)
trace.set_tracer_provider(tracer_provider)
reader = PeriodicExportingMetricReader(
_GuardedExporter(OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=5), "metrics"),
export_interval_millis=5000,
export_timeout_millis=4000,
)
meter_provider = MeterProvider(
resource=resource,
metric_readers=[reader],
views=[
_histogram("http.server.duration", [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]),
_histogram("messenger.job.duration", [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]),
_histogram("messenger.attachment.time_to_ready", [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60]),
_histogram("messenger.db.duration", [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5]),
],
)
from opentelemetry import metrics
metrics.set_meter_provider(meter_provider)
meter = metrics.get_meter("messenger", settings.service_version)
_dropped = meter.create_counter("messenger.telemetry.dropped", unit="1")
_http_requests = meter.create_counter("http.server.requests", unit="1")
_http_duration = meter.create_histogram("http.server.duration", unit="s")
_jobs = meter.create_counter("messenger.jobs", unit="1")
_job_duration = meter.create_histogram("messenger.job.duration", unit="s")
_ready_duration = meter.create_histogram("messenger.attachment.time_to_ready", unit="s")
_db_duration = meter.create_histogram("messenger.db.duration", unit="s")
meter.create_observable_gauge("messenger.queue.depth", callbacks=[_queue_depth])
meter.create_observable_gauge("messenger.queue.oldest_age", callbacks=[_queue_age], unit="s")
meter.create_observable_gauge("messenger.db.pool.used", callbacks=[_pool_used])
meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[_pool_waiting])
meter.create_observable_gauge("messenger.worker.busy", callbacks=[_worker_busy])
meter.create_observable_gauge("messenger.worker.capacity", callbacks=[_worker_capacity])
meter.create_observable_gauge("messenger.worker.consumers", callbacks=[_worker_consumers])
meter.create_observable_gauge("messenger.db.up", callbacks=[_db_up])
meter.create_observable_gauge("messenger.process.cpu.utilization", callbacks=[_cpu])
meter.create_observable_gauge("messenger.process.memory", callbacks=[_memory], unit="By")
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(
_GuardedExporter(OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=5), "logs"),
max_queue_size=256,
schedule_delay_millis=2000,
max_export_batch_size=64,
export_timeout_millis=5000,
)
)
root = logging.getLogger()
root.handlers.clear()
root.setLevel(logging.INFO)
stream = logging.StreamHandler()
stream.setFormatter(_RawFormatter())
root.addHandler(stream)
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
root.addHandler(otel_handler)
for name in ("pika", "botocore", "boto3", "urllib3", "uvicorn", "uvicorn.error", "uvicorn.access"):
logging.getLogger(name).setLevel(logging.WARNING)
quiet = logging.getLogger("opentelemetry")
quiet.handlers.clear()
quiet.propagate = False
quiet.setLevel(logging.CRITICAL)
log_event("service.started", role=settings.role)
def tracer():
return trace.get_tracer("messenger", settings.service_version)
def current_ids():
span = trace.get_current_span()
ctx = span.get_span_context()
if not ctx or not ctx.is_valid:
return None, None
return format(ctx.trace_id, "032x"), format(ctx.span_id, "016x")
def log_event(event, severity="INFO", **fields):
trace_id, span_id = current_ids()
payload = {
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + f".{int(time.time() * 1000000) % 1000000:06d}Z",
"severity": severity,
"service.name": settings.service_name,
"event": event,
"trace_id": trace_id,
"span_id": span_id,
}
for key, value in fields.items():
if key in BANNED_FIELDS or value is None:
continue
if isinstance(value, (str, int, float, bool)):
payload[key] = value
level = getattr(logging, severity, logging.INFO)
logging.getLogger("messenger").log(level, json.dumps(payload, ensure_ascii=False))
def inject_carrier(carrier=None):
target = {} if carrier is None else carrier
inject(target)
return target
def extract_context(carrier):
clean = {}
for key, value in (carrier or {}).items():
if isinstance(value, bytes):
value = value.decode("utf-8", "replace")
if isinstance(value, str):
clean[str(key).lower()] = value
return extract(clean)
def record_http(method, route, status, duration):
if _http_requests is None:
return
klass = f"{int(status) // 100}xx"
attributes = {
"http.request.method": method,
"http.route": route,
"http.response.status_class": klass,
}
_http_requests.add(1, attributes)
_http_duration.record(duration, attributes)
def record_job(outcome, duration):
attributes = {"job_type": "attachment.process.v1", "outcome": outcome}
if _jobs is not None:
_jobs.add(1, attributes)
_job_duration.record(max(duration, 0.0), attributes)
def record_ready(duration):
if _ready_duration is not None:
_ready_duration.record(max(duration, 0.0), {"job_type": "attachment.process.v1"})
def record_db(duration):
if _db_duration is not None:
_db_duration.record(max(duration, 0.0), {"db.system": "postgresql"})
def update_runtime(**values):
with _lock:
_state.update(values)
def note_worker(busy, capacity):
update_runtime(worker_busy=busy, worker_capacity=capacity)
def sample_process():
global _cpu_last
now_cpu = time.process_time()
now_wall = time.monotonic()
ratio = 0.0
if _cpu_last is not None:
dc = now_cpu - _cpu_last[0]
dw = now_wall - _cpu_last[1]
if dw > 0:
ratio = dc / dw
_cpu_last = (now_cpu, now_wall)
memory = 0
try:
with open("/proc/self/statm", encoding="ascii") as handle:
resident = int(handle.read().split()[1])
memory = resident * os_page()
except (OSError, IndexError, ValueError):
memory = 0
update_runtime(cpu=ratio, memory=memory)
_page = None
def os_page():
global _page
if _page is None:
import os
_page = os.sysconf("SC_PAGE_SIZE")
return _page
def _obs(value, attributes=None):
from opentelemetry.metrics import Observation
return [Observation(value, attributes or {})]
def _queue_depth(_options):
return _obs(_state["queue_depth"], {"queue": "attachment.process.v1"})
def _queue_age(_options):
return _obs(_state["queue_age"], {"queue": "attachment.process.v1"})
def _pool_used(_options):
return _obs(_state["pool_used"], {"pool": "postgres"})
def _pool_waiting(_options):
return _obs(_state["pool_waiting"], {"pool": "postgres"})
def _worker_busy(_options):
if settings.role != "worker":
return []
return _obs(_state["worker_busy"], {"queue": "attachment.process.v1"})
def _worker_capacity(_options):
if settings.role != "worker":
return []
return _obs(_state["worker_capacity"], {"queue": "attachment.process.v1"})
def _worker_consumers(_options):
return _obs(_state["worker_consumers"], {"queue": "attachment.process.v1"})
def _db_up(_options):
return _obs(_state["db_up"])
def _cpu(_options):
return _obs(_state["cpu"])
def _memory(_options):
return _obs(_state["memory"])
def mark_error(span):
span.set_status(Status(StatusCode.ERROR))
def attach(ctx):
return otel_context.attach(ctx)
+347
View File
@@ -0,0 +1,347 @@
import datetime as dt
import hashlib
import json
import signal
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pika
from botocore.exceptions import BotoCoreError, ClientError
from opentelemetry.trace import SpanKind
from app import broker, db, storage, telemetry
from app.broker import QUEUE, _params
from app.codec import utcnow
from app.errors import AppError
from app.faults import consume_process_failure
from app.images import thumbnail_jpeg
from app.poller import start
from app.settings import settings
_inflight = 0
_inflight_lock = threading.Lock()
_stop = threading.Event()
class Transient(Exception):
pass
class Permanent(Exception):
pass
@contextmanager
def _busy():
global _inflight
with _inflight_lock:
_inflight += 1
current = _inflight
telemetry.note_worker(current, settings.worker_concurrency)
try:
yield
finally:
with _inflight_lock:
_inflight -= 1
current = _inflight
telemetry.note_worker(current, settings.worker_concurrency)
def _mark_failed(attachment_id, attempt, code):
with db.connection() as conn:
conn.execute(
"""
UPDATE attachments
SET status = 'failed', error_code = %s, lease_until = NULL, worker_id = NULL, updated_at = %s
WHERE id = %s AND status = 'processing' AND attempt = %s
""",
(code, utcnow(), attachment_id, attempt),
)
telemetry.log_event("job.failed", attachment_id=str(attachment_id), attempt=attempt, error_code=code)
def _release(attachment_id, attempt):
with db.connection() as conn:
conn.execute(
"""
UPDATE attachments
SET status = 'queued', lease_until = NULL, worker_id = NULL, updated_at = %s
WHERE id = %s AND status = 'processing' AND attempt = %s
""",
(utcnow(), attachment_id, attempt),
)
def process_delivery(body, headers):
started = time.perf_counter()
try:
payload = json.loads(body.decode("utf-8"))
attachment_id = uuid.UUID(str(payload["attachment_id"]))
pipeline = int(payload["pipeline_version"])
except (ValueError, UnicodeError, KeyError, TypeError):
telemetry.log_event("job.poison", severity="ERROR")
return "ack"
carrier = {}
for key, value in (headers or {}).items():
if isinstance(value, bytes):
value = value.decode("utf-8", "replace")
if isinstance(value, str):
carrier[str(key).lower()] = value
if payload.get("traceparent"):
carrier.setdefault("traceparent", payload["traceparent"])
if payload.get("tracestate"):
carrier.setdefault("tracestate", payload["tracestate"])
parent = telemetry.extract_context(carrier)
outcome = None
with telemetry.tracer().start_as_current_span(
"attachment.process consume",
context=parent,
kind=SpanKind.CONSUMER,
attributes={
"messaging.system": "rabbitmq",
"messaging.destination.name": QUEUE,
"messaging.operation.type": "consume",
},
) as span:
try:
action, outcome, attempt = _consume(payload, attachment_id, pipeline, span)
except Exception:
telemetry.mark_error(span)
telemetry.log_event("job.failed", severity="ERROR", error_type="unexpected", attachment_id=str(attachment_id))
action, outcome = "requeue", "retry"
attempt = None
if outcome:
telemetry.record_job(outcome, time.perf_counter() - started)
if attempt is not None:
span.set_attribute("attempt", attempt)
if outcome in {"failed", "retry"}:
telemetry.mark_error(span)
return action
def _consume(payload, attachment_id, pipeline, span):
now = utcnow()
with db.connection() as conn:
row = conn.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone()
if not row or int(row["pipeline_version"]) != pipeline:
return "ack", None, None
if row["status"] == "ready":
return "ack", None, int(row["attempt"])
if row["status"] == "failed":
return "ack", None, int(row["attempt"])
if row["status"] == "processing" and row["lease_until"] and row["lease_until"] > now:
held_attempt = int(row["attempt"])
staging_key = None
else:
held_attempt = None
attempt = int(row["attempt"]) + 1
if attempt > settings.job_max_attempts:
conn.execute(
"""
UPDATE attachments
SET status = 'failed', error_code = 'retry_exhausted', lease_until = NULL, updated_at = %s
WHERE id = %s
""",
(now, attachment_id),
)
return "ack", "failed", attempt
lease = now + dt.timedelta(seconds=settings.job_timeout)
conn.execute(
"""
UPDATE attachments
SET status = 'processing', attempt = %s, lease_until = %s, worker_id = %s, updated_at = %s
WHERE id = %s
""",
(attempt, lease, settings.instance_id, now, attachment_id),
)
staging_key = row["staging_key"]
content_type = row["content_type"]
expected_sha = row["sha256"]
created_at = row["created_at"]
if staging_key is None:
span.add_event("lease_held")
time.sleep(1)
return "requeue", None, held_attempt
span.set_attribute("attempt", attempt)
telemetry.log_event("job.consumed", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id")))
try:
if consume_process_failure():
span.add_event("retry", {"reason": "injected"})
raise Transient()
data = storage.get(staging_key, "staging")
if hashlib.sha256(data).hexdigest() != expected_sha:
raise Permanent("invalid_image")
thumb, width, height = thumbnail_jpeg(data)
original_key = f"originals/{attachment_id}"
thumb_key = f"variants/{attachment_id}/thumbnail"
storage.put(original_key, data, content_type, "original")
storage.put(thumb_key, thumb, "image/jpeg", "thumbnail")
except Permanent as exc:
_mark_failed(attachment_id, attempt, str(exc) or "processing_failed")
return "ack", "failed", attempt
except AppError as exc:
code = "invalid_image" if exc.status in {413, 415} else "processing_failed"
_mark_failed(attachment_id, attempt, code)
return "ack", "failed", attempt
except Transient:
_release(attachment_id, attempt)
telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="transient")
time.sleep(0.5)
return "requeue", "retry", attempt
except (BotoCoreError, ClientError, OSError, TimeoutError):
_release(attachment_id, attempt)
telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="storage")
time.sleep(min(2 ** max(attempt - 1, 0), 8))
return "requeue", "retry", attempt
thumb_sha = hashlib.sha256(thumb).hexdigest()
ready_at = utcnow()
with db.connection() as conn:
current = conn.execute("SELECT status, attempt FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone()
if not current or current["status"] != "processing" or int(current["attempt"]) != attempt:
return "ack", None, attempt
conn.execute(
"""
INSERT INTO attachment_variants (
attachment_id, name, content_type, size_bytes, width, height, sha256, object_key, pipeline_version
) VALUES (%s, 'thumbnail', 'image/jpeg', %s, %s, %s, %s, %s, %s)
ON CONFLICT (attachment_id, name, pipeline_version) DO UPDATE
SET content_type = EXCLUDED.content_type,
size_bytes = EXCLUDED.size_bytes,
width = EXCLUDED.width,
height = EXCLUDED.height,
sha256 = EXCLUDED.sha256,
object_key = EXCLUDED.object_key
""",
(attachment_id, len(thumb), width, height, thumb_sha, thumb_key, pipeline),
)
conn.execute(
"""
UPDATE attachments
SET status = 'ready', error_code = NULL, original_key = %s, ready_at = %s,
lease_until = NULL, updated_at = %s
WHERE id = %s AND status = 'processing' AND attempt = %s
""",
(original_key, ready_at, ready_at, attachment_id, attempt),
)
try:
storage.delete(staging_key)
except (BotoCoreError, ClientError, OSError):
telemetry.log_event("staging.cleanup_deferred", severity="WARNING", attachment_id=str(attachment_id))
telemetry.record_ready((ready_at - created_at).total_seconds())
telemetry.log_event("job.ready", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id")))
return "ack", "success", attempt
def _serve(connection):
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True)
channel.basic_qos(prefetch_count=settings.worker_concurrency)
executor = ThreadPoolExecutor(max_workers=settings.worker_concurrency, thread_name_prefix="job")
def on_message(ch, method, properties, body):
def work():
with _busy():
try:
action = process_delivery(body, properties.headers if properties else None)
except Exception:
action = "requeue"
def finish():
try:
if action == "ack":
ch.basic_ack(method.delivery_tag)
else:
ch.basic_nack(method.delivery_tag, requeue=True)
except Exception:
telemetry.log_event("job.ack_failed", severity="ERROR")
connection.add_callback_threadsafe(finish)
executor.submit(work)
channel.basic_consume(queue=QUEUE, on_message_callback=on_message, auto_ack=False)
def watch():
_stop.wait()
connection.add_callback_threadsafe(channel.stop_consuming)
threading.Thread(target=watch, name="stop-watch", daemon=True).start()
channel.start_consuming()
executor.shutdown(wait=True, cancel_futures=False)
class _Health(BaseHTTPRequestHandler):
def do_GET(self):
if self.path.startswith("/health/live"):
self._send(200)
return
if self.path.startswith("/health/ready"):
try:
with db.connection() as conn:
conn.execute("SELECT 1")
ok = storage.ready() and broker.ready()
except Exception:
ok = False
self._send(200 if ok else 503)
return
self._send(404)
def _send(self, status):
body = b'{"status":"ok"}' if status == 200 else b'{"status":"unavailable"}'
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, fmt, *args):
return
def _health_server():
server = ThreadingHTTPServer(("0.0.0.0", 8081), _Health)
server.serve_forever()
def _request_stop(signum, _frame):
_stop.set()
def main():
signal.signal(signal.SIGTERM, _request_stop)
signal.signal(signal.SIGINT, _request_stop)
telemetry.init()
from app.main import _retry
_retry("postgres", db.init)
db.migrate()
_retry("s3", storage.init)
_retry("broker", broker.init)
telemetry.note_worker(0, settings.worker_concurrency)
start()
threading.Thread(target=_health_server, name="health", daemon=True).start()
while not _stop.is_set():
connection = None
try:
connection = pika.BlockingConnection(_params())
telemetry.update_runtime(worker_consumers=settings.worker_concurrency)
_serve(connection)
except Exception:
if _stop.is_set():
break
telemetry.log_event("broker.reconnect", severity="WARNING", error_type="connection")
time.sleep(1)
finally:
if connection is not None and connection.is_open:
try:
connection.close()
except Exception:
pass
if __name__ == "__main__":
main()