forked from mtusi2026/mtusi2026lab6
Собрать наблюдаемый стенд, чтобы причину деградации API и worker было видно по метрикам, логам и трассам.
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Messenger application for lab 6."""
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
import asyncio
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from app import faults, http, telemetry
|
||||
from app.errors import AppError
|
||||
from app.settings import settings
|
||||
|
||||
|
||||
class _TooLarge(Exception):
|
||||
pass
|
||||
|
||||
|
||||
async def _read_body(receive, limit):
|
||||
chunks = []
|
||||
size = 0
|
||||
while True:
|
||||
message = await receive()
|
||||
if message["type"] == "http.disconnect":
|
||||
return None
|
||||
if message["type"] != "http.request":
|
||||
continue
|
||||
data = message.get("body", b"") or b""
|
||||
size += len(data)
|
||||
if size > limit:
|
||||
raise _TooLarge()
|
||||
chunks.append(data)
|
||||
if not message.get("more_body"):
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def _too_large(scope, limit):
|
||||
for name, value in scope.get("headers", []):
|
||||
if name == b"content-length":
|
||||
try:
|
||||
return int(value) > limit
|
||||
except ValueError:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def app(scope, receive, send):
|
||||
if scope["type"] == "lifespan":
|
||||
while True:
|
||||
message = await receive()
|
||||
if message["type"] == "lifespan.startup":
|
||||
telemetry.log_event("service.listening", port=8080)
|
||||
await send({"type": "lifespan.startup.complete"})
|
||||
elif message["type"] == "lifespan.shutdown":
|
||||
await send({"type": "lifespan.shutdown.complete"})
|
||||
return
|
||||
return
|
||||
if scope["type"] != "http":
|
||||
return
|
||||
method = scope.get("method", "GET")
|
||||
path = scope.get("path", "/")
|
||||
raw_headers = {}
|
||||
for name, value in scope.get("headers", []):
|
||||
raw_headers[name.decode("latin-1").lower()] = value.decode("latin-1")
|
||||
request_id = http.request_id_from(raw_headers)
|
||||
template, _name, _groups = http.match(method, path)
|
||||
limit = http.body_limit(path)
|
||||
started = time.perf_counter()
|
||||
status = 500
|
||||
response_headers = {}
|
||||
body = b""
|
||||
with telemetry.tracer().start_as_current_span(
|
||||
f"{method} {template}",
|
||||
kind=SpanKind.SERVER,
|
||||
attributes={"http.request.method": method, "http.route": template},
|
||||
) as span:
|
||||
try:
|
||||
if _too_large(scope, limit):
|
||||
raise AppError(413, "payload_too_large", "Слишком большой объект")
|
||||
try:
|
||||
payload = await _read_body(receive, limit)
|
||||
except _TooLarge as exc:
|
||||
raise AppError(413, "payload_too_large", "Слишком большой объект") from exc
|
||||
if payload is None:
|
||||
return
|
||||
if method in {"POST", "PUT"} and path.rstrip("/").endswith("/attachments") and len(payload) > settings.max_upload:
|
||||
raise AppError(413, "payload_too_large", "Слишком большой объект")
|
||||
query = http.query_from(scope.get("query_string", b"").decode("ascii", "replace"))
|
||||
if path.startswith("/api/") and random.random() < faults.http_error_ratio():
|
||||
raise AppError(500, "internal_error", "Внутренняя ошибка")
|
||||
|
||||
def run():
|
||||
return http.handle(method, path, query, raw_headers, payload)
|
||||
|
||||
template, status, response_headers, body = await asyncio.to_thread(run)
|
||||
span.set_attribute("http.route", template)
|
||||
except Exception as exc:
|
||||
status, response_headers, body = http.error_response(exc, request_id)
|
||||
if not isinstance(exc, AppError):
|
||||
telemetry.log_event("internal_error", severity="ERROR", error_type=type(exc).__name__, request_id=request_id)
|
||||
if status >= 500:
|
||||
telemetry.mark_error(span)
|
||||
span.set_attribute("error.type", type(exc).__name__)
|
||||
duration = time.perf_counter() - started
|
||||
span.set_attribute("http.response.status_code", status)
|
||||
telemetry.record_http(method, template, status, duration)
|
||||
telemetry.log_event(
|
||||
"http.request.completed",
|
||||
request_id=request_id,
|
||||
http_route=template,
|
||||
http_method=method,
|
||||
status=status,
|
||||
duration_ms=int(duration * 1000),
|
||||
)
|
||||
header_list = [
|
||||
(b"x-request-id", request_id.encode("ascii")),
|
||||
(b"x-instance-id", settings.instance_id.encode("ascii")),
|
||||
]
|
||||
for key, value in response_headers.items():
|
||||
header_list.append((key.lower().encode("latin-1"), value.encode("latin-1")))
|
||||
await send({"type": "http.response.start", "status": status, "headers": header_list})
|
||||
await send({"type": "http.response.body", "body": body})
|
||||
|
||||
|
||||
def new_request_id():
|
||||
return uuid.uuid4().hex
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
import base64
|
||||
import json
|
||||
import uuid
|
||||
from urllib.parse import quote, urlparse, unquote
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
import pika
|
||||
|
||||
from app import telemetry
|
||||
from app.db import jsonb
|
||||
from app.settings import settings
|
||||
|
||||
QUEUE = "attachment.process.v1"
|
||||
|
||||
|
||||
def _params():
|
||||
parsed = urlparse(settings.broker_url)
|
||||
return pika.ConnectionParameters(
|
||||
host=parsed.hostname or "rabbitmq",
|
||||
port=parsed.port or 5672,
|
||||
virtual_host=unquote(parsed.path[1:] or "/"),
|
||||
credentials=pika.PlainCredentials(unquote(parsed.username or ""), unquote(parsed.password or "")),
|
||||
heartbeat=60,
|
||||
blocked_connection_timeout=2,
|
||||
socket_timeout=2,
|
||||
connection_attempts=1,
|
||||
retry_delay=0,
|
||||
)
|
||||
|
||||
|
||||
def _credentials():
|
||||
parsed = urlparse(settings.broker_url)
|
||||
return unquote(parsed.username or ""), unquote(parsed.password or ""), unquote(parsed.path[1:] or "/")
|
||||
|
||||
|
||||
def init():
|
||||
connection = pika.BlockingConnection(_params())
|
||||
try:
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=QUEUE, durable=True)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def ready():
|
||||
try:
|
||||
connection = pika.BlockingConnection(_params())
|
||||
try:
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=QUEUE, durable=True, passive=True)
|
||||
finally:
|
||||
connection.close()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def insert_outbox(conn, payload, created_at):
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO outbox (job_id, attachment_id, payload, created_at)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
""",
|
||||
(uuid.UUID(str(payload["job_id"])), uuid.UUID(str(payload["attachment_id"])), jsonb(payload), created_at),
|
||||
)
|
||||
|
||||
|
||||
def publish_pending(conn, job_id=None):
|
||||
query = """
|
||||
SELECT job_id, payload
|
||||
FROM outbox
|
||||
WHERE published_at IS NULL
|
||||
"""
|
||||
args = []
|
||||
if job_id is not None:
|
||||
query += " AND job_id = %s"
|
||||
args.append(job_id)
|
||||
query += " ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 20"
|
||||
rows = conn.execute(query, args).fetchall()
|
||||
for row in rows:
|
||||
payload = row["payload"]
|
||||
if isinstance(payload, str):
|
||||
payload = json.loads(payload)
|
||||
_publish(payload)
|
||||
conn.execute("UPDATE outbox SET published_at = NOW() WHERE job_id = %s", (row["job_id"],))
|
||||
return len(rows)
|
||||
|
||||
|
||||
def _publish(payload):
|
||||
carrier = {}
|
||||
if payload.get("traceparent"):
|
||||
carrier["traceparent"] = payload["traceparent"]
|
||||
if payload.get("tracestate"):
|
||||
carrier["tracestate"] = payload["tracestate"]
|
||||
parent = telemetry.extract_context(carrier)
|
||||
with telemetry.tracer().start_as_current_span(
|
||||
"attachment.process publish",
|
||||
context=parent,
|
||||
kind=telemetry.trace.SpanKind.PRODUCER,
|
||||
attributes={
|
||||
"messaging.system": "rabbitmq",
|
||||
"messaging.destination.name": QUEUE,
|
||||
"messaging.operation.type": "publish",
|
||||
},
|
||||
):
|
||||
outgoing = telemetry.inject_carrier()
|
||||
body = dict(payload)
|
||||
if outgoing.get("traceparent"):
|
||||
body["traceparent"] = outgoing["traceparent"]
|
||||
if outgoing.get("tracestate"):
|
||||
body["tracestate"] = outgoing["tracestate"]
|
||||
_send(body, outgoing)
|
||||
telemetry.log_event(
|
||||
"job.published",
|
||||
job_id=str(payload.get("job_id")),
|
||||
attempt=int(payload.get("attempt") or 1),
|
||||
)
|
||||
|
||||
|
||||
def _send(payload, headers):
|
||||
connection = pika.BlockingConnection(_params())
|
||||
try:
|
||||
channel = connection.channel()
|
||||
channel.confirm_delivery()
|
||||
channel.basic_publish(
|
||||
exchange="",
|
||||
routing_key=QUEUE,
|
||||
body=json.dumps(payload, separators=(",", ":")).encode("utf-8"),
|
||||
properties=pika.BasicProperties(
|
||||
delivery_mode=2,
|
||||
content_type="application/json",
|
||||
headers=headers,
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def consumer_count():
|
||||
user, password, vhost = _credentials()
|
||||
url = (
|
||||
settings.rabbit_management_url.rstrip("/")
|
||||
+ "/api/queues/"
|
||||
+ quote(vhost, safe="")
|
||||
+ "/"
|
||||
+ quote(QUEUE, safe="")
|
||||
)
|
||||
token = base64.b64encode(f"{user}:{password}".encode("utf-8")).decode("ascii")
|
||||
request = Request(url, headers={"Authorization": "Basic " + token})
|
||||
with urlopen(request, timeout=2) as response:
|
||||
data = json.loads(response.read().decode("utf-8"))
|
||||
return int(data.get("consumers") or 0)
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
import base64
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from app.errors import AppError, bad_request
|
||||
|
||||
USERNAME = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
|
||||
|
||||
def utcnow():
|
||||
return dt.datetime.now(dt.timezone.utc)
|
||||
|
||||
|
||||
def iso(value):
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=dt.timezone.utc)
|
||||
value = value.astimezone(dt.timezone.utc)
|
||||
return value.strftime("%Y-%m-%dT%H:%M:%S.") + f"{value.microsecond:06d}Z"
|
||||
|
||||
|
||||
def parse_iso(value):
|
||||
if not isinstance(value, str) or not value.endswith("Z"):
|
||||
raise bad_request()
|
||||
try:
|
||||
return dt.datetime.fromisoformat(value[:-1] + "+00:00")
|
||||
except ValueError as exc:
|
||||
raise bad_request() from exc
|
||||
|
||||
|
||||
def parse_uuid(value):
|
||||
if not isinstance(value, str):
|
||||
raise bad_request()
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError as exc:
|
||||
raise bad_request() from exc
|
||||
|
||||
|
||||
def require_text(value, minimum, maximum, allow_blank=False):
|
||||
if not isinstance(value, str):
|
||||
raise bad_request()
|
||||
if "\x00" in value or any(ord(ch) < 32 or ord(ch) == 127 for ch in value):
|
||||
raise bad_request()
|
||||
if not allow_blank and not value.strip():
|
||||
raise bad_request()
|
||||
if not (minimum <= len(value) <= maximum):
|
||||
raise bad_request()
|
||||
return value
|
||||
|
||||
|
||||
def canonical_message(text, attachment_ids):
|
||||
body = {"attachment_ids": list(attachment_ids), "text": text}
|
||||
raw = json.dumps(body, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
return hashlib.sha256(raw.encode("utf-8")).digest()
|
||||
|
||||
|
||||
def encode_cursor(kind, resource_id, created_at, row_id):
|
||||
raw = f"{kind}|{resource_id}|{iso(created_at)}|{row_id}"
|
||||
return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(token, kind, resource_id):
|
||||
if not isinstance(token, str) or not token:
|
||||
raise bad_request()
|
||||
padded = token + ("=" * (-len(token) % 4))
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8")
|
||||
cursor_kind, cursor_resource, stamp, row_id = raw.split("|", 3)
|
||||
created_at = parse_iso(stamp)
|
||||
row_uuid = str(uuid.UUID(row_id))
|
||||
except (ValueError, UnicodeError, AppError) as exc:
|
||||
raise bad_request() from exc
|
||||
if cursor_kind != kind or cursor_resource != resource_id:
|
||||
raise bad_request()
|
||||
return created_at, row_uuid
|
||||
|
||||
|
||||
def fitted_size(width, height, box=256):
|
||||
if width < 1 or height < 1:
|
||||
raise ValueError("dimensions")
|
||||
if width <= box and height <= box:
|
||||
return width, height
|
||||
if width >= height:
|
||||
return box, max(1, (height * box) // width)
|
||||
return max(1, (width * box) // height), box
|
||||
|
||||
|
||||
def detect_image(data, declared):
|
||||
if data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
detected = "image/png"
|
||||
elif data.startswith(b"\xff\xd8\xff"):
|
||||
detected = "image/jpeg"
|
||||
else:
|
||||
return None
|
||||
if declared != detected:
|
||||
return None
|
||||
return detected
|
||||
|
||||
|
||||
def media_type(header):
|
||||
if not header or not isinstance(header, str):
|
||||
return ""
|
||||
return header.split(";", 1)[0].strip().lower()
|
||||
|
||||
|
||||
def loads_object(raw):
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeError as exc:
|
||||
raise bad_request() from exc
|
||||
|
||||
def pairs(items):
|
||||
obj = {}
|
||||
for key, value in items:
|
||||
if key in obj:
|
||||
raise ValueError("duplicate")
|
||||
obj[key] = value
|
||||
return obj
|
||||
|
||||
try:
|
||||
value = json.loads(text, object_pairs_hook=pairs)
|
||||
except (ValueError, UnicodeError) as exc:
|
||||
raise bad_request() from exc
|
||||
if not isinstance(value, dict):
|
||||
raise bad_request()
|
||||
return value
|
||||
|
||||
|
||||
def expect_fields(data, allowed, required):
|
||||
if set(data) - set(allowed):
|
||||
raise bad_request()
|
||||
for key in required:
|
||||
if key not in data:
|
||||
raise bad_request()
|
||||
@@ -0,0 +1,98 @@
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
|
||||
from app import telemetry
|
||||
from app.errors import AppError, unavailable
|
||||
from app.faults import db_fail, db_slow_seconds
|
||||
from app.settings import settings
|
||||
|
||||
pool = None
|
||||
|
||||
|
||||
def init():
|
||||
global pool
|
||||
pool = ConnectionPool(
|
||||
settings.database_url,
|
||||
min_size=1,
|
||||
max_size=settings.pool_max,
|
||||
timeout=2,
|
||||
max_waiting=16,
|
||||
check=ConnectionPool.check_connection,
|
||||
kwargs={
|
||||
"row_factory": dict_row,
|
||||
"autocommit": False,
|
||||
"connect_timeout": 3,
|
||||
"options": "-c timezone=UTC -c statement_timeout=4000",
|
||||
},
|
||||
open=True,
|
||||
reconnect_timeout=2,
|
||||
)
|
||||
|
||||
|
||||
def migrate():
|
||||
script = Path(__file__).with_name("schema.sql").read_text(encoding="utf-8")
|
||||
statements = []
|
||||
buffer = []
|
||||
for line in script.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("--"):
|
||||
continue
|
||||
buffer.append(line)
|
||||
if stripped.endswith(";"):
|
||||
statements.append("\n".join(buffer))
|
||||
buffer = []
|
||||
with pool.connection() as conn:
|
||||
conn.execute("SELECT pg_advisory_lock(861006)")
|
||||
for statement in statements:
|
||||
conn.execute(statement)
|
||||
conn.execute("SELECT pg_advisory_unlock(861006)")
|
||||
conn.commit()
|
||||
|
||||
|
||||
def jsonb(value):
|
||||
return Jsonb(value)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connection():
|
||||
if db_fail():
|
||||
raise unavailable()
|
||||
delay = db_slow_seconds()
|
||||
started = time.perf_counter()
|
||||
span = telemetry.tracer().start_as_current_span(
|
||||
"db.transaction",
|
||||
attributes={"db.system": "postgresql"},
|
||||
)
|
||||
span.__enter__()
|
||||
try:
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
with pool.connection() as conn:
|
||||
try:
|
||||
yield conn
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
except AppError:
|
||||
raise
|
||||
except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError) as exc:
|
||||
raise unavailable() from exc
|
||||
finally:
|
||||
span.__exit__(None, None, None)
|
||||
telemetry.record_db(time.perf_counter() - started)
|
||||
|
||||
|
||||
def stats():
|
||||
if pool is None:
|
||||
return 0, 0
|
||||
data = pool.get_stats()
|
||||
used = int(data.get("pool_size", 0)) - int(data.get("pool_available", 0))
|
||||
waiting = int(data.get("requests_waiting", 0))
|
||||
return max(used, 0), waiting
|
||||
+584
@@ -0,0 +1,584 @@
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import uuid
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
|
||||
from argon2.low_level import Type
|
||||
from psycopg.errors import UniqueViolation
|
||||
|
||||
from app import broker, storage, telemetry
|
||||
from app.codec import (
|
||||
USERNAME,
|
||||
IDEMPOTENCY,
|
||||
canonical_message,
|
||||
decode_cursor,
|
||||
detect_image,
|
||||
encode_cursor,
|
||||
expect_fields,
|
||||
iso,
|
||||
parse_uuid,
|
||||
require_text,
|
||||
utcnow,
|
||||
)
|
||||
from app.db import connection, jsonb
|
||||
from app.errors import bad_request, conflict, not_found, too_large, unauthorized, unsupported
|
||||
from app.images import validate_image
|
||||
from app.settings import settings
|
||||
|
||||
BASIC = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER = 'Bearer realm="messenger"'
|
||||
_hasher = PasswordHasher(time_cost=1, memory_cost=8192, parallelism=1, hash_len=32, salt_len=16, type=Type.ID)
|
||||
_dummy = None
|
||||
|
||||
|
||||
def init_hasher():
|
||||
global _dummy
|
||||
_dummy = _hasher.hash("not-a-real-password")
|
||||
|
||||
|
||||
def _user_json(row):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": row["display_name"],
|
||||
"created_at": iso(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _token_hash(token):
|
||||
return hashlib.sha256(token.encode("utf-8")).digest()
|
||||
|
||||
|
||||
def _password(value):
|
||||
if not isinstance(value, str) or "\x00" in value or not (12 <= len(value) <= 128):
|
||||
raise bad_request()
|
||||
return value
|
||||
|
||||
|
||||
def as_uuid(value):
|
||||
if isinstance(value, uuid.UUID):
|
||||
return value
|
||||
return uuid.UUID(str(value))
|
||||
|
||||
|
||||
def _username(value):
|
||||
if not isinstance(value, str) or USERNAME.fullmatch(value) is None:
|
||||
raise bad_request()
|
||||
return value
|
||||
|
||||
|
||||
def register(data):
|
||||
expect_fields(data, ["username", "display_name", "password"], ["username", "display_name", "password"])
|
||||
username = _username(data["username"])
|
||||
display_name = require_text(data["display_name"], 1, 100)
|
||||
password = _password(data["password"])
|
||||
password_hash = _hasher.hash(password)
|
||||
user_id = uuid.uuid4()
|
||||
created_at = utcnow()
|
||||
try:
|
||||
with connection() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
""",
|
||||
(user_id, username, display_name, password_hash, created_at),
|
||||
)
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
|
||||
(user_id,),
|
||||
).fetchone()
|
||||
except UniqueViolation as exc:
|
||||
raise conflict("Имя пользователя занято") from exc
|
||||
return 201, _user_json(row)
|
||||
|
||||
|
||||
def login(username, password):
|
||||
if not isinstance(username, str) or not isinstance(password, str) or "\x00" in username or "\x00" in password:
|
||||
raise unauthorized(BASIC)
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = %s",
|
||||
(username,),
|
||||
).fetchone()
|
||||
stored = row["password_hash"] if row else _dummy
|
||||
try:
|
||||
_hasher.verify(stored, password)
|
||||
valid = row is not None
|
||||
except (VerifyMismatchError, VerificationError, InvalidHashError):
|
||||
valid = False
|
||||
if not valid:
|
||||
raise unauthorized(BASIC)
|
||||
token = secrets_token()
|
||||
expires_at = utcnow() + dt.timedelta(seconds=settings.session_ttl)
|
||||
with connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
|
||||
(_token_hash(token), row["id"], expires_at),
|
||||
)
|
||||
return 201, {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": iso(expires_at),
|
||||
"user": _user_json(row),
|
||||
}
|
||||
|
||||
|
||||
def secrets_token():
|
||||
import secrets
|
||||
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def current_user(token):
|
||||
if not token or len(token) < 32:
|
||||
raise unauthorized(BEARER)
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT u.id, u.username, u.display_name, u.created_at
|
||||
FROM sessions s
|
||||
JOIN users u ON u.id = s.user_id
|
||||
WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > NOW()
|
||||
""",
|
||||
(_token_hash(token),),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise unauthorized(BEARER)
|
||||
return row
|
||||
|
||||
|
||||
def ready_check():
|
||||
with connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
if not storage.ready() or not broker.ready():
|
||||
raise unavailable()
|
||||
|
||||
|
||||
def revoke(token):
|
||||
user = current_user(token)
|
||||
with connection() as conn:
|
||||
conn.execute(
|
||||
"UPDATE sessions SET revoked_at = NOW() WHERE token_hash = %s AND revoked_at IS NULL",
|
||||
(_token_hash(token),),
|
||||
)
|
||||
return user
|
||||
|
||||
|
||||
def create_chat(user, data):
|
||||
expect_fields(data, ["title", "member_ids"], ["title", "member_ids"])
|
||||
title = require_text(data["title"], 1, 100)
|
||||
raw_members = data["member_ids"]
|
||||
if not isinstance(raw_members, list) or not (1 <= len(raw_members) <= 100):
|
||||
raise bad_request()
|
||||
members = [parse_uuid(item) for item in raw_members]
|
||||
if len(set(members)) != len(members):
|
||||
raise bad_request()
|
||||
if str(user["id"]) not in members:
|
||||
raise bad_request()
|
||||
chat_id = uuid.uuid4()
|
||||
created_at = utcnow()
|
||||
with connection() as conn:
|
||||
found = conn.execute("SELECT id FROM users WHERE id = ANY(%s)", ([as_uuid(item) for item in members],)).fetchall()
|
||||
if len(found) != len(members):
|
||||
raise bad_request()
|
||||
conn.execute(
|
||||
"INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)",
|
||||
(chat_id, title, created_at),
|
||||
)
|
||||
for position, member_id in enumerate(members):
|
||||
conn.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)",
|
||||
(chat_id, as_uuid(member_id), position),
|
||||
)
|
||||
return 201, {
|
||||
"id": str(chat_id),
|
||||
"title": title,
|
||||
"member_ids": members,
|
||||
"created_at": iso(created_at),
|
||||
}
|
||||
|
||||
|
||||
def _chat_json(conn, row):
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY position",
|
||||
(row["id"],),
|
||||
).fetchall()
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": row["title"],
|
||||
"member_ids": [str(item["user_id"]) for item in members],
|
||||
"created_at": iso(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def list_chats(user, limit, cursor):
|
||||
created_at = None
|
||||
row_id = None
|
||||
if cursor is not None:
|
||||
created_at, row_id = decode_cursor(cursor, "chats", str(user["id"]))
|
||||
with connection() as conn:
|
||||
if created_at is None:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members m ON m.chat_id = c.id
|
||||
WHERE m.user_id = %s
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT %s
|
||||
""",
|
||||
(user["id"], limit + 1),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members m ON m.chat_id = c.id
|
||||
WHERE m.user_id = %s AND (c.created_at, c.id) > (%s, %s)
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT %s
|
||||
""",
|
||||
(user["id"], created_at, as_uuid(row_id), limit + 1),
|
||||
).fetchall()
|
||||
extra = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
items = [_chat_json(conn, row) for row in rows]
|
||||
next_cursor = None
|
||||
if extra and rows:
|
||||
last = rows[-1]
|
||||
next_cursor = encode_cursor("chats", str(user["id"]), last["created_at"], str(last["id"]))
|
||||
return {"items": items, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
def get_chat(user, chat_id):
|
||||
with connection() as conn:
|
||||
row = _visible_chat(conn, user["id"], chat_id)
|
||||
return _chat_json(conn, row)
|
||||
|
||||
|
||||
def _visible_chat(conn, user_id, chat_id):
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members m ON m.chat_id = c.id
|
||||
WHERE c.id = %s AND m.user_id = %s
|
||||
""",
|
||||
(as_uuid(chat_id), as_uuid(user_id)),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise not_found()
|
||||
return row
|
||||
|
||||
|
||||
def _message_json(row, attachment_ids):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": row["text"],
|
||||
"created_at": iso(row["created_at"]),
|
||||
"attachment_ids": attachment_ids,
|
||||
}
|
||||
|
||||
|
||||
def _attachment_ids(conn, message_id):
|
||||
rows = conn.execute(
|
||||
"SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position",
|
||||
(message_id,),
|
||||
).fetchall()
|
||||
return [str(row["attachment_id"]) for row in rows]
|
||||
|
||||
|
||||
def create_message(user, chat_id, key, data, path):
|
||||
chat_uuid = as_uuid(chat_id)
|
||||
if IDEMPOTENCY.fullmatch(key or "") is None:
|
||||
raise bad_request()
|
||||
expect_fields(data, ["text", "attachment_ids"], ["text"])
|
||||
if not isinstance(data["text"], str) or "\x00" in data["text"] or len(data["text"]) > 4000:
|
||||
raise bad_request()
|
||||
text = data["text"]
|
||||
if "attachment_ids" in data:
|
||||
raw_ids = data["attachment_ids"]
|
||||
if not isinstance(raw_ids, list) or len(raw_ids) > 10:
|
||||
raise bad_request()
|
||||
attachment_ids = [parse_uuid(item) for item in raw_ids]
|
||||
if len(set(attachment_ids)) != len(attachment_ids):
|
||||
raise bad_request()
|
||||
else:
|
||||
attachment_ids = []
|
||||
if not text.strip() and not attachment_ids:
|
||||
raise bad_request()
|
||||
body_hash = canonical_message(text, attachment_ids)
|
||||
lock = int.from_bytes(hashlib.sha256(f"{user['id']}|POST|{path}|{key}".encode("utf-8")).digest()[:8], "big", signed=True)
|
||||
with connection() as conn:
|
||||
_visible_chat(conn, user["id"], chat_id)
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
|
||||
existing = conn.execute(
|
||||
"""
|
||||
SELECT body_hash, response_json, created_at
|
||||
FROM idempotency
|
||||
WHERE user_id = %s AND method = 'POST' AND path = %s AND idem_key = %s
|
||||
""",
|
||||
(user["id"], path, key),
|
||||
).fetchone()
|
||||
if existing and existing["created_at"] >= utcnow() - dt.timedelta(seconds=settings.idempotency_ttl):
|
||||
if bytes(existing["body_hash"]) != body_hash:
|
||||
raise conflict("Ключ идемпотентности уже использован с другим телом")
|
||||
stored = existing["response_json"]
|
||||
if isinstance(stored, str):
|
||||
import json
|
||||
|
||||
stored = json.loads(stored)
|
||||
return 201, stored
|
||||
if existing:
|
||||
conn.execute(
|
||||
"DELETE FROM idempotency WHERE user_id = %s AND method = 'POST' AND path = %s AND idem_key = %s",
|
||||
(user["id"], path, key),
|
||||
)
|
||||
if attachment_ids:
|
||||
rows = conn.execute(
|
||||
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s)",
|
||||
([as_uuid(item) for item in attachment_ids],),
|
||||
).fetchall()
|
||||
found = {str(row["id"]): row for row in rows}
|
||||
if len(found) != len(attachment_ids):
|
||||
raise not_found()
|
||||
for attachment_id in attachment_ids:
|
||||
row = found[attachment_id]
|
||||
if str(row["chat_id"]) != chat_id:
|
||||
raise not_found()
|
||||
if row["status"] == "failed":
|
||||
raise conflict("Нельзя прикрепить неуспешное вложение")
|
||||
message_id = uuid.uuid4()
|
||||
created_at = utcnow()
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)",
|
||||
(message_id, chat_uuid, user["id"], text, created_at),
|
||||
)
|
||||
for position, attachment_id in enumerate(attachment_ids):
|
||||
conn.execute(
|
||||
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
|
||||
(message_id, as_uuid(attachment_id), position),
|
||||
)
|
||||
stored = conn.execute(
|
||||
"SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = %s",
|
||||
(message_id,),
|
||||
).fetchone()
|
||||
payload = _message_json(stored, attachment_ids)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO idempotency (user_id, method, path, idem_key, body_hash, status_code, response_json, created_at)
|
||||
VALUES (%s, 'POST', %s, %s, %s, 201, %s, %s)
|
||||
""",
|
||||
(user["id"], path, key, body_hash, jsonb(payload), created_at),
|
||||
)
|
||||
return 201, payload
|
||||
|
||||
|
||||
def list_messages(user, chat_id, limit, cursor):
|
||||
chat_uuid = as_uuid(chat_id)
|
||||
created_at = None
|
||||
row_id = None
|
||||
if cursor is not None:
|
||||
created_at, row_id = decode_cursor(cursor, "messages", chat_id)
|
||||
with connection() as conn:
|
||||
_visible_chat(conn, user["id"], chat_uuid)
|
||||
if created_at is None:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = %s
|
||||
ORDER BY created_at, id
|
||||
LIMIT %s
|
||||
""",
|
||||
(chat_uuid, limit + 1),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = %s AND (created_at, id) > (%s, %s)
|
||||
ORDER BY created_at, id
|
||||
LIMIT %s
|
||||
""",
|
||||
(chat_uuid, created_at, as_uuid(row_id), limit + 1),
|
||||
).fetchall()
|
||||
extra = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
items = [_message_json(row, _attachment_ids(conn, row["id"])) for row in rows]
|
||||
next_cursor = None
|
||||
if extra and rows:
|
||||
last = rows[-1]
|
||||
next_cursor = encode_cursor("messages", chat_id, last["created_at"], str(last["id"]))
|
||||
return {"items": items, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
def _variants(conn, attachment_id):
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT name, content_type, size_bytes, width, height, sha256
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = %s AND pipeline_version = 1
|
||||
ORDER BY name
|
||||
""",
|
||||
(attachment_id,),
|
||||
).fetchall()
|
||||
return [
|
||||
{
|
||||
"name": row["name"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"width": row["width"],
|
||||
"height": row["height"],
|
||||
"sha256": row["sha256"],
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def _attachment_json(row, variants):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]),
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": row["status"],
|
||||
"variants": variants,
|
||||
"error_code": row["error_code"],
|
||||
"created_at": iso(row["created_at"]),
|
||||
"updated_at": iso(row["updated_at"]),
|
||||
}
|
||||
|
||||
|
||||
def upload(user, chat_id, filename, content_type, data):
|
||||
import hashlib as hl
|
||||
|
||||
if not data:
|
||||
raise unsupported()
|
||||
if len(data) > settings.max_upload:
|
||||
raise too_large()
|
||||
detected = detect_image(data, content_type)
|
||||
if not detected:
|
||||
raise unsupported()
|
||||
validate_image(data)
|
||||
with connection() as conn:
|
||||
_visible_chat(conn, user["id"], chat_id)
|
||||
attachment_id = uuid.uuid4()
|
||||
created_at = utcnow()
|
||||
digest = hl.sha256(data).hexdigest()
|
||||
staging_key = f"staging/{attachment_id}"
|
||||
storage.put(staging_key, data, detected, "staging")
|
||||
carrier = telemetry.inject_carrier()
|
||||
job_id = uuid.uuid4()
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"job_id": str(job_id),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": str(attachment_id),
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": iso(created_at),
|
||||
}
|
||||
if carrier.get("traceparent"):
|
||||
payload["traceparent"] = carrier["traceparent"]
|
||||
if carrier.get("tracestate"):
|
||||
payload["tracestate"] = carrier["tracestate"]
|
||||
row = {
|
||||
"id": attachment_id,
|
||||
"chat_id": chat_id,
|
||||
"owner_id": user["id"],
|
||||
"filename": filename,
|
||||
"content_type": detected,
|
||||
"size_bytes": len(data),
|
||||
"sha256": digest,
|
||||
"status": "queued",
|
||||
"error_code": None,
|
||||
"created_at": created_at,
|
||||
"updated_at": created_at,
|
||||
}
|
||||
with connection() as conn:
|
||||
_visible_chat(conn, user["id"], chat_id)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO attachments (
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status,
|
||||
error_code, staging_key, pipeline_version, attempt, created_at, updated_at
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', NULL, %s, 1, 0, %s, %s)
|
||||
""",
|
||||
(
|
||||
attachment_id,
|
||||
as_uuid(chat_id),
|
||||
user["id"],
|
||||
filename,
|
||||
detected,
|
||||
len(data),
|
||||
digest,
|
||||
staging_key,
|
||||
created_at,
|
||||
created_at,
|
||||
),
|
||||
)
|
||||
broker.insert_outbox(conn, payload, created_at)
|
||||
try:
|
||||
with connection() as conn:
|
||||
broker.publish_pending(conn, job_id)
|
||||
except Exception:
|
||||
telemetry.log_event("job.publish_deferred", severity="WARNING", job_id=str(job_id))
|
||||
telemetry.log_event("attachment.accepted", attachment_id=str(attachment_id), size_bytes=len(data), content_type=detected)
|
||||
return 202, _attachment_json(row, [])
|
||||
|
||||
|
||||
def get_attachment(user, attachment_id):
|
||||
attachment_uuid = as_uuid(attachment_id)
|
||||
with connection() as conn:
|
||||
row = _visible_attachment(conn, user["id"], attachment_uuid)
|
||||
return _attachment_json(row, _variants(conn, attachment_uuid))
|
||||
|
||||
|
||||
def _visible_attachment(conn, user_id, attachment_id):
|
||||
row = conn.execute("SELECT * FROM attachments WHERE id = %s", (as_uuid(attachment_id),)).fetchone()
|
||||
if not row:
|
||||
raise not_found()
|
||||
member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(row["chat_id"], user_id),
|
||||
).fetchone()
|
||||
if not member:
|
||||
raise not_found()
|
||||
return row
|
||||
|
||||
|
||||
def download(user, attachment_id, variant):
|
||||
attachment_uuid = as_uuid(attachment_id)
|
||||
if variant not in {"original", "thumbnail", "crop", "optimized"}:
|
||||
raise bad_request()
|
||||
with connection() as conn:
|
||||
row = _visible_attachment(conn, user["id"], attachment_uuid)
|
||||
if row["status"] != "ready":
|
||||
raise conflict("Вложение ещё не готово")
|
||||
if variant == "original":
|
||||
key = row["original_key"]
|
||||
content_type = row["content_type"]
|
||||
else:
|
||||
found = conn.execute(
|
||||
"""
|
||||
SELECT object_key, content_type
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = %s AND name = %s AND pipeline_version = 1
|
||||
""",
|
||||
(attachment_uuid, variant),
|
||||
).fetchone()
|
||||
if not found:
|
||||
raise not_found()
|
||||
key = found["object_key"]
|
||||
content_type = found["content_type"]
|
||||
return storage.get(key, variant), content_type
|
||||
@@ -0,0 +1,40 @@
|
||||
class AppError(Exception):
|
||||
def __init__(self, status, code, message, headers=None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def bad_request(message="Неверный запрос"):
|
||||
return AppError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def unauthorized(challenge):
|
||||
return AppError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Требуется аутентификация",
|
||||
{"WWW-Authenticate": challenge},
|
||||
)
|
||||
|
||||
|
||||
def not_found():
|
||||
return AppError(404, "not_found", "Объект не найден")
|
||||
|
||||
|
||||
def conflict(message="Конфликт"):
|
||||
return AppError(409, "conflict", message)
|
||||
|
||||
|
||||
def too_large(message="Слишком большой объект"):
|
||||
return AppError(413, "payload_too_large", message)
|
||||
|
||||
|
||||
def unsupported(message="Неподдерживаемый тип изображения"):
|
||||
return AppError(415, "unsupported_media_type", message)
|
||||
|
||||
|
||||
def unavailable(message="Сервис временно недоступен"):
|
||||
return AppError(503, "unavailable", message)
|
||||
@@ -0,0 +1,73 @@
|
||||
from pathlib import Path
|
||||
|
||||
from app.settings import settings
|
||||
|
||||
|
||||
def _text(name):
|
||||
path = Path(settings.fault_dir) / name
|
||||
try:
|
||||
return path.read_text(encoding="utf-8").strip()
|
||||
except (FileNotFoundError, OSError):
|
||||
return ""
|
||||
|
||||
|
||||
def db_slow_seconds():
|
||||
raw = _text("db_slow_ms")
|
||||
if not raw:
|
||||
return 0.0
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
if value <= 0:
|
||||
return 0.0
|
||||
return min(value, 10000.0) / 1000.0
|
||||
|
||||
|
||||
def db_fail():
|
||||
return _text("db_fail") == "1"
|
||||
|
||||
|
||||
def http_error_ratio():
|
||||
raw = _text("http_error_ratio")
|
||||
if not raw:
|
||||
return 0.0
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
if value <= 0:
|
||||
return 0.0
|
||||
return min(value, 1.0)
|
||||
|
||||
|
||||
def handler_slow_seconds():
|
||||
raw = _text("handler_slow_ms")
|
||||
if not raw:
|
||||
return 0.0
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
if value <= 0:
|
||||
return 0.0
|
||||
return min(value, 10000.0) / 1000.0
|
||||
|
||||
|
||||
def consume_process_failure():
|
||||
path = Path(settings.fault_dir) / "process_fail_remaining"
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8").strip()
|
||||
except (FileNotFoundError, OSError):
|
||||
return False
|
||||
try:
|
||||
remaining = int(raw or "0")
|
||||
except ValueError:
|
||||
return False
|
||||
if remaining <= 0:
|
||||
return False
|
||||
try:
|
||||
path.write_text(str(remaining - 1), encoding="utf-8")
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
+221
@@ -0,0 +1,221 @@
|
||||
import json
|
||||
import re
|
||||
from urllib.parse import parse_qs
|
||||
|
||||
from app import domain, faults
|
||||
from app.codec import REQUEST_ID, loads_object, media_type, require_text
|
||||
from app.errors import AppError, bad_request, not_found, unauthorized
|
||||
from app.settings import settings
|
||||
|
||||
_ROUTES = [
|
||||
("GET", re.compile(r"^/health/live$"), "/health/live", "live"),
|
||||
("GET", re.compile(r"^/health/ready$"), "/health/ready", "ready"),
|
||||
("POST", re.compile(r"^/api/v1/users$"), "/api/v1/users", "register"),
|
||||
("GET", re.compile(r"^/api/v1/users/me$"), "/api/v1/users/me", "me"),
|
||||
("POST", re.compile(r"^/api/v1/auth/sessions$"), "/api/v1/auth/sessions", "login"),
|
||||
("DELETE", re.compile(r"^/api/v1/auth/sessions/current$"), "/api/v1/auth/sessions/current", "logout"),
|
||||
("POST", re.compile(r"^/api/v1/chats$"), "/api/v1/chats", "create_chat"),
|
||||
("GET", re.compile(r"^/api/v1/chats$"), "/api/v1/chats", "list_chats"),
|
||||
("GET", re.compile(r"^/api/v1/chats/([^/]+)$"), "/api/v1/chats/{chat_id}", "get_chat"),
|
||||
("POST", re.compile(r"^/api/v1/chats/([^/]+)/messages$"), "/api/v1/chats/{chat_id}/messages", "create_message"),
|
||||
("GET", re.compile(r"^/api/v1/chats/([^/]+)/messages$"), "/api/v1/chats/{chat_id}/messages", "list_messages"),
|
||||
("POST", re.compile(r"^/api/v1/chats/([^/]+)/attachments$"), "/api/v1/chats/{chat_id}/attachments", "upload"),
|
||||
("GET", re.compile(r"^/api/v1/attachments/([^/]+)$"), "/api/v1/attachments/{attachment_id}", "get_attachment"),
|
||||
("GET", re.compile(r"^/api/v1/attachments/([^/]+)/content$"), "/api/v1/attachments/{attachment_id}/content", "download"),
|
||||
]
|
||||
|
||||
|
||||
def match(method, path):
|
||||
for route_method, pattern, template, name in _ROUTES:
|
||||
found = pattern.fullmatch(path)
|
||||
if found and route_method == method:
|
||||
return template, name, found.groups()
|
||||
return "unmatched", "unmatched", ()
|
||||
|
||||
|
||||
def _single(query, name):
|
||||
values = query.get(name)
|
||||
if not values:
|
||||
return None
|
||||
if len(values) != 1 or values[0] == "":
|
||||
raise bad_request()
|
||||
return values[0]
|
||||
|
||||
|
||||
def _limit(query):
|
||||
raw = _single(query, "limit")
|
||||
if raw is None:
|
||||
return 50
|
||||
if re.fullmatch(r"[1-9][0-9]*", raw) is None:
|
||||
raise bad_request()
|
||||
value = int(raw)
|
||||
if value > 100:
|
||||
raise bad_request()
|
||||
return value
|
||||
|
||||
|
||||
def _json_body(headers, body):
|
||||
if media_type(headers.get("content-type", "")) != "application/json":
|
||||
raise bad_request()
|
||||
if not body:
|
||||
raise bad_request()
|
||||
return loads_object(body)
|
||||
|
||||
|
||||
def _bearer(headers):
|
||||
value = headers.get("authorization", "")
|
||||
if not value.lower().startswith("bearer "):
|
||||
raise unauthorized(domain.BEARER)
|
||||
token = value.split(" ", 1)[1].strip()
|
||||
return token
|
||||
|
||||
|
||||
def _basic(headers):
|
||||
import base64
|
||||
|
||||
value = headers.get("authorization", "")
|
||||
if not value.lower().startswith("basic "):
|
||||
raise unauthorized(domain.BASIC)
|
||||
try:
|
||||
decoded = base64.b64decode(value.split(" ", 1)[1].strip(), validate=True).decode("utf-8")
|
||||
except (ValueError, UnicodeError) as exc:
|
||||
raise unauthorized(domain.BASIC) from exc
|
||||
if ":" not in decoded:
|
||||
raise unauthorized(domain.BASIC)
|
||||
return decoded.split(":", 1)
|
||||
|
||||
|
||||
def _filename(query):
|
||||
raw = _single(query, "filename")
|
||||
if raw is None:
|
||||
raise bad_request()
|
||||
if len(raw) < 1 or len(raw) > 255 or not raw.strip():
|
||||
raise bad_request()
|
||||
if any(ord(ch) < 32 or ord(ch) == 127 or ch in "/\\" for ch in raw):
|
||||
raise bad_request()
|
||||
return raw
|
||||
|
||||
|
||||
def handle(method, path, query, headers, body):
|
||||
template, name, groups = match(method, path)
|
||||
if name == "unmatched":
|
||||
raise not_found()
|
||||
if name not in {"live", "ready"}:
|
||||
delay = faults.handler_slow_seconds()
|
||||
if delay:
|
||||
import time
|
||||
|
||||
time.sleep(delay)
|
||||
if name == "live":
|
||||
return template, 200, {"Content-Type": "application/json; charset=utf-8"}, _dump({"status": "ok"})
|
||||
if name == "ready":
|
||||
domain.ready_check()
|
||||
return template, 200, {"Content-Type": "application/json; charset=utf-8"}, _dump({"status": "ok"})
|
||||
if name == "register":
|
||||
status, payload = domain.register(_json_body(headers, body))
|
||||
return template, status, _json_headers(), _dump(payload)
|
||||
if name == "login":
|
||||
if body:
|
||||
raise bad_request()
|
||||
username, password = _basic(headers)
|
||||
status, payload = domain.login(username, password)
|
||||
return template, status, _json_headers(), _dump(payload)
|
||||
if name == "me":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
return template, 200, _json_headers(), _dump(domain._user_json(user))
|
||||
if name == "logout":
|
||||
domain.revoke(_bearer(headers))
|
||||
return template, 204, {}, b""
|
||||
if name == "create_chat":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
status, payload = domain.create_chat(user, _json_body(headers, body))
|
||||
return template, status, _json_headers(), _dump(payload)
|
||||
if name == "list_chats":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
payload = domain.list_chats(user, _limit(query), _single(query, "cursor"))
|
||||
return template, 200, _json_headers(), _dump(payload)
|
||||
if name == "get_chat":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
payload = domain.get_chat(user, _uuid_segment(groups[0]))
|
||||
return template, 200, _json_headers(), _dump(payload)
|
||||
if name == "create_message":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
chat_id = _uuid_segment(groups[0])
|
||||
status, payload = domain.create_message(
|
||||
user,
|
||||
chat_id,
|
||||
headers.get("idempotency-key"),
|
||||
_json_body(headers, body),
|
||||
path,
|
||||
)
|
||||
return template, status, _json_headers(), _dump(payload)
|
||||
if name == "list_messages":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
payload = domain.list_messages(user, _uuid_segment(groups[0]), _limit(query), _single(query, "cursor"))
|
||||
return template, 200, _json_headers(), _dump(payload)
|
||||
if name == "upload":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
content_type = media_type(headers.get("content-type", ""))
|
||||
status, payload = domain.upload(user, _uuid_segment(groups[0]), _filename(query), content_type, body)
|
||||
return template, status, _json_headers(), _dump(payload)
|
||||
if name == "get_attachment":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
payload = domain.get_attachment(user, _uuid_segment(groups[0]))
|
||||
return template, 200, _json_headers(), _dump(payload)
|
||||
if name == "download":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
variant = _single(query, "variant") or "original"
|
||||
data, content_type = domain.download(user, _uuid_segment(groups[0]), variant)
|
||||
return template, 200, {"Content-Type": content_type}, data
|
||||
raise not_found()
|
||||
|
||||
|
||||
def _uuid_segment(value):
|
||||
from app.codec import parse_uuid
|
||||
|
||||
return parse_uuid(value)
|
||||
|
||||
|
||||
def _json_headers():
|
||||
return {"Content-Type": "application/json; charset=utf-8"}
|
||||
|
||||
|
||||
def _dump(payload):
|
||||
return json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
|
||||
|
||||
def error_response(exc, request_id):
|
||||
if isinstance(exc, AppError):
|
||||
status = exc.status
|
||||
code = exc.code
|
||||
message = exc.message
|
||||
headers = dict(exc.headers)
|
||||
else:
|
||||
status = 500
|
||||
code = "internal_error"
|
||||
message = "Внутренняя ошибка"
|
||||
headers = {}
|
||||
headers["Content-Type"] = "application/json; charset=utf-8"
|
||||
body = _dump({"error": {"code": code, "message": message, "request_id": request_id}})
|
||||
return status, headers, body
|
||||
|
||||
|
||||
def request_id_from(headers):
|
||||
incoming = headers.get("x-request-id", "")
|
||||
if REQUEST_ID.fullmatch(incoming):
|
||||
return incoming
|
||||
import uuid
|
||||
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
def query_from(raw):
|
||||
if not raw:
|
||||
return {}
|
||||
return parse_qs(raw, keep_blank_values=True, strict_parsing=False)
|
||||
|
||||
|
||||
def body_limit(path):
|
||||
if "/attachments" in path and path.rstrip("/").endswith("attachments"):
|
||||
return settings.max_upload + 1
|
||||
return 1_000_000
|
||||
@@ -0,0 +1,62 @@
|
||||
import io
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
from app.codec import fitted_size
|
||||
from app.errors import AppError, too_large, unsupported
|
||||
from app.settings import settings
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = settings.max_pixels
|
||||
|
||||
|
||||
def _open(data):
|
||||
try:
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
image.verify()
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
width, height = image.size
|
||||
if width < 1 or height < 1 or width * height > settings.max_pixels:
|
||||
raise too_large()
|
||||
image.load()
|
||||
return image.copy()
|
||||
except AppError:
|
||||
raise
|
||||
except Image.DecompressionBombError as exc:
|
||||
raise too_large() from exc
|
||||
except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombWarning) as exc:
|
||||
raise unsupported() from exc
|
||||
|
||||
|
||||
def validate_image(data):
|
||||
_open(data).close()
|
||||
|
||||
|
||||
def _rgb(image):
|
||||
if image.mode == "RGB":
|
||||
return image
|
||||
if image.mode in ("RGBA", "LA") or (image.mode == "P" and "transparency" in image.info):
|
||||
rgba = image.convert("RGBA")
|
||||
background = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
background.paste(rgba, mask=rgba.getchannel("A"))
|
||||
return background
|
||||
return image.convert("RGB")
|
||||
|
||||
|
||||
def thumbnail_jpeg(data):
|
||||
image = _open(data)
|
||||
try:
|
||||
image = ImageOps.exif_transpose(image)
|
||||
image = _rgb(image)
|
||||
target = fitted_size(image.width, image.height, 256)
|
||||
if (image.width, image.height) != target:
|
||||
image = image.resize(target, Image.Resampling.LANCZOS)
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, format="JPEG", quality=85, optimize=True)
|
||||
blob = buffer.getvalue()
|
||||
return blob, image.width, image.height
|
||||
except AppError:
|
||||
raise
|
||||
except (UnidentifiedImageError, OSError, ValueError) as exc:
|
||||
raise unsupported() from exc
|
||||
finally:
|
||||
image.close()
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
import os
|
||||
import time
|
||||
|
||||
import uvicorn
|
||||
|
||||
from app import broker, db, domain, storage, telemetry
|
||||
from app.asgi import app
|
||||
from app.poller import start
|
||||
|
||||
|
||||
def _retry(label, action):
|
||||
last = None
|
||||
for _ in range(40):
|
||||
try:
|
||||
action()
|
||||
return
|
||||
except Exception as exc:
|
||||
last = exc
|
||||
telemetry.log_event("dependency.retry", severity="WARNING", dependency=label, error_type=type(exc).__name__)
|
||||
time.sleep(1)
|
||||
raise last
|
||||
|
||||
|
||||
def main():
|
||||
telemetry.init()
|
||||
_retry("postgres", db.init)
|
||||
db.migrate()
|
||||
_retry("s3", storage.init)
|
||||
_retry("broker", broker.init)
|
||||
domain.init_hasher()
|
||||
start()
|
||||
uvicorn.run(
|
||||
app,
|
||||
host=os.environ.get("APP_HOST", "0.0.0.0"),
|
||||
port=int(os.environ.get("APP_PORT", "8080")),
|
||||
access_log=False,
|
||||
log_config={
|
||||
"version": 1,
|
||||
"disable_existing_loggers": False,
|
||||
"loggers": {
|
||||
"uvicorn": {"level": "WARNING"},
|
||||
"uvicorn.error": {"level": "WARNING"},
|
||||
"uvicorn.access": {"level": "WARNING"},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,44 @@
|
||||
import threading
|
||||
|
||||
from app import broker, db, telemetry
|
||||
|
||||
|
||||
def _once():
|
||||
try:
|
||||
with db.connection() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*) AS depth,
|
||||
EXTRACT(EPOCH FROM (NOW() - MIN(created_at))) AS age
|
||||
FROM attachments
|
||||
WHERE status IN ('queued', 'processing')
|
||||
"""
|
||||
).fetchone()
|
||||
depth = int(row["depth"] or 0)
|
||||
age = float(row["age"] or 0) if depth else 0.0
|
||||
used, waiting = db.stats()
|
||||
telemetry.update_runtime(queue_depth=depth, queue_age=age, db_up=1, pool_used=used, pool_waiting=waiting)
|
||||
except Exception:
|
||||
telemetry.update_runtime(db_up=0)
|
||||
try:
|
||||
telemetry.update_runtime(worker_consumers=broker.consumer_count())
|
||||
except Exception:
|
||||
telemetry.log_event("broker.stats_unavailable", severity="WARNING")
|
||||
telemetry.sample_process()
|
||||
try:
|
||||
with db.connection() as conn:
|
||||
broker.publish_pending(conn)
|
||||
except Exception:
|
||||
telemetry.log_event("job.publish_deferred", severity="WARNING")
|
||||
|
||||
|
||||
def start():
|
||||
stop = threading.Event()
|
||||
|
||||
def loop():
|
||||
while not stop.wait(2):
|
||||
_once()
|
||||
|
||||
_once()
|
||||
threading.Thread(target=loop, name="poller", daemon=True).start()
|
||||
return stop
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE,
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS sessions_user_idx ON sessions (user_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
position integer NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
sender_id uuid NOT NULL REFERENCES users (id),
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS messages_order_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
owner_id uuid NOT NULL REFERENCES users (id),
|
||||
filename text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes integer NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
status text NOT NULL CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
error_code text,
|
||||
staging_key text NOT NULL,
|
||||
original_key text,
|
||||
pipeline_version integer NOT NULL DEFAULT 1,
|
||||
attempt integer NOT NULL DEFAULT 0,
|
||||
lease_until timestamptz,
|
||||
worker_id text,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
ready_at timestamptz
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS attachments_open_idx ON attachments (created_at)
|
||||
WHERE status IN ('queued', 'processing');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
name text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes integer NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
object_key text NOT NULL,
|
||||
pipeline_version integer NOT NULL,
|
||||
PRIMARY KEY (attachment_id, name, pipeline_version)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages (id),
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
position integer NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS idempotency (
|
||||
user_id uuid NOT NULL,
|
||||
method text NOT NULL,
|
||||
path text NOT NULL,
|
||||
idem_key text NOT NULL,
|
||||
body_hash bytea NOT NULL,
|
||||
status_code integer NOT NULL,
|
||||
response_json jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, method, path, idem_key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS outbox (
|
||||
job_id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
payload jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
published_at timestamptz
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS outbox_unpublished_idx ON outbox (created_at)
|
||||
WHERE published_at IS NULL;
|
||||
@@ -0,0 +1,45 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _read(name, file_name=None, default=None):
|
||||
value = os.environ.get(name)
|
||||
if value:
|
||||
return value.strip()
|
||||
file_env = os.environ.get(file_name) if file_name else None
|
||||
if file_env:
|
||||
return Path(file_env).read_text(encoding="utf-8").strip()
|
||||
if default is not None:
|
||||
return default
|
||||
raise RuntimeError("missing configuration " + name)
|
||||
|
||||
|
||||
class Settings:
|
||||
def __init__(self):
|
||||
self.service_name = os.environ.get("OTEL_SERVICE_NAME", "messenger-api")
|
||||
self.service_version = os.environ.get("SERVICE_VERSION", "1.6.0")
|
||||
self.instance_id = os.environ.get("INSTANCE_ID", "api-local")
|
||||
self.environment = "lab"
|
||||
self.database_url = _read("DATABASE_URL", "DATABASE_URL_FILE")
|
||||
self.session_ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
|
||||
self.idempotency_ttl = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
self.s3_endpoint = os.environ.get("S3_ENDPOINT", "http://minio:9000")
|
||||
self.s3_region = os.environ.get("S3_REGION", "us-east-1")
|
||||
self.s3_bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
self.s3_access_key = _read("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
|
||||
self.s3_secret_key = _read("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
|
||||
self.broker_url = _read("BROKER_URL", "BROKER_URL_FILE")
|
||||
self.worker_concurrency = int(os.environ.get("WORKER_CONCURRENCY", "2"))
|
||||
self.job_max_attempts = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
|
||||
self.job_timeout = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))
|
||||
self.staging_retention = int(os.environ.get("STAGING_RETENTION_SECONDS", "86400"))
|
||||
self.max_upload = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760"))
|
||||
self.max_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
|
||||
self.pool_max = int(os.environ.get("DB_POOL_MAX", "8"))
|
||||
self.otlp_endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
|
||||
self.fault_dir = os.environ.get("FAULT_DIR", "/faults")
|
||||
self.rabbit_management_url = os.environ.get("RABBITMQ_MANAGEMENT_URL", "http://rabbitmq:15672")
|
||||
self.role = os.environ.get("APP_ROLE", "api")
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,92 @@
|
||||
import threading
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from app import telemetry
|
||||
from app.settings import settings
|
||||
|
||||
_client = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def init():
|
||||
global _client
|
||||
_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.s3_endpoint,
|
||||
aws_access_key_id=settings.s3_access_key,
|
||||
aws_secret_access_key=settings.s3_secret_key,
|
||||
region_name=settings.s3_region,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=2,
|
||||
read_timeout=10,
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
),
|
||||
)
|
||||
try:
|
||||
_client.head_bucket(Bucket=settings.s3_bucket)
|
||||
return
|
||||
except ClientError as exc:
|
||||
code = str(exc.response.get("Error", {}).get("Code", ""))
|
||||
if code not in {"404", "NoSuchBucket", "NotFound"}:
|
||||
raise
|
||||
try:
|
||||
_client.create_bucket(Bucket=settings.s3_bucket)
|
||||
except ClientError as exc:
|
||||
code = str(exc.response.get("Error", {}).get("Code", ""))
|
||||
if code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
|
||||
raise
|
||||
_warm()
|
||||
|
||||
|
||||
def _warm():
|
||||
"""Read one existing object so the first user download is not a cold volume open."""
|
||||
try:
|
||||
listed = _client.list_objects_v2(Bucket=settings.s3_bucket, MaxKeys=1)
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
return
|
||||
contents = listed.get("Contents") or []
|
||||
if not contents:
|
||||
return
|
||||
try:
|
||||
body = _client.get_object(Bucket=settings.s3_bucket, Key=contents[0]["Key"])["Body"]
|
||||
body.read()
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
return
|
||||
|
||||
|
||||
def ready():
|
||||
try:
|
||||
with _lock:
|
||||
_client.head_bucket(Bucket=settings.s3_bucket)
|
||||
return True
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def put(key, data, content_type, variant):
|
||||
with telemetry.tracer().start_as_current_span(
|
||||
"s3.put",
|
||||
attributes={"s3.operation": "put", "variant": variant},
|
||||
):
|
||||
with _lock:
|
||||
_client.put_object(Bucket=settings.s3_bucket, Key=key, Body=data, ContentType=content_type)
|
||||
|
||||
|
||||
def get(key, variant):
|
||||
with telemetry.tracer().start_as_current_span(
|
||||
"s3.get",
|
||||
attributes={"s3.operation": "get", "variant": variant},
|
||||
):
|
||||
with _lock:
|
||||
response = _client.get_object(Bucket=settings.s3_bucket, Key=key)
|
||||
return response["Body"].read()
|
||||
|
||||
|
||||
def delete(key):
|
||||
with _lock:
|
||||
_client.delete_object(Bucket=settings.s3_bucket, Key=key)
|
||||
@@ -0,0 +1,379 @@
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
from opentelemetry import context as otel_context
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.propagate import extract, inject
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk._logs.export import LogExportResult
|
||||
from opentelemetry.sdk.metrics.export import MetricExportResult
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
|
||||
from opentelemetry.trace import Status, StatusCode
|
||||
|
||||
from app.settings import settings
|
||||
|
||||
BANNED_FIELDS = {
|
||||
"password",
|
||||
"token",
|
||||
"authorization",
|
||||
"secret",
|
||||
"filename",
|
||||
"text",
|
||||
"body",
|
||||
"database_url",
|
||||
"broker_url",
|
||||
"image",
|
||||
}
|
||||
|
||||
_dropped = None
|
||||
_http_requests = None
|
||||
_http_duration = None
|
||||
_jobs = None
|
||||
_job_duration = None
|
||||
_ready_duration = None
|
||||
_db_duration = None
|
||||
_state = {
|
||||
"queue_depth": 0,
|
||||
"queue_age": 0.0,
|
||||
"db_up": 0,
|
||||
"pool_used": 0,
|
||||
"pool_waiting": 0,
|
||||
"worker_busy": 0,
|
||||
"worker_capacity": 0,
|
||||
"worker_consumers": 0,
|
||||
"cpu": 0.0,
|
||||
"memory": 0,
|
||||
}
|
||||
_cpu_last = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
class _GuardedExporter:
|
||||
def __init__(self, inner, signal):
|
||||
self.inner = inner
|
||||
self.signal = signal
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.inner, name)
|
||||
|
||||
def export(self, items, **kwargs):
|
||||
try:
|
||||
result = self.inner.export(items)
|
||||
except Exception:
|
||||
_count_drop(self.signal, len(items))
|
||||
return self._failure()
|
||||
if getattr(result, "name", None) != "SUCCESS":
|
||||
_count_drop(self.signal, len(items))
|
||||
return result
|
||||
|
||||
def _failure(self):
|
||||
if self.signal == "metrics":
|
||||
return MetricExportResult.FAILURE
|
||||
if self.signal == "logs":
|
||||
return LogExportResult.FAILURE
|
||||
return SpanExportResult.FAILURE
|
||||
|
||||
def shutdown(self, **kwargs):
|
||||
try:
|
||||
return self.inner.shutdown(**kwargs)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def force_flush(self, timeout_millis=3000):
|
||||
try:
|
||||
return self.inner.force_flush(timeout_millis)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _RawFormatter(logging.Formatter):
|
||||
def format(self, record):
|
||||
return record.getMessage()
|
||||
|
||||
|
||||
def _count_drop(signal, count):
|
||||
if _dropped is not None and count:
|
||||
_dropped.add(count, {"signal": signal})
|
||||
|
||||
|
||||
def _resource():
|
||||
return Resource.create(
|
||||
{
|
||||
"service.name": settings.service_name,
|
||||
"service.version": settings.service_version,
|
||||
"service.instance.id": settings.instance_id,
|
||||
"deployment.environment.name": settings.environment,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _histogram(name, buckets):
|
||||
return View(instrument_name=name, aggregation=ExplicitBucketHistogramAggregation(buckets))
|
||||
|
||||
|
||||
def init():
|
||||
global _dropped, _http_requests, _http_duration, _jobs, _job_duration, _ready_duration, _db_duration
|
||||
resource = _resource()
|
||||
endpoint = settings.otlp_endpoint
|
||||
tracer_provider = TracerProvider(resource=resource)
|
||||
tracer_provider.add_span_processor(
|
||||
BatchSpanProcessor(
|
||||
_GuardedExporter(OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=5), "spans"),
|
||||
max_queue_size=256,
|
||||
schedule_delay_millis=2000,
|
||||
max_export_batch_size=64,
|
||||
export_timeout_millis=5000,
|
||||
)
|
||||
)
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
|
||||
reader = PeriodicExportingMetricReader(
|
||||
_GuardedExporter(OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=5), "metrics"),
|
||||
export_interval_millis=5000,
|
||||
export_timeout_millis=4000,
|
||||
)
|
||||
meter_provider = MeterProvider(
|
||||
resource=resource,
|
||||
metric_readers=[reader],
|
||||
views=[
|
||||
_histogram("http.server.duration", [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]),
|
||||
_histogram("messenger.job.duration", [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]),
|
||||
_histogram("messenger.attachment.time_to_ready", [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60]),
|
||||
_histogram("messenger.db.duration", [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5]),
|
||||
],
|
||||
)
|
||||
from opentelemetry import metrics
|
||||
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
meter = metrics.get_meter("messenger", settings.service_version)
|
||||
_dropped = meter.create_counter("messenger.telemetry.dropped", unit="1")
|
||||
_http_requests = meter.create_counter("http.server.requests", unit="1")
|
||||
_http_duration = meter.create_histogram("http.server.duration", unit="s")
|
||||
_jobs = meter.create_counter("messenger.jobs", unit="1")
|
||||
_job_duration = meter.create_histogram("messenger.job.duration", unit="s")
|
||||
_ready_duration = meter.create_histogram("messenger.attachment.time_to_ready", unit="s")
|
||||
_db_duration = meter.create_histogram("messenger.db.duration", unit="s")
|
||||
meter.create_observable_gauge("messenger.queue.depth", callbacks=[_queue_depth])
|
||||
meter.create_observable_gauge("messenger.queue.oldest_age", callbacks=[_queue_age], unit="s")
|
||||
meter.create_observable_gauge("messenger.db.pool.used", callbacks=[_pool_used])
|
||||
meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[_pool_waiting])
|
||||
meter.create_observable_gauge("messenger.worker.busy", callbacks=[_worker_busy])
|
||||
meter.create_observable_gauge("messenger.worker.capacity", callbacks=[_worker_capacity])
|
||||
meter.create_observable_gauge("messenger.worker.consumers", callbacks=[_worker_consumers])
|
||||
meter.create_observable_gauge("messenger.db.up", callbacks=[_db_up])
|
||||
meter.create_observable_gauge("messenger.process.cpu.utilization", callbacks=[_cpu])
|
||||
meter.create_observable_gauge("messenger.process.memory", callbacks=[_memory], unit="By")
|
||||
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(
|
||||
_GuardedExporter(OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=5), "logs"),
|
||||
max_queue_size=256,
|
||||
schedule_delay_millis=2000,
|
||||
max_export_batch_size=64,
|
||||
export_timeout_millis=5000,
|
||||
)
|
||||
)
|
||||
root = logging.getLogger()
|
||||
root.handlers.clear()
|
||||
root.setLevel(logging.INFO)
|
||||
stream = logging.StreamHandler()
|
||||
stream.setFormatter(_RawFormatter())
|
||||
root.addHandler(stream)
|
||||
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||
root.addHandler(otel_handler)
|
||||
for name in ("pika", "botocore", "boto3", "urllib3", "uvicorn", "uvicorn.error", "uvicorn.access"):
|
||||
logging.getLogger(name).setLevel(logging.WARNING)
|
||||
quiet = logging.getLogger("opentelemetry")
|
||||
quiet.handlers.clear()
|
||||
quiet.propagate = False
|
||||
quiet.setLevel(logging.CRITICAL)
|
||||
log_event("service.started", role=settings.role)
|
||||
|
||||
|
||||
def tracer():
|
||||
return trace.get_tracer("messenger", settings.service_version)
|
||||
|
||||
|
||||
def current_ids():
|
||||
span = trace.get_current_span()
|
||||
ctx = span.get_span_context()
|
||||
if not ctx or not ctx.is_valid:
|
||||
return None, None
|
||||
return format(ctx.trace_id, "032x"), format(ctx.span_id, "016x")
|
||||
|
||||
|
||||
def log_event(event, severity="INFO", **fields):
|
||||
trace_id, span_id = current_ids()
|
||||
payload = {
|
||||
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + f".{int(time.time() * 1000000) % 1000000:06d}Z",
|
||||
"severity": severity,
|
||||
"service.name": settings.service_name,
|
||||
"event": event,
|
||||
"trace_id": trace_id,
|
||||
"span_id": span_id,
|
||||
}
|
||||
for key, value in fields.items():
|
||||
if key in BANNED_FIELDS or value is None:
|
||||
continue
|
||||
if isinstance(value, (str, int, float, bool)):
|
||||
payload[key] = value
|
||||
level = getattr(logging, severity, logging.INFO)
|
||||
logging.getLogger("messenger").log(level, json.dumps(payload, ensure_ascii=False))
|
||||
|
||||
|
||||
def inject_carrier(carrier=None):
|
||||
target = {} if carrier is None else carrier
|
||||
inject(target)
|
||||
return target
|
||||
|
||||
|
||||
def extract_context(carrier):
|
||||
clean = {}
|
||||
for key, value in (carrier or {}).items():
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("utf-8", "replace")
|
||||
if isinstance(value, str):
|
||||
clean[str(key).lower()] = value
|
||||
return extract(clean)
|
||||
|
||||
|
||||
def record_http(method, route, status, duration):
|
||||
if _http_requests is None:
|
||||
return
|
||||
klass = f"{int(status) // 100}xx"
|
||||
attributes = {
|
||||
"http.request.method": method,
|
||||
"http.route": route,
|
||||
"http.response.status_class": klass,
|
||||
}
|
||||
_http_requests.add(1, attributes)
|
||||
_http_duration.record(duration, attributes)
|
||||
|
||||
|
||||
def record_job(outcome, duration):
|
||||
attributes = {"job_type": "attachment.process.v1", "outcome": outcome}
|
||||
if _jobs is not None:
|
||||
_jobs.add(1, attributes)
|
||||
_job_duration.record(max(duration, 0.0), attributes)
|
||||
|
||||
|
||||
def record_ready(duration):
|
||||
if _ready_duration is not None:
|
||||
_ready_duration.record(max(duration, 0.0), {"job_type": "attachment.process.v1"})
|
||||
|
||||
|
||||
def record_db(duration):
|
||||
if _db_duration is not None:
|
||||
_db_duration.record(max(duration, 0.0), {"db.system": "postgresql"})
|
||||
|
||||
|
||||
def update_runtime(**values):
|
||||
with _lock:
|
||||
_state.update(values)
|
||||
|
||||
|
||||
def note_worker(busy, capacity):
|
||||
update_runtime(worker_busy=busy, worker_capacity=capacity)
|
||||
|
||||
|
||||
def sample_process():
|
||||
global _cpu_last
|
||||
now_cpu = time.process_time()
|
||||
now_wall = time.monotonic()
|
||||
ratio = 0.0
|
||||
if _cpu_last is not None:
|
||||
dc = now_cpu - _cpu_last[0]
|
||||
dw = now_wall - _cpu_last[1]
|
||||
if dw > 0:
|
||||
ratio = dc / dw
|
||||
_cpu_last = (now_cpu, now_wall)
|
||||
memory = 0
|
||||
try:
|
||||
with open("/proc/self/statm", encoding="ascii") as handle:
|
||||
resident = int(handle.read().split()[1])
|
||||
memory = resident * os_page()
|
||||
except (OSError, IndexError, ValueError):
|
||||
memory = 0
|
||||
update_runtime(cpu=ratio, memory=memory)
|
||||
|
||||
|
||||
_page = None
|
||||
|
||||
|
||||
def os_page():
|
||||
global _page
|
||||
if _page is None:
|
||||
import os
|
||||
|
||||
_page = os.sysconf("SC_PAGE_SIZE")
|
||||
return _page
|
||||
|
||||
|
||||
def _obs(value, attributes=None):
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
return [Observation(value, attributes or {})]
|
||||
|
||||
|
||||
def _queue_depth(_options):
|
||||
return _obs(_state["queue_depth"], {"queue": "attachment.process.v1"})
|
||||
|
||||
|
||||
def _queue_age(_options):
|
||||
return _obs(_state["queue_age"], {"queue": "attachment.process.v1"})
|
||||
|
||||
|
||||
def _pool_used(_options):
|
||||
return _obs(_state["pool_used"], {"pool": "postgres"})
|
||||
|
||||
|
||||
def _pool_waiting(_options):
|
||||
return _obs(_state["pool_waiting"], {"pool": "postgres"})
|
||||
|
||||
|
||||
def _worker_busy(_options):
|
||||
if settings.role != "worker":
|
||||
return []
|
||||
return _obs(_state["worker_busy"], {"queue": "attachment.process.v1"})
|
||||
|
||||
|
||||
def _worker_capacity(_options):
|
||||
if settings.role != "worker":
|
||||
return []
|
||||
return _obs(_state["worker_capacity"], {"queue": "attachment.process.v1"})
|
||||
|
||||
|
||||
def _worker_consumers(_options):
|
||||
return _obs(_state["worker_consumers"], {"queue": "attachment.process.v1"})
|
||||
|
||||
|
||||
def _db_up(_options):
|
||||
return _obs(_state["db_up"])
|
||||
|
||||
|
||||
def _cpu(_options):
|
||||
return _obs(_state["cpu"])
|
||||
|
||||
|
||||
def _memory(_options):
|
||||
return _obs(_state["memory"])
|
||||
|
||||
|
||||
def mark_error(span):
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
|
||||
|
||||
def attach(ctx):
|
||||
return otel_context.attach(ctx)
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import pika
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from app import broker, db, storage, telemetry
|
||||
from app.broker import QUEUE, _params
|
||||
from app.codec import utcnow
|
||||
from app.errors import AppError
|
||||
from app.faults import consume_process_failure
|
||||
from app.images import thumbnail_jpeg
|
||||
from app.poller import start
|
||||
from app.settings import settings
|
||||
|
||||
_inflight = 0
|
||||
_inflight_lock = threading.Lock()
|
||||
_stop = threading.Event()
|
||||
|
||||
|
||||
class Transient(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Permanent(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _busy():
|
||||
global _inflight
|
||||
with _inflight_lock:
|
||||
_inflight += 1
|
||||
current = _inflight
|
||||
telemetry.note_worker(current, settings.worker_concurrency)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with _inflight_lock:
|
||||
_inflight -= 1
|
||||
current = _inflight
|
||||
telemetry.note_worker(current, settings.worker_concurrency)
|
||||
|
||||
|
||||
def _mark_failed(attachment_id, attempt, code):
|
||||
with db.connection() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = %s, lease_until = NULL, worker_id = NULL, updated_at = %s
|
||||
WHERE id = %s AND status = 'processing' AND attempt = %s
|
||||
""",
|
||||
(code, utcnow(), attachment_id, attempt),
|
||||
)
|
||||
telemetry.log_event("job.failed", attachment_id=str(attachment_id), attempt=attempt, error_code=code)
|
||||
|
||||
|
||||
def _release(attachment_id, attempt):
|
||||
with db.connection() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'queued', lease_until = NULL, worker_id = NULL, updated_at = %s
|
||||
WHERE id = %s AND status = 'processing' AND attempt = %s
|
||||
""",
|
||||
(utcnow(), attachment_id, attempt),
|
||||
)
|
||||
|
||||
|
||||
def process_delivery(body, headers):
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
payload = json.loads(body.decode("utf-8"))
|
||||
attachment_id = uuid.UUID(str(payload["attachment_id"]))
|
||||
pipeline = int(payload["pipeline_version"])
|
||||
except (ValueError, UnicodeError, KeyError, TypeError):
|
||||
telemetry.log_event("job.poison", severity="ERROR")
|
||||
return "ack"
|
||||
carrier = {}
|
||||
for key, value in (headers or {}).items():
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("utf-8", "replace")
|
||||
if isinstance(value, str):
|
||||
carrier[str(key).lower()] = value
|
||||
if payload.get("traceparent"):
|
||||
carrier.setdefault("traceparent", payload["traceparent"])
|
||||
if payload.get("tracestate"):
|
||||
carrier.setdefault("tracestate", payload["tracestate"])
|
||||
parent = telemetry.extract_context(carrier)
|
||||
outcome = None
|
||||
with telemetry.tracer().start_as_current_span(
|
||||
"attachment.process consume",
|
||||
context=parent,
|
||||
kind=SpanKind.CONSUMER,
|
||||
attributes={
|
||||
"messaging.system": "rabbitmq",
|
||||
"messaging.destination.name": QUEUE,
|
||||
"messaging.operation.type": "consume",
|
||||
},
|
||||
) as span:
|
||||
try:
|
||||
action, outcome, attempt = _consume(payload, attachment_id, pipeline, span)
|
||||
except Exception:
|
||||
telemetry.mark_error(span)
|
||||
telemetry.log_event("job.failed", severity="ERROR", error_type="unexpected", attachment_id=str(attachment_id))
|
||||
action, outcome = "requeue", "retry"
|
||||
attempt = None
|
||||
if outcome:
|
||||
telemetry.record_job(outcome, time.perf_counter() - started)
|
||||
if attempt is not None:
|
||||
span.set_attribute("attempt", attempt)
|
||||
if outcome in {"failed", "retry"}:
|
||||
telemetry.mark_error(span)
|
||||
return action
|
||||
|
||||
|
||||
def _consume(payload, attachment_id, pipeline, span):
|
||||
now = utcnow()
|
||||
with db.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone()
|
||||
if not row or int(row["pipeline_version"]) != pipeline:
|
||||
return "ack", None, None
|
||||
if row["status"] == "ready":
|
||||
return "ack", None, int(row["attempt"])
|
||||
if row["status"] == "failed":
|
||||
return "ack", None, int(row["attempt"])
|
||||
if row["status"] == "processing" and row["lease_until"] and row["lease_until"] > now:
|
||||
held_attempt = int(row["attempt"])
|
||||
staging_key = None
|
||||
else:
|
||||
held_attempt = None
|
||||
attempt = int(row["attempt"]) + 1
|
||||
if attempt > settings.job_max_attempts:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = 'retry_exhausted', lease_until = NULL, updated_at = %s
|
||||
WHERE id = %s
|
||||
""",
|
||||
(now, attachment_id),
|
||||
)
|
||||
return "ack", "failed", attempt
|
||||
lease = now + dt.timedelta(seconds=settings.job_timeout)
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'processing', attempt = %s, lease_until = %s, worker_id = %s, updated_at = %s
|
||||
WHERE id = %s
|
||||
""",
|
||||
(attempt, lease, settings.instance_id, now, attachment_id),
|
||||
)
|
||||
staging_key = row["staging_key"]
|
||||
content_type = row["content_type"]
|
||||
expected_sha = row["sha256"]
|
||||
created_at = row["created_at"]
|
||||
if staging_key is None:
|
||||
span.add_event("lease_held")
|
||||
time.sleep(1)
|
||||
return "requeue", None, held_attempt
|
||||
span.set_attribute("attempt", attempt)
|
||||
telemetry.log_event("job.consumed", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id")))
|
||||
try:
|
||||
if consume_process_failure():
|
||||
span.add_event("retry", {"reason": "injected"})
|
||||
raise Transient()
|
||||
data = storage.get(staging_key, "staging")
|
||||
if hashlib.sha256(data).hexdigest() != expected_sha:
|
||||
raise Permanent("invalid_image")
|
||||
thumb, width, height = thumbnail_jpeg(data)
|
||||
original_key = f"originals/{attachment_id}"
|
||||
thumb_key = f"variants/{attachment_id}/thumbnail"
|
||||
storage.put(original_key, data, content_type, "original")
|
||||
storage.put(thumb_key, thumb, "image/jpeg", "thumbnail")
|
||||
except Permanent as exc:
|
||||
_mark_failed(attachment_id, attempt, str(exc) or "processing_failed")
|
||||
return "ack", "failed", attempt
|
||||
except AppError as exc:
|
||||
code = "invalid_image" if exc.status in {413, 415} else "processing_failed"
|
||||
_mark_failed(attachment_id, attempt, code)
|
||||
return "ack", "failed", attempt
|
||||
except Transient:
|
||||
_release(attachment_id, attempt)
|
||||
telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="transient")
|
||||
time.sleep(0.5)
|
||||
return "requeue", "retry", attempt
|
||||
except (BotoCoreError, ClientError, OSError, TimeoutError):
|
||||
_release(attachment_id, attempt)
|
||||
telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="storage")
|
||||
time.sleep(min(2 ** max(attempt - 1, 0), 8))
|
||||
return "requeue", "retry", attempt
|
||||
thumb_sha = hashlib.sha256(thumb).hexdigest()
|
||||
ready_at = utcnow()
|
||||
with db.connection() as conn:
|
||||
current = conn.execute("SELECT status, attempt FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone()
|
||||
if not current or current["status"] != "processing" or int(current["attempt"]) != attempt:
|
||||
return "ack", None, attempt
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO attachment_variants (
|
||||
attachment_id, name, content_type, size_bytes, width, height, sha256, object_key, pipeline_version
|
||||
) VALUES (%s, 'thumbnail', 'image/jpeg', %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (attachment_id, name, pipeline_version) DO UPDATE
|
||||
SET content_type = EXCLUDED.content_type,
|
||||
size_bytes = EXCLUDED.size_bytes,
|
||||
width = EXCLUDED.width,
|
||||
height = EXCLUDED.height,
|
||||
sha256 = EXCLUDED.sha256,
|
||||
object_key = EXCLUDED.object_key
|
||||
""",
|
||||
(attachment_id, len(thumb), width, height, thumb_sha, thumb_key, pipeline),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'ready', error_code = NULL, original_key = %s, ready_at = %s,
|
||||
lease_until = NULL, updated_at = %s
|
||||
WHERE id = %s AND status = 'processing' AND attempt = %s
|
||||
""",
|
||||
(original_key, ready_at, ready_at, attachment_id, attempt),
|
||||
)
|
||||
try:
|
||||
storage.delete(staging_key)
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
telemetry.log_event("staging.cleanup_deferred", severity="WARNING", attachment_id=str(attachment_id))
|
||||
telemetry.record_ready((ready_at - created_at).total_seconds())
|
||||
telemetry.log_event("job.ready", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id")))
|
||||
return "ack", "success", attempt
|
||||
|
||||
|
||||
def _serve(connection):
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=QUEUE, durable=True)
|
||||
channel.basic_qos(prefetch_count=settings.worker_concurrency)
|
||||
executor = ThreadPoolExecutor(max_workers=settings.worker_concurrency, thread_name_prefix="job")
|
||||
|
||||
def on_message(ch, method, properties, body):
|
||||
def work():
|
||||
with _busy():
|
||||
try:
|
||||
action = process_delivery(body, properties.headers if properties else None)
|
||||
except Exception:
|
||||
action = "requeue"
|
||||
|
||||
def finish():
|
||||
try:
|
||||
if action == "ack":
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
else:
|
||||
ch.basic_nack(method.delivery_tag, requeue=True)
|
||||
except Exception:
|
||||
telemetry.log_event("job.ack_failed", severity="ERROR")
|
||||
|
||||
connection.add_callback_threadsafe(finish)
|
||||
|
||||
executor.submit(work)
|
||||
|
||||
channel.basic_consume(queue=QUEUE, on_message_callback=on_message, auto_ack=False)
|
||||
|
||||
def watch():
|
||||
_stop.wait()
|
||||
connection.add_callback_threadsafe(channel.stop_consuming)
|
||||
|
||||
threading.Thread(target=watch, name="stop-watch", daemon=True).start()
|
||||
channel.start_consuming()
|
||||
executor.shutdown(wait=True, cancel_futures=False)
|
||||
|
||||
|
||||
class _Health(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if self.path.startswith("/health/live"):
|
||||
self._send(200)
|
||||
return
|
||||
if self.path.startswith("/health/ready"):
|
||||
try:
|
||||
with db.connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
ok = storage.ready() and broker.ready()
|
||||
except Exception:
|
||||
ok = False
|
||||
self._send(200 if ok else 503)
|
||||
return
|
||||
self._send(404)
|
||||
|
||||
def _send(self, status):
|
||||
body = b'{"status":"ok"}' if status == 200 else b'{"status":"unavailable"}'
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
return
|
||||
|
||||
|
||||
def _health_server():
|
||||
server = ThreadingHTTPServer(("0.0.0.0", 8081), _Health)
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
def _request_stop(signum, _frame):
|
||||
_stop.set()
|
||||
|
||||
|
||||
def main():
|
||||
signal.signal(signal.SIGTERM, _request_stop)
|
||||
signal.signal(signal.SIGINT, _request_stop)
|
||||
telemetry.init()
|
||||
from app.main import _retry
|
||||
|
||||
_retry("postgres", db.init)
|
||||
db.migrate()
|
||||
_retry("s3", storage.init)
|
||||
_retry("broker", broker.init)
|
||||
telemetry.note_worker(0, settings.worker_concurrency)
|
||||
start()
|
||||
threading.Thread(target=_health_server, name="health", daemon=True).start()
|
||||
while not _stop.is_set():
|
||||
connection = None
|
||||
try:
|
||||
connection = pika.BlockingConnection(_params())
|
||||
telemetry.update_runtime(worker_consumers=settings.worker_concurrency)
|
||||
_serve(connection)
|
||||
except Exception:
|
||||
if _stop.is_set():
|
||||
break
|
||||
telemetry.log_event("broker.reconnect", severity="WARNING", error_type="connection")
|
||||
time.sleep(1)
|
||||
finally:
|
||||
if connection is not None and connection.is_open:
|
||||
try:
|
||||
connection.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user