Files
mtusi2026lab5/app/httpapi.py
T

1071 lines
47 KiB
Python

"""HTTP API for the messenger contract. Heavy image work is never done here."""
from __future__ import annotations
import base64
import hashlib
import hmac
import json
import logging
import re
import secrets
import uuid
from datetime import timedelta
from fastapi import FastAPI, Request
from fastapi.responses import Response
from psycopg.errors import UniqueViolation
from psycopg.types.json import Jsonb
from starlette.exceptions import HTTPException as StarletteHTTPException
from app import runtime
from app.db import connection, ping
from app.errors import ApiError, unauthorized
from app.images import ImageRejected, inspect_upload
from app.outbox import add_outbox, audit, relay_tick
from app.timeutil import iso, utcnow
log = logging.getLogger("messenger.api")
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
VARIANTS = {"original", "thumbnail", "crop", "optimized"}
class RequestIdMiddleware:
def __init__(self, app, instance_id: str):
self.app = app
self.instance_id = instance_id
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
await self.app(scope, receive, send)
return
request_id = uuid.uuid4().hex
scope.setdefault("state", {})["request_id"] = request_id
async def send_with_ids(message):
if message["type"] == "http.response.start":
headers = list(message.get("headers") or [])
headers.append((b"x-request-id", request_id.encode("ascii")))
headers.append((b"x-instance-id", self.instance_id.encode("utf-8")))
message = {**message, "headers": headers}
await send(message)
await self.app(scope, receive, send_with_ids)
def respond(status: int, payload, headers: dict | None = None) -> Response:
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
return Response(content=body, status_code=status, media_type="application/json", headers=headers)
def error_payload(request: Request, code: str, message: str) -> dict:
request_id = getattr(request.state, "request_id", None) or uuid.uuid4().hex
return {"error": {"code": code, "message": message, "request_id": request_id}}
def create_app() -> FastAPI:
settings, _store, _publisher = runtime.require()
app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None, redirect_slashes=False)
app.add_middleware(RequestIdMiddleware, instance_id=settings.instance_id)
@app.exception_handler(ApiError)
async def on_api_error(request: Request, exc: ApiError):
response = respond(exc.status, error_payload(request, exc.code, exc.message))
for key, value in exc.headers.items():
response.headers[key] = value
return response
@app.exception_handler(StarletteHTTPException)
async def on_http(request: Request, exc: StarletteHTTPException):
if exc.status_code == 404:
return respond(404, error_payload(request, "not_found", "Объект не найден"))
return respond(400, error_payload(request, "invalid_request", "Некорректный запрос"))
@app.exception_handler(Exception)
async def on_unexpected(request: Request, exc: Exception):
log.exception("unhandled API error")
return respond(500, error_payload(request, "internal_error", "Внутренняя ошибка"))
@app.get("/health/live")
async def live():
return respond(200, {"status": "ok"})
@app.get("/health/ready")
async def ready(request: Request):
store = runtime.store
if not ping() or store is None or not store.healthy():
raise ApiError(503, "unavailable", "Зависимость недоступна")
return respond(200, {"status": "ok"})
@app.post("/api/v1/users")
async def register_user(request: Request):
data = await _json_object(request)
_reject_unknown(data, {"username", "display_name", "password"})
username = _username(data.get("username"))
display_name = _bounded_text(data.get("display_name"), 1, 100, "display_name")
password = _password(data.get("password"))
user_id = uuid.uuid4()
password_hash = runtime.hasher.hash(password)
try:
with connection() as conn:
row = conn.execute(
"""
INSERT INTO users (id, username, display_name, password_hash)
VALUES (%s, %s, %s, %s)
RETURNING id, username, display_name, created_at
""",
(user_id, username, display_name, password_hash),
).fetchone()
except UniqueViolation as exc:
raise ApiError(409, "conflict", "Имя пользователя занято") from exc
return respond(201, _user_json(row))
@app.post("/api/v1/auth/sessions")
async def create_session(request: Request):
username, password = _basic_credentials(request)
with connection() as conn:
row = conn.execute("SELECT * FROM users WHERE username = %s", (username,)).fetchone()
stored = row["password_hash"] if row else runtime.dummy_hash
try:
runtime.hasher.verify(stored, password)
valid = row is not None
except Exception:
valid = False
if not valid or row is None:
raise unauthorized("basic")
if len(password) < 12 or len(password) > 128:
raise unauthorized("basic")
token = secrets.token_urlsafe(32)
token_hash = hashlib.sha256(token.encode("utf-8")).hexdigest()
expires = utcnow() + timedelta(seconds=runtime.settings.session_ttl_seconds)
conn.execute(
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
(token_hash, row["id"], expires),
)
payload = {
"token": token,
"token_type": "Bearer",
"expires_at": iso(expires),
"user": _user_json(row),
}
return respond(201, payload)
@app.delete("/api/v1/auth/sessions/current")
async def revoke_session(request: Request):
with connection() as conn:
_user, token_hash = _bearer(request, conn)
conn.execute(
"UPDATE sessions SET revoked_at = clock_timestamp() WHERE token_hash = %s AND revoked_at IS NULL",
(token_hash,),
)
return Response(status_code=204)
@app.get("/api/v1/users/me")
async def current_user(request: Request):
with connection() as conn:
user, _token = _bearer(request, conn)
return respond(200, _user_json(user))
@app.post("/api/v1/chats")
async def create_chat(request: Request):
data = await _json_object(request)
_reject_unknown(data, {"title", "member_ids"})
title = _bounded_text(data.get("title"), 1, 100, "title")
member_ids = _uuid_list(data.get("member_ids"), minimum=1, maximum=100)
with connection() as conn:
user, _token = _bearer(request, conn)
if user["id"] not in member_ids:
raise ApiError(400, "invalid_request", "Создатель должен входить в список участников")
found = conn.execute("SELECT id FROM users WHERE id = ANY(%s)", (member_ids,)).fetchall()
if len(found) != len(member_ids):
raise ApiError(400, "invalid_request", "Список участников содержит неизвестных пользователей")
chat_id = uuid.uuid4()
row = conn.execute(
"INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at",
(chat_id, title),
).fetchone()
for position, member_id in enumerate(member_ids):
conn.execute(
"INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)",
(chat_id, member_id, position),
)
payload = _chat_json(conn, row)
return respond(201, payload)
@app.get("/api/v1/chats")
async def list_chats(request: Request):
limit = _limit(request)
with connection() as conn:
user, _token = _bearer(request, conn)
cursor = _cursor(request, "chats", str(user["id"]))
rows = conn.execute(
"""
SELECT c.id, c.title, c.created_at
FROM chats AS c
JOIN chat_members AS m ON m.chat_id = c.id AND m.user_id = %s
WHERE (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))
ORDER BY c.created_at, c.id
LIMIT %s
""",
(user["id"], cursor[0], cursor[0], cursor[1], limit + 1),
).fetchall()
page, next_cursor = _page(rows, limit, lambda row: {"scope": "chats", "owner": str(user["id"]), "created_at": iso(row["created_at"]), "id": str(row["id"])})
items = [_chat_json(conn, row) for row in page]
return respond(200, {"items": items, "next_cursor": next_cursor})
@app.get("/api/v1/chats/{chat_id}")
async def get_chat(chat_id: str, request: Request):
parsed = _parse_uuid(chat_id)
with connection() as conn:
user, _token = _bearer(request, conn)
row = _require_chat(conn, parsed, user["id"])
payload = _chat_json(conn, row)
return respond(200, payload)
@app.post("/api/v1/chats/{chat_id}/messages")
async def create_message(chat_id: str, request: Request):
parsed = _parse_uuid(chat_id)
data = await _json_object(request)
_reject_unknown(data, {"text", "attachment_ids"})
if "text" not in data:
raise ApiError(400, "invalid_request", "Текст сообщения обязателен")
text = data["text"]
if not isinstance(text, str) or len(text) > 4000 or "\x00" in text:
raise ApiError(400, "invalid_request", "Некорректный текст сообщения")
if "attachment_ids" in data:
attachment_ids = _uuid_list(data["attachment_ids"], minimum=0, maximum=10)
else:
attachment_ids = []
if text.strip() == "" and not attachment_ids:
raise ApiError(400, "invalid_request", "Пустое сообщение без вложений недопустимо")
key = _idempotency_key(request)
path = "/api/v1/chats/" + str(parsed) + "/messages"
digest = hashlib.sha256(
json.dumps({"attachment_ids": [str(item) for item in attachment_ids], "text": text}, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
with connection() as conn:
user, _token = _bearer(request, conn)
_require_chat(conn, parsed, user["id"])
conn.execute(
"""
DELETE FROM idempotency_keys
WHERE user_id = %s AND http_method = 'POST' AND path = %s AND idem_key = %s
AND created_at < clock_timestamp() - make_interval(secs => %s)
""",
(user["id"], path, key, runtime.settings.idempotency_ttl_seconds),
)
inserted = conn.execute(
"""
INSERT INTO idempotency_keys (user_id, http_method, path, idem_key, request_hash)
VALUES (%s, 'POST', %s, %s, %s)
ON CONFLICT DO NOTHING
RETURNING idem_key
""",
(user["id"], path, key, digest),
).fetchone()
if inserted is None:
existing = conn.execute(
"""
SELECT request_hash, response_json FROM idempotency_keys
WHERE user_id = %s AND http_method = 'POST' AND path = %s AND idem_key = %s
""",
(user["id"], path, key),
).fetchone()
if existing is None or existing["response_json"] is None:
raise ApiError(503, "unavailable", "Повтор запроса ещё выполняется")
if existing["request_hash"] != digest:
raise ApiError(409, "conflict", "Ключ идемпотентности уже использован с другим сообщением")
return respond(201, existing["response_json"])
_claim_attachments(conn, parsed, attachment_ids)
message = conn.execute(
"""
INSERT INTO messages (id, chat_id, sender_id, body)
VALUES (%s, %s, %s, %s)
RETURNING id, chat_id, sender_id, body, created_at
""",
(uuid.uuid4(), parsed, user["id"], text),
).fetchone()
for position, attachment_id in enumerate(attachment_ids):
conn.execute(
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
(message["id"], attachment_id, position),
)
payload = _message_json(message, attachment_ids)
conn.execute(
"""
UPDATE idempotency_keys SET response_json = %s
WHERE user_id = %s AND http_method = 'POST' AND path = %s AND idem_key = %s
""",
(Jsonb(payload), user["id"], path, key),
)
return respond(201, payload)
@app.get("/api/v1/chats/{chat_id}/messages")
async def list_messages(chat_id: str, request: Request):
parsed = _parse_uuid(chat_id)
limit = _limit(request)
with connection() as conn:
user, _token = _bearer(request, conn)
_require_chat(conn, parsed, user["id"])
cursor = _cursor(request, "messages", str(parsed))
rows = conn.execute(
"""
SELECT id, chat_id, sender_id, body, created_at
FROM messages
WHERE chat_id = %s
AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))
ORDER BY created_at, id
LIMIT %s
""",
(parsed, cursor[0], cursor[0], cursor[1], limit + 1),
).fetchall()
page, next_cursor = _page(
rows,
limit,
lambda row: {"scope": "messages", "owner": str(parsed), "created_at": iso(row["created_at"]), "id": str(row["id"])},
)
items = []
for row in page:
ids = _attachment_ids(conn, row["id"])
items.append(_message_json(row, ids))
return respond(200, {"items": items, "next_cursor": next_cursor})
@app.post("/api/v1/chats/{chat_id}/attachments")
async def upload_attachment(chat_id: str, request: Request):
parsed = _parse_uuid(chat_id)
filename = request.query_params.get("filename")
_filename(filename)
declared = _media_type(request)
with connection() as conn:
user, _token = _bearer(request, conn)
_require_chat(conn, parsed, user["id"])
if _queue_limit(conn) <= conn.execute(
"SELECT count(*) AS n FROM attachments WHERE status IN ('queued', 'processing')"
).fetchone()["n"]:
raise ApiError(503, "unavailable", "Очередь обработки заполнена")
owner_id = user["id"]
data = await _read_body(request, runtime.settings.max_upload_bytes)
try:
content_type, _width, _height = inspect_upload(
data, declared, runtime.settings.max_upload_bytes, runtime.settings.max_image_pixels
)
except ImageRejected as exc:
raise ApiError(exc.status, exc.code, exc.message) from exc
digest = hashlib.sha256(data).hexdigest()
attachment_id = uuid.uuid4()
staging_key = f"staging/{attachment_id}"
original_key = f"originals/{attachment_id}"
store = runtime.store
try:
store.put(staging_key, data, content_type)
except Exception as exc:
log.warning("staging upload failed")
raise ApiError(503, "unavailable", "Хранилище временно недоступно") from exc
try:
with connection() as conn:
_require_chat(conn, parsed, owner_id)
if _queue_limit(conn) <= conn.execute(
"SELECT count(*) AS n FROM attachments WHERE status IN ('queued', 'processing')"
).fetchone()["n"]:
raise ApiError(503, "unavailable", "Очередь обработки заполнена")
row = conn.execute(
"""
INSERT INTO attachments (
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status,
pipeline_version, staging_key, original_key
) VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', 1, %s, %s)
RETURNING *
""",
(attachment_id, parsed, owner_id, filename, content_type, len(data), digest, staging_key, original_key),
).fetchone()
job_id = uuid.uuid4()
job = conn.execute(
"""
INSERT INTO jobs (job_id, attachment_id, pipeline_version, attempt, status)
VALUES (%s, %s, 1, 1, 'queued')
RETURNING job_id, attachment_id, pipeline_version, attempt, created_at
""",
(job_id, attachment_id),
).fetchone()
add_outbox(conn, job["job_id"], attachment_id, 1, 1, job["created_at"])
audit(conn, job_id, attachment_id, "accepted", {"pipeline_version": 1})
snapshot = _attachment_json(row, [])
except Exception:
try:
store.delete(staging_key)
except Exception:
log.warning("failed to delete uncommitted staging object")
raise
try:
relay_tick()
except Exception:
log.warning("publish deferred to the outbox relay")
return respond(202, snapshot)
@app.get("/api/v1/attachments/{attachment_id}")
async def get_attachment(attachment_id: str, request: Request):
parsed = _parse_uuid(attachment_id)
with connection() as conn:
user, _token = _bearer(request, conn)
row = _visible_attachment(conn, parsed, user["id"])
payload = _attachment_json(row, _visible_variants(conn, row))
return respond(200, payload)
@app.get("/api/v1/attachments/{attachment_id}/content")
async def download_attachment(attachment_id: str, request: Request):
parsed = _parse_uuid(attachment_id)
variant = request.query_params.get("variant") or "original"
if variant not in VARIANTS:
raise ApiError(400, "invalid_request", "Неизвестный вариант")
with connection() as conn:
user, _token = _bearer(request, conn)
row = _visible_attachment(conn, parsed, user["id"])
if row["status"] != "ready":
raise ApiError(409, "conflict", "Вложение ещё не готово")
if variant == "original":
key = row["original_key"]
content_type = row["content_type"]
else:
found = conn.execute(
"""
SELECT object_key, content_type FROM attachment_variants
WHERE attachment_id = %s AND pipeline_version = %s AND name = %s
""",
(row["id"], row["ready_pipeline_version"], variant),
).fetchone()
if found is None:
raise ApiError(404, "not_found", "Вариант не найден")
key, content_type = found["object_key"], found["content_type"]
try:
blob = runtime.store.get(key)
except Exception as exc:
log.warning("object download failed")
raise ApiError(503, "unavailable", "Хранилище временно недоступно") from exc
if blob is None:
raise ApiError(404, "not_found", "Вариант не найден")
return Response(content=blob, media_type=content_type)
@app.post("/internal/control/outbox")
async def set_outbox(request: Request):
_operator(request)
data = await _json_object(request)
_reject_unknown(data, {"paused"})
if not isinstance(data.get("paused"), bool):
raise ApiError(400, "invalid_request", "paused должен быть boolean")
with connection() as conn:
conn.execute("UPDATE runtime_control SET value = %s WHERE key = 'outbox_paused'", (Jsonb(data["paused"]),))
return respond(200, {"paused": data["paused"]})
@app.post("/internal/control/fault")
async def set_fault(request: Request):
_operator(request)
data = await _json_object(request)
_reject_unknown(data, {"mode", "remaining"})
mode = data.get("mode")
allowed = {None, "transient", "permanent", "duplicate", "before_effect", "after_s3", "after_commit"}
if mode not in allowed:
raise ApiError(400, "invalid_request", "Неизвестный режим fault injection")
if mode is None:
value = {"mode": None, "armed": False}
else:
remaining = data.get("remaining", 1)
if not isinstance(remaining, int) or isinstance(remaining, bool) or remaining < 1:
raise ApiError(400, "invalid_request", "remaining должен быть положительным целым")
value = {"mode": mode, "armed": True, "remaining": remaining}
with connection() as conn:
conn.execute("UPDATE runtime_control SET value = %s WHERE key = 'worker_fault'", (Jsonb(value),))
return respond(200, value)
@app.post("/internal/control/backpressure")
async def set_backpressure(request: Request):
_operator(request)
data = await _json_object(request)
_reject_unknown(data, {"max_queued_jobs"})
value = data.get("max_queued_jobs")
if value is not None and (not isinstance(value, int) or isinstance(value, bool) or value < 0):
raise ApiError(400, "invalid_request", "max_queued_jobs должен быть неотрицательным целым или null")
with connection() as conn:
if value is None:
conn.execute("UPDATE runtime_control SET value = 'null'::jsonb WHERE key = 'max_queued_jobs'")
else:
conn.execute("UPDATE runtime_control SET value = %s WHERE key = 'max_queued_jobs'", (Jsonb(value),))
return respond(200, {"max_queued_jobs": value})
@app.get("/internal/metrics")
async def metrics(request: Request):
_operator(request)
with connection() as conn:
attachments = _counts(conn, "attachments")
jobs = _counts(conn, "jobs")
unpublished = conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL").fetchone()["n"]
failed = conn.execute("SELECT count(*) AS n FROM failed_jobs").fetchone()["n"]
paused = conn.execute("SELECT value FROM runtime_control WHERE key = 'outbox_paused'").fetchone()["value"]
return respond(200, {
"attachments": attachments,
"jobs": jobs,
"outbox_unpublished": unpublished,
"failed_registry": failed,
"outbox_paused": paused is True,
})
@app.get("/internal/attachments/{attachment_id}")
async def internal_attachment(attachment_id: str, request: Request):
_operator(request)
parsed = _parse_uuid(attachment_id)
with connection() as conn:
row = conn.execute("SELECT * FROM attachments WHERE id = %s", (parsed,)).fetchone()
if row is None:
raise ApiError(404, "not_found", "Объект не найден")
jobs = conn.execute("SELECT * FROM jobs WHERE attachment_id = %s ORDER BY pipeline_version", (parsed,)).fetchall()
variants = conn.execute(
"SELECT * FROM attachment_variants WHERE attachment_id = %s ORDER BY pipeline_version, name",
(parsed,),
).fetchall()
audits = conn.execute(
"SELECT id, action, detail, created_at FROM job_audit WHERE attachment_id = %s ORDER BY id",
(parsed,),
).fetchall()
job_ids = [job["job_id"] for job in jobs]
messages = []
if job_ids:
messages = conn.execute(
"SELECT payload FROM outbox WHERE job_id = ANY(%s) ORDER BY id",
(job_ids,),
).fetchall()
return respond(200, {
"attachment": _internal_attachment(row),
"jobs": [_internal_job(job) for job in jobs],
"variants": [_internal_variant(item) for item in variants],
"messages": [item["payload"] for item in messages],
"audits": [{"id": item["id"], "action": item["action"], "detail": item["detail"], "created_at": iso(item["created_at"])} for item in audits],
})
@app.post("/internal/attachments/{attachment_id}/replay")
async def replay_attachment(attachment_id: str, request: Request):
_operator(request)
parsed = _parse_uuid(attachment_id)
with connection() as conn:
row = _lock_attachment(conn, parsed)
job = conn.execute(
"SELECT * FROM jobs WHERE attachment_id = %s AND pipeline_version = %s FOR UPDATE",
(parsed, row["pipeline_version"]),
).fetchone()
if job is None:
raise ApiError(404, "not_found", "Задание не найдено")
if row["status"] == "ready" and row["ready_pipeline_version"] == row["pipeline_version"]:
audit(conn, job["job_id"], parsed, "replay_rejected", {"reason": "already_ready"})
payload = {"replayed": False, "reason": "already_ready", "job_id": str(job["job_id"]), "pipeline_version": row["pipeline_version"]}
elif job["status"] == "processing" and job["lease_until"] is not None and conn.execute("SELECT %s > clock_timestamp() AS active", (job["lease_until"],)).fetchone()["active"]:
payload = {"replayed": False, "reason": "in_progress", "job_id": str(job["job_id"]), "pipeline_version": row["pipeline_version"]}
else:
conn.execute(
"""
UPDATE jobs
SET status = 'queued', attempt = 1, error_code = NULL, error_class = NULL, last_error = NULL,
next_attempt_at = NULL, lease_until = NULL, lease_owner = NULL, updated_at = clock_timestamp()
WHERE job_id = %s
""",
(job["job_id"],),
)
conn.execute(
"""
UPDATE attachments
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
WHERE id = %s AND pipeline_version = %s
""",
(parsed, row["pipeline_version"]),
)
conn.execute(
"UPDATE failed_jobs SET replayed_at = clock_timestamp() WHERE job_id = %s AND replayed_at IS NULL",
(job["job_id"],),
)
add_outbox(conn, job["job_id"], parsed, row["pipeline_version"], 1, job["created_at"])
audit(conn, job["job_id"], parsed, "replay", {"pipeline_version": row["pipeline_version"]})
payload = {"replayed": True, "reason": "queued", "job_id": str(job["job_id"]), "pipeline_version": row["pipeline_version"]}
try:
relay_tick()
except Exception:
log.warning("replay publish deferred")
return respond(200, payload)
@app.post("/internal/attachments/{attachment_id}/reprocess")
async def reprocess_attachment(attachment_id: str, request: Request):
_operator(request)
parsed = _parse_uuid(attachment_id)
data = await _json_object(request)
_reject_unknown(data, {"pipeline_version"})
version = data.get("pipeline_version")
if version not in {1, 2}:
raise ApiError(400, "invalid_request", "Поддерживаются pipeline_version 1 и 2")
with connection() as conn:
row = _lock_attachment(conn, parsed)
if version < int(row["pipeline_version"]):
raise ApiError(409, "conflict", "Нельзя заменить результат более старой версией pipeline")
if version == int(row["pipeline_version"]) and row["status"] == "ready" and row["ready_pipeline_version"] == version:
job = conn.execute(
"SELECT job_id FROM jobs WHERE attachment_id = %s AND pipeline_version = %s",
(parsed, version),
).fetchone()
audit(conn, job["job_id"] if job else None, parsed, "reprocess_rejected", {"reason": "already_ready", "pipeline_version": version})
payload = {"replayed": False, "reason": "already_ready", "job_id": str(job["job_id"]) if job else None, "pipeline_version": version}
else:
job = conn.execute(
"SELECT * FROM jobs WHERE attachment_id = %s AND pipeline_version = %s FOR UPDATE",
(parsed, version),
).fetchone()
if job is None:
job_id = uuid.uuid4()
job = conn.execute(
"""
INSERT INTO jobs (job_id, attachment_id, pipeline_version, attempt, status)
VALUES (%s, %s, %s, 1, 'queued')
RETURNING *
""",
(job_id, parsed, version),
).fetchone()
else:
conn.execute(
"""
UPDATE jobs
SET status = 'queued', attempt = 1, error_code = NULL, error_class = NULL, last_error = NULL,
next_attempt_at = NULL, lease_until = NULL, lease_owner = NULL, updated_at = clock_timestamp()
WHERE job_id = %s
""",
(job["job_id"],),
)
conn.execute(
"""
UPDATE attachments
SET pipeline_version = %s, status = 'queued', error_code = NULL, updated_at = clock_timestamp()
WHERE id = %s
""",
(version, parsed),
)
add_outbox(conn, job["job_id"], parsed, version, 1, job["created_at"])
audit(conn, job["job_id"], parsed, "reprocess", {"pipeline_version": version})
payload = {"replayed": True, "reason": "queued", "job_id": str(job["job_id"]), "pipeline_version": version}
try:
relay_tick()
except Exception:
log.warning("reprocess publish deferred")
return respond(200, payload)
@app.post("/internal/jobs/{job_id}/redeliver")
async def redeliver_job(job_id: str, request: Request):
"""Inject one extra broker delivery of the same logical job without creating a new job."""
_operator(request)
parsed = _parse_uuid(job_id)
with connection() as conn:
job = conn.execute("SELECT * FROM jobs WHERE job_id = %s", (parsed,)).fetchone()
if job is None:
raise ApiError(404, "not_found", "Задание не найдено")
body = {
"schema_version": 1,
"job_id": str(job["job_id"]),
"type": "attachment.process.v1",
"attachment_id": str(job["attachment_id"]),
"pipeline_version": int(job["pipeline_version"]),
"attempt": int(job["attempt"]),
"created_at": iso(job["created_at"]),
}
audit(conn, job["job_id"], job["attachment_id"], "redeliver_requested", {"attempt": int(job["attempt"])})
if not runtime.publisher.publish(body):
raise ApiError(503, "unavailable", "Брокер не подтвердил повторную доставку")
return respond(200, {"enqueued": True, "job_id": str(parsed), "attempt": body["attempt"], "pipeline_version": body["pipeline_version"]})
@app.get("/internal/failed")
async def list_failed(request: Request):
_operator(request)
with connection() as conn:
rows = conn.execute(
"""
SELECT id, job_id, attachment_id, pipeline_version, attempt, error_code, error_class, created_at, replayed_at
FROM failed_jobs ORDER BY id DESC LIMIT 100
"""
).fetchall()
return respond(200, {"items": [
{
"id": row["id"],
"job_id": str(row["job_id"]),
"attachment_id": str(row["attachment_id"]),
"pipeline_version": row["pipeline_version"],
"attempt": row["attempt"],
"error_code": row["error_code"],
"error_class": row["error_class"],
"created_at": iso(row["created_at"]),
"replayed_at": iso(row["replayed_at"]) if row["replayed_at"] else None,
}
for row in rows
]})
return app
async def _json_object(request: Request) -> dict:
raw = await request.body()
content_type = request.headers.get("content-type", "")
if content_type.split(";", 1)[0].strip().lower() != "application/json":
raise ApiError(400, "invalid_request", "Ожидался application/json")
try:
value = json.loads(raw.decode("utf-8"))
except (UnicodeError, json.JSONDecodeError) as exc:
raise ApiError(400, "invalid_request", "Некорректный JSON") from exc
if not isinstance(value, dict):
raise ApiError(400, "invalid_request", "Ожидался JSON-объект")
return value
def _reject_unknown(data: dict, allowed: set[str]) -> None:
if set(data) - allowed:
raise ApiError(400, "invalid_request", "Запрос содержит неизвестные поля")
def _bounded_text(value, minimum: int, maximum: int, field: str) -> str:
if not isinstance(value, str) or "\x00" in value or len(value) < minimum or len(value) > maximum or value.strip() == "":
raise ApiError(400, "invalid_request", "Некорректное поле " + field)
return value
def _username(value) -> str:
if not isinstance(value, str) or USERNAME_RE.fullmatch(value) is None:
raise ApiError(400, "invalid_request", "Некорректное имя пользователя")
return value
def _password(value) -> str:
if not isinstance(value, str) or "\x00" in value or len(value) < 12 or len(value) > 128:
raise ApiError(400, "invalid_request", "Некорректный пароль")
return value
def _parse_uuid(value: str) -> uuid.UUID:
try:
parsed = uuid.UUID(str(value))
except (ValueError, AttributeError, TypeError) as exc:
raise ApiError(400, "invalid_request", "Некорректный UUID") from exc
return parsed
def _uuid_list(value, minimum: int, maximum: int) -> list[uuid.UUID]:
if not isinstance(value, list) or len(value) < minimum or len(value) > maximum:
raise ApiError(400, "invalid_request", "Некорректный список идентификаторов")
parsed = []
for item in value:
if not isinstance(item, str):
raise ApiError(400, "invalid_request", "Некорректный список идентификаторов")
parsed.append(_parse_uuid(item))
if len(set(parsed)) != len(parsed):
raise ApiError(400, "invalid_request", "Список идентификаторов содержит дубликаты")
return parsed
def _idempotency_key(request: Request) -> str:
key = request.headers.get("idempotency-key", "")
if not IDEMPOTENCY_RE.fullmatch(key):
raise ApiError(400, "invalid_request", "Некорректный Idempotency-Key")
return key
def _basic_credentials(request: Request) -> tuple[str, str]:
header = request.headers.get("authorization", "")
if not header.lower().startswith("basic "):
raise unauthorized("basic")
try:
decoded = base64.b64decode(header.split(" ", 1)[1].strip(), validate=True).decode("utf-8")
except Exception as exc:
raise unauthorized("basic") from exc
if ":" not in decoded:
raise unauthorized("basic")
username, password = decoded.split(":", 1)
return username, password
def _bearer(request: Request, conn):
header = request.headers.get("authorization", "")
if not header.startswith("Bearer ") and not header.lower().startswith("bearer "):
raise unauthorized("bearer")
token = header.split(" ", 1)[1].strip()
if token == "":
raise unauthorized("bearer")
token_hash = hashlib.sha256(token.encode("utf-8")).hexdigest()
row = conn.execute(
"""
SELECT u.id, u.username, u.display_name, u.created_at
FROM sessions AS s
JOIN users AS u ON u.id = s.user_id
WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > clock_timestamp()
""",
(token_hash,),
).fetchone()
if row is None:
raise unauthorized("bearer")
return row, token_hash
def _user_json(row) -> dict:
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": iso(row["created_at"]),
}
def _chat_json(conn, row) -> dict:
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY position, user_id",
(row["id"],),
).fetchall()
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": [str(item["user_id"]) for item in members],
"created_at": iso(row["created_at"]),
}
def _require_chat(conn, chat_id: uuid.UUID, user_id):
row = conn.execute(
"""
SELECT c.id, c.title, c.created_at
FROM chats AS c
JOIN chat_members AS m ON m.chat_id = c.id AND m.user_id = %s
WHERE c.id = %s
""",
(user_id, chat_id),
).fetchone()
if row is None:
raise ApiError(404, "not_found", "Объект не найден")
return row
def _message_json(row, attachment_ids) -> dict:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["body"],
"created_at": iso(row["created_at"]),
"attachment_ids": [str(item) for item in attachment_ids],
}
def _attachment_ids(conn, message_id) -> list:
rows = conn.execute(
"SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position",
(message_id,),
).fetchall()
return [row["attachment_id"] for row in rows]
def _claim_attachments(conn, chat_id, attachment_ids: list) -> None:
for attachment_id in attachment_ids:
row = conn.execute("SELECT chat_id, status FROM attachments WHERE id = %s", (attachment_id,)).fetchone()
if row is None or row["chat_id"] != chat_id:
raise ApiError(404, "not_found", "Объект не найден")
if row["status"] == "failed":
raise ApiError(409, "conflict", "Нельзя прикрепить вложение с ошибкой обработки")
def _limit(request: Request) -> int:
raw = request.query_params.get("limit")
if raw is None:
return 50
if not re.fullmatch(r"[0-9]+", raw):
raise ApiError(400, "invalid_request", "Некорректный limit")
value = int(raw)
if value < 1 or value > 100:
raise ApiError(400, "invalid_request", "Некорректный limit")
return value
def _cursor(request: Request, scope: str, owner: str):
raw = request.query_params.get("cursor")
if raw is None:
return None, None
if raw == "":
raise ApiError(400, "invalid_request", "Некорректный cursor")
try:
padding = "=" * (-len(raw) % 4)
payload = json.loads(base64.urlsafe_b64decode(raw + padding).decode("utf-8"))
except Exception as exc:
raise ApiError(400, "invalid_request", "Некорректный cursor") from exc
if not isinstance(payload, dict) or payload.get("scope") != scope or payload.get("owner") != owner:
raise ApiError(400, "invalid_request", "Курсор не относится к этому запросу")
try:
created_at = str(payload["created_at"])
if not created_at.endswith("Z"):
raise ValueError("timestamp")
from datetime import datetime
created = datetime.fromisoformat(created_at[:-1] + "+00:00")
identifier = str(uuid.UUID(str(payload["id"])))
except (KeyError, ValueError, TypeError) as exc:
raise ApiError(400, "invalid_request", "Некорректный cursor") from exc
return created, identifier
def _page(rows, limit: int, cursor_of):
if len(rows) <= limit:
return rows, None
visible = rows[:limit]
payload = cursor_of(visible[-1])
token = base64.urlsafe_b64encode(json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8")).decode("ascii").rstrip("=")
return visible, token
def _filename(value: str | None) -> None:
if not isinstance(value, str) or value == "" or len(value) > 255:
raise ApiError(400, "invalid_request", "Некорректное имя файла")
if any(character in value for character in "/\\") or any(ord(character) < 32 for character in value):
raise ApiError(400, "invalid_request", "Некорректное имя файла")
def _media_type(request: Request) -> str:
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if media not in {"image/png", "image/jpeg"}:
raise ApiError(415, "unsupported_media_type", "Допустимы только PNG и JPEG")
return media
async def _read_body(request: Request, limit: int) -> bytes:
declared = request.headers.get("content-length")
if declared is not None:
if not declared.isdigit() or int(declared) > limit:
raise ApiError(413, "payload_too_large", "Превышен размер файла")
chunks = []
total = 0
async for chunk in request.stream():
total += len(chunk)
if total > limit:
raise ApiError(413, "payload_too_large", "Превышен размер файла")
chunks.append(chunk)
return b"".join(chunks)
def _queue_limit(conn) -> int:
row = conn.execute("SELECT value FROM runtime_control WHERE key = 'max_queued_jobs'").fetchone()
value = row["value"] if row else None
if isinstance(value, int) and not isinstance(value, bool):
return value
return runtime.settings.max_queued_jobs
def _visible_attachment(conn, attachment_id, user_id):
row = conn.execute(
"""
SELECT a.*
FROM attachments AS a
JOIN chat_members AS m ON m.chat_id = a.chat_id AND m.user_id = %s
WHERE a.id = %s
""",
(user_id, attachment_id),
).fetchone()
if row is None:
raise ApiError(404, "not_found", "Объект не найден")
return row
def _visible_variants(conn, attachment) -> list[dict]:
if attachment["status"] != "ready" or attachment["ready_pipeline_version"] is None:
return []
rows = conn.execute(
"""
SELECT name, content_type, size_bytes, width, height, sha256
FROM attachment_variants
WHERE attachment_id = %s AND pipeline_version = %s
ORDER BY name
""",
(attachment["id"], attachment["ready_pipeline_version"]),
).fetchall()
return [
{
"name": row["name"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"width": row["width"],
"height": row["height"],
"sha256": row["sha256"],
}
for row in rows
]
def _attachment_json(row, variants: list[dict]) -> dict:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": row["status"],
"variants": variants,
"error_code": row["error_code"] if row["status"] == "failed" else None,
"created_at": iso(row["created_at"]),
"updated_at": iso(row["updated_at"]),
}
def _operator(request: Request) -> None:
expected = runtime.settings.operator_token
presented = request.headers.get("x-operator-token", "")
if expected == "" or not hmac.compare_digest(
hashlib.sha256(presented.encode("utf-8")).digest(),
hashlib.sha256(expected.encode("utf-8")).digest(),
):
raise unauthorized("bearer")
def _lock_attachment(conn, attachment_id):
row = conn.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone()
if row is None:
raise ApiError(404, "not_found", "Объект не найден")
return row
def _counts(conn, table: str) -> dict:
rows = conn.execute(f"SELECT status, count(*) AS n FROM {table} GROUP BY status").fetchall()
return {row["status"]: row["n"] for row in rows}
def _internal_attachment(row) -> dict:
return {
"id": str(row["id"]),
"status": row["status"],
"error_code": row["error_code"],
"pipeline_version": row["pipeline_version"],
"ready_pipeline_version": row["ready_pipeline_version"],
"sha256": row["sha256"],
"size_bytes": row["size_bytes"],
"staging_deleted": row["staging_deleted_at"] is not None,
"created_at": iso(row["created_at"]),
"updated_at": iso(row["updated_at"]),
}
def _internal_job(row) -> dict:
return {
"job_id": str(row["job_id"]),
"pipeline_version": row["pipeline_version"],
"attempt": row["attempt"],
"status": row["status"],
"error_code": row["error_code"],
"error_class": row["error_class"],
"created_at": iso(row["created_at"]),
}
def _internal_variant(row) -> dict:
return {
"pipeline_version": row["pipeline_version"],
"name": row["name"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"width": row["width"],
"height": row["height"],
"sha256": row["sha256"],
}