Отделить приём вложений от обработки через очередь и worker.

This commit is contained in:
samsik
2026-11-20 01:30:00 +03:00
parent 276dbdad8a
commit b92fa218ba
40 changed files with 4054 additions and 33 deletions
+112
View File
@@ -0,0 +1,112 @@
"""RabbitMQ: durable quorum-очереди и publisher confirms.
Сообщение содержит только идентификаторы задания. Байты и ключи S3 сюда не входят.
Ack потребителя и confirm издателя — разные границы.
"""
from __future__ import annotations
import json
import os
import threading
from pathlib import Path
import pika
EXCHANGE = "attachments"
PROCESS_QUEUE = "attachment.process"
FAILED_QUEUE = "attachment.failed"
PROCESS_KEY = "process"
FAILED_KEY = "failed"
_lock = threading.Lock()
_connection: pika.BlockingConnection | None = None
_channel: pika.adapters.blocking_connection.BlockingChannel | None = None
def broker_url() -> str:
path = os.environ.get("BROKER_URL_FILE", "").strip()
if path and Path(path).is_file():
return Path(path).read_text(encoding="utf-8").strip()
return os.environ.get("BROKER_URL", "").strip()
def declare(channel) -> None:
channel.exchange_declare(EXCHANGE, exchange_type="direct", durable=True)
arguments = {"x-queue-type": "quorum"}
channel.queue_declare(PROCESS_QUEUE, durable=True, arguments=arguments)
channel.queue_declare(FAILED_QUEUE, durable=True, arguments=arguments)
channel.queue_bind(PROCESS_QUEUE, EXCHANGE, PROCESS_KEY)
channel.queue_bind(FAILED_QUEUE, EXCHANGE, FAILED_KEY)
def connect() -> tuple[pika.BlockingConnection, pika.adapters.blocking_connection.BlockingChannel]:
url = broker_url()
if not url:
raise RuntimeError("broker is not configured")
params = pika.URLParameters(url)
params.heartbeat = 60
params.blocked_connection_timeout = 15
params.socket_timeout = 10
params.connection_attempts = 3
params.retry_delay = 1
connection = pika.BlockingConnection(params)
channel = connection.channel()
channel.confirm_delivery()
declare(channel)
return connection, channel
def _ensure():
global _connection, _channel
if _connection is not None and _connection.is_open and _channel is not None and _channel.is_open:
return _channel
if _connection is not None:
try:
_connection.close()
except Exception:
pass
_connection, _channel = connect()
return _channel
def publish_job(payload: dict, routing_key: str = PROCESS_KEY) -> None:
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
if len(body) > 4096:
raise RuntimeError("job message exceeds 4096 bytes")
properties = pika.BasicProperties(
content_type="application/json",
delivery_mode=2,
message_id=str(payload.get("job_id", "")),
type=str(payload.get("type", "")),
)
with _lock:
channel = _ensure()
try:
channel.basic_publish(
EXCHANGE,
routing_key,
body,
properties,
mandatory=True,
)
except Exception:
global _connection, _channel
_connection = None
_channel = None
raise
def publish_on(channel, payload: dict, routing_key: str) -> None:
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
channel.basic_publish(
EXCHANGE,
routing_key,
body,
pika.BasicProperties(
content_type="application/json",
delivery_mode=2,
message_id=str(payload.get("job_id", "")),
type=str(payload.get("type", "")),
),
mandatory=True,
)
+1258
View File
File diff suppressed because it is too large Load Diff
+8
View File
@@ -0,0 +1,8 @@
"""Ошибки API. Текст не содержит SQL, паролей и токенов."""
class ApiError(Exception):
def __init__(self, status: int, code: str, message: str):
self.status = status
self.code = code
self.message = message
+74
View File
@@ -0,0 +1,74 @@
"""Размеры PNG/JPEG по заголовкам, без распаковки пикселей."""
from __future__ import annotations
from errors import ApiError
PNG_SIG = b"\x89PNG\r\n\x1a\n"
# SOF0..SOF3, SOF5..SOF7, SOF9..SOF11, SOF13..SOF15. Не DHT/JPG/DAC.
_JPEG_SOF = {
0xC0, 0xC1, 0xC2, 0xC3,
0xC5, 0xC6, 0xC7,
0xC9, 0xCA, 0xCB,
0xCD, 0xCE, 0xCF,
}
def sniff(data: bytes) -> str:
if data.startswith(PNG_SIG):
return "image/png"
if data.startswith(b"\xff\xd8\xff"):
return "image/jpeg"
raise ApiError(415, "unsupported_media_type", "Файл не PNG и не JPEG")
def dimensions(data: bytes, content_type: str) -> tuple[int, int]:
if content_type == "image/png":
return _png_size(data)
if content_type == "image/jpeg":
return _jpeg_size(data)
raise ApiError(415, "unsupported_media_type", "Файл не PNG и не JPEG")
def _png_size(data: bytes) -> tuple[int, int]:
if len(data) < 24 or not data.startswith(PNG_SIG) or data[12:16] != b"IHDR":
raise ApiError(415, "unsupported_media_type", "Повреждённый PNG")
width = int.from_bytes(data[16:20], "big")
height = int.from_bytes(data[20:24], "big")
if width < 1 or height < 1:
raise ApiError(415, "unsupported_media_type", "Повреждённый PNG")
return width, height
def _jpeg_size(data: bytes) -> tuple[int, int]:
if len(data) < 4 or data[:2] != b"\xff\xd8":
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
index = 2
size = len(data)
while index < size:
if data[index] != 0xFF:
break
while index < size and data[index] == 0xFF:
index += 1
if index >= size:
break
marker = data[index]
index += 1
if marker in (0xD8, 0xD9) or marker == 0xDA:
break
if marker == 0x01 or 0xD0 <= marker <= 0xD7:
continue
if index + 2 > size:
break
segment = int.from_bytes(data[index:index + 2], "big")
if segment < 2 or index + segment > size:
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
if marker in _JPEG_SOF:
if segment < 7:
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
height = int.from_bytes(data[index + 3:index + 5], "big")
width = int.from_bytes(data[index + 5:index + 7], "big")
if width < 1 or height < 1:
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
return width, height
index += segment
raise ApiError(415, "unsupported_media_type", "В JPEG нет размеров")
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env python3
"""Однократные миграции. Реплики API этот файл не запускают."""
import sys
import db
def main() -> None:
db.migrate_with_retry()
sys.stderr.write("migrations applied\n")
if __name__ == "__main__":
main()
+43
View File
@@ -0,0 +1,43 @@
"""Пароли: scrypt из hashlib. Не bcrypt — он режет пароль на 72 байтах."""
from __future__ import annotations
import base64
import hashlib
import hmac
import secrets
N = 2**14
R = 8
P = 1
DKLEN = 32
def hash_password(password: str) -> str:
salt = secrets.token_bytes(16)
digest = hashlib.scrypt(
password.encode("utf-8"), salt=salt, n=N, r=R, p=P, dklen=DKLEN,
maxmem=64 * 1024 * 1024)
return "scrypt$%d$%d$%d$%s$%s" % (
N, R, P,
base64.b64encode(salt).decode("ascii"),
base64.b64encode(digest).decode("ascii"),
)
def verify_password(password: str, stored: str) -> bool:
try:
algo, n, r, p, salt_b64, digest_b64 = stored.split("$")
if algo != "scrypt":
return False
salt = base64.b64decode(salt_b64)
expected = base64.b64decode(digest_b64)
actual = hashlib.scrypt(
password.encode("utf-8"), salt=salt, n=int(n), r=int(r), p=int(p),
dklen=len(expected), maxmem=64 * 1024 * 1024)
except (ValueError, TypeError):
return False
return hmac.compare_digest(actual, expected)
# Одинаковая стоимость проверки, когда пользователя нет.
DUMMY_HASH = hash_password("dummy-password-not-used")
+24
View File
@@ -0,0 +1,24 @@
"""Доставка outbox в брокер. Строка остаётся неопубликованной, пока нет confirm."""
from __future__ import annotations
import os
import sys
import broker
import db
def publish_batch(limit: int = 20) -> int:
if os.environ.get("OUTBOX_PUBLISH_ENABLED", "1") == "0":
return 0
rows = db.take_outbox(limit)
published = 0
for outbox_id, payload in rows:
try:
broker.publish_job(payload)
except Exception:
sys.stderr.write("broker did not confirm job %s\n" % payload.get("job_id", ""))
return published
db.mark_outbox_published(outbox_id)
published += 1
return published
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env python3
"""Однократное создание приватного bucket. Повторный запуск ничего не публикует."""
import sys
import time
import storage
def main() -> None:
storage.init()
last = None
for _ in range(30):
try:
storage.ensure_bucket()
sys.stderr.write("bucket ready\n")
return
except Exception as exc:
last = exc
time.sleep(1)
sys.stderr.write("bucket bootstrap failed\n")
raise SystemExit(1) from last
if __name__ == "__main__":
main()
+730
View File
@@ -0,0 +1,730 @@
#!/usr/bin/env python3
"""Messenger lab5. Приём файла отделён от обработки: staging, outbox, брокер, worker."""
from __future__ import annotations
import base64
import hashlib
import json
import os
import re
import secrets
import signal
import socket
import socketserver
import sys
import threading
import time
import unicodedata
import uuid
from datetime import datetime, timedelta, timezone
from http.server import BaseHTTPRequestHandler
from urllib.parse import parse_qs, urlparse
import db
import images
import passwords
import sessions
import storage
from errors import ApiError
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
APP_PORT = int(os.environ.get("APP_PORT", "8080"))
APP_ENV = os.environ.get("APP_ENV", "lab")
INSTANCE_ID = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname()
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}\Z")
UUID_RE = re.compile(
r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\Z")
TIMESTAMP_RE = re.compile(
r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?Z\Z")
IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}\Z")
MAX_BODY = 1024 * 1024
_draining = threading.Event()
_inflight = 0
_inflight_lock = threading.Lock()
def drain_seconds() -> float:
raw = os.environ.get("DRAIN_SECONDS", "3")
try:
value = float(raw)
except ValueError:
return 3.0
if value < 0:
return 0.0
if value > 6:
return 6.0
return value
def now_utc() -> datetime:
current = datetime.now(timezone.utc)
return current.replace(microsecond=(current.microsecond // 1000) * 1000)
def is_blank(value: str) -> bool:
return value.strip() == ""
def require_object(body):
if not isinstance(body, dict):
raise ApiError(400, "invalid_request", "Тело должно быть JSON-объектом")
def reject_extra(body: dict, allowed: set):
extra = set(body) - allowed
if extra:
raise ApiError(400, "invalid_request",
"Лишние поля: " + ", ".join(sorted(extra)))
def parse_uuid(value, field: str) -> str:
if not isinstance(value, str) or not UUID_RE.match(value):
raise ApiError(400, "invalid_request", f"Некорректный UUID: {field}")
return value.lower()
def parse_display_name(value) -> str:
if not isinstance(value, str):
raise ApiError(400, "invalid_request", "display_name должен быть строкой")
if not (1 <= len(value) <= 100):
raise ApiError(400, "invalid_request", "display_name: 1..100 code points")
if is_blank(value):
raise ApiError(400, "invalid_request", "display_name не может быть пустым")
return value
def parse_username(value) -> str:
if not isinstance(value, str) or not USERNAME_RE.match(value):
raise ApiError(400, "invalid_request", "username: ^[a-z0-9_]{3,32}$")
return value
def parse_password(value) -> str:
if not isinstance(value, str):
raise ApiError(400, "invalid_request", "password должен быть строкой")
if not (12 <= len(value) <= 128):
raise ApiError(400, "invalid_request", "password: 12..128 code points")
if is_blank(value):
raise ApiError(400, "invalid_request", "password не может быть пустым")
return value
def parse_title(value) -> str:
if not isinstance(value, str):
raise ApiError(400, "invalid_request", "title должен быть строкой")
if not (1 <= len(value) <= 100):
raise ApiError(400, "invalid_request", "title: 1..100 code points")
if is_blank(value):
raise ApiError(400, "invalid_request", "title не может быть пустым")
return value
def max_upload_bytes() -> int:
raw = os.environ.get("MAX_UPLOAD_BYTES", "10485760")
try:
value = int(raw)
except ValueError:
return 10485760
return value if value >= 1 else 10485760
def max_image_pixels() -> int:
raw = os.environ.get("MAX_IMAGE_PIXELS", "40000000")
try:
value = int(raw)
except ValueError:
return 40000000
return value if value >= 1 else 40000000
def parse_text(value, has_attachments: bool) -> str:
if not isinstance(value, str):
raise ApiError(400, "invalid_request", "text должен быть строкой")
if len(value) > 4000:
raise ApiError(400, "invalid_request", "text: не больше 4000 code points")
if is_blank(value) and not has_attachments:
raise ApiError(400, "invalid_request", "text не может быть пустым без вложений")
return value
def parse_attachment_ids(value) -> list:
if value is None:
return []
if not isinstance(value, list):
raise ApiError(400, "invalid_request", "attachment_ids должен быть массивом")
if len(value) > 10:
raise ApiError(400, "invalid_request", "attachment_ids: не больше 10")
seen = set()
result = []
for item in value:
att_id = parse_uuid(item, "attachment_ids[]")
if att_id in seen:
raise ApiError(400, "invalid_request", "attachment_ids содержит дубликаты")
seen.add(att_id)
result.append(att_id)
return result
def parse_filename(value) -> str:
if not isinstance(value, str):
raise ApiError(400, "invalid_request", "filename: 1..255 code points")
if not (1 <= len(value) <= 255):
raise ApiError(400, "invalid_request", "filename: 1..255 code points")
for char in value:
if char in "/\\" or unicodedata.category(char) == "Cc":
raise ApiError(400, "invalid_request", "filename содержит запрещённый символ")
return value
def parse_member_ids(value) -> list:
if not isinstance(value, list):
raise ApiError(400, "invalid_request", "member_ids должен быть массивом")
if not (1 <= len(value) <= 100):
raise ApiError(400, "invalid_request", "member_ids: 1..100")
seen = set()
result = []
for item in value:
uid = parse_uuid(item, "member_ids[]")
if uid in seen:
raise ApiError(400, "invalid_request", "member_ids содержит дубликаты")
seen.add(uid)
result.append(uid)
return result
def parse_limit(raw, default=50) -> int:
if raw is None:
return default
if isinstance(raw, str) and raw.strip() == "":
raise ApiError(400, "invalid_request", "limit должен быть целым")
try:
value = int(raw)
except (TypeError, ValueError):
raise ApiError(400, "invalid_request", "limit должен быть целым")
if not (1 <= value <= 100):
raise ApiError(400, "invalid_request", "limit: 1..100")
return value
def parse_optional_cursor(raw):
if raw is None:
return None
if not isinstance(raw, str) or raw.strip() == "":
raise ApiError(400, "invalid_request", "Некорректный cursor")
return raw
def parse_cursor_timestamp(value: str) -> datetime:
if not isinstance(value, str) or not TIMESTAMP_RE.match(value):
raise ApiError(400, "invalid_request", "Некорректный cursor")
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as exc:
raise ApiError(400, "invalid_request", "Некорректный cursor") from exc
return parsed
def parse_idempotency_key(headers) -> str:
raw = headers.get("Idempotency-Key")
if not isinstance(raw, str) or not IDEMPOTENCY_RE.match(raw):
raise ApiError(400, "invalid_request", "Некорректный Idempotency-Key")
return raw
def canonical_hash(text: str, attachment_ids: list) -> str:
# Порядок ключей JSON не важен. Отсутствующий attachment_ids равен [].
# Порядок элементов массива сохраняется: это часть тела запроса.
payload = {"attachment_ids": attachment_ids, "text": text}
raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
def challenge_for(path: str) -> str:
if path == "/api/v1/auth/sessions":
return 'Basic realm="messenger", charset="UTF-8"'
return 'Bearer realm="messenger"'
def bearer_token(headers) -> str:
auth = headers.get("Authorization") or ""
parts = auth.split(None, 1)
if len(parts) != 2 or parts[0].lower() != "bearer" or not parts[1].strip():
raise ApiError(401, "unauthorized", "Требуется Bearer")
return parts[1].strip()
def current_user(headers) -> dict:
token = bearer_token(headers)
user = db.get_user(sessions.user_id(token))
if user is None:
raise ApiError(401, "unauthorized", "Сессия недействительна")
return user
def basic_credentials(headers) -> tuple[str, str]:
auth = headers.get("Authorization") or ""
parts = auth.split(None, 1)
if len(parts) != 2 or parts[0].lower() != "basic" or not parts[1].strip():
raise ApiError(401, "unauthorized", "Требуется Basic")
try:
decoded = base64.b64decode(parts[1].strip(), validate=True).decode("utf-8")
except (ValueError, UnicodeDecodeError) as exc:
raise ApiError(401, "unauthorized", "Неверные учётные данные") from exc
if ":" not in decoded:
raise ApiError(401, "unauthorized", "Неверные учётные данные")
username, password = decoded.split(":", 1)
return username, password
def encode_cursor(chat_id: str, created_at: str, msg_id: str) -> str:
raw = f"{chat_id}|{created_at}|{msg_id}".encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
def decode_cursor(cursor: str, chat_id: str):
pad = "=" * (-len(cursor) % 4)
try:
raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8")
except Exception as exc:
raise ApiError(400, "invalid_request", "Некорректный cursor") from exc
parts = raw.split("|")
if len(parts) != 3:
raise ApiError(400, "invalid_request", "Некорректный cursor")
c_chat, c_created, c_id = parts
if not UUID_RE.match(c_chat):
raise ApiError(400, "invalid_request", "Некорректный cursor")
if c_chat.lower() != chat_id:
raise ApiError(400, "invalid_request", "cursor от другого чата")
return parse_cursor_timestamp(c_created), parse_uuid(c_id, "cursor")
def encode_chat_cursor(created_at: str, chat_id: str) -> str:
raw = f"chat|{created_at}|{chat_id}".encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
def decode_chat_cursor(cursor: str):
pad = "=" * (-len(cursor) % 4)
try:
raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8")
except Exception as exc:
raise ApiError(400, "invalid_request", "Некорректный cursor") from exc
parts = raw.split("|")
if len(parts) != 3 or parts[0] != "chat":
raise ApiError(400, "invalid_request", "Некорректный cursor")
return parse_cursor_timestamp(parts[1]), parse_uuid(parts[2], "cursor")
def create_user(body) -> dict:
require_object(body)
reject_extra(body, {"username", "display_name", "password"})
username = parse_username(body.get("username"))
display_name = parse_display_name(body.get("display_name"))
password = parse_password(body.get("password"))
return db.create_user(username, display_name, passwords.hash_password(password), now_utc())
def create_session(headers) -> dict:
username, password = basic_credentials(headers)
row = None
try:
row = db.user_for_login(username)
except ApiError:
raise
stored = row[3] if row is not None else passwords.DUMMY_HASH
password_ok = passwords.verify_password(password, stored)
if row is None or not password_ok:
raise ApiError(401, "unauthorized", "Неверные учётные данные")
token = secrets.token_urlsafe(32)
ttl = sessions.session_ttl()
expires = now_utc() + timedelta(seconds=ttl)
sessions.create(token, str(row[0]), ttl)
return {
"token": token,
"token_type": "Bearer",
"expires_at": db.format_ts(expires),
"user": db.user_json((row[0], row[1], row[2], row[4])),
}
def logout(headers) -> None:
token = bearer_token(headers)
sessions.user_id(token)
sessions.revoke(token)
def create_chat(body, actor: dict) -> dict:
require_object(body)
reject_extra(body, {"title", "member_ids"})
title = parse_title(body.get("title"))
member_ids = parse_member_ids(body.get("member_ids"))
if actor["id"] not in member_ids:
raise ApiError(400, "invalid_request", "вызывающий должен быть в member_ids")
return db.create_chat(title, member_ids, now_utc())
def list_chats(actor: dict, limit: int, cursor) -> dict:
decoded = decode_chat_cursor(cursor) if cursor else None
page = db.list_chats(actor["id"], limit, decoded)
next_cursor = None
if page["next_cursor_parts"] is not None:
created, chat_id = page["next_cursor_parts"]
next_cursor = encode_chat_cursor(db.format_ts(created), chat_id)
return {"items": page["items"], "next_cursor": next_cursor}
def create_message(chat_id: str, body, actor: dict, idem_key: str) -> dict:
cid = parse_uuid(chat_id, "chat_id")
require_object(body)
reject_extra(body, {"text", "attachment_ids"})
attachment_ids = parse_attachment_ids(body.get("attachment_ids"))
text = parse_text(body.get("text"), bool(attachment_ids))
path = "/api/v1/chats/%s/messages" % cid
return db.create_message_idempotent(
cid,
actor["id"],
text,
attachment_ids,
now_utc(),
"POST",
path,
idem_key,
canonical_hash(text, attachment_ids),
)
def upload_attachment(chat_id: str, actor: dict, filename: str, declared: str, data: bytes) -> dict:
detected = images.sniff(data)
if detected != declared:
raise ApiError(415, "unsupported_media_type", "Содержимое не совпадает с Content-Type")
width, height = images.dimensions(data, detected)
if width * height > max_image_pixels():
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
att_id = str(uuid.uuid4())
created = now_utc()
staging_key = "staging/" + att_id
object_key = "originals/" + att_id
payload = {
"schema_version": 1,
"job_id": str(uuid.uuid4()),
"type": "attachment.process.v1",
"attachment_id": att_id,
"pipeline_version": 1,
"attempt": 1,
"created_at": db.format_ts(created),
}
storage.put_bytes(staging_key, data, detected)
try:
return db.insert_queued_attachment(
att_id, chat_id, actor["id"], filename, detected, len(data),
hashlib.sha256(data).hexdigest(), object_key, staging_key,
width, height, created, payload,
)
except Exception:
storage.delete_quiet(staging_key)
raise
def read_attachment(attachment_id: str, actor: dict) -> dict:
return db.get_attachment(parse_uuid(attachment_id, "attachment_id"), actor["id"])
def read_attachment_bytes(attachment_id: str, actor: dict, variant: str):
if variant not in ("original", "thumbnail", "crop", "optimized"):
raise ApiError(400, "invalid_request", "Некорректный variant")
record = db.get_download(parse_uuid(attachment_id, "attachment_id"), actor["id"], variant)
data = storage.get_bytes(record["object_key"])
if len(data) != record["size_bytes"] or hashlib.sha256(data).hexdigest() != record["sha256"]:
raise ApiError(503, "unavailable", "Объект хранилища не совпал с метаданными")
return record["content_type"], data
def list_messages(chat_id: str, actor: dict, limit: int, cursor) -> dict:
cid = parse_uuid(chat_id, "chat_id")
decoded = decode_cursor(cursor, cid) if cursor else None
page = db.list_messages(cid, actor["id"], limit, decoded)
next_cursor = None
if page["next_cursor_parts"] is not None:
created, message_id = page["next_cursor_parts"]
next_cursor = encode_cursor(cid, db.format_ts(created), message_id)
return {"items": page["items"], "next_cursor": next_cursor}
class FastThreadingHTTPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
daemon_threads = True
block_on_close = False
allow_reuse_address = True
request_queue_size = 256
class Handler(BaseHTTPRequestHandler):
server_version = "mtusi-messenger/5.0"
protocol_version = "HTTP/1.0"
def log_message(self, fmt, *args):
return
def _request_id(self) -> str:
return self.headers.get("X-Request-Id") or str(uuid.uuid4())
def _send_json(self, status: int, payload, request_id: str, extra_headers=None):
body = b""
if status != 204:
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
self.send_response(status)
if status != 204:
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
self.send_header("Connection", "close")
for key, value in (extra_headers or {}).items():
self.send_header(key, value)
self.end_headers()
if body:
self.wfile.write(body)
def _send_bytes(self, content_type: str, payload: bytes, request_id: str):
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(payload)))
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
self.send_header("Connection", "close")
self.end_headers()
if payload:
self.wfile.write(payload)
def _send_error(self, status: int, code: str, message: str, request_id: str, extra_headers=None):
self._send_json(
status,
{"error": {"code": code, "message": message, "request_id": request_id}},
request_id,
extra_headers,
)
def _read_body(self):
length = self.headers.get("Content-Length")
if not length:
return None
try:
size = int(length)
except ValueError as exc:
raise ApiError(400, "invalid_request", "Некорректный Content-Length") from exc
if size > MAX_BODY:
raise ApiError(413, "payload_too_large", "Тело слишком большое")
raw = self.rfile.read(size)
ctype = (self.headers.get("Content-Type") or "").split(";")[0].strip()
if ctype and ctype != "application/json":
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
if not raw:
raise ApiError(400, "invalid_request", "Пустое тело")
try:
return json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise ApiError(400, "invalid_request", "Malformed JSON") from exc
def _read_upload(self) -> tuple[str, bytes]:
length = self.headers.get("Content-Length")
if length is None:
raise ApiError(400, "invalid_request", "Нужен Content-Length")
try:
size = int(length)
except ValueError as exc:
raise ApiError(400, "invalid_request", "Некорректный Content-Length") from exc
if size > max_upload_bytes():
raise ApiError(413, "payload_too_large", "Файл больше допустимого размера")
if size < 1:
raise ApiError(415, "unsupported_media_type", "Пустое тело")
declared = (self.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower()
if declared not in ("image/png", "image/jpeg"):
raise ApiError(415, "unsupported_media_type", "Ожидается image/png или image/jpeg")
data = self.rfile.read(size)
if len(data) != size:
raise ApiError(400, "invalid_request", "Тело короче Content-Length")
return declared, data
def do_GET(self):
self._dispatch("GET")
def do_POST(self):
self._dispatch("POST")
def do_DELETE(self):
self._dispatch("DELETE")
def _dispatch(self, method: str):
global _inflight
with _inflight_lock:
_inflight += 1
rid = self._request_id()
parsed = urlparse(self.path)
path = parsed.path.rstrip("/") or "/"
query = parse_qs(parsed.query, keep_blank_values=True)
try:
self._route(method, path, query, rid)
except ApiError as exc:
headers = {}
if exc.status == 401:
headers["WWW-Authenticate"] = challenge_for(path)
self._send_error(exc.status, exc.code, exc.message, rid, headers)
except BrokenPipeError:
pass
except Exception:
sys.stderr.write("internal error on %s %s\n" % (method, path))
self._send_error(500, "internal_error", "Внутренняя ошибка", rid)
finally:
with _inflight_lock:
_inflight -= 1
def _route(self, method: str, path: str, query: dict, rid: str):
if method == "GET" and path == "/health/live":
return self._send_json(200, {"status": "ok"}, rid)
if method == "GET" and path == "/health/ready":
if _draining.is_set():
raise ApiError(503, "unavailable", "Реплика останавливается")
if db.ping() and sessions.ping() and storage.ping():
return self._send_json(200, {"status": "ok"}, rid)
raise ApiError(503, "unavailable", "Зависимость недоступна")
if _draining.is_set():
raise ApiError(503, "unavailable", "Реплика останавливается")
if method == "POST" and path == "/api/v1/users":
return self._send_json(201, create_user(self._read_body()), rid)
if method == "POST" and path == "/api/v1/auth/sessions":
return self._send_json(201, create_session(self.headers), rid)
if method == "DELETE" and path == "/api/v1/auth/sessions/current":
logout(self.headers)
return self._send_json(204, None, rid)
if method == "GET" and path == "/api/v1/users/me":
return self._send_json(200, current_user(self.headers), rid)
if method == "POST" and path == "/api/v1/chats":
actor = current_user(self.headers)
return self._send_json(201, create_chat(self._read_body(), actor), rid)
if method == "GET" and path == "/api/v1/chats":
actor = current_user(self.headers)
limit = parse_limit(query.get("limit", [None])[0])
cursor = parse_optional_cursor(query.get("cursor", [None])[0])
return self._send_json(200, list_chats(actor, limit, cursor), rid)
match = re.fullmatch(r"/api/v1/chats/([^/]+)", path)
if match and method == "GET":
actor = current_user(self.headers)
return self._send_json(200, db.get_chat(parse_uuid(match.group(1), "chat_id"), actor["id"]), rid)
match = re.fullmatch(r"/api/v1/chats/([^/]+)/attachments", path)
if match and method == "POST":
actor = current_user(self.headers)
chat_id = parse_uuid(match.group(1), "chat_id")
if not db.is_member(chat_id, actor["id"]):
raise ApiError(404, "not_found", "Чат не найден")
names = query.get("filename") or []
if len(names) != 1:
raise ApiError(400, "invalid_request", "Нужен один filename")
filename = parse_filename(names[0])
declared, data = self._read_upload()
body = upload_attachment(chat_id, actor, filename, declared, data)
return self._send_json(202, body, rid)
match = re.fullmatch(r"/api/v1/attachments/([^/]+)/replay", path)
if match and method == "POST":
actor = current_user(self.headers)
att_id = parse_uuid(match.group(1), "attachment_id")
body = db.replay_attachment(att_id, actor["id"])
status = 200 if body["status"] == "ready" else 202
return self._send_json(status, body, rid)
match = re.fullmatch(r"/api/v1/attachments/([^/]+)/content", path)
if match and method == "GET":
actor = current_user(self.headers)
variants = query.get("variant") or ["original"]
if len(variants) != 1 or not isinstance(variants[0], str) or variants[0] == "":
raise ApiError(400, "invalid_request", "Некорректный variant")
content_type, payload = read_attachment_bytes(match.group(1), actor, variants[0])
return self._send_bytes(content_type, payload, rid)
match = re.fullmatch(r"/api/v1/attachments/([^/]+)", path)
if match and method == "GET":
actor = current_user(self.headers)
return self._send_json(200, read_attachment(match.group(1), actor), rid)
match = re.fullmatch(r"/api/v1/chats/([^/]+)/messages", path)
if match and method == "POST":
actor = current_user(self.headers)
idem_key = parse_idempotency_key(self.headers)
body = create_message(match.group(1), self._read_body(), actor, idem_key)
return self._send_json(201, body, rid)
if match and method == "GET":
actor = current_user(self.headers)
limit = parse_limit(query.get("limit", [None])[0])
cursor = parse_optional_cursor(query.get("cursor", [None])[0])
return self._send_json(200, list_messages(match.group(1), actor, limit, cursor), rid)
raise ApiError(404, "not_found", "Маршрут не найден")
def _inflight_now() -> int:
with _inflight_lock:
return _inflight
def _publish_outbox() -> None:
import relay
while not _draining.is_set():
try:
relay.publish_batch()
except Exception:
sys.stderr.write("outbox publish failed\n")
_draining.wait(0.4)
def main():
try:
db.init()
except Exception:
sys.stderr.write("database is not configured\n")
try:
sessions.init()
except Exception:
sys.stderr.write("session store is not configured\n")
try:
storage.init()
except Exception:
sys.stderr.write("object storage is not configured\n")
threading.Thread(target=_publish_outbox, name="outbox", daemon=True).start()
server = FastThreadingHTTPServer((APP_HOST, APP_PORT), Handler)
def drain_and_stop():
time.sleep(drain_seconds())
deadline = time.monotonic() + 2
while time.monotonic() < deadline and _inflight_now() > 0:
time.sleep(0.05)
server.shutdown()
def shutdown(signum, frame):
if _draining.is_set():
return
_draining.set()
sys.stderr.write("SIGTERM instance=%s stop accepting\n" % INSTANCE_ID)
threading.Thread(target=drain_and_stop, daemon=True).start()
signal.signal(signal.SIGTERM, shutdown)
signal.signal(signal.SIGINT, shutdown)
sys.stderr.write(
"listening on %s:%s env=%s instance=%s\n" % (APP_HOST, APP_PORT, APP_ENV, INSTANCE_ID)
)
try:
server.serve_forever(poll_interval=0.1)
finally:
server.server_close()
db.close()
sys.stderr.write("stopped instance=%s\n" % INSTANCE_ID)
if __name__ == "__main__":
main()
+85
View File
@@ -0,0 +1,85 @@
"""Сессии в Valkey. Ключ — SHA-256 токена, TTL равен остатку жизни сессии.
Источник истины сессий — общий Valkey, не память процесса API.
AOF и volume сохраняют ключ при перезапуске хранилища.
"""
from __future__ import annotations
import hashlib
import os
from pathlib import Path
import redis
from errors import ApiError
_client: redis.Redis | None = None
def session_ttl() -> int:
raw = os.environ.get("SESSION_TTL_SECONDS", "3600")
try:
value = int(raw)
except ValueError:
return 3600
return value if value >= 1 else 1
def redis_url() -> str:
path = os.environ.get("CACHE_URL_FILE", "").strip()
if path and Path(path).is_file():
return Path(path).read_text(encoding="utf-8").strip()
return os.environ.get("REDIS_URL", "").strip()
def init() -> None:
global _client
url = redis_url()
if not url:
_client = None
return
_client = redis.Redis.from_url(
url, socket_connect_timeout=0.5, socket_timeout=0.5,
decode_responses=True)
def _key(token: str) -> str:
digest = hashlib.sha256(token.encode("utf-8")).hexdigest()
return "session:" + digest
def _require() -> redis.Redis:
if _client is None:
raise ApiError(503, "unavailable", "Хранилище сессий недоступно")
return _client
def ping() -> bool:
try:
return bool(_require().ping())
except (ApiError, redis.RedisError):
return False
def create(token: str, user_id: str, ttl: int) -> None:
try:
_require().set(_key(token), user_id, ex=ttl)
except (ApiError, redis.RedisError) as exc:
raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc
def user_id(token: str) -> str:
try:
value = _require().get(_key(token))
except (ApiError, redis.RedisError) as exc:
raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc
if not value:
raise ApiError(401, "unauthorized", "Сессия недействительна")
return value
def revoke(token: str) -> None:
try:
_require().delete(_key(token))
except (ApiError, redis.RedisError) as exc:
raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc
+70
View File
@@ -0,0 +1,70 @@
"""Численные пределы worker. Секреты здесь не читаются."""
from __future__ import annotations
import os
import random
def _int(name: str, default: int, low: int, high: int) -> int:
try:
value = int(os.environ.get(name, str(default)))
except ValueError:
return default
if value < low:
return low
if value > high:
return high
return value
def _float(name: str, default: float, low: float, high: float) -> float:
try:
value = float(os.environ.get(name, str(default)))
except ValueError:
return default
if value < low:
return low
if value > high:
return high
return value
def max_attempts() -> int:
return _int("JOB_MAX_ATTEMPTS", 5, 1, 20)
def job_timeout() -> int:
return _int("JOB_TIMEOUT_SECONDS", 60, 5, 600)
def worker_concurrency() -> int:
return _int("WORKER_CONCURRENCY", 2, 1, 8)
def max_image_pixels() -> int:
return _int("MAX_IMAGE_PIXELS", 40000000, 1, 40000000)
def max_upload_bytes() -> int:
return _int("MAX_UPLOAD_BYTES", 10485760, 1, 10485760)
def staging_retention() -> int:
return _int("STAGING_RETENTION_SECONDS", 86400, 60, 7 * 86400)
def jpeg_quality() -> int:
return _int("JPEG_QUALITY", 85, 40, 95)
def webp_quality() -> int:
return _int("WEBP_QUALITY", 80, 40, 95)
def backoff_delay(attempt: int) -> float:
"""Экспоненциальная пауза с jitter. attempt — номер уже совершённой попытки."""
base = _float("JOB_BACKOFF_BASE_SECONDS", 1.0, 0.1, 60.0)
cap = _float("JOB_BACKOFF_CAP_SECONDS", 30.0, 0.1, 300.0)
ratio = _float("JOB_BACKOFF_JITTER_RATIO", 0.2, 0.0, 1.0)
delay = min(cap, base * (2 ** max(0, attempt - 1)))
return delay + random.uniform(0, delay * ratio)
+118
View File
@@ -0,0 +1,118 @@
"""Приватный S3-совместимый bucket. Ключ объекта задаёт сервер, не клиент."""
from __future__ import annotations
import io
import os
import sys
from pathlib import Path
from minio import Minio
from errors import ApiError
_client: Minio | None = None
def _read_setting(env_name: str, file_env: str) -> str:
path = os.environ.get(file_env, "").strip()
if path and Path(path).is_file():
return Path(path).read_text(encoding="utf-8").strip()
return os.environ.get(env_name, "").strip()
def endpoint() -> str:
return os.environ.get("S3_ENDPOINT", "http://s3:9000").strip()
def bucket() -> str:
return os.environ.get("S3_BUCKET", "messenger-attachments").strip()
def _host_and_secure() -> tuple[str, bool]:
raw = endpoint()
secure = raw.startswith("https://")
host = raw.split("://", 1)[-1].rstrip("/")
return host, secure
def init() -> None:
global _client
host, secure = _host_and_secure()
access = _read_setting("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
secret = _read_setting("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
if not host or not access or not secret or not bucket():
raise RuntimeError("object storage is not configured")
_client = Minio(
host,
access_key=access,
secret_key=secret,
secure=secure,
region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1",
)
def _api() -> Minio:
if _client is None:
raise ApiError(503, "unavailable", "Хранилище недоступно")
return _client
def ping() -> bool:
try:
return _api().bucket_exists(bucket())
except Exception:
return False
def ensure_bucket() -> None:
api = _api()
name = bucket()
if not api.bucket_exists(name):
api.make_bucket(name)
def put_bytes(key: str, data: bytes, content_type: str) -> None:
try:
_api().put_object(
bucket(),
key,
io.BytesIO(data),
length=len(data),
content_type=content_type,
)
except ApiError:
raise
except Exception as exc:
sys.stderr.write("s3 put failed\n")
raise ApiError(503, "unavailable", "Хранилище недоступно") from exc
def get_bytes(key: str) -> bytes:
response = None
try:
response = _api().get_object(bucket(), key)
return response.read()
except ApiError:
raise
except Exception as exc:
sys.stderr.write("s3 get failed\n")
raise ApiError(503, "unavailable", "Хранилище недоступно") from exc
finally:
if response is not None:
response.close()
response.release_conn()
def exists(key: str) -> bool:
try:
_api().stat_object(bucket(), key)
return True
except Exception:
return False
def delete_quiet(key: str) -> None:
try:
_api().remove_object(bucket(), key)
except Exception:
sys.stderr.write("s3 delete failed\n")
+124
View File
@@ -0,0 +1,124 @@
"""Варианты изображения. Оригинал этот модуль не перекодирует.
thumbnail: JPEG, вписать в 256×256 без увеличения, пропорции сохраняются,
размеры округляются вниз до минимума 1 px. EXIF orientation применяется до
масштаба. Альфа кладётся на белый. Качество JPEG_QUALITY (по умолчанию 85),
optimize, без EXIF/ICC.
crop: JPEG тех же параметров, центральный квадрат, затем ровно 128×128.
Маленький исходник можно увеличить.
optimized: WebP, вписать в 1280×1280 без увеличения. Качество WEBP_QUALITY
(по умолчанию 80), method=6. EXIF, ICC и XMP в файл не пишутся. Меньший размер
маленького файла не обещается.
"""
from __future__ import annotations
import io
from PIL import Image, ImageOps, UnidentifiedImageError
import settings
THUMBNAIL_LIMIT = 256
CROP_SIZE = 128
OPTIMIZED_LIMIT = 1280
class PermanentJobError(Exception):
def __init__(self, code: str):
super().__init__(code)
self.code = code
class TransientJobError(Exception):
def __init__(self, code: str = "unavailable"):
super().__init__(code)
self.code = code
def _open(data: bytes) -> Image.Image:
if len(data) > settings.max_upload_bytes():
raise PermanentJobError("payload_too_large")
Image.MAX_IMAGE_PIXELS = settings.max_image_pixels()
try:
image = Image.open(io.BytesIO(data))
width, height = image.size
if width < 1 or height < 1 or width * height > settings.max_image_pixels():
raise PermanentJobError("payload_too_large")
image = ImageOps.exif_transpose(image)
image.load()
except PermanentJobError:
raise
except Image.DecompressionBombError as exc:
raise PermanentJobError("payload_too_large") from exc
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc:
raise PermanentJobError("invalid_image") from exc
return image
def _on_white(image: Image.Image) -> Image.Image:
if image.mode == "RGB":
return image
rgba = image.convert("RGBA")
background = Image.new("RGB", rgba.size, (255, 255, 255))
background.paste(rgba, mask=rgba.getchannel("A"))
return background
def _fit(image: Image.Image, limit: int) -> Image.Image:
width, height = image.size
if width <= limit and height <= limit:
return image
scale = min(limit / width, limit / height)
resized = (max(1, int(width * scale)), max(1, int(height * scale)))
return image.resize(resized, Image.Resampling.LANCZOS)
def _crop(image: Image.Image) -> Image.Image:
width, height = image.size
side = min(width, height)
left = (width - side) // 2
top = (height - side) // 2
cropped = image.crop((left, top, left + side, top + side))
if cropped.size != (CROP_SIZE, CROP_SIZE):
cropped = cropped.resize((CROP_SIZE, CROP_SIZE), Image.Resampling.LANCZOS)
return cropped
def _jpeg(image: Image.Image) -> bytes:
buffer = io.BytesIO()
_on_white(image).save(
buffer,
format="JPEG",
quality=settings.jpeg_quality(),
optimize=True,
progressive=False,
)
return buffer.getvalue()
def _webp(image: Image.Image) -> bytes:
buffer = io.BytesIO()
_on_white(image).save(
buffer,
format="WEBP",
quality=settings.webp_quality(),
method=6,
)
return buffer.getvalue()
def render(data: bytes) -> dict:
image = _open(data)
thumb = _fit(image, THUMBNAIL_LIMIT)
crop = _crop(image)
optimized = _fit(image, OPTIMIZED_LIMIT)
thumb_bytes = _jpeg(thumb)
crop_bytes = _jpeg(crop)
optimized_bytes = _webp(optimized)
return {
"thumbnail": (thumb_bytes, "image/jpeg", thumb.size),
"crop": (crop_bytes, "image/jpeg", crop.size),
"optimized": (optimized_bytes, "image/webp", optimized.size),
}
+257
View File
@@ -0,0 +1,257 @@
#!/usr/bin/env python3
"""Отдельный процесс обработки. API этот файл не импортирует и сам не делает resize."""
from __future__ import annotations
import hashlib
import json
import os
import sys
import threading
import time
import uuid
from datetime import datetime, timedelta, timezone
import broker
import db
import relay
import settings
import storage
import variants
from variants import PermanentJobError, TransientJobError
_stop = threading.Event()
def _validate(payload) -> dict:
if not isinstance(payload, dict):
raise ValueError("payload")
allowed = {
"schema_version", "job_id", "type", "attachment_id", "pipeline_version",
"attempt", "created_at", "traceparent", "tracestate",
}
if set(payload) - allowed:
raise ValueError("extra")
if payload.get("schema_version") != 1 or payload.get("type") != "attachment.process.v1":
raise ValueError("type")
for field in ("job_id", "attachment_id"):
uuid.UUID(str(payload.get(field)))
if int(payload.get("pipeline_version")) < 1 or int(payload.get("attempt")) < 1:
raise ValueError("attempt")
if not isinstance(payload.get("created_at"), str):
raise ValueError("created_at")
return payload
def _lab_fault(source: dict, attempt: int) -> None:
if os.environ.get("APP_ENV", "lab") != "lab":
return
name = source["filename"]
if name.startswith("__lab_fault_permanent"):
raise PermanentJobError("invalid_image")
if name.startswith("__lab_fault_transient") and attempt < 3:
raise TransientJobError("unavailable")
def _produce(source: dict, attempt: int) -> list:
_lab_fault(source, attempt)
if source["size_bytes"] > settings.max_upload_bytes():
raise PermanentJobError("payload_too_large")
if source["width"] * source["height"] > settings.max_image_pixels():
raise PermanentJobError("payload_too_large")
staging_key = source["staging_key"]
if not staging_key:
if storage.exists(source["object_key"]):
original = storage.get_bytes(source["object_key"])
else:
raise TransientJobError("unavailable")
else:
original = storage.get_bytes(staging_key)
if len(original) != source["size_bytes"] or hashlib.sha256(original).hexdigest() != source["sha256"]:
raise TransientJobError("unavailable")
rendered = variants.render(original)
storage.put_bytes(source["object_key"], original, source["content_type"])
stored = []
for name, (data, content_type, size) in rendered.items():
key = "variants/%s/%s" % (source["attachment_id"], name)
storage.put_bytes(key, data, content_type)
stored.append({
"name": name,
"content_type": content_type,
"size_bytes": len(data),
"width": size[0],
"height": size[1],
"sha256": hashlib.sha256(data).hexdigest(),
"object_key": key,
})
if os.environ.get("CRASH_AFTER_S3") == "1" or source["filename"].startswith("__lab_fault_crash"):
sys.stderr.write("crash after object write before ack attachment=%s\n" % source["attachment_id"])
os._exit(86)
return stored
def _retry(payload: dict, claim: dict, code: str) -> str:
delay = settings.backoff_delay(claim["attempt"])
available = datetime.now(timezone.utc) + timedelta(seconds=delay)
sys.stderr.write(
"retry job=%s attempt=%s delay=%.3f code=%s\n"
% (payload["job_id"], claim["attempt"], delay, code)
)
return db.retry_job(
payload["job_id"],
claim["lease_token"],
code,
payload,
available,
settings.max_attempts(),
)
def _dlq(channel, payload: dict) -> None:
try:
broker.publish_on(channel, payload, broker.FAILED_KEY)
except Exception:
sys.stderr.write("dlq publish failed job=%s\n" % payload.get("job_id"))
def _cleanup(source: dict) -> None:
key = source.get("staging_key")
if not key:
return
storage.delete_quiet(key)
db.clear_staging_key(source["attachment_id"])
def _work(channel, payload: dict) -> None:
claim = db.claim_job(payload["job_id"], settings.job_timeout())
if claim["action"] == "ack":
return
if claim["action"] != "work":
time.sleep(0.4)
raise TransientJobError("lease_held")
source = db.load_job_source(payload["attachment_id"])
if source is None or source["job_status"] == "ready" or source["attachment_status"] == "ready":
return
if int(payload["pipeline_version"]) != int(source["pipeline_version"]):
return
try:
produced = _produce(source, claim["attempt"])
outcome = db.finish_job(
payload["job_id"], claim["lease_token"], produced, source["pipeline_version"],
)
except PermanentJobError as exc:
outcome = db.fail_job(payload["job_id"], claim["lease_token"], exc.code, claim["attempt"])
if outcome == "failed":
_dlq(channel, payload)
return
except TransientJobError as exc:
if exc.code == "lease_held":
raise
outcome = _retry(payload, claim, exc.code)
if outcome == "failed":
_dlq(channel, payload)
return
except Exception as exc:
sys.stderr.write("transient processing error %s\n" % type(exc).__name__)
outcome = _retry(payload, claim, "unavailable")
if outcome == "failed":
_dlq(channel, payload)
return
if outcome == "lost":
raise TransientJobError("lease_lost")
if outcome == "ready":
_cleanup(source)
def _on_message(channel, method, _properties, body: bytes) -> None:
try:
payload = _validate(json.loads(body.decode("utf-8")))
except Exception:
sys.stderr.write("dropping invalid job message\n")
channel.basic_ack(method.delivery_tag)
return
try:
_work(channel, payload)
except TransientJobError:
channel.basic_nack(method.delivery_tag, requeue=True)
return
except Exception:
sys.stderr.write("worker failed closed for job %s\n" % payload.get("job_id"))
channel.basic_nack(method.delivery_tag, requeue=True)
return
channel.basic_ack(method.delivery_tag)
def _consume() -> None:
connection, channel = broker.connect()
channel.basic_qos(prefetch_count=1)
channel.basic_consume(broker.PROCESS_QUEUE, _on_message, auto_ack=False)
channel.start_consuming()
def _consume_loop() -> None:
while not _stop.is_set():
try:
_consume()
except Exception:
if _stop.is_set():
return
sys.stderr.write("consumer reconnect\n")
time.sleep(1)
def _sweep() -> None:
while not _stop.is_set():
try:
expired = db.recover_expired_leases(settings.max_attempts())
restored = db.reconcile_queued(settings.job_timeout())
if expired or restored:
sys.stderr.write("sweep expired=%s restored=%s\n" % (expired, restored))
for attachment_id, key in db.staging_ready_for_cleanup(40):
storage.delete_quiet(key)
db.clear_staging_key(attachment_id)
for attachment_id, key in db.expired_failed_staging(settings.staging_retention(), 40):
storage.delete_quiet(key)
db.clear_staging_key(attachment_id)
relay.publish_batch()
except Exception:
sys.stderr.write("sweeper failed\n")
_stop.wait(2)
def _memory_limit() -> str:
for path in ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"):
try:
with open(path, encoding="utf-8") as handle:
return handle.read().strip()
except OSError:
continue
return "unknown"
def main() -> None:
db.init()
storage.init()
sys.stderr.write(
"worker concurrency=%s prefetch_per_channel=1 max_attempts=%s timeout=%ss memory_limit=%s\n"
% (
settings.worker_concurrency(),
settings.max_attempts(),
settings.job_timeout(),
_memory_limit(),
)
)
threading.Thread(target=_sweep, name="sweep", daemon=True).start()
threads = []
for index in range(settings.worker_concurrency()):
thread = threading.Thread(target=_consume_loop, name="consumer-%s" % index, daemon=True)
thread.start()
threads.append(thread)
try:
while any(thread.is_alive() for thread in threads):
time.sleep(0.5)
except KeyboardInterrupt:
_stop.set()
if __name__ == "__main__":
main()