forked from mtusi2026/mtusi2026lab5
Отделить приём вложений от обработки через очередь и worker.
This commit is contained in:
+112
@@ -0,0 +1,112 @@
|
||||
"""RabbitMQ: durable quorum-очереди и publisher confirms.
|
||||
|
||||
Сообщение содержит только идентификаторы задания. Байты и ключи S3 сюда не входят.
|
||||
Ack потребителя и confirm издателя — разные границы.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pika
|
||||
|
||||
EXCHANGE = "attachments"
|
||||
PROCESS_QUEUE = "attachment.process"
|
||||
FAILED_QUEUE = "attachment.failed"
|
||||
PROCESS_KEY = "process"
|
||||
FAILED_KEY = "failed"
|
||||
|
||||
_lock = threading.Lock()
|
||||
_connection: pika.BlockingConnection | None = None
|
||||
_channel: pika.adapters.blocking_connection.BlockingChannel | None = None
|
||||
|
||||
|
||||
def broker_url() -> str:
|
||||
path = os.environ.get("BROKER_URL_FILE", "").strip()
|
||||
if path and Path(path).is_file():
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
return os.environ.get("BROKER_URL", "").strip()
|
||||
|
||||
|
||||
def declare(channel) -> None:
|
||||
channel.exchange_declare(EXCHANGE, exchange_type="direct", durable=True)
|
||||
arguments = {"x-queue-type": "quorum"}
|
||||
channel.queue_declare(PROCESS_QUEUE, durable=True, arguments=arguments)
|
||||
channel.queue_declare(FAILED_QUEUE, durable=True, arguments=arguments)
|
||||
channel.queue_bind(PROCESS_QUEUE, EXCHANGE, PROCESS_KEY)
|
||||
channel.queue_bind(FAILED_QUEUE, EXCHANGE, FAILED_KEY)
|
||||
|
||||
|
||||
def connect() -> tuple[pika.BlockingConnection, pika.adapters.blocking_connection.BlockingChannel]:
|
||||
url = broker_url()
|
||||
if not url:
|
||||
raise RuntimeError("broker is not configured")
|
||||
params = pika.URLParameters(url)
|
||||
params.heartbeat = 60
|
||||
params.blocked_connection_timeout = 15
|
||||
params.socket_timeout = 10
|
||||
params.connection_attempts = 3
|
||||
params.retry_delay = 1
|
||||
connection = pika.BlockingConnection(params)
|
||||
channel = connection.channel()
|
||||
channel.confirm_delivery()
|
||||
declare(channel)
|
||||
return connection, channel
|
||||
|
||||
|
||||
def _ensure():
|
||||
global _connection, _channel
|
||||
if _connection is not None and _connection.is_open and _channel is not None and _channel.is_open:
|
||||
return _channel
|
||||
if _connection is not None:
|
||||
try:
|
||||
_connection.close()
|
||||
except Exception:
|
||||
pass
|
||||
_connection, _channel = connect()
|
||||
return _channel
|
||||
|
||||
|
||||
def publish_job(payload: dict, routing_key: str = PROCESS_KEY) -> None:
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
if len(body) > 4096:
|
||||
raise RuntimeError("job message exceeds 4096 bytes")
|
||||
properties = pika.BasicProperties(
|
||||
content_type="application/json",
|
||||
delivery_mode=2,
|
||||
message_id=str(payload.get("job_id", "")),
|
||||
type=str(payload.get("type", "")),
|
||||
)
|
||||
with _lock:
|
||||
channel = _ensure()
|
||||
try:
|
||||
channel.basic_publish(
|
||||
EXCHANGE,
|
||||
routing_key,
|
||||
body,
|
||||
properties,
|
||||
mandatory=True,
|
||||
)
|
||||
except Exception:
|
||||
global _connection, _channel
|
||||
_connection = None
|
||||
_channel = None
|
||||
raise
|
||||
|
||||
|
||||
def publish_on(channel, payload: dict, routing_key: str) -> None:
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
channel.basic_publish(
|
||||
EXCHANGE,
|
||||
routing_key,
|
||||
body,
|
||||
pika.BasicProperties(
|
||||
content_type="application/json",
|
||||
delivery_mode=2,
|
||||
message_id=str(payload.get("job_id", "")),
|
||||
type=str(payload.get("type", "")),
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Ошибки API. Текст не содержит SQL, паролей и токенов."""
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str):
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Размеры PNG/JPEG по заголовкам, без распаковки пикселей."""
|
||||
from __future__ import annotations
|
||||
|
||||
from errors import ApiError
|
||||
|
||||
PNG_SIG = b"\x89PNG\r\n\x1a\n"
|
||||
# SOF0..SOF3, SOF5..SOF7, SOF9..SOF11, SOF13..SOF15. Не DHT/JPG/DAC.
|
||||
_JPEG_SOF = {
|
||||
0xC0, 0xC1, 0xC2, 0xC3,
|
||||
0xC5, 0xC6, 0xC7,
|
||||
0xC9, 0xCA, 0xCB,
|
||||
0xCD, 0xCE, 0xCF,
|
||||
}
|
||||
|
||||
|
||||
def sniff(data: bytes) -> str:
|
||||
if data.startswith(PNG_SIG):
|
||||
return "image/png"
|
||||
if data.startswith(b"\xff\xd8\xff"):
|
||||
return "image/jpeg"
|
||||
raise ApiError(415, "unsupported_media_type", "Файл не PNG и не JPEG")
|
||||
|
||||
|
||||
def dimensions(data: bytes, content_type: str) -> tuple[int, int]:
|
||||
if content_type == "image/png":
|
||||
return _png_size(data)
|
||||
if content_type == "image/jpeg":
|
||||
return _jpeg_size(data)
|
||||
raise ApiError(415, "unsupported_media_type", "Файл не PNG и не JPEG")
|
||||
|
||||
|
||||
def _png_size(data: bytes) -> tuple[int, int]:
|
||||
if len(data) < 24 or not data.startswith(PNG_SIG) or data[12:16] != b"IHDR":
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённый PNG")
|
||||
width = int.from_bytes(data[16:20], "big")
|
||||
height = int.from_bytes(data[20:24], "big")
|
||||
if width < 1 or height < 1:
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённый PNG")
|
||||
return width, height
|
||||
|
||||
|
||||
def _jpeg_size(data: bytes) -> tuple[int, int]:
|
||||
if len(data) < 4 or data[:2] != b"\xff\xd8":
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
|
||||
index = 2
|
||||
size = len(data)
|
||||
while index < size:
|
||||
if data[index] != 0xFF:
|
||||
break
|
||||
while index < size and data[index] == 0xFF:
|
||||
index += 1
|
||||
if index >= size:
|
||||
break
|
||||
marker = data[index]
|
||||
index += 1
|
||||
if marker in (0xD8, 0xD9) or marker == 0xDA:
|
||||
break
|
||||
if marker == 0x01 or 0xD0 <= marker <= 0xD7:
|
||||
continue
|
||||
if index + 2 > size:
|
||||
break
|
||||
segment = int.from_bytes(data[index:index + 2], "big")
|
||||
if segment < 2 or index + segment > size:
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
|
||||
if marker in _JPEG_SOF:
|
||||
if segment < 7:
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
|
||||
height = int.from_bytes(data[index + 3:index + 5], "big")
|
||||
width = int.from_bytes(data[index + 5:index + 7], "big")
|
||||
if width < 1 or height < 1:
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённый JPEG")
|
||||
return width, height
|
||||
index += segment
|
||||
raise ApiError(415, "unsupported_media_type", "В JPEG нет размеров")
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Однократные миграции. Реплики API этот файл не запускают."""
|
||||
import sys
|
||||
|
||||
import db
|
||||
|
||||
|
||||
def main() -> None:
|
||||
db.migrate_with_retry()
|
||||
sys.stderr.write("migrations applied\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Пароли: scrypt из hashlib. Не bcrypt — он режет пароль на 72 байтах."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
|
||||
N = 2**14
|
||||
R = 8
|
||||
P = 1
|
||||
DKLEN = 32
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
salt = secrets.token_bytes(16)
|
||||
digest = hashlib.scrypt(
|
||||
password.encode("utf-8"), salt=salt, n=N, r=R, p=P, dklen=DKLEN,
|
||||
maxmem=64 * 1024 * 1024)
|
||||
return "scrypt$%d$%d$%d$%s$%s" % (
|
||||
N, R, P,
|
||||
base64.b64encode(salt).decode("ascii"),
|
||||
base64.b64encode(digest).decode("ascii"),
|
||||
)
|
||||
|
||||
|
||||
def verify_password(password: str, stored: str) -> bool:
|
||||
try:
|
||||
algo, n, r, p, salt_b64, digest_b64 = stored.split("$")
|
||||
if algo != "scrypt":
|
||||
return False
|
||||
salt = base64.b64decode(salt_b64)
|
||||
expected = base64.b64decode(digest_b64)
|
||||
actual = hashlib.scrypt(
|
||||
password.encode("utf-8"), salt=salt, n=int(n), r=int(r), p=int(p),
|
||||
dklen=len(expected), maxmem=64 * 1024 * 1024)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
return hmac.compare_digest(actual, expected)
|
||||
|
||||
|
||||
# Одинаковая стоимость проверки, когда пользователя нет.
|
||||
DUMMY_HASH = hash_password("dummy-password-not-used")
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Доставка outbox в брокер. Строка остаётся неопубликованной, пока нет confirm."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import broker
|
||||
import db
|
||||
|
||||
|
||||
def publish_batch(limit: int = 20) -> int:
|
||||
if os.environ.get("OUTBOX_PUBLISH_ENABLED", "1") == "0":
|
||||
return 0
|
||||
rows = db.take_outbox(limit)
|
||||
published = 0
|
||||
for outbox_id, payload in rows:
|
||||
try:
|
||||
broker.publish_job(payload)
|
||||
except Exception:
|
||||
sys.stderr.write("broker did not confirm job %s\n" % payload.get("job_id", ""))
|
||||
return published
|
||||
db.mark_outbox_published(outbox_id)
|
||||
published += 1
|
||||
return published
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Однократное создание приватного bucket. Повторный запуск ничего не публикует."""
|
||||
import sys
|
||||
import time
|
||||
|
||||
import storage
|
||||
|
||||
|
||||
def main() -> None:
|
||||
storage.init()
|
||||
last = None
|
||||
for _ in range(30):
|
||||
try:
|
||||
storage.ensure_bucket()
|
||||
sys.stderr.write("bucket ready\n")
|
||||
return
|
||||
except Exception as exc:
|
||||
last = exc
|
||||
time.sleep(1)
|
||||
sys.stderr.write("bucket bootstrap failed\n")
|
||||
raise SystemExit(1) from last
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+730
@@ -0,0 +1,730 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Messenger lab5. Приём файла отделён от обработки: staging, outbox, брокер, worker."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import signal
|
||||
import socket
|
||||
import socketserver
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import unicodedata
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import db
|
||||
import images
|
||||
import passwords
|
||||
import sessions
|
||||
import storage
|
||||
from errors import ApiError
|
||||
|
||||
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = int(os.environ.get("APP_PORT", "8080"))
|
||||
APP_ENV = os.environ.get("APP_ENV", "lab")
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname()
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}\Z")
|
||||
UUID_RE = re.compile(
|
||||
r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
|
||||
r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\Z")
|
||||
TIMESTAMP_RE = re.compile(
|
||||
r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?Z\Z")
|
||||
IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}\Z")
|
||||
MAX_BODY = 1024 * 1024
|
||||
|
||||
_draining = threading.Event()
|
||||
_inflight = 0
|
||||
_inflight_lock = threading.Lock()
|
||||
|
||||
|
||||
def drain_seconds() -> float:
|
||||
raw = os.environ.get("DRAIN_SECONDS", "3")
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return 3.0
|
||||
if value < 0:
|
||||
return 0.0
|
||||
if value > 6:
|
||||
return 6.0
|
||||
return value
|
||||
|
||||
|
||||
def now_utc() -> datetime:
|
||||
current = datetime.now(timezone.utc)
|
||||
return current.replace(microsecond=(current.microsecond // 1000) * 1000)
|
||||
|
||||
|
||||
def is_blank(value: str) -> bool:
|
||||
return value.strip() == ""
|
||||
|
||||
|
||||
def require_object(body):
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(400, "invalid_request", "Тело должно быть JSON-объектом")
|
||||
|
||||
|
||||
def reject_extra(body: dict, allowed: set):
|
||||
extra = set(body) - allowed
|
||||
if extra:
|
||||
raise ApiError(400, "invalid_request",
|
||||
"Лишние поля: " + ", ".join(sorted(extra)))
|
||||
|
||||
|
||||
def parse_uuid(value, field: str) -> str:
|
||||
if not isinstance(value, str) or not UUID_RE.match(value):
|
||||
raise ApiError(400, "invalid_request", f"Некорректный UUID: {field}")
|
||||
return value.lower()
|
||||
|
||||
|
||||
def parse_display_name(value) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ApiError(400, "invalid_request", "display_name должен быть строкой")
|
||||
if not (1 <= len(value) <= 100):
|
||||
raise ApiError(400, "invalid_request", "display_name: 1..100 code points")
|
||||
if is_blank(value):
|
||||
raise ApiError(400, "invalid_request", "display_name не может быть пустым")
|
||||
return value
|
||||
|
||||
|
||||
def parse_username(value) -> str:
|
||||
if not isinstance(value, str) or not USERNAME_RE.match(value):
|
||||
raise ApiError(400, "invalid_request", "username: ^[a-z0-9_]{3,32}$")
|
||||
return value
|
||||
|
||||
|
||||
def parse_password(value) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ApiError(400, "invalid_request", "password должен быть строкой")
|
||||
if not (12 <= len(value) <= 128):
|
||||
raise ApiError(400, "invalid_request", "password: 12..128 code points")
|
||||
if is_blank(value):
|
||||
raise ApiError(400, "invalid_request", "password не может быть пустым")
|
||||
return value
|
||||
|
||||
|
||||
def parse_title(value) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ApiError(400, "invalid_request", "title должен быть строкой")
|
||||
if not (1 <= len(value) <= 100):
|
||||
raise ApiError(400, "invalid_request", "title: 1..100 code points")
|
||||
if is_blank(value):
|
||||
raise ApiError(400, "invalid_request", "title не может быть пустым")
|
||||
return value
|
||||
|
||||
|
||||
def max_upload_bytes() -> int:
|
||||
raw = os.environ.get("MAX_UPLOAD_BYTES", "10485760")
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return 10485760
|
||||
return value if value >= 1 else 10485760
|
||||
|
||||
|
||||
def max_image_pixels() -> int:
|
||||
raw = os.environ.get("MAX_IMAGE_PIXELS", "40000000")
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return 40000000
|
||||
return value if value >= 1 else 40000000
|
||||
|
||||
|
||||
def parse_text(value, has_attachments: bool) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ApiError(400, "invalid_request", "text должен быть строкой")
|
||||
if len(value) > 4000:
|
||||
raise ApiError(400, "invalid_request", "text: не больше 4000 code points")
|
||||
if is_blank(value) and not has_attachments:
|
||||
raise ApiError(400, "invalid_request", "text не может быть пустым без вложений")
|
||||
return value
|
||||
|
||||
|
||||
def parse_attachment_ids(value) -> list:
|
||||
if value is None:
|
||||
return []
|
||||
if not isinstance(value, list):
|
||||
raise ApiError(400, "invalid_request", "attachment_ids должен быть массивом")
|
||||
if len(value) > 10:
|
||||
raise ApiError(400, "invalid_request", "attachment_ids: не больше 10")
|
||||
seen = set()
|
||||
result = []
|
||||
for item in value:
|
||||
att_id = parse_uuid(item, "attachment_ids[]")
|
||||
if att_id in seen:
|
||||
raise ApiError(400, "invalid_request", "attachment_ids содержит дубликаты")
|
||||
seen.add(att_id)
|
||||
result.append(att_id)
|
||||
return result
|
||||
|
||||
|
||||
def parse_filename(value) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ApiError(400, "invalid_request", "filename: 1..255 code points")
|
||||
if not (1 <= len(value) <= 255):
|
||||
raise ApiError(400, "invalid_request", "filename: 1..255 code points")
|
||||
for char in value:
|
||||
if char in "/\\" or unicodedata.category(char) == "Cc":
|
||||
raise ApiError(400, "invalid_request", "filename содержит запрещённый символ")
|
||||
return value
|
||||
|
||||
|
||||
def parse_member_ids(value) -> list:
|
||||
if not isinstance(value, list):
|
||||
raise ApiError(400, "invalid_request", "member_ids должен быть массивом")
|
||||
if not (1 <= len(value) <= 100):
|
||||
raise ApiError(400, "invalid_request", "member_ids: 1..100")
|
||||
seen = set()
|
||||
result = []
|
||||
for item in value:
|
||||
uid = parse_uuid(item, "member_ids[]")
|
||||
if uid in seen:
|
||||
raise ApiError(400, "invalid_request", "member_ids содержит дубликаты")
|
||||
seen.add(uid)
|
||||
result.append(uid)
|
||||
return result
|
||||
|
||||
|
||||
def parse_limit(raw, default=50) -> int:
|
||||
if raw is None:
|
||||
return default
|
||||
if isinstance(raw, str) and raw.strip() == "":
|
||||
raise ApiError(400, "invalid_request", "limit должен быть целым")
|
||||
try:
|
||||
value = int(raw)
|
||||
except (TypeError, ValueError):
|
||||
raise ApiError(400, "invalid_request", "limit должен быть целым")
|
||||
if not (1 <= value <= 100):
|
||||
raise ApiError(400, "invalid_request", "limit: 1..100")
|
||||
return value
|
||||
|
||||
|
||||
def parse_optional_cursor(raw):
|
||||
if raw is None:
|
||||
return None
|
||||
if not isinstance(raw, str) or raw.strip() == "":
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor")
|
||||
return raw
|
||||
|
||||
|
||||
def parse_cursor_timestamp(value: str) -> datetime:
|
||||
if not isinstance(value, str) or not TIMESTAMP_RE.match(value):
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor")
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor") from exc
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_idempotency_key(headers) -> str:
|
||||
raw = headers.get("Idempotency-Key")
|
||||
if not isinstance(raw, str) or not IDEMPOTENCY_RE.match(raw):
|
||||
raise ApiError(400, "invalid_request", "Некорректный Idempotency-Key")
|
||||
return raw
|
||||
|
||||
|
||||
def canonical_hash(text: str, attachment_ids: list) -> str:
|
||||
# Порядок ключей JSON не важен. Отсутствующий attachment_ids равен [].
|
||||
# Порядок элементов массива сохраняется: это часть тела запроса.
|
||||
payload = {"attachment_ids": attachment_ids, "text": text}
|
||||
raw = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def challenge_for(path: str) -> str:
|
||||
if path == "/api/v1/auth/sessions":
|
||||
return 'Basic realm="messenger", charset="UTF-8"'
|
||||
return 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
def bearer_token(headers) -> str:
|
||||
auth = headers.get("Authorization") or ""
|
||||
parts = auth.split(None, 1)
|
||||
if len(parts) != 2 or parts[0].lower() != "bearer" or not parts[1].strip():
|
||||
raise ApiError(401, "unauthorized", "Требуется Bearer")
|
||||
return parts[1].strip()
|
||||
|
||||
|
||||
def current_user(headers) -> dict:
|
||||
token = bearer_token(headers)
|
||||
user = db.get_user(sessions.user_id(token))
|
||||
if user is None:
|
||||
raise ApiError(401, "unauthorized", "Сессия недействительна")
|
||||
return user
|
||||
|
||||
|
||||
def basic_credentials(headers) -> tuple[str, str]:
|
||||
auth = headers.get("Authorization") or ""
|
||||
parts = auth.split(None, 1)
|
||||
if len(parts) != 2 or parts[0].lower() != "basic" or not parts[1].strip():
|
||||
raise ApiError(401, "unauthorized", "Требуется Basic")
|
||||
try:
|
||||
decoded = base64.b64decode(parts[1].strip(), validate=True).decode("utf-8")
|
||||
except (ValueError, UnicodeDecodeError) as exc:
|
||||
raise ApiError(401, "unauthorized", "Неверные учётные данные") from exc
|
||||
if ":" not in decoded:
|
||||
raise ApiError(401, "unauthorized", "Неверные учётные данные")
|
||||
username, password = decoded.split(":", 1)
|
||||
return username, password
|
||||
|
||||
|
||||
def encode_cursor(chat_id: str, created_at: str, msg_id: str) -> str:
|
||||
raw = f"{chat_id}|{created_at}|{msg_id}".encode("utf-8")
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(cursor: str, chat_id: str):
|
||||
pad = "=" * (-len(cursor) % 4)
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8")
|
||||
except Exception as exc:
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor") from exc
|
||||
parts = raw.split("|")
|
||||
if len(parts) != 3:
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor")
|
||||
c_chat, c_created, c_id = parts
|
||||
if not UUID_RE.match(c_chat):
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor")
|
||||
if c_chat.lower() != chat_id:
|
||||
raise ApiError(400, "invalid_request", "cursor от другого чата")
|
||||
return parse_cursor_timestamp(c_created), parse_uuid(c_id, "cursor")
|
||||
|
||||
|
||||
def encode_chat_cursor(created_at: str, chat_id: str) -> str:
|
||||
raw = f"chat|{created_at}|{chat_id}".encode("utf-8")
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def decode_chat_cursor(cursor: str):
|
||||
pad = "=" * (-len(cursor) % 4)
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8")
|
||||
except Exception as exc:
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor") from exc
|
||||
parts = raw.split("|")
|
||||
if len(parts) != 3 or parts[0] != "chat":
|
||||
raise ApiError(400, "invalid_request", "Некорректный cursor")
|
||||
return parse_cursor_timestamp(parts[1]), parse_uuid(parts[2], "cursor")
|
||||
|
||||
|
||||
def create_user(body) -> dict:
|
||||
require_object(body)
|
||||
reject_extra(body, {"username", "display_name", "password"})
|
||||
username = parse_username(body.get("username"))
|
||||
display_name = parse_display_name(body.get("display_name"))
|
||||
password = parse_password(body.get("password"))
|
||||
return db.create_user(username, display_name, passwords.hash_password(password), now_utc())
|
||||
|
||||
|
||||
def create_session(headers) -> dict:
|
||||
username, password = basic_credentials(headers)
|
||||
row = None
|
||||
try:
|
||||
row = db.user_for_login(username)
|
||||
except ApiError:
|
||||
raise
|
||||
stored = row[3] if row is not None else passwords.DUMMY_HASH
|
||||
password_ok = passwords.verify_password(password, stored)
|
||||
if row is None or not password_ok:
|
||||
raise ApiError(401, "unauthorized", "Неверные учётные данные")
|
||||
token = secrets.token_urlsafe(32)
|
||||
ttl = sessions.session_ttl()
|
||||
expires = now_utc() + timedelta(seconds=ttl)
|
||||
sessions.create(token, str(row[0]), ttl)
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": db.format_ts(expires),
|
||||
"user": db.user_json((row[0], row[1], row[2], row[4])),
|
||||
}
|
||||
|
||||
|
||||
def logout(headers) -> None:
|
||||
token = bearer_token(headers)
|
||||
sessions.user_id(token)
|
||||
sessions.revoke(token)
|
||||
|
||||
|
||||
def create_chat(body, actor: dict) -> dict:
|
||||
require_object(body)
|
||||
reject_extra(body, {"title", "member_ids"})
|
||||
title = parse_title(body.get("title"))
|
||||
member_ids = parse_member_ids(body.get("member_ids"))
|
||||
if actor["id"] not in member_ids:
|
||||
raise ApiError(400, "invalid_request", "вызывающий должен быть в member_ids")
|
||||
return db.create_chat(title, member_ids, now_utc())
|
||||
|
||||
|
||||
def list_chats(actor: dict, limit: int, cursor) -> dict:
|
||||
decoded = decode_chat_cursor(cursor) if cursor else None
|
||||
page = db.list_chats(actor["id"], limit, decoded)
|
||||
next_cursor = None
|
||||
if page["next_cursor_parts"] is not None:
|
||||
created, chat_id = page["next_cursor_parts"]
|
||||
next_cursor = encode_chat_cursor(db.format_ts(created), chat_id)
|
||||
return {"items": page["items"], "next_cursor": next_cursor}
|
||||
|
||||
|
||||
def create_message(chat_id: str, body, actor: dict, idem_key: str) -> dict:
|
||||
cid = parse_uuid(chat_id, "chat_id")
|
||||
require_object(body)
|
||||
reject_extra(body, {"text", "attachment_ids"})
|
||||
attachment_ids = parse_attachment_ids(body.get("attachment_ids"))
|
||||
text = parse_text(body.get("text"), bool(attachment_ids))
|
||||
path = "/api/v1/chats/%s/messages" % cid
|
||||
return db.create_message_idempotent(
|
||||
cid,
|
||||
actor["id"],
|
||||
text,
|
||||
attachment_ids,
|
||||
now_utc(),
|
||||
"POST",
|
||||
path,
|
||||
idem_key,
|
||||
canonical_hash(text, attachment_ids),
|
||||
)
|
||||
|
||||
|
||||
def upload_attachment(chat_id: str, actor: dict, filename: str, declared: str, data: bytes) -> dict:
|
||||
detected = images.sniff(data)
|
||||
if detected != declared:
|
||||
raise ApiError(415, "unsupported_media_type", "Содержимое не совпадает с Content-Type")
|
||||
width, height = images.dimensions(data, detected)
|
||||
if width * height > max_image_pixels():
|
||||
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
|
||||
att_id = str(uuid.uuid4())
|
||||
created = now_utc()
|
||||
staging_key = "staging/" + att_id
|
||||
object_key = "originals/" + att_id
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"job_id": str(uuid.uuid4()),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": att_id,
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": db.format_ts(created),
|
||||
}
|
||||
storage.put_bytes(staging_key, data, detected)
|
||||
try:
|
||||
return db.insert_queued_attachment(
|
||||
att_id, chat_id, actor["id"], filename, detected, len(data),
|
||||
hashlib.sha256(data).hexdigest(), object_key, staging_key,
|
||||
width, height, created, payload,
|
||||
)
|
||||
except Exception:
|
||||
storage.delete_quiet(staging_key)
|
||||
raise
|
||||
|
||||
|
||||
def read_attachment(attachment_id: str, actor: dict) -> dict:
|
||||
return db.get_attachment(parse_uuid(attachment_id, "attachment_id"), actor["id"])
|
||||
|
||||
|
||||
def read_attachment_bytes(attachment_id: str, actor: dict, variant: str):
|
||||
if variant not in ("original", "thumbnail", "crop", "optimized"):
|
||||
raise ApiError(400, "invalid_request", "Некорректный variant")
|
||||
record = db.get_download(parse_uuid(attachment_id, "attachment_id"), actor["id"], variant)
|
||||
data = storage.get_bytes(record["object_key"])
|
||||
if len(data) != record["size_bytes"] or hashlib.sha256(data).hexdigest() != record["sha256"]:
|
||||
raise ApiError(503, "unavailable", "Объект хранилища не совпал с метаданными")
|
||||
return record["content_type"], data
|
||||
|
||||
|
||||
def list_messages(chat_id: str, actor: dict, limit: int, cursor) -> dict:
|
||||
cid = parse_uuid(chat_id, "chat_id")
|
||||
decoded = decode_cursor(cursor, cid) if cursor else None
|
||||
page = db.list_messages(cid, actor["id"], limit, decoded)
|
||||
next_cursor = None
|
||||
if page["next_cursor_parts"] is not None:
|
||||
created, message_id = page["next_cursor_parts"]
|
||||
next_cursor = encode_cursor(cid, db.format_ts(created), message_id)
|
||||
return {"items": page["items"], "next_cursor": next_cursor}
|
||||
|
||||
|
||||
class FastThreadingHTTPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
|
||||
daemon_threads = True
|
||||
block_on_close = False
|
||||
allow_reuse_address = True
|
||||
request_queue_size = 256
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "mtusi-messenger/5.0"
|
||||
protocol_version = "HTTP/1.0"
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
return
|
||||
|
||||
def _request_id(self) -> str:
|
||||
return self.headers.get("X-Request-Id") or str(uuid.uuid4())
|
||||
|
||||
def _send_json(self, status: int, payload, request_id: str, extra_headers=None):
|
||||
body = b""
|
||||
if status != 204:
|
||||
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
self.send_response(status)
|
||||
if status != 204:
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
self.send_header("Connection", "close")
|
||||
for key, value in (extra_headers or {}).items():
|
||||
self.send_header(key, value)
|
||||
self.end_headers()
|
||||
if body:
|
||||
self.wfile.write(body)
|
||||
|
||||
def _send_bytes(self, content_type: str, payload: bytes, request_id: str):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
if payload:
|
||||
self.wfile.write(payload)
|
||||
|
||||
def _send_error(self, status: int, code: str, message: str, request_id: str, extra_headers=None):
|
||||
self._send_json(
|
||||
status,
|
||||
{"error": {"code": code, "message": message, "request_id": request_id}},
|
||||
request_id,
|
||||
extra_headers,
|
||||
)
|
||||
|
||||
def _read_body(self):
|
||||
length = self.headers.get("Content-Length")
|
||||
if not length:
|
||||
return None
|
||||
try:
|
||||
size = int(length)
|
||||
except ValueError as exc:
|
||||
raise ApiError(400, "invalid_request", "Некорректный Content-Length") from exc
|
||||
if size > MAX_BODY:
|
||||
raise ApiError(413, "payload_too_large", "Тело слишком большое")
|
||||
raw = self.rfile.read(size)
|
||||
ctype = (self.headers.get("Content-Type") or "").split(";")[0].strip()
|
||||
if ctype and ctype != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
|
||||
if not raw:
|
||||
raise ApiError(400, "invalid_request", "Пустое тело")
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise ApiError(400, "invalid_request", "Malformed JSON") from exc
|
||||
|
||||
def _read_upload(self) -> tuple[str, bytes]:
|
||||
length = self.headers.get("Content-Length")
|
||||
if length is None:
|
||||
raise ApiError(400, "invalid_request", "Нужен Content-Length")
|
||||
try:
|
||||
size = int(length)
|
||||
except ValueError as exc:
|
||||
raise ApiError(400, "invalid_request", "Некорректный Content-Length") from exc
|
||||
if size > max_upload_bytes():
|
||||
raise ApiError(413, "payload_too_large", "Файл больше допустимого размера")
|
||||
if size < 1:
|
||||
raise ApiError(415, "unsupported_media_type", "Пустое тело")
|
||||
declared = (self.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower()
|
||||
if declared not in ("image/png", "image/jpeg"):
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается image/png или image/jpeg")
|
||||
data = self.rfile.read(size)
|
||||
if len(data) != size:
|
||||
raise ApiError(400, "invalid_request", "Тело короче Content-Length")
|
||||
return declared, data
|
||||
|
||||
def do_GET(self):
|
||||
self._dispatch("GET")
|
||||
|
||||
def do_POST(self):
|
||||
self._dispatch("POST")
|
||||
|
||||
def do_DELETE(self):
|
||||
self._dispatch("DELETE")
|
||||
|
||||
def _dispatch(self, method: str):
|
||||
global _inflight
|
||||
with _inflight_lock:
|
||||
_inflight += 1
|
||||
rid = self._request_id()
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||
try:
|
||||
self._route(method, path, query, rid)
|
||||
except ApiError as exc:
|
||||
headers = {}
|
||||
if exc.status == 401:
|
||||
headers["WWW-Authenticate"] = challenge_for(path)
|
||||
self._send_error(exc.status, exc.code, exc.message, rid, headers)
|
||||
except BrokenPipeError:
|
||||
pass
|
||||
except Exception:
|
||||
sys.stderr.write("internal error on %s %s\n" % (method, path))
|
||||
self._send_error(500, "internal_error", "Внутренняя ошибка", rid)
|
||||
finally:
|
||||
with _inflight_lock:
|
||||
_inflight -= 1
|
||||
|
||||
def _route(self, method: str, path: str, query: dict, rid: str):
|
||||
if method == "GET" and path == "/health/live":
|
||||
return self._send_json(200, {"status": "ok"}, rid)
|
||||
if method == "GET" and path == "/health/ready":
|
||||
if _draining.is_set():
|
||||
raise ApiError(503, "unavailable", "Реплика останавливается")
|
||||
if db.ping() and sessions.ping() and storage.ping():
|
||||
return self._send_json(200, {"status": "ok"}, rid)
|
||||
raise ApiError(503, "unavailable", "Зависимость недоступна")
|
||||
if _draining.is_set():
|
||||
raise ApiError(503, "unavailable", "Реплика останавливается")
|
||||
|
||||
if method == "POST" and path == "/api/v1/users":
|
||||
return self._send_json(201, create_user(self._read_body()), rid)
|
||||
if method == "POST" and path == "/api/v1/auth/sessions":
|
||||
return self._send_json(201, create_session(self.headers), rid)
|
||||
if method == "DELETE" and path == "/api/v1/auth/sessions/current":
|
||||
logout(self.headers)
|
||||
return self._send_json(204, None, rid)
|
||||
if method == "GET" and path == "/api/v1/users/me":
|
||||
return self._send_json(200, current_user(self.headers), rid)
|
||||
|
||||
if method == "POST" and path == "/api/v1/chats":
|
||||
actor = current_user(self.headers)
|
||||
return self._send_json(201, create_chat(self._read_body(), actor), rid)
|
||||
if method == "GET" and path == "/api/v1/chats":
|
||||
actor = current_user(self.headers)
|
||||
limit = parse_limit(query.get("limit", [None])[0])
|
||||
cursor = parse_optional_cursor(query.get("cursor", [None])[0])
|
||||
return self._send_json(200, list_chats(actor, limit, cursor), rid)
|
||||
|
||||
match = re.fullmatch(r"/api/v1/chats/([^/]+)", path)
|
||||
if match and method == "GET":
|
||||
actor = current_user(self.headers)
|
||||
return self._send_json(200, db.get_chat(parse_uuid(match.group(1), "chat_id"), actor["id"]), rid)
|
||||
|
||||
match = re.fullmatch(r"/api/v1/chats/([^/]+)/attachments", path)
|
||||
if match and method == "POST":
|
||||
actor = current_user(self.headers)
|
||||
chat_id = parse_uuid(match.group(1), "chat_id")
|
||||
if not db.is_member(chat_id, actor["id"]):
|
||||
raise ApiError(404, "not_found", "Чат не найден")
|
||||
names = query.get("filename") or []
|
||||
if len(names) != 1:
|
||||
raise ApiError(400, "invalid_request", "Нужен один filename")
|
||||
filename = parse_filename(names[0])
|
||||
declared, data = self._read_upload()
|
||||
body = upload_attachment(chat_id, actor, filename, declared, data)
|
||||
return self._send_json(202, body, rid)
|
||||
|
||||
match = re.fullmatch(r"/api/v1/attachments/([^/]+)/replay", path)
|
||||
if match and method == "POST":
|
||||
actor = current_user(self.headers)
|
||||
att_id = parse_uuid(match.group(1), "attachment_id")
|
||||
body = db.replay_attachment(att_id, actor["id"])
|
||||
status = 200 if body["status"] == "ready" else 202
|
||||
return self._send_json(status, body, rid)
|
||||
|
||||
match = re.fullmatch(r"/api/v1/attachments/([^/]+)/content", path)
|
||||
if match and method == "GET":
|
||||
actor = current_user(self.headers)
|
||||
variants = query.get("variant") or ["original"]
|
||||
if len(variants) != 1 or not isinstance(variants[0], str) or variants[0] == "":
|
||||
raise ApiError(400, "invalid_request", "Некорректный variant")
|
||||
content_type, payload = read_attachment_bytes(match.group(1), actor, variants[0])
|
||||
return self._send_bytes(content_type, payload, rid)
|
||||
|
||||
match = re.fullmatch(r"/api/v1/attachments/([^/]+)", path)
|
||||
if match and method == "GET":
|
||||
actor = current_user(self.headers)
|
||||
return self._send_json(200, read_attachment(match.group(1), actor), rid)
|
||||
|
||||
match = re.fullmatch(r"/api/v1/chats/([^/]+)/messages", path)
|
||||
if match and method == "POST":
|
||||
actor = current_user(self.headers)
|
||||
idem_key = parse_idempotency_key(self.headers)
|
||||
body = create_message(match.group(1), self._read_body(), actor, idem_key)
|
||||
return self._send_json(201, body, rid)
|
||||
if match and method == "GET":
|
||||
actor = current_user(self.headers)
|
||||
limit = parse_limit(query.get("limit", [None])[0])
|
||||
cursor = parse_optional_cursor(query.get("cursor", [None])[0])
|
||||
return self._send_json(200, list_messages(match.group(1), actor, limit, cursor), rid)
|
||||
|
||||
raise ApiError(404, "not_found", "Маршрут не найден")
|
||||
|
||||
|
||||
def _inflight_now() -> int:
|
||||
with _inflight_lock:
|
||||
return _inflight
|
||||
|
||||
|
||||
def _publish_outbox() -> None:
|
||||
import relay
|
||||
while not _draining.is_set():
|
||||
try:
|
||||
relay.publish_batch()
|
||||
except Exception:
|
||||
sys.stderr.write("outbox publish failed\n")
|
||||
_draining.wait(0.4)
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
db.init()
|
||||
except Exception:
|
||||
sys.stderr.write("database is not configured\n")
|
||||
try:
|
||||
sessions.init()
|
||||
except Exception:
|
||||
sys.stderr.write("session store is not configured\n")
|
||||
try:
|
||||
storage.init()
|
||||
except Exception:
|
||||
sys.stderr.write("object storage is not configured\n")
|
||||
threading.Thread(target=_publish_outbox, name="outbox", daemon=True).start()
|
||||
|
||||
server = FastThreadingHTTPServer((APP_HOST, APP_PORT), Handler)
|
||||
|
||||
def drain_and_stop():
|
||||
time.sleep(drain_seconds())
|
||||
deadline = time.monotonic() + 2
|
||||
while time.monotonic() < deadline and _inflight_now() > 0:
|
||||
time.sleep(0.05)
|
||||
server.shutdown()
|
||||
|
||||
def shutdown(signum, frame):
|
||||
if _draining.is_set():
|
||||
return
|
||||
_draining.set()
|
||||
sys.stderr.write("SIGTERM instance=%s stop accepting\n" % INSTANCE_ID)
|
||||
threading.Thread(target=drain_and_stop, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, shutdown)
|
||||
signal.signal(signal.SIGINT, shutdown)
|
||||
sys.stderr.write(
|
||||
"listening on %s:%s env=%s instance=%s\n" % (APP_HOST, APP_PORT, APP_ENV, INSTANCE_ID)
|
||||
)
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.1)
|
||||
finally:
|
||||
server.server_close()
|
||||
db.close()
|
||||
sys.stderr.write("stopped instance=%s\n" % INSTANCE_ID)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Сессии в Valkey. Ключ — SHA-256 токена, TTL равен остатку жизни сессии.
|
||||
|
||||
Источник истины сессий — общий Valkey, не память процесса API.
|
||||
AOF и volume сохраняют ключ при перезапуске хранилища.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import redis
|
||||
|
||||
from errors import ApiError
|
||||
|
||||
_client: redis.Redis | None = None
|
||||
|
||||
|
||||
def session_ttl() -> int:
|
||||
raw = os.environ.get("SESSION_TTL_SECONDS", "3600")
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return 3600
|
||||
return value if value >= 1 else 1
|
||||
|
||||
|
||||
def redis_url() -> str:
|
||||
path = os.environ.get("CACHE_URL_FILE", "").strip()
|
||||
if path and Path(path).is_file():
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
return os.environ.get("REDIS_URL", "").strip()
|
||||
|
||||
|
||||
def init() -> None:
|
||||
global _client
|
||||
url = redis_url()
|
||||
if not url:
|
||||
_client = None
|
||||
return
|
||||
_client = redis.Redis.from_url(
|
||||
url, socket_connect_timeout=0.5, socket_timeout=0.5,
|
||||
decode_responses=True)
|
||||
|
||||
|
||||
def _key(token: str) -> str:
|
||||
digest = hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
return "session:" + digest
|
||||
|
||||
|
||||
def _require() -> redis.Redis:
|
||||
if _client is None:
|
||||
raise ApiError(503, "unavailable", "Хранилище сессий недоступно")
|
||||
return _client
|
||||
|
||||
|
||||
def ping() -> bool:
|
||||
try:
|
||||
return bool(_require().ping())
|
||||
except (ApiError, redis.RedisError):
|
||||
return False
|
||||
|
||||
|
||||
def create(token: str, user_id: str, ttl: int) -> None:
|
||||
try:
|
||||
_require().set(_key(token), user_id, ex=ttl)
|
||||
except (ApiError, redis.RedisError) as exc:
|
||||
raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc
|
||||
|
||||
|
||||
def user_id(token: str) -> str:
|
||||
try:
|
||||
value = _require().get(_key(token))
|
||||
except (ApiError, redis.RedisError) as exc:
|
||||
raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc
|
||||
if not value:
|
||||
raise ApiError(401, "unauthorized", "Сессия недействительна")
|
||||
return value
|
||||
|
||||
|
||||
def revoke(token: str) -> None:
|
||||
try:
|
||||
_require().delete(_key(token))
|
||||
except (ApiError, redis.RedisError) as exc:
|
||||
raise ApiError(503, "unavailable", "Хранилище сессий недоступно") from exc
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Численные пределы worker. Секреты здесь не читаются."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import random
|
||||
|
||||
|
||||
def _int(name: str, default: int, low: int, high: int) -> int:
|
||||
try:
|
||||
value = int(os.environ.get(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
if value < low:
|
||||
return low
|
||||
if value > high:
|
||||
return high
|
||||
return value
|
||||
|
||||
|
||||
def _float(name: str, default: float, low: float, high: float) -> float:
|
||||
try:
|
||||
value = float(os.environ.get(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
if value < low:
|
||||
return low
|
||||
if value > high:
|
||||
return high
|
||||
return value
|
||||
|
||||
|
||||
def max_attempts() -> int:
|
||||
return _int("JOB_MAX_ATTEMPTS", 5, 1, 20)
|
||||
|
||||
|
||||
def job_timeout() -> int:
|
||||
return _int("JOB_TIMEOUT_SECONDS", 60, 5, 600)
|
||||
|
||||
|
||||
def worker_concurrency() -> int:
|
||||
return _int("WORKER_CONCURRENCY", 2, 1, 8)
|
||||
|
||||
|
||||
def max_image_pixels() -> int:
|
||||
return _int("MAX_IMAGE_PIXELS", 40000000, 1, 40000000)
|
||||
|
||||
|
||||
def max_upload_bytes() -> int:
|
||||
return _int("MAX_UPLOAD_BYTES", 10485760, 1, 10485760)
|
||||
|
||||
|
||||
def staging_retention() -> int:
|
||||
return _int("STAGING_RETENTION_SECONDS", 86400, 60, 7 * 86400)
|
||||
|
||||
|
||||
def jpeg_quality() -> int:
|
||||
return _int("JPEG_QUALITY", 85, 40, 95)
|
||||
|
||||
|
||||
def webp_quality() -> int:
|
||||
return _int("WEBP_QUALITY", 80, 40, 95)
|
||||
|
||||
|
||||
def backoff_delay(attempt: int) -> float:
|
||||
"""Экспоненциальная пауза с jitter. attempt — номер уже совершённой попытки."""
|
||||
base = _float("JOB_BACKOFF_BASE_SECONDS", 1.0, 0.1, 60.0)
|
||||
cap = _float("JOB_BACKOFF_CAP_SECONDS", 30.0, 0.1, 300.0)
|
||||
ratio = _float("JOB_BACKOFF_JITTER_RATIO", 0.2, 0.0, 1.0)
|
||||
delay = min(cap, base * (2 ** max(0, attempt - 1)))
|
||||
return delay + random.uniform(0, delay * ratio)
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
"""Приватный S3-совместимый bucket. Ключ объекта задаёт сервер, не клиент."""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from minio import Minio
|
||||
|
||||
from errors import ApiError
|
||||
|
||||
_client: Minio | None = None
|
||||
|
||||
|
||||
def _read_setting(env_name: str, file_env: str) -> str:
|
||||
path = os.environ.get(file_env, "").strip()
|
||||
if path and Path(path).is_file():
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
return os.environ.get(env_name, "").strip()
|
||||
|
||||
|
||||
def endpoint() -> str:
|
||||
return os.environ.get("S3_ENDPOINT", "http://s3:9000").strip()
|
||||
|
||||
|
||||
def bucket() -> str:
|
||||
return os.environ.get("S3_BUCKET", "messenger-attachments").strip()
|
||||
|
||||
|
||||
def _host_and_secure() -> tuple[str, bool]:
|
||||
raw = endpoint()
|
||||
secure = raw.startswith("https://")
|
||||
host = raw.split("://", 1)[-1].rstrip("/")
|
||||
return host, secure
|
||||
|
||||
|
||||
def init() -> None:
|
||||
global _client
|
||||
host, secure = _host_and_secure()
|
||||
access = _read_setting("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
|
||||
secret = _read_setting("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
|
||||
if not host or not access or not secret or not bucket():
|
||||
raise RuntimeError("object storage is not configured")
|
||||
_client = Minio(
|
||||
host,
|
||||
access_key=access,
|
||||
secret_key=secret,
|
||||
secure=secure,
|
||||
region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1",
|
||||
)
|
||||
|
||||
|
||||
def _api() -> Minio:
|
||||
if _client is None:
|
||||
raise ApiError(503, "unavailable", "Хранилище недоступно")
|
||||
return _client
|
||||
|
||||
|
||||
def ping() -> bool:
|
||||
try:
|
||||
return _api().bucket_exists(bucket())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def ensure_bucket() -> None:
|
||||
api = _api()
|
||||
name = bucket()
|
||||
if not api.bucket_exists(name):
|
||||
api.make_bucket(name)
|
||||
|
||||
|
||||
def put_bytes(key: str, data: bytes, content_type: str) -> None:
|
||||
try:
|
||||
_api().put_object(
|
||||
bucket(),
|
||||
key,
|
||||
io.BytesIO(data),
|
||||
length=len(data),
|
||||
content_type=content_type,
|
||||
)
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
sys.stderr.write("s3 put failed\n")
|
||||
raise ApiError(503, "unavailable", "Хранилище недоступно") from exc
|
||||
|
||||
|
||||
def get_bytes(key: str) -> bytes:
|
||||
response = None
|
||||
try:
|
||||
response = _api().get_object(bucket(), key)
|
||||
return response.read()
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
sys.stderr.write("s3 get failed\n")
|
||||
raise ApiError(503, "unavailable", "Хранилище недоступно") from exc
|
||||
finally:
|
||||
if response is not None:
|
||||
response.close()
|
||||
response.release_conn()
|
||||
|
||||
|
||||
def exists(key: str) -> bool:
|
||||
try:
|
||||
_api().stat_object(bucket(), key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def delete_quiet(key: str) -> None:
|
||||
try:
|
||||
_api().remove_object(bucket(), key)
|
||||
except Exception:
|
||||
sys.stderr.write("s3 delete failed\n")
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
"""Варианты изображения. Оригинал этот модуль не перекодирует.
|
||||
|
||||
thumbnail: JPEG, вписать в 256×256 без увеличения, пропорции сохраняются,
|
||||
размеры округляются вниз до минимума 1 px. EXIF orientation применяется до
|
||||
масштаба. Альфа кладётся на белый. Качество JPEG_QUALITY (по умолчанию 85),
|
||||
optimize, без EXIF/ICC.
|
||||
|
||||
crop: JPEG тех же параметров, центральный квадрат, затем ровно 128×128.
|
||||
Маленький исходник можно увеличить.
|
||||
|
||||
optimized: WebP, вписать в 1280×1280 без увеличения. Качество WEBP_QUALITY
|
||||
(по умолчанию 80), method=6. EXIF, ICC и XMP в файл не пишутся. Меньший размер
|
||||
маленького файла не обещается.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
import settings
|
||||
|
||||
THUMBNAIL_LIMIT = 256
|
||||
CROP_SIZE = 128
|
||||
OPTIMIZED_LIMIT = 1280
|
||||
|
||||
|
||||
class PermanentJobError(Exception):
|
||||
def __init__(self, code: str):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
class TransientJobError(Exception):
|
||||
def __init__(self, code: str = "unavailable"):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def _open(data: bytes) -> Image.Image:
|
||||
if len(data) > settings.max_upload_bytes():
|
||||
raise PermanentJobError("payload_too_large")
|
||||
Image.MAX_IMAGE_PIXELS = settings.max_image_pixels()
|
||||
try:
|
||||
image = Image.open(io.BytesIO(data))
|
||||
width, height = image.size
|
||||
if width < 1 or height < 1 or width * height > settings.max_image_pixels():
|
||||
raise PermanentJobError("payload_too_large")
|
||||
image = ImageOps.exif_transpose(image)
|
||||
image.load()
|
||||
except PermanentJobError:
|
||||
raise
|
||||
except Image.DecompressionBombError as exc:
|
||||
raise PermanentJobError("payload_too_large") from exc
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc:
|
||||
raise PermanentJobError("invalid_image") from exc
|
||||
return image
|
||||
|
||||
|
||||
def _on_white(image: Image.Image) -> Image.Image:
|
||||
if image.mode == "RGB":
|
||||
return image
|
||||
rgba = image.convert("RGBA")
|
||||
background = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
background.paste(rgba, mask=rgba.getchannel("A"))
|
||||
return background
|
||||
|
||||
|
||||
def _fit(image: Image.Image, limit: int) -> Image.Image:
|
||||
width, height = image.size
|
||||
if width <= limit and height <= limit:
|
||||
return image
|
||||
scale = min(limit / width, limit / height)
|
||||
resized = (max(1, int(width * scale)), max(1, int(height * scale)))
|
||||
return image.resize(resized, Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def _crop(image: Image.Image) -> Image.Image:
|
||||
width, height = image.size
|
||||
side = min(width, height)
|
||||
left = (width - side) // 2
|
||||
top = (height - side) // 2
|
||||
cropped = image.crop((left, top, left + side, top + side))
|
||||
if cropped.size != (CROP_SIZE, CROP_SIZE):
|
||||
cropped = cropped.resize((CROP_SIZE, CROP_SIZE), Image.Resampling.LANCZOS)
|
||||
return cropped
|
||||
|
||||
|
||||
def _jpeg(image: Image.Image) -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
_on_white(image).save(
|
||||
buffer,
|
||||
format="JPEG",
|
||||
quality=settings.jpeg_quality(),
|
||||
optimize=True,
|
||||
progressive=False,
|
||||
)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def _webp(image: Image.Image) -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
_on_white(image).save(
|
||||
buffer,
|
||||
format="WEBP",
|
||||
quality=settings.webp_quality(),
|
||||
method=6,
|
||||
)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def render(data: bytes) -> dict:
|
||||
image = _open(data)
|
||||
thumb = _fit(image, THUMBNAIL_LIMIT)
|
||||
crop = _crop(image)
|
||||
optimized = _fit(image, OPTIMIZED_LIMIT)
|
||||
thumb_bytes = _jpeg(thumb)
|
||||
crop_bytes = _jpeg(crop)
|
||||
optimized_bytes = _webp(optimized)
|
||||
return {
|
||||
"thumbnail": (thumb_bytes, "image/jpeg", thumb.size),
|
||||
"crop": (crop_bytes, "image/jpeg", crop.size),
|
||||
"optimized": (optimized_bytes, "image/webp", optimized.size),
|
||||
}
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Отдельный процесс обработки. API этот файл не импортирует и сам не делает resize."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import broker
|
||||
import db
|
||||
import relay
|
||||
import settings
|
||||
import storage
|
||||
import variants
|
||||
from variants import PermanentJobError, TransientJobError
|
||||
|
||||
_stop = threading.Event()
|
||||
|
||||
|
||||
def _validate(payload) -> dict:
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("payload")
|
||||
allowed = {
|
||||
"schema_version", "job_id", "type", "attachment_id", "pipeline_version",
|
||||
"attempt", "created_at", "traceparent", "tracestate",
|
||||
}
|
||||
if set(payload) - allowed:
|
||||
raise ValueError("extra")
|
||||
if payload.get("schema_version") != 1 or payload.get("type") != "attachment.process.v1":
|
||||
raise ValueError("type")
|
||||
for field in ("job_id", "attachment_id"):
|
||||
uuid.UUID(str(payload.get(field)))
|
||||
if int(payload.get("pipeline_version")) < 1 or int(payload.get("attempt")) < 1:
|
||||
raise ValueError("attempt")
|
||||
if not isinstance(payload.get("created_at"), str):
|
||||
raise ValueError("created_at")
|
||||
return payload
|
||||
|
||||
|
||||
def _lab_fault(source: dict, attempt: int) -> None:
|
||||
if os.environ.get("APP_ENV", "lab") != "lab":
|
||||
return
|
||||
name = source["filename"]
|
||||
if name.startswith("__lab_fault_permanent"):
|
||||
raise PermanentJobError("invalid_image")
|
||||
if name.startswith("__lab_fault_transient") and attempt < 3:
|
||||
raise TransientJobError("unavailable")
|
||||
|
||||
|
||||
def _produce(source: dict, attempt: int) -> list:
|
||||
_lab_fault(source, attempt)
|
||||
if source["size_bytes"] > settings.max_upload_bytes():
|
||||
raise PermanentJobError("payload_too_large")
|
||||
if source["width"] * source["height"] > settings.max_image_pixels():
|
||||
raise PermanentJobError("payload_too_large")
|
||||
staging_key = source["staging_key"]
|
||||
if not staging_key:
|
||||
if storage.exists(source["object_key"]):
|
||||
original = storage.get_bytes(source["object_key"])
|
||||
else:
|
||||
raise TransientJobError("unavailable")
|
||||
else:
|
||||
original = storage.get_bytes(staging_key)
|
||||
if len(original) != source["size_bytes"] or hashlib.sha256(original).hexdigest() != source["sha256"]:
|
||||
raise TransientJobError("unavailable")
|
||||
rendered = variants.render(original)
|
||||
storage.put_bytes(source["object_key"], original, source["content_type"])
|
||||
stored = []
|
||||
for name, (data, content_type, size) in rendered.items():
|
||||
key = "variants/%s/%s" % (source["attachment_id"], name)
|
||||
storage.put_bytes(key, data, content_type)
|
||||
stored.append({
|
||||
"name": name,
|
||||
"content_type": content_type,
|
||||
"size_bytes": len(data),
|
||||
"width": size[0],
|
||||
"height": size[1],
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
"object_key": key,
|
||||
})
|
||||
if os.environ.get("CRASH_AFTER_S3") == "1" or source["filename"].startswith("__lab_fault_crash"):
|
||||
sys.stderr.write("crash after object write before ack attachment=%s\n" % source["attachment_id"])
|
||||
os._exit(86)
|
||||
return stored
|
||||
|
||||
|
||||
def _retry(payload: dict, claim: dict, code: str) -> str:
|
||||
delay = settings.backoff_delay(claim["attempt"])
|
||||
available = datetime.now(timezone.utc) + timedelta(seconds=delay)
|
||||
sys.stderr.write(
|
||||
"retry job=%s attempt=%s delay=%.3f code=%s\n"
|
||||
% (payload["job_id"], claim["attempt"], delay, code)
|
||||
)
|
||||
return db.retry_job(
|
||||
payload["job_id"],
|
||||
claim["lease_token"],
|
||||
code,
|
||||
payload,
|
||||
available,
|
||||
settings.max_attempts(),
|
||||
)
|
||||
|
||||
|
||||
def _dlq(channel, payload: dict) -> None:
|
||||
try:
|
||||
broker.publish_on(channel, payload, broker.FAILED_KEY)
|
||||
except Exception:
|
||||
sys.stderr.write("dlq publish failed job=%s\n" % payload.get("job_id"))
|
||||
|
||||
|
||||
def _cleanup(source: dict) -> None:
|
||||
key = source.get("staging_key")
|
||||
if not key:
|
||||
return
|
||||
storage.delete_quiet(key)
|
||||
db.clear_staging_key(source["attachment_id"])
|
||||
|
||||
|
||||
def _work(channel, payload: dict) -> None:
|
||||
claim = db.claim_job(payload["job_id"], settings.job_timeout())
|
||||
if claim["action"] == "ack":
|
||||
return
|
||||
if claim["action"] != "work":
|
||||
time.sleep(0.4)
|
||||
raise TransientJobError("lease_held")
|
||||
source = db.load_job_source(payload["attachment_id"])
|
||||
if source is None or source["job_status"] == "ready" or source["attachment_status"] == "ready":
|
||||
return
|
||||
if int(payload["pipeline_version"]) != int(source["pipeline_version"]):
|
||||
return
|
||||
try:
|
||||
produced = _produce(source, claim["attempt"])
|
||||
outcome = db.finish_job(
|
||||
payload["job_id"], claim["lease_token"], produced, source["pipeline_version"],
|
||||
)
|
||||
except PermanentJobError as exc:
|
||||
outcome = db.fail_job(payload["job_id"], claim["lease_token"], exc.code, claim["attempt"])
|
||||
if outcome == "failed":
|
||||
_dlq(channel, payload)
|
||||
return
|
||||
except TransientJobError as exc:
|
||||
if exc.code == "lease_held":
|
||||
raise
|
||||
outcome = _retry(payload, claim, exc.code)
|
||||
if outcome == "failed":
|
||||
_dlq(channel, payload)
|
||||
return
|
||||
except Exception as exc:
|
||||
sys.stderr.write("transient processing error %s\n" % type(exc).__name__)
|
||||
outcome = _retry(payload, claim, "unavailable")
|
||||
if outcome == "failed":
|
||||
_dlq(channel, payload)
|
||||
return
|
||||
if outcome == "lost":
|
||||
raise TransientJobError("lease_lost")
|
||||
if outcome == "ready":
|
||||
_cleanup(source)
|
||||
|
||||
|
||||
def _on_message(channel, method, _properties, body: bytes) -> None:
|
||||
try:
|
||||
payload = _validate(json.loads(body.decode("utf-8")))
|
||||
except Exception:
|
||||
sys.stderr.write("dropping invalid job message\n")
|
||||
channel.basic_ack(method.delivery_tag)
|
||||
return
|
||||
try:
|
||||
_work(channel, payload)
|
||||
except TransientJobError:
|
||||
channel.basic_nack(method.delivery_tag, requeue=True)
|
||||
return
|
||||
except Exception:
|
||||
sys.stderr.write("worker failed closed for job %s\n" % payload.get("job_id"))
|
||||
channel.basic_nack(method.delivery_tag, requeue=True)
|
||||
return
|
||||
channel.basic_ack(method.delivery_tag)
|
||||
|
||||
|
||||
def _consume() -> None:
|
||||
connection, channel = broker.connect()
|
||||
channel.basic_qos(prefetch_count=1)
|
||||
channel.basic_consume(broker.PROCESS_QUEUE, _on_message, auto_ack=False)
|
||||
channel.start_consuming()
|
||||
|
||||
|
||||
def _consume_loop() -> None:
|
||||
while not _stop.is_set():
|
||||
try:
|
||||
_consume()
|
||||
except Exception:
|
||||
if _stop.is_set():
|
||||
return
|
||||
sys.stderr.write("consumer reconnect\n")
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def _sweep() -> None:
|
||||
while not _stop.is_set():
|
||||
try:
|
||||
expired = db.recover_expired_leases(settings.max_attempts())
|
||||
restored = db.reconcile_queued(settings.job_timeout())
|
||||
if expired or restored:
|
||||
sys.stderr.write("sweep expired=%s restored=%s\n" % (expired, restored))
|
||||
for attachment_id, key in db.staging_ready_for_cleanup(40):
|
||||
storage.delete_quiet(key)
|
||||
db.clear_staging_key(attachment_id)
|
||||
for attachment_id, key in db.expired_failed_staging(settings.staging_retention(), 40):
|
||||
storage.delete_quiet(key)
|
||||
db.clear_staging_key(attachment_id)
|
||||
relay.publish_batch()
|
||||
except Exception:
|
||||
sys.stderr.write("sweeper failed\n")
|
||||
_stop.wait(2)
|
||||
|
||||
|
||||
def _memory_limit() -> str:
|
||||
for path in ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"):
|
||||
try:
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
return handle.read().strip()
|
||||
except OSError:
|
||||
continue
|
||||
return "unknown"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
db.init()
|
||||
storage.init()
|
||||
sys.stderr.write(
|
||||
"worker concurrency=%s prefetch_per_channel=1 max_attempts=%s timeout=%ss memory_limit=%s\n"
|
||||
% (
|
||||
settings.worker_concurrency(),
|
||||
settings.max_attempts(),
|
||||
settings.job_timeout(),
|
||||
_memory_limit(),
|
||||
)
|
||||
)
|
||||
threading.Thread(target=_sweep, name="sweep", daemon=True).start()
|
||||
threads = []
|
||||
for index in range(settings.worker_concurrency()):
|
||||
thread = threading.Thread(target=_consume_loop, name="consumer-%s" % index, daemon=True)
|
||||
thread.start()
|
||||
threads.append(thread)
|
||||
try:
|
||||
while any(thread.is_alive() for thread in threads):
|
||||
time.sleep(0.5)
|
||||
except KeyboardInterrupt:
|
||||
_stop.set()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user