add level 4

This commit is contained in:
Матвей
2026-10-26 10:29:54 +03:00
parent df15ab0967
commit ff33b8a7c2
9 changed files with 265 additions and 40 deletions
+3 -1
View File
@@ -11,7 +11,9 @@ Python 3.12, PostgreSQL 17.6, SeaweedFS 4.17, RabbitMQ 4.1 и HAProxy 3.0 зап
## Архитектура и решения
PostgreSQL хранит метаданные и состояние, SeaweedFS — private staging/final objects, RabbitMQ — durable сообщения, worker — отдельный процесс. До HTTP 202 API полностью принимает и валидирует bytes, сохраняет staging-object и в одной DB-транзакции метаданные + job, затем ждёт RabbitMQ publisher confirm. Worker ack делает только после durable результата либо после финального failed. Обработка идемпотентна по attachment/pipeline/variant: повтор ready-job не меняет результат.
PostgreSQL хранит метаданные, jobs и transactional outbox; SeaweedFS — private staging/final objects; RabbitMQ — durable сообщения; worker и outbox-dispatcher — отдельные процессы. До HTTP 202 API полностью принимает и валидирует bytes, сохраняет staging-object и в одной DB-транзакции метаданные + job + outbox event. Dispatcher публикует событие с publisher confirm и отмечает event после confirm, поэтому crash между COMMIT и publish восстанавливается без ручной job. Worker ack делает только после durable результата либо после финального failed. Обработка идемпотентна по attachment/pipeline/variant: повтор ready-job не меняет результат. Lease ограничен `JOB_TIMEOUT_SECONDS`; истёкшая processing-job возвращается в очередь dispatcher-ом. Retry ограничен `JOB_MAX_ATTEMPTS`, использует exponential backoff с jitter; исчерпанная job остаётся failed и её служебное сообщение попадает в DLQ. Явный replay: `docker compose run --rm outbox /app/.venv/bin/python -m src.replay <attachment_uuid>`.
На уровне 4 worker создаёт: thumbnail JPEG 256×256 без увеличения; crop JPEG 128×128, центрированный квадрат; optimized WebP до 1280×1280 без увеличения. JPEG quality=85, WebP quality=80; входные EXIF/metadata не сохраняются.
## Доказательства критериев
+24
View File
@@ -19,6 +19,7 @@ x-api: &api
UPLOAD_CONCURRENCY: "1"
UPLOAD_TIMEOUT_SECONDS: "30"
UPLOAD_FAULT_AFTER_S3_PUT: ${UPLOAD_FAULT_AFTER_S3_PUT:-0}
OUTBOX_FAULT_AFTER_COMMIT: ${OUTBOX_FAULT_AFTER_COMMIT:-0}
secrets:
- database_url
- cursor_secret
@@ -197,6 +198,8 @@ services:
BROKER_URL_FILE: /run/secrets/broker_url
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
secrets:
- database_url
- cursor_secret
@@ -212,6 +215,27 @@ services:
condition: service_healthy
stop_grace_period: 30s
outbox:
build:
context: .
command: ["/app/.venv/bin/python", "-m", "src.outbox"]
environment:
APP_ENV: lab5
DATABASE_URL_FILE: /run/secrets/database_url
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
BROKER_URL_FILE: /run/secrets/broker_url
OUTBOX_DISPATCH_INTERVAL_SECONDS: "1"
secrets:
- database_url
- cursor_secret
- broker_url
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
restart: unless-stopped
load_balancer:
image: haproxy:3.0.11-alpine
ports:
+17
View File
@@ -0,0 +1,17 @@
ALTER TABLE attachment_jobs
ADD COLUMN next_attempt_at timestamptz NOT NULL DEFAULT now(),
ADD COLUMN lease_until timestamptz,
ADD COLUMN replayed_at timestamptz;
CREATE INDEX attachment_jobs_due_idx ON attachment_jobs(status, next_attempt_at);
CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs(status, lease_until);
CREATE TABLE attachment_outbox (
id uuid PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
payload jsonb NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
published_at timestamptz,
attempts integer NOT NULL DEFAULT 0,
UNIQUE (job_id, published_at)
);
CREATE INDEX attachment_outbox_pending_idx ON attachment_outbox(created_at) WHERE published_at IS NULL;
+3 -7
View File
@@ -1,5 +1,4 @@
import asyncio
from datetime import datetime, timezone
from hashlib import sha256
import os
from tempfile import TemporaryFile
@@ -16,7 +15,6 @@ from .errors import ApiError
from .models import Attachment
from .routes import get_current_user, get_store
from .storage.postgres import PostgresStore
from .broker import publish
router = APIRouter()
Image.MAX_IMAGE_PIXELS = 40000000
@@ -93,11 +91,9 @@ async def upload(chat_id: UUID, request: Request, filename: str = Query(...),
attachment_id, chat_id, user["id"], filename, content_type,
size, digest.hexdigest(), key, job_id, connection=connection,
)
await publish({
"schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1",
"attachment_id": str(attachment_id), "pipeline_version": 1, "attempt": 1,
"created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
})
# Fault-injection for level-4 proof: durable outbox must recover this commit.
if os.environ.get("OUTBOX_FAULT_AFTER_COMMIT") == "1":
os._exit(70)
attachment["variants"] = []
return attachment
except ClientDisconnect:
+23 -1
View File
@@ -9,6 +9,7 @@ from starlette.concurrency import run_in_threadpool
from .errors import ApiError
QUEUE = "attachment.process.v1"
DLQ = "attachment.process.v1.dlq"
def broker_url() -> str:
@@ -21,7 +22,9 @@ def broker_url() -> str:
def _connection() -> pika.BlockingConnection:
connection = pika.BlockingConnection(pika.URLParameters(broker_url()))
connection.channel().queue_declare(queue=QUEUE, durable=True)
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True)
channel.queue_declare(queue=DLQ, durable=True)
return connection
@@ -47,6 +50,25 @@ async def publish(payload: dict) -> None:
raise ApiError(503, "unavailable", "Брокер недоступен") from error
async def publish_dlq(payload: dict) -> None:
payload = {**payload, "failure": "retry_exhausted"}
try:
await run_in_threadpool(_publish_to, DLQ, payload)
except (pika.exceptions.AMQPError, OSError, ValueError) as error:
raise RuntimeError("DLQ is unavailable") from error
def _publish_to(queue: str, payload: dict) -> None:
connection = _connection()
try:
connection.channel().basic_publish(
exchange="", routing_key=queue, body=json.dumps(payload, separators=(",", ":")).encode("utf-8"),
properties=pika.BasicProperties(delivery_mode=pika.DeliveryMode.Persistent, content_type="application/json"),
)
finally:
connection.close()
async def ready() -> None:
connection = None
try:
+33
View File
@@ -0,0 +1,33 @@
"""Transactional-outbox dispatcher; safe to run as one or more replicas."""
import asyncio
import logging
import os
from .broker import publish
from .config import load_settings
from .storage.postgres import PostgresStore
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s outbox %(message)s")
async def run() -> None:
settings = load_settings()
store = PostgresStore(settings.database_url)
await store.connect()
interval = max(0.1, float(os.environ.get("OUTBOX_DISPATCH_INTERVAL_SECONDS", "1")))
try:
while True:
try:
await store.schedule_due_jobs()
for event in await store.due_outbox_events():
await publish(event["payload"])
await store.mark_outbox_published(event["id"])
except Exception:
logging.exception("outbox dispatch failed; durable event remains pending")
await asyncio.sleep(interval)
finally:
await store.close()
if __name__ == "__main__":
asyncio.run(run())
+24
View File
@@ -0,0 +1,24 @@
"""Operator-only safe replay for a failed attachment job."""
import argparse
import asyncio
from uuid import UUID
from .config import load_settings
from .storage.postgres import PostgresStore
async def run(attachment_id: UUID) -> None:
store = PostgresStore(load_settings().database_url)
await store.connect()
try:
if not await store.replay_failed_attachment(attachment_id):
raise SystemExit("attachment is not in failed state")
finally:
await store.close()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("attachment_id", type=UUID)
args = parser.parse_args()
asyncio.run(run(args.attachment_id))
+83 -15
View File
@@ -1,4 +1,4 @@
from datetime import datetime
from datetime import datetime, timedelta, timezone
from contextlib import asynccontextmanager
from hashlib import sha256
import json
@@ -330,6 +330,18 @@ class PostgresStore:
await (connection or self.pool).execute(
"INSERT INTO attachment_jobs (job_id, attachment_id) VALUES ($1, $2)", job_id, attachment_id
)
payload = {
"schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1",
"attachment_id": str(attachment_id), "pipeline_version": 1, "attempt": 1,
"created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
}
await (connection or self.pool).execute(
"INSERT INTO attachment_outbox (id, job_id, payload) VALUES ($1, $2, $3::jsonb)",
uuid4(), job_id, json.dumps(payload, separators=(",", ":")),
)
await (connection or self.pool).execute(
"UPDATE attachment_jobs SET next_attempt_at = 'infinity' WHERE job_id = $1", job_id
)
except DB_ERRORS as error:
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
return dict(row)
@@ -356,37 +368,44 @@ class PostgresStore:
result["variants"] = [dict(item) for item in variants]
return result
async def claim_attachment_job(self, attachment_id: UUID) -> dict | None:
async def claim_attachment_job(self, attachment_id: UUID, lease_seconds: int) -> dict | None:
try:
async with self.pool.acquire() as connection, connection.transaction():
row = await connection.fetchrow("SELECT * FROM attachments WHERE id = $1 FOR UPDATE", attachment_id)
row = await connection.fetchrow(
"SELECT a.*, j.job_id, j.attempt, j.lease_until FROM attachments a JOIN attachment_jobs j ON j.attachment_id = a.id "
"WHERE a.id = $1 FOR UPDATE", attachment_id
)
if row is None or row["status"] in {"ready", "failed"}:
return None
if row["status"] == "processing" and row["lease_until"] is not None and row["lease_until"] > datetime.now(timezone.utc):
return None
await connection.execute(
"UPDATE attachments SET status = 'processing', processing_started_at = now(), updated_at = now() WHERE id = $1",
attachment_id,
)
await connection.execute(
"UPDATE attachment_jobs SET status = 'processing', updated_at = now() WHERE attachment_id = $1", attachment_id
"UPDATE attachment_jobs SET status = 'processing', lease_until = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1",
attachment_id, lease_seconds,
)
return dict(row)
except DB_ERRORS as error:
raise RuntimeError("PostgreSQL is unavailable") from error
async def complete_attachment_job(self, attachment_id: UUID, original_key: str, variant: dict) -> None:
async def complete_attachment_job(self, attachment_id: UUID, original_key: str, variants: list[dict]) -> None:
try:
async with self.pool.acquire() as connection, connection.transaction():
await connection.execute(
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) "
"VALUES ($1, 1, 'thumbnail', $2, $3, $4, $5, $6, $7) "
"ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET object_key = EXCLUDED.object_key, content_type = EXCLUDED.content_type, size_bytes = EXCLUDED.size_bytes, width = EXCLUDED.width, height = EXCLUDED.height, sha256 = EXCLUDED.sha256",
attachment_id, variant["object_key"], variant["content_type"], variant["size_bytes"], variant["width"], variant["height"], variant["sha256"],
)
for variant in variants:
await connection.execute(
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) "
"VALUES ($1, 1, $2, $3, $4, $5, $6, $7, $8) "
"ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET object_key = EXCLUDED.object_key, content_type = EXCLUDED.content_type, size_bytes = EXCLUDED.size_bytes, width = EXCLUDED.width, height = EXCLUDED.height, sha256 = EXCLUDED.sha256",
attachment_id, variant["name"], variant["object_key"], variant["content_type"], variant["size_bytes"], variant["width"], variant["height"], variant["sha256"],
)
await connection.execute(
"UPDATE attachments SET status = 'ready', original_key = $2, error_code = NULL, updated_at = now() WHERE id = $1",
attachment_id, original_key,
)
await connection.execute("UPDATE attachment_jobs SET status = 'ready', updated_at = now() WHERE attachment_id = $1", attachment_id)
await connection.execute("UPDATE attachment_jobs SET status = 'ready', lease_until = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id)
except DB_ERRORS as error:
raise RuntimeError("PostgreSQL is unavailable") from error
@@ -401,7 +420,7 @@ class PostgresStore:
except DB_ERRORS as error:
raise RuntimeError("PostgreSQL is unavailable") from error
async def retry_attachment_job(self, attachment_id: UUID, max_attempts: int) -> bool:
async def retry_attachment_job(self, attachment_id: UUID, max_attempts: int, delay_seconds: float) -> bool:
"""Return true only when a bounded at-least-once retry was recorded."""
try:
async with self.pool.acquire() as connection, connection.transaction():
@@ -410,8 +429,8 @@ class PostgresStore:
)
if job is not None and job["attempt"] < max_attempts:
await connection.execute(
"UPDATE attachment_jobs SET attempt = attempt + 1, status = 'queued', last_error = 'processing_failed', updated_at = now() WHERE attachment_id = $1",
attachment_id,
"UPDATE attachment_jobs SET attempt = attempt + 1, status = 'queued', last_error = 'processing_failed', lease_until = NULL, next_attempt_at = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1",
attachment_id, delay_seconds,
)
await connection.execute(
"UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1", attachment_id
@@ -422,6 +441,55 @@ class PostgresStore:
except DB_ERRORS as error:
raise RuntimeError("PostgreSQL is unavailable") from error
async def due_outbox_events(self, limit: int = 20) -> list[dict]:
try:
return [dict(row) for row in await self.pool.fetch(
"SELECT id, payload FROM attachment_outbox WHERE published_at IS NULL ORDER BY created_at LIMIT $1", limit
)]
except DB_ERRORS as error:
raise RuntimeError("PostgreSQL is unavailable") from error
async def mark_outbox_published(self, event_id: UUID) -> None:
await self.pool.execute("UPDATE attachment_outbox SET published_at = now(), attempts = attempts + 1 WHERE id = $1", event_id)
async def schedule_due_jobs(self) -> None:
"""Outbox dispatcher also restores expired leases and scheduled retries."""
try:
async with self.pool.acquire() as connection, connection.transaction():
rows = await connection.fetch(
"SELECT job_id, attachment_id, pipeline_version, attempt, status FROM attachment_jobs "
"WHERE (status = 'queued' AND next_attempt_at <= now()) OR (status = 'processing' AND lease_until < now()) "
"FOR UPDATE SKIP LOCKED"
)
for row in rows:
if row["status"] == "processing":
await connection.execute(
"UPDATE attachment_jobs SET status = 'queued', lease_until = NULL, next_attempt_at = now(), last_error = 'lease_expired', updated_at = now() WHERE job_id = $1", row["job_id"]
)
await connection.execute("UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1", row["attachment_id"])
payload = {"schema_version": 1, "job_id": str(row["job_id"]), "type": "attachment.process.v1", "attachment_id": str(row["attachment_id"]), "pipeline_version": row["pipeline_version"], "attempt": row["attempt"], "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")}
await connection.execute("INSERT INTO attachment_outbox (id, job_id, payload) VALUES ($1, $2, $3::jsonb)", uuid4(), row["job_id"], json.dumps(payload, separators=(",", ":")))
await connection.execute("UPDATE attachment_jobs SET next_attempt_at = 'infinity' WHERE job_id = $1", row["job_id"])
except DB_ERRORS as error:
raise RuntimeError("PostgreSQL is unavailable") from error
async def replay_failed_attachment(self, attachment_id: UUID) -> bool:
"""Explicit operator-only replay; never requeues an already successful job."""
try:
async with self.pool.acquire() as connection, connection.transaction():
job = await connection.fetchrow(
"SELECT job_id, status FROM attachment_jobs WHERE attachment_id = $1 FOR UPDATE", attachment_id
)
if job is None or job["status"] != "failed":
return False
await connection.execute(
"UPDATE attachment_jobs SET status = 'queued', attempt = 1, next_attempt_at = now(), lease_until = NULL, replayed_at = now(), last_error = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id
)
await connection.execute("UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = now() WHERE id = $1", attachment_id)
return True
except DB_ERRORS as error:
raise RuntimeError("PostgreSQL is unavailable") from error
async def attachment_keys(self) -> dict[str, UUID]:
"""Inventory used only by the maintenance reconciler."""
try:
+55 -16
View File
@@ -5,11 +5,12 @@ from io import BytesIO
import json
import logging
import os
import random
from uuid import UUID
from PIL import Image, ImageOps
from .broker import QUEUE, _connection
from .broker import QUEUE, _connection, publish_dlq
from .config import load_settings
from .storage.postgres import PostgresStore
from .storage.s3 import S3Settings, S3Store
@@ -32,21 +33,55 @@ def thumbnail(data: bytes) -> tuple[bytes, int, int]:
return output.getvalue(), image.width, image.height
async def process(store: PostgresStore, s3: S3Store, payload: dict) -> None:
def variants(data: bytes, max_pixels: int = 40_000_000) -> list[tuple[str, bytes, str, int, int]]:
"""Render required variants without preserving EXIF or other input metadata."""
with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as source:
if source.width * source.height > max_pixels:
raise ValueError("image exceeds configured pixel limit")
image = ImageOps.exif_transpose(source).convert("RGBA")
white = Image.new("RGBA", image.size, "white")
white.alpha_composite(image)
rgb = white.convert("RGB")
thumb = rgb.copy()
thumb.thumbnail((256, 256), Image.Resampling.LANCZOS)
thumbnail_out = BytesIO(); thumb.save(thumbnail_out, "JPEG", quality=85, optimize=True)
side = min(rgb.width, rgb.height)
left, top = (rgb.width - side) // 2, (rgb.height - side) // 2
crop = rgb.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS)
crop_out = BytesIO(); crop.save(crop_out, "JPEG", quality=85, optimize=True)
optimized = rgb.copy()
optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
optimized_out = BytesIO(); optimized.save(optimized_out, "WEBP", quality=80, method=4)
return [
("thumbnail", thumbnail_out.getvalue(), "image/jpeg", thumb.width, thumb.height),
("crop", crop_out.getvalue(), "image/jpeg", 128, 128),
("optimized", optimized_out.getvalue(), "image/webp", optimized.width, optimized.height),
]
async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> None:
attachment_id = UUID(payload["attachment_id"])
attachment = await store.claim_attachment_job(attachment_id)
attachment = await store.claim_attachment_job(attachment_id, lease_seconds)
if attachment is None:
return
original_key = f"originals/{attachment_id}"
variant_key = f"variants/{attachment_id}/thumbnail.jpg"
await s3.copy(attachment["staging_key"], original_key, attachment["content_type"])
original = await s3.download_key(attachment["staging_key"], attachment["size_bytes"], attachment["sha256"])
rendered, width, height = await asyncio.to_thread(thumbnail, original)
await s3.put(variant_key, BytesIO(rendered), len(rendered), "image/jpeg")
await store.complete_attachment_job(attachment_id, original_key, {
"object_key": variant_key, "content_type": "image/jpeg", "size_bytes": len(rendered),
"width": width, "height": height, "sha256": sha256(rendered).hexdigest(),
})
rendered = await asyncio.wait_for(
asyncio.to_thread(variants, original, s3.settings.max_pixels), timeout=lease_seconds
)
metadata = []
for name, content, content_type, width, height in rendered:
extension = "webp" if name == "optimized" else "jpg"
object_key = f"variants/{attachment_id}/{name}.{extension}"
await s3.put(object_key, BytesIO(content), len(content), content_type)
metadata.append({"name": name, "object_key": object_key, "content_type": content_type,
"size_bytes": len(content), "width": width, "height": height,
"sha256": sha256(content).hexdigest()})
await store.complete_attachment_job(attachment_id, original_key, metadata)
await s3.delete(attachment["staging_key"])
@@ -65,20 +100,24 @@ def main() -> None:
payload = json.loads(body)
if payload["type"] != "attachment.process.v1" or payload["schema_version"] != 1:
raise ValueError("unexpected broker payload")
asyncio.run(process(store, s3, payload))
asyncio.run(process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")))))
except Exception:
logging.exception("job failed")
try:
attachment_id = UUID(payload["attachment_id"])
attempt = int(payload.get("attempt", 1))
delay = min(60.0, float(2 ** min(attempt, 6))) * random.uniform(0.5, 1.5)
retry = asyncio.run(store.retry_attachment_job(
attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5")))
attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))), delay
))
except Exception:
retry = False
if retry:
ch.basic_nack(method.delivery_tag, requeue=True)
else:
ch.basic_ack(method.delivery_tag)
if not retry:
try:
asyncio.run(publish_dlq(payload))
except Exception:
logging.exception("could not publish failed job to DLQ")
ch.basic_ack(method.delivery_tag)
else:
ch.basic_ack(method.delivery_tag)