diff --git a/REPORT.md b/REPORT.md index 8aac7727..6c46d248 100644 --- a/REPORT.md +++ b/REPORT.md @@ -11,7 +11,9 @@ Python 3.12, PostgreSQL 17.6, SeaweedFS 4.17, RabbitMQ 4.1 и HAProxy 3.0 зап ## Архитектура и решения -PostgreSQL хранит метаданные и состояние, SeaweedFS — private staging/final objects, RabbitMQ — durable сообщения, worker — отдельный процесс. До HTTP 202 API полностью принимает и валидирует bytes, сохраняет staging-object и в одной DB-транзакции метаданные + job, затем ждёт RabbitMQ publisher confirm. Worker ack делает только после durable результата либо после финального failed. Обработка идемпотентна по attachment/pipeline/variant: повтор ready-job не меняет результат. +PostgreSQL хранит метаданные, jobs и transactional outbox; SeaweedFS — private staging/final objects; RabbitMQ — durable сообщения; worker и outbox-dispatcher — отдельные процессы. До HTTP 202 API полностью принимает и валидирует bytes, сохраняет staging-object и в одной DB-транзакции метаданные + job + outbox event. Dispatcher публикует событие с publisher confirm и отмечает event после confirm, поэтому crash между COMMIT и publish восстанавливается без ручной job. Worker ack делает только после durable результата либо после финального failed. Обработка идемпотентна по attachment/pipeline/variant: повтор ready-job не меняет результат. Lease ограничен `JOB_TIMEOUT_SECONDS`; истёкшая processing-job возвращается в очередь dispatcher-ом. Retry ограничен `JOB_MAX_ATTEMPTS`, использует exponential backoff с jitter; исчерпанная job остаётся failed и её служебное сообщение попадает в DLQ. Явный replay: `docker compose run --rm outbox /app/.venv/bin/python -m src.replay `. + +На уровне 4 worker создаёт: thumbnail JPEG 256×256 без увеличения; crop JPEG 128×128, центрированный квадрат; optimized WebP до 1280×1280 без увеличения. JPEG quality=85, WebP quality=80; входные EXIF/metadata не сохраняются. ## Доказательства критериев diff --git a/compose.yaml b/compose.yaml index 9fadfa2a..4b4edd5d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -19,6 +19,7 @@ x-api: &api UPLOAD_CONCURRENCY: "1" UPLOAD_TIMEOUT_SECONDS: "30" UPLOAD_FAULT_AFTER_S3_PUT: ${UPLOAD_FAULT_AFTER_S3_PUT:-0} + OUTBOX_FAULT_AFTER_COMMIT: ${OUTBOX_FAULT_AFTER_COMMIT:-0} secrets: - database_url - cursor_secret @@ -197,6 +198,8 @@ services: BROKER_URL_FILE: /run/secrets/broker_url WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} secrets: - database_url - cursor_secret @@ -212,6 +215,27 @@ services: condition: service_healthy stop_grace_period: 30s + outbox: + build: + context: . + command: ["/app/.venv/bin/python", "-m", "src.outbox"] + environment: + APP_ENV: lab5 + DATABASE_URL_FILE: /run/secrets/database_url + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + BROKER_URL_FILE: /run/secrets/broker_url + OUTBOX_DISPATCH_INTERVAL_SECONDS: "1" + secrets: + - database_url + - cursor_secret + - broker_url + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + restart: unless-stopped + load_balancer: image: haproxy:3.0.11-alpine ports: diff --git a/migrations/006_level4_reliability.sql b/migrations/006_level4_reliability.sql new file mode 100644 index 00000000..3be954c0 --- /dev/null +++ b/migrations/006_level4_reliability.sql @@ -0,0 +1,17 @@ +ALTER TABLE attachment_jobs + ADD COLUMN next_attempt_at timestamptz NOT NULL DEFAULT now(), + ADD COLUMN lease_until timestamptz, + ADD COLUMN replayed_at timestamptz; +CREATE INDEX attachment_jobs_due_idx ON attachment_jobs(status, next_attempt_at); +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs(status, lease_until); + +CREATE TABLE attachment_outbox ( + id uuid PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE, + payload jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + attempts integer NOT NULL DEFAULT 0, + UNIQUE (job_id, published_at) +); +CREATE INDEX attachment_outbox_pending_idx ON attachment_outbox(created_at) WHERE published_at IS NULL; diff --git a/src/attachments.py b/src/attachments.py index f3bf8947..790732de 100644 --- a/src/attachments.py +++ b/src/attachments.py @@ -1,5 +1,4 @@ import asyncio -from datetime import datetime, timezone from hashlib import sha256 import os from tempfile import TemporaryFile @@ -16,7 +15,6 @@ from .errors import ApiError from .models import Attachment from .routes import get_current_user, get_store from .storage.postgres import PostgresStore -from .broker import publish router = APIRouter() Image.MAX_IMAGE_PIXELS = 40000000 @@ -93,11 +91,9 @@ async def upload(chat_id: UUID, request: Request, filename: str = Query(...), attachment_id, chat_id, user["id"], filename, content_type, size, digest.hexdigest(), key, job_id, connection=connection, ) - await publish({ - "schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1", - "attachment_id": str(attachment_id), "pipeline_version": 1, "attempt": 1, - "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), - }) + # Fault-injection for level-4 proof: durable outbox must recover this commit. + if os.environ.get("OUTBOX_FAULT_AFTER_COMMIT") == "1": + os._exit(70) attachment["variants"] = [] return attachment except ClientDisconnect: diff --git a/src/broker.py b/src/broker.py index e42358e1..3e5b1262 100644 --- a/src/broker.py +++ b/src/broker.py @@ -9,6 +9,7 @@ from starlette.concurrency import run_in_threadpool from .errors import ApiError QUEUE = "attachment.process.v1" +DLQ = "attachment.process.v1.dlq" def broker_url() -> str: @@ -21,7 +22,9 @@ def broker_url() -> str: def _connection() -> pika.BlockingConnection: connection = pika.BlockingConnection(pika.URLParameters(broker_url())) - connection.channel().queue_declare(queue=QUEUE, durable=True) + channel = connection.channel() + channel.queue_declare(queue=QUEUE, durable=True) + channel.queue_declare(queue=DLQ, durable=True) return connection @@ -47,6 +50,25 @@ async def publish(payload: dict) -> None: raise ApiError(503, "unavailable", "Брокер недоступен") from error +async def publish_dlq(payload: dict) -> None: + payload = {**payload, "failure": "retry_exhausted"} + try: + await run_in_threadpool(_publish_to, DLQ, payload) + except (pika.exceptions.AMQPError, OSError, ValueError) as error: + raise RuntimeError("DLQ is unavailable") from error + + +def _publish_to(queue: str, payload: dict) -> None: + connection = _connection() + try: + connection.channel().basic_publish( + exchange="", routing_key=queue, body=json.dumps(payload, separators=(",", ":")).encode("utf-8"), + properties=pika.BasicProperties(delivery_mode=pika.DeliveryMode.Persistent, content_type="application/json"), + ) + finally: + connection.close() + + async def ready() -> None: connection = None try: diff --git a/src/outbox.py b/src/outbox.py new file mode 100644 index 00000000..340928ec --- /dev/null +++ b/src/outbox.py @@ -0,0 +1,33 @@ +"""Transactional-outbox dispatcher; safe to run as one or more replicas.""" +import asyncio +import logging +import os + +from .broker import publish +from .config import load_settings +from .storage.postgres import PostgresStore + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s outbox %(message)s") + + +async def run() -> None: + settings = load_settings() + store = PostgresStore(settings.database_url) + await store.connect() + interval = max(0.1, float(os.environ.get("OUTBOX_DISPATCH_INTERVAL_SECONDS", "1"))) + try: + while True: + try: + await store.schedule_due_jobs() + for event in await store.due_outbox_events(): + await publish(event["payload"]) + await store.mark_outbox_published(event["id"]) + except Exception: + logging.exception("outbox dispatch failed; durable event remains pending") + await asyncio.sleep(interval) + finally: + await store.close() + + +if __name__ == "__main__": + asyncio.run(run()) diff --git a/src/replay.py b/src/replay.py new file mode 100644 index 00000000..2360cb5e --- /dev/null +++ b/src/replay.py @@ -0,0 +1,24 @@ +"""Operator-only safe replay for a failed attachment job.""" +import argparse +import asyncio +from uuid import UUID + +from .config import load_settings +from .storage.postgres import PostgresStore + + +async def run(attachment_id: UUID) -> None: + store = PostgresStore(load_settings().database_url) + await store.connect() + try: + if not await store.replay_failed_attachment(attachment_id): + raise SystemExit("attachment is not in failed state") + finally: + await store.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("attachment_id", type=UUID) + args = parser.parse_args() + asyncio.run(run(args.attachment_id)) diff --git a/src/storage/postgres.py b/src/storage/postgres.py index 4b609e0c..e87fec81 100644 --- a/src/storage/postgres.py +++ b/src/storage/postgres.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timedelta, timezone from contextlib import asynccontextmanager from hashlib import sha256 import json @@ -330,6 +330,18 @@ class PostgresStore: await (connection or self.pool).execute( "INSERT INTO attachment_jobs (job_id, attachment_id) VALUES ($1, $2)", job_id, attachment_id ) + payload = { + "schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1", + "attachment_id": str(attachment_id), "pipeline_version": 1, "attempt": 1, + "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + } + await (connection or self.pool).execute( + "INSERT INTO attachment_outbox (id, job_id, payload) VALUES ($1, $2, $3::jsonb)", + uuid4(), job_id, json.dumps(payload, separators=(",", ":")), + ) + await (connection or self.pool).execute( + "UPDATE attachment_jobs SET next_attempt_at = 'infinity' WHERE job_id = $1", job_id + ) except DB_ERRORS as error: raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error return dict(row) @@ -356,37 +368,44 @@ class PostgresStore: result["variants"] = [dict(item) for item in variants] return result - async def claim_attachment_job(self, attachment_id: UUID) -> dict | None: + async def claim_attachment_job(self, attachment_id: UUID, lease_seconds: int) -> dict | None: try: async with self.pool.acquire() as connection, connection.transaction(): - row = await connection.fetchrow("SELECT * FROM attachments WHERE id = $1 FOR UPDATE", attachment_id) + row = await connection.fetchrow( + "SELECT a.*, j.job_id, j.attempt, j.lease_until FROM attachments a JOIN attachment_jobs j ON j.attachment_id = a.id " + "WHERE a.id = $1 FOR UPDATE", attachment_id + ) if row is None or row["status"] in {"ready", "failed"}: return None + if row["status"] == "processing" and row["lease_until"] is not None and row["lease_until"] > datetime.now(timezone.utc): + return None await connection.execute( "UPDATE attachments SET status = 'processing', processing_started_at = now(), updated_at = now() WHERE id = $1", attachment_id, ) await connection.execute( - "UPDATE attachment_jobs SET status = 'processing', updated_at = now() WHERE attachment_id = $1", attachment_id + "UPDATE attachment_jobs SET status = 'processing', lease_until = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1", + attachment_id, lease_seconds, ) return dict(row) except DB_ERRORS as error: raise RuntimeError("PostgreSQL is unavailable") from error - async def complete_attachment_job(self, attachment_id: UUID, original_key: str, variant: dict) -> None: + async def complete_attachment_job(self, attachment_id: UUID, original_key: str, variants: list[dict]) -> None: try: async with self.pool.acquire() as connection, connection.transaction(): - await connection.execute( - "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) " - "VALUES ($1, 1, 'thumbnail', $2, $3, $4, $5, $6, $7) " - "ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET object_key = EXCLUDED.object_key, content_type = EXCLUDED.content_type, size_bytes = EXCLUDED.size_bytes, width = EXCLUDED.width, height = EXCLUDED.height, sha256 = EXCLUDED.sha256", - attachment_id, variant["object_key"], variant["content_type"], variant["size_bytes"], variant["width"], variant["height"], variant["sha256"], - ) + for variant in variants: + await connection.execute( + "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) " + "VALUES ($1, 1, $2, $3, $4, $5, $6, $7, $8) " + "ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET object_key = EXCLUDED.object_key, content_type = EXCLUDED.content_type, size_bytes = EXCLUDED.size_bytes, width = EXCLUDED.width, height = EXCLUDED.height, sha256 = EXCLUDED.sha256", + attachment_id, variant["name"], variant["object_key"], variant["content_type"], variant["size_bytes"], variant["width"], variant["height"], variant["sha256"], + ) await connection.execute( "UPDATE attachments SET status = 'ready', original_key = $2, error_code = NULL, updated_at = now() WHERE id = $1", attachment_id, original_key, ) - await connection.execute("UPDATE attachment_jobs SET status = 'ready', updated_at = now() WHERE attachment_id = $1", attachment_id) + await connection.execute("UPDATE attachment_jobs SET status = 'ready', lease_until = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id) except DB_ERRORS as error: raise RuntimeError("PostgreSQL is unavailable") from error @@ -401,7 +420,7 @@ class PostgresStore: except DB_ERRORS as error: raise RuntimeError("PostgreSQL is unavailable") from error - async def retry_attachment_job(self, attachment_id: UUID, max_attempts: int) -> bool: + async def retry_attachment_job(self, attachment_id: UUID, max_attempts: int, delay_seconds: float) -> bool: """Return true only when a bounded at-least-once retry was recorded.""" try: async with self.pool.acquire() as connection, connection.transaction(): @@ -410,8 +429,8 @@ class PostgresStore: ) if job is not None and job["attempt"] < max_attempts: await connection.execute( - "UPDATE attachment_jobs SET attempt = attempt + 1, status = 'queued', last_error = 'processing_failed', updated_at = now() WHERE attachment_id = $1", - attachment_id, + "UPDATE attachment_jobs SET attempt = attempt + 1, status = 'queued', last_error = 'processing_failed', lease_until = NULL, next_attempt_at = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1", + attachment_id, delay_seconds, ) await connection.execute( "UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1", attachment_id @@ -422,6 +441,55 @@ class PostgresStore: except DB_ERRORS as error: raise RuntimeError("PostgreSQL is unavailable") from error + async def due_outbox_events(self, limit: int = 20) -> list[dict]: + try: + return [dict(row) for row in await self.pool.fetch( + "SELECT id, payload FROM attachment_outbox WHERE published_at IS NULL ORDER BY created_at LIMIT $1", limit + )] + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def mark_outbox_published(self, event_id: UUID) -> None: + await self.pool.execute("UPDATE attachment_outbox SET published_at = now(), attempts = attempts + 1 WHERE id = $1", event_id) + + async def schedule_due_jobs(self) -> None: + """Outbox dispatcher also restores expired leases and scheduled retries.""" + try: + async with self.pool.acquire() as connection, connection.transaction(): + rows = await connection.fetch( + "SELECT job_id, attachment_id, pipeline_version, attempt, status FROM attachment_jobs " + "WHERE (status = 'queued' AND next_attempt_at <= now()) OR (status = 'processing' AND lease_until < now()) " + "FOR UPDATE SKIP LOCKED" + ) + for row in rows: + if row["status"] == "processing": + await connection.execute( + "UPDATE attachment_jobs SET status = 'queued', lease_until = NULL, next_attempt_at = now(), last_error = 'lease_expired', updated_at = now() WHERE job_id = $1", row["job_id"] + ) + await connection.execute("UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1", row["attachment_id"]) + payload = {"schema_version": 1, "job_id": str(row["job_id"]), "type": "attachment.process.v1", "attachment_id": str(row["attachment_id"]), "pipeline_version": row["pipeline_version"], "attempt": row["attempt"], "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")} + await connection.execute("INSERT INTO attachment_outbox (id, job_id, payload) VALUES ($1, $2, $3::jsonb)", uuid4(), row["job_id"], json.dumps(payload, separators=(",", ":"))) + await connection.execute("UPDATE attachment_jobs SET next_attempt_at = 'infinity' WHERE job_id = $1", row["job_id"]) + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def replay_failed_attachment(self, attachment_id: UUID) -> bool: + """Explicit operator-only replay; never requeues an already successful job.""" + try: + async with self.pool.acquire() as connection, connection.transaction(): + job = await connection.fetchrow( + "SELECT job_id, status FROM attachment_jobs WHERE attachment_id = $1 FOR UPDATE", attachment_id + ) + if job is None or job["status"] != "failed": + return False + await connection.execute( + "UPDATE attachment_jobs SET status = 'queued', attempt = 1, next_attempt_at = now(), lease_until = NULL, replayed_at = now(), last_error = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id + ) + await connection.execute("UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = now() WHERE id = $1", attachment_id) + return True + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + async def attachment_keys(self) -> dict[str, UUID]: """Inventory used only by the maintenance reconciler.""" try: diff --git a/src/worker.py b/src/worker.py index c80c5f60..730f94e9 100644 --- a/src/worker.py +++ b/src/worker.py @@ -5,11 +5,12 @@ from io import BytesIO import json import logging import os +import random from uuid import UUID from PIL import Image, ImageOps -from .broker import QUEUE, _connection +from .broker import QUEUE, _connection, publish_dlq from .config import load_settings from .storage.postgres import PostgresStore from .storage.s3 import S3Settings, S3Store @@ -32,21 +33,55 @@ def thumbnail(data: bytes) -> tuple[bytes, int, int]: return output.getvalue(), image.width, image.height -async def process(store: PostgresStore, s3: S3Store, payload: dict) -> None: +def variants(data: bytes, max_pixels: int = 40_000_000) -> list[tuple[str, bytes, str, int, int]]: + """Render required variants without preserving EXIF or other input metadata.""" + with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as source: + if source.width * source.height > max_pixels: + raise ValueError("image exceeds configured pixel limit") + image = ImageOps.exif_transpose(source).convert("RGBA") + white = Image.new("RGBA", image.size, "white") + white.alpha_composite(image) + rgb = white.convert("RGB") + + thumb = rgb.copy() + thumb.thumbnail((256, 256), Image.Resampling.LANCZOS) + thumbnail_out = BytesIO(); thumb.save(thumbnail_out, "JPEG", quality=85, optimize=True) + + side = min(rgb.width, rgb.height) + left, top = (rgb.width - side) // 2, (rgb.height - side) // 2 + crop = rgb.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS) + crop_out = BytesIO(); crop.save(crop_out, "JPEG", quality=85, optimize=True) + + optimized = rgb.copy() + optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS) + optimized_out = BytesIO(); optimized.save(optimized_out, "WEBP", quality=80, method=4) + return [ + ("thumbnail", thumbnail_out.getvalue(), "image/jpeg", thumb.width, thumb.height), + ("crop", crop_out.getvalue(), "image/jpeg", 128, 128), + ("optimized", optimized_out.getvalue(), "image/webp", optimized.width, optimized.height), + ] + + +async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> None: attachment_id = UUID(payload["attachment_id"]) - attachment = await store.claim_attachment_job(attachment_id) + attachment = await store.claim_attachment_job(attachment_id, lease_seconds) if attachment is None: return original_key = f"originals/{attachment_id}" - variant_key = f"variants/{attachment_id}/thumbnail.jpg" await s3.copy(attachment["staging_key"], original_key, attachment["content_type"]) original = await s3.download_key(attachment["staging_key"], attachment["size_bytes"], attachment["sha256"]) - rendered, width, height = await asyncio.to_thread(thumbnail, original) - await s3.put(variant_key, BytesIO(rendered), len(rendered), "image/jpeg") - await store.complete_attachment_job(attachment_id, original_key, { - "object_key": variant_key, "content_type": "image/jpeg", "size_bytes": len(rendered), - "width": width, "height": height, "sha256": sha256(rendered).hexdigest(), - }) + rendered = await asyncio.wait_for( + asyncio.to_thread(variants, original, s3.settings.max_pixels), timeout=lease_seconds + ) + metadata = [] + for name, content, content_type, width, height in rendered: + extension = "webp" if name == "optimized" else "jpg" + object_key = f"variants/{attachment_id}/{name}.{extension}" + await s3.put(object_key, BytesIO(content), len(content), content_type) + metadata.append({"name": name, "object_key": object_key, "content_type": content_type, + "size_bytes": len(content), "width": width, "height": height, + "sha256": sha256(content).hexdigest()}) + await store.complete_attachment_job(attachment_id, original_key, metadata) await s3.delete(attachment["staging_key"]) @@ -65,20 +100,24 @@ def main() -> None: payload = json.loads(body) if payload["type"] != "attachment.process.v1" or payload["schema_version"] != 1: raise ValueError("unexpected broker payload") - asyncio.run(process(store, s3, payload)) + asyncio.run(process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))))) except Exception: logging.exception("job failed") try: attachment_id = UUID(payload["attachment_id"]) + attempt = int(payload.get("attempt", 1)) + delay = min(60.0, float(2 ** min(attempt, 6))) * random.uniform(0.5, 1.5) retry = asyncio.run(store.retry_attachment_job( - attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))) + attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))), delay )) except Exception: retry = False - if retry: - ch.basic_nack(method.delivery_tag, requeue=True) - else: - ch.basic_ack(method.delivery_tag) + if not retry: + try: + asyncio.run(publish_dlq(payload)) + except Exception: + logging.exception("could not publish failed job to DLQ") + ch.basic_ack(method.delivery_tag) else: ch.basic_ack(method.delivery_tag)