From 655f264104db050a1edec13b5352de167af03aeb Mon Sep 17 00:00:00 2001 From: Alexey Bodanyuk <144894236+wfa22@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:12:52 +0300 Subject: [PATCH] =?UTF-8?q?lab5=20API:=20upload=20202=20queued=20=D1=87?= =?UTF-8?q?=D0=B5=D1=80=D0=B5=D0=B7=20staging=20+=20=D0=B7=D0=B0=D0=B4?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D0=B5=20+=20outbox=20=D0=BE=D0=B4=D0=BD?= =?UTF-8?q?=D0=BE=D0=B9=20=D1=82=D1=80=D0=B0=D0=BD=D0=B7=D0=B0=D0=BA=D1=86?= =?UTF-8?q?=D0=B8=D0=B5=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Порядок: pending-намерение -> PUT в приватный staging bucket -> COMMIT queued + jobs + outbox -> 202. Проверки 413/415 lab4 остаются синхронными. GET видит queued/processing/failed, content: 409 до ready, варианты thumbnail/crop/optimized; к сообщению можно прикрепить queued/processing, failed -> 409. API не связан с брокером. --- src/messenger/app.py | 93 +++++++++++++++--------- src/messenger/attachments.py | 133 ++++++++++++++++++++++------------- src/messenger/db.py | 6 +- src/messenger/repo.py | 15 ++-- src/messenger/storage.py | 17 ++++- src/messenger/validation.py | 16 ++--- 6 files changed, 176 insertions(+), 104 deletions(-) diff --git a/src/messenger/app.py b/src/messenger/app.py index 38100ad..378ea33 100644 --- a/src/messenger/app.py +++ b/src/messenger/app.py @@ -22,7 +22,7 @@ from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Sen from . import attachments, authhdr, cursor, docs, passwords, repo, validation from .config import Settings from .db import Database -from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unauthorized, Unavailable +from .errors import BASIC_CHALLENGE, ApiError, Conflict, InvalidRequest, NotFound, PayloadTooLarge, Unauthorized, Unavailable from .sessions import SessionStore, build_store from .storage import ObjectMissing, Storage from .timefmt import rfc3339 @@ -132,6 +132,10 @@ def storage_of(request: Request) -> Storage: return request.app.state.storage +def staging_of(request: Request) -> Storage: + return request.app.state.staging + + async def authenticate(request: Request) -> str: """User id of a valid Bearer session, otherwise 401. X-User-Id is ignored since lab 2. @@ -363,16 +367,17 @@ async def list_messages(request: Request) -> JSONResponse: return ok({"items": [m.to_json() for m in items], "next_cursor": next_cursor}) -# ------------------------------------------------------------------ attachments (lab 4) +# ------------------------------------------------------------------ attachments (lab 4, lab 5) def fault(request: Request, point: str) -> None: - """Failure injection for the level 4 experiments (FAULT_POINTS, compose.faults.yaml only).""" + """Failure injection for the experiments (FAULT_POINTS of the api service, compose variable + API_FAULT_POINTS; empty in normal runs).""" if point in request.app.state.settings.fault_points: log.warning("fault_injected", extra={"fields": {"request_id": request.state.request_id, "point": point}}) raise Unavailable("Внедрённый отказ: " + point) async def upload_attachment(request: Request) -> JSONResponse: - """POST /chats/{chat_id}/attachments?filename=...: raw PNG/JPEG -> 201 ready (attachments.py).""" + """POST /chats/{chat_id}/attachments?filename=...: raw PNG/JPEG -> 202 queued (attachments.py).""" settings: Settings = request.app.state.settings reader = attachments.BodyReader(request) slots: asyncio.Semaphore = request.app.state.upload_slots @@ -406,23 +411,25 @@ async def upload_attachment(request: Request) -> JSONResponse: async def store_upload(request: Request, db: Database, chat_id: str, user_id: str, filename: str, upload: attachments.Upload) -> JSONResponse: - """intent (pending) -> S3 PUT -> ready -> 201 (order and failure analysis: docs/CONSISTENCY.md).""" + """intent (pending) -> staging PUT -> queued + job + outbox -> 202 (docs/PIPELINE.md). + No resize here: the worker does it; a stopped worker only delays `ready`.""" + settings: Settings = request.app.state.settings attachment_id = uuid.uuid4() - object_key = attachments.new_object_key() - storage = storage_of(request) + staging_key = attachments.new_staging_key() await attachments.create_intent(db, attachment_id, chat_id, user_id, filename, upload.info, len(upload.data), - upload.sha256, object_key) - await storage.put(object_key, upload.data, upload.info.content_type, - {"attachment-id": str(attachment_id), "sha256": upload.sha256.hex()}) + upload.sha256, staging_key) + await staging_of(request).put(staging_key, upload.data, upload.info.content_type, + {"attachment-id": str(attachment_id), "sha256": upload.sha256.hex()}) fault(request, "after_s3_put") - attachment = await attachments.mark_ready(db, attachment_id) + attachment = await attachments.enqueue(db, attachment_id, settings) if attachment is None: raise Unavailable("Загрузка не завершена, повторите запрос") - fault(request, "after_ready_commit") - log.info("attachment_created", extra={"fields": {"request_id": request.state.request_id, - "attachment_id": attachment.id, "chat_id": chat_id, - "size": attachment.size_bytes, "type": attachment.content_type}}) - return ok(attachment.to_json(), 201) + fault(request, "after_enqueue_commit") + log.info("attachment_queued", extra={"fields": {"request_id": request.state.request_id, + "attachment_id": attachment.id, "chat_id": chat_id, + "size": attachment.size_bytes, "type": attachment.content_type, + "pipeline_version": settings.pipeline_version}}) + return ok(attachment.to_json(), 202) async def get_attachment(request: Request) -> JSONResponse: @@ -439,46 +446,60 @@ def content_disposition(filename: str) -> str: async def download_attachment(request: Request) -> Response: - """Original bytes through the authorized API: no redirect, no public S3 URL. Streamed from - S3 in 64 KiB chunks (memory per download does not depend on the file size).""" + """Original or variant bytes through the authorized API: no redirect, no public S3 URL. + queued/processing/failed -> 409 (not ready); a variant a ready attachment lacks -> 404. + Streamed from S3 in 64 KiB chunks (memory per download does not depend on the file size).""" user_id = await authenticate(request) attachment_id = path_uuid(request, "attachment_id") query = validation.query_items(request.scope["query_string"]) try: - validation.variant_param(query) - except validation.NotFoundVariant: + variant = validation.variant_param(query) + except ApiError: await attachments.get_visible(db_of(request), user_id, attachment_id) # 404 for outsiders first - raise NotFound("Такого варианта у вложения нет") from None + raise attachment = await attachments.get_visible(db_of(request), user_id, attachment_id) + if attachment.status != "ready": + raise Conflict("Обработка вложения не удалась" if attachment.status == "failed" else "Вложение ещё не готово") if attachment.lost: raise NotFound("Оригинал вложения утрачен") + if variant == "original": + key, size_expected, media_type, etag = (attachment.object_key, attachment.size_bytes, + attachment.content_type, attachment.sha256) + name = attachment.filename + else: + found = attachment.variant(variant) + if found is None: + raise NotFound("Такого варианта у вложения нет") + key, size_expected, media_type, etag = found.object_key, found.size_bytes, found.content_type, found.sha256 + stem = attachment.filename.rsplit(".", 1)[0] or "image" + name = stem + "-" + variant + (".webp" if media_type == "image/webp" else ".jpg") try: - size, chunks = await storage_of(request).open_stream(attachment.object_key) + size, chunks = await storage_of(request).open_stream(key) except ObjectMissing: - # A ready row without its object: data loss, reported by the reconciler (quarantine). log.error("attachment_object_missing", extra={"fields": {"request_id": request.state.request_id, - "attachment_id": attachment.id}}) - raise NotFound("Оригинал вложения утрачен") from None - if size != attachment.size_bytes: + "attachment_id": attachment.id, "variant": variant}}) + raise Unavailable("Файл вложения временно недоступен") from None + if size != size_expected: log.error("attachment_size_mismatch", extra={"fields": {"request_id": request.state.request_id, - "attachment_id": attachment.id}}) + "attachment_id": attachment.id, "variant": variant}}) await chunks.aclose() - raise Unavailable("Оригинал вложения временно недоступен") + raise Unavailable("Файл вложения временно недоступен") headers = { "Content-Length": str(size), - "Content-Disposition": content_disposition(attachment.filename), + "Content-Disposition": content_disposition(name), "X-Content-Type-Options": "nosniff", "Cache-Control": "private, no-store", - "ETag": '"' + attachment.sha256 + '"', + "ETag": '"' + etag + '"', } - return StreamingResponse(chunks, media_type=attachment.content_type, headers=headers) + return StreamingResponse(chunks, media_type=media_type, headers=headers) def create_app(settings: Settings | None = None) -> Starlette: settings = settings or Settings.from_env() db = Database(settings) sessions = build_store(settings, db) - storage = Storage(settings) + storage = Storage(settings) # final bucket: Read + staging = Storage(settings, bucket=settings.s3_staging_bucket) # staging bucket: Write (+ janitor DELETE) async def purge_loop() -> None: """Housekeeping: expired Idempotency-Key rows (older than IDEMPOTENCY_TTL_SECONDS).""" @@ -492,12 +513,12 @@ def create_app(settings: Settings | None = None) -> Starlette: pass # database unavailable right now; next round async def janitor_loop(every: float) -> None: - """Level 4 cleanup: intents of failed uploads older than PENDING_GRACE_SECONDS (attachments.py). + """Cleanup of intents of failed uploads older than PENDING_GRACE_SECONDS (attachments.py). Every replica runs it; the advisory lock lets one round run at a time.""" while True: await asyncio.sleep(every * (0.5 + random.random())) try: - removed = await attachments.clean_stale_pending(db, storage, settings.pending_grace_seconds) + removed = await attachments.clean_stale_pending(db, staging, settings.pending_grace_seconds) if removed: log.info("stale_uploads_removed", extra={"fields": {"count": removed}}) except ApiError: @@ -524,11 +545,13 @@ def create_app(settings: Settings | None = None) -> Starlette: passwords.configure(settings.hash_concurrency) await db.open() await storage.open() + await staging.open() log.info("dependencies_configured", extra={"fields": { "session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds, "db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds, "instance_id": settings.instance_id, "drain_s": settings.drain_seconds, "message_write_path": settings.message_write_path, "s3_bucket": settings.s3_bucket, + "s3_staging_bucket": settings.s3_staging_bucket, "pipeline_version": settings.pipeline_version, "upload_concurrency": settings.upload_concurrency, "verify_concurrency": settings.image_verify_concurrency, "pending_grace_s": settings.pending_grace_seconds, "fault_points": sorted(settings.fault_points)}}) @@ -543,6 +566,7 @@ def create_app(settings: Settings | None = None) -> Starlette: for task in tasks: task.cancel() await sessions.close() + await staging.close() await storage.close() await db.close() @@ -578,6 +602,7 @@ def create_app(settings: Settings | None = None) -> Starlette: app.state.db = db app.state.sessions = sessions app.state.storage = storage + app.state.staging = staging app.state.upload_slots = asyncio.Semaphore(settings.upload_concurrency) app.state.verify_slots = asyncio.Semaphore(settings.image_verify_concurrency) app.state.draining = False diff --git a/src/messenger/attachments.py b/src/messenger/attachments.py index ec670a6..74e5775 100644 --- a/src/messenger/attachments.py +++ b/src/messenger/attachments.py @@ -1,26 +1,29 @@ -"""Attachments (lab 4): metadata in PostgreSQL, original bytes in the private S3 bucket. +"""Attachments: metadata in PostgreSQL, bytes in private S3 buckets (lab 4), processed +asynchronously by the worker (lab 5, docs/PIPELINE.md). Upload (POST /chats/{id}/attachments, raw PNG/JPEG body), in this order: auth -> membership (404) -> filename / Content-Type / Content-Length (400/415/413) -> upload slot (at most UPLOAD_CONCURRENCY bodies per replica, else 503 + Retry-After) -> body in chunks: size limit, read deadline, format + pixel count from the first bytes - -> full integrity check (images.verify) -> SHA-256 / size known - -> INSERT row status='pending' with a fresh random object_key, COMMIT (intent) - -> PUT object to S3 (Content-MD5) - -> UPDATE status='ready' WHERE status='pending', COMMIT -> 201 ready. + -> full integrity check (images.verify) -> SHA-256 / size known (415 is decided HERE) + -> INSERT row status='pending' with a fresh random staging key, COMMIT (intent) + -> PUT the original into the STAGING bucket (Content-MD5) + -> ONE transaction: status 'pending' -> 'queued' + jobs row + outbox row, COMMIT + -> 202 queued. -Why 201 is honest: it is sent only after BOTH the object and the ready row are durable. -What a failure between the steps leaves behind, and who removes it (docs/CONSISTENCY.md): +What is durable at 202: the complete, verified original in the staging bucket (S3 confirmed the +PUT and its MD5), the metadata row, the job and the outbox row that makes the relay publish it. +A crash of the API after the answer loses nothing; a broker that is down only delays the publish. -* before the intent COMMIT: nothing (no object was written); -* after the intent, before/while PUT, before the ready COMMIT: a 'pending' row (+ maybe the - object). The client got 503 and may retry (a new attachment). The janitor (below) deletes the - object - idempotent, a missing key is fine - and then the row, once it is older than - PENDING_GRACE_SECONDS. Nothing is compensated in the request path: after a lost answer of - COMMIT we do not know whether the row is ready, and deleting the object could destroy a - ready attachment. Only the janitor decides, under a row lock, from the committed state; -* after the ready COMMIT (answer lost): a correct ready attachment the client does not know. +Failures between the steps (docs/PIPELINE.md, failure windows): +* before the intent COMMIT: nothing was written; +* after the intent, before/while PUT, or between PUT and the job COMMIT (FAULT_POINTS=after_s3_put): + a 'pending' row (+ maybe the staging object), invisible to everybody (404). The client got 503 + and may retry (a new attachment). The janitor deletes the staging object - idempotent - and then + the row once it is older than PENDING_GRACE_SECONDS, deciding under the row lock; +* after the job COMMIT, answer lost (FAULT_POINTS=after_enqueue_commit): a correct queued + attachment the client does not know; the worker processes it as usual. """ from __future__ import annotations @@ -36,11 +39,11 @@ import psycopg from psycopg import errors as pg_errors from starlette.requests import Request -from . import images +from . import images, jobs from .config import Settings from .db import Database from .errors import ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType -from .storage import OBJECT_PREFIX, Storage +from .storage import STAGING_PREFIX, Storage from .timefmt import rfc3339 log = logging.getLogger("messenger.attachments") @@ -48,6 +51,21 @@ log = logging.getLogger("messenger.attachments") JANITOR_LOCK_ID = 2026_04 # advisory lock: one janitor round at a time across all replicas +@dataclass(frozen=True, slots=True) +class Variant: + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + object_key: str + + def to_json(self) -> dict: + return {"name": self.name, "content_type": self.content_type, "size_bytes": self.size_bytes, + "width": self.width, "height": self.height, "sha256": self.sha256} + + @dataclass(frozen=True, slots=True) class Attachment: id: str @@ -57,100 +75,121 @@ class Attachment: content_type: str size_bytes: int sha256: str - object_key: str + object_key: str | None # published original in the final bucket (None until the worker publishes it) status: str created_at: dt.datetime updated_at: dt.datetime lost: bool = False + error_code: str | None = None + published_version: int | None = None + variants: tuple[Variant, ...] = () def to_json(self) -> dict: - """Public representation: no object key, no bucket, no internal state.""" + """Public representation: no object keys, no buckets, no job internals.""" return {"id": self.id, "chat_id": self.chat_id, "owner_id": self.owner_id, "filename": self.filename, "content_type": self.content_type, "size_bytes": self.size_bytes, "sha256": self.sha256, - "status": "ready", "variants": [], "error_code": None, + "status": self.status, "variants": [v.to_json() for v in self.variants] if self.status == "ready" else [], + "error_code": self.error_code if self.status == "failed" else None, "created_at": rfc3339(self.created_at), "updated_at": rfc3339(self.updated_at)} + def variant(self, name: str) -> Variant | None: + return next((v for v in self.variants if v.name == name), None) + COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.object_key, " - "a.status, a.created_at, a.updated_at, a.lost_at IS NOT NULL") + "a.status, a.created_at, a.updated_at, a.lost_at IS NOT NULL, a.error_code, a.published_version") -def _attachment(row) -> Attachment: +def _attachment(row, variants: tuple[Variant, ...] = ()) -> Attachment: return Attachment(str(row[0]), str(row[1]), str(row[2]), row[3], row[4], row[5], bytes(row[6]).hex(), row[7], - row[8], row[9], row[10], row[11]) + row[8], row[9], row[10], row[11], row[12], row[13], variants) -def new_object_key() -> str: +def new_staging_key() -> str: """Server-generated, 128 random bits, unrelated to the attachment id and the filename.""" - return OBJECT_PREFIX + secrets.token_hex(16) + return STAGING_PREFIX + secrets.token_hex(16) # ------------------------------------------------------------------ SQL async def create_intent(db: Database, attachment_id: uuid.UUID, chat_id: str, owner_id: str, filename: str, - info: images.ImageInfo, size: int, sha256: bytes, object_key: str) -> None: + info: images.ImageInfo, size: int, sha256: bytes, staging_key: str) -> None: async def work(conn: psycopg.AsyncConnection) -> None: await conn.execute( "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " - "width, height, object_key, status) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending')", + "width, height, staging_key, staging_state, status) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'present', 'pending')", (attachment_id, chat_id, owner_id, filename, info.content_type, size, sha256, info.width, info.height, - object_key)) + staging_key)) try: await db.run(work, transaction=False) except pg_errors.ForeignKeyViolation: raise NotFound("Чат не найден") from None -async def mark_ready(db: Database, attachment_id: uuid.UUID) -> Attachment | None: - """pending -> ready. None if the row is no longer pending (the janitor removed a stale intent).""" +async def enqueue(db: Database, attachment_id: uuid.UUID, settings: Settings) -> Attachment | None: + """pending -> queued + job + outbox row in ONE transaction (the durable handoff before 202). + None if the row is no longer pending (the janitor removed a stale intent).""" async def work(conn: psycopg.AsyncConnection) -> Attachment | None: cur = await conn.execute( - "UPDATE attachments a SET status = 'ready', updated_at = clock_timestamp() " + "UPDATE attachments a SET status = 'queued', updated_at = clock_timestamp() " f"WHERE a.id = %s AND a.status = 'pending' RETURNING {COLUMNS}", (attachment_id,)) row = await cur.fetchone() - return _attachment(row) if row else None - return await db.run(work, transaction=False) + if row is None: + return None + await jobs.create_job(conn, attachment_id, settings.pipeline_version, "upload", settings.job_max_attempts, + "api:" + settings.instance_id) + return _attachment(row) + return await db.run(work) async def get_visible(db: Database, user_id: str, attachment_id: str) -> Attachment: - """A READY attachment of a chat the user is a member of; anything else is 404 (unknown, - foreign chat, unfinished upload) - existence of foreign objects is never revealed.""" + """An attachment of a chat the user is a member of, in any state but the invisible intent + 'pending'; anything else is 404 (unknown, foreign chat, unfinished upload) - existence of + foreign objects is never revealed. A ready attachment comes with its published variants.""" async def work(conn: psycopg.AsyncConnection) -> Attachment: cur = await conn.execute( f"SELECT {COLUMNS} FROM attachments a " "JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s " - "WHERE a.id = %s AND a.status = 'ready'", (user_id, attachment_id)) + "WHERE a.id = %s AND a.status <> 'pending'", (user_id, attachment_id)) row = await cur.fetchone() if row is None: raise NotFound("Вложение не найдено") - return _attachment(row) - return await db.run(work, transaction=False) + variants: tuple[Variant, ...] = () + if row[8] == "ready": + cur = await conn.execute( + "SELECT name, content_type, size_bytes, width, height, sha256, object_key FROM attachment_variants " + "WHERE attachment_id = %s AND pipeline_version = %s " + "ORDER BY array_position(ARRAY['thumbnail', 'crop', 'optimized'], name)", (row[0], row[13])) + variants = tuple(Variant(r[0], r[1], r[2], r[3], r[4], bytes(r[5]).hex(), r[6]) for r in await cur.fetchall()) + return _attachment(row, variants) + # REPEATABLE READ is not needed: the variant rows of a published version never change. + return await db.run(work) -async def clean_stale_pending(db: Database, storage: Storage, grace_seconds: int, batch: int = 20) -> int: - """Janitor round: remove intents of uploads that never became ready. +async def clean_stale_pending(db: Database, staging: Storage, grace_seconds: int, batch: int = 20) -> int: + """Janitor round: remove intents of uploads that never got their job (no 202 was sent). Under FOR UPDATE: an upload that is (impossibly) still running would block on the row in - mark_ready and then find it gone (-> 503), so a ready attachment can never lose its object. - Order: object first (idempotent), row second; a crash in between leaves the pending row and - the next round repeats. Only status='pending' rows older than the grace period are touched: - fresh uploads and ready/used attachments are never selected. + enqueue() and then find it gone (-> 503). Order: staging object first (idempotent), row second; + a crash in between leaves the pending row and the next round repeats. Only status='pending' + rows older than the grace period are touched: queued/processing/ready/failed never. """ async def work(conn: psycopg.AsyncConnection) -> int: cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (JANITOR_LOCK_ID,)) if not (await cur.fetchone())[0]: return 0 cur = await conn.execute( - "SELECT id, object_key FROM attachments WHERE status = 'pending' " + "SELECT id, staging_key FROM attachments WHERE status = 'pending' " "AND created_at < clock_timestamp() - make_interval(secs => %s) " "ORDER BY created_at LIMIT %s FOR UPDATE SKIP LOCKED", (grace_seconds, batch)) rows = await cur.fetchall() if not rows: return 0 - await asyncio.gather(*(storage.delete(key) for _, key in rows)) + await asyncio.gather(*(staging.delete(key) for _, key in rows)) await conn.execute("DELETE FROM attachments WHERE id = ANY(%s) AND status = 'pending'", ([r[0] for r in rows],)) return len(rows) - return await db.run(work, deadline=storage.timeout + 5) + return await db.run(work, deadline=staging.timeout + 5) # ------------------------------------------------------------------ request body diff --git a/src/messenger/db.py b/src/messenger/db.py index 1c81bfd..c8f0866 100644 --- a/src/messenger/db.py +++ b/src/messenger/db.py @@ -55,7 +55,7 @@ def _forget(task: asyncio.Task) -> None: class Database: - def __init__(self, settings: Settings): + def __init__(self, settings: Settings, name: str = "messenger-api"): self.deadline = settings.db_timeout_seconds options = ( f"-c statement_timeout={settings.db_statement_timeout_ms} " @@ -68,12 +68,12 @@ class Database: max_size=settings.db_pool_max, timeout=settings.db_timeout_seconds, # max wait for a free connection open=False, - name="messenger", + name=name, check=AsyncConnectionPool.check_connection, # drop connections broken by a DB restart kwargs={ "options": options, "connect_timeout": max(1, int(settings.db_timeout_seconds)), - "application_name": "messenger-api", + "application_name": name, # visible in pg_stat_activity: api / relay / worker # Autocommit connections: a single statement (post_message function, ping) is its # own transaction with ONE round trip; multi-statement work uses an explicit # conn.transaction() (BEGIN ... COMMIT), see run(). diff --git a/src/messenger/repo.py b/src/messenger/repo.py index 05791c0..5ab8bbc 100644 --- a/src/messenger/repo.py +++ b/src/messenger/repo.py @@ -230,11 +230,12 @@ async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, return created_at -# Attachments a message may reference: ready, of THIS chat (else 404), not quarantined (else 409). -# FOR SHARE keeps a concurrent quarantine (reconciler) from slipping in before COMMIT. +# Attachments a message may reference (lab 5): of THIS chat and not the invisible 'pending' intent +# (else 404); queued/processing/ready are usable, failed or quarantined -> 409. +# FOR SHARE keeps a concurrent transition to failed from slipping in before COMMIT. CHECK_ATTACHMENTS = ( - "SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL) FROM (" - "SELECT lost_at FROM attachments WHERE id = ANY(%s::uuid[]) AND chat_id = %s AND status = 'ready' " + "SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL AND a.status <> 'failed') FROM (" + "SELECT lost_at, status FROM attachments WHERE id = ANY(%s::uuid[]) AND chat_id = %s AND status <> 'pending' " "FOR SHARE) a") @@ -280,7 +281,7 @@ async def post_message(db: Database, user_id: str, chat_id: str, key: str, reque ever left behind: a crash or 5xx rolls everything back and the client may retry); * same key, same body (any replica, also concurrently) -> the stored Message, 201, replayed; * same key, other body -> 409; the membership check comes first, so outsiders always get 404; - * attachments (lab 4) are checked before the key is claimed (same order as post_message_v2). + * attachments (lab 4) are checked before the key is claimed (same order as post_message_v3). """ message_id = uuid.uuid4() attachment_ids = list(attachment_ids) @@ -308,12 +309,12 @@ async def post_message(db: Database, user_id: str, chat_id: str, key: str, reque POST_MESSAGE_CALL = ("SELECT o_status, o_id, o_text, o_created_at, o_attachments " - "FROM post_message_v2(%s, %s, %s, %s, %s, %s, %s, %s::uuid[])") + "FROM post_message_v3(%s, %s, %s, %s, %s, %s, %s, %s::uuid[])") async def post_message_single_call(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, text: str, ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult: - """Same contract as post_message, ONE round trip: SQL function post_message_v2 (migration 0004 = + """Same contract as post_message, ONE round trip: SQL function post_message_v3 (migration 0005 = 0003 + attachment_ids) in autocommit mode, i.e. the call is the whole transaction.""" message_id = uuid.uuid4() diff --git a/src/messenger/storage.py b/src/messenger/storage.py index a8d7e6d..53b28a7 100644 --- a/src/messenger/storage.py +++ b/src/messenger/storage.py @@ -58,8 +58,10 @@ class ListedObject: class Storage: - def __init__(self, settings: Settings, *, access_key: str | None = None, secret_key: str | None = None): - self.bucket = settings.s3_bucket + def __init__(self, settings: Settings, *, access_key: str | None = None, secret_key: str | None = None, + bucket: str | None = None): + # Lab 5: one client per bucket - the final bucket (S3_BUCKET) or the staging bucket. + self.bucket = bucket or settings.s3_bucket self.timeout = settings.s3_timeout_seconds self._endpoint = settings.s3_endpoint self._region = settings.s3_region @@ -174,4 +176,13 @@ class Storage: SENTINEL_KEY = "health/sentinel" -OBJECT_PREFIX = "o/" # every attachment original lives under o/ +# Lab 4 originals: o/ in the final bucket (still readable). +OBJECT_PREFIX = "o/" +# Lab 5: s/ in the STAGING bucket (written by the API before 202) and +# a/// in the final bucket +# (written by the worker; the token makes every attempt write its own keys). +STAGING_PREFIX = "s/" + + +def attempt_key(attachment_id: str, token: str, name: str) -> str: + return f"a/{attachment_id}/{token}/{name}" diff --git a/src/messenger/validation.py b/src/messenger/validation.py index 3ad26ec..bd62fc4 100644 --- a/src/messenger/validation.py +++ b/src/messenger/validation.py @@ -181,19 +181,15 @@ def filename_param(query: list[tuple[str, str]]) -> str: return value -ATTACHMENT_VARIANTS = frozenset({"thumbnail", "crop", "optimized"}) # exist in the schema, not in lab 4 +ATTACHMENT_VARIANTS = ("original", "thumbnail", "crop", "optimized") def variant_param(query: list[tuple[str, str]]) -> str: - """`variant` of GET /attachments/{id}/content: only `original` exists in lab 4. A known variant - name is a missing variant of a ready object (404); anything else is an invalid value (400).""" + """`variant` of GET /attachments/{id}/content: original (default) or a lab 5 variant name. + Any other value is 400; a known name that a ready attachment lacks is 404 (handler).""" value = single_query_param(query, "variant") - if value is None or value == "original": + if value is None: return "original" if value in ATTACHMENT_VARIANTS: - raise NotFoundVariant() - raise InvalidRequest("variant должен быть original") - - -class NotFoundVariant(Exception): - """Raised for a variant that lab 4 does not produce; mapped to 404 by the handler.""" + return value + raise InvalidRequest("variant должен быть original, thumbnail, crop или optimized")