From a633949919e2153a6f5ec29a7e9625e41c86a782 Mon Sep 17 00:00:00 2001 From: Alexey Bodanyuk <144894236+wfa22@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:12:51 +0300 Subject: [PATCH] =?UTF-8?q?lab5:=20=D1=81=D1=85=D0=B5=D0=BC=D0=B0=200005?= =?UTF-8?q?=20(jobs,=20outbox,=20=D0=BF=D0=BE=D0=BF=D1=8B=D1=82=D0=BA?= =?UTF-8?q?=D0=B8,=20=D0=B2=D0=B0=D1=80=D0=B8=D0=B0=D0=BD=D1=82=D1=8B,=20?= =?UTF-8?q?=D0=B0=D1=83=D0=B4=D0=B8=D1=82)=20=D0=B8=20=D0=BC=D0=BE=D0=B4?= =?UTF-8?q?=D1=83=D0=BB=D0=B8=20pipeline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Миграция только добавляет: статусы pending/queued/processing/ready/failed, staging-ключ, published_version, jobs с lease и токеном попытки, job_attempts с заранее известными ключами, attachment_variants (PK = ключ эффекта), outbox, job_events, view failed_jobs, post_message_v3. CHECK attachments_active_has_source: у queued/processing всегда есть источник. pipeline.py — версии 1/2/3 и геометрия по контракту; imaging.py — варианты в дочернем процессе (RLIMIT_AS, таймаут, лимит пикселей); jobs.py — все переходы заданий одной транзакцией с fencing по токену; broker.py — топология и строгая схема сообщения. Зависимости: aio-pika 9.6.2, pillow 12.3.0 (lock с хешами). --- bench/reconcile_cost.py | 156 ------------ bench/reconcile_faults.py | 174 ------------- migrations/0005_pipeline.sql | 260 +++++++++++++++++++ requirements.in | 6 +- requirements.txt | 106 +++++++- src/messenger/broker.py | 86 +++++++ src/messenger/config.py | 131 +++++++++- src/messenger/imaging.py | 208 +++++++++++++++ src/messenger/jobs.py | 398 +++++++++++++++++++++++++++++ src/messenger/pipeline.py | 63 +++++ src/messenger/reconcile.py | 437 -------------------------------- student-tests/test_reconcile.py | 217 ---------------- 12 files changed, 1244 insertions(+), 998 deletions(-) delete mode 100644 bench/reconcile_cost.py delete mode 100644 bench/reconcile_faults.py create mode 100644 migrations/0005_pipeline.sql create mode 100644 src/messenger/broker.py create mode 100644 src/messenger/imaging.py create mode 100644 src/messenger/jobs.py create mode 100644 src/messenger/pipeline.py delete mode 100644 src/messenger/reconcile.py delete mode 100644 student-tests/test_reconcile.py diff --git a/bench/reconcile_cost.py b/bench/reconcile_cost.py deleted file mode 100644 index 8de1ba6..0000000 --- a/bench/reconcile_cost.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python3 -"""Track 2 experiment: cost of one reconciler pass as a function of the number of attachments. - -Hypothesis: a pass is O(N) with ~N/1000 S3 list requests + N/1000 DB pages and constant memory -(merge-join of two key-ordered streams); --verify-checksums adds reading every byte (O(total size)). - -Method: synthetic, CONSISTENT attachments (row 'ready' + 1 KiB object with the matching size and -SHA-256, filename 'bench-reconcile') are added in steps up to the largest N; after each step the -reconciler runs in dry-run 3 times (median reported), plus page-size and checksum variants. -Everything synthetic is deleted at the end (objects first, then rows). - - python bench/reconcile_cost.py --steps 1000 10000 50000 --output evidence/reconcile -""" -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import platform -import statistics -import subprocess -import sys -import time -import uuid -from pathlib import Path - -import httpx2 as httpx - -ROOT = Path(__file__).resolve().parents[1] -BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080") -PASSWORD = "Bench-password-Ω-12345" -OBJECT = bytes(range(256)) * 4 # 1 KiB, the same content for every object -SHA = hashlib.sha256(OBJECT).hexdigest() - -PUT_SCRIPT = """ -import asyncio, sys -from messenger.config import Settings, s3_settings -from messenger.storage import Storage -DATA = bytes(range(256)) * 4 -async def main(): - s = Storage(Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings())) - await s.open() - keys = sys.stdin.read().split() - sem = asyncio.Semaphore(32) - async def one(k): - async with sem: - await s.{op}(k{args}) - await asyncio.gather(*(one(k) for k in keys)) - await s.close() - print(len(keys)) -asyncio.run(main()) -""" - - -def compose(*args: str, stdin: str | None = None, timeout: float = 1800, check: bool = True) -> subprocess.CompletedProcess: - return subprocess.run(["docker", "compose", *args], cwd=ROOT, input=stdin, capture_output=True, text=True, - encoding="utf-8", timeout=timeout, check=check) - - -def psql(sql: str) -> str: - """SQL through stdin: a 5000-row INSERT does not fit into a Windows command line (32 KiB).""" - return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v", - "ON_ERROR_STOP=1", "-f", "-", stdin=sql).stdout.strip() - - -def s3_bulk(op: str, keys: list[str]) -> None: - script = PUT_SCRIPT.replace("{op}", op).replace("{args}", ", DATA, 'image/png', {}" if op == "put" else "") - compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "python", "reconciler", "-c", script, stdin="\n".join(keys)) - - -def owner_and_chat() -> tuple[str, str]: - import base64 - c = httpx.Client(base_url=BASE, timeout=30) - name = "rc_" + uuid.uuid4().hex[:10] - c.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status() - token = c.post("/api/v1/auth/sessions", headers={ - "Authorization": "Basic " + base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()}).json()["token"] - me = c.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json() - chat = c.post("/api/v1/chats", headers={"Authorization": "Bearer " + token}, - json={"title": "bench", "member_ids": [me["id"]]}).json() - return me["id"], chat["id"] - - -def add(n: int, owner: str, chat: str) -> None: - keys = ["o/" + uuid.uuid4().hex for _ in range(n)] - for i in range(0, n, 5000): - part = keys[i:i + 5000] - values = ",".join(f"('{uuid.uuid4()}','{k}')" for k in part) - psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, " - "object_key, status, created_at, updated_at) " - f"SELECT v.id::uuid, '{chat}', '{owner}', 'bench-reconcile', 'image/png', {len(OBJECT)}, " - f"decode('{SHA}', 'hex'), 1, 1, v.k, 'ready', clock_timestamp() - interval '2 hours', " - f"clock_timestamp() - interval '2 hours' FROM (VALUES {values}) AS v(id, k)") - s3_bulk("put", keys) - - -def reconcile(*args: str) -> dict: - out = compose("run", "--rm", "--no-deps", "-T", "reconciler", *args).stdout - return json.loads(out) - - -def measure(label: str, args: list[str], repeats: int) -> dict: - runs = [reconcile(*args) for _ in range(repeats)] - durations = [r["duration_seconds"] for r in runs] - last = runs[-1] - return {"label": label, "args": args, "runs": repeats, "duration_s_median": statistics.median(durations), - "duration_s_all": durations, "objects": last["scanned"]["objects"], "rows": last["scanned"]["rows"], - "findings": {k: v for k, v in last["findings"].items() if v}, "cost": last["cost"]} - - -def main() -> int: - p = argparse.ArgumentParser() - p.add_argument("--steps", type=int, nargs="+", default=[1000, 10000, 50000]) - p.add_argument("--repeats", type=int, default=3) - p.add_argument("--output", default="evidence/reconcile") - args = p.parse_args() - out = ROOT / args.output - out.mkdir(parents=True, exist_ok=True) - owner, chat = owner_and_chat() - present = 0 - results = [] - steps = sorted(args.steps) - variant_step = steps[min(1, len(steps) - 1)] - try: - for target in sorted(args.steps): - started = time.perf_counter() - add(target - present, owner, chat) - present = target - print(f"populated {present} synthetic attachments in {time.perf_counter() - started:.1f} s", flush=True) - results.append({"synthetic": present, **measure("dry-run, page 1000", [], args.repeats)}) - print(json.dumps(results[-1], ensure_ascii=False), flush=True) - if present == variant_step: # page-size and checksum variants at one size - for label, extra in (("dry-run, page 100", ["--page-size", "100"]), - ("dry-run, --verify-checksums", ["--verify-checksums"])): - results.append({"synthetic": present, **measure(label, extra, args.repeats)}) - print(json.dumps(results[-1], ensure_ascii=False), flush=True) - finally: - keys = psql("SELECT object_key FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = " - f"'{chat}'").split() - if keys: - s3_bulk("delete", keys) - psql(f"DELETE FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = '{chat}'") - print(f"removed {len(keys)} synthetic attachments", flush=True) - meta = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(), - "commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, - text=True).stdout.strip(), - "object_bytes": len(OBJECT), "reconciler_limits": "cpus 1.0, memory 256m (compose)", - "note": "the bucket/table also contain the attachments of earlier test runs (counted in objects/rows)"} - (out / "cost.json").write_text(json.dumps({"meta": meta, "results": results}, ensure_ascii=False, indent=1), - encoding="utf-8") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bench/reconcile_faults.py b/bench/reconcile_faults.py deleted file mode 100644 index 7cb11d4..0000000 --- a/bench/reconcile_faults.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python3 -"""Track 2 evidence: inject faults on both sides, run the reconciler, keep every JSON report. - -Scenario A (all fault kinds; the same as student-tests/test_reconcile.py, but the reports are saved): - S3 side: old orphan objects, object deleted under a USED ready row, object truncated under an - unused ready row, same-size corruption (found only with --verify-checksums); - DB side: stale intents (with / without object), quarantine mark on a healthy row; - fresh: orphan + intent younger than --grace. - Runs: dry-run (+ snapshot of both stores before/after) -> apply -> apply again -> restore of the - lost object from a "backup" -> apply (quarantine lifted). -Scenario B (interrupted apply): 2000 old orphans, S3 is paused while --apply deletes them -> - the run stops with an error, nothing else is damaged; the next run finishes the rest; the - third run finds nothing. - - python bench/reconcile_faults.py --output evidence/reconcile -""" -from __future__ import annotations - -import argparse -import json -import subprocess -import sys -import threading -import time -import uuid -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "student-tests")) -import imagegen # noqa: E402 -from conftest import (Api, attachment_row, compose, findings_by_key, psql, s3_delete, s3_exec, s3_keys, # noqa: E402 - s3_put) -from test_reconcile import insert_intent, snapshot # noqa: E402 - -GRACE = ["--grace", "30", "--allow-small-grace"] -SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() - - -def reconcile(*args: str) -> tuple[int, dict]: - r = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "1000", *args, check=False, timeout=600) - return r.returncode, json.loads(r.stdout) - - -def save(out: Path, name: str, data) -> None: - (out / name).write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8") - - -def ours(report: dict, keys: dict[str, str]) -> dict: - """The findings of this scenario's keys only (the stand also holds other test data).""" - f = findings_by_key(report) - return {label: ({"kind": f[k]["kind"], "result": f[k]["result"], "used": f[k]["used"]} if k in f else None) - for label, k in keys.items()} - - -def scenario_a(out: Path) -> dict: - api = Api() - alice, bob = api.user("alice"), api.user("bob") - chat = api.chat(alice, bob) - up = {n: api.upload(alice, chat, imagegen.png(40 + i, 30, noise=True)).json() for i, n in enumerate("ABCDEF")} - data = {n: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content - for n, a in up.items()} - key = {n: attachment_row(a["id"])["object_key"] for n, a in up.items()} - api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]}) - keys = {"healthy used A": key["A"], "healthy unused B": key["B"], "used, object deleted C": key["C"], - "unused, object truncated D": key["D"], "quarantined but healthy E": key["E"], - "same-size corruption F": key["F"]} - orphan = "o/" + uuid.uuid4().hex - stale_with, stale_without = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex - s3_put(orphan, SAMPLE) - s3_put(stale_with, SAMPLE) - insert_intent(chat, alice, stale_with, SAMPLE, 600) - insert_intent(chat, alice, stale_without, SAMPLE, 600) - s3_delete(key["C"]) - s3_put(key["D"], data["D"][: len(data["D"]) // 2]) - s3_put(key["F"], data["F"][:200] + bytes([data["F"][200] ^ 1]) + data["F"][201:]) - psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' WHERE id = '{up['E']['id']}'") - keys.update({"old orphan object": orphan, "stale intent + object": stale_with, "stale intent, no object": stale_without}) - time.sleep(32) - fresh_orphan, fresh_intent = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex - s3_put(fresh_orphan, SAMPLE) - s3_put(fresh_intent, SAMPLE) - insert_intent(chat, alice, fresh_intent, SAMPLE, 0) - keys.update({"fresh orphan (in-flight)": fresh_orphan, "fresh intent (in-flight)": fresh_intent}) - - before = snapshot() - _, dry = reconcile("--verify-checksums", *GRACE) - after = snapshot() - _, apply1 = reconcile("--apply", "--verify-checksums", *GRACE) - _, apply2 = reconcile("--apply", "--verify-checksums", *GRACE) - s3_put(key["C"], data["C"]) # the lost object restored from a backup - _, apply3 = reconcile("--apply", "--verify-checksums", *GRACE) - for name, report in (("faults-1-dry-run.json", dry), ("faults-2-apply.json", apply1), - ("faults-3-apply-again.json", apply2), ("faults-4-after-restore.json", apply3)): - save(out, name, report) - rows = {label: attachment_row(up[n]["id"]) for label, n in (("C", "C"), ("D", "D"), ("E", "E"), ("F", "F"), ("A", "A"), ("B", "B"))} - content = {n: api.client.get(f"/api/v1/attachments/{up[n]['id']}/content", - headers={"Authorization": "Bearer " + bob["token"]}).status_code for n in "ABCDEF"} - present = s3_keys() - summary = { - "dry_run_changed_nothing": before == after, - "per_key": {"dry-run": ours(dry, keys), "apply": ours(apply1, keys), "apply again": ours(apply2, keys), - "after restore of C": ours(apply3, keys)}, - "objects_present_at_end (fresh items were older than --grace by the later runs)": {label: k in present for label, k in keys.items()}, - "rows_after": {k: (v and {"status": v["status"], "lost_reason": v["lost_reason"]}) for k, v in rows.items()}, - "content_status_after": content, - "durations_s": [dry["duration_seconds"], apply1["duration_seconds"], apply2["duration_seconds"], - apply3["duration_seconds"]], - } - print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True) - return summary - - -def scenario_b(out: Path, count: int = 2000) -> dict: - keys = ["o/" + uuid.uuid4().hex for _ in range(count)] - s3_exec("ks = sys.stdin.read().split()\n" - "sem = asyncio.Semaphore(32)\n" - "async def one(k):\n" - " async with sem:\n" - " await s.put(k, b'orphan', 'image/png', {})\n" - "await asyncio.gather(*(one(k) for k in ks))", stdin="\n".join(keys)) - time.sleep(32) - result: dict = {} - - def run_apply() -> None: - result["code"], result["report"] = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE) - - t = threading.Thread(target=run_apply) - t.start() - # pause S3 as soon as deletions are under way (the reconciler is planning/listing first) - deadline = time.monotonic() + 60 - while time.monotonic() < deadline: - left = len(s3_keys() & set(keys)) - if left < count: - break - time.sleep(0.2) - subprocess.run(["docker", "compose", "pause", "s3"], cwd=ROOT, check=True, capture_output=True) - t.join() - subprocess.run(["docker", "compose", "unpause", "s3"], cwd=ROOT, check=True, capture_output=True) - time.sleep(5) # let S3 settle after unpause before counting - left_after_interrupt = len(s3_keys() & set(keys)) - code2, second = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE) - code3, third = reconcile("--apply", *GRACE) - save(out, "interrupted-1-apply.json", result["report"]) - save(out, "interrupted-2-rerun.json", second) - save(out, "interrupted-3-third.json", third) - summary = { - "orphans_injected": count, - "interrupted_run": {"exit_code": result["code"], "error": result["report"].get("error"), - "applied": result["report"]["actions"]["applied"]}, - "orphans_left_after_interrupt": left_after_interrupt, - "rerun": {"exit_code": code2, "applied": second["actions"]["applied"], "error": second["error"]}, - "third": {"exit_code": code3, "planned": third["actions"]["planned"]}, - "orphans_left_at_end": len(s3_keys() & set(keys)), - } - print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True) - return summary - - -def main() -> int: - p = argparse.ArgumentParser() - p.add_argument("--output", default="evidence/reconcile") - args = p.parse_args() - out = ROOT / args.output - out.mkdir(parents=True, exist_ok=True) - a = scenario_a(out) - b = scenario_b(out) - commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip() - save(out, "faults-summary.json", {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "commit": commit, - "grace_seconds": 30, "scenario_a": a, "scenario_b_interrupted": b}) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/migrations/0005_pipeline.sql b/migrations/0005_pipeline.sql new file mode 100644 index 0000000..ac44388 --- /dev/null +++ b/migrations/0005_pipeline.sql @@ -0,0 +1,260 @@ +-- 0005 (lab 5): asynchronous processing of attachments. Details: docs/PIPELINE.md. +-- +-- Upload (API): INSERT attachment 'pending' (intent: the staging key is known before the PUT) +-- -> PUT original into the PRIVATE STAGING bucket +-- -> ONE transaction: attachment 'queued' + jobs row + outbox row -> 202. +-- Relay: outbox row -> publish to RabbitMQ with publisher confirm -> outbox.published_at. +-- Worker: claim job (lease + attempt token) -> variants into the final bucket under keys of +-- THIS attempt -> ONE transaction: variants + attachment 'ready' (fenced by the +-- lease token, never downgrading published_version) -> ack -> staging deleted. +-- +-- PostgreSQL is the source of truth for the work: state, attempts, lease, backoff (available_at) +-- and the failed registry. The broker only delivers job ids (at least once). + +-- ------------------------------------------------------------------ attachments +ALTER TABLE attachments DROP CONSTRAINT attachments_status; +ALTER TABLE attachments ADD CONSTRAINT attachments_status + CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed')); +-- object_key = key of the ORIGINAL in the final bucket. NULL until the worker publishes it. +-- Keys of lab 4 ('o/') stay valid; lab 5 writes 'a///original'. +ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL; +ALTER TABLE attachments DROP CONSTRAINT attachments_object_key_format; +ALTER TABLE attachments ADD CONSTRAINT attachments_object_key_format + CHECK (object_key ~ '^(o/[0-9a-f]{32}|a/[0-9a-f-]{36}/[0-9a-f]{32}/original)$'); +ALTER TABLE attachments + -- Key of the original in the staging bucket ('s/<128 random bits>'), written by the API. + ADD COLUMN staging_key text COLLATE "C", + -- none (lab 4 row) | present | deleting (marked, S3 DELETE pending) | deleted + ADD COLUMN staging_state text NOT NULL DEFAULT 'none', + -- pipeline version whose variants are published (GET shows them); only ever grows. + ADD COLUMN published_version integer, + ADD COLUMN error_code text, + ADD COLUMN failed_at timestamptz, + ADD CONSTRAINT attachments_staging_key_key UNIQUE (staging_key), + ADD CONSTRAINT attachments_staging_key_format CHECK (staging_key ~ '^s/[0-9a-f]{32}$'), + ADD CONSTRAINT attachments_staging_state CHECK (staging_state IN ('none', 'present', 'deleting', 'deleted')), + ADD CONSTRAINT attachments_staging_has_key CHECK (staging_state = 'none' OR staging_key IS NOT NULL), + ADD CONSTRAINT attachments_published_version CHECK (published_version >= 1), + ADD CONSTRAINT attachments_error_code CHECK (error_code ~ '^[a-z_]{1,64}$'); + +-- Lab 4 rows: a 'pending' intent of lab 4 never got its object confirmed and has no staging copy; +-- it becomes a failed upload (invisible: its id was never returned). A lab 4 'ready' row has its +-- original in the final bucket but no thumbnail yet: it is queued for pipeline 1 (below). +UPDATE attachments SET status = 'failed', error_code = 'upload_incomplete', failed_at = clock_timestamp() +WHERE status = 'pending'; +UPDATE attachments SET status = 'queued' WHERE status = 'ready'; + +-- Structural guarantees (the database refuses a buggy cleanup or worker): +-- * work that is queued/processing always has a source: the published original or the staging +-- copy - the cleanup can never mark the staging of an accepted upload as deleting; +-- * ready = published original + published pipeline version; failed = error code. +ALTER TABLE attachments + ADD CONSTRAINT attachments_pending_staging CHECK (status <> 'pending' OR staging_state = 'present'), + ADD CONSTRAINT attachments_active_has_source + CHECK (status NOT IN ('queued', 'processing') OR object_key IS NOT NULL OR staging_state = 'present'), + ADD CONSTRAINT attachments_ready_published + CHECK (status <> 'ready' OR (object_key IS NOT NULL AND published_version IS NOT NULL)), + ADD CONSTRAINT attachments_failed_code + CHECK ((status = 'failed') = (error_code IS NOT NULL AND failed_at IS NOT NULL)); +-- Janitor of the API: the lab 4 partial index on pending rows stays as is. +-- Cleanup of the worker: staging copies that may be deleted (ready: at once; failed: after retention). +CREATE INDEX attachments_staging_cleanup ON attachments (updated_at) + WHERE staging_state IN ('present', 'deleting') AND status IN ('ready', 'failed'); + +-- ------------------------------------------------------------------ jobs +-- One row = one logical piece of work: "build pipeline version V for attachment A". +-- job_id never changes on redelivery; `attempt` counts claims by workers (managed attempts). +CREATE TABLE jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL, + kind text NOT NULL, -- upload | replay | reprocess | migrate + replay_of uuid REFERENCES jobs (id), + state text NOT NULL DEFAULT 'queued', + attempt integer NOT NULL DEFAULT 0, + max_attempts integer NOT NULL, + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), -- backoff: not before + lease_token uuid, -- token of the attempt that holds the job + lease_owner text, + lease_until timestamptz, + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + finished_at timestamptz, + CONSTRAINT jobs_pipeline_version CHECK (pipeline_version >= 1), + CONSTRAINT jobs_kind CHECK (kind IN ('upload', 'replay', 'reprocess', 'migrate')), + CONSTRAINT jobs_replay CHECK ((kind = 'replay') = (replay_of IS NOT NULL)), + CONSTRAINT jobs_state CHECK (state IN ('queued', 'processing', 'succeeded', 'failed', 'superseded')), + CONSTRAINT jobs_attempts CHECK (attempt >= 0 AND max_attempts BETWEEN 1 AND 100 AND attempt <= max_attempts), + -- A lease exists exactly while the job is processing. + CONSTRAINT jobs_lease CHECK ((state = 'processing') = (lease_token IS NOT NULL AND lease_until IS NOT NULL)), + CONSTRAINT jobs_finished CHECK ((state IN ('succeeded', 'failed', 'superseded')) = (finished_at IS NOT NULL)), + CONSTRAINT jobs_failed_code CHECK (state <> 'failed' OR error_code IS NOT NULL), + CONSTRAINT jobs_error_code CHECK (error_code ~ '^[a-z_]{1,64}$') +); +-- At most ONE active job per effect set (attachment, pipeline version): a second upload job, +-- replay or reprocess of the same version cannot exist while one is queued/processing. +CREATE UNIQUE INDEX jobs_one_active ON jobs (attachment_id, pipeline_version) WHERE state IN ('queued', 'processing'); +CREATE INDEX jobs_attachment ON jobs (attachment_id, created_at); +CREATE INDEX jobs_lease_expiry ON jobs (lease_until) WHERE state = 'processing'; -- reaper +CREATE INDEX jobs_queued ON jobs (updated_at) WHERE state = 'queued'; -- lost-message guard +CREATE INDEX jobs_failed ON jobs (finished_at) WHERE state = 'failed'; -- failed registry + +-- One row per claim of a job by a worker. The token is part of every object key the attempt +-- writes, so the keys of an attempt are known without listing the bucket (planned_keys), and an +-- attempt that is no longer the lease holder can never publish (fencing by token). +CREATE TABLE job_attempts ( + token uuid PRIMARY KEY, + job_id uuid NOT NULL REFERENCES jobs (id), + attempt integer NOT NULL, + worker text NOT NULL, + source text NOT NULL, -- staging | original + planned_keys text[] NOT NULL, + outcome text NOT NULL DEFAULT 'running', + error_code text, + started_at timestamptz NOT NULL DEFAULT clock_timestamp(), + finished_at timestamptz, + objects_cleaned_at timestamptz, + CONSTRAINT job_attempts_number UNIQUE (job_id, attempt), + CONSTRAINT job_attempts_source CHECK (source IN ('staging', 'original')), + CONSTRAINT job_attempts_outcome + CHECK (outcome IN ('running', 'succeeded', 'retry', 'failed', 'abandoned', 'superseded')), + CONSTRAINT job_attempts_finished CHECK ((outcome = 'running') = (finished_at IS NULL)), + -- Objects of a successful attempt are published: they are never cleaned. + CONSTRAINT job_attempts_cleaned CHECK (objects_cleaned_at IS NULL OR outcome NOT IN ('running', 'succeeded')) +); +CREATE INDEX job_attempts_cleanup ON job_attempts (finished_at) + WHERE outcome IN ('retry', 'failed', 'abandoned', 'superseded') AND objects_cleaned_at IS NULL; + +-- Published variants. The primary key is the idempotency key of the EFFECT from the contract: +-- (attachment_id, pipeline_version, variant) exists at most once, whatever the deliveries. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL, + name text NOT NULL, + object_key text COLLATE "C" NOT NULL, + content_type text NOT NULL, + size_bytes integer NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + sha256 bytea NOT NULL, + attempt_token uuid NOT NULL REFERENCES job_attempts (token), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT attachment_variants_key UNIQUE (object_key), + CONSTRAINT attachment_variants_key_format CHECK (object_key ~ '^a/[0-9a-f-]{36}/[0-9a-f]{32}/(thumbnail|crop|optimized)$'), + CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')), + CONSTRAINT attachment_variants_type CHECK (content_type IN ('image/jpeg', 'image/webp')), + CONSTRAINT attachment_variants_size CHECK (size_bytes >= 1 AND width >= 1 AND height >= 1), + CONSTRAINT attachment_variants_sha256 CHECK (octet_length(sha256) = 32) +); + +-- Transactional outbox: written in the SAME transaction as the job (upload, retry, replay, +-- reprocess, lease expiry). The relay publishes due rows and marks them; a crash after the +-- publish but before the mark publishes again (a duplicate the worker ignores). +CREATE TABLE outbox ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES jobs (id), + attempt integer NOT NULL, -- the attempt this delivery is meant for + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz, + publish_failures integer NOT NULL DEFAULT 0, + CONSTRAINT outbox_attempt CHECK (attempt >= 1) +); +CREATE INDEX outbox_due ON outbox (available_at, id) WHERE published_at IS NULL; +CREATE INDEX outbox_job ON outbox (job_id); +CREATE INDEX outbox_published ON outbox (published_at) WHERE published_at IS NOT NULL; -- purge + +-- Audit of every transition (no secrets, no file names): replay/reprocess requests with the +-- operator and reason, claims, retries with their delay, lease expiries, results. +CREATE TABLE job_events ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES jobs (id), + at timestamptz NOT NULL DEFAULT clock_timestamp(), + event text NOT NULL, + attempt integer, + actor text NOT NULL, + error_code text, + detail jsonb NOT NULL DEFAULT '{}' +); +CREATE INDEX job_events_job ON job_events (job_id, id); + +-- Failed registry for the operator (jobctl list-failed). +CREATE VIEW failed_jobs AS +SELECT j.id AS job_id, j.attachment_id, j.pipeline_version, j.kind, j.attempt, j.max_attempts, j.error_code, + j.finished_at, a.status AS attachment_status, a.staging_state +FROM jobs j JOIN attachments a ON a.id = j.attachment_id +WHERE j.state = 'failed'; + +-- Lab 4 'ready' rows (now queued): a job for pipeline 1 from their published original. +WITH queued AS ( + SELECT a.id, gen_random_uuid() AS job_id FROM attachments a WHERE a.status = 'queued' +), created AS ( + INSERT INTO jobs (id, attachment_id, pipeline_version, kind, max_attempts) + SELECT q.job_id, q.id, 1, 'migrate', 5 FROM queued q RETURNING id +) +INSERT INTO outbox (job_id, attempt) SELECT c.id, 1 FROM created c; + +-- post_message_v3 = post_message_v2 (0004) with the lab 5 attachment states: queued, processing +-- and ready attachments of this chat may be attached; failed (or quarantined) -> 409; pending, +-- foreign or unknown -> 404. Everything else (key claim, chat clock, one transaction) unchanged. +CREATE FUNCTION post_message_v3(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid, + p_text bytea, p_ttl_seconds integer, p_attachments uuid[]) +RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz, o_attachments uuid[]) +LANGUAGE plpgsql VOLATILE AS $$ +DECLARE + v_created timestamptz; + v_found integer; + v_usable integer; + v_check text := 'ok'; +BEGIN + PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user; + IF NOT FOUND THEN + RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[]; + RETURN; + END IF; + + IF cardinality(p_attachments) > 0 THEN + -- FOR SHARE: a concurrent transition to failed/quarantine waits until this message commits. + SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL AND a.status <> 'failed') INTO v_found, v_usable + FROM (SELECT a.lost_at, a.status FROM attachments a + WHERE a.id = ANY (p_attachments) AND a.chat_id = p_chat AND a.status <> 'pending' + FOR SHARE) a; + IF v_found <> cardinality(p_attachments) THEN + v_check := 'attachment_not_found'; + ELSIF v_usable <> v_found THEN + v_check := 'attachment_conflict'; + END IF; + END IF; + + IF v_check = 'ok' THEN + INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id) + VALUES (p_user, p_chat, p_key, p_hash, p_message) + ON CONFLICT (user_id, chat_id, key) DO UPDATE + SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() + WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds); + IF FOUND THEN + UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond') + WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created; + INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) + VALUES (p_message, p_chat, p_user, p_text, v_created, p_attachments); + INSERT INTO message_attachments (message_id, position, attachment_id) + SELECT p_message, (a.ord - 1)::smallint, a.id + FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord); + RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created, p_attachments; + RETURN; + END IF; + END IF; + + RETURN QUERY + SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at, + m.attachment_ids + FROM idempotency_keys k JOIN messages m ON m.id = k.message_id + WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key; + IF NOT FOUND THEN + RETURN QUERY SELECT CASE WHEN v_check = 'ok' THEN 'retry' ELSE v_check END, + NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[]; + END IF; +END +$$; diff --git a/requirements.in b/requirements.in index 22ea682..623432a 100644 --- a/requirements.in +++ b/requirements.in @@ -16,5 +16,9 @@ redis==8.1.0 argon2-cffi==25.1.0 # Lab 4: S3 client (async botocore over aiohttp; requests are cancellable, unlike libpq cancel). aiobotocore==3.9.2 -# Images are checked without Pillow: PNG by src/messenger/images.py, JPEG by libjpeg-turbo +# Lab 5: AMQP 0-9-1 client for RabbitMQ (publisher confirms, consumer acks, prefetch); last 9.x line. +aio-pika==9.6.2 +# Lab 5: image variants in the worker only (thumbnail/crop JPEG, WebP). Same version as the tests. +pillow==12.3.0 +# Uploads are still checked without Pillow: PNG by src/messenger/images.py, JPEG by libjpeg-turbo # `djpeg -strict` (Debian package libjpeg-turbo-progs, see Dockerfile). diff --git a/requirements.txt b/requirements.txt index c7c583c..65b73e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,10 @@ # # pip-compile --generate-hashes --no-emit-index-url --no-index --output-file=requirements.txt --strip-extras requirements.in # +aio-pika==9.6.2 \ + --hash=sha256:2a5478af920d169795071c9c09c7542cd8cdece60438cf7804533dcbcce93b7f \ + --hash=sha256:c49e9246080dc8ffa1bb0e4aca407bf3d8ad78c3ee3a93df88b68fe65d7a49b9 + # via -r requirements.in aiobotocore==3.9.2 \ --hash=sha256:363b4892423b272eb84afbb58000da0be6ccd0cf8a68b98438c957d05b841efc \ --hash=sha256:8e32238c5bd77717ab1ac90077ff51ab141d025528115ccc7cc9b50b2d176e5f @@ -179,6 +183,10 @@ aioitertools==0.13.0 \ --hash=sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be \ --hash=sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c # via aiobotocore +aiormq==6.9.4 \ + --hash=sha256:0e7c01b662804e1cc7ace9a17794e8c1192a27fc2afa96162362a6e61ae8e8ef \ + --hash=sha256:726a8586695e863fba68cf88842065ab12348c9438dcebdfc9d0bddaf6083277 + # via aio-pika aiosignal==1.4.0 \ --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 @@ -794,6 +802,99 @@ multidict==6.9.1 \ # aiobotocore # aiohttp # yarl +pamqp==3.3.0 \ + --hash=sha256:40b8795bd4efcf2b0f8821c1de83d12ca16d5760f4507836267fd7a02b06763b \ + --hash=sha256:c901a684794157ae39b52cbf700db8c9aae7a470f13528b9d7b4e5f7202f8eb0 + # via aiormq +pillow==12.3.0 \ + --hash=sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756 \ + --hash=sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a \ + --hash=sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59 \ + --hash=sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45 \ + --hash=sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3 \ + --hash=sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df \ + --hash=sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139 \ + --hash=sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b \ + --hash=sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39 \ + --hash=sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e \ + --hash=sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8 \ + --hash=sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1 \ + --hash=sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8 \ + --hash=sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89 \ + --hash=sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5 \ + --hash=sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130 \ + --hash=sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd \ + --hash=sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d \ + --hash=sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b \ + --hash=sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed \ + --hash=sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace \ + --hash=sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb \ + --hash=sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931 \ + --hash=sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510 \ + --hash=sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6 \ + --hash=sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1 \ + --hash=sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce \ + --hash=sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385 \ + --hash=sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e \ + --hash=sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c \ + --hash=sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7 \ + --hash=sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace \ + --hash=sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c \ + --hash=sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f \ + --hash=sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64 \ + --hash=sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f \ + --hash=sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a \ + --hash=sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827 \ + --hash=sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17 \ + --hash=sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4 \ + --hash=sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a \ + --hash=sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701 \ + --hash=sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e \ + --hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 \ + --hash=sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66 \ + --hash=sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468 \ + --hash=sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217 \ + --hash=sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658 \ + --hash=sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418 \ + --hash=sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a \ + --hash=sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c \ + --hash=sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330 \ + --hash=sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402 \ + --hash=sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09 \ + --hash=sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930 \ + --hash=sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f \ + --hash=sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec \ + --hash=sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a \ + --hash=sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94 \ + --hash=sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468 \ + --hash=sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b \ + --hash=sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965 \ + --hash=sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8 \ + --hash=sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd \ + --hash=sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7 \ + --hash=sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c \ + --hash=sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777 \ + --hash=sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35 \ + --hash=sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9 \ + --hash=sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f \ + --hash=sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f \ + --hash=sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0 \ + --hash=sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c \ + --hash=sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71 \ + --hash=sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3 \ + --hash=sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838 \ + --hash=sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf \ + --hash=sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321 \ + --hash=sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26 \ + --hash=sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec \ + --hash=sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9 \ + --hash=sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65 \ + --hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \ + --hash=sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e \ + --hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \ + --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 \ + --hash=sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7 + # via -r requirements.in propcache==0.5.4 \ --hash=sha256:004e685b315646c410771836e72a44f143bbe624f29653a42687815069a303d5 \ --hash=sha256:02c0a34f16889cf800f10f0247a564d8ce6eeab6ffcd7c87198f769067eb8432 \ @@ -1353,4 +1454,7 @@ yarl==1.25.1 \ --hash=sha256:f53dcd26694f148f738edc052b5a69234833e739f10f4c3287bdfd8ec0f7b326 \ --hash=sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc \ --hash=sha256:fe01645169a2112aa1d4ebc3e4c5f029c5c8f97adfc32e5d37c993b39a994d75 - # via aiohttp + # via + # aio-pika + # aiohttp + # aiormq diff --git a/src/messenger/broker.py b/src/messenger/broker.py new file mode 100644 index 0000000..5b58651 --- /dev/null +++ b/src/messenger/broker.py @@ -0,0 +1,86 @@ +"""RabbitMQ (AMQP 0-9-1) for the attachment jobs: topology names and the message format. + +Topology (declared by the broker itself from its definitions file at boot, infra/rabbitmq and +scripts/init_secrets.py; the relay and the worker have NO configure permission and declare nothing): + + exchange messenger.jobs direct, durable + queue messenger.attachment-jobs quorum (Raft log on disk), bound with routing key + attachment.process.v1; x-delivery-limit and dead-lettering to + exchange messenger.jobs.dlx fanout -> queue messenger.attachment-jobs.dlq (quorum) + +Publisher side (relay): persistent message, `mandatory` (an unroutable message is returned, not +silently dropped), channel in confirm mode - `publish()` returns only after the broker has +written the message to the quorum log (publisher confirm). Consumer side (worker): manual ack +after the durable effect, prefetch = WORKER_CONCURRENCY. + +The message carries IDs and service fields only (contracts/jobs.schema.json): no bytes, no URL, +no S3 key, no filename, no credentials. The worker takes every trusted detail from PostgreSQL. +""" +from __future__ import annotations + +import datetime as dt +import json +import re +import uuid + +EXCHANGE = "messenger.jobs" +QUEUE = "messenger.attachment-jobs" +DLX = "messenger.jobs.dlx" +DLQ = "messenger.attachment-jobs.dlq" +ROUTING_KEY = "attachment.process.v1" +JOB_TYPE = "attachment.process.v1" +MAX_MESSAGE_BYTES = 4096 + +REQUIRED = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"} +OPTIONAL = {"traceparent", "tracestate"} +TRACEPARENT_RE = re.compile(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}") +DATETIME_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?Z") + + +class BadMessage(ValueError): + """Not a valid job message (schema v1): dead-lettered, never processed.""" + + +def rfc3339(value: dt.datetime) -> str: + return value.astimezone(dt.timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def encode(job_id: str, attachment_id: str, pipeline_version: int, attempt: int, created_at: dt.datetime) -> bytes: + body = {"schema_version": 1, "job_id": job_id, "type": JOB_TYPE, "attachment_id": attachment_id, + "pipeline_version": pipeline_version, "attempt": attempt, "created_at": rfc3339(created_at)} + return json.dumps(body, separators=(",", ":")).encode() + + +def decode(raw: bytes) -> dict: + """Strict check against contracts/jobs.schema.json (additionalProperties: false).""" + if len(raw) > MAX_MESSAGE_BYTES: + raise BadMessage("too large") + try: + body = json.loads(raw) + except (ValueError, UnicodeError): + raise BadMessage("not JSON") from None + if not isinstance(body, dict): + raise BadMessage("not an object") + keys = set(body) + if not REQUIRED <= keys or keys - REQUIRED - OPTIONAL: + raise BadMessage("wrong fields") + if body["schema_version"] != 1 or isinstance(body["schema_version"], bool) or body["type"] != JOB_TYPE: + raise BadMessage("unsupported schema/type") + for name in ("job_id", "attachment_id"): + if not isinstance(body[name], str): + raise BadMessage(name) + try: + uuid.UUID(body[name]) + except ValueError: + raise BadMessage(name) from None + for name in ("pipeline_version", "attempt"): + value = body[name] + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise BadMessage(name) + if not isinstance(body["created_at"], str) or not DATETIME_RE.fullmatch(body["created_at"]): + raise BadMessage("created_at") + if "traceparent" in body and not (isinstance(body["traceparent"], str) and TRACEPARENT_RE.fullmatch(body["traceparent"])): + raise BadMessage("traceparent") + if "tracestate" in body and not (isinstance(body["tracestate"], str) and len(body["tracestate"]) <= 512): + raise BadMessage("tracestate") + return body diff --git a/src/messenger/config.py b/src/messenger/config.py index 1772442..edb6e0f 100644 --- a/src/messenger/config.py +++ b/src/messenger/config.py @@ -10,7 +10,7 @@ from __future__ import annotations import os import re import socket -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path @@ -134,7 +134,29 @@ class Settings: jpeg_max_memory_mb: int = 256 pending_grace_seconds: int = 900 janitor_interval_seconds: float = 60.0 + # Lab 5: staging bucket (API writes, worker reads) and the asynchronous pipeline (jobs_settings()). + s3_staging_bucket: str = "messenger-staging" + broker_url: str = "" + pipeline_version: int = 1 + job_max_attempts: int = 5 + job_timeout_seconds: float = 60.0 + job_lease_seconds: float = 30.0 + job_retry_base_seconds: float = 2.0 + job_retry_max_seconds: float = 60.0 + worker_concurrency: int = 2 + worker_shutdown_seconds: float = 20.0 + image_max_memory_mb: int = 768 + staging_retention_seconds: int = 86400 + orphan_grace_seconds: int = 600 + cleanup_interval_seconds: float = 30.0 + relay_poll_seconds: float = 0.5 + relay_batch: int = 50 + queued_republish_seconds: float = 300.0 fault_points: frozenset[str] = frozenset() + fault_values: dict[str, int] = field(default_factory=dict) + + def fault_value(self, point: str, default: int = 0) -> int: + return self.fault_values.get(point, default) @property def db_statement_timeout_ms(self) -> int: @@ -142,7 +164,8 @@ class Settings: return max(100, int(self.db_timeout_seconds * 1000) - 500) @classmethod - def from_env(cls) -> "Settings": + def from_env(cls, *, s3_credentials: bool = True) -> "Settings": + """s3_credentials=False: processes without S3 access (relay, jobctl).""" log_level = os.environ.get("LOG_LEVEL", "info").strip().lower() or "info" if log_level not in {"debug", "info", "warning", "error"}: raise ConfigError(f"LOG_LEVEL must be debug/info/warning/error, got {log_level!r}") @@ -185,32 +208,71 @@ class Settings: # Track 1: `function` = one SQL call per send (migration 0003), `multi` = the previous # multi-statement transaction; both have identical semantics (A/B on one image). message_write_path=_choice("MESSAGE_WRITE_PATH", "function", {"function", "multi"}), - **s3_settings(), + **s3_settings(credentials_required=s3_credentials), + **jobs_settings(broker_required=False), ) # Contract limits (contracts/README.md): configurable DOWN for experiments, never above them. CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024 CONTRACT_MAX_IMAGE_PIXELS = 40_000_000 -# Failure points for the level 4 experiments only (compose.faults.yaml); empty in normal runs. -FAULT_POINTS = {"after_s3_put", "after_ready_commit"} +# Failure points for the experiments only (FAULT_POINTS of ONE service, compose variables +# API_/RELAY_/WORKER_FAULT_POINTS); empty in normal runs. `name` or `name=`. +FAULT_POINTS = { + # API, upload: between the staging PUT and the job COMMIT / after the job COMMIT, before 202. + "after_s3_put", "after_enqueue_commit", + # Relay: crash after the broker confirmed a publish but before outbox.published_at is + # committed; publish every message twice; publish a batch in reverse order. + "crash_after_publish", "duplicate_publish", "reverse_order", + # Worker: crash after the claim (before any side effect), after the S3 writes (before the + # result COMMIT), after the result COMMIT (before the ack); transient failure of the first N + # attempts; permanent failure; slow processing (N s, lease kept alive) of every job or only of + # pipeline version `slow_pipeline`; hang: stop the heartbeat and sleep N s inside processing. + "crash_after_claim", "crash_after_s3_write", "crash_after_commit", "transient_error", "permanent_error", + "slow", "slow_pipeline", "hang", + # Worker cleanup: pause N s after reading the candidates, before locking/deleting them. + "pause_cleanup", +} + + +def _faults() -> tuple[frozenset[str], dict[str, int]]: + names, values = set(), {} + for item in os.environ.get("FAULT_POINTS", "").split(","): + name, _, raw = item.strip().partition("=") + if not name: + continue + if name not in FAULT_POINTS: + raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}") + if raw: + if not raw.isdigit(): + raise ConfigError(f"FAULT_POINTS value of {name} must be an integer") + values[name] = int(raw) + names.add(name) + return frozenset(names), values + + +def _bucket(name: str, default: str) -> str: + bucket = os.environ.get(name, default).strip() or default + if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket): + raise ConfigError(f"{name} is not a valid bucket name: {bucket!r}") + return bucket def s3_settings(*, credentials_required: bool = True) -> dict: - """S3 part of Settings; also used by the reconciler/bootstrap CLIs with their own keys.""" - faults = frozenset(f.strip() for f in os.environ.get("FAULT_POINTS", "").split(",") if f.strip()) - if faults - FAULT_POINTS: - raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}") + """S3 part of Settings; also used by the bootstrap CLI with its own keys.""" + faults, fault_values = _faults() endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").strip() or "http://s3:8333" if not re.fullmatch(r"https?://[A-Za-z0-9.\-]+(:[0-9]{1,5})?/?", endpoint): raise ConfigError(f"S3_ENDPOINT must be http(s)://host[:port], got {endpoint!r}") - bucket = os.environ.get("S3_BUCKET", "messenger-attachments").strip() or "messenger-attachments" - if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket): - raise ConfigError(f"S3_BUCKET is not a valid bucket name: {bucket!r}") + bucket = _bucket("S3_BUCKET", "messenger-attachments") + staging = _bucket("S3_STAGING_BUCKET", "messenger-staging") + if staging == bucket: + raise ConfigError("S3_STAGING_BUCKET must differ from S3_BUCKET") return dict( s3_endpoint=endpoint, s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1", s3_bucket=bucket, + s3_staging_bucket=staging, s3_access_key_id=_secret("S3_ACCESS_KEY_ID", required=credentials_required), s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY", required=credentials_required), s3_timeout_seconds=_float("S3_TIMEOUT_SECONDS", 3.0, 0.2, 30.0), @@ -233,4 +295,49 @@ def s3_settings(*, credentials_required: bool = True) -> dict: pending_grace_seconds=_int("PENDING_GRACE_SECONDS", 900, 1, 7 * 24 * 3600), janitor_interval_seconds=_float("JANITOR_INTERVAL_SECONDS", 60.0, 0.0, 3600.0), fault_points=faults, + fault_values=fault_values, + ) + + +def jobs_settings(*, broker_required: bool) -> dict: + """Lab 5 pipeline: retries, lease, worker limits, cleanup, relay (docs/PIPELINE.md).""" + from .pipeline import PIPELINES # local import: pipeline.py imports nothing from here + timeout = _float("JOB_TIMEOUT_SECONDS", 60.0, 1.0, 600.0) + lease = _float("JOB_LEASE_SECONDS", 30.0, 2.0, 600.0) + base = _float("JOB_RETRY_BASE_SECONDS", 2.0, 0.0, 600.0) + cap = _float("JOB_RETRY_MAX_SECONDS", 60.0, 0.0, 3600.0) + if base > cap: + raise ConfigError("JOB_RETRY_BASE_SECONDS must not exceed JOB_RETRY_MAX_SECONDS") + orphan_grace = _int("ORPHAN_GRACE_SECONDS", 600, 1, 7 * 24 * 3600) + version = _int("PIPELINE_VERSION", 1, 1, 1000) + if version not in PIPELINES: + raise ConfigError(f"PIPELINE_VERSION must be one of {sorted(PIPELINES)}") + return dict( + broker_url=_secret("BROKER_URL", required=broker_required), + # Pipeline version of the jobs created by new uploads (track 2: versioned pipeline). + pipeline_version=version, + # Managed attempts (claims) per job before failed/retry_exhausted. + job_max_attempts=_int("JOB_MAX_ATTEMPTS", 5, 1, 100), + # Hard limit of ONE attempt: the image child process is killed, the lease is no longer renewed. + job_timeout_seconds=timeout, + # Lease of a claimed job, renewed by the heartbeat every lease/3 while the attempt runs. + job_lease_seconds=lease, + # Backoff of transient failures: full jitter, uniform(0, min(max, base * 2^(attempt-1))). + job_retry_base_seconds=base, + job_retry_max_seconds=cap, + # Jobs processed at once by ONE worker process = RabbitMQ prefetch (basic.qos). + worker_concurrency=_int("WORKER_CONCURRENCY", 2, 1, 32), + worker_shutdown_seconds=_float("WORKER_SHUTDOWN_SECONDS", 20.0, 0.0, 120.0), + # Address-space limit (RLIMIT_AS) of the child process that decodes/encodes ONE image. + image_max_memory_mb=_int("IMAGE_MAX_MEMORY_MB", 768, 128, 8192), + # Staging copy of a FAILED attachment is kept this long for a replay, then deleted. + staging_retention_seconds=_int("STAGING_RETENTION_SECONDS", 86400, 0, 30 * 24 * 3600), + # Objects of attempts that did not publish are deleted once the attempt ended this long ago. + orphan_grace_seconds=orphan_grace, + cleanup_interval_seconds=_float("CLEANUP_INTERVAL_SECONDS", 30.0, 0.0, 3600.0), + relay_poll_seconds=_float("RELAY_POLL_SECONDS", 0.5, 0.05, 60.0), + relay_batch=_int("RELAY_BATCH", 50, 1, 1000), + # A queued job whose message has not led to a claim for this long is published again + # (message lost in the broker, dead-lettered, purged): no accepted job waits forever. + queued_republish_seconds=_float("QUEUED_REPUBLISH_SECONDS", 300.0, 5.0, 86400.0), ) diff --git a/src/messenger/imaging.py b/src/messenger/imaging.py new file mode 100644 index 0000000..2bf15cc --- /dev/null +++ b/src/messenger/imaging.py @@ -0,0 +1,208 @@ +"""Variants of ONE image, built in a separate child process with hard limits (lab 5 worker). + + parent (worker): await render(data, content_type, version, ...) -> list[Rendered] + child: python -m messenger.imaging --type image/png --pipeline 1 ... + stdin: original bytes; stdout: one JSON header line + variant bytes + +Why a child process per image: +* memory: RLIMIT_AS (IMAGE_MAX_MEMORY_MB) caps the address space of the decoder; a picture that + needs more fails with MemoryError -> `image_too_large` (permanent), the worker keeps running; +* time: the parent kills the child at JOB_TIMEOUT_SECONDS (C code of Pillow cannot be + interrupted inside a thread) -> `processing_timeout` (transient, bounded by JOB_MAX_ATTEMPTS); +* pixels: the size is read from the header and checked against MAX_IMAGE_PIXELS BEFORE the + pixels are decoded (`Image.open` is lazy). +A crash of the decoder (signal) is a transient `processing_killed`: the next attempt may run +with less memory pressure; JOB_MAX_ATTEMPTS bounds it. + +Geometry and encoder settings: pipeline.py. Alpha is handled in premultiplied form (RGBa) while +resizing, so transparent pixels do not bleed their colour into the edges; for JPEG the result is +then composited onto white. +""" +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import io +import json +import sys +import warnings +from dataclasses import dataclass + +from .pipeline import CONTENT_TYPES, CROP_SIDE, OPTIMIZED_BOX, PIPELINES, THUMBNAIL_BOX, center_square, fit + +EXIT_INVALID = 3 # the bytes are not a decodable PNG/JPEG +EXIT_TOO_LARGE = 4 # pixel limit or memory limit +HEADER_LIMIT = 4096 + + +class ProcessingError(Exception): + """Outcome of an attempt that did not produce variants. `permanent`: another attempt with the + same input cannot succeed (job -> failed at once); otherwise retry with backoff.""" + + def __init__(self, code: str, permanent: bool): + super().__init__(code) + self.code = code + self.permanent = permanent + + +@dataclass(frozen=True, slots=True) +class Rendered: + name: str + content_type: str + width: int + height: int + data: bytes + + @property + def sha256(self) -> bytes: + return hashlib.sha256(self.data).digest() + + +# ------------------------------------------------------------------ parent side +async def render(data: bytes, content_type: str, pipeline_version: int, *, max_pixels: int, timeout: float, + memory_mb: int) -> list[Rendered]: + process = await asyncio.create_subprocess_exec( + sys.executable, "-m", "messenger.imaging", "--type", content_type, "--pipeline", str(pipeline_version), + "--max-pixels", str(max_pixels), "--memory-mb", str(memory_mb), + stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) + try: + out, _ = await asyncio.wait_for(process.communicate(data), timeout) + except asyncio.TimeoutError: + process.kill() + await process.wait() + raise ProcessingError("processing_timeout", permanent=False) from None + except asyncio.CancelledError: + process.kill() + await process.wait() + raise + code = process.returncode + if code == EXIT_INVALID: + raise ProcessingError("invalid_image", permanent=True) + if code == EXIT_TOO_LARGE: + raise ProcessingError("image_too_large", permanent=True) + if code is None or code < 0: + raise ProcessingError("processing_killed", permanent=False) # OOM killer or another signal + if code != 0: + raise ProcessingError("processing_failed", permanent=False) + return parse_output(out) + + +def parse_output(out: bytes) -> list[Rendered]: + newline = out.find(b"\n", 0, HEADER_LIMIT) + if newline < 0: + raise ProcessingError("processing_failed", permanent=False) + header = json.loads(out[:newline]) + pos = newline + 1 + result = [] + for item in header["variants"]: + size = int(item["size"]) + result.append(Rendered(item["name"], item["content_type"], int(item["width"]), int(item["height"]), + out[pos:pos + size])) + pos += size + if pos != len(out) or [r.name for r in result] != list(CONTENT_TYPES): + raise ProcessingError("processing_failed", permanent=False) + return result + + +# ------------------------------------------------------------------ child side +def build(data: bytes, content_type: str, pipeline_version: int, max_pixels: int) -> list[Rendered]: + """Decode once, build every variant. Raises the Pillow/OS errors of a bad file.""" + from PIL import Image, ImageOps + + pipeline = PIPELINES[pipeline_version] + warnings.simplefilter("ignore", Image.DecompressionBombWarning) # our own pixel limit below + Image.MAX_IMAGE_PIXELS = max_pixels + expected = {"image/png": "PNG", "image/jpeg": "JPEG"}[content_type] + img = Image.open(io.BytesIO(data), formats=[expected]) # header only: nothing decoded yet + width, height = img.size + if width * height > max_pixels: + raise TooLarge() + if expected == "JPEG": + # DCT scaling: decode a big JPEG directly at 1/2..1/8 of its size, never below what the + # largest variant needs (the box stays square, so EXIF rotation does not matter). + img.draft("RGB", (OPTIMIZED_BOX, OPTIMIZED_BOX)) + img.load() + img = ImageOps.exif_transpose(img) # orientation goes into the pixels, the tag is dropped + img = _normalize(img) + alpha = img.mode == "RGBA" + base = img.convert("RGBa") if alpha else img # premultiplied alpha for resampling + del img + + def finish(im, *, flatten: bool): + if alpha: + im = im.convert("RGBA") + if flatten: + white = Image.new("RGB", im.size, (255, 255, 255)) + white.paste(im, mask=im.getchannel("A")) + im = white + im.info = {} # no EXIF/XMP/ICC/comments of the source in a variant + return im + + def resized(im, size): + return im if im.size == size else im.resize(size, Image.Resampling.LANCZOS, reducing_gap=3.0) + + thumbnail = finish(resized(base, fit(*base.size, THUMBNAIL_BOX)), flatten=True) + crop = finish(resized(base.crop(center_square(*base.size)), (CROP_SIDE, CROP_SIDE)), flatten=True) + optimized = finish(resized(base, fit(*base.size, OPTIMIZED_BOX)), flatten=False) + out = [] + for name, im in (("thumbnail", thumbnail), ("crop", crop), ("optimized", optimized)): + buf = io.BytesIO() + if CONTENT_TYPES[name] == "image/jpeg": + im.save(buf, "JPEG", quality=pipeline.jpeg_quality, optimize=True, subsampling=2, progressive=False) + else: + im.save(buf, "WEBP", quality=pipeline.webp_quality, method=pipeline.webp_method, exif=b"", + icc_profile="", xmp=b"") + out.append(Rendered(name, CONTENT_TYPES[name], im.width, im.height, buf.getvalue())) + return out + + +class TooLarge(Exception): + pass + + +def _normalize(img): + """Any PNG/JPEG mode -> RGB, or RGBA when the image has transparency.""" + if img.mode.startswith("I;16") or img.mode == "I": + # 16-bit grey: scale 0..65535 to 0..255 (a plain convert("L") would clip to white). + img = img.convert("I").point(lambda v: v * (1 / 257)).convert("L") + has_alpha = img.mode in ("RGBA", "LA", "PA", "La", "RGBa") or "transparency" in img.info + return img.convert("RGBA" if has_alpha else "RGB") if img.mode != ("RGBA" if has_alpha else "RGB") else img + + +def child_main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--type", required=True, choices=["image/png", "image/jpeg"]) + parser.add_argument("--pipeline", type=int, required=True, choices=sorted(PIPELINES)) + parser.add_argument("--max-pixels", type=int, required=True) + parser.add_argument("--memory-mb", type=int, required=True) + args = parser.parse_args(argv) + try: + import resource + limit = args.memory_mb * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (limit, limit)) + except (ImportError, ValueError, OSError): + pass # not Linux (unit tests on Windows): the container memory limit still applies + data = sys.stdin.buffer.read() + try: + variants = build(data, args.type, args.pipeline, args.max_pixels) + except (TooLarge, MemoryError): + return EXIT_TOO_LARGE + except Exception as exc: # noqa: BLE001 - every decoder error of the CONTENT is "invalid image" + from PIL import Image + if isinstance(exc, Image.DecompressionBombError): + return EXIT_TOO_LARGE + if isinstance(exc, (OSError, SyntaxError, ValueError, EOFError, KeyError, IndexError, TypeError)): + return EXIT_INVALID + raise + header = {"variants": [{"name": v.name, "content_type": v.content_type, "width": v.width, "height": v.height, + "size": len(v.data)} for v in variants]} + sys.stdout.buffer.write(json.dumps(header).encode() + b"\n") + for v in variants: + sys.stdout.buffer.write(v.data) + sys.stdout.buffer.flush() + return 0 + + +if __name__ == "__main__": + sys.exit(child_main()) diff --git a/src/messenger/jobs.py b/src/messenger/jobs.py new file mode 100644 index 0000000..5eebe9d --- /dev/null +++ b/src/messenger/jobs.py @@ -0,0 +1,398 @@ +"""State machine of attachment jobs in PostgreSQL (lab 5). Every transition is ONE transaction. + +Lock order, everywhere both rows are locked: the jobs row FIRST, then the attachments row +(claim, complete, retry, lease expiry). Replay/reprocess/cleanup lock only the attachments row +and insert NEW job rows. So two transitions never wait on each other in opposite order. + +Fencing: a claim gives the attempt a fresh random `lease_token`. Every later write of that attempt +(heartbeat, result, retry) is conditional on `jobs.lease_token = ` under the jobs row +lock. When the lease expires, the reaper clears the token; from then on the old attempt - even if +its process is still alive (paused, slow) - can neither publish nor schedule anything, and the +objects it wrote under its own keys are garbage for the cleanup. + +Monotonic publication: a result of pipeline version V is published only if V is greater than the +published version of the attachment; otherwise the job ends as `superseded` and nothing visible +changes. `ready` never goes back to queued/processing. +""" +from __future__ import annotations + +import datetime as dt +import json +import random +import uuid +from dataclasses import dataclass + +import psycopg + +from .pipeline import VARIANT_NAMES +from .storage import attempt_key + +ACTIVE = ("queued", "processing") + + +def backoff_delay(attempt: int, base: float, cap: float, rnd=random.random) -> float: + """Full jitter (AWS Architecture Blog, "Exponential Backoff And Jitter"): uniform in + [0, min(cap, base * 2^(attempt-1))]. Spreads the retries of many jobs that failed together.""" + return rnd() * min(cap, base * (2 ** max(0, attempt - 1))) + + +async def event(conn: psycopg.AsyncConnection, job_id, name: str, actor: str, *, attempt: int | None = None, + error_code: str | None = None, **detail) -> None: + await conn.execute( + "INSERT INTO job_events (job_id, event, attempt, actor, error_code, detail) VALUES (%s, %s, %s, %s, %s, %s)", + (job_id, name, attempt, actor, error_code, json.dumps(detail, default=str))) + + +async def create_job(conn: psycopg.AsyncConnection, attachment_id, pipeline_version: int, kind: str, + max_attempts: int, actor: str, *, replay_of=None, **detail) -> uuid.UUID: + """jobs row + its first outbox row + audit, in the CALLER's transaction.""" + job_id = uuid.uuid4() + await conn.execute( + "INSERT INTO jobs (id, attachment_id, pipeline_version, kind, replay_of, max_attempts) " + "VALUES (%s, %s, %s, %s, %s, %s)", (job_id, attachment_id, pipeline_version, kind, replay_of, max_attempts)) + await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, 1)", (job_id,)) + await event(conn, job_id, "created", actor, kind=kind, pipeline_version=pipeline_version, + replay_of=str(replay_of) if replay_of else None, **detail) + return job_id + + +# ------------------------------------------------------------------ relay +@dataclass(frozen=True, slots=True) +class Due: + outbox_id: int + job_id: str + attachment_id: str + pipeline_version: int + attempt: int + created_at: dt.datetime + job_state: str + + +async def due_outbox(conn: psycopg.AsyncConnection, batch: int) -> list[Due]: + """Due, unpublished outbox rows, locked for this relay pass (SKIP LOCKED: several relays + never publish the same row at the same time - a crash may still publish it twice).""" + cur = await conn.execute( + "SELECT o.id, o.job_id, j.attachment_id, j.pipeline_version, o.attempt, j.created_at, j.state " + "FROM outbox o JOIN jobs j ON j.id = o.job_id " + "WHERE o.published_at IS NULL AND o.available_at <= clock_timestamp() " + "ORDER BY o.available_at, o.id LIMIT %s FOR UPDATE OF o SKIP LOCKED", (batch,)) + return [Due(r[0], str(r[1]), str(r[2]), r[3], r[4], r[5], r[6]) for r in await cur.fetchall()] + + +async def mark_published(conn: psycopg.AsyncConnection, ids: list[int]) -> None: + if ids: + await conn.execute("UPDATE outbox SET published_at = clock_timestamp() WHERE id = ANY(%s)", (ids,)) + + +async def count_publish_failure(conn: psycopg.AsyncConnection, ids: list[int]) -> None: + if ids: + await conn.execute("UPDATE outbox SET publish_failures = publish_failures + 1 WHERE id = ANY(%s)", (ids,)) + + +async def expire_leases(conn: psycopg.AsyncConnection, actor: str, limit: int = 50) -> int: + """Reaper: processing jobs whose lease ran out (the worker died, hung or lost its database). + The attempt is abandoned; the job goes back to queued with a new outbox row, or to failed + `retry_exhausted` when it used all its attempts.""" + cur = await conn.execute( + "SELECT id, attempt, max_attempts, lease_token, attachment_id FROM jobs " + "WHERE state = 'processing' AND lease_until < clock_timestamp() " + "ORDER BY lease_until LIMIT %s FOR UPDATE SKIP LOCKED", (limit,)) + rows = await cur.fetchall() + for job_id, attempt, max_attempts, token, attachment_id in rows: + await conn.execute( + "UPDATE job_attempts SET outcome = 'abandoned', error_code = 'lease_expired', finished_at = clock_timestamp() " + "WHERE token = %s AND outcome = 'running'", (token,)) + if attempt >= max_attempts: + await _fail(conn, job_id, attachment_id, "retry_exhausted", actor, attempt, cause="lease_expired") + else: + await conn.execute( + "UPDATE jobs SET state = 'queued', lease_token = NULL, lease_owner = NULL, lease_until = NULL, " + "available_at = clock_timestamp(), updated_at = clock_timestamp() WHERE id = %s", (job_id,)) + await conn.execute( + "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() " + "WHERE id = %s AND status = 'processing'", (attachment_id,)) + await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, %s)", (job_id, attempt + 1)) + await event(conn, job_id, "lease_expired", actor, attempt=attempt) + return len(rows) + + +async def republish_stale(conn: psycopg.AsyncConnection, older_than: float, actor: str, limit: int = 50) -> int: + """Lost-message guard: a queued, due job that has had no unpublished outbox row and no + publish for `older_than` seconds gets a new outbox row (its message was lost by the broker, + dead-lettered after too many redeliveries, or the queue was purged).""" + cur = await conn.execute( + "SELECT j.id, j.attempt FROM jobs j " + "WHERE j.state = 'queued' AND j.available_at < clock_timestamp() - make_interval(secs => %s) " + "AND j.updated_at < clock_timestamp() - make_interval(secs => %s) " + "AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.id AND (o.published_at IS NULL " + " OR o.published_at > clock_timestamp() - make_interval(secs => %s))) " + "ORDER BY j.updated_at LIMIT %s FOR UPDATE OF j SKIP LOCKED", (older_than, older_than, older_than, limit)) + rows = await cur.fetchall() + for job_id, attempt in rows: + await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, %s)", (job_id, attempt + 1)) + await event(conn, job_id, "republished", actor, attempt=attempt) + return len(rows) + + +async def purge_outbox(conn: psycopg.AsyncConnection, keep_seconds: int = 7 * 86400) -> int: + cur = await conn.execute( + "DELETE FROM outbox WHERE id IN (SELECT id FROM outbox WHERE published_at < clock_timestamp() " + "- make_interval(secs => %s) LIMIT 1000)", (keep_seconds,)) + return cur.rowcount + + +# ------------------------------------------------------------------ worker +@dataclass(frozen=True, slots=True) +class Claim: + job_id: str + attachment_id: str + pipeline_version: int + attempt: int + max_attempts: int + token: str + source: str # staging | original + source_key: str + content_type: str + size_bytes: int + sha256: bytes + write_original: bool # the original is not published yet: this attempt publishes it too + + def key(self, name: str) -> str: + return attempt_key(self.attachment_id, self.token, name) + + +@dataclass(frozen=True, slots=True) +class Noop: + reason: str # finished | leased | not_due | superseded | unknown_job + + +async def claim(conn: psycopg.AsyncConnection, job_id: str, worker: str, lease_seconds: float) -> Claim | Noop: + cur = await conn.execute( + "SELECT state, attempt, max_attempts, available_at > clock_timestamp(), attachment_id, pipeline_version " + "FROM jobs WHERE id = %s FOR UPDATE", (job_id,)) + row = await cur.fetchone() + if row is None: + return Noop("unknown_job") + state, attempt, max_attempts, not_due, attachment_id, version = row + if state not in ACTIVE: + return Noop("finished") # duplicate/redelivery of a finished job: ack, nothing else + if state == "processing": + return Noop("leased") # another attempt holds the lease; the reaper handles its death + if not_due: + return Noop("not_due") # backoff not over: the outbox publishes it at available_at + cur = await conn.execute( + "SELECT status, published_version, object_key, staging_key, staging_state, content_type, size_bytes, sha256 " + "FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)) + status, published, object_key, staging_key, staging_state, content_type, size, sha = await cur.fetchone() + actor = "worker:" + worker + if published is not None and published >= version: + # A newer (or the same) version is already published: this job can only be stale. + await _finish(conn, job_id, "superseded", None) + await event(conn, job_id, "superseded", actor, attempt=attempt, published_version=published) + return Noop("superseded") + if object_key is not None: + source, source_key = "original", object_key + elif staging_state == "present": + source, source_key = "staging", staging_key + else: + # Cannot happen while the CHECK attachments_active_has_source holds; fail visibly anyway. + await _fail(conn, job_id, attachment_id, "source_missing", actor, attempt) + return Noop("finished") + token = uuid.uuid4().hex + attempt += 1 + write_original = object_key is None + names = (("original",) if write_original else ()) + VARIANT_NAMES + planned = [attempt_key(str(attachment_id), token, n) for n in names] + await conn.execute( + "UPDATE jobs SET state = 'processing', attempt = %s, lease_token = %s, lease_owner = %s, " + "lease_until = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp() WHERE id = %s", + (attempt, token, worker, lease_seconds, job_id)) + await conn.execute( + "INSERT INTO job_attempts (token, job_id, attempt, worker, source, planned_keys) VALUES (%s, %s, %s, %s, %s, %s)", + (token, job_id, attempt, worker, source, planned)) + await conn.execute( + "UPDATE attachments SET status = 'processing', updated_at = clock_timestamp() WHERE id = %s AND status = 'queued'", + (attachment_id,)) + await event(conn, job_id, "claimed", actor, attempt=attempt, source=source, token=token) + return Claim(str(job_id), str(attachment_id), version, attempt, max_attempts, token, source, source_key, + content_type, size, bytes(sha), write_original) + + +async def heartbeat(conn: psycopg.AsyncConnection, claim: Claim, lease_seconds: float) -> bool: + """Extend the lease; False = the lease was lost (expired and taken over): stop the attempt.""" + cur = await conn.execute( + "UPDATE jobs SET lease_until = clock_timestamp() + make_interval(secs => %s) " + "WHERE id = %s AND state = 'processing' AND lease_token = %s", (lease_seconds, claim.job_id, claim.token)) + return cur.rowcount == 1 + + +async def _holds_lease(conn: psycopg.AsyncConnection, claim: Claim) -> bool: + cur = await conn.execute("SELECT state = 'processing' AND lease_token = %s FROM jobs WHERE id = %s FOR UPDATE", + (claim.token, claim.job_id)) + row = await cur.fetchone() + return bool(row and row[0]) + + +async def complete(conn: psycopg.AsyncConnection, claim: Claim, variants, worker: str) -> str: + """Publish the result of an attempt atomically: variant rows + attachment ready + job succeeded. + Returns succeeded | superseded | fenced (this attempt lost its lease: nothing written).""" + actor = "worker:" + worker + if not await _holds_lease(conn, claim): + return "fenced" + cur = await conn.execute("SELECT published_version FROM attachments WHERE id = %s FOR UPDATE", (claim.attachment_id,)) + (published,) = await cur.fetchone() + if published is not None and published >= claim.pipeline_version: + await _end_attempt(conn, claim, "superseded", None) + await _finish(conn, claim.job_id, "superseded", None) + await event(conn, claim.job_id, "superseded", actor, attempt=claim.attempt, published_version=published) + return "superseded" + for v in variants: + await conn.execute( + "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, " + "size_bytes, width, height, sha256, attempt_token) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", + (claim.attachment_id, claim.pipeline_version, v.name, claim.key(v.name), v.content_type, len(v.data), + v.width, v.height, v.sha256, claim.token)) + await conn.execute( + "UPDATE attachments SET status = 'ready', published_version = %s, error_code = NULL, failed_at = NULL, " + "object_key = COALESCE(object_key, %s), updated_at = clock_timestamp() WHERE id = %s", + (claim.pipeline_version, claim.key("original") if claim.write_original else None, claim.attachment_id)) + await _end_attempt(conn, claim, "succeeded", None) + await _finish(conn, claim.job_id, "succeeded", None) + await event(conn, claim.job_id, "succeeded", actor, attempt=claim.attempt, pipeline_version=claim.pipeline_version) + return "succeeded" + + +async def retry_or_fail(conn: psycopg.AsyncConnection, claim: Claim, code: str, permanent: bool, worker: str, + base: float, cap: float) -> tuple[str, float]: + """After a failed attempt: permanent error or no attempts left -> failed (failed registry); + otherwise queued again after a jittered backoff. Returns (outcome, delay seconds).""" + actor = "worker:" + worker + if not await _holds_lease(conn, claim): + return "fenced", 0.0 + if permanent or claim.attempt >= claim.max_attempts: + final = code if permanent else "retry_exhausted" + await _end_attempt(conn, claim, "failed", code) + await _fail(conn, claim.job_id, claim.attachment_id, final, actor, claim.attempt, cause=code) + return "failed", 0.0 + delay = backoff_delay(claim.attempt, base, cap) + await _end_attempt(conn, claim, "retry", code) + await conn.execute( + "UPDATE jobs SET state = 'queued', lease_token = NULL, lease_owner = NULL, lease_until = NULL, error_code = %s, " + "available_at = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp() WHERE id = %s", + (code, delay, claim.job_id)) + await conn.execute( + "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() WHERE id = %s AND status = 'processing'", + (claim.attachment_id,)) + await conn.execute( + "INSERT INTO outbox (job_id, attempt, available_at) " + "VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s))", (claim.job_id, claim.attempt + 1, delay)) + await event(conn, claim.job_id, "retry_scheduled", actor, attempt=claim.attempt, error_code=code, + delay_s=round(delay, 3)) + return "retry", delay + + +async def release(conn: psycopg.AsyncConnection, claim: Claim, worker: str) -> str: + """Graceful stop of a worker with an attempt still running: give the job back at once + (without waiting for the lease to expire). The claim already counted as an attempt.""" + if not await _holds_lease(conn, claim): + return "fenced" + await _end_attempt(conn, claim, "abandoned", "worker_shutdown") + if claim.attempt >= claim.max_attempts: + await _fail(conn, claim.job_id, claim.attachment_id, "retry_exhausted", "worker:" + worker, claim.attempt, + cause="worker_shutdown") + return "failed" + await conn.execute( + "UPDATE jobs SET state = 'queued', lease_token = NULL, lease_owner = NULL, lease_until = NULL, " + "available_at = clock_timestamp(), updated_at = clock_timestamp() WHERE id = %s", (claim.job_id,)) + await conn.execute( + "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() WHERE id = %s AND status = 'processing'", + (claim.attachment_id,)) + await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, %s)", (claim.job_id, claim.attempt + 1)) + await event(conn, claim.job_id, "released", "worker:" + worker, attempt=claim.attempt) + return "released" + + +async def _end_attempt(conn: psycopg.AsyncConnection, claim: Claim, outcome: str, code: str | None) -> None: + await conn.execute( + "UPDATE job_attempts SET outcome = %s, error_code = %s, finished_at = clock_timestamp() " + "WHERE token = %s AND outcome = 'running'", (outcome, code, claim.token)) + + +async def _finish(conn: psycopg.AsyncConnection, job_id, state: str, code: str | None) -> None: + await conn.execute( + "UPDATE jobs SET state = %s, error_code = %s, lease_token = NULL, lease_owner = NULL, lease_until = NULL, " + "finished_at = clock_timestamp(), updated_at = clock_timestamp() WHERE id = %s", (state, code, job_id)) + + +async def _fail(conn: psycopg.AsyncConnection, job_id, attachment_id, code: str, actor: str, attempt: int, + **detail) -> None: + """Job -> failed (registry). The attachment becomes failed only if it is not ready: a failed + reprocess of a newer version leaves the published result in place.""" + await _finish(conn, job_id, "failed", code) + await conn.execute( + "UPDATE attachments SET status = 'failed', error_code = %s, failed_at = clock_timestamp(), " + "updated_at = clock_timestamp() WHERE id = %s AND status IN ('queued', 'processing')", (code, attachment_id)) + await event(conn, job_id, "failed", actor, attempt=attempt, error_code=code, **detail) + + +# ------------------------------------------------------------------ cleanup (worker) +async def staging_to_delete(conn: psycopg.AsyncConnection, retention_seconds: int, limit: int = 50) -> list[str]: + """Candidates only (no lock): ready attachments (staging no longer needed) and failed ones + older than the retention. The decision is re-checked under the row lock in mark_staging_deleting.""" + cur = await conn.execute( + "SELECT id FROM attachments WHERE staging_state IN ('present', 'deleting') AND (status = 'ready' OR " + "(status = 'failed' AND failed_at < clock_timestamp() - make_interval(secs => %s))) " + "ORDER BY updated_at LIMIT %s", (retention_seconds, limit)) + return [str(r[0]) for r in await cur.fetchall()] + + +async def mark_staging_deleting(conn: psycopg.AsyncConnection, attachment_id: str, retention_seconds: int) -> str | None: + """Under the attachment row lock, decide AGAIN from the committed state and mark the staging copy + `deleting` BEFORE the S3 DELETE. A replay that committed in between made the row queued, so it + no longer matches; a replay that comes later sees `deleting` and refuses. Queued/processing + rows never match (and the CHECK attachments_active_has_source forbids it anyway).""" + cur = await conn.execute( + "UPDATE attachments SET staging_state = 'deleting' " # internal state: updated_at stays (public field) + "WHERE id = %s AND staging_state IN ('present', 'deleting') AND (status = 'ready' OR (status = 'failed' " + "AND failed_at < clock_timestamp() - make_interval(secs => %s) AND NOT EXISTS (SELECT 1 FROM jobs j " + "WHERE j.attachment_id = attachments.id AND j.state IN ('queued', 'processing')))) RETURNING staging_key", + (attachment_id, retention_seconds)) + row = await cur.fetchone() + return row[0] if row else None + + +async def mark_staging_deleted(conn: psycopg.AsyncConnection, attachment_id: str) -> None: + await conn.execute( + "UPDATE attachments SET staging_state = 'deleted' WHERE id = %s AND staging_state = 'deleting'", (attachment_id,)) + + +async def attempts_to_clean(conn: psycopg.AsyncConnection, grace_seconds: int, limit: int = 50) -> list[str]: + cur = await conn.execute( + "SELECT token FROM job_attempts WHERE outcome IN ('retry', 'failed', 'abandoned', 'superseded') " + "AND objects_cleaned_at IS NULL AND finished_at < clock_timestamp() - make_interval(secs => %s) " + "ORDER BY finished_at LIMIT %s", (grace_seconds, limit)) + return [r[0].hex for r in await cur.fetchall()] + + +async def lock_attempt_for_cleanup(conn: psycopg.AsyncConnection, token: str) -> list[str] | None: + """Keys of an attempt that can never publish: it ended without success, it is not the lease + holder of its job (tokens are never reused) and nothing published references its keys.""" + cur = await conn.execute( + "SELECT ja.planned_keys FROM job_attempts ja JOIN jobs j ON j.id = ja.job_id " + "WHERE ja.token = %s AND ja.objects_cleaned_at IS NULL " + "AND ja.outcome IN ('retry', 'failed', 'abandoned', 'superseded') " + "AND j.lease_token IS DISTINCT FROM ja.token FOR UPDATE OF ja", (token,)) + row = await cur.fetchone() + if row is None: + return None + keys = list(row[0]) + cur = await conn.execute( + "SELECT EXISTS (SELECT 1 FROM attachment_variants WHERE object_key = ANY(%s)) " + "OR EXISTS (SELECT 1 FROM attachments WHERE object_key = ANY(%s))", (keys, keys)) + if (await cur.fetchone())[0]: + return None # published: never delete (would be a bug elsewhere; logged by the caller) + return keys + + +async def mark_attempt_cleaned(conn: psycopg.AsyncConnection, token: str) -> None: + await conn.execute("UPDATE job_attempts SET objects_cleaned_at = clock_timestamp() WHERE token = %s", (token,)) diff --git a/src/messenger/pipeline.py b/src/messenger/pipeline.py new file mode 100644 index 0000000..9534ac5 --- /dev/null +++ b/src/messenger/pipeline.py @@ -0,0 +1,63 @@ +"""Versioned image pipeline (lab 5): WHAT the worker builds for one attachment. Pure definitions, +no I/O; the pixels are produced by imaging.py in a child process. + +Every pipeline version builds the same three variants with the geometry of the contract; versions +differ only in encoder parameters, so the published version is visible in the bytes (sha256): + + thumbnail JPEG, fits into 256x256, keeps the aspect ratio, never upscales, floor rounding + (minimum 1 px), EXIF orientation applied, alpha composited onto white; + crop JPEG, centred square 128x128 (small images ARE upscaled, the contract allows it); + optimized WebP, fits into 1280x1280, never upscales, alpha kept. + +Metadata: nothing of the source is copied into a variant - no EXIF (orientation is applied to +the pixels instead), no XMP, no ICC profile, no comments (the pixels are treated as sRGB). +The original is published byte for byte and keeps everything. + +Track 2: the version of a job is fixed when the job is created (PIPELINE_VERSION for uploads, +`jobctl reprocess --pipeline N` later). A result is published only if its version is greater +than the published one, so an old job can never overwrite a newer result. +""" +from __future__ import annotations + +from dataclasses import dataclass + +THUMBNAIL_BOX = 256 +CROP_SIDE = 128 +OPTIMIZED_BOX = 1280 +VARIANT_NAMES = ("thumbnail", "crop", "optimized") +CONTENT_TYPES = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"} + + +@dataclass(frozen=True, slots=True) +class Pipeline: + version: int + jpeg_quality: int # thumbnail and crop: baseline JPEG, 4:2:0, optimized Huffman tables + webp_quality: int # optimized: lossy WebP + webp_method: int # 0 (fast) .. 6 (slow, smaller) + + +PIPELINES = { + 1: Pipeline(1, jpeg_quality=85, webp_quality=80, webp_method=4), + # Versions 2 and 3 exist for the track 2 experiments (reprocess, reorder, stale jobs): + # different encoder settings -> different bytes, identical geometry. + 2: Pipeline(2, jpeg_quality=80, webp_quality=75, webp_method=4), + 3: Pipeline(3, jpeg_quality=90, webp_quality=85, webp_method=5), +} + + +def fit(width: int, height: int, box: int) -> tuple[int, int]: + """Size inside box x box keeping the aspect ratio, never upscaling, rounded DOWN, >= 1 px. + Integer arithmetic: 320x200 -> 256x160 exactly, 1x1000 -> 1x256 (0.256 -> 1 px minimum).""" + if width <= box and height <= box: + return width, height + if width >= height: + return box, max(1, height * box // width) + return max(1, width * box // height), box + + +def center_square(width: int, height: int) -> tuple[int, int, int, int]: + """Crop box (left, top, right, bottom) of the centred square with side min(width, height).""" + side = min(width, height) + left = (width - side) // 2 + top = (height - side) // 2 + return left, top, left + side, top + side diff --git a/src/messenger/reconcile.py b/src/messenger/reconcile.py deleted file mode 100644 index e2457b3..0000000 --- a/src/messenger/reconcile.py +++ /dev/null @@ -1,437 +0,0 @@ -"""Reconciler of the two stores (lab 4, track 2): S3 objects vs attachment rows in PostgreSQL. - - docker compose run --rm reconciler # dry-run: report only (default) - docker compose run --rm reconciler --apply # fix what is safe to fix - docker compose run --rm reconciler --verify-checksums # also read every object (SHA-256) - -How it walks: both sides are read as key-ordered streams and merge-joined - ListObjectsV2 -pages (prefix o/, byte order) against keyset pages of `attachments ORDER BY object_key` -(COLLATE "C" = the same byte order, unique index). Memory does not depend on the bucket size; -cost is ~N/1000 S3 list requests + N/page DB pages (measured: docs/CONSISTENCY.md). - -What it finds and what --apply does (nothing else is ever changed): - - kind condition --apply - orphan_object object, no row, older than --grace delete the object - stale_intent row 'pending' older than --grace (+/- object) delete object, then row - missing_object row 'ready', object absent, older than --grace quarantine the row - size_mismatch row 'ready', object size != size_bytes quarantine the row - checksum_mismatch row 'ready', SHA-256 differs (--verify-checksums) quarantine the row - restored_object row quarantined, object present and correct again remove the quarantine - (a checksum quarantine only with --verify-checksums) - fresh_* the same, but younger than --grace nothing (in-flight upload) - -Safety: -* dry-run is read-only BY CONSTRUCTION: the PostgreSQL session runs with - default_transaction_read_only=on and the S3 wrapper has no write methods, so even a bug in - the planner cannot modify a store; -* ready rows are never deleted - a used (or unused) ready attachment can only be quarantined, - which is reversible; objects that have a row are never deleted unless the row is a stale - intent; anything younger than --grace is never touched; -* every action re-checks its condition right before acting, under a row lock (FOR UPDATE) - where a row exists, so the live API racing with the reconciler cannot be hurt; -* object first, row second: an interrupted run leaves a state the next run repairs; - re-running is idempotent (the second run finds nothing to do); -* safety valve: --apply refuses to delete more than --max-orphan-fraction of the scanned - objects (pointing it at the wrong/empty database would otherwise wipe the bucket). -""" -from __future__ import annotations - -import argparse -import asyncio -import datetime as dt -import hashlib -import json -import sys -import time -from collections import Counter -from collections.abc import AsyncIterator -from dataclasses import dataclass, field - -import psycopg - -from .config import ConfigError, Settings, _float, _secret, s3_settings -from .errors import Unavailable -from .storage import OBJECT_PREFIX, ListedObject, ObjectMissing, Storage - -KINDS = ("orphan_object", "stale_intent", "missing_object", "size_mismatch", "checksum_mismatch", - "restored_object", "fresh_orphan_object", "fresh_intent", "fresh_missing_object") -MIN_SAFE_GRACE = 60.0 # > the longest possible upload (queue 5 s + body 30 s + verify 10 s + S3) -ACTION_OF = {"orphan_object": "delete_object", "stale_intent": "delete_intent", "missing_object": "quarantine", - "size_mismatch": "quarantine", "checksum_mismatch": "quarantine", "restored_object": "unquarantine"} - - -def log(text: str) -> None: - print(f"reconcile: {text}", file=sys.stderr, flush=True) - - -@dataclass(frozen=True, slots=True) -class Row: - id: str - key: str - status: str - size: int - sha256: bytes - created_at: dt.datetime - updated_at: dt.datetime - lost_reason: str | None - used: bool - - -@dataclass(slots=True) -class Finding: - kind: str - key: str - attachment_id: str | None = None - used: bool = False - detail: str = "" - result: str = "planned" # planned | applied | skipped: | failed: - - -@dataclass -class Stats: - objects: int = 0 - rows: int = 0 - consistent: int = 0 - s3_list_requests: int = 0 - s3_head_requests: int = 0 - s3_get_bytes: int = 0 - db_pages: int = 0 - - -class ReadOnlyStorage: - """The dry-run view of S3: listing and reading only - there is no delete to call.""" - - def __init__(self, storage: Storage): - self._s = storage - - async def list_page(self, *args, **kwargs): - return await self._s.list_page(*args, **kwargs) - - async def head(self, key: str): - return await self._s.head(key) - - async def read_all(self, key: str, limit: int) -> bytes: - return await self._s.read_all(key, limit) - - -class Reconciler: - def __init__(self, conn: psycopg.AsyncConnection, storage: Storage | ReadOnlyStorage, *, grace: float, - page_size: int, verify: bool, apply: bool): - self.conn = conn - self.s3 = storage - self.grace = grace - self.page_size = page_size - self.verify = verify - self.apply = apply - self.stats = Stats() - self.findings: list[Finding] = [] - - # ---------------------------------------------------------- streams - async def objects(self) -> AsyncIterator[ListedObject]: - after = None - while True: - page, more = await self.s3.list_page(OBJECT_PREFIX, after, self.page_size) - self.stats.s3_list_requests += 1 - for item in page: - self.stats.objects += 1 - yield item - if not more or not page: - return - after = page[-1].key - - async def rows(self) -> AsyncIterator[Row]: - after = "" - while True: - cur = await self.conn.execute( - "SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256, a.created_at, a.updated_at, " - "a.lost_reason, EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) " - "FROM attachments a WHERE a.object_key > %s ORDER BY a.object_key LIMIT %s", - (after, self.page_size)) - page = await cur.fetchall() - self.stats.db_pages += 1 - for r in page: - self.stats.rows += 1 - yield Row(str(r[0]), r[1], r[2], r[3], bytes(r[4]), r[5], r[6], r[7], r[8]) - if len(page) < self.page_size: - return - after = page[-1][1] - - # ---------------------------------------------------------- planning (read-only) - async def scan(self) -> None: - cur = await self.conn.execute("SELECT clock_timestamp()") - self.db_now = (await cur.fetchone())[0] - self.s3_now = time.time() - objects, rows = self.objects(), self.rows() - obj = await anext(objects, None) - row = await anext(rows, None) - while obj is not None or row is not None: - if row is None or (obj is not None and obj.key < row.key): - self.object_only(obj) - obj = await anext(objects, None) - elif obj is None or row.key < obj.key: - self.row_only(row) - row = await anext(rows, None) - else: - await self.both(obj, row) - obj = await anext(objects, None) - row = await anext(rows, None) - - def row_old(self, row: Row) -> bool: - return (self.db_now - max(row.created_at, row.updated_at)).total_seconds() > self.grace - - def object_only(self, obj: ListedObject) -> None: - old = self.s3_now - obj.last_modified_ts > self.grace - self.add(Finding("orphan_object" if old else "fresh_orphan_object", obj.key)) - - def row_only(self, row: Row) -> None: - if row.status == "pending": - self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id, - detail="object absent")) - elif row.lost_reason is not None: - self.stats.consistent += 1 # already quarantined, still missing: nothing new - else: - self.add(Finding("missing_object" if self.row_old(row) else "fresh_missing_object", row.key, row.id, - used=row.used)) - - async def both(self, obj: ListedObject, row: Row) -> None: - if row.status == "pending": - self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id, - detail="object present")) - return - problem = await self.object_problem(row, obj.size) - if problem: - if row.lost_reason is None: - self.add(Finding(problem, row.key, row.id, used=row.used)) - else: - self.stats.consistent += 1 # quarantined and still broken - elif row.lost_reason == "checksum_mismatch" and not self.verify: - # Same size, but this pass does not read the bytes: it cannot prove the object is the - # original again, so a checksum quarantine is lifted only by --verify-checksums. - self.stats.consistent += 1 - elif row.lost_reason is not None: - self.add(Finding("restored_object", row.key, row.id, used=row.used)) - else: - self.stats.consistent += 1 - - async def object_problem(self, row: Row, size: int) -> str | None: - if size != row.size: - return "size_mismatch" - if self.verify: - data = await self.s3.read_all(row.key, row.size) - self.stats.s3_get_bytes += len(data) - if hashlib.sha256(data).digest() != row.sha256: - return "checksum_mismatch" - return None - - def add(self, finding: Finding) -> None: - if finding.kind not in ACTION_OF: - finding.result = "not_actionable" - self.findings.append(finding) - - # ---------------------------------------------------------- actions (--apply only) - async def run_actions(self, max_orphan_fraction: float) -> None: - orphans = sum(1 for f in self.findings if f.kind == "orphan_object") - if self.stats.objects and orphans > max_orphan_fraction * self.stats.objects: - raise SafetyValve(f"{orphans} of {self.stats.objects} objects look orphaned " - f"(> {max_orphan_fraction:.0%}); wrong database? nothing was changed") - handlers = {"delete_object": self.delete_object, "delete_intent": self.delete_intent, - "quarantine": self.quarantine, "unquarantine": self.unquarantine} - for finding in self.findings: - action = ACTION_OF.get(finding.kind) - if action is None: - continue - try: - finding.result = await handlers[action](finding) - except (Unavailable, psycopg.OperationalError) as exc: - finding.result = f"failed:{type(exc).__name__}" - raise # a store is down: stop; the next run continues from the committed state - - async def delete_object(self, f: Finding) -> str: - cur = await self.conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (f.key,)) - if await cur.fetchone(): - return "skipped:row_appeared" - try: - head = await self.s3.head(f.key) - self.stats.s3_head_requests += 1 - except ObjectMissing: - return "skipped:already_gone" - if time.time() - head.last_modified_ts <= self.grace: - return "skipped:object_rewritten_recently" - # No row names this key and the object is older than the grace period. Rows are always - # written BEFORE their object and keys are random and never reused, so no upload can - # still claim it. - await self.s3.delete(f.key) - return "applied" - - async def delete_intent(self, f: Finding) -> str: - async with self.conn.transaction(): - cur = await self.conn.execute( - "SELECT 1 FROM attachments WHERE id = %s AND status = 'pending' " - "AND greatest(created_at, updated_at) < clock_timestamp() - make_interval(secs => %s) FOR UPDATE", - (f.attachment_id, self.grace)) - if await cur.fetchone() is None: - return "skipped:no_longer_stale_intent" - await self.s3.delete(f.key) # object first (idempotent), then the row - await self.conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'pending'", - (f.attachment_id,)) - return "applied" - - async def quarantine(self, f: Finding) -> str: - async with self.conn.transaction(): - cur = await self.conn.execute( - "SELECT size_bytes, sha256 FROM attachments WHERE id = %s AND status = 'ready' " - "AND lost_at IS NULL FOR UPDATE", (f.attachment_id,)) - row = await cur.fetchone() - if row is None: - return "skipped:row_changed" - reason = await self.current_problem(f.key, row[0], bytes(row[1])) - if reason is None: - return "skipped:object_ok_now" - await self.conn.execute( - "UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = %s WHERE id = %s", - (reason, f.attachment_id)) - return "applied" - - async def unquarantine(self, f: Finding) -> str: - async with self.conn.transaction(): - cur = await self.conn.execute( - "SELECT size_bytes, sha256, lost_reason FROM attachments WHERE id = %s AND lost_at IS NOT NULL FOR UPDATE", - (f.attachment_id,)) - row = await cur.fetchone() - if row is None: - return "skipped:row_changed" - if row[2] == "checksum_mismatch" and not self.verify: - return "skipped:needs_verify_checksums" - if await self.current_problem(f.key, row[0], bytes(row[1])) is not None: - return "skipped:object_still_bad" - await self.conn.execute("UPDATE attachments SET lost_at = NULL, lost_reason = NULL WHERE id = %s", - (f.attachment_id,)) - return "applied" - - async def current_problem(self, key: str, size: int, sha256: bytes) -> str | None: - """Fresh look at the object right before changing the row (the listing may be stale).""" - try: - head = await self.s3.head(key) - self.stats.s3_head_requests += 1 - except ObjectMissing: - return "missing_object" - if head.size != size: - return "size_mismatch" - if self.verify: - data = await self.s3.read_all(key, size) - self.stats.s3_get_bytes += len(data) - if hashlib.sha256(data).digest() != sha256: - return "checksum_mismatch" - return None - - # ---------------------------------------------------------- report - def report(self, mode: str, seconds: float, error: str | None, sample: int) -> dict: - kinds = Counter(f.kind for f in self.findings) - results = Counter(f.result.split(":")[0] for f in self.findings if f.kind in ACTION_OF) - samples = {} - for f in self.findings: - items = samples.setdefault(f.kind, []) - if len(items) < sample: - items.append({"key": f.key, "attachment_id": f.attachment_id, "used": f.used, - "result": f.result, **({"detail": f.detail} if f.detail else {})}) - return { - "mode": mode, "grace_seconds": self.grace, "verify_checksums": self.verify, - "duration_seconds": round(seconds, 3), "error": error, - "scanned": {"objects": self.stats.objects, "rows": self.stats.rows, "consistent": self.stats.consistent}, - "findings": {k: kinds.get(k, 0) for k in KINDS}, - "used_attachments_in_findings": sum(1 for f in self.findings if f.used), - "actions": {"planned": sum(1 for f in self.findings if f.kind in ACTION_OF), - "applied": results.get("applied", 0), "skipped": results.get("skipped", 0), - "failed": results.get("failed", 0)}, - "cost": {"s3_list_requests": self.stats.s3_list_requests, "s3_head_requests": self.stats.s3_head_requests, - "s3_get_bytes": self.stats.s3_get_bytes, "db_pages": self.stats.db_pages, - "page_size": self.page_size}, - "samples": samples, - } - - -class SafetyValve(Exception): - pass - - -def arguments(argv: list[str] | None = None) -> argparse.Namespace: - p = argparse.ArgumentParser(prog="python -m messenger.reconcile", description=__doc__.split("\n")[0]) - p.add_argument("--apply", action="store_true", help="fix findings (default: dry-run, read-only)") - p.add_argument("--grace", type=float, default=3600.0, - help="seconds; anything younger is never touched (default 3600, >= PENDING_GRACE_SECONDS)") - p.add_argument("--allow-small-grace", action="store_true", - help="permit --grace below 60 s (tests only: a grace shorter than an upload could remove the " - "intent of an upload that is still running; that upload then fails with 503)") - p.add_argument("--page-size", type=int, default=1000, help="S3 list / DB page size (1..1000)") - p.add_argument("--verify-checksums", action="store_true", help="read every ready object and compare SHA-256") - p.add_argument("--samples", type=int, default=20, help="keys listed per finding kind in the report") - p.add_argument("--max-orphan-fraction", type=float, default=0.5, - help="--apply refuses if more than this share of objects look orphaned (default 0.5)") - args = p.parse_args(argv) - if not 1 <= args.page_size <= 1000: - p.error("--page-size must be 1..1000") - if args.grace < 1 or (args.grace < MIN_SAFE_GRACE and not args.allow_small_grace): - p.error(f"--grace must be >= {MIN_SAFE_GRACE:.0f} s (or >= 1 s with --allow-small-grace)") - if not 0 < args.max_orphan_fraction <= 1: - p.error("--max-orphan-fraction must be in (0, 1]") - return args - - -async def run(args: argparse.Namespace) -> int: - url = _secret("DATABASE_URL", required=True) - base = s3_settings() - settings = Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024, - database_url=url, **base) - timeout_ms = int(_float("DB_TIMEOUT_SECONDS", 10.0, 1.0, 120.0) * 1000) - options = f"-c statement_timeout={timeout_ms} -c lock_timeout={timeout_ms // 2}" - if not args.apply: - options += " -c default_transaction_read_only=on" # dry-run: PostgreSQL itself refuses writes - mode = "apply" if args.apply else "dry-run" - storage = Storage(settings) - await storage.open() - started = time.perf_counter() - error = None - code = 0 - try: - conn = await psycopg.AsyncConnection.connect(url, autocommit=True, connect_timeout=5, options=options, - application_name=f"messenger-reconcile-{mode}") - except psycopg.OperationalError as exc: - await storage.close() - error = f"store_unavailable: database {type(exc).__name__}" - print(json.dumps({"mode": mode, "error": error}, ensure_ascii=False)) - log(error) - return 1 - rec = Reconciler(conn, storage if args.apply else ReadOnlyStorage(storage), grace=args.grace, - page_size=args.page_size, verify=args.verify_checksums, apply=args.apply) - try: - log(f"{mode}: scanning bucket {settings.s3_bucket} (prefix {OBJECT_PREFIX}) against table attachments") - await rec.scan() - log(f"scan done: {rec.stats.objects} objects, {rec.stats.rows} rows, {len(rec.findings)} findings") - if args.apply: - await rec.run_actions(args.max_orphan_fraction) - except SafetyValve as exc: - error, code = f"safety_valve: {exc}", 3 - except (Unavailable, psycopg.OperationalError) as exc: - error, code = f"store_unavailable: {type(exc).__name__}", 1 - finally: - await conn.close() - await storage.close() - print(json.dumps(rec.report(mode, time.perf_counter() - started, error, args.samples), ensure_ascii=False, - indent=1)) - if error: - log(error) - return code - - -def main(argv: list[str] | None = None) -> int: - args = arguments(argv) - try: - return asyncio.run(run(args)) - except ConfigError as exc: - print(f"configuration error: {exc}", file=sys.stderr) - return 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/student-tests/test_reconcile.py b/student-tests/test_reconcile.py deleted file mode 100644 index d32a750..0000000 --- a/student-tests/test_reconcile.py +++ /dev/null @@ -1,217 +0,0 @@ -"""Track 2: the reconciler of S3 and PostgreSQL (python -m messenger.reconcile). - -Faults are injected on BOTH sides, directly into the stores (S3 with the API/reconciler key -pairs from inside the storage network, PostgreSQL with psql): - - S3 side: orphan object (no row), object deleted under a ready row, object overwritten with - other bytes (size / same-size checksum), object of a quarantined row restored; - DB side: stale intent with and without object, quarantine mark on a healthy row; - fresh: orphan and intent younger than --grace (an upload in progress). - -The tests prove: dry-run changes nothing in either store; --apply fixes exactly the old findings; -fresh and used attachments are never deleted (ready rows only ever get a reversible quarantine); -the second run finds nothing to do; the safety valve and store outages stop the run without -damage. A small --grace (8-30 s, --allow-small-grace) keeps the tests fast.""" -from __future__ import annotations - -import concurrent.futures -import hashlib -import time -import uuid -from pathlib import Path - -import pytest - -import imagegen -from conftest import (attachment_row, findings_by_key, psql, reconcile, s3_delete, s3_keys, s3_put, s3_size, - service_down) - -pytestmark = pytest.mark.stack - -ROOT = Path(__file__).resolve().parents[1] -SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() -GRACE = ["--grace", "8", "--allow-small-grace"] -# The main scenario needs "fresh" items to stay younger than --grace for the whole dry-run + snapshot -# (each docker exec costs 1-3 s on Windows), hence a longer grace there. -LONG_GRACE = ["--grace", "30", "--allow-small-grace"] - - -def new_key() -> str: - return "o/" + uuid.uuid4().hex - - -def insert_intent(chat: dict, owner: dict, key: str, data: bytes, age_seconds: int) -> str: - """A 'pending' row as an upload leaves it after the intent COMMIT (crash afterwards).""" - attachment_id = str(uuid.uuid4()) - psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, " - "object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'x.png', 'image/png', %d, " - "decode('%s', 'hex'), 320, 200, '%s', 'pending', clock_timestamp() - make_interval(secs => %d), " - "clock_timestamp() - make_interval(secs => %d))" - % (attachment_id, chat["id"], owner["id"], len(data), hashlib.sha256(data).hexdigest(), key, age_seconds, - age_seconds)) - return attachment_id - - -def snapshot() -> tuple[str, frozenset]: - """Fingerprint of everything the reconciler could change: every attachment row + the key list.""" - rows = psql("SELECT count(*) || ':' || coalesce(md5(string_agg(id::text || status || object_key || " - "coalesce(lost_reason, '-') || updated_at::text, ',' ORDER BY id)), '') FROM attachments") - return rows, frozenset(s3_keys()) - - -def content_status(api, actor, attachment_id) -> int: - return api.client.get(f"/api/v1/attachments/{attachment_id}/content", - headers={"Authorization": "Bearer " + actor["token"]}).status_code - - -def test_detect_and_safely_fix_faults_on_both_sides(api): - alice, bob = api.user("alice"), api.user("bob") - chat = api.chat(alice, bob) - up = {name: api.upload(alice, chat, imagegen.png(30 + i, 20)).json() for i, name in enumerate("ABCDE")} - data = {name: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content - for name, a in up.items()} - key = {name: attachment_row(a["id"])["object_key"] for name, a in up.items()} - api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]}) - - # ---- inject (old) - orphan_old = new_key() - s3_put(orphan_old, SAMPLE) - stale_with, stale_without = new_key(), new_key() - s3_put(stale_with, SAMPLE) - stale_with_id = insert_intent(chat, alice, stale_with, SAMPLE, 600) # < janitor grace 900 s - stale_without_id = insert_intent(chat, alice, stale_without, SAMPLE, 600) - s3_delete(key["C"]) # ready + used, bytes lost - s3_put(key["D"], data["D"][:len(data["D"]) // 2]) # ready, unused, wrong size - psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' " - f"WHERE id = '{up['E']['id']}'") # quarantined, but bytes are fine - time.sleep(32) # everything above is now older than --grace - # ---- inject (fresh: an upload in progress) - fresh_orphan, fresh_intent = new_key(), new_key() - s3_put(fresh_orphan, SAMPLE) - s3_put(fresh_intent, SAMPLE) - fresh_intent_id = insert_intent(chat, alice, fresh_intent, SAMPLE, 0) - - # ---- dry-run: report everything, change nothing - before = snapshot() - dry = reconcile(*LONG_GRACE) - assert snapshot() == before, "dry-run modified a store" - f = findings_by_key(dry) - expected = {orphan_old: "orphan_object", fresh_orphan: "fresh_orphan_object", stale_with: "stale_intent", - stale_without: "stale_intent", fresh_intent: "fresh_intent", key["C"]: "missing_object", - key["D"]: "size_mismatch", key["E"]: "restored_object"} - assert {k: f[k]["kind"] for k in expected} == expected - assert key["A"] not in f and key["B"] not in f - assert f[key["C"]]["used"] is True and dry["mode"] == "dry-run" - assert all(f[k]["result"] in ("planned", "not_actionable") for k in expected) - - # ---- apply - applied = findings_by_key(reconcile("--apply", *LONG_GRACE)) - for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"]): - assert applied[k]["result"] == "applied", (k, applied[k]) - keys = s3_keys() - assert orphan_old not in keys and stale_with not in keys - assert attachment_row(stale_with_id) is None and attachment_row(stale_without_id) is None - # fresh: untouched - assert fresh_orphan in keys and fresh_intent in keys and attachment_row(fresh_intent_id)["status"] == "pending" - # used, lost bytes: quarantined, NOT deleted; the message still references it - row_c = attachment_row(up["C"]["id"]) - assert row_c["status"] == "ready" and row_c["lost_reason"] == "missing_object" - items, _ = api.history(bob, chat, 100) - assert any(up["C"]["id"] in m["attachment_ids"] for m in items) - assert content_status(api, bob, up["C"]["id"]) == 404 - api.post_message(alice, chat, {"text": "", "attachment_ids": [up["C"]["id"]]}, 409) # cannot attach a lost file - assert attachment_row(up["D"]["id"])["lost_reason"] == "size_mismatch" - assert attachment_row(up["E"]["id"])["lost_reason"] is None # quarantine lifted - for name in "ABE": - got = api.call("GET", f"/api/v1/attachments/{up[name]['id']}/content", 200, token=bob["token"]) - assert got.content == data[name] - - # ---- second run: idempotent, nothing left to do for these keys - again = findings_by_key(reconcile("--apply", *LONG_GRACE)) - for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"], key["A"], key["B"]): - assert k not in again, (k, again.get(k)) - - # ---- the lost object comes back from a backup: the quarantine is lifted, the file is served again - s3_put(key["C"], data["C"]) - restored = findings_by_key(reconcile("--apply", *LONG_GRACE)) - assert restored[key["C"]]["kind"] == "restored_object" and restored[key["C"]]["result"] == "applied" - assert api.call("GET", f"/api/v1/attachments/{up['C']['id']}/content", 200, token=bob["token"]).content == data["C"] - # tidy up the fresh items of this test once they are old enough - time.sleep(31) - reconcile("--apply", *LONG_GRACE) - assert fresh_orphan not in s3_keys() and attachment_row(fresh_intent_id) is None - - -def test_checksum_mode_finds_same_size_corruption(api): - alice = api.user("alice") - chat = api.chat(alice) - att = api.upload(alice, chat, SAMPLE).json() - k = attachment_row(att["id"])["object_key"] - s3_put(k, SAMPLE[:1000] + bytes([SAMPLE[1000] ^ 1]) + SAMPLE[1001:]) # same size, other bytes - time.sleep(9) - assert k not in findings_by_key(reconcile(*GRACE)) # sizes match: listing cannot see it - report = reconcile("--verify-checksums", *GRACE) - assert findings_by_key(report)[k]["kind"] == "checksum_mismatch" - assert report["cost"]["s3_get_bytes"] >= len(SAMPLE) - assert findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]["result"] == "applied" - assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch" - # A later pass WITHOUT checksums sees "same size" but cannot prove the bytes: the quarantine stays. - assert k not in findings_by_key(reconcile("--apply", *GRACE)) - assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch" - # Restoring the original and verifying lifts it. - s3_put(k, SAMPLE) - lifted = findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k] - assert (lifted["kind"], lifted["result"]) == ("restored_object", "applied") - assert attachment_row(att["id"])["lost_reason"] is None - - -def test_safety_valve_refuses_mass_deletion(): - """If (almost) every object looked orphaned - e.g. the reconciler pointed at an empty database - - --apply must stop before deleting anything.""" - orphan = new_key() - s3_put(orphan, SAMPLE) - time.sleep(9) - report = reconcile("--apply", "--max-orphan-fraction", "0.000001", *GRACE, expect_code=3) - assert report["error"].startswith("safety_valve") and report["actions"]["applied"] == 0 - assert orphan in s3_keys() - reconcile("--apply", *GRACE) - assert orphan not in s3_keys() - - -@pytest.mark.parametrize("service", ["s3", "postgres"]) -def test_store_outage_stops_the_run_without_damage(api, service): - with service_down(service, "stop"): - report = reconcile(*GRACE, expect_code=1) - assert report["error"].startswith("store_unavailable") - assert reconcile(*GRACE)["error"] is None - - -def test_live_uploads_are_never_hurt_by_concurrent_apply(api): - """Uploads and messages run while --apply runs repeatedly: every upload that got 201 stays - ready and downloadable byte for byte, every message keeps its attachments.""" - alice = api.user("alice") - chat = api.chat(alice) - payloads = [imagegen.png(16 + i, 16) for i in range(40)] - done: list[tuple[dict, bytes]] = [] - - def worker(chunk: list[bytes]) -> None: - for p in chunk: - att = api.upload(alice, chat, p).json() - api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]}) - done.append((att, p)) - - with concurrent.futures.ThreadPoolExecutor(4) as pool: - futures = [pool.submit(worker, payloads[i::4]) for i in range(4)] - runs = 0 - while not all(x.done() for x in futures): - report = reconcile("--apply", *GRACE) - assert report["error"] is None - runs += 1 - for x in futures: - x.result() - assert runs >= 1 and len(done) == len(payloads) - for att, p in done: - row = attachment_row(att["id"]) - assert row["status"] == "ready" and row["lost_reason"] is None and s3_size(row["object_key"]) == len(p) - for att, p in done[::8]: - assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]).content == p