lab5: схема 0005 (jobs, outbox, попытки, варианты, аудит) и модули pipeline

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Миграция только добавляет: статусы pending/queued/processing/ready/failed, staging-ключ,
published_version, jobs с lease и токеном попытки, job_attempts с заранее известными
ключами, attachment_variants (PK = ключ эффекта), outbox, job_events, view failed_jobs,
post_message_v3. CHECK attachments_active_has_source: у queued/processing всегда есть источник.
pipeline.py — версии 1/2/3 и геометрия по контракту; imaging.py — варианты в дочернем
процессе (RLIMIT_AS, таймаут, лимит пикселей); jobs.py — все переходы заданий одной
транзакцией с fencing по токену; broker.py — топология и строгая схема сообщения.
Зависимости: aio-pika 9.6.2, pillow 12.3.0 (lock с хешами).
This commit is contained in:
Alexey Bodanyuk
2026-10-31 16:12:51 +03:00
parent 5112be83cc
commit a633949919
12 changed files with 1244 additions and 998 deletions
-156
View File
@@ -1,156 +0,0 @@
#!/usr/bin/env python3
"""Track 2 experiment: cost of one reconciler pass as a function of the number of attachments.
Hypothesis: a pass is O(N) with ~N/1000 S3 list requests + N/1000 DB pages and constant memory
(merge-join of two key-ordered streams); --verify-checksums adds reading every byte (O(total size)).
Method: synthetic, CONSISTENT attachments (row 'ready' + 1 KiB object with the matching size and
SHA-256, filename 'bench-reconcile') are added in steps up to the largest N; after each step the
reconciler runs in dry-run 3 times (median reported), plus page-size and checksum variants.
Everything synthetic is deleted at the end (objects first, then rows).
python bench/reconcile_cost.py --steps 1000 10000 50000 --output evidence/reconcile
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import platform
import statistics
import subprocess
import sys
import time
import uuid
from pathlib import Path
import httpx2 as httpx
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
PASSWORD = "Bench-password-Ω-12345"
OBJECT = bytes(range(256)) * 4 # 1 KiB, the same content for every object
SHA = hashlib.sha256(OBJECT).hexdigest()
PUT_SCRIPT = """
import asyncio, sys
from messenger.config import Settings, s3_settings
from messenger.storage import Storage
DATA = bytes(range(256)) * 4
async def main():
s = Storage(Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings()))
await s.open()
keys = sys.stdin.read().split()
sem = asyncio.Semaphore(32)
async def one(k):
async with sem:
await s.{op}(k{args})
await asyncio.gather(*(one(k) for k in keys))
await s.close()
print(len(keys))
asyncio.run(main())
"""
def compose(*args: str, stdin: str | None = None, timeout: float = 1800, check: bool = True) -> subprocess.CompletedProcess:
return subprocess.run(["docker", "compose", *args], cwd=ROOT, input=stdin, capture_output=True, text=True,
encoding="utf-8", timeout=timeout, check=check)
def psql(sql: str) -> str:
"""SQL through stdin: a 5000-row INSERT does not fit into a Windows command line (32 KiB)."""
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v",
"ON_ERROR_STOP=1", "-f", "-", stdin=sql).stdout.strip()
def s3_bulk(op: str, keys: list[str]) -> None:
script = PUT_SCRIPT.replace("{op}", op).replace("{args}", ", DATA, 'image/png', {}" if op == "put" else "")
compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "python", "reconciler", "-c", script, stdin="\n".join(keys))
def owner_and_chat() -> tuple[str, str]:
import base64
c = httpx.Client(base_url=BASE, timeout=30)
name = "rc_" + uuid.uuid4().hex[:10]
c.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
token = c.post("/api/v1/auth/sessions", headers={
"Authorization": "Basic " + base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()}).json()["token"]
me = c.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
chat = c.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
json={"title": "bench", "member_ids": [me["id"]]}).json()
return me["id"], chat["id"]
def add(n: int, owner: str, chat: str) -> None:
keys = ["o/" + uuid.uuid4().hex for _ in range(n)]
for i in range(0, n, 5000):
part = keys[i:i + 5000]
values = ",".join(f"('{uuid.uuid4()}','{k}')" for k in part)
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
"object_key, status, created_at, updated_at) "
f"SELECT v.id::uuid, '{chat}', '{owner}', 'bench-reconcile', 'image/png', {len(OBJECT)}, "
f"decode('{SHA}', 'hex'), 1, 1, v.k, 'ready', clock_timestamp() - interval '2 hours', "
f"clock_timestamp() - interval '2 hours' FROM (VALUES {values}) AS v(id, k)")
s3_bulk("put", keys)
def reconcile(*args: str) -> dict:
out = compose("run", "--rm", "--no-deps", "-T", "reconciler", *args).stdout
return json.loads(out)
def measure(label: str, args: list[str], repeats: int) -> dict:
runs = [reconcile(*args) for _ in range(repeats)]
durations = [r["duration_seconds"] for r in runs]
last = runs[-1]
return {"label": label, "args": args, "runs": repeats, "duration_s_median": statistics.median(durations),
"duration_s_all": durations, "objects": last["scanned"]["objects"], "rows": last["scanned"]["rows"],
"findings": {k: v for k, v in last["findings"].items() if v}, "cost": last["cost"]}
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--steps", type=int, nargs="+", default=[1000, 10000, 50000])
p.add_argument("--repeats", type=int, default=3)
p.add_argument("--output", default="evidence/reconcile")
args = p.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
owner, chat = owner_and_chat()
present = 0
results = []
steps = sorted(args.steps)
variant_step = steps[min(1, len(steps) - 1)]
try:
for target in sorted(args.steps):
started = time.perf_counter()
add(target - present, owner, chat)
present = target
print(f"populated {present} synthetic attachments in {time.perf_counter() - started:.1f} s", flush=True)
results.append({"synthetic": present, **measure("dry-run, page 1000", [], args.repeats)})
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
if present == variant_step: # page-size and checksum variants at one size
for label, extra in (("dry-run, page 100", ["--page-size", "100"]),
("dry-run, --verify-checksums", ["--verify-checksums"])):
results.append({"synthetic": present, **measure(label, extra, args.repeats)})
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
finally:
keys = psql("SELECT object_key FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = "
f"'{chat}'").split()
if keys:
s3_bulk("delete", keys)
psql(f"DELETE FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = '{chat}'")
print(f"removed {len(keys)} synthetic attachments", flush=True)
meta = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
text=True).stdout.strip(),
"object_bytes": len(OBJECT), "reconciler_limits": "cpus 1.0, memory 256m (compose)",
"note": "the bucket/table also contain the attachments of earlier test runs (counted in objects/rows)"}
(out / "cost.json").write_text(json.dumps({"meta": meta, "results": results}, ensure_ascii=False, indent=1),
encoding="utf-8")
return 0
if __name__ == "__main__":
sys.exit(main())
-174
View File
@@ -1,174 +0,0 @@
#!/usr/bin/env python3
"""Track 2 evidence: inject faults on both sides, run the reconciler, keep every JSON report.
Scenario A (all fault kinds; the same as student-tests/test_reconcile.py, but the reports are saved):
S3 side: old orphan objects, object deleted under a USED ready row, object truncated under an
unused ready row, same-size corruption (found only with --verify-checksums);
DB side: stale intents (with / without object), quarantine mark on a healthy row;
fresh: orphan + intent younger than --grace.
Runs: dry-run (+ snapshot of both stores before/after) -> apply -> apply again -> restore of the
lost object from a "backup" -> apply (quarantine lifted).
Scenario B (interrupted apply): 2000 old orphans, S3 is paused while --apply deletes them ->
the run stops with an error, nothing else is damaged; the next run finishes the rest; the
third run finds nothing.
python bench/reconcile_faults.py --output evidence/reconcile
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
from conftest import (Api, attachment_row, compose, findings_by_key, psql, s3_delete, s3_exec, s3_keys, # noqa: E402
s3_put)
from test_reconcile import insert_intent, snapshot # noqa: E402
GRACE = ["--grace", "30", "--allow-small-grace"]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
def reconcile(*args: str) -> tuple[int, dict]:
r = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "1000", *args, check=False, timeout=600)
return r.returncode, json.loads(r.stdout)
def save(out: Path, name: str, data) -> None:
(out / name).write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
def ours(report: dict, keys: dict[str, str]) -> dict:
"""The findings of this scenario's keys only (the stand also holds other test data)."""
f = findings_by_key(report)
return {label: ({"kind": f[k]["kind"], "result": f[k]["result"], "used": f[k]["used"]} if k in f else None)
for label, k in keys.items()}
def scenario_a(out: Path) -> dict:
api = Api()
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
up = {n: api.upload(alice, chat, imagegen.png(40 + i, 30, noise=True)).json() for i, n in enumerate("ABCDEF")}
data = {n: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
for n, a in up.items()}
key = {n: attachment_row(a["id"])["object_key"] for n, a in up.items()}
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
keys = {"healthy used A": key["A"], "healthy unused B": key["B"], "used, object deleted C": key["C"],
"unused, object truncated D": key["D"], "quarantined but healthy E": key["E"],
"same-size corruption F": key["F"]}
orphan = "o/" + uuid.uuid4().hex
stale_with, stale_without = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex
s3_put(orphan, SAMPLE)
s3_put(stale_with, SAMPLE)
insert_intent(chat, alice, stale_with, SAMPLE, 600)
insert_intent(chat, alice, stale_without, SAMPLE, 600)
s3_delete(key["C"])
s3_put(key["D"], data["D"][: len(data["D"]) // 2])
s3_put(key["F"], data["F"][:200] + bytes([data["F"][200] ^ 1]) + data["F"][201:])
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' WHERE id = '{up['E']['id']}'")
keys.update({"old orphan object": orphan, "stale intent + object": stale_with, "stale intent, no object": stale_without})
time.sleep(32)
fresh_orphan, fresh_intent = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex
s3_put(fresh_orphan, SAMPLE)
s3_put(fresh_intent, SAMPLE)
insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
keys.update({"fresh orphan (in-flight)": fresh_orphan, "fresh intent (in-flight)": fresh_intent})
before = snapshot()
_, dry = reconcile("--verify-checksums", *GRACE)
after = snapshot()
_, apply1 = reconcile("--apply", "--verify-checksums", *GRACE)
_, apply2 = reconcile("--apply", "--verify-checksums", *GRACE)
s3_put(key["C"], data["C"]) # the lost object restored from a backup
_, apply3 = reconcile("--apply", "--verify-checksums", *GRACE)
for name, report in (("faults-1-dry-run.json", dry), ("faults-2-apply.json", apply1),
("faults-3-apply-again.json", apply2), ("faults-4-after-restore.json", apply3)):
save(out, name, report)
rows = {label: attachment_row(up[n]["id"]) for label, n in (("C", "C"), ("D", "D"), ("E", "E"), ("F", "F"), ("A", "A"), ("B", "B"))}
content = {n: api.client.get(f"/api/v1/attachments/{up[n]['id']}/content",
headers={"Authorization": "Bearer " + bob["token"]}).status_code for n in "ABCDEF"}
present = s3_keys()
summary = {
"dry_run_changed_nothing": before == after,
"per_key": {"dry-run": ours(dry, keys), "apply": ours(apply1, keys), "apply again": ours(apply2, keys),
"after restore of C": ours(apply3, keys)},
"objects_present_at_end (fresh items were older than --grace by the later runs)": {label: k in present for label, k in keys.items()},
"rows_after": {k: (v and {"status": v["status"], "lost_reason": v["lost_reason"]}) for k, v in rows.items()},
"content_status_after": content,
"durations_s": [dry["duration_seconds"], apply1["duration_seconds"], apply2["duration_seconds"],
apply3["duration_seconds"]],
}
print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True)
return summary
def scenario_b(out: Path, count: int = 2000) -> dict:
keys = ["o/" + uuid.uuid4().hex for _ in range(count)]
s3_exec("ks = sys.stdin.read().split()\n"
"sem = asyncio.Semaphore(32)\n"
"async def one(k):\n"
" async with sem:\n"
" await s.put(k, b'orphan', 'image/png', {})\n"
"await asyncio.gather(*(one(k) for k in ks))", stdin="\n".join(keys))
time.sleep(32)
result: dict = {}
def run_apply() -> None:
result["code"], result["report"] = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE)
t = threading.Thread(target=run_apply)
t.start()
# pause S3 as soon as deletions are under way (the reconciler is planning/listing first)
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
left = len(s3_keys() & set(keys))
if left < count:
break
time.sleep(0.2)
subprocess.run(["docker", "compose", "pause", "s3"], cwd=ROOT, check=True, capture_output=True)
t.join()
subprocess.run(["docker", "compose", "unpause", "s3"], cwd=ROOT, check=True, capture_output=True)
time.sleep(5) # let S3 settle after unpause before counting
left_after_interrupt = len(s3_keys() & set(keys))
code2, second = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE)
code3, third = reconcile("--apply", *GRACE)
save(out, "interrupted-1-apply.json", result["report"])
save(out, "interrupted-2-rerun.json", second)
save(out, "interrupted-3-third.json", third)
summary = {
"orphans_injected": count,
"interrupted_run": {"exit_code": result["code"], "error": result["report"].get("error"),
"applied": result["report"]["actions"]["applied"]},
"orphans_left_after_interrupt": left_after_interrupt,
"rerun": {"exit_code": code2, "applied": second["actions"]["applied"], "error": second["error"]},
"third": {"exit_code": code3, "planned": third["actions"]["planned"]},
"orphans_left_at_end": len(s3_keys() & set(keys)),
}
print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True)
return summary
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--output", default="evidence/reconcile")
args = p.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
a = scenario_a(out)
b = scenario_b(out)
commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip()
save(out, "faults-summary.json", {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "commit": commit,
"grace_seconds": 30, "scenario_a": a, "scenario_b_interrupted": b})
return 0
if __name__ == "__main__":
sys.exit(main())
+260
View File
@@ -0,0 +1,260 @@
-- 0005 (lab 5): asynchronous processing of attachments. Details: docs/PIPELINE.md.
--
-- Upload (API): INSERT attachment 'pending' (intent: the staging key is known before the PUT)
-- -> PUT original into the PRIVATE STAGING bucket
-- -> ONE transaction: attachment 'queued' + jobs row + outbox row -> 202.
-- Relay: outbox row -> publish to RabbitMQ with publisher confirm -> outbox.published_at.
-- Worker: claim job (lease + attempt token) -> variants into the final bucket under keys of
-- THIS attempt -> ONE transaction: variants + attachment 'ready' (fenced by the
-- lease token, never downgrading published_version) -> ack -> staging deleted.
--
-- PostgreSQL is the source of truth for the work: state, attempts, lease, backoff (available_at)
-- and the failed registry. The broker only delivers job ids (at least once).
-- ------------------------------------------------------------------ attachments
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
ALTER TABLE attachments ADD CONSTRAINT attachments_status
CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed'));
-- object_key = key of the ORIGINAL in the final bucket. NULL until the worker publishes it.
-- Keys of lab 4 ('o/<hex>') stay valid; lab 5 writes 'a/<attachment>/<attempt token>/original'.
ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL;
ALTER TABLE attachments DROP CONSTRAINT attachments_object_key_format;
ALTER TABLE attachments ADD CONSTRAINT attachments_object_key_format
CHECK (object_key ~ '^(o/[0-9a-f]{32}|a/[0-9a-f-]{36}/[0-9a-f]{32}/original)$');
ALTER TABLE attachments
-- Key of the original in the staging bucket ('s/<128 random bits>'), written by the API.
ADD COLUMN staging_key text COLLATE "C",
-- none (lab 4 row) | present | deleting (marked, S3 DELETE pending) | deleted
ADD COLUMN staging_state text NOT NULL DEFAULT 'none',
-- pipeline version whose variants are published (GET shows them); only ever grows.
ADD COLUMN published_version integer,
ADD COLUMN error_code text,
ADD COLUMN failed_at timestamptz,
ADD CONSTRAINT attachments_staging_key_key UNIQUE (staging_key),
ADD CONSTRAINT attachments_staging_key_format CHECK (staging_key ~ '^s/[0-9a-f]{32}$'),
ADD CONSTRAINT attachments_staging_state CHECK (staging_state IN ('none', 'present', 'deleting', 'deleted')),
ADD CONSTRAINT attachments_staging_has_key CHECK (staging_state = 'none' OR staging_key IS NOT NULL),
ADD CONSTRAINT attachments_published_version CHECK (published_version >= 1),
ADD CONSTRAINT attachments_error_code CHECK (error_code ~ '^[a-z_]{1,64}$');
-- Lab 4 rows: a 'pending' intent of lab 4 never got its object confirmed and has no staging copy;
-- it becomes a failed upload (invisible: its id was never returned). A lab 4 'ready' row has its
-- original in the final bucket but no thumbnail yet: it is queued for pipeline 1 (below).
UPDATE attachments SET status = 'failed', error_code = 'upload_incomplete', failed_at = clock_timestamp()
WHERE status = 'pending';
UPDATE attachments SET status = 'queued' WHERE status = 'ready';
-- Structural guarantees (the database refuses a buggy cleanup or worker):
-- * work that is queued/processing always has a source: the published original or the staging
-- copy - the cleanup can never mark the staging of an accepted upload as deleting;
-- * ready = published original + published pipeline version; failed = error code.
ALTER TABLE attachments
ADD CONSTRAINT attachments_pending_staging CHECK (status <> 'pending' OR staging_state = 'present'),
ADD CONSTRAINT attachments_active_has_source
CHECK (status NOT IN ('queued', 'processing') OR object_key IS NOT NULL OR staging_state = 'present'),
ADD CONSTRAINT attachments_ready_published
CHECK (status <> 'ready' OR (object_key IS NOT NULL AND published_version IS NOT NULL)),
ADD CONSTRAINT attachments_failed_code
CHECK ((status = 'failed') = (error_code IS NOT NULL AND failed_at IS NOT NULL));
-- Janitor of the API: the lab 4 partial index on pending rows stays as is.
-- Cleanup of the worker: staging copies that may be deleted (ready: at once; failed: after retention).
CREATE INDEX attachments_staging_cleanup ON attachments (updated_at)
WHERE staging_state IN ('present', 'deleting') AND status IN ('ready', 'failed');
-- ------------------------------------------------------------------ jobs
-- One row = one logical piece of work: "build pipeline version V for attachment A".
-- job_id never changes on redelivery; `attempt` counts claims by workers (managed attempts).
CREATE TABLE jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments (id),
pipeline_version integer NOT NULL,
kind text NOT NULL, -- upload | replay | reprocess | migrate
replay_of uuid REFERENCES jobs (id),
state text NOT NULL DEFAULT 'queued',
attempt integer NOT NULL DEFAULT 0,
max_attempts integer NOT NULL,
available_at timestamptz NOT NULL DEFAULT clock_timestamp(), -- backoff: not before
lease_token uuid, -- token of the attempt that holds the job
lease_owner text,
lease_until timestamptz,
error_code text,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
finished_at timestamptz,
CONSTRAINT jobs_pipeline_version CHECK (pipeline_version >= 1),
CONSTRAINT jobs_kind CHECK (kind IN ('upload', 'replay', 'reprocess', 'migrate')),
CONSTRAINT jobs_replay CHECK ((kind = 'replay') = (replay_of IS NOT NULL)),
CONSTRAINT jobs_state CHECK (state IN ('queued', 'processing', 'succeeded', 'failed', 'superseded')),
CONSTRAINT jobs_attempts CHECK (attempt >= 0 AND max_attempts BETWEEN 1 AND 100 AND attempt <= max_attempts),
-- A lease exists exactly while the job is processing.
CONSTRAINT jobs_lease CHECK ((state = 'processing') = (lease_token IS NOT NULL AND lease_until IS NOT NULL)),
CONSTRAINT jobs_finished CHECK ((state IN ('succeeded', 'failed', 'superseded')) = (finished_at IS NOT NULL)),
CONSTRAINT jobs_failed_code CHECK (state <> 'failed' OR error_code IS NOT NULL),
CONSTRAINT jobs_error_code CHECK (error_code ~ '^[a-z_]{1,64}$')
);
-- At most ONE active job per effect set (attachment, pipeline version): a second upload job,
-- replay or reprocess of the same version cannot exist while one is queued/processing.
CREATE UNIQUE INDEX jobs_one_active ON jobs (attachment_id, pipeline_version) WHERE state IN ('queued', 'processing');
CREATE INDEX jobs_attachment ON jobs (attachment_id, created_at);
CREATE INDEX jobs_lease_expiry ON jobs (lease_until) WHERE state = 'processing'; -- reaper
CREATE INDEX jobs_queued ON jobs (updated_at) WHERE state = 'queued'; -- lost-message guard
CREATE INDEX jobs_failed ON jobs (finished_at) WHERE state = 'failed'; -- failed registry
-- One row per claim of a job by a worker. The token is part of every object key the attempt
-- writes, so the keys of an attempt are known without listing the bucket (planned_keys), and an
-- attempt that is no longer the lease holder can never publish (fencing by token).
CREATE TABLE job_attempts (
token uuid PRIMARY KEY,
job_id uuid NOT NULL REFERENCES jobs (id),
attempt integer NOT NULL,
worker text NOT NULL,
source text NOT NULL, -- staging | original
planned_keys text[] NOT NULL,
outcome text NOT NULL DEFAULT 'running',
error_code text,
started_at timestamptz NOT NULL DEFAULT clock_timestamp(),
finished_at timestamptz,
objects_cleaned_at timestamptz,
CONSTRAINT job_attempts_number UNIQUE (job_id, attempt),
CONSTRAINT job_attempts_source CHECK (source IN ('staging', 'original')),
CONSTRAINT job_attempts_outcome
CHECK (outcome IN ('running', 'succeeded', 'retry', 'failed', 'abandoned', 'superseded')),
CONSTRAINT job_attempts_finished CHECK ((outcome = 'running') = (finished_at IS NULL)),
-- Objects of a successful attempt are published: they are never cleaned.
CONSTRAINT job_attempts_cleaned CHECK (objects_cleaned_at IS NULL OR outcome NOT IN ('running', 'succeeded'))
);
CREATE INDEX job_attempts_cleanup ON job_attempts (finished_at)
WHERE outcome IN ('retry', 'failed', 'abandoned', 'superseded') AND objects_cleaned_at IS NULL;
-- Published variants. The primary key is the idempotency key of the EFFECT from the contract:
-- (attachment_id, pipeline_version, variant) exists at most once, whatever the deliveries.
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments (id),
pipeline_version integer NOT NULL,
name text NOT NULL,
object_key text COLLATE "C" NOT NULL,
content_type text NOT NULL,
size_bytes integer NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
sha256 bytea NOT NULL,
attempt_token uuid NOT NULL REFERENCES job_attempts (token),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (attachment_id, pipeline_version, name),
CONSTRAINT attachment_variants_key UNIQUE (object_key),
CONSTRAINT attachment_variants_key_format CHECK (object_key ~ '^a/[0-9a-f-]{36}/[0-9a-f]{32}/(thumbnail|crop|optimized)$'),
CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')),
CONSTRAINT attachment_variants_type CHECK (content_type IN ('image/jpeg', 'image/webp')),
CONSTRAINT attachment_variants_size CHECK (size_bytes >= 1 AND width >= 1 AND height >= 1),
CONSTRAINT attachment_variants_sha256 CHECK (octet_length(sha256) = 32)
);
-- Transactional outbox: written in the SAME transaction as the job (upload, retry, replay,
-- reprocess, lease expiry). The relay publishes due rows and marks them; a crash after the
-- publish but before the mark publishes again (a duplicate the worker ignores).
CREATE TABLE outbox (
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
job_id uuid NOT NULL REFERENCES jobs (id),
attempt integer NOT NULL, -- the attempt this delivery is meant for
available_at timestamptz NOT NULL DEFAULT clock_timestamp(),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
published_at timestamptz,
publish_failures integer NOT NULL DEFAULT 0,
CONSTRAINT outbox_attempt CHECK (attempt >= 1)
);
CREATE INDEX outbox_due ON outbox (available_at, id) WHERE published_at IS NULL;
CREATE INDEX outbox_job ON outbox (job_id);
CREATE INDEX outbox_published ON outbox (published_at) WHERE published_at IS NOT NULL; -- purge
-- Audit of every transition (no secrets, no file names): replay/reprocess requests with the
-- operator and reason, claims, retries with their delay, lease expiries, results.
CREATE TABLE job_events (
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
job_id uuid NOT NULL REFERENCES jobs (id),
at timestamptz NOT NULL DEFAULT clock_timestamp(),
event text NOT NULL,
attempt integer,
actor text NOT NULL,
error_code text,
detail jsonb NOT NULL DEFAULT '{}'
);
CREATE INDEX job_events_job ON job_events (job_id, id);
-- Failed registry for the operator (jobctl list-failed).
CREATE VIEW failed_jobs AS
SELECT j.id AS job_id, j.attachment_id, j.pipeline_version, j.kind, j.attempt, j.max_attempts, j.error_code,
j.finished_at, a.status AS attachment_status, a.staging_state
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
WHERE j.state = 'failed';
-- Lab 4 'ready' rows (now queued): a job for pipeline 1 from their published original.
WITH queued AS (
SELECT a.id, gen_random_uuid() AS job_id FROM attachments a WHERE a.status = 'queued'
), created AS (
INSERT INTO jobs (id, attachment_id, pipeline_version, kind, max_attempts)
SELECT q.job_id, q.id, 1, 'migrate', 5 FROM queued q RETURNING id
)
INSERT INTO outbox (job_id, attempt) SELECT c.id, 1 FROM created c;
-- post_message_v3 = post_message_v2 (0004) with the lab 5 attachment states: queued, processing
-- and ready attachments of this chat may be attached; failed (or quarantined) -> 409; pending,
-- foreign or unknown -> 404. Everything else (key claim, chat clock, one transaction) unchanged.
CREATE FUNCTION post_message_v3(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid,
p_text bytea, p_ttl_seconds integer, p_attachments uuid[])
RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz, o_attachments uuid[])
LANGUAGE plpgsql VOLATILE AS $$
DECLARE
v_created timestamptz;
v_found integer;
v_usable integer;
v_check text := 'ok';
BEGIN
PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user;
IF NOT FOUND THEN
RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
RETURN;
END IF;
IF cardinality(p_attachments) > 0 THEN
-- FOR SHARE: a concurrent transition to failed/quarantine waits until this message commits.
SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL AND a.status <> 'failed') INTO v_found, v_usable
FROM (SELECT a.lost_at, a.status FROM attachments a
WHERE a.id = ANY (p_attachments) AND a.chat_id = p_chat AND a.status <> 'pending'
FOR SHARE) a;
IF v_found <> cardinality(p_attachments) THEN
v_check := 'attachment_not_found';
ELSIF v_usable <> v_found THEN
v_check := 'attachment_conflict';
END IF;
END IF;
IF v_check = 'ok' THEN
INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id)
VALUES (p_user, p_chat, p_key, p_hash, p_message)
ON CONFLICT (user_id, chat_id, key) DO UPDATE
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp()
WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds);
IF FOUND THEN
UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond')
WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created;
INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
VALUES (p_message, p_chat, p_user, p_text, v_created, p_attachments);
INSERT INTO message_attachments (message_id, position, attachment_id)
SELECT p_message, (a.ord - 1)::smallint, a.id
FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord);
RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created, p_attachments;
RETURN;
END IF;
END IF;
RETURN QUERY
SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at,
m.attachment_ids
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key;
IF NOT FOUND THEN
RETURN QUERY SELECT CASE WHEN v_check = 'ok' THEN 'retry' ELSE v_check END,
NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
END IF;
END
$$;
+5 -1
View File
@@ -16,5 +16,9 @@ redis==8.1.0
argon2-cffi==25.1.0
# Lab 4: S3 client (async botocore over aiohttp; requests are cancellable, unlike libpq cancel).
aiobotocore==3.9.2
# Images are checked without Pillow: PNG by src/messenger/images.py, JPEG by libjpeg-turbo
# Lab 5: AMQP 0-9-1 client for RabbitMQ (publisher confirms, consumer acks, prefetch); last 9.x line.
aio-pika==9.6.2
# Lab 5: image variants in the worker only (thumbnail/crop JPEG, WebP). Same version as the tests.
pillow==12.3.0
# Uploads are still checked without Pillow: PNG by src/messenger/images.py, JPEG by libjpeg-turbo
# `djpeg -strict` (Debian package libjpeg-turbo-progs, see Dockerfile).
+105 -1
View File
@@ -4,6 +4,10 @@
#
# pip-compile --generate-hashes --no-emit-index-url --no-index --output-file=requirements.txt --strip-extras requirements.in
#
aio-pika==9.6.2 \
--hash=sha256:2a5478af920d169795071c9c09c7542cd8cdece60438cf7804533dcbcce93b7f \
--hash=sha256:c49e9246080dc8ffa1bb0e4aca407bf3d8ad78c3ee3a93df88b68fe65d7a49b9
# via -r requirements.in
aiobotocore==3.9.2 \
--hash=sha256:363b4892423b272eb84afbb58000da0be6ccd0cf8a68b98438c957d05b841efc \
--hash=sha256:8e32238c5bd77717ab1ac90077ff51ab141d025528115ccc7cc9b50b2d176e5f
@@ -179,6 +183,10 @@ aioitertools==0.13.0 \
--hash=sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be \
--hash=sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c
# via aiobotocore
aiormq==6.9.4 \
--hash=sha256:0e7c01b662804e1cc7ace9a17794e8c1192a27fc2afa96162362a6e61ae8e8ef \
--hash=sha256:726a8586695e863fba68cf88842065ab12348c9438dcebdfc9d0bddaf6083277
# via aio-pika
aiosignal==1.4.0 \
--hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \
--hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7
@@ -794,6 +802,99 @@ multidict==6.9.1 \
# aiobotocore
# aiohttp
# yarl
pamqp==3.3.0 \
--hash=sha256:40b8795bd4efcf2b0f8821c1de83d12ca16d5760f4507836267fd7a02b06763b \
--hash=sha256:c901a684794157ae39b52cbf700db8c9aae7a470f13528b9d7b4e5f7202f8eb0
# via aiormq
pillow==12.3.0 \
--hash=sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756 \
--hash=sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a \
--hash=sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59 \
--hash=sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45 \
--hash=sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3 \
--hash=sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df \
--hash=sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139 \
--hash=sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b \
--hash=sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39 \
--hash=sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e \
--hash=sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8 \
--hash=sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1 \
--hash=sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8 \
--hash=sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89 \
--hash=sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5 \
--hash=sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130 \
--hash=sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd \
--hash=sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d \
--hash=sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b \
--hash=sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed \
--hash=sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace \
--hash=sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb \
--hash=sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931 \
--hash=sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510 \
--hash=sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6 \
--hash=sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1 \
--hash=sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce \
--hash=sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385 \
--hash=sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e \
--hash=sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c \
--hash=sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7 \
--hash=sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace \
--hash=sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c \
--hash=sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f \
--hash=sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64 \
--hash=sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f \
--hash=sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a \
--hash=sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827 \
--hash=sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17 \
--hash=sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4 \
--hash=sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a \
--hash=sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701 \
--hash=sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e \
--hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 \
--hash=sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66 \
--hash=sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468 \
--hash=sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217 \
--hash=sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658 \
--hash=sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418 \
--hash=sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a \
--hash=sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c \
--hash=sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330 \
--hash=sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402 \
--hash=sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09 \
--hash=sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930 \
--hash=sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f \
--hash=sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec \
--hash=sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a \
--hash=sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94 \
--hash=sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468 \
--hash=sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b \
--hash=sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965 \
--hash=sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8 \
--hash=sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd \
--hash=sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7 \
--hash=sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c \
--hash=sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777 \
--hash=sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35 \
--hash=sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9 \
--hash=sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f \
--hash=sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f \
--hash=sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0 \
--hash=sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c \
--hash=sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71 \
--hash=sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3 \
--hash=sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838 \
--hash=sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf \
--hash=sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321 \
--hash=sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26 \
--hash=sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec \
--hash=sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9 \
--hash=sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65 \
--hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \
--hash=sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e \
--hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \
--hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 \
--hash=sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7
# via -r requirements.in
propcache==0.5.4 \
--hash=sha256:004e685b315646c410771836e72a44f143bbe624f29653a42687815069a303d5 \
--hash=sha256:02c0a34f16889cf800f10f0247a564d8ce6eeab6ffcd7c87198f769067eb8432 \
@@ -1353,4 +1454,7 @@ yarl==1.25.1 \
--hash=sha256:f53dcd26694f148f738edc052b5a69234833e739f10f4c3287bdfd8ec0f7b326 \
--hash=sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc \
--hash=sha256:fe01645169a2112aa1d4ebc3e4c5f029c5c8f97adfc32e5d37c993b39a994d75
# via aiohttp
# via
# aio-pika
# aiohttp
# aiormq
+86
View File
@@ -0,0 +1,86 @@
"""RabbitMQ (AMQP 0-9-1) for the attachment jobs: topology names and the message format.
Topology (declared by the broker itself from its definitions file at boot, infra/rabbitmq and
scripts/init_secrets.py; the relay and the worker have NO configure permission and declare nothing):
exchange messenger.jobs direct, durable
queue messenger.attachment-jobs quorum (Raft log on disk), bound with routing key
attachment.process.v1; x-delivery-limit and dead-lettering to
exchange messenger.jobs.dlx fanout -> queue messenger.attachment-jobs.dlq (quorum)
Publisher side (relay): persistent message, `mandatory` (an unroutable message is returned, not
silently dropped), channel in confirm mode - `publish()` returns only after the broker has
written the message to the quorum log (publisher confirm). Consumer side (worker): manual ack
after the durable effect, prefetch = WORKER_CONCURRENCY.
The message carries IDs and service fields only (contracts/jobs.schema.json): no bytes, no URL,
no S3 key, no filename, no credentials. The worker takes every trusted detail from PostgreSQL.
"""
from __future__ import annotations
import datetime as dt
import json
import re
import uuid
EXCHANGE = "messenger.jobs"
QUEUE = "messenger.attachment-jobs"
DLX = "messenger.jobs.dlx"
DLQ = "messenger.attachment-jobs.dlq"
ROUTING_KEY = "attachment.process.v1"
JOB_TYPE = "attachment.process.v1"
MAX_MESSAGE_BYTES = 4096
REQUIRED = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
OPTIONAL = {"traceparent", "tracestate"}
TRACEPARENT_RE = re.compile(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}")
DATETIME_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?Z")
class BadMessage(ValueError):
"""Not a valid job message (schema v1): dead-lettered, never processed."""
def rfc3339(value: dt.datetime) -> str:
return value.astimezone(dt.timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def encode(job_id: str, attachment_id: str, pipeline_version: int, attempt: int, created_at: dt.datetime) -> bytes:
body = {"schema_version": 1, "job_id": job_id, "type": JOB_TYPE, "attachment_id": attachment_id,
"pipeline_version": pipeline_version, "attempt": attempt, "created_at": rfc3339(created_at)}
return json.dumps(body, separators=(",", ":")).encode()
def decode(raw: bytes) -> dict:
"""Strict check against contracts/jobs.schema.json (additionalProperties: false)."""
if len(raw) > MAX_MESSAGE_BYTES:
raise BadMessage("too large")
try:
body = json.loads(raw)
except (ValueError, UnicodeError):
raise BadMessage("not JSON") from None
if not isinstance(body, dict):
raise BadMessage("not an object")
keys = set(body)
if not REQUIRED <= keys or keys - REQUIRED - OPTIONAL:
raise BadMessage("wrong fields")
if body["schema_version"] != 1 or isinstance(body["schema_version"], bool) or body["type"] != JOB_TYPE:
raise BadMessage("unsupported schema/type")
for name in ("job_id", "attachment_id"):
if not isinstance(body[name], str):
raise BadMessage(name)
try:
uuid.UUID(body[name])
except ValueError:
raise BadMessage(name) from None
for name in ("pipeline_version", "attempt"):
value = body[name]
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
raise BadMessage(name)
if not isinstance(body["created_at"], str) or not DATETIME_RE.fullmatch(body["created_at"]):
raise BadMessage("created_at")
if "traceparent" in body and not (isinstance(body["traceparent"], str) and TRACEPARENT_RE.fullmatch(body["traceparent"])):
raise BadMessage("traceparent")
if "tracestate" in body and not (isinstance(body["tracestate"], str) and len(body["tracestate"]) <= 512):
raise BadMessage("tracestate")
return body
+119 -12
View File
@@ -10,7 +10,7 @@ from __future__ import annotations
import os
import re
import socket
from dataclasses import dataclass
from dataclasses import dataclass, field
from pathlib import Path
@@ -134,7 +134,29 @@ class Settings:
jpeg_max_memory_mb: int = 256
pending_grace_seconds: int = 900
janitor_interval_seconds: float = 60.0
# Lab 5: staging bucket (API writes, worker reads) and the asynchronous pipeline (jobs_settings()).
s3_staging_bucket: str = "messenger-staging"
broker_url: str = ""
pipeline_version: int = 1
job_max_attempts: int = 5
job_timeout_seconds: float = 60.0
job_lease_seconds: float = 30.0
job_retry_base_seconds: float = 2.0
job_retry_max_seconds: float = 60.0
worker_concurrency: int = 2
worker_shutdown_seconds: float = 20.0
image_max_memory_mb: int = 768
staging_retention_seconds: int = 86400
orphan_grace_seconds: int = 600
cleanup_interval_seconds: float = 30.0
relay_poll_seconds: float = 0.5
relay_batch: int = 50
queued_republish_seconds: float = 300.0
fault_points: frozenset[str] = frozenset()
fault_values: dict[str, int] = field(default_factory=dict)
def fault_value(self, point: str, default: int = 0) -> int:
return self.fault_values.get(point, default)
@property
def db_statement_timeout_ms(self) -> int:
@@ -142,7 +164,8 @@ class Settings:
return max(100, int(self.db_timeout_seconds * 1000) - 500)
@classmethod
def from_env(cls) -> "Settings":
def from_env(cls, *, s3_credentials: bool = True) -> "Settings":
"""s3_credentials=False: processes without S3 access (relay, jobctl)."""
log_level = os.environ.get("LOG_LEVEL", "info").strip().lower() or "info"
if log_level not in {"debug", "info", "warning", "error"}:
raise ConfigError(f"LOG_LEVEL must be debug/info/warning/error, got {log_level!r}")
@@ -185,32 +208,71 @@ class Settings:
# Track 1: `function` = one SQL call per send (migration 0003), `multi` = the previous
# multi-statement transaction; both have identical semantics (A/B on one image).
message_write_path=_choice("MESSAGE_WRITE_PATH", "function", {"function", "multi"}),
**s3_settings(),
**s3_settings(credentials_required=s3_credentials),
**jobs_settings(broker_required=False),
)
# Contract limits (contracts/README.md): configurable DOWN for experiments, never above them.
CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024
CONTRACT_MAX_IMAGE_PIXELS = 40_000_000
# Failure points for the level 4 experiments only (compose.faults.yaml); empty in normal runs.
FAULT_POINTS = {"after_s3_put", "after_ready_commit"}
# Failure points for the experiments only (FAULT_POINTS of ONE service, compose variables
# API_/RELAY_/WORKER_FAULT_POINTS); empty in normal runs. `name` or `name=<integer>`.
FAULT_POINTS = {
# API, upload: between the staging PUT and the job COMMIT / after the job COMMIT, before 202.
"after_s3_put", "after_enqueue_commit",
# Relay: crash after the broker confirmed a publish but before outbox.published_at is
# committed; publish every message twice; publish a batch in reverse order.
"crash_after_publish", "duplicate_publish", "reverse_order",
# Worker: crash after the claim (before any side effect), after the S3 writes (before the
# result COMMIT), after the result COMMIT (before the ack); transient failure of the first N
# attempts; permanent failure; slow processing (N s, lease kept alive) of every job or only of
# pipeline version `slow_pipeline`; hang: stop the heartbeat and sleep N s inside processing.
"crash_after_claim", "crash_after_s3_write", "crash_after_commit", "transient_error", "permanent_error",
"slow", "slow_pipeline", "hang",
# Worker cleanup: pause N s after reading the candidates, before locking/deleting them.
"pause_cleanup",
}
def _faults() -> tuple[frozenset[str], dict[str, int]]:
names, values = set(), {}
for item in os.environ.get("FAULT_POINTS", "").split(","):
name, _, raw = item.strip().partition("=")
if not name:
continue
if name not in FAULT_POINTS:
raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}")
if raw:
if not raw.isdigit():
raise ConfigError(f"FAULT_POINTS value of {name} must be an integer")
values[name] = int(raw)
names.add(name)
return frozenset(names), values
def _bucket(name: str, default: str) -> str:
bucket = os.environ.get(name, default).strip() or default
if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket):
raise ConfigError(f"{name} is not a valid bucket name: {bucket!r}")
return bucket
def s3_settings(*, credentials_required: bool = True) -> dict:
"""S3 part of Settings; also used by the reconciler/bootstrap CLIs with their own keys."""
faults = frozenset(f.strip() for f in os.environ.get("FAULT_POINTS", "").split(",") if f.strip())
if faults - FAULT_POINTS:
raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}")
"""S3 part of Settings; also used by the bootstrap CLI with its own keys."""
faults, fault_values = _faults()
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").strip() or "http://s3:8333"
if not re.fullmatch(r"https?://[A-Za-z0-9.\-]+(:[0-9]{1,5})?/?", endpoint):
raise ConfigError(f"S3_ENDPOINT must be http(s)://host[:port], got {endpoint!r}")
bucket = os.environ.get("S3_BUCKET", "messenger-attachments").strip() or "messenger-attachments"
if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket):
raise ConfigError(f"S3_BUCKET is not a valid bucket name: {bucket!r}")
bucket = _bucket("S3_BUCKET", "messenger-attachments")
staging = _bucket("S3_STAGING_BUCKET", "messenger-staging")
if staging == bucket:
raise ConfigError("S3_STAGING_BUCKET must differ from S3_BUCKET")
return dict(
s3_endpoint=endpoint,
s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1",
s3_bucket=bucket,
s3_staging_bucket=staging,
s3_access_key_id=_secret("S3_ACCESS_KEY_ID", required=credentials_required),
s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY", required=credentials_required),
s3_timeout_seconds=_float("S3_TIMEOUT_SECONDS", 3.0, 0.2, 30.0),
@@ -233,4 +295,49 @@ def s3_settings(*, credentials_required: bool = True) -> dict:
pending_grace_seconds=_int("PENDING_GRACE_SECONDS", 900, 1, 7 * 24 * 3600),
janitor_interval_seconds=_float("JANITOR_INTERVAL_SECONDS", 60.0, 0.0, 3600.0),
fault_points=faults,
fault_values=fault_values,
)
def jobs_settings(*, broker_required: bool) -> dict:
"""Lab 5 pipeline: retries, lease, worker limits, cleanup, relay (docs/PIPELINE.md)."""
from .pipeline import PIPELINES # local import: pipeline.py imports nothing from here
timeout = _float("JOB_TIMEOUT_SECONDS", 60.0, 1.0, 600.0)
lease = _float("JOB_LEASE_SECONDS", 30.0, 2.0, 600.0)
base = _float("JOB_RETRY_BASE_SECONDS", 2.0, 0.0, 600.0)
cap = _float("JOB_RETRY_MAX_SECONDS", 60.0, 0.0, 3600.0)
if base > cap:
raise ConfigError("JOB_RETRY_BASE_SECONDS must not exceed JOB_RETRY_MAX_SECONDS")
orphan_grace = _int("ORPHAN_GRACE_SECONDS", 600, 1, 7 * 24 * 3600)
version = _int("PIPELINE_VERSION", 1, 1, 1000)
if version not in PIPELINES:
raise ConfigError(f"PIPELINE_VERSION must be one of {sorted(PIPELINES)}")
return dict(
broker_url=_secret("BROKER_URL", required=broker_required),
# Pipeline version of the jobs created by new uploads (track 2: versioned pipeline).
pipeline_version=version,
# Managed attempts (claims) per job before failed/retry_exhausted.
job_max_attempts=_int("JOB_MAX_ATTEMPTS", 5, 1, 100),
# Hard limit of ONE attempt: the image child process is killed, the lease is no longer renewed.
job_timeout_seconds=timeout,
# Lease of a claimed job, renewed by the heartbeat every lease/3 while the attempt runs.
job_lease_seconds=lease,
# Backoff of transient failures: full jitter, uniform(0, min(max, base * 2^(attempt-1))).
job_retry_base_seconds=base,
job_retry_max_seconds=cap,
# Jobs processed at once by ONE worker process = RabbitMQ prefetch (basic.qos).
worker_concurrency=_int("WORKER_CONCURRENCY", 2, 1, 32),
worker_shutdown_seconds=_float("WORKER_SHUTDOWN_SECONDS", 20.0, 0.0, 120.0),
# Address-space limit (RLIMIT_AS) of the child process that decodes/encodes ONE image.
image_max_memory_mb=_int("IMAGE_MAX_MEMORY_MB", 768, 128, 8192),
# Staging copy of a FAILED attachment is kept this long for a replay, then deleted.
staging_retention_seconds=_int("STAGING_RETENTION_SECONDS", 86400, 0, 30 * 24 * 3600),
# Objects of attempts that did not publish are deleted once the attempt ended this long ago.
orphan_grace_seconds=orphan_grace,
cleanup_interval_seconds=_float("CLEANUP_INTERVAL_SECONDS", 30.0, 0.0, 3600.0),
relay_poll_seconds=_float("RELAY_POLL_SECONDS", 0.5, 0.05, 60.0),
relay_batch=_int("RELAY_BATCH", 50, 1, 1000),
# A queued job whose message has not led to a claim for this long is published again
# (message lost in the broker, dead-lettered, purged): no accepted job waits forever.
queued_republish_seconds=_float("QUEUED_REPUBLISH_SECONDS", 300.0, 5.0, 86400.0),
)
+208
View File
@@ -0,0 +1,208 @@
"""Variants of ONE image, built in a separate child process with hard limits (lab 5 worker).
parent (worker): await render(data, content_type, version, ...) -> list[Rendered]
child: python -m messenger.imaging --type image/png --pipeline 1 ...
stdin: original bytes; stdout: one JSON header line + variant bytes
Why a child process per image:
* memory: RLIMIT_AS (IMAGE_MAX_MEMORY_MB) caps the address space of the decoder; a picture that
needs more fails with MemoryError -> `image_too_large` (permanent), the worker keeps running;
* time: the parent kills the child at JOB_TIMEOUT_SECONDS (C code of Pillow cannot be
interrupted inside a thread) -> `processing_timeout` (transient, bounded by JOB_MAX_ATTEMPTS);
* pixels: the size is read from the header and checked against MAX_IMAGE_PIXELS BEFORE the
pixels are decoded (`Image.open` is lazy).
A crash of the decoder (signal) is a transient `processing_killed`: the next attempt may run
with less memory pressure; JOB_MAX_ATTEMPTS bounds it.
Geometry and encoder settings: pipeline.py. Alpha is handled in premultiplied form (RGBa) while
resizing, so transparent pixels do not bleed their colour into the edges; for JPEG the result is
then composited onto white.
"""
from __future__ import annotations
import argparse
import asyncio
import hashlib
import io
import json
import sys
import warnings
from dataclasses import dataclass
from .pipeline import CONTENT_TYPES, CROP_SIDE, OPTIMIZED_BOX, PIPELINES, THUMBNAIL_BOX, center_square, fit
EXIT_INVALID = 3 # the bytes are not a decodable PNG/JPEG
EXIT_TOO_LARGE = 4 # pixel limit or memory limit
HEADER_LIMIT = 4096
class ProcessingError(Exception):
"""Outcome of an attempt that did not produce variants. `permanent`: another attempt with the
same input cannot succeed (job -> failed at once); otherwise retry with backoff."""
def __init__(self, code: str, permanent: bool):
super().__init__(code)
self.code = code
self.permanent = permanent
@dataclass(frozen=True, slots=True)
class Rendered:
name: str
content_type: str
width: int
height: int
data: bytes
@property
def sha256(self) -> bytes:
return hashlib.sha256(self.data).digest()
# ------------------------------------------------------------------ parent side
async def render(data: bytes, content_type: str, pipeline_version: int, *, max_pixels: int, timeout: float,
memory_mb: int) -> list[Rendered]:
process = await asyncio.create_subprocess_exec(
sys.executable, "-m", "messenger.imaging", "--type", content_type, "--pipeline", str(pipeline_version),
"--max-pixels", str(max_pixels), "--memory-mb", str(memory_mb),
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
try:
out, _ = await asyncio.wait_for(process.communicate(data), timeout)
except asyncio.TimeoutError:
process.kill()
await process.wait()
raise ProcessingError("processing_timeout", permanent=False) from None
except asyncio.CancelledError:
process.kill()
await process.wait()
raise
code = process.returncode
if code == EXIT_INVALID:
raise ProcessingError("invalid_image", permanent=True)
if code == EXIT_TOO_LARGE:
raise ProcessingError("image_too_large", permanent=True)
if code is None or code < 0:
raise ProcessingError("processing_killed", permanent=False) # OOM killer or another signal
if code != 0:
raise ProcessingError("processing_failed", permanent=False)
return parse_output(out)
def parse_output(out: bytes) -> list[Rendered]:
newline = out.find(b"\n", 0, HEADER_LIMIT)
if newline < 0:
raise ProcessingError("processing_failed", permanent=False)
header = json.loads(out[:newline])
pos = newline + 1
result = []
for item in header["variants"]:
size = int(item["size"])
result.append(Rendered(item["name"], item["content_type"], int(item["width"]), int(item["height"]),
out[pos:pos + size]))
pos += size
if pos != len(out) or [r.name for r in result] != list(CONTENT_TYPES):
raise ProcessingError("processing_failed", permanent=False)
return result
# ------------------------------------------------------------------ child side
def build(data: bytes, content_type: str, pipeline_version: int, max_pixels: int) -> list[Rendered]:
"""Decode once, build every variant. Raises the Pillow/OS errors of a bad file."""
from PIL import Image, ImageOps
pipeline = PIPELINES[pipeline_version]
warnings.simplefilter("ignore", Image.DecompressionBombWarning) # our own pixel limit below
Image.MAX_IMAGE_PIXELS = max_pixels
expected = {"image/png": "PNG", "image/jpeg": "JPEG"}[content_type]
img = Image.open(io.BytesIO(data), formats=[expected]) # header only: nothing decoded yet
width, height = img.size
if width * height > max_pixels:
raise TooLarge()
if expected == "JPEG":
# DCT scaling: decode a big JPEG directly at 1/2..1/8 of its size, never below what the
# largest variant needs (the box stays square, so EXIF rotation does not matter).
img.draft("RGB", (OPTIMIZED_BOX, OPTIMIZED_BOX))
img.load()
img = ImageOps.exif_transpose(img) # orientation goes into the pixels, the tag is dropped
img = _normalize(img)
alpha = img.mode == "RGBA"
base = img.convert("RGBa") if alpha else img # premultiplied alpha for resampling
del img
def finish(im, *, flatten: bool):
if alpha:
im = im.convert("RGBA")
if flatten:
white = Image.new("RGB", im.size, (255, 255, 255))
white.paste(im, mask=im.getchannel("A"))
im = white
im.info = {} # no EXIF/XMP/ICC/comments of the source in a variant
return im
def resized(im, size):
return im if im.size == size else im.resize(size, Image.Resampling.LANCZOS, reducing_gap=3.0)
thumbnail = finish(resized(base, fit(*base.size, THUMBNAIL_BOX)), flatten=True)
crop = finish(resized(base.crop(center_square(*base.size)), (CROP_SIDE, CROP_SIDE)), flatten=True)
optimized = finish(resized(base, fit(*base.size, OPTIMIZED_BOX)), flatten=False)
out = []
for name, im in (("thumbnail", thumbnail), ("crop", crop), ("optimized", optimized)):
buf = io.BytesIO()
if CONTENT_TYPES[name] == "image/jpeg":
im.save(buf, "JPEG", quality=pipeline.jpeg_quality, optimize=True, subsampling=2, progressive=False)
else:
im.save(buf, "WEBP", quality=pipeline.webp_quality, method=pipeline.webp_method, exif=b"",
icc_profile="", xmp=b"")
out.append(Rendered(name, CONTENT_TYPES[name], im.width, im.height, buf.getvalue()))
return out
class TooLarge(Exception):
pass
def _normalize(img):
"""Any PNG/JPEG mode -> RGB, or RGBA when the image has transparency."""
if img.mode.startswith("I;16") or img.mode == "I":
# 16-bit grey: scale 0..65535 to 0..255 (a plain convert("L") would clip to white).
img = img.convert("I").point(lambda v: v * (1 / 257)).convert("L")
has_alpha = img.mode in ("RGBA", "LA", "PA", "La", "RGBa") or "transparency" in img.info
return img.convert("RGBA" if has_alpha else "RGB") if img.mode != ("RGBA" if has_alpha else "RGB") else img
def child_main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--type", required=True, choices=["image/png", "image/jpeg"])
parser.add_argument("--pipeline", type=int, required=True, choices=sorted(PIPELINES))
parser.add_argument("--max-pixels", type=int, required=True)
parser.add_argument("--memory-mb", type=int, required=True)
args = parser.parse_args(argv)
try:
import resource
limit = args.memory_mb * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
except (ImportError, ValueError, OSError):
pass # not Linux (unit tests on Windows): the container memory limit still applies
data = sys.stdin.buffer.read()
try:
variants = build(data, args.type, args.pipeline, args.max_pixels)
except (TooLarge, MemoryError):
return EXIT_TOO_LARGE
except Exception as exc: # noqa: BLE001 - every decoder error of the CONTENT is "invalid image"
from PIL import Image
if isinstance(exc, Image.DecompressionBombError):
return EXIT_TOO_LARGE
if isinstance(exc, (OSError, SyntaxError, ValueError, EOFError, KeyError, IndexError, TypeError)):
return EXIT_INVALID
raise
header = {"variants": [{"name": v.name, "content_type": v.content_type, "width": v.width, "height": v.height,
"size": len(v.data)} for v in variants]}
sys.stdout.buffer.write(json.dumps(header).encode() + b"\n")
for v in variants:
sys.stdout.buffer.write(v.data)
sys.stdout.buffer.flush()
return 0
if __name__ == "__main__":
sys.exit(child_main())
+398
View File
@@ -0,0 +1,398 @@
"""State machine of attachment jobs in PostgreSQL (lab 5). Every transition is ONE transaction.
Lock order, everywhere both rows are locked: the jobs row FIRST, then the attachments row
(claim, complete, retry, lease expiry). Replay/reprocess/cleanup lock only the attachments row
and insert NEW job rows. So two transitions never wait on each other in opposite order.
Fencing: a claim gives the attempt a fresh random `lease_token`. Every later write of that attempt
(heartbeat, result, retry) is conditional on `jobs.lease_token = <its token>` under the jobs row
lock. When the lease expires, the reaper clears the token; from then on the old attempt - even if
its process is still alive (paused, slow) - can neither publish nor schedule anything, and the
objects it wrote under its own keys are garbage for the cleanup.
Monotonic publication: a result of pipeline version V is published only if V is greater than the
published version of the attachment; otherwise the job ends as `superseded` and nothing visible
changes. `ready` never goes back to queued/processing.
"""
from __future__ import annotations
import datetime as dt
import json
import random
import uuid
from dataclasses import dataclass
import psycopg
from .pipeline import VARIANT_NAMES
from .storage import attempt_key
ACTIVE = ("queued", "processing")
def backoff_delay(attempt: int, base: float, cap: float, rnd=random.random) -> float:
"""Full jitter (AWS Architecture Blog, "Exponential Backoff And Jitter"): uniform in
[0, min(cap, base * 2^(attempt-1))]. Spreads the retries of many jobs that failed together."""
return rnd() * min(cap, base * (2 ** max(0, attempt - 1)))
async def event(conn: psycopg.AsyncConnection, job_id, name: str, actor: str, *, attempt: int | None = None,
error_code: str | None = None, **detail) -> None:
await conn.execute(
"INSERT INTO job_events (job_id, event, attempt, actor, error_code, detail) VALUES (%s, %s, %s, %s, %s, %s)",
(job_id, name, attempt, actor, error_code, json.dumps(detail, default=str)))
async def create_job(conn: psycopg.AsyncConnection, attachment_id, pipeline_version: int, kind: str,
max_attempts: int, actor: str, *, replay_of=None, **detail) -> uuid.UUID:
"""jobs row + its first outbox row + audit, in the CALLER's transaction."""
job_id = uuid.uuid4()
await conn.execute(
"INSERT INTO jobs (id, attachment_id, pipeline_version, kind, replay_of, max_attempts) "
"VALUES (%s, %s, %s, %s, %s, %s)", (job_id, attachment_id, pipeline_version, kind, replay_of, max_attempts))
await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, 1)", (job_id,))
await event(conn, job_id, "created", actor, kind=kind, pipeline_version=pipeline_version,
replay_of=str(replay_of) if replay_of else None, **detail)
return job_id
# ------------------------------------------------------------------ relay
@dataclass(frozen=True, slots=True)
class Due:
outbox_id: int
job_id: str
attachment_id: str
pipeline_version: int
attempt: int
created_at: dt.datetime
job_state: str
async def due_outbox(conn: psycopg.AsyncConnection, batch: int) -> list[Due]:
"""Due, unpublished outbox rows, locked for this relay pass (SKIP LOCKED: several relays
never publish the same row at the same time - a crash may still publish it twice)."""
cur = await conn.execute(
"SELECT o.id, o.job_id, j.attachment_id, j.pipeline_version, o.attempt, j.created_at, j.state "
"FROM outbox o JOIN jobs j ON j.id = o.job_id "
"WHERE o.published_at IS NULL AND o.available_at <= clock_timestamp() "
"ORDER BY o.available_at, o.id LIMIT %s FOR UPDATE OF o SKIP LOCKED", (batch,))
return [Due(r[0], str(r[1]), str(r[2]), r[3], r[4], r[5], r[6]) for r in await cur.fetchall()]
async def mark_published(conn: psycopg.AsyncConnection, ids: list[int]) -> None:
if ids:
await conn.execute("UPDATE outbox SET published_at = clock_timestamp() WHERE id = ANY(%s)", (ids,))
async def count_publish_failure(conn: psycopg.AsyncConnection, ids: list[int]) -> None:
if ids:
await conn.execute("UPDATE outbox SET publish_failures = publish_failures + 1 WHERE id = ANY(%s)", (ids,))
async def expire_leases(conn: psycopg.AsyncConnection, actor: str, limit: int = 50) -> int:
"""Reaper: processing jobs whose lease ran out (the worker died, hung or lost its database).
The attempt is abandoned; the job goes back to queued with a new outbox row, or to failed
`retry_exhausted` when it used all its attempts."""
cur = await conn.execute(
"SELECT id, attempt, max_attempts, lease_token, attachment_id FROM jobs "
"WHERE state = 'processing' AND lease_until < clock_timestamp() "
"ORDER BY lease_until LIMIT %s FOR UPDATE SKIP LOCKED", (limit,))
rows = await cur.fetchall()
for job_id, attempt, max_attempts, token, attachment_id in rows:
await conn.execute(
"UPDATE job_attempts SET outcome = 'abandoned', error_code = 'lease_expired', finished_at = clock_timestamp() "
"WHERE token = %s AND outcome = 'running'", (token,))
if attempt >= max_attempts:
await _fail(conn, job_id, attachment_id, "retry_exhausted", actor, attempt, cause="lease_expired")
else:
await conn.execute(
"UPDATE jobs SET state = 'queued', lease_token = NULL, lease_owner = NULL, lease_until = NULL, "
"available_at = clock_timestamp(), updated_at = clock_timestamp() WHERE id = %s", (job_id,))
await conn.execute(
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() "
"WHERE id = %s AND status = 'processing'", (attachment_id,))
await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, %s)", (job_id, attempt + 1))
await event(conn, job_id, "lease_expired", actor, attempt=attempt)
return len(rows)
async def republish_stale(conn: psycopg.AsyncConnection, older_than: float, actor: str, limit: int = 50) -> int:
"""Lost-message guard: a queued, due job that has had no unpublished outbox row and no
publish for `older_than` seconds gets a new outbox row (its message was lost by the broker,
dead-lettered after too many redeliveries, or the queue was purged)."""
cur = await conn.execute(
"SELECT j.id, j.attempt FROM jobs j "
"WHERE j.state = 'queued' AND j.available_at < clock_timestamp() - make_interval(secs => %s) "
"AND j.updated_at < clock_timestamp() - make_interval(secs => %s) "
"AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.id AND (o.published_at IS NULL "
" OR o.published_at > clock_timestamp() - make_interval(secs => %s))) "
"ORDER BY j.updated_at LIMIT %s FOR UPDATE OF j SKIP LOCKED", (older_than, older_than, older_than, limit))
rows = await cur.fetchall()
for job_id, attempt in rows:
await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, %s)", (job_id, attempt + 1))
await event(conn, job_id, "republished", actor, attempt=attempt)
return len(rows)
async def purge_outbox(conn: psycopg.AsyncConnection, keep_seconds: int = 7 * 86400) -> int:
cur = await conn.execute(
"DELETE FROM outbox WHERE id IN (SELECT id FROM outbox WHERE published_at < clock_timestamp() "
"- make_interval(secs => %s) LIMIT 1000)", (keep_seconds,))
return cur.rowcount
# ------------------------------------------------------------------ worker
@dataclass(frozen=True, slots=True)
class Claim:
job_id: str
attachment_id: str
pipeline_version: int
attempt: int
max_attempts: int
token: str
source: str # staging | original
source_key: str
content_type: str
size_bytes: int
sha256: bytes
write_original: bool # the original is not published yet: this attempt publishes it too
def key(self, name: str) -> str:
return attempt_key(self.attachment_id, self.token, name)
@dataclass(frozen=True, slots=True)
class Noop:
reason: str # finished | leased | not_due | superseded | unknown_job
async def claim(conn: psycopg.AsyncConnection, job_id: str, worker: str, lease_seconds: float) -> Claim | Noop:
cur = await conn.execute(
"SELECT state, attempt, max_attempts, available_at > clock_timestamp(), attachment_id, pipeline_version "
"FROM jobs WHERE id = %s FOR UPDATE", (job_id,))
row = await cur.fetchone()
if row is None:
return Noop("unknown_job")
state, attempt, max_attempts, not_due, attachment_id, version = row
if state not in ACTIVE:
return Noop("finished") # duplicate/redelivery of a finished job: ack, nothing else
if state == "processing":
return Noop("leased") # another attempt holds the lease; the reaper handles its death
if not_due:
return Noop("not_due") # backoff not over: the outbox publishes it at available_at
cur = await conn.execute(
"SELECT status, published_version, object_key, staging_key, staging_state, content_type, size_bytes, sha256 "
"FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,))
status, published, object_key, staging_key, staging_state, content_type, size, sha = await cur.fetchone()
actor = "worker:" + worker
if published is not None and published >= version:
# A newer (or the same) version is already published: this job can only be stale.
await _finish(conn, job_id, "superseded", None)
await event(conn, job_id, "superseded", actor, attempt=attempt, published_version=published)
return Noop("superseded")
if object_key is not None:
source, source_key = "original", object_key
elif staging_state == "present":
source, source_key = "staging", staging_key
else:
# Cannot happen while the CHECK attachments_active_has_source holds; fail visibly anyway.
await _fail(conn, job_id, attachment_id, "source_missing", actor, attempt)
return Noop("finished")
token = uuid.uuid4().hex
attempt += 1
write_original = object_key is None
names = (("original",) if write_original else ()) + VARIANT_NAMES
planned = [attempt_key(str(attachment_id), token, n) for n in names]
await conn.execute(
"UPDATE jobs SET state = 'processing', attempt = %s, lease_token = %s, lease_owner = %s, "
"lease_until = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp() WHERE id = %s",
(attempt, token, worker, lease_seconds, job_id))
await conn.execute(
"INSERT INTO job_attempts (token, job_id, attempt, worker, source, planned_keys) VALUES (%s, %s, %s, %s, %s, %s)",
(token, job_id, attempt, worker, source, planned))
await conn.execute(
"UPDATE attachments SET status = 'processing', updated_at = clock_timestamp() WHERE id = %s AND status = 'queued'",
(attachment_id,))
await event(conn, job_id, "claimed", actor, attempt=attempt, source=source, token=token)
return Claim(str(job_id), str(attachment_id), version, attempt, max_attempts, token, source, source_key,
content_type, size, bytes(sha), write_original)
async def heartbeat(conn: psycopg.AsyncConnection, claim: Claim, lease_seconds: float) -> bool:
"""Extend the lease; False = the lease was lost (expired and taken over): stop the attempt."""
cur = await conn.execute(
"UPDATE jobs SET lease_until = clock_timestamp() + make_interval(secs => %s) "
"WHERE id = %s AND state = 'processing' AND lease_token = %s", (lease_seconds, claim.job_id, claim.token))
return cur.rowcount == 1
async def _holds_lease(conn: psycopg.AsyncConnection, claim: Claim) -> bool:
cur = await conn.execute("SELECT state = 'processing' AND lease_token = %s FROM jobs WHERE id = %s FOR UPDATE",
(claim.token, claim.job_id))
row = await cur.fetchone()
return bool(row and row[0])
async def complete(conn: psycopg.AsyncConnection, claim: Claim, variants, worker: str) -> str:
"""Publish the result of an attempt atomically: variant rows + attachment ready + job succeeded.
Returns succeeded | superseded | fenced (this attempt lost its lease: nothing written)."""
actor = "worker:" + worker
if not await _holds_lease(conn, claim):
return "fenced"
cur = await conn.execute("SELECT published_version FROM attachments WHERE id = %s FOR UPDATE", (claim.attachment_id,))
(published,) = await cur.fetchone()
if published is not None and published >= claim.pipeline_version:
await _end_attempt(conn, claim, "superseded", None)
await _finish(conn, claim.job_id, "superseded", None)
await event(conn, claim.job_id, "superseded", actor, attempt=claim.attempt, published_version=published)
return "superseded"
for v in variants:
await conn.execute(
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, "
"size_bytes, width, height, sha256, attempt_token) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
(claim.attachment_id, claim.pipeline_version, v.name, claim.key(v.name), v.content_type, len(v.data),
v.width, v.height, v.sha256, claim.token))
await conn.execute(
"UPDATE attachments SET status = 'ready', published_version = %s, error_code = NULL, failed_at = NULL, "
"object_key = COALESCE(object_key, %s), updated_at = clock_timestamp() WHERE id = %s",
(claim.pipeline_version, claim.key("original") if claim.write_original else None, claim.attachment_id))
await _end_attempt(conn, claim, "succeeded", None)
await _finish(conn, claim.job_id, "succeeded", None)
await event(conn, claim.job_id, "succeeded", actor, attempt=claim.attempt, pipeline_version=claim.pipeline_version)
return "succeeded"
async def retry_or_fail(conn: psycopg.AsyncConnection, claim: Claim, code: str, permanent: bool, worker: str,
base: float, cap: float) -> tuple[str, float]:
"""After a failed attempt: permanent error or no attempts left -> failed (failed registry);
otherwise queued again after a jittered backoff. Returns (outcome, delay seconds)."""
actor = "worker:" + worker
if not await _holds_lease(conn, claim):
return "fenced", 0.0
if permanent or claim.attempt >= claim.max_attempts:
final = code if permanent else "retry_exhausted"
await _end_attempt(conn, claim, "failed", code)
await _fail(conn, claim.job_id, claim.attachment_id, final, actor, claim.attempt, cause=code)
return "failed", 0.0
delay = backoff_delay(claim.attempt, base, cap)
await _end_attempt(conn, claim, "retry", code)
await conn.execute(
"UPDATE jobs SET state = 'queued', lease_token = NULL, lease_owner = NULL, lease_until = NULL, error_code = %s, "
"available_at = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp() WHERE id = %s",
(code, delay, claim.job_id))
await conn.execute(
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() WHERE id = %s AND status = 'processing'",
(claim.attachment_id,))
await conn.execute(
"INSERT INTO outbox (job_id, attempt, available_at) "
"VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s))", (claim.job_id, claim.attempt + 1, delay))
await event(conn, claim.job_id, "retry_scheduled", actor, attempt=claim.attempt, error_code=code,
delay_s=round(delay, 3))
return "retry", delay
async def release(conn: psycopg.AsyncConnection, claim: Claim, worker: str) -> str:
"""Graceful stop of a worker with an attempt still running: give the job back at once
(without waiting for the lease to expire). The claim already counted as an attempt."""
if not await _holds_lease(conn, claim):
return "fenced"
await _end_attempt(conn, claim, "abandoned", "worker_shutdown")
if claim.attempt >= claim.max_attempts:
await _fail(conn, claim.job_id, claim.attachment_id, "retry_exhausted", "worker:" + worker, claim.attempt,
cause="worker_shutdown")
return "failed"
await conn.execute(
"UPDATE jobs SET state = 'queued', lease_token = NULL, lease_owner = NULL, lease_until = NULL, "
"available_at = clock_timestamp(), updated_at = clock_timestamp() WHERE id = %s", (claim.job_id,))
await conn.execute(
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() WHERE id = %s AND status = 'processing'",
(claim.attachment_id,))
await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, %s)", (claim.job_id, claim.attempt + 1))
await event(conn, claim.job_id, "released", "worker:" + worker, attempt=claim.attempt)
return "released"
async def _end_attempt(conn: psycopg.AsyncConnection, claim: Claim, outcome: str, code: str | None) -> None:
await conn.execute(
"UPDATE job_attempts SET outcome = %s, error_code = %s, finished_at = clock_timestamp() "
"WHERE token = %s AND outcome = 'running'", (outcome, code, claim.token))
async def _finish(conn: psycopg.AsyncConnection, job_id, state: str, code: str | None) -> None:
await conn.execute(
"UPDATE jobs SET state = %s, error_code = %s, lease_token = NULL, lease_owner = NULL, lease_until = NULL, "
"finished_at = clock_timestamp(), updated_at = clock_timestamp() WHERE id = %s", (state, code, job_id))
async def _fail(conn: psycopg.AsyncConnection, job_id, attachment_id, code: str, actor: str, attempt: int,
**detail) -> None:
"""Job -> failed (registry). The attachment becomes failed only if it is not ready: a failed
reprocess of a newer version leaves the published result in place."""
await _finish(conn, job_id, "failed", code)
await conn.execute(
"UPDATE attachments SET status = 'failed', error_code = %s, failed_at = clock_timestamp(), "
"updated_at = clock_timestamp() WHERE id = %s AND status IN ('queued', 'processing')", (code, attachment_id))
await event(conn, job_id, "failed", actor, attempt=attempt, error_code=code, **detail)
# ------------------------------------------------------------------ cleanup (worker)
async def staging_to_delete(conn: psycopg.AsyncConnection, retention_seconds: int, limit: int = 50) -> list[str]:
"""Candidates only (no lock): ready attachments (staging no longer needed) and failed ones
older than the retention. The decision is re-checked under the row lock in mark_staging_deleting."""
cur = await conn.execute(
"SELECT id FROM attachments WHERE staging_state IN ('present', 'deleting') AND (status = 'ready' OR "
"(status = 'failed' AND failed_at < clock_timestamp() - make_interval(secs => %s))) "
"ORDER BY updated_at LIMIT %s", (retention_seconds, limit))
return [str(r[0]) for r in await cur.fetchall()]
async def mark_staging_deleting(conn: psycopg.AsyncConnection, attachment_id: str, retention_seconds: int) -> str | None:
"""Under the attachment row lock, decide AGAIN from the committed state and mark the staging copy
`deleting` BEFORE the S3 DELETE. A replay that committed in between made the row queued, so it
no longer matches; a replay that comes later sees `deleting` and refuses. Queued/processing
rows never match (and the CHECK attachments_active_has_source forbids it anyway)."""
cur = await conn.execute(
"UPDATE attachments SET staging_state = 'deleting' " # internal state: updated_at stays (public field)
"WHERE id = %s AND staging_state IN ('present', 'deleting') AND (status = 'ready' OR (status = 'failed' "
"AND failed_at < clock_timestamp() - make_interval(secs => %s) AND NOT EXISTS (SELECT 1 FROM jobs j "
"WHERE j.attachment_id = attachments.id AND j.state IN ('queued', 'processing')))) RETURNING staging_key",
(attachment_id, retention_seconds))
row = await cur.fetchone()
return row[0] if row else None
async def mark_staging_deleted(conn: psycopg.AsyncConnection, attachment_id: str) -> None:
await conn.execute(
"UPDATE attachments SET staging_state = 'deleted' WHERE id = %s AND staging_state = 'deleting'", (attachment_id,))
async def attempts_to_clean(conn: psycopg.AsyncConnection, grace_seconds: int, limit: int = 50) -> list[str]:
cur = await conn.execute(
"SELECT token FROM job_attempts WHERE outcome IN ('retry', 'failed', 'abandoned', 'superseded') "
"AND objects_cleaned_at IS NULL AND finished_at < clock_timestamp() - make_interval(secs => %s) "
"ORDER BY finished_at LIMIT %s", (grace_seconds, limit))
return [r[0].hex for r in await cur.fetchall()]
async def lock_attempt_for_cleanup(conn: psycopg.AsyncConnection, token: str) -> list[str] | None:
"""Keys of an attempt that can never publish: it ended without success, it is not the lease
holder of its job (tokens are never reused) and nothing published references its keys."""
cur = await conn.execute(
"SELECT ja.planned_keys FROM job_attempts ja JOIN jobs j ON j.id = ja.job_id "
"WHERE ja.token = %s AND ja.objects_cleaned_at IS NULL "
"AND ja.outcome IN ('retry', 'failed', 'abandoned', 'superseded') "
"AND j.lease_token IS DISTINCT FROM ja.token FOR UPDATE OF ja", (token,))
row = await cur.fetchone()
if row is None:
return None
keys = list(row[0])
cur = await conn.execute(
"SELECT EXISTS (SELECT 1 FROM attachment_variants WHERE object_key = ANY(%s)) "
"OR EXISTS (SELECT 1 FROM attachments WHERE object_key = ANY(%s))", (keys, keys))
if (await cur.fetchone())[0]:
return None # published: never delete (would be a bug elsewhere; logged by the caller)
return keys
async def mark_attempt_cleaned(conn: psycopg.AsyncConnection, token: str) -> None:
await conn.execute("UPDATE job_attempts SET objects_cleaned_at = clock_timestamp() WHERE token = %s", (token,))
+63
View File
@@ -0,0 +1,63 @@
"""Versioned image pipeline (lab 5): WHAT the worker builds for one attachment. Pure definitions,
no I/O; the pixels are produced by imaging.py in a child process.
Every pipeline version builds the same three variants with the geometry of the contract; versions
differ only in encoder parameters, so the published version is visible in the bytes (sha256):
thumbnail JPEG, fits into 256x256, keeps the aspect ratio, never upscales, floor rounding
(minimum 1 px), EXIF orientation applied, alpha composited onto white;
crop JPEG, centred square 128x128 (small images ARE upscaled, the contract allows it);
optimized WebP, fits into 1280x1280, never upscales, alpha kept.
Metadata: nothing of the source is copied into a variant - no EXIF (orientation is applied to
the pixels instead), no XMP, no ICC profile, no comments (the pixels are treated as sRGB).
The original is published byte for byte and keeps everything.
Track 2: the version of a job is fixed when the job is created (PIPELINE_VERSION for uploads,
`jobctl reprocess --pipeline N` later). A result is published only if its version is greater
than the published one, so an old job can never overwrite a newer result.
"""
from __future__ import annotations
from dataclasses import dataclass
THUMBNAIL_BOX = 256
CROP_SIDE = 128
OPTIMIZED_BOX = 1280
VARIANT_NAMES = ("thumbnail", "crop", "optimized")
CONTENT_TYPES = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"}
@dataclass(frozen=True, slots=True)
class Pipeline:
version: int
jpeg_quality: int # thumbnail and crop: baseline JPEG, 4:2:0, optimized Huffman tables
webp_quality: int # optimized: lossy WebP
webp_method: int # 0 (fast) .. 6 (slow, smaller)
PIPELINES = {
1: Pipeline(1, jpeg_quality=85, webp_quality=80, webp_method=4),
# Versions 2 and 3 exist for the track 2 experiments (reprocess, reorder, stale jobs):
# different encoder settings -> different bytes, identical geometry.
2: Pipeline(2, jpeg_quality=80, webp_quality=75, webp_method=4),
3: Pipeline(3, jpeg_quality=90, webp_quality=85, webp_method=5),
}
def fit(width: int, height: int, box: int) -> tuple[int, int]:
"""Size inside box x box keeping the aspect ratio, never upscaling, rounded DOWN, >= 1 px.
Integer arithmetic: 320x200 -> 256x160 exactly, 1x1000 -> 1x256 (0.256 -> 1 px minimum)."""
if width <= box and height <= box:
return width, height
if width >= height:
return box, max(1, height * box // width)
return max(1, width * box // height), box
def center_square(width: int, height: int) -> tuple[int, int, int, int]:
"""Crop box (left, top, right, bottom) of the centred square with side min(width, height)."""
side = min(width, height)
left = (width - side) // 2
top = (height - side) // 2
return left, top, left + side, top + side
-437
View File
@@ -1,437 +0,0 @@
"""Reconciler of the two stores (lab 4, track 2): S3 objects vs attachment rows in PostgreSQL.
docker compose run --rm reconciler # dry-run: report only (default)
docker compose run --rm reconciler --apply # fix what is safe to fix
docker compose run --rm reconciler --verify-checksums # also read every object (SHA-256)
How it walks: both sides are read as key-ordered streams and merge-joined - ListObjectsV2
pages (prefix o/, byte order) against keyset pages of `attachments ORDER BY object_key`
(COLLATE "C" = the same byte order, unique index). Memory does not depend on the bucket size;
cost is ~N/1000 S3 list requests + N/page DB pages (measured: docs/CONSISTENCY.md).
What it finds and what --apply does (nothing else is ever changed):
kind condition --apply
orphan_object object, no row, older than --grace delete the object
stale_intent row 'pending' older than --grace (+/- object) delete object, then row
missing_object row 'ready', object absent, older than --grace quarantine the row
size_mismatch row 'ready', object size != size_bytes quarantine the row
checksum_mismatch row 'ready', SHA-256 differs (--verify-checksums) quarantine the row
restored_object row quarantined, object present and correct again remove the quarantine
(a checksum quarantine only with --verify-checksums)
fresh_* the same, but younger than --grace nothing (in-flight upload)
Safety:
* dry-run is read-only BY CONSTRUCTION: the PostgreSQL session runs with
default_transaction_read_only=on and the S3 wrapper has no write methods, so even a bug in
the planner cannot modify a store;
* ready rows are never deleted - a used (or unused) ready attachment can only be quarantined,
which is reversible; objects that have a row are never deleted unless the row is a stale
intent; anything younger than --grace is never touched;
* every action re-checks its condition right before acting, under a row lock (FOR UPDATE)
where a row exists, so the live API racing with the reconciler cannot be hurt;
* object first, row second: an interrupted run leaves a state the next run repairs;
re-running is idempotent (the second run finds nothing to do);
* safety valve: --apply refuses to delete more than --max-orphan-fraction of the scanned
objects (pointing it at the wrong/empty database would otherwise wipe the bucket).
"""
from __future__ import annotations
import argparse
import asyncio
import datetime as dt
import hashlib
import json
import sys
import time
from collections import Counter
from collections.abc import AsyncIterator
from dataclasses import dataclass, field
import psycopg
from .config import ConfigError, Settings, _float, _secret, s3_settings
from .errors import Unavailable
from .storage import OBJECT_PREFIX, ListedObject, ObjectMissing, Storage
KINDS = ("orphan_object", "stale_intent", "missing_object", "size_mismatch", "checksum_mismatch",
"restored_object", "fresh_orphan_object", "fresh_intent", "fresh_missing_object")
MIN_SAFE_GRACE = 60.0 # > the longest possible upload (queue 5 s + body 30 s + verify 10 s + S3)
ACTION_OF = {"orphan_object": "delete_object", "stale_intent": "delete_intent", "missing_object": "quarantine",
"size_mismatch": "quarantine", "checksum_mismatch": "quarantine", "restored_object": "unquarantine"}
def log(text: str) -> None:
print(f"reconcile: {text}", file=sys.stderr, flush=True)
@dataclass(frozen=True, slots=True)
class Row:
id: str
key: str
status: str
size: int
sha256: bytes
created_at: dt.datetime
updated_at: dt.datetime
lost_reason: str | None
used: bool
@dataclass(slots=True)
class Finding:
kind: str
key: str
attachment_id: str | None = None
used: bool = False
detail: str = ""
result: str = "planned" # planned | applied | skipped:<why> | failed:<why>
@dataclass
class Stats:
objects: int = 0
rows: int = 0
consistent: int = 0
s3_list_requests: int = 0
s3_head_requests: int = 0
s3_get_bytes: int = 0
db_pages: int = 0
class ReadOnlyStorage:
"""The dry-run view of S3: listing and reading only - there is no delete to call."""
def __init__(self, storage: Storage):
self._s = storage
async def list_page(self, *args, **kwargs):
return await self._s.list_page(*args, **kwargs)
async def head(self, key: str):
return await self._s.head(key)
async def read_all(self, key: str, limit: int) -> bytes:
return await self._s.read_all(key, limit)
class Reconciler:
def __init__(self, conn: psycopg.AsyncConnection, storage: Storage | ReadOnlyStorage, *, grace: float,
page_size: int, verify: bool, apply: bool):
self.conn = conn
self.s3 = storage
self.grace = grace
self.page_size = page_size
self.verify = verify
self.apply = apply
self.stats = Stats()
self.findings: list[Finding] = []
# ---------------------------------------------------------- streams
async def objects(self) -> AsyncIterator[ListedObject]:
after = None
while True:
page, more = await self.s3.list_page(OBJECT_PREFIX, after, self.page_size)
self.stats.s3_list_requests += 1
for item in page:
self.stats.objects += 1
yield item
if not more or not page:
return
after = page[-1].key
async def rows(self) -> AsyncIterator[Row]:
after = ""
while True:
cur = await self.conn.execute(
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256, a.created_at, a.updated_at, "
"a.lost_reason, EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) "
"FROM attachments a WHERE a.object_key > %s ORDER BY a.object_key LIMIT %s",
(after, self.page_size))
page = await cur.fetchall()
self.stats.db_pages += 1
for r in page:
self.stats.rows += 1
yield Row(str(r[0]), r[1], r[2], r[3], bytes(r[4]), r[5], r[6], r[7], r[8])
if len(page) < self.page_size:
return
after = page[-1][1]
# ---------------------------------------------------------- planning (read-only)
async def scan(self) -> None:
cur = await self.conn.execute("SELECT clock_timestamp()")
self.db_now = (await cur.fetchone())[0]
self.s3_now = time.time()
objects, rows = self.objects(), self.rows()
obj = await anext(objects, None)
row = await anext(rows, None)
while obj is not None or row is not None:
if row is None or (obj is not None and obj.key < row.key):
self.object_only(obj)
obj = await anext(objects, None)
elif obj is None or row.key < obj.key:
self.row_only(row)
row = await anext(rows, None)
else:
await self.both(obj, row)
obj = await anext(objects, None)
row = await anext(rows, None)
def row_old(self, row: Row) -> bool:
return (self.db_now - max(row.created_at, row.updated_at)).total_seconds() > self.grace
def object_only(self, obj: ListedObject) -> None:
old = self.s3_now - obj.last_modified_ts > self.grace
self.add(Finding("orphan_object" if old else "fresh_orphan_object", obj.key))
def row_only(self, row: Row) -> None:
if row.status == "pending":
self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id,
detail="object absent"))
elif row.lost_reason is not None:
self.stats.consistent += 1 # already quarantined, still missing: nothing new
else:
self.add(Finding("missing_object" if self.row_old(row) else "fresh_missing_object", row.key, row.id,
used=row.used))
async def both(self, obj: ListedObject, row: Row) -> None:
if row.status == "pending":
self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id,
detail="object present"))
return
problem = await self.object_problem(row, obj.size)
if problem:
if row.lost_reason is None:
self.add(Finding(problem, row.key, row.id, used=row.used))
else:
self.stats.consistent += 1 # quarantined and still broken
elif row.lost_reason == "checksum_mismatch" and not self.verify:
# Same size, but this pass does not read the bytes: it cannot prove the object is the
# original again, so a checksum quarantine is lifted only by --verify-checksums.
self.stats.consistent += 1
elif row.lost_reason is not None:
self.add(Finding("restored_object", row.key, row.id, used=row.used))
else:
self.stats.consistent += 1
async def object_problem(self, row: Row, size: int) -> str | None:
if size != row.size:
return "size_mismatch"
if self.verify:
data = await self.s3.read_all(row.key, row.size)
self.stats.s3_get_bytes += len(data)
if hashlib.sha256(data).digest() != row.sha256:
return "checksum_mismatch"
return None
def add(self, finding: Finding) -> None:
if finding.kind not in ACTION_OF:
finding.result = "not_actionable"
self.findings.append(finding)
# ---------------------------------------------------------- actions (--apply only)
async def run_actions(self, max_orphan_fraction: float) -> None:
orphans = sum(1 for f in self.findings if f.kind == "orphan_object")
if self.stats.objects and orphans > max_orphan_fraction * self.stats.objects:
raise SafetyValve(f"{orphans} of {self.stats.objects} objects look orphaned "
f"(> {max_orphan_fraction:.0%}); wrong database? nothing was changed")
handlers = {"delete_object": self.delete_object, "delete_intent": self.delete_intent,
"quarantine": self.quarantine, "unquarantine": self.unquarantine}
for finding in self.findings:
action = ACTION_OF.get(finding.kind)
if action is None:
continue
try:
finding.result = await handlers[action](finding)
except (Unavailable, psycopg.OperationalError) as exc:
finding.result = f"failed:{type(exc).__name__}"
raise # a store is down: stop; the next run continues from the committed state
async def delete_object(self, f: Finding) -> str:
cur = await self.conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (f.key,))
if await cur.fetchone():
return "skipped:row_appeared"
try:
head = await self.s3.head(f.key)
self.stats.s3_head_requests += 1
except ObjectMissing:
return "skipped:already_gone"
if time.time() - head.last_modified_ts <= self.grace:
return "skipped:object_rewritten_recently"
# No row names this key and the object is older than the grace period. Rows are always
# written BEFORE their object and keys are random and never reused, so no upload can
# still claim it.
await self.s3.delete(f.key)
return "applied"
async def delete_intent(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT 1 FROM attachments WHERE id = %s AND status = 'pending' "
"AND greatest(created_at, updated_at) < clock_timestamp() - make_interval(secs => %s) FOR UPDATE",
(f.attachment_id, self.grace))
if await cur.fetchone() is None:
return "skipped:no_longer_stale_intent"
await self.s3.delete(f.key) # object first (idempotent), then the row
await self.conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'pending'",
(f.attachment_id,))
return "applied"
async def quarantine(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT size_bytes, sha256 FROM attachments WHERE id = %s AND status = 'ready' "
"AND lost_at IS NULL FOR UPDATE", (f.attachment_id,))
row = await cur.fetchone()
if row is None:
return "skipped:row_changed"
reason = await self.current_problem(f.key, row[0], bytes(row[1]))
if reason is None:
return "skipped:object_ok_now"
await self.conn.execute(
"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = %s WHERE id = %s",
(reason, f.attachment_id))
return "applied"
async def unquarantine(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT size_bytes, sha256, lost_reason FROM attachments WHERE id = %s AND lost_at IS NOT NULL FOR UPDATE",
(f.attachment_id,))
row = await cur.fetchone()
if row is None:
return "skipped:row_changed"
if row[2] == "checksum_mismatch" and not self.verify:
return "skipped:needs_verify_checksums"
if await self.current_problem(f.key, row[0], bytes(row[1])) is not None:
return "skipped:object_still_bad"
await self.conn.execute("UPDATE attachments SET lost_at = NULL, lost_reason = NULL WHERE id = %s",
(f.attachment_id,))
return "applied"
async def current_problem(self, key: str, size: int, sha256: bytes) -> str | None:
"""Fresh look at the object right before changing the row (the listing may be stale)."""
try:
head = await self.s3.head(key)
self.stats.s3_head_requests += 1
except ObjectMissing:
return "missing_object"
if head.size != size:
return "size_mismatch"
if self.verify:
data = await self.s3.read_all(key, size)
self.stats.s3_get_bytes += len(data)
if hashlib.sha256(data).digest() != sha256:
return "checksum_mismatch"
return None
# ---------------------------------------------------------- report
def report(self, mode: str, seconds: float, error: str | None, sample: int) -> dict:
kinds = Counter(f.kind for f in self.findings)
results = Counter(f.result.split(":")[0] for f in self.findings if f.kind in ACTION_OF)
samples = {}
for f in self.findings:
items = samples.setdefault(f.kind, [])
if len(items) < sample:
items.append({"key": f.key, "attachment_id": f.attachment_id, "used": f.used,
"result": f.result, **({"detail": f.detail} if f.detail else {})})
return {
"mode": mode, "grace_seconds": self.grace, "verify_checksums": self.verify,
"duration_seconds": round(seconds, 3), "error": error,
"scanned": {"objects": self.stats.objects, "rows": self.stats.rows, "consistent": self.stats.consistent},
"findings": {k: kinds.get(k, 0) for k in KINDS},
"used_attachments_in_findings": sum(1 for f in self.findings if f.used),
"actions": {"planned": sum(1 for f in self.findings if f.kind in ACTION_OF),
"applied": results.get("applied", 0), "skipped": results.get("skipped", 0),
"failed": results.get("failed", 0)},
"cost": {"s3_list_requests": self.stats.s3_list_requests, "s3_head_requests": self.stats.s3_head_requests,
"s3_get_bytes": self.stats.s3_get_bytes, "db_pages": self.stats.db_pages,
"page_size": self.page_size},
"samples": samples,
}
class SafetyValve(Exception):
pass
def arguments(argv: list[str] | None = None) -> argparse.Namespace:
p = argparse.ArgumentParser(prog="python -m messenger.reconcile", description=__doc__.split("\n")[0])
p.add_argument("--apply", action="store_true", help="fix findings (default: dry-run, read-only)")
p.add_argument("--grace", type=float, default=3600.0,
help="seconds; anything younger is never touched (default 3600, >= PENDING_GRACE_SECONDS)")
p.add_argument("--allow-small-grace", action="store_true",
help="permit --grace below 60 s (tests only: a grace shorter than an upload could remove the "
"intent of an upload that is still running; that upload then fails with 503)")
p.add_argument("--page-size", type=int, default=1000, help="S3 list / DB page size (1..1000)")
p.add_argument("--verify-checksums", action="store_true", help="read every ready object and compare SHA-256")
p.add_argument("--samples", type=int, default=20, help="keys listed per finding kind in the report")
p.add_argument("--max-orphan-fraction", type=float, default=0.5,
help="--apply refuses if more than this share of objects look orphaned (default 0.5)")
args = p.parse_args(argv)
if not 1 <= args.page_size <= 1000:
p.error("--page-size must be 1..1000")
if args.grace < 1 or (args.grace < MIN_SAFE_GRACE and not args.allow_small_grace):
p.error(f"--grace must be >= {MIN_SAFE_GRACE:.0f} s (or >= 1 s with --allow-small-grace)")
if not 0 < args.max_orphan_fraction <= 1:
p.error("--max-orphan-fraction must be in (0, 1]")
return args
async def run(args: argparse.Namespace) -> int:
url = _secret("DATABASE_URL", required=True)
base = s3_settings()
settings = Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024,
database_url=url, **base)
timeout_ms = int(_float("DB_TIMEOUT_SECONDS", 10.0, 1.0, 120.0) * 1000)
options = f"-c statement_timeout={timeout_ms} -c lock_timeout={timeout_ms // 2}"
if not args.apply:
options += " -c default_transaction_read_only=on" # dry-run: PostgreSQL itself refuses writes
mode = "apply" if args.apply else "dry-run"
storage = Storage(settings)
await storage.open()
started = time.perf_counter()
error = None
code = 0
try:
conn = await psycopg.AsyncConnection.connect(url, autocommit=True, connect_timeout=5, options=options,
application_name=f"messenger-reconcile-{mode}")
except psycopg.OperationalError as exc:
await storage.close()
error = f"store_unavailable: database {type(exc).__name__}"
print(json.dumps({"mode": mode, "error": error}, ensure_ascii=False))
log(error)
return 1
rec = Reconciler(conn, storage if args.apply else ReadOnlyStorage(storage), grace=args.grace,
page_size=args.page_size, verify=args.verify_checksums, apply=args.apply)
try:
log(f"{mode}: scanning bucket {settings.s3_bucket} (prefix {OBJECT_PREFIX}) against table attachments")
await rec.scan()
log(f"scan done: {rec.stats.objects} objects, {rec.stats.rows} rows, {len(rec.findings)} findings")
if args.apply:
await rec.run_actions(args.max_orphan_fraction)
except SafetyValve as exc:
error, code = f"safety_valve: {exc}", 3
except (Unavailable, psycopg.OperationalError) as exc:
error, code = f"store_unavailable: {type(exc).__name__}", 1
finally:
await conn.close()
await storage.close()
print(json.dumps(rec.report(mode, time.perf_counter() - started, error, args.samples), ensure_ascii=False,
indent=1))
if error:
log(error)
return code
def main(argv: list[str] | None = None) -> int:
args = arguments(argv)
try:
return asyncio.run(run(args))
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())
-217
View File
@@ -1,217 +0,0 @@
"""Track 2: the reconciler of S3 and PostgreSQL (python -m messenger.reconcile).
Faults are injected on BOTH sides, directly into the stores (S3 with the API/reconciler key
pairs from inside the storage network, PostgreSQL with psql):
S3 side: orphan object (no row), object deleted under a ready row, object overwritten with
other bytes (size / same-size checksum), object of a quarantined row restored;
DB side: stale intent with and without object, quarantine mark on a healthy row;
fresh: orphan and intent younger than --grace (an upload in progress).
The tests prove: dry-run changes nothing in either store; --apply fixes exactly the old findings;
fresh and used attachments are never deleted (ready rows only ever get a reversible quarantine);
the second run finds nothing to do; the safety valve and store outages stop the run without
damage. A small --grace (8-30 s, --allow-small-grace) keeps the tests fast."""
from __future__ import annotations
import concurrent.futures
import hashlib
import time
import uuid
from pathlib import Path
import pytest
import imagegen
from conftest import (attachment_row, findings_by_key, psql, reconcile, s3_delete, s3_keys, s3_put, s3_size,
service_down)
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
GRACE = ["--grace", "8", "--allow-small-grace"]
# The main scenario needs "fresh" items to stay younger than --grace for the whole dry-run + snapshot
# (each docker exec costs 1-3 s on Windows), hence a longer grace there.
LONG_GRACE = ["--grace", "30", "--allow-small-grace"]
def new_key() -> str:
return "o/" + uuid.uuid4().hex
def insert_intent(chat: dict, owner: dict, key: str, data: bytes, age_seconds: int) -> str:
"""A 'pending' row as an upload leaves it after the intent COMMIT (crash afterwards)."""
attachment_id = str(uuid.uuid4())
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
"object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'x.png', 'image/png', %d, "
"decode('%s', 'hex'), 320, 200, '%s', 'pending', clock_timestamp() - make_interval(secs => %d), "
"clock_timestamp() - make_interval(secs => %d))"
% (attachment_id, chat["id"], owner["id"], len(data), hashlib.sha256(data).hexdigest(), key, age_seconds,
age_seconds))
return attachment_id
def snapshot() -> tuple[str, frozenset]:
"""Fingerprint of everything the reconciler could change: every attachment row + the key list."""
rows = psql("SELECT count(*) || ':' || coalesce(md5(string_agg(id::text || status || object_key || "
"coalesce(lost_reason, '-') || updated_at::text, ',' ORDER BY id)), '') FROM attachments")
return rows, frozenset(s3_keys())
def content_status(api, actor, attachment_id) -> int:
return api.client.get(f"/api/v1/attachments/{attachment_id}/content",
headers={"Authorization": "Bearer " + actor["token"]}).status_code
def test_detect_and_safely_fix_faults_on_both_sides(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
up = {name: api.upload(alice, chat, imagegen.png(30 + i, 20)).json() for i, name in enumerate("ABCDE")}
data = {name: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
for name, a in up.items()}
key = {name: attachment_row(a["id"])["object_key"] for name, a in up.items()}
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
# ---- inject (old)
orphan_old = new_key()
s3_put(orphan_old, SAMPLE)
stale_with, stale_without = new_key(), new_key()
s3_put(stale_with, SAMPLE)
stale_with_id = insert_intent(chat, alice, stale_with, SAMPLE, 600) # < janitor grace 900 s
stale_without_id = insert_intent(chat, alice, stale_without, SAMPLE, 600)
s3_delete(key["C"]) # ready + used, bytes lost
s3_put(key["D"], data["D"][:len(data["D"]) // 2]) # ready, unused, wrong size
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' "
f"WHERE id = '{up['E']['id']}'") # quarantined, but bytes are fine
time.sleep(32) # everything above is now older than --grace
# ---- inject (fresh: an upload in progress)
fresh_orphan, fresh_intent = new_key(), new_key()
s3_put(fresh_orphan, SAMPLE)
s3_put(fresh_intent, SAMPLE)
fresh_intent_id = insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
# ---- dry-run: report everything, change nothing
before = snapshot()
dry = reconcile(*LONG_GRACE)
assert snapshot() == before, "dry-run modified a store"
f = findings_by_key(dry)
expected = {orphan_old: "orphan_object", fresh_orphan: "fresh_orphan_object", stale_with: "stale_intent",
stale_without: "stale_intent", fresh_intent: "fresh_intent", key["C"]: "missing_object",
key["D"]: "size_mismatch", key["E"]: "restored_object"}
assert {k: f[k]["kind"] for k in expected} == expected
assert key["A"] not in f and key["B"] not in f
assert f[key["C"]]["used"] is True and dry["mode"] == "dry-run"
assert all(f[k]["result"] in ("planned", "not_actionable") for k in expected)
# ---- apply
applied = findings_by_key(reconcile("--apply", *LONG_GRACE))
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"]):
assert applied[k]["result"] == "applied", (k, applied[k])
keys = s3_keys()
assert orphan_old not in keys and stale_with not in keys
assert attachment_row(stale_with_id) is None and attachment_row(stale_without_id) is None
# fresh: untouched
assert fresh_orphan in keys and fresh_intent in keys and attachment_row(fresh_intent_id)["status"] == "pending"
# used, lost bytes: quarantined, NOT deleted; the message still references it
row_c = attachment_row(up["C"]["id"])
assert row_c["status"] == "ready" and row_c["lost_reason"] == "missing_object"
items, _ = api.history(bob, chat, 100)
assert any(up["C"]["id"] in m["attachment_ids"] for m in items)
assert content_status(api, bob, up["C"]["id"]) == 404
api.post_message(alice, chat, {"text": "", "attachment_ids": [up["C"]["id"]]}, 409) # cannot attach a lost file
assert attachment_row(up["D"]["id"])["lost_reason"] == "size_mismatch"
assert attachment_row(up["E"]["id"])["lost_reason"] is None # quarantine lifted
for name in "ABE":
got = api.call("GET", f"/api/v1/attachments/{up[name]['id']}/content", 200, token=bob["token"])
assert got.content == data[name]
# ---- second run: idempotent, nothing left to do for these keys
again = findings_by_key(reconcile("--apply", *LONG_GRACE))
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"], key["A"], key["B"]):
assert k not in again, (k, again.get(k))
# ---- the lost object comes back from a backup: the quarantine is lifted, the file is served again
s3_put(key["C"], data["C"])
restored = findings_by_key(reconcile("--apply", *LONG_GRACE))
assert restored[key["C"]]["kind"] == "restored_object" and restored[key["C"]]["result"] == "applied"
assert api.call("GET", f"/api/v1/attachments/{up['C']['id']}/content", 200, token=bob["token"]).content == data["C"]
# tidy up the fresh items of this test once they are old enough
time.sleep(31)
reconcile("--apply", *LONG_GRACE)
assert fresh_orphan not in s3_keys() and attachment_row(fresh_intent_id) is None
def test_checksum_mode_finds_same_size_corruption(api):
alice = api.user("alice")
chat = api.chat(alice)
att = api.upload(alice, chat, SAMPLE).json()
k = attachment_row(att["id"])["object_key"]
s3_put(k, SAMPLE[:1000] + bytes([SAMPLE[1000] ^ 1]) + SAMPLE[1001:]) # same size, other bytes
time.sleep(9)
assert k not in findings_by_key(reconcile(*GRACE)) # sizes match: listing cannot see it
report = reconcile("--verify-checksums", *GRACE)
assert findings_by_key(report)[k]["kind"] == "checksum_mismatch"
assert report["cost"]["s3_get_bytes"] >= len(SAMPLE)
assert findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]["result"] == "applied"
assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch"
# A later pass WITHOUT checksums sees "same size" but cannot prove the bytes: the quarantine stays.
assert k not in findings_by_key(reconcile("--apply", *GRACE))
assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch"
# Restoring the original and verifying lifts it.
s3_put(k, SAMPLE)
lifted = findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]
assert (lifted["kind"], lifted["result"]) == ("restored_object", "applied")
assert attachment_row(att["id"])["lost_reason"] is None
def test_safety_valve_refuses_mass_deletion():
"""If (almost) every object looked orphaned - e.g. the reconciler pointed at an empty database -
--apply must stop before deleting anything."""
orphan = new_key()
s3_put(orphan, SAMPLE)
time.sleep(9)
report = reconcile("--apply", "--max-orphan-fraction", "0.000001", *GRACE, expect_code=3)
assert report["error"].startswith("safety_valve") and report["actions"]["applied"] == 0
assert orphan in s3_keys()
reconcile("--apply", *GRACE)
assert orphan not in s3_keys()
@pytest.mark.parametrize("service", ["s3", "postgres"])
def test_store_outage_stops_the_run_without_damage(api, service):
with service_down(service, "stop"):
report = reconcile(*GRACE, expect_code=1)
assert report["error"].startswith("store_unavailable")
assert reconcile(*GRACE)["error"] is None
def test_live_uploads_are_never_hurt_by_concurrent_apply(api):
"""Uploads and messages run while --apply runs repeatedly: every upload that got 201 stays
ready and downloadable byte for byte, every message keeps its attachments."""
alice = api.user("alice")
chat = api.chat(alice)
payloads = [imagegen.png(16 + i, 16) for i in range(40)]
done: list[tuple[dict, bytes]] = []
def worker(chunk: list[bytes]) -> None:
for p in chunk:
att = api.upload(alice, chat, p).json()
api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]})
done.append((att, p))
with concurrent.futures.ThreadPoolExecutor(4) as pool:
futures = [pool.submit(worker, payloads[i::4]) for i in range(4)]
runs = 0
while not all(x.done() for x in futures):
report = reconcile("--apply", *GRACE)
assert report["error"] is None
runs += 1
for x in futures:
x.result()
assert runs >= 1 and len(done) == len(payloads)
for att, p in done:
row = attachment_row(att["id"])
assert row["status"] == "ready" and row["lost_reason"] is None and s3_size(row["object_key"]) == len(p)
for att, p in done[::8]:
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]).content == p