forked from mtusi2026/mtusi2026lab5
lab5: схема 0005 (jobs, outbox, попытки, варианты, аудит) и модули pipeline
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Миграция только добавляет: статусы pending/queued/processing/ready/failed, staging-ключ, published_version, jobs с lease и токеном попытки, job_attempts с заранее известными ключами, attachment_variants (PK = ключ эффекта), outbox, job_events, view failed_jobs, post_message_v3. CHECK attachments_active_has_source: у queued/processing всегда есть источник. pipeline.py — версии 1/2/3 и геометрия по контракту; imaging.py — варианты в дочернем процессе (RLIMIT_AS, таймаут, лимит пикселей); jobs.py — все переходы заданий одной транзакцией с fencing по токену; broker.py — топология и строгая схема сообщения. Зависимости: aio-pika 9.6.2, pillow 12.3.0 (lock с хешами).
This commit is contained in:
@@ -1,156 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 2 experiment: cost of one reconciler pass as a function of the number of attachments.
|
||||
|
||||
Hypothesis: a pass is O(N) with ~N/1000 S3 list requests + N/1000 DB pages and constant memory
|
||||
(merge-join of two key-ordered streams); --verify-checksums adds reading every byte (O(total size)).
|
||||
|
||||
Method: synthetic, CONSISTENT attachments (row 'ready' + 1 KiB object with the matching size and
|
||||
SHA-256, filename 'bench-reconcile') are added in steps up to the largest N; after each step the
|
||||
reconciler runs in dry-run 3 times (median reported), plus page-size and checksum variants.
|
||||
Everything synthetic is deleted at the end (objects first, then rows).
|
||||
|
||||
python bench/reconcile_cost.py --steps 1000 10000 50000 --output evidence/reconcile
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import httpx2 as httpx
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
PASSWORD = "Bench-password-Ω-12345"
|
||||
OBJECT = bytes(range(256)) * 4 # 1 KiB, the same content for every object
|
||||
SHA = hashlib.sha256(OBJECT).hexdigest()
|
||||
|
||||
PUT_SCRIPT = """
|
||||
import asyncio, sys
|
||||
from messenger.config import Settings, s3_settings
|
||||
from messenger.storage import Storage
|
||||
DATA = bytes(range(256)) * 4
|
||||
async def main():
|
||||
s = Storage(Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings()))
|
||||
await s.open()
|
||||
keys = sys.stdin.read().split()
|
||||
sem = asyncio.Semaphore(32)
|
||||
async def one(k):
|
||||
async with sem:
|
||||
await s.{op}(k{args})
|
||||
await asyncio.gather(*(one(k) for k in keys))
|
||||
await s.close()
|
||||
print(len(keys))
|
||||
asyncio.run(main())
|
||||
"""
|
||||
|
||||
|
||||
def compose(*args: str, stdin: str | None = None, timeout: float = 1800, check: bool = True) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(["docker", "compose", *args], cwd=ROOT, input=stdin, capture_output=True, text=True,
|
||||
encoding="utf-8", timeout=timeout, check=check)
|
||||
|
||||
|
||||
def psql(sql: str) -> str:
|
||||
"""SQL through stdin: a 5000-row INSERT does not fit into a Windows command line (32 KiB)."""
|
||||
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v",
|
||||
"ON_ERROR_STOP=1", "-f", "-", stdin=sql).stdout.strip()
|
||||
|
||||
|
||||
def s3_bulk(op: str, keys: list[str]) -> None:
|
||||
script = PUT_SCRIPT.replace("{op}", op).replace("{args}", ", DATA, 'image/png', {}" if op == "put" else "")
|
||||
compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "python", "reconciler", "-c", script, stdin="\n".join(keys))
|
||||
|
||||
|
||||
def owner_and_chat() -> tuple[str, str]:
|
||||
import base64
|
||||
c = httpx.Client(base_url=BASE, timeout=30)
|
||||
name = "rc_" + uuid.uuid4().hex[:10]
|
||||
c.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
|
||||
token = c.post("/api/v1/auth/sessions", headers={
|
||||
"Authorization": "Basic " + base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()}).json()["token"]
|
||||
me = c.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
|
||||
chat = c.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
|
||||
json={"title": "bench", "member_ids": [me["id"]]}).json()
|
||||
return me["id"], chat["id"]
|
||||
|
||||
|
||||
def add(n: int, owner: str, chat: str) -> None:
|
||||
keys = ["o/" + uuid.uuid4().hex for _ in range(n)]
|
||||
for i in range(0, n, 5000):
|
||||
part = keys[i:i + 5000]
|
||||
values = ",".join(f"('{uuid.uuid4()}','{k}')" for k in part)
|
||||
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
|
||||
"object_key, status, created_at, updated_at) "
|
||||
f"SELECT v.id::uuid, '{chat}', '{owner}', 'bench-reconcile', 'image/png', {len(OBJECT)}, "
|
||||
f"decode('{SHA}', 'hex'), 1, 1, v.k, 'ready', clock_timestamp() - interval '2 hours', "
|
||||
f"clock_timestamp() - interval '2 hours' FROM (VALUES {values}) AS v(id, k)")
|
||||
s3_bulk("put", keys)
|
||||
|
||||
|
||||
def reconcile(*args: str) -> dict:
|
||||
out = compose("run", "--rm", "--no-deps", "-T", "reconciler", *args).stdout
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def measure(label: str, args: list[str], repeats: int) -> dict:
|
||||
runs = [reconcile(*args) for _ in range(repeats)]
|
||||
durations = [r["duration_seconds"] for r in runs]
|
||||
last = runs[-1]
|
||||
return {"label": label, "args": args, "runs": repeats, "duration_s_median": statistics.median(durations),
|
||||
"duration_s_all": durations, "objects": last["scanned"]["objects"], "rows": last["scanned"]["rows"],
|
||||
"findings": {k: v for k, v in last["findings"].items() if v}, "cost": last["cost"]}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--steps", type=int, nargs="+", default=[1000, 10000, 50000])
|
||||
p.add_argument("--repeats", type=int, default=3)
|
||||
p.add_argument("--output", default="evidence/reconcile")
|
||||
args = p.parse_args()
|
||||
out = ROOT / args.output
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
owner, chat = owner_and_chat()
|
||||
present = 0
|
||||
results = []
|
||||
steps = sorted(args.steps)
|
||||
variant_step = steps[min(1, len(steps) - 1)]
|
||||
try:
|
||||
for target in sorted(args.steps):
|
||||
started = time.perf_counter()
|
||||
add(target - present, owner, chat)
|
||||
present = target
|
||||
print(f"populated {present} synthetic attachments in {time.perf_counter() - started:.1f} s", flush=True)
|
||||
results.append({"synthetic": present, **measure("dry-run, page 1000", [], args.repeats)})
|
||||
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
|
||||
if present == variant_step: # page-size and checksum variants at one size
|
||||
for label, extra in (("dry-run, page 100", ["--page-size", "100"]),
|
||||
("dry-run, --verify-checksums", ["--verify-checksums"])):
|
||||
results.append({"synthetic": present, **measure(label, extra, args.repeats)})
|
||||
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
|
||||
finally:
|
||||
keys = psql("SELECT object_key FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = "
|
||||
f"'{chat}'").split()
|
||||
if keys:
|
||||
s3_bulk("delete", keys)
|
||||
psql(f"DELETE FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = '{chat}'")
|
||||
print(f"removed {len(keys)} synthetic attachments", flush=True)
|
||||
meta = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
|
||||
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
|
||||
text=True).stdout.strip(),
|
||||
"object_bytes": len(OBJECT), "reconciler_limits": "cpus 1.0, memory 256m (compose)",
|
||||
"note": "the bucket/table also contain the attachments of earlier test runs (counted in objects/rows)"}
|
||||
(out / "cost.json").write_text(json.dumps({"meta": meta, "results": results}, ensure_ascii=False, indent=1),
|
||||
encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,174 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 2 evidence: inject faults on both sides, run the reconciler, keep every JSON report.
|
||||
|
||||
Scenario A (all fault kinds; the same as student-tests/test_reconcile.py, but the reports are saved):
|
||||
S3 side: old orphan objects, object deleted under a USED ready row, object truncated under an
|
||||
unused ready row, same-size corruption (found only with --verify-checksums);
|
||||
DB side: stale intents (with / without object), quarantine mark on a healthy row;
|
||||
fresh: orphan + intent younger than --grace.
|
||||
Runs: dry-run (+ snapshot of both stores before/after) -> apply -> apply again -> restore of the
|
||||
lost object from a "backup" -> apply (quarantine lifted).
|
||||
Scenario B (interrupted apply): 2000 old orphans, S3 is paused while --apply deletes them ->
|
||||
the run stops with an error, nothing else is damaged; the next run finishes the rest; the
|
||||
third run finds nothing.
|
||||
|
||||
python bench/reconcile_faults.py --output evidence/reconcile
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
import imagegen # noqa: E402
|
||||
from conftest import (Api, attachment_row, compose, findings_by_key, psql, s3_delete, s3_exec, s3_keys, # noqa: E402
|
||||
s3_put)
|
||||
from test_reconcile import insert_intent, snapshot # noqa: E402
|
||||
|
||||
GRACE = ["--grace", "30", "--allow-small-grace"]
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
|
||||
|
||||
def reconcile(*args: str) -> tuple[int, dict]:
|
||||
r = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "1000", *args, check=False, timeout=600)
|
||||
return r.returncode, json.loads(r.stdout)
|
||||
|
||||
|
||||
def save(out: Path, name: str, data) -> None:
|
||||
(out / name).write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
|
||||
|
||||
def ours(report: dict, keys: dict[str, str]) -> dict:
|
||||
"""The findings of this scenario's keys only (the stand also holds other test data)."""
|
||||
f = findings_by_key(report)
|
||||
return {label: ({"kind": f[k]["kind"], "result": f[k]["result"], "used": f[k]["used"]} if k in f else None)
|
||||
for label, k in keys.items()}
|
||||
|
||||
|
||||
def scenario_a(out: Path) -> dict:
|
||||
api = Api()
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
chat = api.chat(alice, bob)
|
||||
up = {n: api.upload(alice, chat, imagegen.png(40 + i, 30, noise=True)).json() for i, n in enumerate("ABCDEF")}
|
||||
data = {n: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
|
||||
for n, a in up.items()}
|
||||
key = {n: attachment_row(a["id"])["object_key"] for n, a in up.items()}
|
||||
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
|
||||
keys = {"healthy used A": key["A"], "healthy unused B": key["B"], "used, object deleted C": key["C"],
|
||||
"unused, object truncated D": key["D"], "quarantined but healthy E": key["E"],
|
||||
"same-size corruption F": key["F"]}
|
||||
orphan = "o/" + uuid.uuid4().hex
|
||||
stale_with, stale_without = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex
|
||||
s3_put(orphan, SAMPLE)
|
||||
s3_put(stale_with, SAMPLE)
|
||||
insert_intent(chat, alice, stale_with, SAMPLE, 600)
|
||||
insert_intent(chat, alice, stale_without, SAMPLE, 600)
|
||||
s3_delete(key["C"])
|
||||
s3_put(key["D"], data["D"][: len(data["D"]) // 2])
|
||||
s3_put(key["F"], data["F"][:200] + bytes([data["F"][200] ^ 1]) + data["F"][201:])
|
||||
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' WHERE id = '{up['E']['id']}'")
|
||||
keys.update({"old orphan object": orphan, "stale intent + object": stale_with, "stale intent, no object": stale_without})
|
||||
time.sleep(32)
|
||||
fresh_orphan, fresh_intent = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex
|
||||
s3_put(fresh_orphan, SAMPLE)
|
||||
s3_put(fresh_intent, SAMPLE)
|
||||
insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
|
||||
keys.update({"fresh orphan (in-flight)": fresh_orphan, "fresh intent (in-flight)": fresh_intent})
|
||||
|
||||
before = snapshot()
|
||||
_, dry = reconcile("--verify-checksums", *GRACE)
|
||||
after = snapshot()
|
||||
_, apply1 = reconcile("--apply", "--verify-checksums", *GRACE)
|
||||
_, apply2 = reconcile("--apply", "--verify-checksums", *GRACE)
|
||||
s3_put(key["C"], data["C"]) # the lost object restored from a backup
|
||||
_, apply3 = reconcile("--apply", "--verify-checksums", *GRACE)
|
||||
for name, report in (("faults-1-dry-run.json", dry), ("faults-2-apply.json", apply1),
|
||||
("faults-3-apply-again.json", apply2), ("faults-4-after-restore.json", apply3)):
|
||||
save(out, name, report)
|
||||
rows = {label: attachment_row(up[n]["id"]) for label, n in (("C", "C"), ("D", "D"), ("E", "E"), ("F", "F"), ("A", "A"), ("B", "B"))}
|
||||
content = {n: api.client.get(f"/api/v1/attachments/{up[n]['id']}/content",
|
||||
headers={"Authorization": "Bearer " + bob["token"]}).status_code for n in "ABCDEF"}
|
||||
present = s3_keys()
|
||||
summary = {
|
||||
"dry_run_changed_nothing": before == after,
|
||||
"per_key": {"dry-run": ours(dry, keys), "apply": ours(apply1, keys), "apply again": ours(apply2, keys),
|
||||
"after restore of C": ours(apply3, keys)},
|
||||
"objects_present_at_end (fresh items were older than --grace by the later runs)": {label: k in present for label, k in keys.items()},
|
||||
"rows_after": {k: (v and {"status": v["status"], "lost_reason": v["lost_reason"]}) for k, v in rows.items()},
|
||||
"content_status_after": content,
|
||||
"durations_s": [dry["duration_seconds"], apply1["duration_seconds"], apply2["duration_seconds"],
|
||||
apply3["duration_seconds"]],
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True)
|
||||
return summary
|
||||
|
||||
|
||||
def scenario_b(out: Path, count: int = 2000) -> dict:
|
||||
keys = ["o/" + uuid.uuid4().hex for _ in range(count)]
|
||||
s3_exec("ks = sys.stdin.read().split()\n"
|
||||
"sem = asyncio.Semaphore(32)\n"
|
||||
"async def one(k):\n"
|
||||
" async with sem:\n"
|
||||
" await s.put(k, b'orphan', 'image/png', {})\n"
|
||||
"await asyncio.gather(*(one(k) for k in ks))", stdin="\n".join(keys))
|
||||
time.sleep(32)
|
||||
result: dict = {}
|
||||
|
||||
def run_apply() -> None:
|
||||
result["code"], result["report"] = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE)
|
||||
|
||||
t = threading.Thread(target=run_apply)
|
||||
t.start()
|
||||
# pause S3 as soon as deletions are under way (the reconciler is planning/listing first)
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
left = len(s3_keys() & set(keys))
|
||||
if left < count:
|
||||
break
|
||||
time.sleep(0.2)
|
||||
subprocess.run(["docker", "compose", "pause", "s3"], cwd=ROOT, check=True, capture_output=True)
|
||||
t.join()
|
||||
subprocess.run(["docker", "compose", "unpause", "s3"], cwd=ROOT, check=True, capture_output=True)
|
||||
time.sleep(5) # let S3 settle after unpause before counting
|
||||
left_after_interrupt = len(s3_keys() & set(keys))
|
||||
code2, second = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE)
|
||||
code3, third = reconcile("--apply", *GRACE)
|
||||
save(out, "interrupted-1-apply.json", result["report"])
|
||||
save(out, "interrupted-2-rerun.json", second)
|
||||
save(out, "interrupted-3-third.json", third)
|
||||
summary = {
|
||||
"orphans_injected": count,
|
||||
"interrupted_run": {"exit_code": result["code"], "error": result["report"].get("error"),
|
||||
"applied": result["report"]["actions"]["applied"]},
|
||||
"orphans_left_after_interrupt": left_after_interrupt,
|
||||
"rerun": {"exit_code": code2, "applied": second["actions"]["applied"], "error": second["error"]},
|
||||
"third": {"exit_code": code3, "planned": third["actions"]["planned"]},
|
||||
"orphans_left_at_end": len(s3_keys() & set(keys)),
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True)
|
||||
return summary
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--output", default="evidence/reconcile")
|
||||
args = p.parse_args()
|
||||
out = ROOT / args.output
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
a = scenario_a(out)
|
||||
b = scenario_b(out)
|
||||
commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip()
|
||||
save(out, "faults-summary.json", {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "commit": commit,
|
||||
"grace_seconds": 30, "scenario_a": a, "scenario_b_interrupted": b})
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,260 @@
|
||||
-- 0005 (lab 5): asynchronous processing of attachments. Details: docs/PIPELINE.md.
|
||||
--
|
||||
-- Upload (API): INSERT attachment 'pending' (intent: the staging key is known before the PUT)
|
||||
-- -> PUT original into the PRIVATE STAGING bucket
|
||||
-- -> ONE transaction: attachment 'queued' + jobs row + outbox row -> 202.
|
||||
-- Relay: outbox row -> publish to RabbitMQ with publisher confirm -> outbox.published_at.
|
||||
-- Worker: claim job (lease + attempt token) -> variants into the final bucket under keys of
|
||||
-- THIS attempt -> ONE transaction: variants + attachment 'ready' (fenced by the
|
||||
-- lease token, never downgrading published_version) -> ack -> staging deleted.
|
||||
--
|
||||
-- PostgreSQL is the source of truth for the work: state, attempts, lease, backoff (available_at)
|
||||
-- and the failed registry. The broker only delivers job ids (at least once).
|
||||
|
||||
-- ------------------------------------------------------------------ attachments
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status
|
||||
CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed'));
|
||||
-- object_key = key of the ORIGINAL in the final bucket. NULL until the worker publishes it.
|
||||
-- Keys of lab 4 ('o/<hex>') stay valid; lab 5 writes 'a/<attachment>/<attempt token>/original'.
|
||||
ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL;
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_object_key_format;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_object_key_format
|
||||
CHECK (object_key ~ '^(o/[0-9a-f]{32}|a/[0-9a-f-]{36}/[0-9a-f]{32}/original)$');
|
||||
ALTER TABLE attachments
|
||||
-- Key of the original in the staging bucket ('s/<128 random bits>'), written by the API.
|
||||
ADD COLUMN staging_key text COLLATE "C",
|
||||
-- none (lab 4 row) | present | deleting (marked, S3 DELETE pending) | deleted
|
||||
ADD COLUMN staging_state text NOT NULL DEFAULT 'none',
|
||||
-- pipeline version whose variants are published (GET shows them); only ever grows.
|
||||
ADD COLUMN published_version integer,
|
||||
ADD COLUMN error_code text,
|
||||
ADD COLUMN failed_at timestamptz,
|
||||
ADD CONSTRAINT attachments_staging_key_key UNIQUE (staging_key),
|
||||
ADD CONSTRAINT attachments_staging_key_format CHECK (staging_key ~ '^s/[0-9a-f]{32}$'),
|
||||
ADD CONSTRAINT attachments_staging_state CHECK (staging_state IN ('none', 'present', 'deleting', 'deleted')),
|
||||
ADD CONSTRAINT attachments_staging_has_key CHECK (staging_state = 'none' OR staging_key IS NOT NULL),
|
||||
ADD CONSTRAINT attachments_published_version CHECK (published_version >= 1),
|
||||
ADD CONSTRAINT attachments_error_code CHECK (error_code ~ '^[a-z_]{1,64}$');
|
||||
|
||||
-- Lab 4 rows: a 'pending' intent of lab 4 never got its object confirmed and has no staging copy;
|
||||
-- it becomes a failed upload (invisible: its id was never returned). A lab 4 'ready' row has its
|
||||
-- original in the final bucket but no thumbnail yet: it is queued for pipeline 1 (below).
|
||||
UPDATE attachments SET status = 'failed', error_code = 'upload_incomplete', failed_at = clock_timestamp()
|
||||
WHERE status = 'pending';
|
||||
UPDATE attachments SET status = 'queued' WHERE status = 'ready';
|
||||
|
||||
-- Structural guarantees (the database refuses a buggy cleanup or worker):
|
||||
-- * work that is queued/processing always has a source: the published original or the staging
|
||||
-- copy - the cleanup can never mark the staging of an accepted upload as deleting;
|
||||
-- * ready = published original + published pipeline version; failed = error code.
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_pending_staging CHECK (status <> 'pending' OR staging_state = 'present'),
|
||||
ADD CONSTRAINT attachments_active_has_source
|
||||
CHECK (status NOT IN ('queued', 'processing') OR object_key IS NOT NULL OR staging_state = 'present'),
|
||||
ADD CONSTRAINT attachments_ready_published
|
||||
CHECK (status <> 'ready' OR (object_key IS NOT NULL AND published_version IS NOT NULL)),
|
||||
ADD CONSTRAINT attachments_failed_code
|
||||
CHECK ((status = 'failed') = (error_code IS NOT NULL AND failed_at IS NOT NULL));
|
||||
-- Janitor of the API: the lab 4 partial index on pending rows stays as is.
|
||||
-- Cleanup of the worker: staging copies that may be deleted (ready: at once; failed: after retention).
|
||||
CREATE INDEX attachments_staging_cleanup ON attachments (updated_at)
|
||||
WHERE staging_state IN ('present', 'deleting') AND status IN ('ready', 'failed');
|
||||
|
||||
-- ------------------------------------------------------------------ jobs
|
||||
-- One row = one logical piece of work: "build pipeline version V for attachment A".
|
||||
-- job_id never changes on redelivery; `attempt` counts claims by workers (managed attempts).
|
||||
CREATE TABLE jobs (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
pipeline_version integer NOT NULL,
|
||||
kind text NOT NULL, -- upload | replay | reprocess | migrate
|
||||
replay_of uuid REFERENCES jobs (id),
|
||||
state text NOT NULL DEFAULT 'queued',
|
||||
attempt integer NOT NULL DEFAULT 0,
|
||||
max_attempts integer NOT NULL,
|
||||
available_at timestamptz NOT NULL DEFAULT clock_timestamp(), -- backoff: not before
|
||||
lease_token uuid, -- token of the attempt that holds the job
|
||||
lease_owner text,
|
||||
lease_until timestamptz,
|
||||
error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
finished_at timestamptz,
|
||||
CONSTRAINT jobs_pipeline_version CHECK (pipeline_version >= 1),
|
||||
CONSTRAINT jobs_kind CHECK (kind IN ('upload', 'replay', 'reprocess', 'migrate')),
|
||||
CONSTRAINT jobs_replay CHECK ((kind = 'replay') = (replay_of IS NOT NULL)),
|
||||
CONSTRAINT jobs_state CHECK (state IN ('queued', 'processing', 'succeeded', 'failed', 'superseded')),
|
||||
CONSTRAINT jobs_attempts CHECK (attempt >= 0 AND max_attempts BETWEEN 1 AND 100 AND attempt <= max_attempts),
|
||||
-- A lease exists exactly while the job is processing.
|
||||
CONSTRAINT jobs_lease CHECK ((state = 'processing') = (lease_token IS NOT NULL AND lease_until IS NOT NULL)),
|
||||
CONSTRAINT jobs_finished CHECK ((state IN ('succeeded', 'failed', 'superseded')) = (finished_at IS NOT NULL)),
|
||||
CONSTRAINT jobs_failed_code CHECK (state <> 'failed' OR error_code IS NOT NULL),
|
||||
CONSTRAINT jobs_error_code CHECK (error_code ~ '^[a-z_]{1,64}$')
|
||||
);
|
||||
-- At most ONE active job per effect set (attachment, pipeline version): a second upload job,
|
||||
-- replay or reprocess of the same version cannot exist while one is queued/processing.
|
||||
CREATE UNIQUE INDEX jobs_one_active ON jobs (attachment_id, pipeline_version) WHERE state IN ('queued', 'processing');
|
||||
CREATE INDEX jobs_attachment ON jobs (attachment_id, created_at);
|
||||
CREATE INDEX jobs_lease_expiry ON jobs (lease_until) WHERE state = 'processing'; -- reaper
|
||||
CREATE INDEX jobs_queued ON jobs (updated_at) WHERE state = 'queued'; -- lost-message guard
|
||||
CREATE INDEX jobs_failed ON jobs (finished_at) WHERE state = 'failed'; -- failed registry
|
||||
|
||||
-- One row per claim of a job by a worker. The token is part of every object key the attempt
|
||||
-- writes, so the keys of an attempt are known without listing the bucket (planned_keys), and an
|
||||
-- attempt that is no longer the lease holder can never publish (fencing by token).
|
||||
CREATE TABLE job_attempts (
|
||||
token uuid PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES jobs (id),
|
||||
attempt integer NOT NULL,
|
||||
worker text NOT NULL,
|
||||
source text NOT NULL, -- staging | original
|
||||
planned_keys text[] NOT NULL,
|
||||
outcome text NOT NULL DEFAULT 'running',
|
||||
error_code text,
|
||||
started_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
finished_at timestamptz,
|
||||
objects_cleaned_at timestamptz,
|
||||
CONSTRAINT job_attempts_number UNIQUE (job_id, attempt),
|
||||
CONSTRAINT job_attempts_source CHECK (source IN ('staging', 'original')),
|
||||
CONSTRAINT job_attempts_outcome
|
||||
CHECK (outcome IN ('running', 'succeeded', 'retry', 'failed', 'abandoned', 'superseded')),
|
||||
CONSTRAINT job_attempts_finished CHECK ((outcome = 'running') = (finished_at IS NULL)),
|
||||
-- Objects of a successful attempt are published: they are never cleaned.
|
||||
CONSTRAINT job_attempts_cleaned CHECK (objects_cleaned_at IS NULL OR outcome NOT IN ('running', 'succeeded'))
|
||||
);
|
||||
CREATE INDEX job_attempts_cleanup ON job_attempts (finished_at)
|
||||
WHERE outcome IN ('retry', 'failed', 'abandoned', 'superseded') AND objects_cleaned_at IS NULL;
|
||||
|
||||
-- Published variants. The primary key is the idempotency key of the EFFECT from the contract:
|
||||
-- (attachment_id, pipeline_version, variant) exists at most once, whatever the deliveries.
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
pipeline_version integer NOT NULL,
|
||||
name text NOT NULL,
|
||||
object_key text COLLATE "C" NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes integer NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
sha256 bytea NOT NULL,
|
||||
attempt_token uuid NOT NULL REFERENCES job_attempts (token),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, name),
|
||||
CONSTRAINT attachment_variants_key UNIQUE (object_key),
|
||||
CONSTRAINT attachment_variants_key_format CHECK (object_key ~ '^a/[0-9a-f-]{36}/[0-9a-f]{32}/(thumbnail|crop|optimized)$'),
|
||||
CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')),
|
||||
CONSTRAINT attachment_variants_type CHECK (content_type IN ('image/jpeg', 'image/webp')),
|
||||
CONSTRAINT attachment_variants_size CHECK (size_bytes >= 1 AND width >= 1 AND height >= 1),
|
||||
CONSTRAINT attachment_variants_sha256 CHECK (octet_length(sha256) = 32)
|
||||
);
|
||||
|
||||
-- Transactional outbox: written in the SAME transaction as the job (upload, retry, replay,
|
||||
-- reprocess, lease expiry). The relay publishes due rows and marks them; a crash after the
|
||||
-- publish but before the mark publishes again (a duplicate the worker ignores).
|
||||
CREATE TABLE outbox (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES jobs (id),
|
||||
attempt integer NOT NULL, -- the attempt this delivery is meant for
|
||||
available_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
published_at timestamptz,
|
||||
publish_failures integer NOT NULL DEFAULT 0,
|
||||
CONSTRAINT outbox_attempt CHECK (attempt >= 1)
|
||||
);
|
||||
CREATE INDEX outbox_due ON outbox (available_at, id) WHERE published_at IS NULL;
|
||||
CREATE INDEX outbox_job ON outbox (job_id);
|
||||
CREATE INDEX outbox_published ON outbox (published_at) WHERE published_at IS NOT NULL; -- purge
|
||||
|
||||
-- Audit of every transition (no secrets, no file names): replay/reprocess requests with the
|
||||
-- operator and reason, claims, retries with their delay, lease expiries, results.
|
||||
CREATE TABLE job_events (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES jobs (id),
|
||||
at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
event text NOT NULL,
|
||||
attempt integer,
|
||||
actor text NOT NULL,
|
||||
error_code text,
|
||||
detail jsonb NOT NULL DEFAULT '{}'
|
||||
);
|
||||
CREATE INDEX job_events_job ON job_events (job_id, id);
|
||||
|
||||
-- Failed registry for the operator (jobctl list-failed).
|
||||
CREATE VIEW failed_jobs AS
|
||||
SELECT j.id AS job_id, j.attachment_id, j.pipeline_version, j.kind, j.attempt, j.max_attempts, j.error_code,
|
||||
j.finished_at, a.status AS attachment_status, a.staging_state
|
||||
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.state = 'failed';
|
||||
|
||||
-- Lab 4 'ready' rows (now queued): a job for pipeline 1 from their published original.
|
||||
WITH queued AS (
|
||||
SELECT a.id, gen_random_uuid() AS job_id FROM attachments a WHERE a.status = 'queued'
|
||||
), created AS (
|
||||
INSERT INTO jobs (id, attachment_id, pipeline_version, kind, max_attempts)
|
||||
SELECT q.job_id, q.id, 1, 'migrate', 5 FROM queued q RETURNING id
|
||||
)
|
||||
INSERT INTO outbox (job_id, attempt) SELECT c.id, 1 FROM created c;
|
||||
|
||||
-- post_message_v3 = post_message_v2 (0004) with the lab 5 attachment states: queued, processing
|
||||
-- and ready attachments of this chat may be attached; failed (or quarantined) -> 409; pending,
|
||||
-- foreign or unknown -> 404. Everything else (key claim, chat clock, one transaction) unchanged.
|
||||
CREATE FUNCTION post_message_v3(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid,
|
||||
p_text bytea, p_ttl_seconds integer, p_attachments uuid[])
|
||||
RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz, o_attachments uuid[])
|
||||
LANGUAGE plpgsql VOLATILE AS $$
|
||||
DECLARE
|
||||
v_created timestamptz;
|
||||
v_found integer;
|
||||
v_usable integer;
|
||||
v_check text := 'ok';
|
||||
BEGIN
|
||||
PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user;
|
||||
IF NOT FOUND THEN
|
||||
RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF cardinality(p_attachments) > 0 THEN
|
||||
-- FOR SHARE: a concurrent transition to failed/quarantine waits until this message commits.
|
||||
SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL AND a.status <> 'failed') INTO v_found, v_usable
|
||||
FROM (SELECT a.lost_at, a.status FROM attachments a
|
||||
WHERE a.id = ANY (p_attachments) AND a.chat_id = p_chat AND a.status <> 'pending'
|
||||
FOR SHARE) a;
|
||||
IF v_found <> cardinality(p_attachments) THEN
|
||||
v_check := 'attachment_not_found';
|
||||
ELSIF v_usable <> v_found THEN
|
||||
v_check := 'attachment_conflict';
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
IF v_check = 'ok' THEN
|
||||
INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id)
|
||||
VALUES (p_user, p_chat, p_key, p_hash, p_message)
|
||||
ON CONFLICT (user_id, chat_id, key) DO UPDATE
|
||||
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp()
|
||||
WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds);
|
||||
IF FOUND THEN
|
||||
UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond')
|
||||
WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created;
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
|
||||
VALUES (p_message, p_chat, p_user, p_text, v_created, p_attachments);
|
||||
INSERT INTO message_attachments (message_id, position, attachment_id)
|
||||
SELECT p_message, (a.ord - 1)::smallint, a.id
|
||||
FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord);
|
||||
RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created, p_attachments;
|
||||
RETURN;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
RETURN QUERY
|
||||
SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at,
|
||||
m.attachment_ids
|
||||
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
|
||||
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key;
|
||||
IF NOT FOUND THEN
|
||||
RETURN QUERY SELECT CASE WHEN v_check = 'ok' THEN 'retry' ELSE v_check END,
|
||||
NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
+5
-1
@@ -16,5 +16,9 @@ redis==8.1.0
|
||||
argon2-cffi==25.1.0
|
||||
# Lab 4: S3 client (async botocore over aiohttp; requests are cancellable, unlike libpq cancel).
|
||||
aiobotocore==3.9.2
|
||||
# Images are checked without Pillow: PNG by src/messenger/images.py, JPEG by libjpeg-turbo
|
||||
# Lab 5: AMQP 0-9-1 client for RabbitMQ (publisher confirms, consumer acks, prefetch); last 9.x line.
|
||||
aio-pika==9.6.2
|
||||
# Lab 5: image variants in the worker only (thumbnail/crop JPEG, WebP). Same version as the tests.
|
||||
pillow==12.3.0
|
||||
# Uploads are still checked without Pillow: PNG by src/messenger/images.py, JPEG by libjpeg-turbo
|
||||
# `djpeg -strict` (Debian package libjpeg-turbo-progs, see Dockerfile).
|
||||
|
||||
+105
-1
@@ -4,6 +4,10 @@
|
||||
#
|
||||
# pip-compile --generate-hashes --no-emit-index-url --no-index --output-file=requirements.txt --strip-extras requirements.in
|
||||
#
|
||||
aio-pika==9.6.2 \
|
||||
--hash=sha256:2a5478af920d169795071c9c09c7542cd8cdece60438cf7804533dcbcce93b7f \
|
||||
--hash=sha256:c49e9246080dc8ffa1bb0e4aca407bf3d8ad78c3ee3a93df88b68fe65d7a49b9
|
||||
# via -r requirements.in
|
||||
aiobotocore==3.9.2 \
|
||||
--hash=sha256:363b4892423b272eb84afbb58000da0be6ccd0cf8a68b98438c957d05b841efc \
|
||||
--hash=sha256:8e32238c5bd77717ab1ac90077ff51ab141d025528115ccc7cc9b50b2d176e5f
|
||||
@@ -179,6 +183,10 @@ aioitertools==0.13.0 \
|
||||
--hash=sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be \
|
||||
--hash=sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c
|
||||
# via aiobotocore
|
||||
aiormq==6.9.4 \
|
||||
--hash=sha256:0e7c01b662804e1cc7ace9a17794e8c1192a27fc2afa96162362a6e61ae8e8ef \
|
||||
--hash=sha256:726a8586695e863fba68cf88842065ab12348c9438dcebdfc9d0bddaf6083277
|
||||
# via aio-pika
|
||||
aiosignal==1.4.0 \
|
||||
--hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \
|
||||
--hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7
|
||||
@@ -794,6 +802,99 @@ multidict==6.9.1 \
|
||||
# aiobotocore
|
||||
# aiohttp
|
||||
# yarl
|
||||
pamqp==3.3.0 \
|
||||
--hash=sha256:40b8795bd4efcf2b0f8821c1de83d12ca16d5760f4507836267fd7a02b06763b \
|
||||
--hash=sha256:c901a684794157ae39b52cbf700db8c9aae7a470f13528b9d7b4e5f7202f8eb0
|
||||
# via aiormq
|
||||
pillow==12.3.0 \
|
||||
--hash=sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756 \
|
||||
--hash=sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a \
|
||||
--hash=sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59 \
|
||||
--hash=sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45 \
|
||||
--hash=sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3 \
|
||||
--hash=sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df \
|
||||
--hash=sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139 \
|
||||
--hash=sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b \
|
||||
--hash=sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39 \
|
||||
--hash=sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e \
|
||||
--hash=sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8 \
|
||||
--hash=sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1 \
|
||||
--hash=sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8 \
|
||||
--hash=sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89 \
|
||||
--hash=sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5 \
|
||||
--hash=sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130 \
|
||||
--hash=sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd \
|
||||
--hash=sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d \
|
||||
--hash=sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b \
|
||||
--hash=sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed \
|
||||
--hash=sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace \
|
||||
--hash=sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb \
|
||||
--hash=sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931 \
|
||||
--hash=sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510 \
|
||||
--hash=sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6 \
|
||||
--hash=sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1 \
|
||||
--hash=sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce \
|
||||
--hash=sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385 \
|
||||
--hash=sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e \
|
||||
--hash=sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c \
|
||||
--hash=sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7 \
|
||||
--hash=sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace \
|
||||
--hash=sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c \
|
||||
--hash=sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f \
|
||||
--hash=sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64 \
|
||||
--hash=sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f \
|
||||
--hash=sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a \
|
||||
--hash=sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827 \
|
||||
--hash=sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17 \
|
||||
--hash=sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4 \
|
||||
--hash=sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a \
|
||||
--hash=sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701 \
|
||||
--hash=sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e \
|
||||
--hash=sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91 \
|
||||
--hash=sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66 \
|
||||
--hash=sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468 \
|
||||
--hash=sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217 \
|
||||
--hash=sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658 \
|
||||
--hash=sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418 \
|
||||
--hash=sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a \
|
||||
--hash=sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c \
|
||||
--hash=sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330 \
|
||||
--hash=sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402 \
|
||||
--hash=sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09 \
|
||||
--hash=sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930 \
|
||||
--hash=sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f \
|
||||
--hash=sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec \
|
||||
--hash=sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a \
|
||||
--hash=sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94 \
|
||||
--hash=sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468 \
|
||||
--hash=sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b \
|
||||
--hash=sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965 \
|
||||
--hash=sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8 \
|
||||
--hash=sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd \
|
||||
--hash=sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7 \
|
||||
--hash=sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c \
|
||||
--hash=sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777 \
|
||||
--hash=sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35 \
|
||||
--hash=sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9 \
|
||||
--hash=sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f \
|
||||
--hash=sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f \
|
||||
--hash=sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0 \
|
||||
--hash=sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c \
|
||||
--hash=sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71 \
|
||||
--hash=sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3 \
|
||||
--hash=sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838 \
|
||||
--hash=sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf \
|
||||
--hash=sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321 \
|
||||
--hash=sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26 \
|
||||
--hash=sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec \
|
||||
--hash=sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9 \
|
||||
--hash=sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65 \
|
||||
--hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \
|
||||
--hash=sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e \
|
||||
--hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \
|
||||
--hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 \
|
||||
--hash=sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7
|
||||
# via -r requirements.in
|
||||
propcache==0.5.4 \
|
||||
--hash=sha256:004e685b315646c410771836e72a44f143bbe624f29653a42687815069a303d5 \
|
||||
--hash=sha256:02c0a34f16889cf800f10f0247a564d8ce6eeab6ffcd7c87198f769067eb8432 \
|
||||
@@ -1353,4 +1454,7 @@ yarl==1.25.1 \
|
||||
--hash=sha256:f53dcd26694f148f738edc052b5a69234833e739f10f4c3287bdfd8ec0f7b326 \
|
||||
--hash=sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc \
|
||||
--hash=sha256:fe01645169a2112aa1d4ebc3e4c5f029c5c8f97adfc32e5d37c993b39a994d75
|
||||
# via aiohttp
|
||||
# via
|
||||
# aio-pika
|
||||
# aiohttp
|
||||
# aiormq
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""RabbitMQ (AMQP 0-9-1) for the attachment jobs: topology names and the message format.
|
||||
|
||||
Topology (declared by the broker itself from its definitions file at boot, infra/rabbitmq and
|
||||
scripts/init_secrets.py; the relay and the worker have NO configure permission and declare nothing):
|
||||
|
||||
exchange messenger.jobs direct, durable
|
||||
queue messenger.attachment-jobs quorum (Raft log on disk), bound with routing key
|
||||
attachment.process.v1; x-delivery-limit and dead-lettering to
|
||||
exchange messenger.jobs.dlx fanout -> queue messenger.attachment-jobs.dlq (quorum)
|
||||
|
||||
Publisher side (relay): persistent message, `mandatory` (an unroutable message is returned, not
|
||||
silently dropped), channel in confirm mode - `publish()` returns only after the broker has
|
||||
written the message to the quorum log (publisher confirm). Consumer side (worker): manual ack
|
||||
after the durable effect, prefetch = WORKER_CONCURRENCY.
|
||||
|
||||
The message carries IDs and service fields only (contracts/jobs.schema.json): no bytes, no URL,
|
||||
no S3 key, no filename, no credentials. The worker takes every trusted detail from PostgreSQL.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
EXCHANGE = "messenger.jobs"
|
||||
QUEUE = "messenger.attachment-jobs"
|
||||
DLX = "messenger.jobs.dlx"
|
||||
DLQ = "messenger.attachment-jobs.dlq"
|
||||
ROUTING_KEY = "attachment.process.v1"
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
MAX_MESSAGE_BYTES = 4096
|
||||
|
||||
REQUIRED = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
|
||||
OPTIONAL = {"traceparent", "tracestate"}
|
||||
TRACEPARENT_RE = re.compile(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}")
|
||||
DATETIME_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?Z")
|
||||
|
||||
|
||||
class BadMessage(ValueError):
|
||||
"""Not a valid job message (schema v1): dead-lettered, never processed."""
|
||||
|
||||
|
||||
def rfc3339(value: dt.datetime) -> str:
|
||||
return value.astimezone(dt.timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def encode(job_id: str, attachment_id: str, pipeline_version: int, attempt: int, created_at: dt.datetime) -> bytes:
|
||||
body = {"schema_version": 1, "job_id": job_id, "type": JOB_TYPE, "attachment_id": attachment_id,
|
||||
"pipeline_version": pipeline_version, "attempt": attempt, "created_at": rfc3339(created_at)}
|
||||
return json.dumps(body, separators=(",", ":")).encode()
|
||||
|
||||
|
||||
def decode(raw: bytes) -> dict:
|
||||
"""Strict check against contracts/jobs.schema.json (additionalProperties: false)."""
|
||||
if len(raw) > MAX_MESSAGE_BYTES:
|
||||
raise BadMessage("too large")
|
||||
try:
|
||||
body = json.loads(raw)
|
||||
except (ValueError, UnicodeError):
|
||||
raise BadMessage("not JSON") from None
|
||||
if not isinstance(body, dict):
|
||||
raise BadMessage("not an object")
|
||||
keys = set(body)
|
||||
if not REQUIRED <= keys or keys - REQUIRED - OPTIONAL:
|
||||
raise BadMessage("wrong fields")
|
||||
if body["schema_version"] != 1 or isinstance(body["schema_version"], bool) or body["type"] != JOB_TYPE:
|
||||
raise BadMessage("unsupported schema/type")
|
||||
for name in ("job_id", "attachment_id"):
|
||||
if not isinstance(body[name], str):
|
||||
raise BadMessage(name)
|
||||
try:
|
||||
uuid.UUID(body[name])
|
||||
except ValueError:
|
||||
raise BadMessage(name) from None
|
||||
for name in ("pipeline_version", "attempt"):
|
||||
value = body[name]
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
|
||||
raise BadMessage(name)
|
||||
if not isinstance(body["created_at"], str) or not DATETIME_RE.fullmatch(body["created_at"]):
|
||||
raise BadMessage("created_at")
|
||||
if "traceparent" in body and not (isinstance(body["traceparent"], str) and TRACEPARENT_RE.fullmatch(body["traceparent"])):
|
||||
raise BadMessage("traceparent")
|
||||
if "tracestate" in body and not (isinstance(body["tracestate"], str) and len(body["tracestate"]) <= 512):
|
||||
raise BadMessage("tracestate")
|
||||
return body
|
||||
+119
-12
@@ -10,7 +10,7 @@ from __future__ import annotations
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@@ -134,7 +134,29 @@ class Settings:
|
||||
jpeg_max_memory_mb: int = 256
|
||||
pending_grace_seconds: int = 900
|
||||
janitor_interval_seconds: float = 60.0
|
||||
# Lab 5: staging bucket (API writes, worker reads) and the asynchronous pipeline (jobs_settings()).
|
||||
s3_staging_bucket: str = "messenger-staging"
|
||||
broker_url: str = ""
|
||||
pipeline_version: int = 1
|
||||
job_max_attempts: int = 5
|
||||
job_timeout_seconds: float = 60.0
|
||||
job_lease_seconds: float = 30.0
|
||||
job_retry_base_seconds: float = 2.0
|
||||
job_retry_max_seconds: float = 60.0
|
||||
worker_concurrency: int = 2
|
||||
worker_shutdown_seconds: float = 20.0
|
||||
image_max_memory_mb: int = 768
|
||||
staging_retention_seconds: int = 86400
|
||||
orphan_grace_seconds: int = 600
|
||||
cleanup_interval_seconds: float = 30.0
|
||||
relay_poll_seconds: float = 0.5
|
||||
relay_batch: int = 50
|
||||
queued_republish_seconds: float = 300.0
|
||||
fault_points: frozenset[str] = frozenset()
|
||||
fault_values: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
def fault_value(self, point: str, default: int = 0) -> int:
|
||||
return self.fault_values.get(point, default)
|
||||
|
||||
@property
|
||||
def db_statement_timeout_ms(self) -> int:
|
||||
@@ -142,7 +164,8 @@ class Settings:
|
||||
return max(100, int(self.db_timeout_seconds * 1000) - 500)
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "Settings":
|
||||
def from_env(cls, *, s3_credentials: bool = True) -> "Settings":
|
||||
"""s3_credentials=False: processes without S3 access (relay, jobctl)."""
|
||||
log_level = os.environ.get("LOG_LEVEL", "info").strip().lower() or "info"
|
||||
if log_level not in {"debug", "info", "warning", "error"}:
|
||||
raise ConfigError(f"LOG_LEVEL must be debug/info/warning/error, got {log_level!r}")
|
||||
@@ -185,32 +208,71 @@ class Settings:
|
||||
# Track 1: `function` = one SQL call per send (migration 0003), `multi` = the previous
|
||||
# multi-statement transaction; both have identical semantics (A/B on one image).
|
||||
message_write_path=_choice("MESSAGE_WRITE_PATH", "function", {"function", "multi"}),
|
||||
**s3_settings(),
|
||||
**s3_settings(credentials_required=s3_credentials),
|
||||
**jobs_settings(broker_required=False),
|
||||
)
|
||||
|
||||
|
||||
# Contract limits (contracts/README.md): configurable DOWN for experiments, never above them.
|
||||
CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024
|
||||
CONTRACT_MAX_IMAGE_PIXELS = 40_000_000
|
||||
# Failure points for the level 4 experiments only (compose.faults.yaml); empty in normal runs.
|
||||
FAULT_POINTS = {"after_s3_put", "after_ready_commit"}
|
||||
# Failure points for the experiments only (FAULT_POINTS of ONE service, compose variables
|
||||
# API_/RELAY_/WORKER_FAULT_POINTS); empty in normal runs. `name` or `name=<integer>`.
|
||||
FAULT_POINTS = {
|
||||
# API, upload: between the staging PUT and the job COMMIT / after the job COMMIT, before 202.
|
||||
"after_s3_put", "after_enqueue_commit",
|
||||
# Relay: crash after the broker confirmed a publish but before outbox.published_at is
|
||||
# committed; publish every message twice; publish a batch in reverse order.
|
||||
"crash_after_publish", "duplicate_publish", "reverse_order",
|
||||
# Worker: crash after the claim (before any side effect), after the S3 writes (before the
|
||||
# result COMMIT), after the result COMMIT (before the ack); transient failure of the first N
|
||||
# attempts; permanent failure; slow processing (N s, lease kept alive) of every job or only of
|
||||
# pipeline version `slow_pipeline`; hang: stop the heartbeat and sleep N s inside processing.
|
||||
"crash_after_claim", "crash_after_s3_write", "crash_after_commit", "transient_error", "permanent_error",
|
||||
"slow", "slow_pipeline", "hang",
|
||||
# Worker cleanup: pause N s after reading the candidates, before locking/deleting them.
|
||||
"pause_cleanup",
|
||||
}
|
||||
|
||||
|
||||
def _faults() -> tuple[frozenset[str], dict[str, int]]:
|
||||
names, values = set(), {}
|
||||
for item in os.environ.get("FAULT_POINTS", "").split(","):
|
||||
name, _, raw = item.strip().partition("=")
|
||||
if not name:
|
||||
continue
|
||||
if name not in FAULT_POINTS:
|
||||
raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}")
|
||||
if raw:
|
||||
if not raw.isdigit():
|
||||
raise ConfigError(f"FAULT_POINTS value of {name} must be an integer")
|
||||
values[name] = int(raw)
|
||||
names.add(name)
|
||||
return frozenset(names), values
|
||||
|
||||
|
||||
def _bucket(name: str, default: str) -> str:
|
||||
bucket = os.environ.get(name, default).strip() or default
|
||||
if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket):
|
||||
raise ConfigError(f"{name} is not a valid bucket name: {bucket!r}")
|
||||
return bucket
|
||||
|
||||
|
||||
def s3_settings(*, credentials_required: bool = True) -> dict:
|
||||
"""S3 part of Settings; also used by the reconciler/bootstrap CLIs with their own keys."""
|
||||
faults = frozenset(f.strip() for f in os.environ.get("FAULT_POINTS", "").split(",") if f.strip())
|
||||
if faults - FAULT_POINTS:
|
||||
raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}")
|
||||
"""S3 part of Settings; also used by the bootstrap CLI with its own keys."""
|
||||
faults, fault_values = _faults()
|
||||
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").strip() or "http://s3:8333"
|
||||
if not re.fullmatch(r"https?://[A-Za-z0-9.\-]+(:[0-9]{1,5})?/?", endpoint):
|
||||
raise ConfigError(f"S3_ENDPOINT must be http(s)://host[:port], got {endpoint!r}")
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments").strip() or "messenger-attachments"
|
||||
if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket):
|
||||
raise ConfigError(f"S3_BUCKET is not a valid bucket name: {bucket!r}")
|
||||
bucket = _bucket("S3_BUCKET", "messenger-attachments")
|
||||
staging = _bucket("S3_STAGING_BUCKET", "messenger-staging")
|
||||
if staging == bucket:
|
||||
raise ConfigError("S3_STAGING_BUCKET must differ from S3_BUCKET")
|
||||
return dict(
|
||||
s3_endpoint=endpoint,
|
||||
s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1",
|
||||
s3_bucket=bucket,
|
||||
s3_staging_bucket=staging,
|
||||
s3_access_key_id=_secret("S3_ACCESS_KEY_ID", required=credentials_required),
|
||||
s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY", required=credentials_required),
|
||||
s3_timeout_seconds=_float("S3_TIMEOUT_SECONDS", 3.0, 0.2, 30.0),
|
||||
@@ -233,4 +295,49 @@ def s3_settings(*, credentials_required: bool = True) -> dict:
|
||||
pending_grace_seconds=_int("PENDING_GRACE_SECONDS", 900, 1, 7 * 24 * 3600),
|
||||
janitor_interval_seconds=_float("JANITOR_INTERVAL_SECONDS", 60.0, 0.0, 3600.0),
|
||||
fault_points=faults,
|
||||
fault_values=fault_values,
|
||||
)
|
||||
|
||||
|
||||
def jobs_settings(*, broker_required: bool) -> dict:
|
||||
"""Lab 5 pipeline: retries, lease, worker limits, cleanup, relay (docs/PIPELINE.md)."""
|
||||
from .pipeline import PIPELINES # local import: pipeline.py imports nothing from here
|
||||
timeout = _float("JOB_TIMEOUT_SECONDS", 60.0, 1.0, 600.0)
|
||||
lease = _float("JOB_LEASE_SECONDS", 30.0, 2.0, 600.0)
|
||||
base = _float("JOB_RETRY_BASE_SECONDS", 2.0, 0.0, 600.0)
|
||||
cap = _float("JOB_RETRY_MAX_SECONDS", 60.0, 0.0, 3600.0)
|
||||
if base > cap:
|
||||
raise ConfigError("JOB_RETRY_BASE_SECONDS must not exceed JOB_RETRY_MAX_SECONDS")
|
||||
orphan_grace = _int("ORPHAN_GRACE_SECONDS", 600, 1, 7 * 24 * 3600)
|
||||
version = _int("PIPELINE_VERSION", 1, 1, 1000)
|
||||
if version not in PIPELINES:
|
||||
raise ConfigError(f"PIPELINE_VERSION must be one of {sorted(PIPELINES)}")
|
||||
return dict(
|
||||
broker_url=_secret("BROKER_URL", required=broker_required),
|
||||
# Pipeline version of the jobs created by new uploads (track 2: versioned pipeline).
|
||||
pipeline_version=version,
|
||||
# Managed attempts (claims) per job before failed/retry_exhausted.
|
||||
job_max_attempts=_int("JOB_MAX_ATTEMPTS", 5, 1, 100),
|
||||
# Hard limit of ONE attempt: the image child process is killed, the lease is no longer renewed.
|
||||
job_timeout_seconds=timeout,
|
||||
# Lease of a claimed job, renewed by the heartbeat every lease/3 while the attempt runs.
|
||||
job_lease_seconds=lease,
|
||||
# Backoff of transient failures: full jitter, uniform(0, min(max, base * 2^(attempt-1))).
|
||||
job_retry_base_seconds=base,
|
||||
job_retry_max_seconds=cap,
|
||||
# Jobs processed at once by ONE worker process = RabbitMQ prefetch (basic.qos).
|
||||
worker_concurrency=_int("WORKER_CONCURRENCY", 2, 1, 32),
|
||||
worker_shutdown_seconds=_float("WORKER_SHUTDOWN_SECONDS", 20.0, 0.0, 120.0),
|
||||
# Address-space limit (RLIMIT_AS) of the child process that decodes/encodes ONE image.
|
||||
image_max_memory_mb=_int("IMAGE_MAX_MEMORY_MB", 768, 128, 8192),
|
||||
# Staging copy of a FAILED attachment is kept this long for a replay, then deleted.
|
||||
staging_retention_seconds=_int("STAGING_RETENTION_SECONDS", 86400, 0, 30 * 24 * 3600),
|
||||
# Objects of attempts that did not publish are deleted once the attempt ended this long ago.
|
||||
orphan_grace_seconds=orphan_grace,
|
||||
cleanup_interval_seconds=_float("CLEANUP_INTERVAL_SECONDS", 30.0, 0.0, 3600.0),
|
||||
relay_poll_seconds=_float("RELAY_POLL_SECONDS", 0.5, 0.05, 60.0),
|
||||
relay_batch=_int("RELAY_BATCH", 50, 1, 1000),
|
||||
# A queued job whose message has not led to a claim for this long is published again
|
||||
# (message lost in the broker, dead-lettered, purged): no accepted job waits forever.
|
||||
queued_republish_seconds=_float("QUEUED_REPUBLISH_SECONDS", 300.0, 5.0, 86400.0),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Variants of ONE image, built in a separate child process with hard limits (lab 5 worker).
|
||||
|
||||
parent (worker): await render(data, content_type, version, ...) -> list[Rendered]
|
||||
child: python -m messenger.imaging --type image/png --pipeline 1 ...
|
||||
stdin: original bytes; stdout: one JSON header line + variant bytes
|
||||
|
||||
Why a child process per image:
|
||||
* memory: RLIMIT_AS (IMAGE_MAX_MEMORY_MB) caps the address space of the decoder; a picture that
|
||||
needs more fails with MemoryError -> `image_too_large` (permanent), the worker keeps running;
|
||||
* time: the parent kills the child at JOB_TIMEOUT_SECONDS (C code of Pillow cannot be
|
||||
interrupted inside a thread) -> `processing_timeout` (transient, bounded by JOB_MAX_ATTEMPTS);
|
||||
* pixels: the size is read from the header and checked against MAX_IMAGE_PIXELS BEFORE the
|
||||
pixels are decoded (`Image.open` is lazy).
|
||||
A crash of the decoder (signal) is a transient `processing_killed`: the next attempt may run
|
||||
with less memory pressure; JOB_MAX_ATTEMPTS bounds it.
|
||||
|
||||
Geometry and encoder settings: pipeline.py. Alpha is handled in premultiplied form (RGBa) while
|
||||
resizing, so transparent pixels do not bleed their colour into the edges; for JPEG the result is
|
||||
then composited onto white.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import warnings
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .pipeline import CONTENT_TYPES, CROP_SIDE, OPTIMIZED_BOX, PIPELINES, THUMBNAIL_BOX, center_square, fit
|
||||
|
||||
EXIT_INVALID = 3 # the bytes are not a decodable PNG/JPEG
|
||||
EXIT_TOO_LARGE = 4 # pixel limit or memory limit
|
||||
HEADER_LIMIT = 4096
|
||||
|
||||
|
||||
class ProcessingError(Exception):
|
||||
"""Outcome of an attempt that did not produce variants. `permanent`: another attempt with the
|
||||
same input cannot succeed (job -> failed at once); otherwise retry with backoff."""
|
||||
|
||||
def __init__(self, code: str, permanent: bool):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
self.permanent = permanent
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Rendered:
|
||||
name: str
|
||||
content_type: str
|
||||
width: int
|
||||
height: int
|
||||
data: bytes
|
||||
|
||||
@property
|
||||
def sha256(self) -> bytes:
|
||||
return hashlib.sha256(self.data).digest()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ parent side
|
||||
async def render(data: bytes, content_type: str, pipeline_version: int, *, max_pixels: int, timeout: float,
|
||||
memory_mb: int) -> list[Rendered]:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "-m", "messenger.imaging", "--type", content_type, "--pipeline", str(pipeline_version),
|
||||
"--max-pixels", str(max_pixels), "--memory-mb", str(memory_mb),
|
||||
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
|
||||
try:
|
||||
out, _ = await asyncio.wait_for(process.communicate(data), timeout)
|
||||
except asyncio.TimeoutError:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
raise ProcessingError("processing_timeout", permanent=False) from None
|
||||
except asyncio.CancelledError:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
raise
|
||||
code = process.returncode
|
||||
if code == EXIT_INVALID:
|
||||
raise ProcessingError("invalid_image", permanent=True)
|
||||
if code == EXIT_TOO_LARGE:
|
||||
raise ProcessingError("image_too_large", permanent=True)
|
||||
if code is None or code < 0:
|
||||
raise ProcessingError("processing_killed", permanent=False) # OOM killer or another signal
|
||||
if code != 0:
|
||||
raise ProcessingError("processing_failed", permanent=False)
|
||||
return parse_output(out)
|
||||
|
||||
|
||||
def parse_output(out: bytes) -> list[Rendered]:
|
||||
newline = out.find(b"\n", 0, HEADER_LIMIT)
|
||||
if newline < 0:
|
||||
raise ProcessingError("processing_failed", permanent=False)
|
||||
header = json.loads(out[:newline])
|
||||
pos = newline + 1
|
||||
result = []
|
||||
for item in header["variants"]:
|
||||
size = int(item["size"])
|
||||
result.append(Rendered(item["name"], item["content_type"], int(item["width"]), int(item["height"]),
|
||||
out[pos:pos + size]))
|
||||
pos += size
|
||||
if pos != len(out) or [r.name for r in result] != list(CONTENT_TYPES):
|
||||
raise ProcessingError("processing_failed", permanent=False)
|
||||
return result
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ child side
|
||||
def build(data: bytes, content_type: str, pipeline_version: int, max_pixels: int) -> list[Rendered]:
|
||||
"""Decode once, build every variant. Raises the Pillow/OS errors of a bad file."""
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
pipeline = PIPELINES[pipeline_version]
|
||||
warnings.simplefilter("ignore", Image.DecompressionBombWarning) # our own pixel limit below
|
||||
Image.MAX_IMAGE_PIXELS = max_pixels
|
||||
expected = {"image/png": "PNG", "image/jpeg": "JPEG"}[content_type]
|
||||
img = Image.open(io.BytesIO(data), formats=[expected]) # header only: nothing decoded yet
|
||||
width, height = img.size
|
||||
if width * height > max_pixels:
|
||||
raise TooLarge()
|
||||
if expected == "JPEG":
|
||||
# DCT scaling: decode a big JPEG directly at 1/2..1/8 of its size, never below what the
|
||||
# largest variant needs (the box stays square, so EXIF rotation does not matter).
|
||||
img.draft("RGB", (OPTIMIZED_BOX, OPTIMIZED_BOX))
|
||||
img.load()
|
||||
img = ImageOps.exif_transpose(img) # orientation goes into the pixels, the tag is dropped
|
||||
img = _normalize(img)
|
||||
alpha = img.mode == "RGBA"
|
||||
base = img.convert("RGBa") if alpha else img # premultiplied alpha for resampling
|
||||
del img
|
||||
|
||||
def finish(im, *, flatten: bool):
|
||||
if alpha:
|
||||
im = im.convert("RGBA")
|
||||
if flatten:
|
||||
white = Image.new("RGB", im.size, (255, 255, 255))
|
||||
white.paste(im, mask=im.getchannel("A"))
|
||||
im = white
|
||||
im.info = {} # no EXIF/XMP/ICC/comments of the source in a variant
|
||||
return im
|
||||
|
||||
def resized(im, size):
|
||||
return im if im.size == size else im.resize(size, Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||
|
||||
thumbnail = finish(resized(base, fit(*base.size, THUMBNAIL_BOX)), flatten=True)
|
||||
crop = finish(resized(base.crop(center_square(*base.size)), (CROP_SIDE, CROP_SIDE)), flatten=True)
|
||||
optimized = finish(resized(base, fit(*base.size, OPTIMIZED_BOX)), flatten=False)
|
||||
out = []
|
||||
for name, im in (("thumbnail", thumbnail), ("crop", crop), ("optimized", optimized)):
|
||||
buf = io.BytesIO()
|
||||
if CONTENT_TYPES[name] == "image/jpeg":
|
||||
im.save(buf, "JPEG", quality=pipeline.jpeg_quality, optimize=True, subsampling=2, progressive=False)
|
||||
else:
|
||||
im.save(buf, "WEBP", quality=pipeline.webp_quality, method=pipeline.webp_method, exif=b"",
|
||||
icc_profile="", xmp=b"")
|
||||
out.append(Rendered(name, CONTENT_TYPES[name], im.width, im.height, buf.getvalue()))
|
||||
return out
|
||||
|
||||
|
||||
class TooLarge(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _normalize(img):
|
||||
"""Any PNG/JPEG mode -> RGB, or RGBA when the image has transparency."""
|
||||
if img.mode.startswith("I;16") or img.mode == "I":
|
||||
# 16-bit grey: scale 0..65535 to 0..255 (a plain convert("L") would clip to white).
|
||||
img = img.convert("I").point(lambda v: v * (1 / 257)).convert("L")
|
||||
has_alpha = img.mode in ("RGBA", "LA", "PA", "La", "RGBa") or "transparency" in img.info
|
||||
return img.convert("RGBA" if has_alpha else "RGB") if img.mode != ("RGBA" if has_alpha else "RGB") else img
|
||||
|
||||
|
||||
def child_main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--type", required=True, choices=["image/png", "image/jpeg"])
|
||||
parser.add_argument("--pipeline", type=int, required=True, choices=sorted(PIPELINES))
|
||||
parser.add_argument("--max-pixels", type=int, required=True)
|
||||
parser.add_argument("--memory-mb", type=int, required=True)
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
import resource
|
||||
limit = args.memory_mb * 1024 * 1024
|
||||
resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
|
||||
except (ImportError, ValueError, OSError):
|
||||
pass # not Linux (unit tests on Windows): the container memory limit still applies
|
||||
data = sys.stdin.buffer.read()
|
||||
try:
|
||||
variants = build(data, args.type, args.pipeline, args.max_pixels)
|
||||
except (TooLarge, MemoryError):
|
||||
return EXIT_TOO_LARGE
|
||||
except Exception as exc: # noqa: BLE001 - every decoder error of the CONTENT is "invalid image"
|
||||
from PIL import Image
|
||||
if isinstance(exc, Image.DecompressionBombError):
|
||||
return EXIT_TOO_LARGE
|
||||
if isinstance(exc, (OSError, SyntaxError, ValueError, EOFError, KeyError, IndexError, TypeError)):
|
||||
return EXIT_INVALID
|
||||
raise
|
||||
header = {"variants": [{"name": v.name, "content_type": v.content_type, "width": v.width, "height": v.height,
|
||||
"size": len(v.data)} for v in variants]}
|
||||
sys.stdout.buffer.write(json.dumps(header).encode() + b"\n")
|
||||
for v in variants:
|
||||
sys.stdout.buffer.write(v.data)
|
||||
sys.stdout.buffer.flush()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(child_main())
|
||||
@@ -0,0 +1,398 @@
|
||||
"""State machine of attachment jobs in PostgreSQL (lab 5). Every transition is ONE transaction.
|
||||
|
||||
Lock order, everywhere both rows are locked: the jobs row FIRST, then the attachments row
|
||||
(claim, complete, retry, lease expiry). Replay/reprocess/cleanup lock only the attachments row
|
||||
and insert NEW job rows. So two transitions never wait on each other in opposite order.
|
||||
|
||||
Fencing: a claim gives the attempt a fresh random `lease_token`. Every later write of that attempt
|
||||
(heartbeat, result, retry) is conditional on `jobs.lease_token = <its token>` under the jobs row
|
||||
lock. When the lease expires, the reaper clears the token; from then on the old attempt - even if
|
||||
its process is still alive (paused, slow) - can neither publish nor schedule anything, and the
|
||||
objects it wrote under its own keys are garbage for the cleanup.
|
||||
|
||||
Monotonic publication: a result of pipeline version V is published only if V is greater than the
|
||||
published version of the attachment; otherwise the job ends as `superseded` and nothing visible
|
||||
changes. `ready` never goes back to queued/processing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import random
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
import psycopg
|
||||
|
||||
from .pipeline import VARIANT_NAMES
|
||||
from .storage import attempt_key
|
||||
|
||||
ACTIVE = ("queued", "processing")
|
||||
|
||||
|
||||
def backoff_delay(attempt: int, base: float, cap: float, rnd=random.random) -> float:
|
||||
"""Full jitter (AWS Architecture Blog, "Exponential Backoff And Jitter"): uniform in
|
||||
[0, min(cap, base * 2^(attempt-1))]. Spreads the retries of many jobs that failed together."""
|
||||
return rnd() * min(cap, base * (2 ** max(0, attempt - 1)))
|
||||
|
||||
|
||||
async def event(conn: psycopg.AsyncConnection, job_id, name: str, actor: str, *, attempt: int | None = None,
|
||||
error_code: str | None = None, **detail) -> None:
|
||||
await conn.execute(
|
||||
"INSERT INTO job_events (job_id, event, attempt, actor, error_code, detail) VALUES (%s, %s, %s, %s, %s, %s)",
|
||||
(job_id, name, attempt, actor, error_code, json.dumps(detail, default=str)))
|
||||
|
||||
|
||||
async def create_job(conn: psycopg.AsyncConnection, attachment_id, pipeline_version: int, kind: str,
|
||||
max_attempts: int, actor: str, *, replay_of=None, **detail) -> uuid.UUID:
|
||||
"""jobs row + its first outbox row + audit, in the CALLER's transaction."""
|
||||
job_id = uuid.uuid4()
|
||||
await conn.execute(
|
||||
"INSERT INTO jobs (id, attachment_id, pipeline_version, kind, replay_of, max_attempts) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s)", (job_id, attachment_id, pipeline_version, kind, replay_of, max_attempts))
|
||||
await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, 1)", (job_id,))
|
||||
await event(conn, job_id, "created", actor, kind=kind, pipeline_version=pipeline_version,
|
||||
replay_of=str(replay_of) if replay_of else None, **detail)
|
||||
return job_id
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ relay
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Due:
|
||||
outbox_id: int
|
||||
job_id: str
|
||||
attachment_id: str
|
||||
pipeline_version: int
|
||||
attempt: int
|
||||
created_at: dt.datetime
|
||||
job_state: str
|
||||
|
||||
|
||||
async def due_outbox(conn: psycopg.AsyncConnection, batch: int) -> list[Due]:
|
||||
"""Due, unpublished outbox rows, locked for this relay pass (SKIP LOCKED: several relays
|
||||
never publish the same row at the same time - a crash may still publish it twice)."""
|
||||
cur = await conn.execute(
|
||||
"SELECT o.id, o.job_id, j.attachment_id, j.pipeline_version, o.attempt, j.created_at, j.state "
|
||||
"FROM outbox o JOIN jobs j ON j.id = o.job_id "
|
||||
"WHERE o.published_at IS NULL AND o.available_at <= clock_timestamp() "
|
||||
"ORDER BY o.available_at, o.id LIMIT %s FOR UPDATE OF o SKIP LOCKED", (batch,))
|
||||
return [Due(r[0], str(r[1]), str(r[2]), r[3], r[4], r[5], r[6]) for r in await cur.fetchall()]
|
||||
|
||||
|
||||
async def mark_published(conn: psycopg.AsyncConnection, ids: list[int]) -> None:
|
||||
if ids:
|
||||
await conn.execute("UPDATE outbox SET published_at = clock_timestamp() WHERE id = ANY(%s)", (ids,))
|
||||
|
||||
|
||||
async def count_publish_failure(conn: psycopg.AsyncConnection, ids: list[int]) -> None:
|
||||
if ids:
|
||||
await conn.execute("UPDATE outbox SET publish_failures = publish_failures + 1 WHERE id = ANY(%s)", (ids,))
|
||||
|
||||
|
||||
async def expire_leases(conn: psycopg.AsyncConnection, actor: str, limit: int = 50) -> int:
|
||||
"""Reaper: processing jobs whose lease ran out (the worker died, hung or lost its database).
|
||||
The attempt is abandoned; the job goes back to queued with a new outbox row, or to failed
|
||||
`retry_exhausted` when it used all its attempts."""
|
||||
cur = await conn.execute(
|
||||
"SELECT id, attempt, max_attempts, lease_token, attachment_id FROM jobs "
|
||||
"WHERE state = 'processing' AND lease_until < clock_timestamp() "
|
||||
"ORDER BY lease_until LIMIT %s FOR UPDATE SKIP LOCKED", (limit,))
|
||||
rows = await cur.fetchall()
|
||||
for job_id, attempt, max_attempts, token, attachment_id in rows:
|
||||
await conn.execute(
|
||||
"UPDATE job_attempts SET outcome = 'abandoned', error_code = 'lease_expired', finished_at = clock_timestamp() "
|
||||
"WHERE token = %s AND outcome = 'running'", (token,))
|
||||
if attempt >= max_attempts:
|
||||
await _fail(conn, job_id, attachment_id, "retry_exhausted", actor, attempt, cause="lease_expired")
|
||||
else:
|
||||
await conn.execute(
|
||||
"UPDATE jobs SET state = 'queued', lease_token = NULL, lease_owner = NULL, lease_until = NULL, "
|
||||
"available_at = clock_timestamp(), updated_at = clock_timestamp() WHERE id = %s", (job_id,))
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() "
|
||||
"WHERE id = %s AND status = 'processing'", (attachment_id,))
|
||||
await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, %s)", (job_id, attempt + 1))
|
||||
await event(conn, job_id, "lease_expired", actor, attempt=attempt)
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def republish_stale(conn: psycopg.AsyncConnection, older_than: float, actor: str, limit: int = 50) -> int:
|
||||
"""Lost-message guard: a queued, due job that has had no unpublished outbox row and no
|
||||
publish for `older_than` seconds gets a new outbox row (its message was lost by the broker,
|
||||
dead-lettered after too many redeliveries, or the queue was purged)."""
|
||||
cur = await conn.execute(
|
||||
"SELECT j.id, j.attempt FROM jobs j "
|
||||
"WHERE j.state = 'queued' AND j.available_at < clock_timestamp() - make_interval(secs => %s) "
|
||||
"AND j.updated_at < clock_timestamp() - make_interval(secs => %s) "
|
||||
"AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.id AND (o.published_at IS NULL "
|
||||
" OR o.published_at > clock_timestamp() - make_interval(secs => %s))) "
|
||||
"ORDER BY j.updated_at LIMIT %s FOR UPDATE OF j SKIP LOCKED", (older_than, older_than, older_than, limit))
|
||||
rows = await cur.fetchall()
|
||||
for job_id, attempt in rows:
|
||||
await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, %s)", (job_id, attempt + 1))
|
||||
await event(conn, job_id, "republished", actor, attempt=attempt)
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def purge_outbox(conn: psycopg.AsyncConnection, keep_seconds: int = 7 * 86400) -> int:
|
||||
cur = await conn.execute(
|
||||
"DELETE FROM outbox WHERE id IN (SELECT id FROM outbox WHERE published_at < clock_timestamp() "
|
||||
"- make_interval(secs => %s) LIMIT 1000)", (keep_seconds,))
|
||||
return cur.rowcount
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ worker
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Claim:
|
||||
job_id: str
|
||||
attachment_id: str
|
||||
pipeline_version: int
|
||||
attempt: int
|
||||
max_attempts: int
|
||||
token: str
|
||||
source: str # staging | original
|
||||
source_key: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: bytes
|
||||
write_original: bool # the original is not published yet: this attempt publishes it too
|
||||
|
||||
def key(self, name: str) -> str:
|
||||
return attempt_key(self.attachment_id, self.token, name)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Noop:
|
||||
reason: str # finished | leased | not_due | superseded | unknown_job
|
||||
|
||||
|
||||
async def claim(conn: psycopg.AsyncConnection, job_id: str, worker: str, lease_seconds: float) -> Claim | Noop:
|
||||
cur = await conn.execute(
|
||||
"SELECT state, attempt, max_attempts, available_at > clock_timestamp(), attachment_id, pipeline_version "
|
||||
"FROM jobs WHERE id = %s FOR UPDATE", (job_id,))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return Noop("unknown_job")
|
||||
state, attempt, max_attempts, not_due, attachment_id, version = row
|
||||
if state not in ACTIVE:
|
||||
return Noop("finished") # duplicate/redelivery of a finished job: ack, nothing else
|
||||
if state == "processing":
|
||||
return Noop("leased") # another attempt holds the lease; the reaper handles its death
|
||||
if not_due:
|
||||
return Noop("not_due") # backoff not over: the outbox publishes it at available_at
|
||||
cur = await conn.execute(
|
||||
"SELECT status, published_version, object_key, staging_key, staging_state, content_type, size_bytes, sha256 "
|
||||
"FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,))
|
||||
status, published, object_key, staging_key, staging_state, content_type, size, sha = await cur.fetchone()
|
||||
actor = "worker:" + worker
|
||||
if published is not None and published >= version:
|
||||
# A newer (or the same) version is already published: this job can only be stale.
|
||||
await _finish(conn, job_id, "superseded", None)
|
||||
await event(conn, job_id, "superseded", actor, attempt=attempt, published_version=published)
|
||||
return Noop("superseded")
|
||||
if object_key is not None:
|
||||
source, source_key = "original", object_key
|
||||
elif staging_state == "present":
|
||||
source, source_key = "staging", staging_key
|
||||
else:
|
||||
# Cannot happen while the CHECK attachments_active_has_source holds; fail visibly anyway.
|
||||
await _fail(conn, job_id, attachment_id, "source_missing", actor, attempt)
|
||||
return Noop("finished")
|
||||
token = uuid.uuid4().hex
|
||||
attempt += 1
|
||||
write_original = object_key is None
|
||||
names = (("original",) if write_original else ()) + VARIANT_NAMES
|
||||
planned = [attempt_key(str(attachment_id), token, n) for n in names]
|
||||
await conn.execute(
|
||||
"UPDATE jobs SET state = 'processing', attempt = %s, lease_token = %s, lease_owner = %s, "
|
||||
"lease_until = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp() WHERE id = %s",
|
||||
(attempt, token, worker, lease_seconds, job_id))
|
||||
await conn.execute(
|
||||
"INSERT INTO job_attempts (token, job_id, attempt, worker, source, planned_keys) VALUES (%s, %s, %s, %s, %s, %s)",
|
||||
(token, job_id, attempt, worker, source, planned))
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'processing', updated_at = clock_timestamp() WHERE id = %s AND status = 'queued'",
|
||||
(attachment_id,))
|
||||
await event(conn, job_id, "claimed", actor, attempt=attempt, source=source, token=token)
|
||||
return Claim(str(job_id), str(attachment_id), version, attempt, max_attempts, token, source, source_key,
|
||||
content_type, size, bytes(sha), write_original)
|
||||
|
||||
|
||||
async def heartbeat(conn: psycopg.AsyncConnection, claim: Claim, lease_seconds: float) -> bool:
|
||||
"""Extend the lease; False = the lease was lost (expired and taken over): stop the attempt."""
|
||||
cur = await conn.execute(
|
||||
"UPDATE jobs SET lease_until = clock_timestamp() + make_interval(secs => %s) "
|
||||
"WHERE id = %s AND state = 'processing' AND lease_token = %s", (lease_seconds, claim.job_id, claim.token))
|
||||
return cur.rowcount == 1
|
||||
|
||||
|
||||
async def _holds_lease(conn: psycopg.AsyncConnection, claim: Claim) -> bool:
|
||||
cur = await conn.execute("SELECT state = 'processing' AND lease_token = %s FROM jobs WHERE id = %s FOR UPDATE",
|
||||
(claim.token, claim.job_id))
|
||||
row = await cur.fetchone()
|
||||
return bool(row and row[0])
|
||||
|
||||
|
||||
async def complete(conn: psycopg.AsyncConnection, claim: Claim, variants, worker: str) -> str:
|
||||
"""Publish the result of an attempt atomically: variant rows + attachment ready + job succeeded.
|
||||
Returns succeeded | superseded | fenced (this attempt lost its lease: nothing written)."""
|
||||
actor = "worker:" + worker
|
||||
if not await _holds_lease(conn, claim):
|
||||
return "fenced"
|
||||
cur = await conn.execute("SELECT published_version FROM attachments WHERE id = %s FOR UPDATE", (claim.attachment_id,))
|
||||
(published,) = await cur.fetchone()
|
||||
if published is not None and published >= claim.pipeline_version:
|
||||
await _end_attempt(conn, claim, "superseded", None)
|
||||
await _finish(conn, claim.job_id, "superseded", None)
|
||||
await event(conn, claim.job_id, "superseded", actor, attempt=claim.attempt, published_version=published)
|
||||
return "superseded"
|
||||
for v in variants:
|
||||
await conn.execute(
|
||||
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, "
|
||||
"size_bytes, width, height, sha256, attempt_token) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
|
||||
(claim.attachment_id, claim.pipeline_version, v.name, claim.key(v.name), v.content_type, len(v.data),
|
||||
v.width, v.height, v.sha256, claim.token))
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'ready', published_version = %s, error_code = NULL, failed_at = NULL, "
|
||||
"object_key = COALESCE(object_key, %s), updated_at = clock_timestamp() WHERE id = %s",
|
||||
(claim.pipeline_version, claim.key("original") if claim.write_original else None, claim.attachment_id))
|
||||
await _end_attempt(conn, claim, "succeeded", None)
|
||||
await _finish(conn, claim.job_id, "succeeded", None)
|
||||
await event(conn, claim.job_id, "succeeded", actor, attempt=claim.attempt, pipeline_version=claim.pipeline_version)
|
||||
return "succeeded"
|
||||
|
||||
|
||||
async def retry_or_fail(conn: psycopg.AsyncConnection, claim: Claim, code: str, permanent: bool, worker: str,
|
||||
base: float, cap: float) -> tuple[str, float]:
|
||||
"""After a failed attempt: permanent error or no attempts left -> failed (failed registry);
|
||||
otherwise queued again after a jittered backoff. Returns (outcome, delay seconds)."""
|
||||
actor = "worker:" + worker
|
||||
if not await _holds_lease(conn, claim):
|
||||
return "fenced", 0.0
|
||||
if permanent or claim.attempt >= claim.max_attempts:
|
||||
final = code if permanent else "retry_exhausted"
|
||||
await _end_attempt(conn, claim, "failed", code)
|
||||
await _fail(conn, claim.job_id, claim.attachment_id, final, actor, claim.attempt, cause=code)
|
||||
return "failed", 0.0
|
||||
delay = backoff_delay(claim.attempt, base, cap)
|
||||
await _end_attempt(conn, claim, "retry", code)
|
||||
await conn.execute(
|
||||
"UPDATE jobs SET state = 'queued', lease_token = NULL, lease_owner = NULL, lease_until = NULL, error_code = %s, "
|
||||
"available_at = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp() WHERE id = %s",
|
||||
(code, delay, claim.job_id))
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() WHERE id = %s AND status = 'processing'",
|
||||
(claim.attachment_id,))
|
||||
await conn.execute(
|
||||
"INSERT INTO outbox (job_id, attempt, available_at) "
|
||||
"VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s))", (claim.job_id, claim.attempt + 1, delay))
|
||||
await event(conn, claim.job_id, "retry_scheduled", actor, attempt=claim.attempt, error_code=code,
|
||||
delay_s=round(delay, 3))
|
||||
return "retry", delay
|
||||
|
||||
|
||||
async def release(conn: psycopg.AsyncConnection, claim: Claim, worker: str) -> str:
|
||||
"""Graceful stop of a worker with an attempt still running: give the job back at once
|
||||
(without waiting for the lease to expire). The claim already counted as an attempt."""
|
||||
if not await _holds_lease(conn, claim):
|
||||
return "fenced"
|
||||
await _end_attempt(conn, claim, "abandoned", "worker_shutdown")
|
||||
if claim.attempt >= claim.max_attempts:
|
||||
await _fail(conn, claim.job_id, claim.attachment_id, "retry_exhausted", "worker:" + worker, claim.attempt,
|
||||
cause="worker_shutdown")
|
||||
return "failed"
|
||||
await conn.execute(
|
||||
"UPDATE jobs SET state = 'queued', lease_token = NULL, lease_owner = NULL, lease_until = NULL, "
|
||||
"available_at = clock_timestamp(), updated_at = clock_timestamp() WHERE id = %s", (claim.job_id,))
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() WHERE id = %s AND status = 'processing'",
|
||||
(claim.attachment_id,))
|
||||
await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, %s)", (claim.job_id, claim.attempt + 1))
|
||||
await event(conn, claim.job_id, "released", "worker:" + worker, attempt=claim.attempt)
|
||||
return "released"
|
||||
|
||||
|
||||
async def _end_attempt(conn: psycopg.AsyncConnection, claim: Claim, outcome: str, code: str | None) -> None:
|
||||
await conn.execute(
|
||||
"UPDATE job_attempts SET outcome = %s, error_code = %s, finished_at = clock_timestamp() "
|
||||
"WHERE token = %s AND outcome = 'running'", (outcome, code, claim.token))
|
||||
|
||||
|
||||
async def _finish(conn: psycopg.AsyncConnection, job_id, state: str, code: str | None) -> None:
|
||||
await conn.execute(
|
||||
"UPDATE jobs SET state = %s, error_code = %s, lease_token = NULL, lease_owner = NULL, lease_until = NULL, "
|
||||
"finished_at = clock_timestamp(), updated_at = clock_timestamp() WHERE id = %s", (state, code, job_id))
|
||||
|
||||
|
||||
async def _fail(conn: psycopg.AsyncConnection, job_id, attachment_id, code: str, actor: str, attempt: int,
|
||||
**detail) -> None:
|
||||
"""Job -> failed (registry). The attachment becomes failed only if it is not ready: a failed
|
||||
reprocess of a newer version leaves the published result in place."""
|
||||
await _finish(conn, job_id, "failed", code)
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'failed', error_code = %s, failed_at = clock_timestamp(), "
|
||||
"updated_at = clock_timestamp() WHERE id = %s AND status IN ('queued', 'processing')", (code, attachment_id))
|
||||
await event(conn, job_id, "failed", actor, attempt=attempt, error_code=code, **detail)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ cleanup (worker)
|
||||
async def staging_to_delete(conn: psycopg.AsyncConnection, retention_seconds: int, limit: int = 50) -> list[str]:
|
||||
"""Candidates only (no lock): ready attachments (staging no longer needed) and failed ones
|
||||
older than the retention. The decision is re-checked under the row lock in mark_staging_deleting."""
|
||||
cur = await conn.execute(
|
||||
"SELECT id FROM attachments WHERE staging_state IN ('present', 'deleting') AND (status = 'ready' OR "
|
||||
"(status = 'failed' AND failed_at < clock_timestamp() - make_interval(secs => %s))) "
|
||||
"ORDER BY updated_at LIMIT %s", (retention_seconds, limit))
|
||||
return [str(r[0]) for r in await cur.fetchall()]
|
||||
|
||||
|
||||
async def mark_staging_deleting(conn: psycopg.AsyncConnection, attachment_id: str, retention_seconds: int) -> str | None:
|
||||
"""Under the attachment row lock, decide AGAIN from the committed state and mark the staging copy
|
||||
`deleting` BEFORE the S3 DELETE. A replay that committed in between made the row queued, so it
|
||||
no longer matches; a replay that comes later sees `deleting` and refuses. Queued/processing
|
||||
rows never match (and the CHECK attachments_active_has_source forbids it anyway)."""
|
||||
cur = await conn.execute(
|
||||
"UPDATE attachments SET staging_state = 'deleting' " # internal state: updated_at stays (public field)
|
||||
"WHERE id = %s AND staging_state IN ('present', 'deleting') AND (status = 'ready' OR (status = 'failed' "
|
||||
"AND failed_at < clock_timestamp() - make_interval(secs => %s) AND NOT EXISTS (SELECT 1 FROM jobs j "
|
||||
"WHERE j.attachment_id = attachments.id AND j.state IN ('queued', 'processing')))) RETURNING staging_key",
|
||||
(attachment_id, retention_seconds))
|
||||
row = await cur.fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
async def mark_staging_deleted(conn: psycopg.AsyncConnection, attachment_id: str) -> None:
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET staging_state = 'deleted' WHERE id = %s AND staging_state = 'deleting'", (attachment_id,))
|
||||
|
||||
|
||||
async def attempts_to_clean(conn: psycopg.AsyncConnection, grace_seconds: int, limit: int = 50) -> list[str]:
|
||||
cur = await conn.execute(
|
||||
"SELECT token FROM job_attempts WHERE outcome IN ('retry', 'failed', 'abandoned', 'superseded') "
|
||||
"AND objects_cleaned_at IS NULL AND finished_at < clock_timestamp() - make_interval(secs => %s) "
|
||||
"ORDER BY finished_at LIMIT %s", (grace_seconds, limit))
|
||||
return [r[0].hex for r in await cur.fetchall()]
|
||||
|
||||
|
||||
async def lock_attempt_for_cleanup(conn: psycopg.AsyncConnection, token: str) -> list[str] | None:
|
||||
"""Keys of an attempt that can never publish: it ended without success, it is not the lease
|
||||
holder of its job (tokens are never reused) and nothing published references its keys."""
|
||||
cur = await conn.execute(
|
||||
"SELECT ja.planned_keys FROM job_attempts ja JOIN jobs j ON j.id = ja.job_id "
|
||||
"WHERE ja.token = %s AND ja.objects_cleaned_at IS NULL "
|
||||
"AND ja.outcome IN ('retry', 'failed', 'abandoned', 'superseded') "
|
||||
"AND j.lease_token IS DISTINCT FROM ja.token FOR UPDATE OF ja", (token,))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
keys = list(row[0])
|
||||
cur = await conn.execute(
|
||||
"SELECT EXISTS (SELECT 1 FROM attachment_variants WHERE object_key = ANY(%s)) "
|
||||
"OR EXISTS (SELECT 1 FROM attachments WHERE object_key = ANY(%s))", (keys, keys))
|
||||
if (await cur.fetchone())[0]:
|
||||
return None # published: never delete (would be a bug elsewhere; logged by the caller)
|
||||
return keys
|
||||
|
||||
|
||||
async def mark_attempt_cleaned(conn: psycopg.AsyncConnection, token: str) -> None:
|
||||
await conn.execute("UPDATE job_attempts SET objects_cleaned_at = clock_timestamp() WHERE token = %s", (token,))
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Versioned image pipeline (lab 5): WHAT the worker builds for one attachment. Pure definitions,
|
||||
no I/O; the pixels are produced by imaging.py in a child process.
|
||||
|
||||
Every pipeline version builds the same three variants with the geometry of the contract; versions
|
||||
differ only in encoder parameters, so the published version is visible in the bytes (sha256):
|
||||
|
||||
thumbnail JPEG, fits into 256x256, keeps the aspect ratio, never upscales, floor rounding
|
||||
(minimum 1 px), EXIF orientation applied, alpha composited onto white;
|
||||
crop JPEG, centred square 128x128 (small images ARE upscaled, the contract allows it);
|
||||
optimized WebP, fits into 1280x1280, never upscales, alpha kept.
|
||||
|
||||
Metadata: nothing of the source is copied into a variant - no EXIF (orientation is applied to
|
||||
the pixels instead), no XMP, no ICC profile, no comments (the pixels are treated as sRGB).
|
||||
The original is published byte for byte and keeps everything.
|
||||
|
||||
Track 2: the version of a job is fixed when the job is created (PIPELINE_VERSION for uploads,
|
||||
`jobctl reprocess --pipeline N` later). A result is published only if its version is greater
|
||||
than the published one, so an old job can never overwrite a newer result.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
THUMBNAIL_BOX = 256
|
||||
CROP_SIDE = 128
|
||||
OPTIMIZED_BOX = 1280
|
||||
VARIANT_NAMES = ("thumbnail", "crop", "optimized")
|
||||
CONTENT_TYPES = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Pipeline:
|
||||
version: int
|
||||
jpeg_quality: int # thumbnail and crop: baseline JPEG, 4:2:0, optimized Huffman tables
|
||||
webp_quality: int # optimized: lossy WebP
|
||||
webp_method: int # 0 (fast) .. 6 (slow, smaller)
|
||||
|
||||
|
||||
PIPELINES = {
|
||||
1: Pipeline(1, jpeg_quality=85, webp_quality=80, webp_method=4),
|
||||
# Versions 2 and 3 exist for the track 2 experiments (reprocess, reorder, stale jobs):
|
||||
# different encoder settings -> different bytes, identical geometry.
|
||||
2: Pipeline(2, jpeg_quality=80, webp_quality=75, webp_method=4),
|
||||
3: Pipeline(3, jpeg_quality=90, webp_quality=85, webp_method=5),
|
||||
}
|
||||
|
||||
|
||||
def fit(width: int, height: int, box: int) -> tuple[int, int]:
|
||||
"""Size inside box x box keeping the aspect ratio, never upscaling, rounded DOWN, >= 1 px.
|
||||
Integer arithmetic: 320x200 -> 256x160 exactly, 1x1000 -> 1x256 (0.256 -> 1 px minimum)."""
|
||||
if width <= box and height <= box:
|
||||
return width, height
|
||||
if width >= height:
|
||||
return box, max(1, height * box // width)
|
||||
return max(1, width * box // height), box
|
||||
|
||||
|
||||
def center_square(width: int, height: int) -> tuple[int, int, int, int]:
|
||||
"""Crop box (left, top, right, bottom) of the centred square with side min(width, height)."""
|
||||
side = min(width, height)
|
||||
left = (width - side) // 2
|
||||
top = (height - side) // 2
|
||||
return left, top, left + side, top + side
|
||||
@@ -1,437 +0,0 @@
|
||||
"""Reconciler of the two stores (lab 4, track 2): S3 objects vs attachment rows in PostgreSQL.
|
||||
|
||||
docker compose run --rm reconciler # dry-run: report only (default)
|
||||
docker compose run --rm reconciler --apply # fix what is safe to fix
|
||||
docker compose run --rm reconciler --verify-checksums # also read every object (SHA-256)
|
||||
|
||||
How it walks: both sides are read as key-ordered streams and merge-joined - ListObjectsV2
|
||||
pages (prefix o/, byte order) against keyset pages of `attachments ORDER BY object_key`
|
||||
(COLLATE "C" = the same byte order, unique index). Memory does not depend on the bucket size;
|
||||
cost is ~N/1000 S3 list requests + N/page DB pages (measured: docs/CONSISTENCY.md).
|
||||
|
||||
What it finds and what --apply does (nothing else is ever changed):
|
||||
|
||||
kind condition --apply
|
||||
orphan_object object, no row, older than --grace delete the object
|
||||
stale_intent row 'pending' older than --grace (+/- object) delete object, then row
|
||||
missing_object row 'ready', object absent, older than --grace quarantine the row
|
||||
size_mismatch row 'ready', object size != size_bytes quarantine the row
|
||||
checksum_mismatch row 'ready', SHA-256 differs (--verify-checksums) quarantine the row
|
||||
restored_object row quarantined, object present and correct again remove the quarantine
|
||||
(a checksum quarantine only with --verify-checksums)
|
||||
fresh_* the same, but younger than --grace nothing (in-flight upload)
|
||||
|
||||
Safety:
|
||||
* dry-run is read-only BY CONSTRUCTION: the PostgreSQL session runs with
|
||||
default_transaction_read_only=on and the S3 wrapper has no write methods, so even a bug in
|
||||
the planner cannot modify a store;
|
||||
* ready rows are never deleted - a used (or unused) ready attachment can only be quarantined,
|
||||
which is reversible; objects that have a row are never deleted unless the row is a stale
|
||||
intent; anything younger than --grace is never touched;
|
||||
* every action re-checks its condition right before acting, under a row lock (FOR UPDATE)
|
||||
where a row exists, so the live API racing with the reconciler cannot be hurt;
|
||||
* object first, row second: an interrupted run leaves a state the next run repairs;
|
||||
re-running is idempotent (the second run finds nothing to do);
|
||||
* safety valve: --apply refuses to delete more than --max-orphan-fraction of the scanned
|
||||
objects (pointing it at the wrong/empty database would otherwise wipe the bucket).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import psycopg
|
||||
|
||||
from .config import ConfigError, Settings, _float, _secret, s3_settings
|
||||
from .errors import Unavailable
|
||||
from .storage import OBJECT_PREFIX, ListedObject, ObjectMissing, Storage
|
||||
|
||||
KINDS = ("orphan_object", "stale_intent", "missing_object", "size_mismatch", "checksum_mismatch",
|
||||
"restored_object", "fresh_orphan_object", "fresh_intent", "fresh_missing_object")
|
||||
MIN_SAFE_GRACE = 60.0 # > the longest possible upload (queue 5 s + body 30 s + verify 10 s + S3)
|
||||
ACTION_OF = {"orphan_object": "delete_object", "stale_intent": "delete_intent", "missing_object": "quarantine",
|
||||
"size_mismatch": "quarantine", "checksum_mismatch": "quarantine", "restored_object": "unquarantine"}
|
||||
|
||||
|
||||
def log(text: str) -> None:
|
||||
print(f"reconcile: {text}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Row:
|
||||
id: str
|
||||
key: str
|
||||
status: str
|
||||
size: int
|
||||
sha256: bytes
|
||||
created_at: dt.datetime
|
||||
updated_at: dt.datetime
|
||||
lost_reason: str | None
|
||||
used: bool
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Finding:
|
||||
kind: str
|
||||
key: str
|
||||
attachment_id: str | None = None
|
||||
used: bool = False
|
||||
detail: str = ""
|
||||
result: str = "planned" # planned | applied | skipped:<why> | failed:<why>
|
||||
|
||||
|
||||
@dataclass
|
||||
class Stats:
|
||||
objects: int = 0
|
||||
rows: int = 0
|
||||
consistent: int = 0
|
||||
s3_list_requests: int = 0
|
||||
s3_head_requests: int = 0
|
||||
s3_get_bytes: int = 0
|
||||
db_pages: int = 0
|
||||
|
||||
|
||||
class ReadOnlyStorage:
|
||||
"""The dry-run view of S3: listing and reading only - there is no delete to call."""
|
||||
|
||||
def __init__(self, storage: Storage):
|
||||
self._s = storage
|
||||
|
||||
async def list_page(self, *args, **kwargs):
|
||||
return await self._s.list_page(*args, **kwargs)
|
||||
|
||||
async def head(self, key: str):
|
||||
return await self._s.head(key)
|
||||
|
||||
async def read_all(self, key: str, limit: int) -> bytes:
|
||||
return await self._s.read_all(key, limit)
|
||||
|
||||
|
||||
class Reconciler:
|
||||
def __init__(self, conn: psycopg.AsyncConnection, storage: Storage | ReadOnlyStorage, *, grace: float,
|
||||
page_size: int, verify: bool, apply: bool):
|
||||
self.conn = conn
|
||||
self.s3 = storage
|
||||
self.grace = grace
|
||||
self.page_size = page_size
|
||||
self.verify = verify
|
||||
self.apply = apply
|
||||
self.stats = Stats()
|
||||
self.findings: list[Finding] = []
|
||||
|
||||
# ---------------------------------------------------------- streams
|
||||
async def objects(self) -> AsyncIterator[ListedObject]:
|
||||
after = None
|
||||
while True:
|
||||
page, more = await self.s3.list_page(OBJECT_PREFIX, after, self.page_size)
|
||||
self.stats.s3_list_requests += 1
|
||||
for item in page:
|
||||
self.stats.objects += 1
|
||||
yield item
|
||||
if not more or not page:
|
||||
return
|
||||
after = page[-1].key
|
||||
|
||||
async def rows(self) -> AsyncIterator[Row]:
|
||||
after = ""
|
||||
while True:
|
||||
cur = await self.conn.execute(
|
||||
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256, a.created_at, a.updated_at, "
|
||||
"a.lost_reason, EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) "
|
||||
"FROM attachments a WHERE a.object_key > %s ORDER BY a.object_key LIMIT %s",
|
||||
(after, self.page_size))
|
||||
page = await cur.fetchall()
|
||||
self.stats.db_pages += 1
|
||||
for r in page:
|
||||
self.stats.rows += 1
|
||||
yield Row(str(r[0]), r[1], r[2], r[3], bytes(r[4]), r[5], r[6], r[7], r[8])
|
||||
if len(page) < self.page_size:
|
||||
return
|
||||
after = page[-1][1]
|
||||
|
||||
# ---------------------------------------------------------- planning (read-only)
|
||||
async def scan(self) -> None:
|
||||
cur = await self.conn.execute("SELECT clock_timestamp()")
|
||||
self.db_now = (await cur.fetchone())[0]
|
||||
self.s3_now = time.time()
|
||||
objects, rows = self.objects(), self.rows()
|
||||
obj = await anext(objects, None)
|
||||
row = await anext(rows, None)
|
||||
while obj is not None or row is not None:
|
||||
if row is None or (obj is not None and obj.key < row.key):
|
||||
self.object_only(obj)
|
||||
obj = await anext(objects, None)
|
||||
elif obj is None or row.key < obj.key:
|
||||
self.row_only(row)
|
||||
row = await anext(rows, None)
|
||||
else:
|
||||
await self.both(obj, row)
|
||||
obj = await anext(objects, None)
|
||||
row = await anext(rows, None)
|
||||
|
||||
def row_old(self, row: Row) -> bool:
|
||||
return (self.db_now - max(row.created_at, row.updated_at)).total_seconds() > self.grace
|
||||
|
||||
def object_only(self, obj: ListedObject) -> None:
|
||||
old = self.s3_now - obj.last_modified_ts > self.grace
|
||||
self.add(Finding("orphan_object" if old else "fresh_orphan_object", obj.key))
|
||||
|
||||
def row_only(self, row: Row) -> None:
|
||||
if row.status == "pending":
|
||||
self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id,
|
||||
detail="object absent"))
|
||||
elif row.lost_reason is not None:
|
||||
self.stats.consistent += 1 # already quarantined, still missing: nothing new
|
||||
else:
|
||||
self.add(Finding("missing_object" if self.row_old(row) else "fresh_missing_object", row.key, row.id,
|
||||
used=row.used))
|
||||
|
||||
async def both(self, obj: ListedObject, row: Row) -> None:
|
||||
if row.status == "pending":
|
||||
self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id,
|
||||
detail="object present"))
|
||||
return
|
||||
problem = await self.object_problem(row, obj.size)
|
||||
if problem:
|
||||
if row.lost_reason is None:
|
||||
self.add(Finding(problem, row.key, row.id, used=row.used))
|
||||
else:
|
||||
self.stats.consistent += 1 # quarantined and still broken
|
||||
elif row.lost_reason == "checksum_mismatch" and not self.verify:
|
||||
# Same size, but this pass does not read the bytes: it cannot prove the object is the
|
||||
# original again, so a checksum quarantine is lifted only by --verify-checksums.
|
||||
self.stats.consistent += 1
|
||||
elif row.lost_reason is not None:
|
||||
self.add(Finding("restored_object", row.key, row.id, used=row.used))
|
||||
else:
|
||||
self.stats.consistent += 1
|
||||
|
||||
async def object_problem(self, row: Row, size: int) -> str | None:
|
||||
if size != row.size:
|
||||
return "size_mismatch"
|
||||
if self.verify:
|
||||
data = await self.s3.read_all(row.key, row.size)
|
||||
self.stats.s3_get_bytes += len(data)
|
||||
if hashlib.sha256(data).digest() != row.sha256:
|
||||
return "checksum_mismatch"
|
||||
return None
|
||||
|
||||
def add(self, finding: Finding) -> None:
|
||||
if finding.kind not in ACTION_OF:
|
||||
finding.result = "not_actionable"
|
||||
self.findings.append(finding)
|
||||
|
||||
# ---------------------------------------------------------- actions (--apply only)
|
||||
async def run_actions(self, max_orphan_fraction: float) -> None:
|
||||
orphans = sum(1 for f in self.findings if f.kind == "orphan_object")
|
||||
if self.stats.objects and orphans > max_orphan_fraction * self.stats.objects:
|
||||
raise SafetyValve(f"{orphans} of {self.stats.objects} objects look orphaned "
|
||||
f"(> {max_orphan_fraction:.0%}); wrong database? nothing was changed")
|
||||
handlers = {"delete_object": self.delete_object, "delete_intent": self.delete_intent,
|
||||
"quarantine": self.quarantine, "unquarantine": self.unquarantine}
|
||||
for finding in self.findings:
|
||||
action = ACTION_OF.get(finding.kind)
|
||||
if action is None:
|
||||
continue
|
||||
try:
|
||||
finding.result = await handlers[action](finding)
|
||||
except (Unavailable, psycopg.OperationalError) as exc:
|
||||
finding.result = f"failed:{type(exc).__name__}"
|
||||
raise # a store is down: stop; the next run continues from the committed state
|
||||
|
||||
async def delete_object(self, f: Finding) -> str:
|
||||
cur = await self.conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (f.key,))
|
||||
if await cur.fetchone():
|
||||
return "skipped:row_appeared"
|
||||
try:
|
||||
head = await self.s3.head(f.key)
|
||||
self.stats.s3_head_requests += 1
|
||||
except ObjectMissing:
|
||||
return "skipped:already_gone"
|
||||
if time.time() - head.last_modified_ts <= self.grace:
|
||||
return "skipped:object_rewritten_recently"
|
||||
# No row names this key and the object is older than the grace period. Rows are always
|
||||
# written BEFORE their object and keys are random and never reused, so no upload can
|
||||
# still claim it.
|
||||
await self.s3.delete(f.key)
|
||||
return "applied"
|
||||
|
||||
async def delete_intent(self, f: Finding) -> str:
|
||||
async with self.conn.transaction():
|
||||
cur = await self.conn.execute(
|
||||
"SELECT 1 FROM attachments WHERE id = %s AND status = 'pending' "
|
||||
"AND greatest(created_at, updated_at) < clock_timestamp() - make_interval(secs => %s) FOR UPDATE",
|
||||
(f.attachment_id, self.grace))
|
||||
if await cur.fetchone() is None:
|
||||
return "skipped:no_longer_stale_intent"
|
||||
await self.s3.delete(f.key) # object first (idempotent), then the row
|
||||
await self.conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'pending'",
|
||||
(f.attachment_id,))
|
||||
return "applied"
|
||||
|
||||
async def quarantine(self, f: Finding) -> str:
|
||||
async with self.conn.transaction():
|
||||
cur = await self.conn.execute(
|
||||
"SELECT size_bytes, sha256 FROM attachments WHERE id = %s AND status = 'ready' "
|
||||
"AND lost_at IS NULL FOR UPDATE", (f.attachment_id,))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return "skipped:row_changed"
|
||||
reason = await self.current_problem(f.key, row[0], bytes(row[1]))
|
||||
if reason is None:
|
||||
return "skipped:object_ok_now"
|
||||
await self.conn.execute(
|
||||
"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = %s WHERE id = %s",
|
||||
(reason, f.attachment_id))
|
||||
return "applied"
|
||||
|
||||
async def unquarantine(self, f: Finding) -> str:
|
||||
async with self.conn.transaction():
|
||||
cur = await self.conn.execute(
|
||||
"SELECT size_bytes, sha256, lost_reason FROM attachments WHERE id = %s AND lost_at IS NOT NULL FOR UPDATE",
|
||||
(f.attachment_id,))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return "skipped:row_changed"
|
||||
if row[2] == "checksum_mismatch" and not self.verify:
|
||||
return "skipped:needs_verify_checksums"
|
||||
if await self.current_problem(f.key, row[0], bytes(row[1])) is not None:
|
||||
return "skipped:object_still_bad"
|
||||
await self.conn.execute("UPDATE attachments SET lost_at = NULL, lost_reason = NULL WHERE id = %s",
|
||||
(f.attachment_id,))
|
||||
return "applied"
|
||||
|
||||
async def current_problem(self, key: str, size: int, sha256: bytes) -> str | None:
|
||||
"""Fresh look at the object right before changing the row (the listing may be stale)."""
|
||||
try:
|
||||
head = await self.s3.head(key)
|
||||
self.stats.s3_head_requests += 1
|
||||
except ObjectMissing:
|
||||
return "missing_object"
|
||||
if head.size != size:
|
||||
return "size_mismatch"
|
||||
if self.verify:
|
||||
data = await self.s3.read_all(key, size)
|
||||
self.stats.s3_get_bytes += len(data)
|
||||
if hashlib.sha256(data).digest() != sha256:
|
||||
return "checksum_mismatch"
|
||||
return None
|
||||
|
||||
# ---------------------------------------------------------- report
|
||||
def report(self, mode: str, seconds: float, error: str | None, sample: int) -> dict:
|
||||
kinds = Counter(f.kind for f in self.findings)
|
||||
results = Counter(f.result.split(":")[0] for f in self.findings if f.kind in ACTION_OF)
|
||||
samples = {}
|
||||
for f in self.findings:
|
||||
items = samples.setdefault(f.kind, [])
|
||||
if len(items) < sample:
|
||||
items.append({"key": f.key, "attachment_id": f.attachment_id, "used": f.used,
|
||||
"result": f.result, **({"detail": f.detail} if f.detail else {})})
|
||||
return {
|
||||
"mode": mode, "grace_seconds": self.grace, "verify_checksums": self.verify,
|
||||
"duration_seconds": round(seconds, 3), "error": error,
|
||||
"scanned": {"objects": self.stats.objects, "rows": self.stats.rows, "consistent": self.stats.consistent},
|
||||
"findings": {k: kinds.get(k, 0) for k in KINDS},
|
||||
"used_attachments_in_findings": sum(1 for f in self.findings if f.used),
|
||||
"actions": {"planned": sum(1 for f in self.findings if f.kind in ACTION_OF),
|
||||
"applied": results.get("applied", 0), "skipped": results.get("skipped", 0),
|
||||
"failed": results.get("failed", 0)},
|
||||
"cost": {"s3_list_requests": self.stats.s3_list_requests, "s3_head_requests": self.stats.s3_head_requests,
|
||||
"s3_get_bytes": self.stats.s3_get_bytes, "db_pages": self.stats.db_pages,
|
||||
"page_size": self.page_size},
|
||||
"samples": samples,
|
||||
}
|
||||
|
||||
|
||||
class SafetyValve(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def arguments(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(prog="python -m messenger.reconcile", description=__doc__.split("\n")[0])
|
||||
p.add_argument("--apply", action="store_true", help="fix findings (default: dry-run, read-only)")
|
||||
p.add_argument("--grace", type=float, default=3600.0,
|
||||
help="seconds; anything younger is never touched (default 3600, >= PENDING_GRACE_SECONDS)")
|
||||
p.add_argument("--allow-small-grace", action="store_true",
|
||||
help="permit --grace below 60 s (tests only: a grace shorter than an upload could remove the "
|
||||
"intent of an upload that is still running; that upload then fails with 503)")
|
||||
p.add_argument("--page-size", type=int, default=1000, help="S3 list / DB page size (1..1000)")
|
||||
p.add_argument("--verify-checksums", action="store_true", help="read every ready object and compare SHA-256")
|
||||
p.add_argument("--samples", type=int, default=20, help="keys listed per finding kind in the report")
|
||||
p.add_argument("--max-orphan-fraction", type=float, default=0.5,
|
||||
help="--apply refuses if more than this share of objects look orphaned (default 0.5)")
|
||||
args = p.parse_args(argv)
|
||||
if not 1 <= args.page_size <= 1000:
|
||||
p.error("--page-size must be 1..1000")
|
||||
if args.grace < 1 or (args.grace < MIN_SAFE_GRACE and not args.allow_small_grace):
|
||||
p.error(f"--grace must be >= {MIN_SAFE_GRACE:.0f} s (or >= 1 s with --allow-small-grace)")
|
||||
if not 0 < args.max_orphan_fraction <= 1:
|
||||
p.error("--max-orphan-fraction must be in (0, 1]")
|
||||
return args
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> int:
|
||||
url = _secret("DATABASE_URL", required=True)
|
||||
base = s3_settings()
|
||||
settings = Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024,
|
||||
database_url=url, **base)
|
||||
timeout_ms = int(_float("DB_TIMEOUT_SECONDS", 10.0, 1.0, 120.0) * 1000)
|
||||
options = f"-c statement_timeout={timeout_ms} -c lock_timeout={timeout_ms // 2}"
|
||||
if not args.apply:
|
||||
options += " -c default_transaction_read_only=on" # dry-run: PostgreSQL itself refuses writes
|
||||
mode = "apply" if args.apply else "dry-run"
|
||||
storage = Storage(settings)
|
||||
await storage.open()
|
||||
started = time.perf_counter()
|
||||
error = None
|
||||
code = 0
|
||||
try:
|
||||
conn = await psycopg.AsyncConnection.connect(url, autocommit=True, connect_timeout=5, options=options,
|
||||
application_name=f"messenger-reconcile-{mode}")
|
||||
except psycopg.OperationalError as exc:
|
||||
await storage.close()
|
||||
error = f"store_unavailable: database {type(exc).__name__}"
|
||||
print(json.dumps({"mode": mode, "error": error}, ensure_ascii=False))
|
||||
log(error)
|
||||
return 1
|
||||
rec = Reconciler(conn, storage if args.apply else ReadOnlyStorage(storage), grace=args.grace,
|
||||
page_size=args.page_size, verify=args.verify_checksums, apply=args.apply)
|
||||
try:
|
||||
log(f"{mode}: scanning bucket {settings.s3_bucket} (prefix {OBJECT_PREFIX}) against table attachments")
|
||||
await rec.scan()
|
||||
log(f"scan done: {rec.stats.objects} objects, {rec.stats.rows} rows, {len(rec.findings)} findings")
|
||||
if args.apply:
|
||||
await rec.run_actions(args.max_orphan_fraction)
|
||||
except SafetyValve as exc:
|
||||
error, code = f"safety_valve: {exc}", 3
|
||||
except (Unavailable, psycopg.OperationalError) as exc:
|
||||
error, code = f"store_unavailable: {type(exc).__name__}", 1
|
||||
finally:
|
||||
await conn.close()
|
||||
await storage.close()
|
||||
print(json.dumps(rec.report(mode, time.perf_counter() - started, error, args.samples), ensure_ascii=False,
|
||||
indent=1))
|
||||
if error:
|
||||
log(error)
|
||||
return code
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = arguments(argv)
|
||||
try:
|
||||
return asyncio.run(run(args))
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,217 +0,0 @@
|
||||
"""Track 2: the reconciler of S3 and PostgreSQL (python -m messenger.reconcile).
|
||||
|
||||
Faults are injected on BOTH sides, directly into the stores (S3 with the API/reconciler key
|
||||
pairs from inside the storage network, PostgreSQL with psql):
|
||||
|
||||
S3 side: orphan object (no row), object deleted under a ready row, object overwritten with
|
||||
other bytes (size / same-size checksum), object of a quarantined row restored;
|
||||
DB side: stale intent with and without object, quarantine mark on a healthy row;
|
||||
fresh: orphan and intent younger than --grace (an upload in progress).
|
||||
|
||||
The tests prove: dry-run changes nothing in either store; --apply fixes exactly the old findings;
|
||||
fresh and used attachments are never deleted (ready rows only ever get a reversible quarantine);
|
||||
the second run finds nothing to do; the safety valve and store outages stop the run without
|
||||
damage. A small --grace (8-30 s, --allow-small-grace) keeps the tests fast."""
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import imagegen
|
||||
from conftest import (attachment_row, findings_by_key, psql, reconcile, s3_delete, s3_keys, s3_put, s3_size,
|
||||
service_down)
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
GRACE = ["--grace", "8", "--allow-small-grace"]
|
||||
# The main scenario needs "fresh" items to stay younger than --grace for the whole dry-run + snapshot
|
||||
# (each docker exec costs 1-3 s on Windows), hence a longer grace there.
|
||||
LONG_GRACE = ["--grace", "30", "--allow-small-grace"]
|
||||
|
||||
|
||||
def new_key() -> str:
|
||||
return "o/" + uuid.uuid4().hex
|
||||
|
||||
|
||||
def insert_intent(chat: dict, owner: dict, key: str, data: bytes, age_seconds: int) -> str:
|
||||
"""A 'pending' row as an upload leaves it after the intent COMMIT (crash afterwards)."""
|
||||
attachment_id = str(uuid.uuid4())
|
||||
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
|
||||
"object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'x.png', 'image/png', %d, "
|
||||
"decode('%s', 'hex'), 320, 200, '%s', 'pending', clock_timestamp() - make_interval(secs => %d), "
|
||||
"clock_timestamp() - make_interval(secs => %d))"
|
||||
% (attachment_id, chat["id"], owner["id"], len(data), hashlib.sha256(data).hexdigest(), key, age_seconds,
|
||||
age_seconds))
|
||||
return attachment_id
|
||||
|
||||
|
||||
def snapshot() -> tuple[str, frozenset]:
|
||||
"""Fingerprint of everything the reconciler could change: every attachment row + the key list."""
|
||||
rows = psql("SELECT count(*) || ':' || coalesce(md5(string_agg(id::text || status || object_key || "
|
||||
"coalesce(lost_reason, '-') || updated_at::text, ',' ORDER BY id)), '') FROM attachments")
|
||||
return rows, frozenset(s3_keys())
|
||||
|
||||
|
||||
def content_status(api, actor, attachment_id) -> int:
|
||||
return api.client.get(f"/api/v1/attachments/{attachment_id}/content",
|
||||
headers={"Authorization": "Bearer " + actor["token"]}).status_code
|
||||
|
||||
|
||||
def test_detect_and_safely_fix_faults_on_both_sides(api):
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
chat = api.chat(alice, bob)
|
||||
up = {name: api.upload(alice, chat, imagegen.png(30 + i, 20)).json() for i, name in enumerate("ABCDE")}
|
||||
data = {name: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
|
||||
for name, a in up.items()}
|
||||
key = {name: attachment_row(a["id"])["object_key"] for name, a in up.items()}
|
||||
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
|
||||
|
||||
# ---- inject (old)
|
||||
orphan_old = new_key()
|
||||
s3_put(orphan_old, SAMPLE)
|
||||
stale_with, stale_without = new_key(), new_key()
|
||||
s3_put(stale_with, SAMPLE)
|
||||
stale_with_id = insert_intent(chat, alice, stale_with, SAMPLE, 600) # < janitor grace 900 s
|
||||
stale_without_id = insert_intent(chat, alice, stale_without, SAMPLE, 600)
|
||||
s3_delete(key["C"]) # ready + used, bytes lost
|
||||
s3_put(key["D"], data["D"][:len(data["D"]) // 2]) # ready, unused, wrong size
|
||||
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' "
|
||||
f"WHERE id = '{up['E']['id']}'") # quarantined, but bytes are fine
|
||||
time.sleep(32) # everything above is now older than --grace
|
||||
# ---- inject (fresh: an upload in progress)
|
||||
fresh_orphan, fresh_intent = new_key(), new_key()
|
||||
s3_put(fresh_orphan, SAMPLE)
|
||||
s3_put(fresh_intent, SAMPLE)
|
||||
fresh_intent_id = insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
|
||||
|
||||
# ---- dry-run: report everything, change nothing
|
||||
before = snapshot()
|
||||
dry = reconcile(*LONG_GRACE)
|
||||
assert snapshot() == before, "dry-run modified a store"
|
||||
f = findings_by_key(dry)
|
||||
expected = {orphan_old: "orphan_object", fresh_orphan: "fresh_orphan_object", stale_with: "stale_intent",
|
||||
stale_without: "stale_intent", fresh_intent: "fresh_intent", key["C"]: "missing_object",
|
||||
key["D"]: "size_mismatch", key["E"]: "restored_object"}
|
||||
assert {k: f[k]["kind"] for k in expected} == expected
|
||||
assert key["A"] not in f and key["B"] not in f
|
||||
assert f[key["C"]]["used"] is True and dry["mode"] == "dry-run"
|
||||
assert all(f[k]["result"] in ("planned", "not_actionable") for k in expected)
|
||||
|
||||
# ---- apply
|
||||
applied = findings_by_key(reconcile("--apply", *LONG_GRACE))
|
||||
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"]):
|
||||
assert applied[k]["result"] == "applied", (k, applied[k])
|
||||
keys = s3_keys()
|
||||
assert orphan_old not in keys and stale_with not in keys
|
||||
assert attachment_row(stale_with_id) is None and attachment_row(stale_without_id) is None
|
||||
# fresh: untouched
|
||||
assert fresh_orphan in keys and fresh_intent in keys and attachment_row(fresh_intent_id)["status"] == "pending"
|
||||
# used, lost bytes: quarantined, NOT deleted; the message still references it
|
||||
row_c = attachment_row(up["C"]["id"])
|
||||
assert row_c["status"] == "ready" and row_c["lost_reason"] == "missing_object"
|
||||
items, _ = api.history(bob, chat, 100)
|
||||
assert any(up["C"]["id"] in m["attachment_ids"] for m in items)
|
||||
assert content_status(api, bob, up["C"]["id"]) == 404
|
||||
api.post_message(alice, chat, {"text": "", "attachment_ids": [up["C"]["id"]]}, 409) # cannot attach a lost file
|
||||
assert attachment_row(up["D"]["id"])["lost_reason"] == "size_mismatch"
|
||||
assert attachment_row(up["E"]["id"])["lost_reason"] is None # quarantine lifted
|
||||
for name in "ABE":
|
||||
got = api.call("GET", f"/api/v1/attachments/{up[name]['id']}/content", 200, token=bob["token"])
|
||||
assert got.content == data[name]
|
||||
|
||||
# ---- second run: idempotent, nothing left to do for these keys
|
||||
again = findings_by_key(reconcile("--apply", *LONG_GRACE))
|
||||
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"], key["A"], key["B"]):
|
||||
assert k not in again, (k, again.get(k))
|
||||
|
||||
# ---- the lost object comes back from a backup: the quarantine is lifted, the file is served again
|
||||
s3_put(key["C"], data["C"])
|
||||
restored = findings_by_key(reconcile("--apply", *LONG_GRACE))
|
||||
assert restored[key["C"]]["kind"] == "restored_object" and restored[key["C"]]["result"] == "applied"
|
||||
assert api.call("GET", f"/api/v1/attachments/{up['C']['id']}/content", 200, token=bob["token"]).content == data["C"]
|
||||
# tidy up the fresh items of this test once they are old enough
|
||||
time.sleep(31)
|
||||
reconcile("--apply", *LONG_GRACE)
|
||||
assert fresh_orphan not in s3_keys() and attachment_row(fresh_intent_id) is None
|
||||
|
||||
|
||||
def test_checksum_mode_finds_same_size_corruption(api):
|
||||
alice = api.user("alice")
|
||||
chat = api.chat(alice)
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
k = attachment_row(att["id"])["object_key"]
|
||||
s3_put(k, SAMPLE[:1000] + bytes([SAMPLE[1000] ^ 1]) + SAMPLE[1001:]) # same size, other bytes
|
||||
time.sleep(9)
|
||||
assert k not in findings_by_key(reconcile(*GRACE)) # sizes match: listing cannot see it
|
||||
report = reconcile("--verify-checksums", *GRACE)
|
||||
assert findings_by_key(report)[k]["kind"] == "checksum_mismatch"
|
||||
assert report["cost"]["s3_get_bytes"] >= len(SAMPLE)
|
||||
assert findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]["result"] == "applied"
|
||||
assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch"
|
||||
# A later pass WITHOUT checksums sees "same size" but cannot prove the bytes: the quarantine stays.
|
||||
assert k not in findings_by_key(reconcile("--apply", *GRACE))
|
||||
assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch"
|
||||
# Restoring the original and verifying lifts it.
|
||||
s3_put(k, SAMPLE)
|
||||
lifted = findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]
|
||||
assert (lifted["kind"], lifted["result"]) == ("restored_object", "applied")
|
||||
assert attachment_row(att["id"])["lost_reason"] is None
|
||||
|
||||
|
||||
def test_safety_valve_refuses_mass_deletion():
|
||||
"""If (almost) every object looked orphaned - e.g. the reconciler pointed at an empty database -
|
||||
--apply must stop before deleting anything."""
|
||||
orphan = new_key()
|
||||
s3_put(orphan, SAMPLE)
|
||||
time.sleep(9)
|
||||
report = reconcile("--apply", "--max-orphan-fraction", "0.000001", *GRACE, expect_code=3)
|
||||
assert report["error"].startswith("safety_valve") and report["actions"]["applied"] == 0
|
||||
assert orphan in s3_keys()
|
||||
reconcile("--apply", *GRACE)
|
||||
assert orphan not in s3_keys()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("service", ["s3", "postgres"])
|
||||
def test_store_outage_stops_the_run_without_damage(api, service):
|
||||
with service_down(service, "stop"):
|
||||
report = reconcile(*GRACE, expect_code=1)
|
||||
assert report["error"].startswith("store_unavailable")
|
||||
assert reconcile(*GRACE)["error"] is None
|
||||
|
||||
|
||||
def test_live_uploads_are_never_hurt_by_concurrent_apply(api):
|
||||
"""Uploads and messages run while --apply runs repeatedly: every upload that got 201 stays
|
||||
ready and downloadable byte for byte, every message keeps its attachments."""
|
||||
alice = api.user("alice")
|
||||
chat = api.chat(alice)
|
||||
payloads = [imagegen.png(16 + i, 16) for i in range(40)]
|
||||
done: list[tuple[dict, bytes]] = []
|
||||
|
||||
def worker(chunk: list[bytes]) -> None:
|
||||
for p in chunk:
|
||||
att = api.upload(alice, chat, p).json()
|
||||
api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]})
|
||||
done.append((att, p))
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(4) as pool:
|
||||
futures = [pool.submit(worker, payloads[i::4]) for i in range(4)]
|
||||
runs = 0
|
||||
while not all(x.done() for x in futures):
|
||||
report = reconcile("--apply", *GRACE)
|
||||
assert report["error"] is None
|
||||
runs += 1
|
||||
for x in futures:
|
||||
x.result()
|
||||
assert runs >= 1 and len(done) == len(payloads)
|
||||
for att, p in done:
|
||||
row = attachment_row(att["id"])
|
||||
assert row["status"] == "ready" and row["lost_reason"] is None and s3_size(row["object_key"]) == len(p)
|
||||
for att, p in done[::8]:
|
||||
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]).content == p
|
||||
Reference in New Issue
Block a user