feat: Carry audited pipeline replay and fault probes into lab6 (U4)

This commit is contained in:
Самохвалов Андрей Ремович
2026-10-02 15:12:27 +03:00
parent 8eab283f6f
commit e5d8796dc6
9 changed files with 547 additions and 1 deletions
+16
View File
@@ -0,0 +1,16 @@
-- A replay starts a new bounded run without deleting earlier outbox history.
ALTER TABLE attachment_jobs
ADD COLUMN replay_start_attempt integer NOT NULL DEFAULT 1
CHECK (replay_start_attempt > 0 AND replay_start_attempt <= attempt);
CREATE TABLE attachment_job_replays (
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE,
previous_attempt integer NOT NULL CHECK (previous_attempt > 0),
new_attempt integer NOT NULL CHECK (new_attempt > previous_attempt),
previous_error_code text NOT NULL,
operator_name text NOT NULL CHECK (length(operator_name) BETWEEN 1 AND 80),
reason text NOT NULL CHECK (length(reason) BETWEEN 1 AND 200),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE (job_id, new_attempt)
);
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env python3
"""Small synthetic image workload; reports upload and time-to-ready separately."""
import argparse
import hashlib
import json
import math
import os
import struct
import sys
import time
import uuid
import zlib
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
def png_noise(width, height):
raw = b"".join(b"\0" + os.urandom(width * 3) for _ in range(height))
def chunk(kind, data):
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
return (b"\x89PNG\r\n\x1a\n" +
chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) +
chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b""))
def percentile(values, percentile_value):
values = sorted(values)
return round(values[max(0, math.ceil(len(values) * percentile_value) - 1)], 3)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--concurrency", type=int, default=4)
parser.add_argument("--per-size", type=int, default=4)
parser.add_argument("--output", required=True)
args = parser.parse_args()
if not 1 <= args.concurrency <= 16 or not 1 <= args.per_size <= 100:
parser.error("invalid workload size")
setup = Client(args.base_url)
alice, _bob, _eve, chat = setup.bootstrap()
images = {
"small_320x200": (ROOT / "tests/fixtures/sample.png").read_bytes(),
"noise_800x600": png_noise(800, 600),
}
items = [(name, data) for name, data in images.items() for _ in range(args.per_size)]
def work(item):
name, data = item
client = Client(args.base_url, timeout=20)
start = time.monotonic()
response = client.call(
"POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}.png",
202, actor=alice, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment",
)
accepted = time.monotonic()
attachment = response["json"]
if attachment["sha256"] != hashlib.sha256(data).hexdigest():
raise Failure("accepted checksum mismatch")
client.wait_ready(alice, attachment, wait_seconds=180)
ready = time.monotonic()
return {"profile": name, "size_bytes": len(data),
"upload_ms": round((accepted - start) * 1000, 3),
"ready_ms": round((ready - start) * 1000, 3)}
started = time.monotonic()
completed, failures = [], []
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
for future in as_completed([executor.submit(work, item) for item in items]):
try:
completed.append(future.result())
except (Failure, OSError, ValueError) as exc:
failures.append(type(exc).__name__)
elapsed = time.monotonic() - started
result = {
"lab": 6, "workload": "synthetic PNG upload and poll until ready",
"concurrency": args.concurrency, "per_size": args.per_size,
"elapsed_seconds": round(elapsed, 3), "accepted_ready": len(completed),
"failed": len(failures), "failure_types": sorted(set(failures)),
"profiles": {},
}
for name, data in images.items():
rows = [item for item in completed if item["profile"] == name]
result["profiles"][name] = {
"size_bytes": len(data), "successful": len(rows),
"upload_ms": {f"p{p}": percentile([r["upload_ms"] for r in rows], p / 100)
for p in (50, 95, 99)} if rows else {},
"ready_ms": {f"p{p}": percentile([r["ready_ms"] for r in rows], p / 100)
for p in (50, 95, 99)} if rows else {},
}
Path(args.output).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps(result, indent=2, sort_keys=True))
if failures or len(completed) != len(items):
raise SystemExit(1)
if __name__ == "__main__":
main()
+1 -1
View File
@@ -155,7 +155,7 @@ def claim_job(conn, job_id, delivery_attempt, lease_seconds=60):
def _reschedule_locked(conn, job, *, max_attempts, delay_seconds, error_code, permanent):
if max_attempts < 1 or delay_seconds < 0:
raise ValueError("invalid retry configuration")
failed = permanent or job["attempt"] >= max_attempts
failed = permanent or job["attempt"] - job["replay_start_attempt"] + 1 >= max_attempts
if failed:
code = error_code if permanent else "retry_exhausted"
conn.execute(
+64
View File
@@ -0,0 +1,64 @@
"""Explicit, audited replay of one failed current-version job."""
import argparse
from uuid import UUID
from .db import pool
def replay_failed(conn, job_id: UUID, operator: str, reason: str) -> int:
if not 1 <= len(operator) <= 80 or not 1 <= len(reason) <= 200:
raise ValueError("operator and reason are required")
with conn.transaction():
job = conn.execute(
"SELECT j.*,a.status AS attachment_status,"
"a.pipeline_version AS current_pipeline_version "
"FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.id=%s FOR UPDATE OF j,a",
(job_id,),
).fetchone()
if (job is None or job["status"] != "failed" or
job["pipeline_version"] != job["current_pipeline_version"] or
job["attachment_status"] != "failed"):
raise ValueError("only a failed current-version job can be replayed")
attempt = job["attempt"] + 1
conn.execute(
"UPDATE attachment_jobs SET status='queued',attempt=%s,replay_start_attempt=%s,"
"next_attempt_at=clock_timestamp(),error_code=NULL,updated_at=clock_timestamp() "
"WHERE id=%s",
(attempt, attempt, job_id),
)
conn.execute(
"UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() "
"WHERE id=%s AND pipeline_version=%s",
(job["attachment_id"], job["pipeline_version"]),
)
conn.execute(
"INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,%s)",
(job_id, attempt),
)
conn.execute(
"INSERT INTO attachment_job_replays "
"(job_id,previous_attempt,new_attempt,previous_error_code,operator_name,reason) "
"VALUES (%s,%s,%s,%s,%s,%s)",
(job_id, job["attempt"], attempt, job["error_code"], operator, reason),
)
return attempt
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("job_id", type=UUID)
parser.add_argument("--operator", required=True)
parser.add_argument("--reason", required=True)
args = parser.parse_args()
try:
with pool().connection() as conn:
attempt = replay_failed(conn, args.job_id, args.operator, args.reason)
except ValueError as exc:
parser.error(str(exc))
print(f"Queued job {args.job_id} attempt {attempt}")
if __name__ == "__main__":
main()
+57
View File
@@ -0,0 +1,57 @@
"""Start a new pipeline version from a verified, already ready original."""
import argparse
from uuid import UUID, uuid4
from .db import pool
def start_new_version(conn, attachment_id: UUID) -> tuple[int, UUID]:
with conn.transaction():
row = conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
).fetchone()
if row is None or row["status"] != "ready":
raise ValueError("only a ready attachment can start a new version")
original = conn.execute(
"SELECT object_key,size_bytes,sha256 FROM attachment_variants "
"WHERE attachment_id=%s AND pipeline_version=%s AND name='original'",
(attachment_id, row["pipeline_version"]),
).fetchone()
if (original is None or original["object_key"] != row["object_key"] or
original["size_bytes"] != row["size_bytes"] or
original["sha256"] != row["sha256"]):
raise ValueError("verified original is missing")
version = row["pipeline_version"] + 1
job_id = uuid4()
conn.execute(
"UPDATE attachments SET pipeline_version=%s,status='queued',error_code=NULL,"
"updated_at=clock_timestamp() WHERE id=%s",
(version, attachment_id),
)
conn.execute(
"INSERT INTO attachment_jobs (id,attachment_id,pipeline_version) "
"VALUES (%s,%s,%s)",
(job_id, attachment_id, version),
)
conn.execute(
"INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,1)",
(job_id,),
)
return version, job_id
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("attachment_id", type=UUID)
args = parser.parse_args()
try:
with pool().connection() as conn:
version, job_id = start_new_version(conn, args.attachment_id)
except ValueError as exc:
parser.error(str(exc))
print(f"Queued attachment {args.attachment_id} version {version} job {job_id}")
if __name__ == "__main__":
main()
+10
View File
@@ -110,6 +110,13 @@ def _delay(attempt: int) -> float:
return base + random.uniform(0, base / 2)
def _fault(point: str, job_id) -> None:
"""Test-only process death at a named durable boundary for one job."""
if (os.environ.get("LAB5_FAULT_POINT") == point and
os.environ.get("LAB5_FAULT_JOB_ID") == str(job_id)):
os._exit(72)
@contextmanager
def _deadline(seconds: int):
def expired(_signal, _frame):
@@ -130,6 +137,7 @@ def process_message(item: dict) -> str:
if job is None:
return "duplicate_or_stale"
token = job["lease_token"]
_fault("after_claim_before_s3", job["id"])
try:
with _deadline(LEASE_SECONDS):
with tracer.start_as_current_span("s3.read_source", record_exception=False, set_status_on_exception=False):
@@ -138,9 +146,11 @@ def process_message(item: dict) -> str:
variants = render(data, job["content_type"])
with tracer.start_as_current_span("s3.write_variants", record_exception=False, set_status_on_exception=False):
stored = _write_variants(job, variants)
_fault("after_s3_before_result_commit", job["id"])
with pool().connection() as conn:
with tracer.start_as_current_span("db.complete_job", record_exception=False, set_status_on_exception=False):
current = complete_job(conn, job["id"], token, stored)
_fault("after_result_commit_before_ack", job["id"])
if current and job["staging_key"]:
try:
storage().remove_object(BUCKET, job["staging_key"])
+59
View File
@@ -0,0 +1,59 @@
"""Run in a one-shot publisher container to inject publish-before-confirm gap."""
import json
import os
from pathlib import Path
from uuid import UUID, uuid4
import pika
from src.db import pool
from src.publisher import QUEUE, job_payload, publish_batch
attachment_id = UUID(os.environ["FAULT_ATTACHMENT_ID"])
url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip()
broker = pika.BlockingConnection(pika.URLParameters(url))
try:
channel = broker.channel()
channel.queue_declare(queue=QUEUE, passive=True)
channel.confirm_delivery()
token = uuid4()
with pool().connection() as conn:
row = conn.execute(
"SELECT o.id,o.job_id,o.attempt,j.attachment_id,j.pipeline_version,j.created_at "
"FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id "
"WHERE j.attachment_id=%s AND j.pipeline_version=1 AND o.attempt=1",
(attachment_id,),
).fetchone()
assert row is not None
claimed = conn.execute(
"UPDATE attachment_outbox SET claim_token=%s,"
"claim_expires_at=clock_timestamp()+interval '30 seconds' "
"WHERE id=%s AND claim_token IS NULL AND published_at IS NULL RETURNING id",
(token, row["id"]),
).fetchone()
assert claimed is not None
body = json.dumps(job_payload(row), separators=(",", ":")).encode()
channel.basic_publish(
exchange="", routing_key=QUEUE, mandatory=True, body=body,
properties=pika.BasicProperties(content_type="application/json", delivery_mode=2),
)
# Simulate process death after broker confirm, before PostgreSQL confirm.
with pool().connection() as conn:
conn.execute(
"UPDATE attachment_outbox SET claim_expires_at=clock_timestamp()-interval '1 second' "
"WHERE id=%s AND claim_token=%s AND published_at IS NULL",
(row["id"], token),
)
published = publish_batch(channel)
with pool().connection() as conn:
outbox = conn.execute(
"SELECT published_at IS NOT NULL AS confirmed FROM attachment_outbox "
"WHERE id=%s", (row["id"],),
).fetchone()
count = channel.queue_declare(queue=QUEUE, passive=True).method.message_count
assert published >= 1 and outbox["confirmed"] and count >= 2
print(json.dumps({"confirmed_after_republish": True, "queued_copies_before_worker": count}))
finally:
broker.close()
+188
View File
@@ -0,0 +1,188 @@
#!/usr/bin/env python3
"""Deterministic fault windows for the versioned attachment pipeline.
Run in a one-shot worker image while the normal worker and publisher are stopped.
All rows and objects are synthetic and stay in the educational volumes.
"""
import hashlib
import json
import os
import subprocess
import sys
import time
from io import BytesIO
from uuid import uuid4
from PIL import Image
from src.attachments import BUCKET, storage
from src.db import pool
from src.pipeline_jobs import (
claim_job, finish_attempt, queue_attachment, recover_expired, reserve_upload,
)
from src.replay import replay_failed
from src.reprocess import start_new_version
from src.worker import process_message
image = BytesIO()
Image.new("RGB", (320, 200), "#2765a8").save(image, format="PNG")
DATA = image.getvalue()
SHA = hashlib.sha256(DATA).hexdigest()
def create_case():
user_id, chat_id, attachment_id = uuid4(), uuid4(), uuid4()
staging_key = f"staging/{attachment_id}"
with pool().connection() as conn:
with conn.transaction():
conn.execute(
"INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,%s,%s,%s)",
(user_id, f"fault_{user_id.hex[:20]}", "Pipeline fault fixture", "synthetic"),
)
conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s)", (chat_id, "Pipeline fault fixture"))
conn.execute(
"INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)",
(chat_id, user_id),
)
reserve_upload(conn, attachment_id, staging_key)
storage().put_object(BUCKET, staging_key, BytesIO(DATA), len(DATA), content_type="image/png")
with pool().connection() as conn:
_, job_id = queue_attachment(
conn, attachment_id=attachment_id, chat_id=chat_id, owner_id=user_id,
filename="fixture.png", staging_key=staging_key,
object_key=f"originals/{attachment_id}", content_type="image/png",
size_bytes=len(DATA), sha256=SHA,
)
return attachment_id, job_id
def item(job_id, attempt=1):
return {"job_id": str(job_id), "attempt": attempt}
def state(attachment_id, version):
with pool().connection() as conn:
attachment = conn.execute(
"SELECT status,pipeline_version,sha256 FROM attachments WHERE id=%s", (attachment_id,)
).fetchone()
rows = conn.execute(
"SELECT name,sha256 FROM attachment_variants "
"WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name",
(attachment_id, version),
).fetchall()
return attachment, {row["name"]: row["sha256"] for row in rows}
def assert_ready(attachment_id, version):
attachment, variants = state(attachment_id, version)
assert attachment["status"] == "ready" and attachment["pipeline_version"] == version
assert attachment["sha256"] == SHA
assert set(variants) == {"original", "thumbnail", "crop", "optimized"}
assert variants["original"] == SHA
return variants
def expire_and_recover(job_id):
with pool().connection() as conn:
conn.execute(
"UPDATE attachment_jobs SET lease_expires_at=clock_timestamp()-interval '1 second' "
"WHERE id=%s AND status='processing'",
(job_id,),
)
recovered = recover_expired(conn, max_attempts=5)
assert (job_id, "queued") in recovered
def crash(job_id, point):
child = subprocess.run(
[sys.executable, "-c", "from src.worker import process_message; "
f"process_message({{'job_id': '{job_id}', 'attempt': 1}})"],
env={**os.environ, "LAB5_FAULT_POINT": point, "LAB5_FAULT_JOB_ID": str(job_id)},
capture_output=True, text=True,
)
assert child.returncode == 72, (point, child.returncode, child.stderr)
def main():
results = {}
attachment, job = create_case()
crash(job, "after_result_commit_before_ack")
first = assert_ready(attachment, 1)
assert process_message(item(job)) == "duplicate_or_stale"
assert assert_ready(attachment, 1) == first
results["after_result_commit_before_ack_duplicate"] = "pass"
attachment, job = create_case()
crash(job, "after_claim_before_s3")
expire_and_recover(job)
assert process_message(item(job, 2)) == "ready"
assert_ready(attachment, 1)
results["before_side_effect_lease_recovery"] = "pass"
attachment, job = create_case()
crash(job, "after_s3_before_result_commit")
expire_and_recover(job)
assert process_message(item(job, 2)) == "ready"
assert_ready(attachment, 1)
results["after_s3_before_result_commit"] = "pass"
attachment, old_job = create_case()
assert process_message(item(old_job)) == "ready"
with pool().connection() as conn:
version, new_job = start_new_version(conn, attachment)
assert version == 2
assert process_message(item(old_job)) == "duplicate_or_stale"
assert process_message(item(new_job)) == "ready"
newer = assert_ready(attachment, 2)
assert process_message(item(old_job)) == "duplicate_or_stale"
assert assert_ready(attachment, 2) == newer
assert len(state(attachment, 1)[1]) == 4
results["duplicate_reorder_new_version"] = "pass"
attachment, job = create_case()
with pool().connection() as conn:
claimed = claim_job(conn, job, 1)
assert finish_attempt(
conn, job, claimed["lease_token"], max_attempts=5,
error_code="invalid_image", permanent=True,
) == "failed"
attempt = replay_failed(conn, job, "lab5-test", "verified synthetic replay")
assert attempt == 2
assert process_message(item(job, attempt)) == "ready"
assert_ready(attachment, 1)
with pool().connection() as conn:
count = conn.execute(
"SELECT count(*) AS n FROM attachment_job_replays WHERE job_id=%s", (job,)
).fetchone()["n"]
assert count == 1
results["failed_registry_operator_replay"] = "pass"
attachment, job = create_case()
with pool().connection() as conn:
claimed = claim_job(conn, job, 1)
assert finish_attempt(
conn, job, claimed["lease_token"], max_attempts=2,
delay_seconds=0.01, error_code="storage_unavailable",
) == "queued"
time.sleep(0.03)
with pool().connection() as conn:
claimed = claim_job(conn, job, 2)
assert claimed is not None
assert finish_attempt(
conn, job, claimed["lease_token"], max_attempts=2,
error_code="storage_unavailable",
) == "failed"
attempt = replay_failed(conn, job, "lab5-test", "retry budget reset after repair")
assert attempt == 3
assert process_message(item(job, attempt)) == "ready"
assert_ready(attachment, 1)
results["bounded_transient_retry_then_replay"] = "pass"
print(json.dumps(results, sort_keys=True))
if __name__ == "__main__":
main()
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Show that broker confirm without outbox confirmation safely republishes."""
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run # noqa: E402
def compose(*args, input_data=None):
return subprocess.run(
["docker", "compose", *args], cwd=ROOT,
input=input_data, check=True,
)
def main():
args = arguments("Lab6 publish/confirm gap")
client = Client(args.base_url, args.ca_file)
compose("stop", "worker", "publisher")
attachment = None
try:
alice, bob, _eve, chat = client.bootstrap()
attachment, _original = client.attachment(alice, chat)
check(attachment["status"] == "queued", "upload was not queued")
source = (ROOT / "student-tests/publish_gap_inner.py").read_bytes()
compose(
"run", "--rm", "--no-deps", "-T",
"-e", f"FAULT_ATTACHMENT_ID={attachment['id']}", "publisher", "python", "-",
input_data=source,
)
finally:
compose("up", "-d", "publisher", "worker")
ready = client.wait_ready(bob, attachment)
check(ready["status"] == "ready", "republished job did not finish")
check(len(ready["variants"]) == 3, "duplicate delivery changed variant set")
print("PASS: confirmed broker copy republished after lost DB confirm; worker reached ready once")
if __name__ == "__main__":
run(main)