feat: Carry audited pipeline replay and fault probes into lab6 (U4)
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
-- A replay starts a new bounded run without deleting earlier outbox history.
|
||||
ALTER TABLE attachment_jobs
|
||||
ADD COLUMN replay_start_attempt integer NOT NULL DEFAULT 1
|
||||
CHECK (replay_start_attempt > 0 AND replay_start_attempt <= attempt);
|
||||
|
||||
CREATE TABLE attachment_job_replays (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE,
|
||||
previous_attempt integer NOT NULL CHECK (previous_attempt > 0),
|
||||
new_attempt integer NOT NULL CHECK (new_attempt > previous_attempt),
|
||||
previous_error_code text NOT NULL,
|
||||
operator_name text NOT NULL CHECK (length(operator_name) BETWEEN 1 AND 80),
|
||||
reason text NOT NULL CHECK (length(reason) BETWEEN 1 AND 200),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
UNIQUE (job_id, new_attempt)
|
||||
);
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Small synthetic image workload; reports upload and time-to-ready separately."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
import zlib
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure # noqa: E402
|
||||
|
||||
|
||||
def png_noise(width, height):
|
||||
raw = b"".join(b"\0" + os.urandom(width * 3) for _ in range(height))
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
|
||||
|
||||
return (b"\x89PNG\r\n\x1a\n" +
|
||||
chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) +
|
||||
chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b""))
|
||||
|
||||
|
||||
def percentile(values, percentile_value):
|
||||
values = sorted(values)
|
||||
return round(values[max(0, math.ceil(len(values) * percentile_value) - 1)], 3)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--concurrency", type=int, default=4)
|
||||
parser.add_argument("--per-size", type=int, default=4)
|
||||
parser.add_argument("--output", required=True)
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.concurrency <= 16 or not 1 <= args.per_size <= 100:
|
||||
parser.error("invalid workload size")
|
||||
|
||||
setup = Client(args.base_url)
|
||||
alice, _bob, _eve, chat = setup.bootstrap()
|
||||
images = {
|
||||
"small_320x200": (ROOT / "tests/fixtures/sample.png").read_bytes(),
|
||||
"noise_800x600": png_noise(800, 600),
|
||||
}
|
||||
items = [(name, data) for name, data in images.items() for _ in range(args.per_size)]
|
||||
|
||||
def work(item):
|
||||
name, data = item
|
||||
client = Client(args.base_url, timeout=20)
|
||||
start = time.monotonic()
|
||||
response = client.call(
|
||||
"POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}.png",
|
||||
202, actor=alice, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment",
|
||||
)
|
||||
accepted = time.monotonic()
|
||||
attachment = response["json"]
|
||||
if attachment["sha256"] != hashlib.sha256(data).hexdigest():
|
||||
raise Failure("accepted checksum mismatch")
|
||||
client.wait_ready(alice, attachment, wait_seconds=180)
|
||||
ready = time.monotonic()
|
||||
return {"profile": name, "size_bytes": len(data),
|
||||
"upload_ms": round((accepted - start) * 1000, 3),
|
||||
"ready_ms": round((ready - start) * 1000, 3)}
|
||||
|
||||
started = time.monotonic()
|
||||
completed, failures = [], []
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
|
||||
for future in as_completed([executor.submit(work, item) for item in items]):
|
||||
try:
|
||||
completed.append(future.result())
|
||||
except (Failure, OSError, ValueError) as exc:
|
||||
failures.append(type(exc).__name__)
|
||||
elapsed = time.monotonic() - started
|
||||
result = {
|
||||
"lab": 6, "workload": "synthetic PNG upload and poll until ready",
|
||||
"concurrency": args.concurrency, "per_size": args.per_size,
|
||||
"elapsed_seconds": round(elapsed, 3), "accepted_ready": len(completed),
|
||||
"failed": len(failures), "failure_types": sorted(set(failures)),
|
||||
"profiles": {},
|
||||
}
|
||||
for name, data in images.items():
|
||||
rows = [item for item in completed if item["profile"] == name]
|
||||
result["profiles"][name] = {
|
||||
"size_bytes": len(data), "successful": len(rows),
|
||||
"upload_ms": {f"p{p}": percentile([r["upload_ms"] for r in rows], p / 100)
|
||||
for p in (50, 95, 99)} if rows else {},
|
||||
"ready_ms": {f"p{p}": percentile([r["ready_ms"] for r in rows], p / 100)
|
||||
for p in (50, 95, 99)} if rows else {},
|
||||
}
|
||||
Path(args.output).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
if failures or len(completed) != len(items):
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -155,7 +155,7 @@ def claim_job(conn, job_id, delivery_attempt, lease_seconds=60):
|
||||
def _reschedule_locked(conn, job, *, max_attempts, delay_seconds, error_code, permanent):
|
||||
if max_attempts < 1 or delay_seconds < 0:
|
||||
raise ValueError("invalid retry configuration")
|
||||
failed = permanent or job["attempt"] >= max_attempts
|
||||
failed = permanent or job["attempt"] - job["replay_start_attempt"] + 1 >= max_attempts
|
||||
if failed:
|
||||
code = error_code if permanent else "retry_exhausted"
|
||||
conn.execute(
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Explicit, audited replay of one failed current-version job."""
|
||||
|
||||
import argparse
|
||||
from uuid import UUID
|
||||
|
||||
from .db import pool
|
||||
|
||||
|
||||
def replay_failed(conn, job_id: UUID, operator: str, reason: str) -> int:
|
||||
if not 1 <= len(operator) <= 80 or not 1 <= len(reason) <= 200:
|
||||
raise ValueError("operator and reason are required")
|
||||
with conn.transaction():
|
||||
job = conn.execute(
|
||||
"SELECT j.*,a.status AS attachment_status,"
|
||||
"a.pipeline_version AS current_pipeline_version "
|
||||
"FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
|
||||
"WHERE j.id=%s FOR UPDATE OF j,a",
|
||||
(job_id,),
|
||||
).fetchone()
|
||||
if (job is None or job["status"] != "failed" or
|
||||
job["pipeline_version"] != job["current_pipeline_version"] or
|
||||
job["attachment_status"] != "failed"):
|
||||
raise ValueError("only a failed current-version job can be replayed")
|
||||
attempt = job["attempt"] + 1
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET status='queued',attempt=%s,replay_start_attempt=%s,"
|
||||
"next_attempt_at=clock_timestamp(),error_code=NULL,updated_at=clock_timestamp() "
|
||||
"WHERE id=%s",
|
||||
(attempt, attempt, job_id),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() "
|
||||
"WHERE id=%s AND pipeline_version=%s",
|
||||
(job["attachment_id"], job["pipeline_version"]),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,%s)",
|
||||
(job_id, attempt),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_job_replays "
|
||||
"(job_id,previous_attempt,new_attempt,previous_error_code,operator_name,reason) "
|
||||
"VALUES (%s,%s,%s,%s,%s,%s)",
|
||||
(job_id, job["attempt"], attempt, job["error_code"], operator, reason),
|
||||
)
|
||||
return attempt
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("job_id", type=UUID)
|
||||
parser.add_argument("--operator", required=True)
|
||||
parser.add_argument("--reason", required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
with pool().connection() as conn:
|
||||
attempt = replay_failed(conn, args.job_id, args.operator, args.reason)
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
print(f"Queued job {args.job_id} attempt {attempt}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Start a new pipeline version from a verified, already ready original."""
|
||||
|
||||
import argparse
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from .db import pool
|
||||
|
||||
|
||||
def start_new_version(conn, attachment_id: UUID) -> tuple[int, UUID]:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)
|
||||
).fetchone()
|
||||
if row is None or row["status"] != "ready":
|
||||
raise ValueError("only a ready attachment can start a new version")
|
||||
original = conn.execute(
|
||||
"SELECT object_key,size_bytes,sha256 FROM attachment_variants "
|
||||
"WHERE attachment_id=%s AND pipeline_version=%s AND name='original'",
|
||||
(attachment_id, row["pipeline_version"]),
|
||||
).fetchone()
|
||||
if (original is None or original["object_key"] != row["object_key"] or
|
||||
original["size_bytes"] != row["size_bytes"] or
|
||||
original["sha256"] != row["sha256"]):
|
||||
raise ValueError("verified original is missing")
|
||||
version = row["pipeline_version"] + 1
|
||||
job_id = uuid4()
|
||||
conn.execute(
|
||||
"UPDATE attachments SET pipeline_version=%s,status='queued',error_code=NULL,"
|
||||
"updated_at=clock_timestamp() WHERE id=%s",
|
||||
(version, attachment_id),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_jobs (id,attachment_id,pipeline_version) "
|
||||
"VALUES (%s,%s,%s)",
|
||||
(job_id, attachment_id, version),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,1)",
|
||||
(job_id,),
|
||||
)
|
||||
return version, job_id
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("attachment_id", type=UUID)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
with pool().connection() as conn:
|
||||
version, job_id = start_new_version(conn, args.attachment_id)
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
print(f"Queued attachment {args.attachment_id} version {version} job {job_id}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -110,6 +110,13 @@ def _delay(attempt: int) -> float:
|
||||
return base + random.uniform(0, base / 2)
|
||||
|
||||
|
||||
def _fault(point: str, job_id) -> None:
|
||||
"""Test-only process death at a named durable boundary for one job."""
|
||||
if (os.environ.get("LAB5_FAULT_POINT") == point and
|
||||
os.environ.get("LAB5_FAULT_JOB_ID") == str(job_id)):
|
||||
os._exit(72)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _deadline(seconds: int):
|
||||
def expired(_signal, _frame):
|
||||
@@ -130,6 +137,7 @@ def process_message(item: dict) -> str:
|
||||
if job is None:
|
||||
return "duplicate_or_stale"
|
||||
token = job["lease_token"]
|
||||
_fault("after_claim_before_s3", job["id"])
|
||||
try:
|
||||
with _deadline(LEASE_SECONDS):
|
||||
with tracer.start_as_current_span("s3.read_source", record_exception=False, set_status_on_exception=False):
|
||||
@@ -138,9 +146,11 @@ def process_message(item: dict) -> str:
|
||||
variants = render(data, job["content_type"])
|
||||
with tracer.start_as_current_span("s3.write_variants", record_exception=False, set_status_on_exception=False):
|
||||
stored = _write_variants(job, variants)
|
||||
_fault("after_s3_before_result_commit", job["id"])
|
||||
with pool().connection() as conn:
|
||||
with tracer.start_as_current_span("db.complete_job", record_exception=False, set_status_on_exception=False):
|
||||
current = complete_job(conn, job["id"], token, stored)
|
||||
_fault("after_result_commit_before_ack", job["id"])
|
||||
if current and job["staging_key"]:
|
||||
try:
|
||||
storage().remove_object(BUCKET, job["staging_key"])
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Run in a one-shot publisher container to inject publish-before-confirm gap."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pika
|
||||
|
||||
from src.db import pool
|
||||
from src.publisher import QUEUE, job_payload, publish_batch
|
||||
|
||||
|
||||
attachment_id = UUID(os.environ["FAULT_ATTACHMENT_ID"])
|
||||
url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip()
|
||||
broker = pika.BlockingConnection(pika.URLParameters(url))
|
||||
try:
|
||||
channel = broker.channel()
|
||||
channel.queue_declare(queue=QUEUE, passive=True)
|
||||
channel.confirm_delivery()
|
||||
token = uuid4()
|
||||
with pool().connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT o.id,o.job_id,o.attempt,j.attachment_id,j.pipeline_version,j.created_at "
|
||||
"FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id "
|
||||
"WHERE j.attachment_id=%s AND j.pipeline_version=1 AND o.attempt=1",
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
assert row is not None
|
||||
claimed = conn.execute(
|
||||
"UPDATE attachment_outbox SET claim_token=%s,"
|
||||
"claim_expires_at=clock_timestamp()+interval '30 seconds' "
|
||||
"WHERE id=%s AND claim_token IS NULL AND published_at IS NULL RETURNING id",
|
||||
(token, row["id"]),
|
||||
).fetchone()
|
||||
assert claimed is not None
|
||||
body = json.dumps(job_payload(row), separators=(",", ":")).encode()
|
||||
channel.basic_publish(
|
||||
exchange="", routing_key=QUEUE, mandatory=True, body=body,
|
||||
properties=pika.BasicProperties(content_type="application/json", delivery_mode=2),
|
||||
)
|
||||
# Simulate process death after broker confirm, before PostgreSQL confirm.
|
||||
with pool().connection() as conn:
|
||||
conn.execute(
|
||||
"UPDATE attachment_outbox SET claim_expires_at=clock_timestamp()-interval '1 second' "
|
||||
"WHERE id=%s AND claim_token=%s AND published_at IS NULL",
|
||||
(row["id"], token),
|
||||
)
|
||||
published = publish_batch(channel)
|
||||
with pool().connection() as conn:
|
||||
outbox = conn.execute(
|
||||
"SELECT published_at IS NOT NULL AS confirmed FROM attachment_outbox "
|
||||
"WHERE id=%s", (row["id"],),
|
||||
).fetchone()
|
||||
count = channel.queue_declare(queue=QUEUE, passive=True).method.message_count
|
||||
assert published >= 1 and outbox["confirmed"] and count >= 2
|
||||
print(json.dumps({"confirmed_after_republish": True, "queued_copies_before_worker": count}))
|
||||
finally:
|
||||
broker.close()
|
||||
@@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic fault windows for the versioned attachment pipeline.
|
||||
|
||||
Run in a one-shot worker image while the normal worker and publisher are stopped.
|
||||
All rows and objects are synthetic and stay in the educational volumes.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from io import BytesIO
|
||||
from uuid import uuid4
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from src.attachments import BUCKET, storage
|
||||
from src.db import pool
|
||||
from src.pipeline_jobs import (
|
||||
claim_job, finish_attempt, queue_attachment, recover_expired, reserve_upload,
|
||||
)
|
||||
from src.replay import replay_failed
|
||||
from src.reprocess import start_new_version
|
||||
from src.worker import process_message
|
||||
|
||||
|
||||
image = BytesIO()
|
||||
Image.new("RGB", (320, 200), "#2765a8").save(image, format="PNG")
|
||||
DATA = image.getvalue()
|
||||
SHA = hashlib.sha256(DATA).hexdigest()
|
||||
|
||||
|
||||
def create_case():
|
||||
user_id, chat_id, attachment_id = uuid4(), uuid4(), uuid4()
|
||||
staging_key = f"staging/{attachment_id}"
|
||||
with pool().connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute(
|
||||
"INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,%s,%s,%s)",
|
||||
(user_id, f"fault_{user_id.hex[:20]}", "Pipeline fault fixture", "synthetic"),
|
||||
)
|
||||
conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s)", (chat_id, "Pipeline fault fixture"))
|
||||
conn.execute(
|
||||
"INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)",
|
||||
(chat_id, user_id),
|
||||
)
|
||||
reserve_upload(conn, attachment_id, staging_key)
|
||||
storage().put_object(BUCKET, staging_key, BytesIO(DATA), len(DATA), content_type="image/png")
|
||||
with pool().connection() as conn:
|
||||
_, job_id = queue_attachment(
|
||||
conn, attachment_id=attachment_id, chat_id=chat_id, owner_id=user_id,
|
||||
filename="fixture.png", staging_key=staging_key,
|
||||
object_key=f"originals/{attachment_id}", content_type="image/png",
|
||||
size_bytes=len(DATA), sha256=SHA,
|
||||
)
|
||||
return attachment_id, job_id
|
||||
|
||||
|
||||
def item(job_id, attempt=1):
|
||||
return {"job_id": str(job_id), "attempt": attempt}
|
||||
|
||||
|
||||
def state(attachment_id, version):
|
||||
with pool().connection() as conn:
|
||||
attachment = conn.execute(
|
||||
"SELECT status,pipeline_version,sha256 FROM attachments WHERE id=%s", (attachment_id,)
|
||||
).fetchone()
|
||||
rows = conn.execute(
|
||||
"SELECT name,sha256 FROM attachment_variants "
|
||||
"WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name",
|
||||
(attachment_id, version),
|
||||
).fetchall()
|
||||
return attachment, {row["name"]: row["sha256"] for row in rows}
|
||||
|
||||
|
||||
def assert_ready(attachment_id, version):
|
||||
attachment, variants = state(attachment_id, version)
|
||||
assert attachment["status"] == "ready" and attachment["pipeline_version"] == version
|
||||
assert attachment["sha256"] == SHA
|
||||
assert set(variants) == {"original", "thumbnail", "crop", "optimized"}
|
||||
assert variants["original"] == SHA
|
||||
return variants
|
||||
|
||||
|
||||
def expire_and_recover(job_id):
|
||||
with pool().connection() as conn:
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET lease_expires_at=clock_timestamp()-interval '1 second' "
|
||||
"WHERE id=%s AND status='processing'",
|
||||
(job_id,),
|
||||
)
|
||||
recovered = recover_expired(conn, max_attempts=5)
|
||||
assert (job_id, "queued") in recovered
|
||||
|
||||
|
||||
def crash(job_id, point):
|
||||
child = subprocess.run(
|
||||
[sys.executable, "-c", "from src.worker import process_message; "
|
||||
f"process_message({{'job_id': '{job_id}', 'attempt': 1}})"],
|
||||
env={**os.environ, "LAB5_FAULT_POINT": point, "LAB5_FAULT_JOB_ID": str(job_id)},
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
assert child.returncode == 72, (point, child.returncode, child.stderr)
|
||||
|
||||
|
||||
def main():
|
||||
results = {}
|
||||
|
||||
attachment, job = create_case()
|
||||
crash(job, "after_result_commit_before_ack")
|
||||
first = assert_ready(attachment, 1)
|
||||
assert process_message(item(job)) == "duplicate_or_stale"
|
||||
assert assert_ready(attachment, 1) == first
|
||||
results["after_result_commit_before_ack_duplicate"] = "pass"
|
||||
|
||||
attachment, job = create_case()
|
||||
crash(job, "after_claim_before_s3")
|
||||
expire_and_recover(job)
|
||||
assert process_message(item(job, 2)) == "ready"
|
||||
assert_ready(attachment, 1)
|
||||
results["before_side_effect_lease_recovery"] = "pass"
|
||||
|
||||
attachment, job = create_case()
|
||||
crash(job, "after_s3_before_result_commit")
|
||||
expire_and_recover(job)
|
||||
assert process_message(item(job, 2)) == "ready"
|
||||
assert_ready(attachment, 1)
|
||||
results["after_s3_before_result_commit"] = "pass"
|
||||
|
||||
attachment, old_job = create_case()
|
||||
assert process_message(item(old_job)) == "ready"
|
||||
with pool().connection() as conn:
|
||||
version, new_job = start_new_version(conn, attachment)
|
||||
assert version == 2
|
||||
assert process_message(item(old_job)) == "duplicate_or_stale"
|
||||
assert process_message(item(new_job)) == "ready"
|
||||
newer = assert_ready(attachment, 2)
|
||||
assert process_message(item(old_job)) == "duplicate_or_stale"
|
||||
assert assert_ready(attachment, 2) == newer
|
||||
assert len(state(attachment, 1)[1]) == 4
|
||||
results["duplicate_reorder_new_version"] = "pass"
|
||||
|
||||
attachment, job = create_case()
|
||||
with pool().connection() as conn:
|
||||
claimed = claim_job(conn, job, 1)
|
||||
assert finish_attempt(
|
||||
conn, job, claimed["lease_token"], max_attempts=5,
|
||||
error_code="invalid_image", permanent=True,
|
||||
) == "failed"
|
||||
attempt = replay_failed(conn, job, "lab5-test", "verified synthetic replay")
|
||||
assert attempt == 2
|
||||
assert process_message(item(job, attempt)) == "ready"
|
||||
assert_ready(attachment, 1)
|
||||
with pool().connection() as conn:
|
||||
count = conn.execute(
|
||||
"SELECT count(*) AS n FROM attachment_job_replays WHERE job_id=%s", (job,)
|
||||
).fetchone()["n"]
|
||||
assert count == 1
|
||||
results["failed_registry_operator_replay"] = "pass"
|
||||
|
||||
attachment, job = create_case()
|
||||
with pool().connection() as conn:
|
||||
claimed = claim_job(conn, job, 1)
|
||||
assert finish_attempt(
|
||||
conn, job, claimed["lease_token"], max_attempts=2,
|
||||
delay_seconds=0.01, error_code="storage_unavailable",
|
||||
) == "queued"
|
||||
time.sleep(0.03)
|
||||
with pool().connection() as conn:
|
||||
claimed = claim_job(conn, job, 2)
|
||||
assert claimed is not None
|
||||
assert finish_attempt(
|
||||
conn, job, claimed["lease_token"], max_attempts=2,
|
||||
error_code="storage_unavailable",
|
||||
) == "failed"
|
||||
attempt = replay_failed(conn, job, "lab5-test", "retry budget reset after repair")
|
||||
assert attempt == 3
|
||||
assert process_message(item(job, attempt)) == "ready"
|
||||
assert_ready(attachment, 1)
|
||||
results["bounded_transient_retry_then_replay"] = "pass"
|
||||
|
||||
print(json.dumps(results, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Show that broker confirm without outbox confirmation safely republishes."""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run # noqa: E402
|
||||
|
||||
|
||||
def compose(*args, input_data=None):
|
||||
return subprocess.run(
|
||||
["docker", "compose", *args], cwd=ROOT,
|
||||
input=input_data, check=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Lab6 publish/confirm gap")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
compose("stop", "worker", "publisher")
|
||||
attachment = None
|
||||
try:
|
||||
alice, bob, _eve, chat = client.bootstrap()
|
||||
attachment, _original = client.attachment(alice, chat)
|
||||
check(attachment["status"] == "queued", "upload was not queued")
|
||||
source = (ROOT / "student-tests/publish_gap_inner.py").read_bytes()
|
||||
compose(
|
||||
"run", "--rm", "--no-deps", "-T",
|
||||
"-e", f"FAULT_ATTACHMENT_ID={attachment['id']}", "publisher", "python", "-",
|
||||
input_data=source,
|
||||
)
|
||||
finally:
|
||||
compose("up", "-d", "publisher", "worker")
|
||||
ready = client.wait_ready(bob, attachment)
|
||||
check(ready["status"] == "ready", "republished job did not finish")
|
||||
check(len(ready["variants"]) == 3, "duplicate delivery changed variant set")
|
||||
print("PASS: confirmed broker copy republished after lost DB confirm; worker reached ready once")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Reference in New Issue
Block a user