Add the level 3 pipeline scenarios and the stand evidence
Bring the stdlib suite and image generator of the lab 5 preparation: make suite-preflight checks services, readiness, schema columns, broker topology and the faults override, and make level3 runs the grade 3 scenarios (smoke through the pipeline, worker down with API, relay and broker restarts, duplicate delivery after ready, poison image, S3 outage with recovery, retries up to JOB_MAX_ATTEMPTS) and checks the invariants into evidence/level3.json. make pipeline-evidence shows the queued job on the broker while the worker is stopped, its properties and contract payload, the 409 before ready and SIGTERM of the relay and the worker; make topology dumps queues, consumers with prefetch, permissions and users. The lab 4 failover and S3 outage scenarios and the upload fault suite now expect 202 and wait for ready, the stack evidence lists the broker and its client packages, and the CI acceptance job runs the pipeline scenarios and the SIGTERM check of the new processes. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c55fb1b998
commit
fe2693870e
@@ -37,7 +37,10 @@ HEAD = (
|
||||
FAULTS = """
|
||||
CREATE TABLE lab4_fault_seen (id uuid PRIMARY KEY, object_key text, filename text);
|
||||
CREATE FUNCTION lab4_fault_seen() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN INSERT INTO lab4_fault_seen VALUES (NEW.id, NEW.object_key, NEW.filename); RETURN NEW; END $$;
|
||||
BEGIN
|
||||
INSERT INTO lab4_fault_seen VALUES (NEW.id, NEW.staging_key, NEW.filename);
|
||||
RETURN NEW;
|
||||
END $$;
|
||||
CREATE FUNCTION lab4_fault_commit() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NEW.filename LIKE 'fault-fail-%' THEN
|
||||
@@ -61,7 +64,7 @@ END $$;
|
||||
CREATE TRIGGER lab4_fault_seen AFTER INSERT ON attachments FOR EACH ROW
|
||||
WHEN (NEW.filename LIKE 'fault-%') EXECUTE FUNCTION lab4_fault_seen();
|
||||
CREATE TRIGGER lab4_fault_commit BEFORE UPDATE ON attachments FOR EACH ROW
|
||||
WHEN (NEW.filename LIKE 'fault-%' AND OLD.status = 'pending' AND NEW.status = 'ready')
|
||||
WHEN (NEW.filename LIKE 'fault-%' AND OLD.status = 'pending' AND NEW.status = 'queued')
|
||||
EXECUTE FUNCTION lab4_fault_commit();
|
||||
"""
|
||||
CLEANUP = """
|
||||
@@ -94,7 +97,7 @@ def bucket_keys() -> set[str]:
|
||||
"list",
|
||||
"--recursive",
|
||||
"--json",
|
||||
f"root/{BUCKET}/attachments/",
|
||||
f"root/{BUCKET}/staging/",
|
||||
)
|
||||
return {item["key"] for item in json.loads(listing)["items"]}
|
||||
|
||||
@@ -221,7 +224,7 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None:
|
||||
created = 0
|
||||
attachment = await upload(http, alice, chat)
|
||||
exact = await upload(http, alice, chat, png(16, 16, size=LIMIT))
|
||||
assert (attachment.status_code, exact.status_code) == (201, 201)
|
||||
assert (attachment.status_code, exact.status_code) == (202, 202)
|
||||
assert exact.json()["size_bytes"] == LIMIT
|
||||
created += 2
|
||||
mine = attachment.json()["id"]
|
||||
@@ -319,9 +322,9 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None:
|
||||
400,
|
||||
)
|
||||
expect(
|
||||
"variant=thumbnail",
|
||||
"variant=preview",
|
||||
await http.get(
|
||||
f"/api/v1/attachments/{mine}/content?variant=thumbnail", headers=bearer(alice)
|
||||
f"/api/v1/attachments/{mine}/content?variant=preview", headers=bearer(alice)
|
||||
),
|
||||
400,
|
||||
)
|
||||
@@ -332,7 +335,7 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None:
|
||||
)
|
||||
statuses = [entry["status"] for entry in logs(since) if entry.get("event") == "request.handled"]
|
||||
print(
|
||||
f"строк вложений в чате: {rows} (= ответов 201); ответов в журнале реплик: "
|
||||
f"строк вложений в чате: {rows} (= ответов 202); ответов в журнале реплик: "
|
||||
f"{len(statuses)}, из них 5xx: {sum(code >= 500 for code in statuses)}"
|
||||
)
|
||||
assert not [status for status in statuses if status >= 500]
|
||||
@@ -375,7 +378,7 @@ async def crash(http: AsyncClient, token: str, chat: str) -> tuple[str, str]:
|
||||
name = f"fault-crash-{uuid4().hex}.png"
|
||||
sent = asyncio.create_task(upload(http, token, chat, filename=name))
|
||||
address = await asyncio.to_thread(sql, "SELECT lab4_fault_sleeper()")
|
||||
assert address, "UPDATE ready не дошёл до триггера"
|
||||
assert address, "UPDATE queued не дошёл до триггера"
|
||||
victim = replica_at(address)
|
||||
kill(victim)
|
||||
try:
|
||||
@@ -396,6 +399,7 @@ async def test_crash_between_put_and_commit_is_swept_after_grace(
|
||||
s = await scene(http)
|
||||
alice, chat = s["alice"], s["chats"][0]
|
||||
kept = (await upload(http, alice, chat, filename="kept.png")).json()
|
||||
await waited(lambda: status_of(kept["id"]) == "ready", 60)
|
||||
assert (
|
||||
await message(http, alice, chat, {"text": "", "attachment_ids": [kept["id"]]})
|
||||
).status_code == 201
|
||||
@@ -465,4 +469,4 @@ async def test_stalled_upload_is_cut_and_leaves_nothing(http: AsyncClient, fault
|
||||
assert (status, body["error"]["code"]) == (503, "unavailable")
|
||||
assert elapsed <= REQUEST_TIMEOUT + 2
|
||||
assert seen(name) is None
|
||||
assert (await upload(http, s["alice"], s["chats"][0])).status_code == 201
|
||||
assert (await upload(http, s["alice"], s["chats"][0])).status_code == 202
|
||||
|
||||
Reference in New Issue
Block a user