Files
mtusi2026lab6/tests/level5_idempotency.py
T

234 lines
8.5 KiB
Python

#!/usr/bin/env python3
"""Grade-5 track 2: repeatable versioned-pipeline idempotency experiment."""
import hashlib
import json
import time
from client import Client, arguments, check, run
from level4_resilience import compose, restart_count, sql, verify_variants, wait_until
def jobctl(service, *args, expect_success=True):
result = compose(
"run", "--rm", "--no-deps", "--entrypoint", "/app/jobctl",
service, *args,
capture=True,
check_result=False,
)
if expect_success:
check(result.returncode == 0, "jobctl failed: " + result.stderr)
else:
check(result.returncode != 0, "fault-injection command unexpectedly succeeded")
return result
def job_id(attachment_id, version):
return sql(
"SELECT job_id::text FROM attachment_jobs WHERE attachment_id='" +
attachment_id + "' AND pipeline_version=" + str(version)
)
def variant_hashes(attachment_id, version):
rows = sql(
"SELECT name || ':' || encode(sha256,'hex') FROM attachment_variants "
"WHERE attachment_id='" + attachment_id + "' AND pipeline_version=" +
str(version) + " ORDER BY name"
)
return tuple(rows.splitlines()) if rows else ()
def wait_pipeline(client, actor, attachment, version, timeout=45):
current = attachment
def poll():
nonlocal current
current = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"],
actor=actor,
schema="Attachment",
)["json"]
state = sql(
"SELECT status || ':' || pipeline_version FROM attachments WHERE id='" +
attachment["id"] + "'"
)
return current if current["status"] == "ready" and len(current["variants"]) == 3 and state == "ready:" + str(version) else None
return wait_until(poll, timeout, "pipeline v" + str(version) + " did not become ready")
def expire_processing_lease(attachment_id, version):
sql(
"UPDATE attachment_jobs SET lease_until=CURRENT_TIMESTAMP-INTERVAL '1 second' "
"WHERE attachment_id='" + attachment_id + "' AND pipeline_version=" +
str(version) + " AND status='processing'"
)
def wait_queue_drained():
return wait_until(
lambda: sql(
"SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing')"
) == "0",
20,
"logical job queue did not drain",
)
def main():
args = arguments("Lab5 grade-5 versioned pipeline idempotency")
client = Client(args.base_url, args.ca_file)
alice, bob, _, chat = client.bootstrap()
# H1: a crash before the first object-store side effect loses neither the
# confirmed upload nor its logical job. The final original must not exist
# yet, demonstrated by the rejected v2 upgrade source read.
before_restarts = restart_count("worker")
compose("exec", "-T", "worker", "touch", "/tmp/mtusi-fault-before-objects")
before, before_original = client.attachment(alice, chat)
wait_until(
lambda: restart_count("worker") > before_restarts,
20,
"worker did not crash before object writes",
)
check(
sql(
"SELECT status FROM attachment_jobs WHERE attachment_id='" +
before["id"] + "'"
) == "processing",
"before-side-effect crash did not leave a leased job",
)
rejected = jobctl("worker", "upgrade", before["id"], "2", expect_success=False)
check(
"upgrade source original is unavailable" in rejected.stderr,
"before-side-effect crash unexpectedly published the final original",
)
expire_processing_lease(before["id"], 1)
before_ready = wait_pipeline(client, bob, before, 1)
verify_variants(client, bob, before_ready, before_original)
# H2: after an object side effect but before result COMMIT/ack, a new
# pipeline can supersede the in-flight v1 job. The copied original is the
# durable handoff for v2; v1 cannot publish metadata after the upgrade.
after_restarts = restart_count("worker")
compose("exec", "-T", "worker", "touch", "/tmp/mtusi-fault-after-objects")
raced, raced_original = client.attachment(alice, chat)
wait_until(
lambda: restart_count("worker") > after_restarts,
20,
"worker did not crash after object writes",
)
v1_raced_job = job_id(raced["id"], 1)
check(
sql(
"SELECT status FROM attachment_jobs WHERE job_id='" + v1_raced_job + "'"
) == "processing",
"after-side-effect crash did not preserve the leased v1 job",
)
upgraded = json.loads(jobctl("worker", "upgrade", raced["id"], "2").stdout)
check(
upgraded["pipeline_version"] == 2 and upgraded["status"] == "queued",
"v2 upgrade did not create a queued versioned job",
)
raced_ready = wait_pipeline(client, bob, raced, 2)
verify_variants(client, bob, raced_ready, raced_original)
check(
sql(
"SELECT status FROM attachment_jobs WHERE job_id='" + v1_raced_job + "'"
) == "superseded",
"in-flight v1 job was not superseded",
)
check(
sql(
"SELECT count(*) FROM attachment_variants WHERE attachment_id='" +
raced["id"] + "' AND pipeline_version=1"
) == "0",
"stale v1 metadata committed after v2 became active",
)
check(
sql(
"SELECT count(*) FROM attachment_job_events WHERE job_id='" +
v1_raced_job + "' AND event_type='superseded'"
) == "1",
"superseded in-flight pipeline was not audited",
)
# H3: a completed v1 can be reprocessed by the distinct v2 profile. Both
# version histories remain, while API reads only active v2.
versioned, versioned_original = client.attachment(alice, chat)
v1_ready = wait_pipeline(client, bob, versioned, 1)
verify_variants(client, bob, v1_ready, versioned_original)
v1_job = job_id(versioned["id"], 1)
v1_hashes = variant_hashes(versioned["id"], 1)
check(len(v1_hashes) == 3, "v1 variant history is incomplete")
upgraded = json.loads(jobctl("worker", "upgrade", versioned["id"], "2").stdout)
v2_job = upgraded["job_id"]
v2_ready = wait_pipeline(client, bob, versioned, 2)
verify_variants(client, bob, v2_ready, versioned_original)
v2_hashes = variant_hashes(versioned["id"], 2)
check(len(v2_hashes) == 3, "v2 variant history is incomplete")
check(v1_hashes != v2_hashes, "v1 and v2 profiles produced identical variant hashes")
check(
sql(
"SELECT count(*) FROM attachment_variants WHERE attachment_id='" +
versioned["id"] + "'"
) == "6",
"version upgrade did not preserve both variant histories",
)
# Duplicate v2 deliveries followed by a reordered stale v1 delivery are
# all no-ops. Record the active result before injection and prove the old
# job cannot overwrite it.
active_before = variant_hashes(versioned["id"], 2)
original_sha_before = hashlib.sha256(
client.call(
"GET",
"/api/v1/attachments/" + versioned["id"] + "/content",
actor=bob,
)["bytes"]
).hexdigest()
jobctl("dispatcher", "inject", v2_job)
jobctl("dispatcher", "inject", v2_job)
jobctl("dispatcher", "inject", v1_job)
time.sleep(2)
wait_queue_drained()
active_after = variant_hashes(versioned["id"], 2)
original_sha_after = hashlib.sha256(
client.call(
"GET",
"/api/v1/attachments/" + versioned["id"] + "/content",
actor=bob,
)["bytes"]
).hexdigest()
check(active_after == active_before, "duplicate/reordered delivery changed active v2 hashes")
check(original_sha_after == original_sha_before, "duplicate/reordered delivery changed original")
check(
sql(
"SELECT status || ':' || pipeline_version FROM attachments WHERE id='" +
versioned["id"] + "'"
) == "ready:2",
"stale v1 delivery rolled back the active pipeline",
)
check(
sql(
"SELECT count(*) FROM attachment_variants WHERE attachment_id='" +
versioned["id"] + "'"
) == "6",
"duplicate/reordered messages multiplied variant rows",
)
check(
sql("SELECT status FROM attachment_jobs WHERE job_id='" + v2_job + "'") == "succeeded",
"duplicate v2 delivery changed successful job state",
)
print(
"PASS: pipeline v1/v2 history, crash before/after side effect, "
"superseded in-flight work, duplicate and reordered stale deliveries verified."
)
if __name__ == "__main__":
run(main)