forked from mtusi2026/mtusi2026lab5
234 lines
8.5 KiB
Python
234 lines
8.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Grade-5 track 2: repeatable versioned-pipeline idempotency experiment."""
|
|
|
|
import hashlib
|
|
import json
|
|
import time
|
|
|
|
from client import Client, arguments, check, run
|
|
from level4_resilience import compose, restart_count, sql, verify_variants, wait_until
|
|
|
|
|
|
def jobctl(service, *args, expect_success=True):
|
|
result = compose(
|
|
"run", "--rm", "--no-deps", "--entrypoint", "/app/jobctl",
|
|
service, *args,
|
|
capture=True,
|
|
check_result=False,
|
|
)
|
|
if expect_success:
|
|
check(result.returncode == 0, "jobctl failed: " + result.stderr)
|
|
else:
|
|
check(result.returncode != 0, "fault-injection command unexpectedly succeeded")
|
|
return result
|
|
|
|
|
|
def job_id(attachment_id, version):
|
|
return sql(
|
|
"SELECT job_id::text FROM attachment_jobs WHERE attachment_id='" +
|
|
attachment_id + "' AND pipeline_version=" + str(version)
|
|
)
|
|
|
|
|
|
def variant_hashes(attachment_id, version):
|
|
rows = sql(
|
|
"SELECT name || ':' || encode(sha256,'hex') FROM attachment_variants "
|
|
"WHERE attachment_id='" + attachment_id + "' AND pipeline_version=" +
|
|
str(version) + " ORDER BY name"
|
|
)
|
|
return tuple(rows.splitlines()) if rows else ()
|
|
|
|
|
|
def wait_pipeline(client, actor, attachment, version, timeout=45):
|
|
current = attachment
|
|
|
|
def poll():
|
|
nonlocal current
|
|
current = client.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + attachment["id"],
|
|
actor=actor,
|
|
schema="Attachment",
|
|
)["json"]
|
|
state = sql(
|
|
"SELECT status || ':' || pipeline_version FROM attachments WHERE id='" +
|
|
attachment["id"] + "'"
|
|
)
|
|
return current if current["status"] == "ready" and len(current["variants"]) == 3 and state == "ready:" + str(version) else None
|
|
|
|
return wait_until(poll, timeout, "pipeline v" + str(version) + " did not become ready")
|
|
|
|
|
|
def expire_processing_lease(attachment_id, version):
|
|
sql(
|
|
"UPDATE attachment_jobs SET lease_until=CURRENT_TIMESTAMP-INTERVAL '1 second' "
|
|
"WHERE attachment_id='" + attachment_id + "' AND pipeline_version=" +
|
|
str(version) + " AND status='processing'"
|
|
)
|
|
|
|
|
|
def wait_queue_drained():
|
|
return wait_until(
|
|
lambda: sql(
|
|
"SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing')"
|
|
) == "0",
|
|
20,
|
|
"logical job queue did not drain",
|
|
)
|
|
|
|
|
|
def main():
|
|
args = arguments("Lab5 grade-5 versioned pipeline idempotency")
|
|
client = Client(args.base_url, args.ca_file)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
|
|
# H1: a crash before the first object-store side effect loses neither the
|
|
# confirmed upload nor its logical job. The final original must not exist
|
|
# yet, demonstrated by the rejected v2 upgrade source read.
|
|
before_restarts = restart_count("worker")
|
|
compose("exec", "-T", "worker", "touch", "/tmp/mtusi-fault-before-objects")
|
|
before, before_original = client.attachment(alice, chat)
|
|
wait_until(
|
|
lambda: restart_count("worker") > before_restarts,
|
|
20,
|
|
"worker did not crash before object writes",
|
|
)
|
|
check(
|
|
sql(
|
|
"SELECT status FROM attachment_jobs WHERE attachment_id='" +
|
|
before["id"] + "'"
|
|
) == "processing",
|
|
"before-side-effect crash did not leave a leased job",
|
|
)
|
|
rejected = jobctl("worker", "upgrade", before["id"], "2", expect_success=False)
|
|
check(
|
|
"upgrade source original is unavailable" in rejected.stderr,
|
|
"before-side-effect crash unexpectedly published the final original",
|
|
)
|
|
expire_processing_lease(before["id"], 1)
|
|
before_ready = wait_pipeline(client, bob, before, 1)
|
|
verify_variants(client, bob, before_ready, before_original)
|
|
|
|
# H2: after an object side effect but before result COMMIT/ack, a new
|
|
# pipeline can supersede the in-flight v1 job. The copied original is the
|
|
# durable handoff for v2; v1 cannot publish metadata after the upgrade.
|
|
after_restarts = restart_count("worker")
|
|
compose("exec", "-T", "worker", "touch", "/tmp/mtusi-fault-after-objects")
|
|
raced, raced_original = client.attachment(alice, chat)
|
|
wait_until(
|
|
lambda: restart_count("worker") > after_restarts,
|
|
20,
|
|
"worker did not crash after object writes",
|
|
)
|
|
v1_raced_job = job_id(raced["id"], 1)
|
|
check(
|
|
sql(
|
|
"SELECT status FROM attachment_jobs WHERE job_id='" + v1_raced_job + "'"
|
|
) == "processing",
|
|
"after-side-effect crash did not preserve the leased v1 job",
|
|
)
|
|
upgraded = json.loads(jobctl("worker", "upgrade", raced["id"], "2").stdout)
|
|
check(
|
|
upgraded["pipeline_version"] == 2 and upgraded["status"] == "queued",
|
|
"v2 upgrade did not create a queued versioned job",
|
|
)
|
|
raced_ready = wait_pipeline(client, bob, raced, 2)
|
|
verify_variants(client, bob, raced_ready, raced_original)
|
|
check(
|
|
sql(
|
|
"SELECT status FROM attachment_jobs WHERE job_id='" + v1_raced_job + "'"
|
|
) == "superseded",
|
|
"in-flight v1 job was not superseded",
|
|
)
|
|
check(
|
|
sql(
|
|
"SELECT count(*) FROM attachment_variants WHERE attachment_id='" +
|
|
raced["id"] + "' AND pipeline_version=1"
|
|
) == "0",
|
|
"stale v1 metadata committed after v2 became active",
|
|
)
|
|
check(
|
|
sql(
|
|
"SELECT count(*) FROM attachment_job_events WHERE job_id='" +
|
|
v1_raced_job + "' AND event_type='superseded'"
|
|
) == "1",
|
|
"superseded in-flight pipeline was not audited",
|
|
)
|
|
|
|
# H3: a completed v1 can be reprocessed by the distinct v2 profile. Both
|
|
# version histories remain, while API reads only active v2.
|
|
versioned, versioned_original = client.attachment(alice, chat)
|
|
v1_ready = wait_pipeline(client, bob, versioned, 1)
|
|
verify_variants(client, bob, v1_ready, versioned_original)
|
|
v1_job = job_id(versioned["id"], 1)
|
|
v1_hashes = variant_hashes(versioned["id"], 1)
|
|
check(len(v1_hashes) == 3, "v1 variant history is incomplete")
|
|
|
|
upgraded = json.loads(jobctl("worker", "upgrade", versioned["id"], "2").stdout)
|
|
v2_job = upgraded["job_id"]
|
|
v2_ready = wait_pipeline(client, bob, versioned, 2)
|
|
verify_variants(client, bob, v2_ready, versioned_original)
|
|
v2_hashes = variant_hashes(versioned["id"], 2)
|
|
check(len(v2_hashes) == 3, "v2 variant history is incomplete")
|
|
check(v1_hashes != v2_hashes, "v1 and v2 profiles produced identical variant hashes")
|
|
check(
|
|
sql(
|
|
"SELECT count(*) FROM attachment_variants WHERE attachment_id='" +
|
|
versioned["id"] + "'"
|
|
) == "6",
|
|
"version upgrade did not preserve both variant histories",
|
|
)
|
|
|
|
# Duplicate v2 deliveries followed by a reordered stale v1 delivery are
|
|
# all no-ops. Record the active result before injection and prove the old
|
|
# job cannot overwrite it.
|
|
active_before = variant_hashes(versioned["id"], 2)
|
|
original_sha_before = hashlib.sha256(
|
|
client.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + versioned["id"] + "/content",
|
|
actor=bob,
|
|
)["bytes"]
|
|
).hexdigest()
|
|
jobctl("dispatcher", "inject", v2_job)
|
|
jobctl("dispatcher", "inject", v2_job)
|
|
jobctl("dispatcher", "inject", v1_job)
|
|
time.sleep(2)
|
|
wait_queue_drained()
|
|
active_after = variant_hashes(versioned["id"], 2)
|
|
original_sha_after = hashlib.sha256(
|
|
client.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + versioned["id"] + "/content",
|
|
actor=bob,
|
|
)["bytes"]
|
|
).hexdigest()
|
|
check(active_after == active_before, "duplicate/reordered delivery changed active v2 hashes")
|
|
check(original_sha_after == original_sha_before, "duplicate/reordered delivery changed original")
|
|
check(
|
|
sql(
|
|
"SELECT status || ':' || pipeline_version FROM attachments WHERE id='" +
|
|
versioned["id"] + "'"
|
|
) == "ready:2",
|
|
"stale v1 delivery rolled back the active pipeline",
|
|
)
|
|
check(
|
|
sql(
|
|
"SELECT count(*) FROM attachment_variants WHERE attachment_id='" +
|
|
versioned["id"] + "'"
|
|
) == "6",
|
|
"duplicate/reordered messages multiplied variant rows",
|
|
)
|
|
check(
|
|
sql("SELECT status FROM attachment_jobs WHERE job_id='" + v2_job + "'") == "succeeded",
|
|
"duplicate v2 delivery changed successful job state",
|
|
)
|
|
|
|
print(
|
|
"PASS: pipeline v1/v2 history, crash before/after side effect, "
|
|
"superseded in-flight work, duplicate and reordered stale deliveries verified."
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main) |