forked from mtusi2026/mtusi2026lab5
Durable staging handoff, transactional outbox, at-least-once worker with lease fencing, idempotent versioned pipeline, three variants, operator tooling, contract tests and evidence. Includes the review fixes: reprocess source selection and failure settlement, replay guards, stale-job retirement, PNG structural/raster validation, migration rollback safety and audit events.
166 lines
7.2 KiB
Python
Executable File
166 lines
7.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Track 2 evidence: versioned pipeline, duplicate delivery, stale-version guard.
|
|
|
|
Runs N repetitions against the live stand and writes raw CSV/Markdown evidence.
|
|
This is a correctness harness, not a load test.
|
|
|
|
python3 scripts/lab5_track2.py --reps 5
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
sys.path.insert(0, str(ROOT / 'student-tests'))
|
|
|
|
from client import Client, Failure, check # noqa: E402
|
|
from tests_helpers import compose, jobctl, psql # noqa: E402
|
|
|
|
FIXTURE = ROOT / 'tests/fixtures/sample.png'
|
|
EVIDENCE = ROOT / 'evidence/lab5/track2'
|
|
|
|
|
|
def wait_ready(client, actor, attachment, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
current = client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json']
|
|
check(current['status'] != 'failed', 'attachment failed: ' + str(current.get('error_code')))
|
|
if current['status'] == 'ready':
|
|
return current
|
|
time.sleep(0.2)
|
|
raise Failure('attachment did not become ready')
|
|
|
|
|
|
def wait_version(attachment_id, version, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
row = psql("select pipeline_version || '|' || status from attachments where id='%s'" % attachment_id)
|
|
if row == str(version) + '|ready':
|
|
return
|
|
time.sleep(0.2)
|
|
raise Failure('attachment did not reach pipeline version %d' % version)
|
|
|
|
|
|
def variants_by_name(attachment):
|
|
return {v['name']: v for v in attachment['variants']}
|
|
|
|
|
|
def scenario_duplicate_before_effect(client, actor, chat):
|
|
compose('stop', 'worker')
|
|
try:
|
|
attached, _ = client.attachment(actor, chat)
|
|
time.sleep(1)
|
|
job = psql("select job_id from attachment_jobs where attachment_id='%s'" % attached['id'])
|
|
check(bool(job), 'job id missing')
|
|
jobctl('duplicate', '-job', job)
|
|
jobctl('duplicate', '-job', job)
|
|
finally:
|
|
compose('up', '-d', '--no-deps', 'worker', check=False)
|
|
|
|
ready = wait_ready(client, actor, attached)
|
|
names = sorted(variants_by_name(ready))
|
|
check(names == ['crop', 'optimized', 'thumbnail'], 'duplicate delivery changed the variant set')
|
|
check(psql("select count(*) from attachment_variants where attachment_id='%s'" % attached['id']) == '3',
|
|
'duplicate delivery created extra variant rows')
|
|
|
|
return 'duplicate_before_effect', True, 'attempts=' + psql(
|
|
"select attempt from attachment_jobs where attachment_id='%s'" % attached['id'])
|
|
|
|
|
|
def scenario_stale_version(client, actor, chat):
|
|
attached, content = client.attachment(actor, chat)
|
|
v1 = wait_ready(client, actor, attached)
|
|
v1_variants = variants_by_name(v1)
|
|
original_sha = hashlib.sha256(content).hexdigest()
|
|
check(v1['sha256'] == original_sha, 'original checksum changed before reprocess')
|
|
|
|
v1_job = psql("select job_id from attachment_jobs where attachment_id='%s' and pipeline_version=1" % attached['id'])
|
|
check(bool(v1_job), 'v1 job missing')
|
|
|
|
reprocess = jobctl('reprocess', '-attachment', attached['id'], '-pipeline', '2')
|
|
check(reprocess.returncode == 0, 'reprocess failed: ' + reprocess.stderr)
|
|
wait_version(attached['id'], 2)
|
|
v2 = wait_ready(client, actor, attached)
|
|
v2_variants = variants_by_name(v2)
|
|
check(sorted(v2_variants) == ['crop', 'optimized', 'thumbnail'], 'v2 variant set incomplete')
|
|
check(v2_variants['thumbnail']['sha256'] != v1_variants['thumbnail']['sha256'],
|
|
'v2 bytes must differ from v1 (pipeline parameters changed)')
|
|
check(v2['sha256'] == original_sha, 'original changed during reprocess')
|
|
before = json.dumps(v2_variants, sort_keys=True)
|
|
|
|
# Deliver the old v1 message after v2 is published. It must be acknowledged
|
|
# without touching the serving version or its artifacts.
|
|
jobctl('duplicate', '-job', v1_job)
|
|
time.sleep(2)
|
|
|
|
after_attachment = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=actor, schema='Attachment')['json']
|
|
after_variants = variants_by_name(after_attachment)
|
|
check(json.dumps(after_variants, sort_keys=True) == before, 'stale v1 delivery overwrote v2 variants')
|
|
check(psql("select pipeline_version from attachments where id='%s'" % attached['id']) == '2',
|
|
'serving pipeline version regressed')
|
|
check(psql("select status from attachment_jobs where job_id='%s'" % v1_job) == 'ready',
|
|
'v1 job state changed unexpectedly')
|
|
|
|
return 'stale_version_after_v2', True, 'v1_job=' + v1_job
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='Lab5 track 2 correctness harness')
|
|
parser.add_argument('--base-url', default='http://localhost:8080')
|
|
parser.add_argument('--reps', type=int, default=3)
|
|
args = parser.parse_args()
|
|
|
|
EVIDENCE.mkdir(parents=True, exist_ok=True)
|
|
rows = []
|
|
|
|
client = Client(args.base_url, None)
|
|
|
|
for rep in range(1, args.reps + 1):
|
|
alice, _, _, chat = client.bootstrap()
|
|
for scenario in (scenario_duplicate_before_effect, scenario_stale_version):
|
|
name, passed, detail = scenario(client, alice, chat)
|
|
rows.append({'scenario': name, 'rep': rep, 'pass': int(passed), 'detail': detail})
|
|
print('D%s rep=%d: %s' % (name, rep, 'PASS' if passed else 'FAIL'))
|
|
|
|
with (EVIDENCE / 'runs.csv').open('w', newline='') as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=['scenario', 'rep', 'pass', 'detail'])
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
|
|
summary = [
|
|
'# Track 2 — versioned pipeline and duplicate/stale delivery',
|
|
'',
|
|
'Hypothesis: with the effect key `(attachment_id, pipeline_version, variant)`,',
|
|
'deterministic object keys, a serving version that changes only when all v2',
|
|
'variants are committed, and stale-version acknowledgements, an at-least-once',
|
|
'pipeline loses no accepted work and never overwrites a newer result with an',
|
|
'older duplicate.',
|
|
'',
|
|
'Method: N=%d repetitions. Each repetition creates fresh users/chat and runs:' % args.reps,
|
|
'- duplicate pipeline publication while the worker is stopped, followed by recovery;',
|
|
'- v1 ready -> explicit reprocess to v2 -> old v1 message delivered after v2 ready.',
|
|
'',
|
|
'Invariants checked: unique variant rows per effect key; ready remains ready;',
|
|
'serving pipeline version never regresses; original bytes/checksum unchanged;',
|
|
'stale job state is not rewritten.',
|
|
'',
|
|
'Limitations: this harness covers duplicate and stale/reorder delivery. Crash',
|
|
'windows (before/after side effect, before/after ack, outbox gap) and lease',
|
|
'takeover are covered by student-tests/lab5_faults.py; both are single-node',
|
|
'Compose evidence, not a production availability claim. Delivery is',
|
|
'at-least-once; exactly-once is not claimed.',
|
|
]
|
|
(EVIDENCE / 'summary.md').write_text('\n'.join(summary) + '\n')
|
|
|
|
passed = sum(row['pass'] for row in rows)
|
|
print('PASS: track2 %d/%d scenarios' % (passed, len(rows)))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|