Files
mtusi2026lab5/scripts/track2_experiment.py
dawudukandCursor cf9d9a918f Добавить версионированный pipeline и проверку идемпотентности.
Трек 2 фиксирует дубль, устаревшую версию, падение до и после записи объекта и replay без второй копии результата.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-11-09 01:23:49 +03:00

252 lines
10 KiB
Python

#!/usr/bin/env python3
"""Track 2: versioned pipeline under duplicate, reorder, crash and a stale job.
Prints statuses and attachment ids only. Tokens stay in memory.
"""
import base64
import hashlib
import json
import os
import subprocess
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "evidence" / "track2.json"
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
def compose(*args: str) -> None:
command = ["docker", "compose", "--env-file", ".env"]
extra = os.environ.get("LAB5_COMPOSE_FILES", "")
for name in extra.split(os.pathsep):
if name:
command.extend(["-f", name])
subprocess.run([*command, *args], cwd=ROOT, check=True)
class Api:
def __init__(self) -> None:
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
def call(self, method, path, body=None, token=None, raw=None, content_type=None, headers=None):
hdrs = dict(headers or {})
data = raw
if body is not None:
data = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if content_type:
hdrs["Content-Type"] = content_type
if token:
hdrs["Authorization"] = "Bearer " + token
request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method)
try:
response = self.opener.open(request, timeout=30)
except urllib.error.HTTPError as error:
response = error
with response:
payload = response.read()
parsed = json.loads(payload) if payload and response.headers.get_content_type() == "application/json" else None
return response.status, parsed, payload
def expect(status, got, label):
if got != status:
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
def wait_applied(api: Api, attachment_id: str, token: str, version: int, seconds: int = 150, allow_failed: bool = False):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
expect(200, status, "poll")
state = jobs(api, attachment_id)
if body["status"] == "failed" and allow_failed:
return body
if (
body["status"] == "ready"
and state["applied_pipeline_version"] == version
and any(item["name"] == "thumbnail" for item in body["variants"])
):
return body
time.sleep(0.4)
raise SystemExit("FAIL timeout waiting for pipeline %s" % version)
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
expect(200, status, "poll")
last = body
if body["status"] == "ready" and any(item["name"] == "thumbnail" for item in body["variants"]):
return body
if body["status"] == "failed":
return body
time.sleep(0.4)
raise SystemExit("FAIL timeout last=%s" % (last or {}).get("status"))
def jobs(api: Api, attachment_id: str):
status, body, _ = api.call("GET", "/api/v1/_lab/attachments/%s/jobs" % attachment_id)
expect(200, status, "jobs")
return body
def thumb(body):
found = [item for item in body["variants"] if item["name"] == "thumbnail"]
if len(found) != 1:
raise SystemExit("FAIL thumbnail count %s" % len(found))
return found[0]["sha256"]
def user(api: Api):
username = "t2_" + uuid.uuid4().hex[:10]
password = "T3st-" + uuid.uuid4().hex
status, body, _ = api.call(
"POST",
"/api/v1/users",
{"username": username, "display_name": "Track2", "password": password},
)
expect(201, status, "register")
basic = base64.b64encode((username + ":" + password).encode()).decode()
status, session, _ = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
expect(201, status, "login")
return body["id"], session["token"]
def upload(api: Api, chat_id: str, token: str, filename: str):
status, body, _ = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=%s" % (chat_id, filename),
token=token,
raw=FIXTURE.read_bytes(),
content_type="image/png",
)
expect(202, status, "upload " + filename)
return body
def main() -> None:
api = Api()
content = FIXTURE.read_bytes()
original = hashlib.sha256(content).hexdigest()
alice, token = user(api)
status, chat, _ = api.call(
"POST",
"/api/v1/chats",
{"title": "track2", "member_ids": [alice]},
token=token,
)
expect(201, status, "chat")
chat_id = chat["id"]
report = {"hypothesis": "at-least-once plus a versioned effect key keeps the newest committed pipeline", "cases": []}
first = upload(api, chat_id, token, "v1.png")
ready = wait_applied(api, first["id"], token, 1)
sha_v1 = thumb(ready)
state = jobs(api, first["id"])
v1_job = state["jobs"][0]["job_id"]
expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "duplicate 1")
expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "duplicate 2")
time.sleep(2)
again = wait_applied(api, first["id"], token, 1)
state = jobs(api, first["id"])
v1_rows = [item for item in state["variants"] if item["pipeline_version"] == 1]
if thumb(again) != sha_v1 or len(v1_rows) != 3:
raise SystemExit("FAIL duplicate changed the effect")
report["cases"].append({"name": "duplicate", "attachment_id": first["id"], "thumbnail_sha256": sha_v1, "variant_rows": len(v1_rows)})
expect(202, api.call("POST", "/api/v1/_lab/attachments/%s/pipelines" % first["id"])[0], "bump")
bumped = wait_applied(api, first["id"], token, 2)
sha_v2 = thumb(bumped)
state = jobs(api, first["id"])
if sha_v2 == sha_v1 or state["applied_pipeline_version"] != 2:
raise SystemExit("FAIL version 2 did not replace the served bytes")
v1_job = [item["job_id"] for item in state["jobs"] if item["pipeline_version"] == 1][0]
expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "stale redelivery")
time.sleep(2)
after_stale = wait_applied(api, first["id"], token, 2)
state = jobs(api, first["id"])
if thumb(after_stale) != sha_v2 or state["applied_pipeline_version"] != 2:
raise SystemExit("FAIL stale job overwrote version 2")
report["cases"].append(
{
"name": "reorder_and_stale_version",
"attachment_id": first["id"],
"version1_sha256": sha_v1,
"version2_sha256": sha_v2,
"applied_pipeline_version": state["applied_pipeline_version"],
"variant_rows": len(state["variants"]),
}
)
def crashed(mode: str):
expect(200, api.call("POST", "/api/v1/_lab/faults", {"mode": mode, "remaining": 1})[0], mode)
created = upload(api, chat_id, token, mode + ".png")
# The worker process exits. Docker restarts it; the lease must expire before retry.
body = wait_applied(api, created["id"], token, 1, seconds=180)
if body["status"] != "ready":
raise SystemExit("FAIL %s ended %s" % (mode, body["status"]))
status_code, _, payload = api.call("GET", "/api/v1/attachments/%s/content" % created["id"], token=token)
expect(200, status_code, mode + " original")
if hashlib.sha256(payload).hexdigest() != original:
raise SystemExit("FAIL %s lost the original" % mode)
state_now = jobs(api, created["id"])
rows = [item for item in state_now["variants"] if item["pipeline_version"] == 1]
if len(rows) != 3:
raise SystemExit("FAIL %s variant rows %s" % (mode, len(rows)))
report["cases"].append(
{
"name": mode,
"attachment_id": created["id"],
"original_sha256": original,
"variant_rows": len(rows),
"thumbnail_sha256": thumb(body),
}
)
crashed("crash_before_s3")
crashed("crash_after_s3")
expect(200, api.call("POST", "/api/v1/_lab/faults", {"mode": "fail_once", "remaining": 1})[0], "fail_once")
failed = upload(api, chat_id, token, "fail.png")
failed_body = wait_applied(api, failed["id"], token, 1, allow_failed=True)
if failed_body["status"] != "failed" or failed_body["error_code"] != "processing_failed":
raise SystemExit("FAIL expected a terminal processing_failed")
expect(200, api.call("POST", "/api/v1/_lab/attachments/%s/replay" % failed["id"])[0], "replay")
restored = wait_applied(api, failed["id"], token, 1)
if restored["status"] != "ready":
raise SystemExit("FAIL replay did not reach ready")
state = jobs(api, failed["id"])
replay_job = state["jobs"][0]["job_id"]
expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % replay_job)[0], "replay duplicate")
time.sleep(2)
state = jobs(api, failed["id"])
if len(state["variants"]) != 3 or state["jobs"][0]["status"] != "ready":
raise SystemExit("FAIL replay produced another effect")
report["cases"].append(
{
"name": "replay_then_duplicate",
"attachment_id": failed["id"],
"variant_rows": len(state["variants"]),
"job_status": state["jobs"][0]["status"],
}
)
report["not_exactly_once"] = (
"The broker delivers a message more than once. A crash after the object PUT and before ack "
"runs the S3 write again. The database commit of a pipeline version happens once, and an older "
"version cannot replace the applied one. That is idempotent effect, not exactly-once delivery."
)
OUT.parent.mkdir(exist_ok=True)
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print("PASS track2", OUT)
if __name__ == "__main__":
main()