252 lines
10 KiB
Python
252 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Track 2: versioned pipeline under duplicate, reorder, crash and a stale job.
|
|
|
|
Prints statuses and attachment ids only. Tokens stay in memory.
|
|
"""
|
|
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
OUT = ROOT / "evidence" / "track2.json"
|
|
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
|
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
|
|
|
|
|
|
def compose(*args: str) -> None:
|
|
command = ["docker", "compose", "--env-file", ".env"]
|
|
extra = os.environ.get("LAB5_COMPOSE_FILES", "")
|
|
for name in extra.split(os.pathsep):
|
|
if name:
|
|
command.extend(["-f", name])
|
|
subprocess.run([*command, *args], cwd=ROOT, check=True)
|
|
|
|
|
|
class Api:
|
|
def __init__(self) -> None:
|
|
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
|
|
def call(self, method, path, body=None, token=None, raw=None, content_type=None, headers=None):
|
|
hdrs = dict(headers or {})
|
|
data = raw
|
|
if body is not None:
|
|
data = json.dumps(body).encode("utf-8")
|
|
hdrs["Content-Type"] = "application/json"
|
|
if content_type:
|
|
hdrs["Content-Type"] = content_type
|
|
if token:
|
|
hdrs["Authorization"] = "Bearer " + token
|
|
request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method)
|
|
try:
|
|
response = self.opener.open(request, timeout=30)
|
|
except urllib.error.HTTPError as error:
|
|
response = error
|
|
with response:
|
|
payload = response.read()
|
|
parsed = json.loads(payload) if payload and response.headers.get_content_type() == "application/json" else None
|
|
return response.status, parsed, payload
|
|
|
|
|
|
def expect(status, got, label):
|
|
if got != status:
|
|
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
|
|
|
|
|
|
def wait_applied(api: Api, attachment_id: str, token: str, version: int, seconds: int = 150, allow_failed: bool = False):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
|
|
expect(200, status, "poll")
|
|
state = jobs(api, attachment_id)
|
|
if body["status"] == "failed" and allow_failed:
|
|
return body
|
|
if (
|
|
body["status"] == "ready"
|
|
and state["applied_pipeline_version"] == version
|
|
and any(item["name"] == "thumbnail" for item in body["variants"])
|
|
):
|
|
return body
|
|
time.sleep(0.4)
|
|
raise SystemExit("FAIL timeout waiting for pipeline %s" % version)
|
|
deadline = time.monotonic() + seconds
|
|
last = None
|
|
while time.monotonic() < deadline:
|
|
status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
|
|
expect(200, status, "poll")
|
|
last = body
|
|
if body["status"] == "ready" and any(item["name"] == "thumbnail" for item in body["variants"]):
|
|
return body
|
|
if body["status"] == "failed":
|
|
return body
|
|
time.sleep(0.4)
|
|
raise SystemExit("FAIL timeout last=%s" % (last or {}).get("status"))
|
|
|
|
|
|
def jobs(api: Api, attachment_id: str):
|
|
status, body, _ = api.call("GET", "/api/v1/_lab/attachments/%s/jobs" % attachment_id)
|
|
expect(200, status, "jobs")
|
|
return body
|
|
|
|
|
|
def thumb(body):
|
|
found = [item for item in body["variants"] if item["name"] == "thumbnail"]
|
|
if len(found) != 1:
|
|
raise SystemExit("FAIL thumbnail count %s" % len(found))
|
|
return found[0]["sha256"]
|
|
|
|
|
|
def user(api: Api):
|
|
username = "t2_" + uuid.uuid4().hex[:10]
|
|
password = "T3st-" + uuid.uuid4().hex
|
|
status, body, _ = api.call(
|
|
"POST",
|
|
"/api/v1/users",
|
|
{"username": username, "display_name": "Track2", "password": password},
|
|
)
|
|
expect(201, status, "register")
|
|
basic = base64.b64encode((username + ":" + password).encode()).decode()
|
|
status, session, _ = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
|
expect(201, status, "login")
|
|
return body["id"], session["token"]
|
|
|
|
|
|
def upload(api: Api, chat_id: str, token: str, filename: str):
|
|
status, body, _ = api.call(
|
|
"POST",
|
|
"/api/v1/chats/%s/attachments?filename=%s" % (chat_id, filename),
|
|
token=token,
|
|
raw=FIXTURE.read_bytes(),
|
|
content_type="image/png",
|
|
)
|
|
expect(202, status, "upload " + filename)
|
|
return body
|
|
|
|
|
|
def main() -> None:
|
|
api = Api()
|
|
content = FIXTURE.read_bytes()
|
|
original = hashlib.sha256(content).hexdigest()
|
|
alice, token = user(api)
|
|
status, chat, _ = api.call(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
{"title": "track2", "member_ids": [alice]},
|
|
token=token,
|
|
)
|
|
expect(201, status, "chat")
|
|
chat_id = chat["id"]
|
|
report = {"hypothesis": "at-least-once plus a versioned effect key keeps the newest committed pipeline", "cases": []}
|
|
|
|
first = upload(api, chat_id, token, "v1.png")
|
|
ready = wait_applied(api, first["id"], token, 1)
|
|
sha_v1 = thumb(ready)
|
|
state = jobs(api, first["id"])
|
|
v1_job = state["jobs"][0]["job_id"]
|
|
expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "duplicate 1")
|
|
expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "duplicate 2")
|
|
time.sleep(2)
|
|
again = wait_applied(api, first["id"], token, 1)
|
|
state = jobs(api, first["id"])
|
|
v1_rows = [item for item in state["variants"] if item["pipeline_version"] == 1]
|
|
if thumb(again) != sha_v1 or len(v1_rows) != 3:
|
|
raise SystemExit("FAIL duplicate changed the effect")
|
|
report["cases"].append({"name": "duplicate", "attachment_id": first["id"], "thumbnail_sha256": sha_v1, "variant_rows": len(v1_rows)})
|
|
|
|
expect(202, api.call("POST", "/api/v1/_lab/attachments/%s/pipelines" % first["id"])[0], "bump")
|
|
bumped = wait_applied(api, first["id"], token, 2)
|
|
sha_v2 = thumb(bumped)
|
|
state = jobs(api, first["id"])
|
|
if sha_v2 == sha_v1 or state["applied_pipeline_version"] != 2:
|
|
raise SystemExit("FAIL version 2 did not replace the served bytes")
|
|
v1_job = [item["job_id"] for item in state["jobs"] if item["pipeline_version"] == 1][0]
|
|
expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "stale redelivery")
|
|
time.sleep(2)
|
|
after_stale = wait_applied(api, first["id"], token, 2)
|
|
state = jobs(api, first["id"])
|
|
if thumb(after_stale) != sha_v2 or state["applied_pipeline_version"] != 2:
|
|
raise SystemExit("FAIL stale job overwrote version 2")
|
|
report["cases"].append(
|
|
{
|
|
"name": "reorder_and_stale_version",
|
|
"attachment_id": first["id"],
|
|
"version1_sha256": sha_v1,
|
|
"version2_sha256": sha_v2,
|
|
"applied_pipeline_version": state["applied_pipeline_version"],
|
|
"variant_rows": len(state["variants"]),
|
|
}
|
|
)
|
|
|
|
def crashed(mode: str):
|
|
expect(200, api.call("POST", "/api/v1/_lab/faults", {"mode": mode, "remaining": 1})[0], mode)
|
|
created = upload(api, chat_id, token, mode + ".png")
|
|
# The worker process exits. Docker restarts it; the lease must expire before retry.
|
|
body = wait_applied(api, created["id"], token, 1, seconds=180)
|
|
if body["status"] != "ready":
|
|
raise SystemExit("FAIL %s ended %s" % (mode, body["status"]))
|
|
status_code, _, payload = api.call("GET", "/api/v1/attachments/%s/content" % created["id"], token=token)
|
|
expect(200, status_code, mode + " original")
|
|
if hashlib.sha256(payload).hexdigest() != original:
|
|
raise SystemExit("FAIL %s lost the original" % mode)
|
|
state_now = jobs(api, created["id"])
|
|
rows = [item for item in state_now["variants"] if item["pipeline_version"] == 1]
|
|
if len(rows) != 3:
|
|
raise SystemExit("FAIL %s variant rows %s" % (mode, len(rows)))
|
|
report["cases"].append(
|
|
{
|
|
"name": mode,
|
|
"attachment_id": created["id"],
|
|
"original_sha256": original,
|
|
"variant_rows": len(rows),
|
|
"thumbnail_sha256": thumb(body),
|
|
}
|
|
)
|
|
|
|
crashed("crash_before_s3")
|
|
crashed("crash_after_s3")
|
|
|
|
expect(200, api.call("POST", "/api/v1/_lab/faults", {"mode": "fail_once", "remaining": 1})[0], "fail_once")
|
|
failed = upload(api, chat_id, token, "fail.png")
|
|
failed_body = wait_applied(api, failed["id"], token, 1, allow_failed=True)
|
|
if failed_body["status"] != "failed" or failed_body["error_code"] != "processing_failed":
|
|
raise SystemExit("FAIL expected a terminal processing_failed")
|
|
expect(200, api.call("POST", "/api/v1/_lab/attachments/%s/replay" % failed["id"])[0], "replay")
|
|
restored = wait_applied(api, failed["id"], token, 1)
|
|
if restored["status"] != "ready":
|
|
raise SystemExit("FAIL replay did not reach ready")
|
|
state = jobs(api, failed["id"])
|
|
replay_job = state["jobs"][0]["job_id"]
|
|
expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % replay_job)[0], "replay duplicate")
|
|
time.sleep(2)
|
|
state = jobs(api, failed["id"])
|
|
if len(state["variants"]) != 3 or state["jobs"][0]["status"] != "ready":
|
|
raise SystemExit("FAIL replay produced another effect")
|
|
report["cases"].append(
|
|
{
|
|
"name": "replay_then_duplicate",
|
|
"attachment_id": failed["id"],
|
|
"variant_rows": len(state["variants"]),
|
|
"job_status": state["jobs"][0]["status"],
|
|
}
|
|
)
|
|
|
|
report["not_exactly_once"] = (
|
|
"The broker delivers a message more than once. A crash after the object PUT and before ack "
|
|
"runs the S3 write again. The database commit of a pipeline version happens once, and an older "
|
|
"version cannot replace the applied one. That is idempotent effect, not exactly-once delivery."
|
|
)
|
|
OUT.parent.mkdir(exist_ok=True)
|
|
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
print("PASS track2", OUT)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|