Files
mtusi2026lab5/tests/custom/attachment_flow.py
T
Mikhail Verkovykh 9e43faa536 feat(lab5): durable async attachment pipeline through grade 4
- RabbitMQ quorum queue, persistent messages and publisher confirms
- S3 staging plus PostgreSQL attachments/jobs outbox transaction
- worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper
- jittered retry, permanent failures, explicit replay and admission cap
- deterministic crash hooks, resilience script and evidence for grade 4
- async-aware custom tests, oversized pixel case, docs and changelog
2026-11-15 00:30:36 +03:00

138 lines
4.4 KiB
Python
Executable File

#!/usr/bin/env python3
"""Two-step attachment durability probe used by verify-attachment-failover.sh.
Step 1 (`upload`) uploads a fixture and writes a short-lived state file. The
caller then stops the replica named in `instance_id`. Step 2 (`download`) uses
the same session token through the load balancer, retries transient errors and
checks that another replica returns the exact original bytes.
The state file contains a test session token. It lives in a private temporary
directory and is never copied into evidence.
"""
import argparse
import hashlib
import json
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
def upload(args):
c = Client(args.base_url, args.ca_file)
alice, _, _, chat = c.bootstrap()
content = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
response = c.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png",
202,
actor=alice,
raw=content,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
accepted_by = response["headers"].get("X-Instance-Id")
attachment = c.wait_ready(alice, response["json"], wait_seconds=60)
check(attachment["status"] == "ready", "upload did not become ready")
check(
attachment["sha256"] == hashlib.sha256(content).hexdigest(),
"upload checksum mismatch",
)
state = {
"attachment_id": attachment["id"],
"sha256": attachment["sha256"],
"size_bytes": attachment["size_bytes"],
"instance_id": accepted_by,
"token": alice["token"],
"chat_id": chat["id"],
"content_length": len(content),
}
check(bool(state["instance_id"]), "upload response has no instance id")
args.state.write_text(json.dumps(state))
print(
json.dumps(
{
"uploaded_by": state["instance_id"],
"attachment_id": state["attachment_id"],
"size_bytes": state["size_bytes"],
}
)
)
def download(args):
state = json.loads(args.state.read_text())
expected_sha = state["sha256"]
expected_size = state["size_bytes"]
actor = {"token": state["token"]}
c = Client(args.base_url, args.ca_file)
deadline = time.monotonic() + args.timeout
attempts = 0
errors = []
while time.monotonic() < deadline:
attempts += 1
try:
response = c.call(
"GET",
f"/api/v1/attachments/{state['attachment_id']}/content",
actor=actor,
)
except Failure as error:
errors.append(str(error))
time.sleep(0.5)
continue
served_by = response["headers"].get("X-Instance-Id")
check(served_by is not None, "download response has no instance id")
check(served_by != state["instance_id"], "download was served by the stopped replica")
check(
len(response["bytes"]) == expected_size,
f"download size {len(response['bytes'])} != {expected_size}",
)
check(
hashlib.sha256(response["bytes"]).hexdigest() == expected_sha,
"download checksum differs from the uploaded original",
)
print(
json.dumps(
{
"downloaded_by": served_by,
"stopped_replica": state["instance_id"],
"attempts": attempts,
"transient_errors": len(errors),
"size_bytes": len(response["bytes"]),
"sha256": expected_sha,
}
)
)
return
raise Failure(
f"download did not recover within {args.timeout}s after {attempts} attempts: "
+ "; ".join(errors[-3:])
)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--action", choices=("upload", "download"), required=True)
parser.add_argument("--state", type=Path, required=True)
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--ca-file", default=None)
parser.add_argument("--timeout", type=float, default=30)
args = parser.parse_args()
if args.action == "upload":
upload(args)
else:
download(args)
if __name__ == "__main__":
main()