Bring the stdlib suite and image generator of the lab 5 preparation: make suite-preflight checks services, readiness, schema columns, broker topology and the faults override, and make level3 runs the grade 3 scenarios (smoke through the pipeline, worker down with API, relay and broker restarts, duplicate delivery after ready, poison image, S3 outage with recovery, retries up to JOB_MAX_ATTEMPTS) and checks the invariants into evidence/level3.json. make pipeline-evidence shows the queued job on the broker while the worker is stopped, its properties and contract payload, the 409 before ready and SIGTERM of the relay and the worker; make topology dumps queues, consumers with prefetch, permissions and users. The lab 4 failover and S3 outage scenarios and the upload fault suite now expect 202 and wait for ready, the stack evidence lists the broker and its client packages, and the CI acceptance job runs the pipeline scenarios and the SIGTERM check of the new processes. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
106 lines
3.9 KiB
Python
106 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
|
|
from client import Client, check
|
|
|
|
COMPOSE = os.environ.get("COMPOSE", "docker compose").split()
|
|
PEEK = (
|
|
'RABBITMQADMIN_USERNAME=messenger-ops RABBITMQADMIN_PASSWORD="$(cat /run/secrets/'
|
|
'rabbitmq_ops_password)" exec rabbitmqadmin -V messenger '
|
|
"--table-style ascii get messages -q attachments.process -a ack_requeue_true"
|
|
)
|
|
QUEUES = "rabbitmqctl -q list_queues -p messenger name messages_ready messages_unacknowledged"
|
|
|
|
|
|
def compose(*argv):
|
|
done = subprocess.run(
|
|
[*COMPOSE, *argv], cwd=ROOT, capture_output=True, text=True, timeout=180, check=False
|
|
)
|
|
check(done.returncode == 0, f"{' '.join(argv[:3])}: {done.stderr.strip()[-300:]}")
|
|
return done.stdout
|
|
|
|
|
|
def ready_messages():
|
|
for line in compose("exec", "-T", "broker", *QUEUES.split()).splitlines():
|
|
name, ready, _ = [*line.split("\t"), "", ""][:3]
|
|
if name == "attachments.process":
|
|
return int(ready)
|
|
return 0
|
|
|
|
|
|
def say(line):
|
|
print(line, flush=True)
|
|
|
|
|
|
def stop(service):
|
|
began = time.monotonic()
|
|
compose("stop", "-t", "10", service)
|
|
seconds = time.monotonic() - began
|
|
container = compose("ps", "-aq", service).split()[0]
|
|
code = subprocess.run(
|
|
["docker", "inspect", "-f", "{{.State.ExitCode}}", container],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
).stdout.strip()
|
|
say(f"$ docker compose stop -t 10 {service} -> {seconds:.1f} s, exit code {code}")
|
|
check(seconds <= 10 and code == "0", f"{service} did not stop cleanly on SIGTERM")
|
|
|
|
|
|
def start(service):
|
|
compose("up", "-d", "--no-deps", "--wait", "--wait-timeout", "90", service)
|
|
say(f"$ docker compose up -d --no-deps --wait {service} -> healthy")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Lab5 stand evidence: queue, message, SIGTERM")
|
|
parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
|
|
args = parser.parse_args()
|
|
client = Client(args.base_url)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
say(f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}")
|
|
stop("worker")
|
|
attachment, _ = client.attachment(alice, chat)
|
|
say(f"upload -> 202 status={attachment['status']} id={attachment['id']}")
|
|
path = f"/api/v1/attachments/{attachment['id']}"
|
|
for variant in ("original", "thumbnail"):
|
|
client.call("GET", f"{path}/content?variant={variant}", 409, actor=bob)
|
|
say(f"GET content?variant={variant} while worker is stopped -> 409")
|
|
deadline = time.monotonic() + 30
|
|
while ready_messages() < 1:
|
|
check(time.monotonic() < deadline, "the job did not reach the broker")
|
|
time.sleep(0.5)
|
|
say("$ rabbitmqctl list_queues -p messenger name messages_ready messages_unacknowledged")
|
|
say(compose("exec", "-T", "broker", *QUEUES.split()).rstrip())
|
|
say("$ rabbitmqadmin get messages -q attachments.process -a ack_requeue_true")
|
|
peeked = compose("exec", "-T", "broker", "sh", "-c", PEEK)
|
|
say(peeked.rstrip())
|
|
body = json.loads(peeked[peeked.index('{"') : peeked.index("}", peeked.index('{"')) + 1])
|
|
say(f"payload keys: {sorted(body)}")
|
|
check(body["attachment_id"] == attachment["id"], "the queued job is for another attachment")
|
|
stop("relay")
|
|
start("relay")
|
|
start("worker")
|
|
ready = client.wait_ready(bob, attachment)
|
|
names = [(v["name"], v["width"], v["height"]) for v in ready["variants"]]
|
|
say(f"after worker start -> status={ready['status']} variants={names}")
|
|
deadline = time.monotonic() + 30
|
|
while ready_messages():
|
|
check(time.monotonic() < deadline, "the queue did not drain")
|
|
time.sleep(0.5)
|
|
say(compose("exec", "-T", "broker", *QUEUES.split()).rstrip())
|
|
say("PASS: pipeline stand evidence")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|