add lab6
This commit is contained in:
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sequential lab3-5 verification; capture actual output and exit status, fail fast."""
|
||||
from datetime import datetime, timezone
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DESTINATION = ROOT / "evidence/audit"
|
||||
CHECKS = [
|
||||
("contracts", ["make", "check", "validate", "PYTHON=.venv/bin/python"]),
|
||||
("compose", ["docker", "compose", "config", "--quiet"]),
|
||||
("smoke", ["make", "test"]),
|
||||
("attachments", ["make", "test-attachments"]),
|
||||
("idempotency", [sys.executable, "student-tests/idempotency_concurrency.py"]),
|
||||
("regressions", [sys.executable, "student-tests/review_regressions.py"]),
|
||||
("private-bucket", ["make", "test-private"]),
|
||||
("s3-permissions", ["make", "test-s3-permissions"]),
|
||||
("bootstrap", ["docker", "compose", "run", "--rm", "--no-deps", "s3-init"]),
|
||||
("upload-failover", ["make", "test-upload-failover"]),
|
||||
("public-failover", ["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh"]),
|
||||
("restore-replicas", ["docker", "compose", "up", "-d", "--wait", "api-a", "api-b", "load_balancer"]),
|
||||
("s3-outage", ["make", "test-s3-outage"]),
|
||||
("persistence", [sys.executable, "scripts/verify-persistence.py"]),
|
||||
("reconcile", ["make", "test-reconcile"]),
|
||||
("reconcile-audit", ["make", "test-reconcile-audit"]),
|
||||
("upload-memory", ["make", "test-upload-memory"]),
|
||||
("reconcile-benchmark", ["make", "test-reconcile-benchmark"]),
|
||||
("final-smoke", ["make", "test"]),
|
||||
("environment", [sys.executable, "scripts/record-environment.py"]),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--resume-from', choices=[name for name, _ in CHECKS])
|
||||
args = parser.parse_args()
|
||||
DESTINATION.mkdir(parents=True, exist_ok=True)
|
||||
results = []
|
||||
checks = CHECKS
|
||||
if args.resume_from:
|
||||
offset = [name for name, _ in CHECKS].index(args.resume_from)
|
||||
previous = json.loads((DESTINATION / 'results.json').read_text())['checks']
|
||||
results = previous[:offset]
|
||||
if [item['name'] for item in results] != [name for name, _ in CHECKS[:offset]] or any(item['exit_code'] for item in results):
|
||||
parser.error('resume requires successful evidence for every earlier step')
|
||||
checks = CHECKS[offset:]
|
||||
for name, command in checks:
|
||||
print(f"RUN {name}", flush=True)
|
||||
started = time.monotonic()
|
||||
with (DESTINATION / f"{name}.txt").open("w") as stream:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT)
|
||||
results.append({"name": name, "command": command, "exit_code": result.returncode,
|
||||
"seconds": round(time.monotonic() - started, 3)})
|
||||
(DESTINATION / "results.json").write_text(json.dumps({
|
||||
"recorded_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"checks": results, "complete": len(results) == len(CHECKS),
|
||||
}, indent=2) + "\n")
|
||||
print(f"{'PASS' if result.returncode == 0 else 'FAIL'} {name}: exit={result.returncode}", flush=True)
|
||||
if result.returncode:
|
||||
print((DESTINATION / f"{name}.txt").read_text()[-8000:], flush=True)
|
||||
return result.returncode
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local credentials without printing or overwriting existing values."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
root = Path(__file__).resolve().parents[1] / "secrets"
|
||||
root.mkdir(mode=0o700, exist_ok=True)
|
||||
|
||||
|
||||
def write(name, value):
|
||||
path = root / name
|
||||
if path.exists():
|
||||
return path.read_text().strip()
|
||||
with path.open("x") as stream:
|
||||
path.chmod(0o600)
|
||||
stream.write(value + "\n")
|
||||
return value
|
||||
|
||||
|
||||
write("grafana_password", secrets.token_urlsafe(24))
|
||||
password = write("postgres_password", secrets.token_hex(32))
|
||||
write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger")
|
||||
write("cursor_secret", secrets.token_hex(32))
|
||||
admin_key = write("s3_admin_key", secrets.token_hex(16))
|
||||
admin_secret = write("s3_admin_secret", secrets.token_hex(32))
|
||||
app_key = write("s3_access_key_id", secrets.token_hex(16))
|
||||
app_secret = write("s3_secret_access_key", secrets.token_hex(32))
|
||||
rabbitmq_password = write("rabbitmq_password", secrets.token_urlsafe(32))
|
||||
write("broker_url", f"amqp://messenger:{rabbitmq_password}@broker:5672/%2F")
|
||||
config_path = root / "s3.json"
|
||||
config_path.write_text(json.dumps({"identities": [
|
||||
{"name": "bootstrap", "credentials": [{"accessKey": admin_key, "secretKey": admin_secret}],
|
||||
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
|
||||
{"name": "api", "credentials": [{"accessKey": app_key, "secretKey": app_secret}],
|
||||
"actions": ["Read:messenger-attachments", "Write:messenger-attachments", "List:messenger-attachments"]},
|
||||
]}, indent=2) + "\n")
|
||||
config_path.chmod(0o600)
|
||||
print("Local secrets ready; existing values preserved.")
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sanitized environment and source fingerprint for local test evidence."""
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import subprocess
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(*args):
|
||||
return subprocess.check_output(args, cwd=root, text=True).strip()
|
||||
|
||||
|
||||
paths = [root / name for name in ('Dockerfile', 'compose.yaml', 'pyproject.toml', 'uv.lock', 'Makefile')]
|
||||
for directory in ('src', 'migrations', 'infra', 'scripts', 'student-tests'):
|
||||
paths.extend(path for path in (root / directory).rglob('*')
|
||||
if path.is_file() and '__pycache__' not in path.parts)
|
||||
report = {
|
||||
'recorded_at_utc': datetime.now(timezone.utc).isoformat(),
|
||||
'base_commit': command('git', 'rev-parse', 'HEAD'),
|
||||
'branch': command('git', 'branch', '--show-current'),
|
||||
'note': 'Results apply to this working tree, not only the base commit. Per-profile load settings are recorded in real-c*-p*-s*.json.',
|
||||
'host': platform.platform(),
|
||||
'docker': json.loads(command('docker', 'info', '--format',
|
||||
'{"server":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}')),
|
||||
'compose_version': command('docker', 'compose', 'version', '--short'),
|
||||
'sources_sha256': {str(path.relative_to(root)): sha256(path.read_bytes()).hexdigest() for path in sorted(paths)},
|
||||
'services': {},
|
||||
}
|
||||
for service in ('postgres', 's3', 'broker', 'worker', 'outbox', 'reconcile', 'api-a', 'api-b', 'load_balancer'):
|
||||
container = command('docker', 'compose', 'ps', '-q', service)
|
||||
info = json.loads(command('docker', 'inspect', container))[0]
|
||||
report['services'][service] = {
|
||||
'image': info['Config']['Image'], 'image_id': info['Image'],
|
||||
'state': info['State']['Status'], 'health': info['State'].get('Health', {}).get('Status'),
|
||||
'memory_limit_bytes': info['HostConfig']['Memory'],
|
||||
'nano_cpus_limit': info['HostConfig']['NanoCpus'],
|
||||
'volumes': [mount['Name'] for mount in info['Mounts'] if mount['Type'] == 'volume'],
|
||||
}
|
||||
(root / 'evidence/environment.json').write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
|
||||
print('PASS: sanitized environment and source hashes saved')
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Deliberately preserves named volumes and test data.
|
||||
docker compose restart postgres s3 broker
|
||||
docker compose up -d --wait postgres s3 broker
|
||||
docker compose run --rm --no-deps s3-init
|
||||
docker compose restart worker outbox reconcile api-a api-b load_balancer
|
||||
docker compose up -d --wait worker outbox reconcile api-a api-b load_balancer
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run real, sequential Docker profiles; restore normal settings on exit."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
for workers in (1, 2, 4):
|
||||
env = dict(os.environ, WORKER_CONCURRENCY=str(workers),
|
||||
WORKER_PREFETCH=str(workers), MAX_PENDING_JOBS='32')
|
||||
subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait',
|
||||
'worker', 'api-a', 'api-b'], cwd=ROOT, env=env, check=True)
|
||||
for size in (512, 2400):
|
||||
subprocess.run([sys.executable, 'scripts/track1.py',
|
||||
'--run-name', f'real-c{workers}-p{workers}-s{size}',
|
||||
'--workers', str(workers), '--prefetch', str(workers),
|
||||
'--max-pending', '32', '--images', '160',
|
||||
'--concurrency', '16', '--size', str(size),
|
||||
'--require-backpressure'], cwd=ROOT, check=True)
|
||||
finally:
|
||||
subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait',
|
||||
'worker', 'api-a', 'api-b'], cwd=ROOT, check=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
for service in api-a api-b; do
|
||||
container="$(docker compose ps -q "$service")"
|
||||
[ -n "$container" ] || continue
|
||||
hostname="$(docker inspect --format '{{.Config.Hostname}}' "$container")"
|
||||
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker compose stop "$service"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
printf '%s\n' 'TARGET_INSTANCE_ID does not match a running API replica' >&2
|
||||
exit 2
|
||||
Executable
+217
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Closed-loop upload burst with concurrent readiness observations and drain proof."""
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from io import BytesIO
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def percentile(values, point):
|
||||
values = sorted(values)
|
||||
return round(values[max(0, math.ceil(len(values) * point) - 1)] * 1000, 3) if values else None
|
||||
|
||||
|
||||
def command_json(*args):
|
||||
result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, timeout=20)
|
||||
if result.returncode:
|
||||
raise Failure("metrics command failed: " + args[0])
|
||||
try:
|
||||
return json.loads(result.stdout)
|
||||
except ValueError:
|
||||
raise Failure("metrics command returned invalid JSON") from None
|
||||
|
||||
|
||||
def metrics(chat_id):
|
||||
return command_json("docker", "compose", "exec", "-T", "worker",
|
||||
"/app/.venv/bin/python", "-m", "src.pipeline_metrics", chat_id)
|
||||
|
||||
|
||||
def broker_depth():
|
||||
rows = command_json("docker", "compose", "exec", "-T", "broker",
|
||||
"rabbitmqctl", "list_queues", "name", "messages_ready",
|
||||
"messages_unacknowledged", "--formatter", "json")
|
||||
return next(row for row in rows if row["name"] == "attachment.process.v1")
|
||||
|
||||
|
||||
def container_info(service):
|
||||
result = subprocess.run(['docker', 'compose', 'ps', '-q', service], cwd=ROOT,
|
||||
capture_output=True, text=True, timeout=10, check=True)
|
||||
ids = result.stdout.split()
|
||||
check(len(ids) == 1, 'expected exactly one container: ' + service)
|
||||
return command_json('docker', 'inspect', ids[0])[0]
|
||||
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument("--run-name", required=True)
|
||||
p.add_argument("--workers", type=int, required=True)
|
||||
p.add_argument("--prefetch", type=int, required=True)
|
||||
p.add_argument("--max-pending", type=int, required=True)
|
||||
p.add_argument("--images", type=int, default=40)
|
||||
p.add_argument("--concurrency", type=int, default=8)
|
||||
p.add_argument("--size", type=int, default=1600)
|
||||
p.add_argument("--max-ready-seconds", type=float, default=180)
|
||||
p.add_argument("--require-backpressure", action="store_true")
|
||||
p.add_argument("--output")
|
||||
args = arguments(__doc__, configure)
|
||||
check(1 <= args.images <= 1000 and 1 <= args.concurrency <= 128, "invalid workload")
|
||||
check(1 <= args.size <= 6000 and 0 < args.max_ready_seconds <= 3600, "invalid size/deadline")
|
||||
check(args.workers > 0 and args.prefetch > 0 and args.max_pending > 0, "invalid profile")
|
||||
from PIL import Image
|
||||
# Reproducible, nonuniform synthetic image; no real personal data.
|
||||
row = bytes(v for x in range(args.size) for v in ((x * 13) % 256, (x * 37) % 256, (x * 71) % 256))
|
||||
image = Image.frombytes("RGB", (args.size, args.size), row * args.size)
|
||||
stream = BytesIO()
|
||||
image.save(stream, "PNG")
|
||||
data = stream.getvalue()
|
||||
client = Client(args.base_url, args.ca_file, timeout=30)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
baseline = metrics(chat["id"])
|
||||
check(baseline["concurrency"] == args.workers and baseline["prefetch"] == args.prefetch,
|
||||
"worker settings do not match requested profile")
|
||||
check(baseline["pending"] == 0, "use an idle isolated stack")
|
||||
for service in ("api-a", "api-b"):
|
||||
env = dict(item.split('=', 1) for item in container_info(service)['Config']['Env'])
|
||||
check(int(env['MAX_PENDING_JOBS']) == args.max_pending,
|
||||
'running API MAX_PENDING_JOBS does not match profile')
|
||||
resources = container_info('worker')['HostConfig']
|
||||
path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=track1.png"
|
||||
pending, records, samples, errors = {}, [], [], []
|
||||
lock, finished = threading.Lock(), threading.Event()
|
||||
started = time.monotonic()
|
||||
def observe():
|
||||
local = Client(args.base_url, args.ca_file, timeout=5)
|
||||
while not finished.wait(.1):
|
||||
with lock:
|
||||
batch = list(pending.items())
|
||||
for key, record in batch:
|
||||
if finished.is_set():
|
||||
break
|
||||
try:
|
||||
att = local.call("GET", "/api/v1/attachments/" + key, actor=bob, schema="Attachment")["json"]
|
||||
if att["status"] in ("ready", "failed"):
|
||||
with lock:
|
||||
record["terminal"] = att["status"]
|
||||
record["terminal_at"] = time.monotonic()
|
||||
pending.pop(key, None)
|
||||
except Failure:
|
||||
continue
|
||||
def sample():
|
||||
while not finished.is_set():
|
||||
try:
|
||||
sample = metrics(chat["id"])
|
||||
sample["queue"] = broker_depth()
|
||||
sample["seconds"] = time.monotonic() - started
|
||||
with lock:
|
||||
samples.append(sample)
|
||||
except (Failure, OSError, subprocess.TimeoutExpired) as exc:
|
||||
with lock:
|
||||
errors.append(type(exc).__name__)
|
||||
if finished.wait(1):
|
||||
break
|
||||
observer = threading.Thread(target=observe, daemon=True)
|
||||
sampler = threading.Thread(target=sample, daemon=True)
|
||||
observer.start()
|
||||
sampler.start()
|
||||
def upload(index):
|
||||
local = Client(args.base_url, args.ca_file, timeout=30)
|
||||
before = time.monotonic()
|
||||
try:
|
||||
response = local.call("POST", path, (202, 429), actor=alice, raw=data,
|
||||
headers={"Content-Type": "image/png"}, schema="Attachment")
|
||||
received = time.monotonic()
|
||||
record = {"index": index, "status": response["status"], "upload_seconds": received - before}
|
||||
if response["status"] == 202:
|
||||
record.update(id=response["json"]["id"], accepted_at=received)
|
||||
with lock:
|
||||
records.append(record)
|
||||
if response["status"] == 202:
|
||||
pending[record["id"]] = record
|
||||
except Failure:
|
||||
with lock:
|
||||
records.append({"index": index, "status": "transport_or_http_error"})
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
list(pool.map(upload, range(args.images)))
|
||||
upload_finished = time.monotonic()
|
||||
deadline = upload_finished + args.max_ready_seconds
|
||||
while time.monotonic() < deadline:
|
||||
with lock:
|
||||
empty = not pending
|
||||
if empty:
|
||||
break
|
||||
time.sleep(.1)
|
||||
finally:
|
||||
measured_end = time.monotonic()
|
||||
finished.set()
|
||||
observer.join(timeout=10)
|
||||
sampler.join(timeout=45)
|
||||
elapsed = measured_end - started
|
||||
final = metrics(chat["id"])
|
||||
queue = broker_depth()
|
||||
for _ in range(20):
|
||||
if queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0:
|
||||
break
|
||||
time.sleep(.25)
|
||||
queue = broker_depth()
|
||||
accepted = [r for r in records if r["status"] == 202]
|
||||
ready = [r for r in accepted if r.get("terminal") == "ready"]
|
||||
latencies = [r["terminal_at"] - r["accepted_at"] for r in ready]
|
||||
waits = [r["queue_wait_seconds"] for r in final["jobs"] if r["queue_wait_seconds"] is not None]
|
||||
drained = (bool(accepted) and len(ready) == len(accepted) and final["pending"] == 0
|
||||
and queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0)
|
||||
recovery = False
|
||||
if drained:
|
||||
try:
|
||||
att, _ = client.attachment(alice, chat)
|
||||
client.wait_ready(bob, att, wait_seconds=args.max_ready_seconds)
|
||||
recovery = True
|
||||
except Failure:
|
||||
pass
|
||||
rejected = sum(r["status"] == 429 for r in records)
|
||||
report = {
|
||||
"run_name": args.run_name,
|
||||
"settings": {"workers": baseline["concurrency"], "prefetch": baseline["prefetch"],
|
||||
"max_pending": args.max_pending,
|
||||
"worker_limits": {key: resources[key] for key in ('Memory', 'NanoCpus')}},
|
||||
"workload": {"images": args.images, "concurrency": args.concurrency,
|
||||
"size": args.size, "bytes": len(data), "type": "closed-loop burst"},
|
||||
"results": {"accepted": len(accepted), "rejected_429": rejected, "ready": len(ready),
|
||||
"failed": sum(r.get("terminal") == "failed" for r in accepted),
|
||||
"timed_out": len(pending), "elapsed_seconds": elapsed,
|
||||
"ready_per_second": len(ready) / elapsed,
|
||||
"ready_latency_ms": {str(p): percentile(latencies, p) for p in (.5, .95, .99)},
|
||||
"queue_wait_ms": {str(p): percentile(waits, p) for p in (.5, .95, .99)},
|
||||
"max_sampled_rss_bytes": max((s["rss_bytes"] for s in samples), default=None),
|
||||
"max_sampled_pending": max((s["pending"] for s in samples), default=None),
|
||||
"sample_count": len(samples),
|
||||
"drained": drained, "recovery_upload_ready": recovery},
|
||||
"records": [{k: v for k, v in r.items() if k not in ("accepted_at", "terminal_at")} |
|
||||
({"ready_seconds": r["terminal_at"] - r["accepted_at"]} if "terminal_at" in r else {})
|
||||
for r in records],
|
||||
"samples": samples, "metric_errors": errors,
|
||||
"limitations": "Closed-loop; ready time is polling upper bound. RSS is sum of worker/container processes, shared pages may be counted twice. Sampling may miss peaks. Queue wait is first claim minus durable job creation.",
|
||||
}
|
||||
target = Path(args.output or ROOT / "evidence" / (args.run_name + ".json"))
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(json.dumps(report, indent=2) + "\n")
|
||||
print(json.dumps(report["results"], indent=2))
|
||||
check(not errors and bool(samples), "incomplete metrics; result saved but not valid evidence")
|
||||
check(all(s['pending'] <= args.max_pending for s in samples), "admission limit exceeded")
|
||||
check(all(r["status"] in (202, 429) for r in records), "unexpected HTTP/transport failures")
|
||||
check(drained and recovery, "pipeline did not drain/recover; see saved result")
|
||||
if args.require_backpressure:
|
||||
check(rejected > 0, "backpressure was not observed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Public persistence test plus proof of actual container restarts."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def snapshot():
|
||||
result = {}
|
||||
for service in ('postgres', 's3', 'api-a', 'api-b', 'load_balancer'):
|
||||
container = subprocess.check_output(['docker', 'compose', 'ps', '-q', service], cwd=ROOT, text=True).strip()
|
||||
started = subprocess.check_output(['docker', 'inspect', '--format', '{{.State.StartedAt}}', container], text=True).strip()
|
||||
result[service] = {'container': container[:12], 'started_at': started}
|
||||
return result
|
||||
|
||||
|
||||
before = snapshot()
|
||||
print('Before:', json.dumps(before, sort_keys=True), flush=True)
|
||||
subprocess.run([sys.executable, 'tests/resilience.py', '--mode', 'persistence',
|
||||
'--action-script', 'scripts/restart.sh'], cwd=ROOT, check=True)
|
||||
after = snapshot()
|
||||
print('After:', json.dumps(after, sort_keys=True), flush=True)
|
||||
assert all(before[key]['started_at'] != after[key]['started_at'] for key in before)
|
||||
print('PASS: PostgreSQL, S3, both API replicas and LB actually restarted; volumes preserved')
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sequential public tests with safe container-state evidence; no credentials."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def states():
|
||||
ids = subprocess.check_output(['docker', 'compose', 'ps', '-aq'], text=True).split()
|
||||
rows = json.loads(subprocess.check_output(['docker', 'inspect', *ids], text=True))
|
||||
return {r['Config']['Labels']['com.docker.compose.service']:
|
||||
{'id': r['Id'][:12], 'started_at': r['State']['StartedAt'], 'status': r['State']['Status']}
|
||||
for r in rows}
|
||||
|
||||
|
||||
def main():
|
||||
results, stopped = [], set()
|
||||
try:
|
||||
for mode in ['persistence'] + ['failover'] * 6:
|
||||
before = states()
|
||||
hook = 'restart' if mode == 'persistence' else 'stop-one'
|
||||
result = subprocess.run([sys.executable, 'tests/resilience.py', '--mode', mode,
|
||||
'--action-script', 'scripts/' + hook + '.sh'],
|
||||
text=True, capture_output=True, timeout=200)
|
||||
after = states()
|
||||
results.append(dict(mode=mode, returncode=result.returncode,
|
||||
output=result.stdout + result.stderr, before=before, after=after))
|
||||
(ROOT / 'evidence/resilience.json').write_text(json.dumps(results, indent=2) + '\n')
|
||||
print(result.stdout + result.stderr, flush=True)
|
||||
assert result.returncode == 0, 'public resilience failed'
|
||||
if mode == 'persistence':
|
||||
for service in ('postgres', 's3', 'broker', 'worker', 'outbox', 'api-a', 'api-b'):
|
||||
assert before[service]['started_at'] != after[service]['started_at'], service
|
||||
else:
|
||||
stopped.update(s for s in ('api-a', 'api-b') if after[s]['status'] == 'exited')
|
||||
subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait', 'api-a', 'api-b'], check=True)
|
||||
if len(stopped) == 2:
|
||||
break
|
||||
assert len(stopped) == 2, 'did not exercise both API replicas'
|
||||
finally:
|
||||
subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait', 'api-a', 'api-b'], check=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user