This commit is contained in:
Матвей
2026-11-07 13:15:01 +03:00
parent 20afc2c790
commit eafb825657
93 changed files with 7549 additions and 1 deletions
+70
View File
@@ -0,0 +1,70 @@
#!/usr/bin/env python3
"""Sequential lab3-5 verification; capture actual output and exit status, fail fast."""
from datetime import datetime, timezone
import argparse
import json
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
DESTINATION = ROOT / "evidence/audit"
CHECKS = [
("contracts", ["make", "check", "validate", "PYTHON=.venv/bin/python"]),
("compose", ["docker", "compose", "config", "--quiet"]),
("smoke", ["make", "test"]),
("attachments", ["make", "test-attachments"]),
("idempotency", [sys.executable, "student-tests/idempotency_concurrency.py"]),
("regressions", [sys.executable, "student-tests/review_regressions.py"]),
("private-bucket", ["make", "test-private"]),
("s3-permissions", ["make", "test-s3-permissions"]),
("bootstrap", ["docker", "compose", "run", "--rm", "--no-deps", "s3-init"]),
("upload-failover", ["make", "test-upload-failover"]),
("public-failover", ["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh"]),
("restore-replicas", ["docker", "compose", "up", "-d", "--wait", "api-a", "api-b", "load_balancer"]),
("s3-outage", ["make", "test-s3-outage"]),
("persistence", [sys.executable, "scripts/verify-persistence.py"]),
("reconcile", ["make", "test-reconcile"]),
("reconcile-audit", ["make", "test-reconcile-audit"]),
("upload-memory", ["make", "test-upload-memory"]),
("reconcile-benchmark", ["make", "test-reconcile-benchmark"]),
("final-smoke", ["make", "test"]),
("environment", [sys.executable, "scripts/record-environment.py"]),
]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--resume-from', choices=[name for name, _ in CHECKS])
args = parser.parse_args()
DESTINATION.mkdir(parents=True, exist_ok=True)
results = []
checks = CHECKS
if args.resume_from:
offset = [name for name, _ in CHECKS].index(args.resume_from)
previous = json.loads((DESTINATION / 'results.json').read_text())['checks']
results = previous[:offset]
if [item['name'] for item in results] != [name for name, _ in CHECKS[:offset]] or any(item['exit_code'] for item in results):
parser.error('resume requires successful evidence for every earlier step')
checks = CHECKS[offset:]
for name, command in checks:
print(f"RUN {name}", flush=True)
started = time.monotonic()
with (DESTINATION / f"{name}.txt").open("w") as stream:
result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT)
results.append({"name": name, "command": command, "exit_code": result.returncode,
"seconds": round(time.monotonic() - started, 3)})
(DESTINATION / "results.json").write_text(json.dumps({
"recorded_at_utc": datetime.now(timezone.utc).isoformat(),
"checks": results, "complete": len(results) == len(CHECKS),
}, indent=2) + "\n")
print(f"{'PASS' if result.returncode == 0 else 'FAIL'} {name}: exit={result.returncode}", flush=True)
if result.returncode:
print((DESTINATION / f"{name}.txt").read_text()[-8000:], flush=True)
return result.returncode
return 0
if __name__ == "__main__":
raise SystemExit(main())
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env python3
"""Create local credentials without printing or overwriting existing values."""
import json
from pathlib import Path
import secrets
root = Path(__file__).resolve().parents[1] / "secrets"
root.mkdir(mode=0o700, exist_ok=True)
def write(name, value):
path = root / name
if path.exists():
return path.read_text().strip()
with path.open("x") as stream:
path.chmod(0o600)
stream.write(value + "\n")
return value
write("grafana_password", secrets.token_urlsafe(24))
password = write("postgres_password", secrets.token_hex(32))
write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger")
write("cursor_secret", secrets.token_hex(32))
admin_key = write("s3_admin_key", secrets.token_hex(16))
admin_secret = write("s3_admin_secret", secrets.token_hex(32))
app_key = write("s3_access_key_id", secrets.token_hex(16))
app_secret = write("s3_secret_access_key", secrets.token_hex(32))
rabbitmq_password = write("rabbitmq_password", secrets.token_urlsafe(32))
write("broker_url", f"amqp://messenger:{rabbitmq_password}@broker:5672/%2F")
config_path = root / "s3.json"
config_path.write_text(json.dumps({"identities": [
{"name": "bootstrap", "credentials": [{"accessKey": admin_key, "secretKey": admin_secret}],
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
{"name": "api", "credentials": [{"accessKey": app_key, "secretKey": app_secret}],
"actions": ["Read:messenger-attachments", "Write:messenger-attachments", "List:messenger-attachments"]},
]}, indent=2) + "\n")
config_path.chmod(0o600)
print("Local secrets ready; existing values preserved.")
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env python3
"""Sanitized environment and source fingerprint for local test evidence."""
from datetime import datetime, timezone
from hashlib import sha256
import json
from pathlib import Path
import platform
import subprocess
root = Path(__file__).resolve().parents[1]
def command(*args):
return subprocess.check_output(args, cwd=root, text=True).strip()
paths = [root / name for name in ('Dockerfile', 'compose.yaml', 'pyproject.toml', 'uv.lock', 'Makefile')]
for directory in ('src', 'migrations', 'infra', 'scripts', 'student-tests'):
paths.extend(path for path in (root / directory).rglob('*')
if path.is_file() and '__pycache__' not in path.parts)
report = {
'recorded_at_utc': datetime.now(timezone.utc).isoformat(),
'base_commit': command('git', 'rev-parse', 'HEAD'),
'branch': command('git', 'branch', '--show-current'),
'note': 'Results apply to this working tree, not only the base commit. Per-profile load settings are recorded in real-c*-p*-s*.json.',
'host': platform.platform(),
'docker': json.loads(command('docker', 'info', '--format',
'{"server":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}')),
'compose_version': command('docker', 'compose', 'version', '--short'),
'sources_sha256': {str(path.relative_to(root)): sha256(path.read_bytes()).hexdigest() for path in sorted(paths)},
'services': {},
}
for service in ('postgres', 's3', 'broker', 'worker', 'outbox', 'reconcile', 'api-a', 'api-b', 'load_balancer'):
container = command('docker', 'compose', 'ps', '-q', service)
info = json.loads(command('docker', 'inspect', container))[0]
report['services'][service] = {
'image': info['Config']['Image'], 'image_id': info['Image'],
'state': info['State']['Status'], 'health': info['State'].get('Health', {}).get('Status'),
'memory_limit_bytes': info['HostConfig']['Memory'],
'nano_cpus_limit': info['HostConfig']['NanoCpus'],
'volumes': [mount['Name'] for mount in info['Mounts'] if mount['Type'] == 'volume'],
}
(root / 'evidence/environment.json').write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
print('PASS: sanitized environment and source hashes saved')
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Deliberately preserves named volumes and test data.
docker compose restart postgres s3 broker
docker compose up -d --wait postgres s3 broker
docker compose run --rm --no-deps s3-init
docker compose restart worker outbox reconcile api-a api-b load_balancer
docker compose up -d --wait worker outbox reconcile api-a api-b load_balancer
+31
View File
@@ -0,0 +1,31 @@
#!/usr/bin/env python3
"""Run real, sequential Docker profiles; restore normal settings on exit."""
import os
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
def main():
try:
for workers in (1, 2, 4):
env = dict(os.environ, WORKER_CONCURRENCY=str(workers),
WORKER_PREFETCH=str(workers), MAX_PENDING_JOBS='32')
subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait',
'worker', 'api-a', 'api-b'], cwd=ROOT, env=env, check=True)
for size in (512, 2400):
subprocess.run([sys.executable, 'scripts/track1.py',
'--run-name', f'real-c{workers}-p{workers}-s{size}',
'--workers', str(workers), '--prefetch', str(workers),
'--max-pending', '32', '--images', '160',
'--concurrency', '16', '--size', str(size),
'--require-backpressure'], cwd=ROOT, check=True)
finally:
subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait',
'worker', 'api-a', 'api-b'], cwd=ROOT, check=True)
if __name__ == '__main__':
main()
+18
View File
@@ -0,0 +1,18 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cd "$(dirname "$0")/.."
for service in api-a api-b; do
container="$(docker compose ps -q "$service")"
[ -n "$container" ] || continue
hostname="$(docker inspect --format '{{.Config.Hostname}}' "$container")"
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
docker compose stop "$service"
exit 0
fi
done
printf '%s\n' 'TARGET_INSTANCE_ID does not match a running API replica' >&2
exit 2
+217
View File
@@ -0,0 +1,217 @@
#!/usr/bin/env python3
"""Closed-loop upload burst with concurrent readiness observations and drain proof."""
from concurrent.futures import ThreadPoolExecutor, as_completed
from io import BytesIO
import json
import math
from pathlib import Path
import subprocess
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, arguments, check, run
def percentile(values, point):
values = sorted(values)
return round(values[max(0, math.ceil(len(values) * point) - 1)] * 1000, 3) if values else None
def command_json(*args):
result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, timeout=20)
if result.returncode:
raise Failure("metrics command failed: " + args[0])
try:
return json.loads(result.stdout)
except ValueError:
raise Failure("metrics command returned invalid JSON") from None
def metrics(chat_id):
return command_json("docker", "compose", "exec", "-T", "worker",
"/app/.venv/bin/python", "-m", "src.pipeline_metrics", chat_id)
def broker_depth():
rows = command_json("docker", "compose", "exec", "-T", "broker",
"rabbitmqctl", "list_queues", "name", "messages_ready",
"messages_unacknowledged", "--formatter", "json")
return next(row for row in rows if row["name"] == "attachment.process.v1")
def container_info(service):
result = subprocess.run(['docker', 'compose', 'ps', '-q', service], cwd=ROOT,
capture_output=True, text=True, timeout=10, check=True)
ids = result.stdout.split()
check(len(ids) == 1, 'expected exactly one container: ' + service)
return command_json('docker', 'inspect', ids[0])[0]
def main():
def configure(p):
p.add_argument("--run-name", required=True)
p.add_argument("--workers", type=int, required=True)
p.add_argument("--prefetch", type=int, required=True)
p.add_argument("--max-pending", type=int, required=True)
p.add_argument("--images", type=int, default=40)
p.add_argument("--concurrency", type=int, default=8)
p.add_argument("--size", type=int, default=1600)
p.add_argument("--max-ready-seconds", type=float, default=180)
p.add_argument("--require-backpressure", action="store_true")
p.add_argument("--output")
args = arguments(__doc__, configure)
check(1 <= args.images <= 1000 and 1 <= args.concurrency <= 128, "invalid workload")
check(1 <= args.size <= 6000 and 0 < args.max_ready_seconds <= 3600, "invalid size/deadline")
check(args.workers > 0 and args.prefetch > 0 and args.max_pending > 0, "invalid profile")
from PIL import Image
# Reproducible, nonuniform synthetic image; no real personal data.
row = bytes(v for x in range(args.size) for v in ((x * 13) % 256, (x * 37) % 256, (x * 71) % 256))
image = Image.frombytes("RGB", (args.size, args.size), row * args.size)
stream = BytesIO()
image.save(stream, "PNG")
data = stream.getvalue()
client = Client(args.base_url, args.ca_file, timeout=30)
alice, bob, _, chat = client.bootstrap()
baseline = metrics(chat["id"])
check(baseline["concurrency"] == args.workers and baseline["prefetch"] == args.prefetch,
"worker settings do not match requested profile")
check(baseline["pending"] == 0, "use an idle isolated stack")
for service in ("api-a", "api-b"):
env = dict(item.split('=', 1) for item in container_info(service)['Config']['Env'])
check(int(env['MAX_PENDING_JOBS']) == args.max_pending,
'running API MAX_PENDING_JOBS does not match profile')
resources = container_info('worker')['HostConfig']
path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=track1.png"
pending, records, samples, errors = {}, [], [], []
lock, finished = threading.Lock(), threading.Event()
started = time.monotonic()
def observe():
local = Client(args.base_url, args.ca_file, timeout=5)
while not finished.wait(.1):
with lock:
batch = list(pending.items())
for key, record in batch:
if finished.is_set():
break
try:
att = local.call("GET", "/api/v1/attachments/" + key, actor=bob, schema="Attachment")["json"]
if att["status"] in ("ready", "failed"):
with lock:
record["terminal"] = att["status"]
record["terminal_at"] = time.monotonic()
pending.pop(key, None)
except Failure:
continue
def sample():
while not finished.is_set():
try:
sample = metrics(chat["id"])
sample["queue"] = broker_depth()
sample["seconds"] = time.monotonic() - started
with lock:
samples.append(sample)
except (Failure, OSError, subprocess.TimeoutExpired) as exc:
with lock:
errors.append(type(exc).__name__)
if finished.wait(1):
break
observer = threading.Thread(target=observe, daemon=True)
sampler = threading.Thread(target=sample, daemon=True)
observer.start()
sampler.start()
def upload(index):
local = Client(args.base_url, args.ca_file, timeout=30)
before = time.monotonic()
try:
response = local.call("POST", path, (202, 429), actor=alice, raw=data,
headers={"Content-Type": "image/png"}, schema="Attachment")
received = time.monotonic()
record = {"index": index, "status": response["status"], "upload_seconds": received - before}
if response["status"] == 202:
record.update(id=response["json"]["id"], accepted_at=received)
with lock:
records.append(record)
if response["status"] == 202:
pending[record["id"]] = record
except Failure:
with lock:
records.append({"index": index, "status": "transport_or_http_error"})
try:
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
list(pool.map(upload, range(args.images)))
upload_finished = time.monotonic()
deadline = upload_finished + args.max_ready_seconds
while time.monotonic() < deadline:
with lock:
empty = not pending
if empty:
break
time.sleep(.1)
finally:
measured_end = time.monotonic()
finished.set()
observer.join(timeout=10)
sampler.join(timeout=45)
elapsed = measured_end - started
final = metrics(chat["id"])
queue = broker_depth()
for _ in range(20):
if queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0:
break
time.sleep(.25)
queue = broker_depth()
accepted = [r for r in records if r["status"] == 202]
ready = [r for r in accepted if r.get("terminal") == "ready"]
latencies = [r["terminal_at"] - r["accepted_at"] for r in ready]
waits = [r["queue_wait_seconds"] for r in final["jobs"] if r["queue_wait_seconds"] is not None]
drained = (bool(accepted) and len(ready) == len(accepted) and final["pending"] == 0
and queue["messages_ready"] == 0 and queue["messages_unacknowledged"] == 0)
recovery = False
if drained:
try:
att, _ = client.attachment(alice, chat)
client.wait_ready(bob, att, wait_seconds=args.max_ready_seconds)
recovery = True
except Failure:
pass
rejected = sum(r["status"] == 429 for r in records)
report = {
"run_name": args.run_name,
"settings": {"workers": baseline["concurrency"], "prefetch": baseline["prefetch"],
"max_pending": args.max_pending,
"worker_limits": {key: resources[key] for key in ('Memory', 'NanoCpus')}},
"workload": {"images": args.images, "concurrency": args.concurrency,
"size": args.size, "bytes": len(data), "type": "closed-loop burst"},
"results": {"accepted": len(accepted), "rejected_429": rejected, "ready": len(ready),
"failed": sum(r.get("terminal") == "failed" for r in accepted),
"timed_out": len(pending), "elapsed_seconds": elapsed,
"ready_per_second": len(ready) / elapsed,
"ready_latency_ms": {str(p): percentile(latencies, p) for p in (.5, .95, .99)},
"queue_wait_ms": {str(p): percentile(waits, p) for p in (.5, .95, .99)},
"max_sampled_rss_bytes": max((s["rss_bytes"] for s in samples), default=None),
"max_sampled_pending": max((s["pending"] for s in samples), default=None),
"sample_count": len(samples),
"drained": drained, "recovery_upload_ready": recovery},
"records": [{k: v for k, v in r.items() if k not in ("accepted_at", "terminal_at")} |
({"ready_seconds": r["terminal_at"] - r["accepted_at"]} if "terminal_at" in r else {})
for r in records],
"samples": samples, "metric_errors": errors,
"limitations": "Closed-loop; ready time is polling upper bound. RSS is sum of worker/container processes, shared pages may be counted twice. Sampling may miss peaks. Queue wait is first claim minus durable job creation.",
}
target = Path(args.output or ROOT / "evidence" / (args.run_name + ".json"))
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(json.dumps(report, indent=2) + "\n")
print(json.dumps(report["results"], indent=2))
check(not errors and bool(samples), "incomplete metrics; result saved but not valid evidence")
check(all(s['pending'] <= args.max_pending for s in samples), "admission limit exceeded")
check(all(r["status"] in (202, 429) for r in records), "unexpected HTTP/transport failures")
check(drained and recovery, "pipeline did not drain/recover; see saved result")
if args.require_backpressure:
check(rejected > 0, "backpressure was not observed")
if __name__ == "__main__":
run(main)
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env python3
"""Public persistence test plus proof of actual container restarts."""
import json
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
def snapshot():
result = {}
for service in ('postgres', 's3', 'api-a', 'api-b', 'load_balancer'):
container = subprocess.check_output(['docker', 'compose', 'ps', '-q', service], cwd=ROOT, text=True).strip()
started = subprocess.check_output(['docker', 'inspect', '--format', '{{.State.StartedAt}}', container], text=True).strip()
result[service] = {'container': container[:12], 'started_at': started}
return result
before = snapshot()
print('Before:', json.dumps(before, sort_keys=True), flush=True)
subprocess.run([sys.executable, 'tests/resilience.py', '--mode', 'persistence',
'--action-script', 'scripts/restart.sh'], cwd=ROOT, check=True)
after = snapshot()
print('After:', json.dumps(after, sort_keys=True), flush=True)
assert all(before[key]['started_at'] != after[key]['started_at'] for key in before)
print('PASS: PostgreSQL, S3, both API replicas and LB actually restarted; volumes preserved')
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Sequential public tests with safe container-state evidence; no credentials."""
import json
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
def states():
ids = subprocess.check_output(['docker', 'compose', 'ps', '-aq'], text=True).split()
rows = json.loads(subprocess.check_output(['docker', 'inspect', *ids], text=True))
return {r['Config']['Labels']['com.docker.compose.service']:
{'id': r['Id'][:12], 'started_at': r['State']['StartedAt'], 'status': r['State']['Status']}
for r in rows}
def main():
results, stopped = [], set()
try:
for mode in ['persistence'] + ['failover'] * 6:
before = states()
hook = 'restart' if mode == 'persistence' else 'stop-one'
result = subprocess.run([sys.executable, 'tests/resilience.py', '--mode', mode,
'--action-script', 'scripts/' + hook + '.sh'],
text=True, capture_output=True, timeout=200)
after = states()
results.append(dict(mode=mode, returncode=result.returncode,
output=result.stdout + result.stderr, before=before, after=after))
(ROOT / 'evidence/resilience.json').write_text(json.dumps(results, indent=2) + '\n')
print(result.stdout + result.stderr, flush=True)
assert result.returncode == 0, 'public resilience failed'
if mode == 'persistence':
for service in ('postgres', 's3', 'broker', 'worker', 'outbox', 'api-a', 'api-b'):
assert before[service]['started_at'] != after[service]['started_at'], service
else:
stopped.update(s for s in ('api-a', 'api-b') if after[s]['status'] == 'exited')
subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait', 'api-a', 'api-b'], check=True)
if len(stopped) == 2:
break
assert len(stopped) == 2, 'did not exercise both API replicas'
finally:
subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--wait', 'api-a', 'api-b'], check=True)
if __name__ == '__main__':
main()