Files

595 lines
33 KiB
Python
Executable File

#!/usr/bin/env python3
"""Repeatable destructive PKI study, restricted to fresh local synthetic projects.
python3 scripts/lab7_track2.py --reps 3 --duration 30 --output evidence/lab7/track2.json
Each repetition creates its own mtusi-lab7-track2-* Compose project, volumes,
loopback ports and operator state outside checkout; cleanup removes only that
project. Existing projects/volumes are never stopped or mutated. Images must
already exist: all application recreation uses --no-build. One stable HTTPS
client continuously POSTs messages; these total-expiry phases are NOT failover.
Requires securityctl bootstrap/up/issue/renew --watch/revoke/rotate-secret/
ca-rollover/audit-summary and automatic complete trust/CRL consumer reloads.
"""
import argparse
import collections
import concurrent.futures
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import shutil
import socket
import subprocess
import sys
import threading
import time
import urllib.parse
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'student-tests'))
from client import Client, Failure
from lab7_helpers import Stand, ProbeFailure, command, require, http, temporary_directory
from lab7_security import isolation
def percentile(values, fraction):
if not values:
return None
ordered = sorted(values)
return ordered[min(len(ordered) - 1, int((len(ordered) - 1) * fraction))]
def certificate(path):
result = command(['openssl', 'x509', '-in', str(path), '-noout', '-serial', '-enddate', '-issuer'])
fields = dict(line.split('=', 1) for line in result.stdout.splitlines() if '=' in line)
require('serial' in fields and 'notAfter' in fields, 'certificate_metadata_missing')
expires = dt.datetime.strptime(fields['notAfter'], '%b %d %H:%M:%S %Y %Z').replace(tzinfo=dt.timezone.utc)
return {'serial_number': fields['serial'].upper(), 'not_after': expires.isoformat(),
'expires_unix': expires.timestamp(), 'issuer': fields.get('issuer')}
def current_api_certificate(stand):
directory = stand.secrets / 'pki/api'
version = (directory / 'current-version').read_text().strip()
require(version and version not in ('.', '..') and Path(version).name == version,
'invalid_api_version_selector')
return certificate(directory / 'versions' / version / 'tls.crt')
def snapshot(stand, role, directory):
destination = Path(directory) / role
destination.mkdir(mode=0o700)
current = destination / 'current'
current.mkdir(mode=0o700)
for name in ('tls.crt', 'tls.key', 'ca.crt', 'ca.crl'):
shutil.copyfile(stand.secrets / 'pki' / role / 'current' / name, current / name)
(current / name).chmod(0o600)
return destination
def until(predicate, deadline=90, classification='recovery_deadline_exceeded'):
end = time.monotonic() + deadline
while time.monotonic() < end:
try:
result = predicate()
if result:
return result
except (ProbeFailure, Failure, OSError):
pass
time.sleep(0.2)
raise ProbeFailure(classification)
class Load:
"""A single bounded closed-loop HTTPS load client with stable actors."""
def __init__(self, base, ca, actor, chat, rate, started=None):
self.client = Client(base, str(ca), timeout=3)
self.actor, self.chat, self.rate = actor, chat, rate
self.stop = threading.Event()
self.lock = threading.Lock()
self.phase = 'baseline'
self.started = started if started is not None else time.monotonic()
self.samples = []
self.thread = threading.Thread(target=self.run, daemon=True)
def set_phase(self, phase):
with self.lock:
self.phase = phase
def run(self):
count = 0
while not self.stop.is_set():
started = time.monotonic()
with self.lock:
phase = self.phase
try:
result = self.client.message(self.actor, self.chat, 'lab7-load-' + str(count))
status, error = result['status'], None
except Failure as exc:
# Client messages omit bodies/credentials; retain only status or
# a fixed exception category, never arbitrary error text.
text = str(exc)
status = int(text.rsplit('received ', 1)[1]) if 'received ' in text and text.rsplit('received ', 1)[1].isdigit() else None
error = 'http_' + str(status) if status else 'connection_or_tls'
except Exception:
status, error = None, 'client_exception'
sample = {'at_seconds': round(started - self.started, 6), 'phase': phase,
'latency_seconds': round(time.monotonic() - started, 6), 'status': status, 'error': error}
with self.lock:
self.samples.append(sample)
count += 1
self.stop.wait(max(0, 1 / self.rate - (time.monotonic() - started)))
def close(self):
self.stop.set()
self.thread.join(timeout=8)
require(not self.thread.is_alive(), 'load_client_shutdown_timeout')
def refresh_trust(self, ca):
# Only the load thread uses self.client; pointer swap retains actor/token
# and creates a fresh CA-verified opener after trust-store publication.
self.client = Client(self.client.base, str(ca), timeout=3)
def summary(self):
with self.lock:
samples = list(self.samples)
phases = {}
for name in dict.fromkeys(s['phase'] for s in samples):
selected = [s for s in samples if s['phase'] == name]
successes = [s for s in selected if s['status'] == 201]
elapsed = max(0.001, selected[-1]['at_seconds'] + selected[-1]['latency_seconds'] - selected[0]['at_seconds'])
failures = [s for s in selected if s['error']]
phases[name] = {'requests': len(selected), 'successes': len(successes), 'errors': len(failures),
'elapsed_seconds': elapsed, 'rps': len(successes) / elapsed,
'p50_seconds': percentile([s['latency_seconds'] for s in successes], 0.50),
'p95_seconds': percentile([s['latency_seconds'] for s in successes], 0.95),
'errors_by_class': dict(collections.Counter(s['error'] for s in failures)),
'failed_span_seconds': failures[-1]['at_seconds'] - failures[0]['at_seconds'] if failures else 0}
outages = []
began = None
for sample in samples:
if sample['error'] and began is None:
began = sample
if not sample['error'] and began is not None:
outages.append({'phase_started': began['phase'], 'start_seconds': began['at_seconds'],
'recovered_seconds': sample['at_seconds'],
'observed_outage_seconds': sample['at_seconds'] - began['at_seconds'],
'recovery_request_latency_seconds': sample['latency_seconds']})
began = None
if began is not None:
outages.append({'phase_started': began['phase'], 'start_seconds': began['at_seconds'],
'recovered_seconds': None, 'observed_outage_seconds': samples[-1]['at_seconds'] - began['at_seconds']})
return {'phases': phases, 'raw_samples': samples, 'observed_outage_windows': outages,
'outage_measurement': 'Failure-to-next-success sampling windows; resolution is the actual closed-loop request cadence.'}
def free_ports(count):
sockets = []
try:
for _ in range(count):
sock = socket.socket()
sock.bind(('127.0.0.1', 0))
sockets.append(sock)
return [sock.getsockname()[1] for sock in sockets]
finally:
for sock in sockets:
sock.close()
def isolated_stand(directory, run_number):
root = Path(directory)
project = 'mtusi-lab7-track2-' + str(run_number) + '-' + uuid.uuid4().hex[:10]
ingress, s3, bao, grafana, prometheus, loki, tempo, broker, stats = free_ports(9)
env = {'OTEL_TRACES_SAMPLER': 'always_on', 'LAB7_OPERATOR_STATE': str(root / 'operator'),
'BAO_PORT': str(bao), 'GRAFANA_PORT': str(grafana), 'PUBLIC_PORT': str(ingress), 'S3_PORT': str(s3),
'PROMETHEUS_PORT': str(prometheus), 'LOKI_PORT': str(loki), 'TEMPO_PORT': str(tempo),
'BROKER_MANAGEMENT_PORT': str(broker), 'LB_STATS_PORT': str(stats)}
stand = Stand(project, root / 'secrets', 'https://localhost:' + str(ingress), env)
override = root / 'compose-study.yaml'
# Compose ports !override prevents the normal fixed loopback ports merging.
# Namespace consumers do not receive ports; publication belongs to owners.
owners = ('lb', 'seaweedfs', 'openbao', 'grafana', 'prometheus', 'loki', 'tempo', 'broker', 'otel-collector')
mappings = {'lb': [f'127.0.0.1:{ingress}:8443'], 'seaweedfs': [f'127.0.0.1:{s3}:8333'],
'openbao': [f'127.0.0.1:{bao}:8200'], 'grafana': [f'127.0.0.1:{grafana}:3000']}
lines = ['services:']
for owner in owners:
lines += [' ' + owner + ':', ' ports: !override ' + json.dumps(mappings.get(owner, []))]
for service in ('api', 'worker', 'relay'):
lines += [' ' + service + ':', ' environment:', ' OTEL_TRACES_SAMPLER: "always_on"']
override.write_text('\n'.join(lines) + '\n')
stand.overrides.append(override)
# securityctl subprocess Compose invocations must consume the same override.
stand.env['LAB7_COMPOSE_OVERRIDE'] = str(override)
return stand
def server_serial(stand):
result = stand.probe('tls', roles=('lb',), config={'host': 'api-replica', 'port': 8080, 'role': 'lb'})
require(result.get('status') == 200 and result.get('serial'), 'fresh_api_tls_control_failed')
return result['serial'].replace(':', '').upper()
def api_ready(stand, ca):
parsed = urllib.parse.urlsplit(stand.base_url)
return http('localhost', parsed.port, 'GET', '/health/ready', ca=str(ca))['status'] == 200
def retained(client, actor, chat, sentinel, attachment, content):
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
found = False
seen = set()
for _ in range(100):
page = client.call('GET', path, actor=actor, schema='MessagePage')['json']
if any(m['id'] == sentinel['id'] and m['text'] == sentinel['text'] for m in page['items']):
found = True
break
cursor = page['next_cursor']
if cursor is None:
break
require(cursor not in seen, 'persisted_history_cursor_loop')
seen.add(cursor)
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + urllib.parse.quote(cursor, safe='')
require(found, 'persisted_message_sentinel_missing')
got = client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=actor)['bytes']
require(hashlib.sha256(got).digest() == hashlib.sha256(content).digest(), 'persisted_attachment_sentinel_changed')
return {'user_id': actor['id'], 'chat_id': chat['id'], 'message_id': sentinel['id'], 'attachment_id': attachment['id'],
'attachment_sha256': hashlib.sha256(content).hexdigest()}
def resume_negative(stand, identity, mutate, control_root, expect_detail=''):
control = Path(control_root)
control.mkdir(mode=0o700)
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
future = pool.submit(stand.probe, 'session', roles=('operator',),
config={'role': 'oldlb', 'host': 'api-replica', 'port': 8080, 'wait_seconds': 480},
extra_mounts=((identity, '/pki/oldlb'),), control_dir=control, timeout=540)
try:
until(lambda: (control / 'armed').exists() or (future.done() and future.result()), deadline=45,
classification='resumption_probe_not_armed')
mutate()
finally:
(control / 'proceed').write_text('go')
result = future.result(timeout=240)
require(all(x['classification'] == 'tls_certificate_rejected' for x in result['after']),
'old_identity_fresh_or_resumed_accepted')
if expect_detail:
require(all(expect_detail.upper() in (x.get('detail') or '').upper() for x in result['after']),
'old_identity_rejected_for_unexpected_reason', expect_detail)
return result
def run_study(args, repetition, directory):
stand = isolated_stand(directory, repetition)
ca = stand.secrets / 'pki/operator/current/ca.crt'
actions, errors = [], []
load, watcher = None, None
started = time.monotonic()
started_at = dt.datetime.now(dt.timezone.utc).isoformat()
stage = 'start'
def mark(name):
nonlocal stage
stage = name
def action(name, fn):
nonlocal stage
stage = name
before = time.monotonic()
if load:
load.set_phase(name)
result = fn()
actions.append({'phase': name, 'at_seconds': before - started, 'seconds': time.monotonic() - before, 'result': result or {}})
return result
try:
# No image build, and never bootstrap an existing project/namespace.
require(not stand.containers(), 'isolated_project_already_exists')
action('bootstrap', lambda: stand.security('bootstrap', timeout=300))
action('startup', lambda: stand.security('up', timeout=360))
stand.compose('up', '-d', '--no-build', '--scale', 'api=2', timeout=240)
until(lambda: api_ready(stand, ca), deadline=120)
client = Client(stand.base_url, str(ca), timeout=5)
alice, _, _, chat = client.bootstrap()
attachment, content = client.attachment(alice, chat)
attachment = client.wait_ready(alice, attachment, wait_seconds=120)
sentinel = client.message(alice, chat, 'lab7-persisted-' + uuid.uuid4().hex)['json']
image_ids = sorted(c['Image'] for c in stand.containers('api') + stand.containers('worker'))
action('initial_native_tls_control', lambda: {'serial_number': server_serial(stand)})
action('initial_namespace_isolation', lambda: isolation(stand))
load = Load(stand.base_url, ca, alice, chat, args.rate, started=started)
load.thread.start()
action('baseline', lambda: time.sleep(args.duration))
baseline = load.summary()['phases'].get('baseline', {})
require(baseline.get('successes', 0) > 0 and baseline.get('errors', 0) == 0, 'baseline_not_healthy')
def renewal():
nonlocal watcher
stand.security('issue', 'api', '--ttl', '20s')
original = current_api_certificate(stand)
old_expiry = original['expires_unix']
until(lambda: server_serial(stand) == original['serial_number'], deadline=10)
# Start independent cert-auth watcher at half of remaining lifetime.
time.sleep(max(0, (old_expiry - time.time()) / 2))
watcher = subprocess.Popen([sys.executable, 'scripts/securityctl.py', 'renew', 'api', '--ttl', '20s', '--watch', '--interval', '1'],
cwd=ROOT, env={**os.environ, **stand.env}, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
seen = {original['serial_number']}
records = [original]
end = time.monotonic() + max(60, args.duration)
while time.monotonic() < end:
require(watcher.poll() is None, 'leaf_renew_watcher_exited')
current = current_api_certificate(stand)
if current['serial_number'] not in seen:
# A polling timestamp is not the publication timestamp.
# Prove adoption on fresh TLS and continuity past old expiry.
until(lambda: server_serial(stand) == current['serial_number'],
deadline=5, classification='published_leaf_not_used_on_fresh_tls')
seen.add(current['serial_number'])
records.append(current)
time.sleep(0.5)
require(len(seen) >= 3 and time.time() > old_expiry, 'renewal_not_repeated_before_old_expiry')
require(api_ready(stand, ca), 'api_stopped_at_old_certificate_expiry')
return {'original_expiry': original['not_after'], 'leaf_versions': records,
'replacement_count': len(seen) - 1, 'old_expiry_survived': True}
action('leaf_renewal', renewal)
require(load.summary()['phases']['leaf_renewal']['errors'] == 0, 'renewal_load_interrupted')
def expiry():
nonlocal watcher
watcher.terminate()
try:
watcher.wait(timeout=10)
except subprocess.TimeoutExpired:
watcher.kill()
watcher.wait(timeout=5)
watcher = None
last = current_api_certificate(stand)
before = {c['Id']: c.get('RestartCount', 0) for c in stand.containers('api')}
time.sleep(max(0, last['expires_unix'] - time.time() + 1))
require(not api_ready(stand, ca), 'expired_api_remains_ready')
# The process may be in restart policy after bounded drain. A
# restart count is evidence of exit, not a fabricated state check.
def exited():
state = stand.containers('api')
return state and all(not c['State']['Running'] or c.get('RestartCount', 0) > before.get(c['Id'], 0) for c in state)
remaining = max(0.1, last['expires_unix'] + 29 - time.time())
until(exited, deadline=remaining, classification='expired_api_did_not_exit_within_shutdown_bound')
time.sleep(2)
require(not api_ready(stand, ca), 'expired_api_reopened_admission')
return {'expired_leaf': last, 'ready_closed': True, 'bounded_process_exit': True,
'shutdown_deadline_unix': last['expires_unix'] + 29, 'total_outage_not_failover': True}
action('intentional_expiry', expiry)
require(load.summary()['phases']['intentional_expiry']['errors'] > 0, 'expiry_outage_not_exercised')
def recover():
before = time.monotonic()
stand.security('issue', 'api', '--ttl', '1h')
stand.compose('up', '-d', '--no-build', '--no-deps', '--force-recreate', '--scale', 'api=2', 'api', timeout=150)
until(lambda: api_ready(stand, ca), deadline=90)
return {'recovery_seconds': time.monotonic() - before, 'retained': retained(client, alice, chat, sentinel, attachment, content)}
action('expiry_recovery', recover)
action('recovery_control', lambda: time.sleep(args.duration))
def sealed_bao():
before = time.monotonic()
stand.security('seal')
until(lambda: not api_ready(stand, ca), deadline=65,
classification='sealed_bao_did_not_close_admission')
time.sleep(2)
require(not api_ready(stand, ca), 'sealed_bao_reopened_admission')
return {'admission_closed_seconds': time.monotonic() - before,
'max_authorization_freshness_seconds': 60, 'ready_closed': True}
action('sealed_bao', sealed_bao)
require(load.summary()['phases']['sealed_bao']['errors'] > 0, 'sealed_bao_outage_not_exercised')
def unseal_recovery():
before = time.monotonic()
stand.security('unseal')
stand.compose('up', '-d', '--no-build', '--no-deps', 'api', 'worker', 'relay', timeout=150)
until(lambda: api_ready(stand, ca), deadline=90)
return {'recovery_seconds': time.monotonic() - before,
'retained': retained(client, alice, chat, sentinel, attachment, content)}
action('sealed_bao_recovery', unseal_recovery)
action('unsealed_recovery_control', lambda: time.sleep(args.duration))
require(load.summary()['phases']['unsealed_recovery_control']['errors'] == 0,
'unsealed_bao_not_recovered')
def revoke():
old = snapshot(stand, 'lb', Path(directory) / 'revoke-private')
metadata = certificate(old / 'current/tls.crt')
def mutation():
stand.security('issue', 'lb', '--ttl', '1h')
stand.security('revoke', 'lb', '--serial', metadata['serial_number'])
result = resume_negative(stand, old, mutation, Path(directory) / 'revoke-control')
until(lambda: api_ready(stand, ca), deadline=90)
return {'revoked_serial': metadata['serial_number'], 'old_identity': result, 'fresh_current_serial': server_serial(stand)}
(Path(directory) / 'revoke-private').mkdir(mode=0o700)
action('lb_leaf_rotate_revoke', revoke)
def rotate_secret():
secret_path = stand.secrets / 'backends/credentials.json'
old = json.loads(secret_path.read_text())
# Old material lives only in private temporary operator state; not
# stdout, argv, image, logs, or evidence. Only worker key is copied.
oldfile = Path(directory) / 'old-worker-credentials.json'
oldfile.write_text(json.dumps({'s3': {'worker': old['s3']['worker']}}))
oldfile.chmod(0o600)
key = 'attachments/lab7-rotation-missing-' + uuid.uuid4().hex
control = stand.probe('s3', roles=('operator',), credentials=True,
config={'single': True, 'role': 'worker', 'key': key})
require(control.get('status') == 404, 'old_s3_credential_control_not_authorized')
stand.security('rotate-secret', 'worker', timeout=180)
denied = stand.probe('s3', roles=('operator',), extra_mounts=((oldfile, '/old-credentials.json'),),
config={'credentials': '/old-credentials.json', 'single': True, 'role': 'worker', 'key': key})
require(denied.get('status') == 403, 'old_worker_s3_credential_not_403')
fresh, _ = client.attachment(alice, chat)
fresh = client.wait_ready(alice, fresh, wait_seconds=120)
oldfile.unlink()
return {'old_credential_status': 403, 'new_worker_attachment_id': fresh['id'], 'status': fresh['status']}
action('worker_s3_secret_rotation', rotate_secret)
def warm_pipeline(phase):
# Every CA phase restarts stores and both application identities.
# Retry only transport cold starts; a failed attachment is a real
# pipeline defect and must fail the run immediately.
started = time.monotonic()
deadline = started + 180
errors = []
attempts = 0
while time.monotonic() < deadline:
attempts += 1
try:
probe = Client(stand.base_url, str(ca), timeout=15)
ready, _ = probe.attachment(alice, chat)
probe.wait_ready(alice, ready, wait_seconds=60)
return {'phase': phase, 'attempts': attempts,
'seconds': round(time.monotonic() - started, 3), 'errors': errors}
except Failure as exc:
text = str(exc)
errors.append(text)
if 'attachment reached failed' in text or 'incorrect upload status' in text:
raise ProbeFailure('attachment_pipeline_failed', phase + ':' + text[:300]) from None
if time.monotonic() >= deadline:
raise ProbeFailure('attachment_pipeline_not_recovered',
phase + ':' + str(attempts) + ':' + text[:240]) from None
time.sleep(2)
raise ProbeFailure('attachment_pipeline_not_recovered', phase + ':' + str(attempts))
def rollover():
old = snapshot(stand, 'lb', Path(directory) / 'ca-private')
metadata = certificate(old / 'current/tls.crt')
warm_results = []
mark('ca_prepare')
prepare_result = stand.security('ca-rollover', '--phase', 'prepare', timeout=300)
load.refresh_trust(ca)
warm_results.append(warm_pipeline('prepare'))
mark('ca_overlap_probe')
old_control = stand.probe('tls', roles=('operator',), extra_mounts=((old, '/pki/oldlb'),),
config={'role': 'oldlb', 'host': 'api-replica', 'port': 8080, 'ca': '/pki/operator/current/ca.crt'})
require(old_control.get('status') == 200, 'overlap_old_identity_not_valid')
mark('ca_issue')
issue_result = stand.security('ca-rollover', '--phase', 'issue', timeout=360)
warm_results.append(warm_pipeline('issue'))
load.refresh_trust(ca)
mark('ca_new_replica_probe')
new_control = server_serial(stand)
again = stand.probe('tls', roles=('operator',), extra_mounts=((old, '/pki/oldlb'),),
config={'role': 'oldlb', 'host': 'api-replica', 'port': 8080, 'ca': '/pki/operator/current/ca.crt'})
require(again.get('status') == 200, 'overlap_old_identity_lost_before_retire')
mark('ca_retire')
retire_result = {}
def retire():
retire_result.update(stand.security('ca-rollover', '--phase', 'retire', timeout=360))
result = resume_negative(stand, old, retire, Path(directory) / 'ca-control',
expect_detail='UNKNOWN_CA')
load.refresh_trust(ca)
mark('ca_retire_ready')
until(lambda: api_ready(stand, ca), deadline=120)
warm_results.append(warm_pipeline('retire'))
mark('ca_attachment')
current = Client(stand.base_url, str(ca), timeout=10)
fresh, _ = current.attachment(alice, chat)
fresh = current.wait_ready(alice, fresh, wait_seconds=120)
mark('ca_backends')
bao = stand.probe('bao', roles=('api',), config={'role': 'api'})
redis = stand.probe('redis', roles=('api',), credentials=True, config={'role': 'api'})
mark('ca_telemetry')
telemetry = {}
for name, host, port, role, method, path, body in (
('api_otlp', 'otel-collector', 4318, 'api', 'POST', '/v1/traces', '{}'),
('grafana_prometheus', 'prometheus', 9090, 'grafana', 'GET', '/api/v1/query?query=1', ''),
('grafana_loki', 'loki', 3100, 'grafana', 'GET', '/loki/api/v1/query?query=vector%281%29', ''),
('prometheus_collector_scrape', 'otel-collector', 8888, 'prometheus', 'GET', '/metrics', '')):
response = stand.probe('tls', roles=(role,), config={'host': host, 'port': port, 'role': role,
'method': method, 'path': path, 'body': body, 'headers': {'Content-Type': 'application/json'}}, network='observability')
require(response.get('status') == 200, 'new_ca_telemetry_path_failed', name)
telemetry[name] = response
mark('ca_isolation')
namespace = isolation(stand)
mark('ca_retained')
return {'old_leaf_serial': metadata['serial_number'], 'overlap_old': old_control, 'overlap_new_api_serial': new_control,
'retired_old_identity': result, 'prepare': prepare_result, 'issue': issue_result,
'retire': retire_result, 'warm_pipeline': warm_results,
'new_pipeline_attachment_id': fresh['id'], 'new_bao_policy': bao,
'new_redis_cache_path': redis, 'new_telemetry_paths': telemetry, 'new_namespace_isolation': namespace,
'retained': retained(current, alice, chat, sentinel, attachment, content)}
(Path(directory) / 'ca-private').mkdir(mode=0o700)
action('ca_prepare_issue_retire', rollover)
action('final_recovery_control', lambda: time.sleep(args.duration))
require(load.summary()['phases']['final_recovery_control']['errors'] == 0, 'new_trust_paths_not_recovered')
require(sorted(c['Image'] for c in stand.containers('api') + stand.containers('worker')) == image_ids, 'study_rebuilt_or_changed_application_image')
action('bao_audit', lambda: stand.security('audit-summary'))
except ProbeFailure as exc:
errors.append({'classification': exc.classification, 'stage': stage, 'detail': exc.detail})
except Failure as exc:
# Public client errors deliberately omit credentials and query strings.
errors.append({'classification': 'public_api_contract_failure', 'stage': stage, 'detail': str(exc)})
except Exception as exc:
errors.append({'classification': 'study_exception', 'type': type(exc).__name__})
finally:
if watcher and watcher.poll() is None:
watcher.terminate()
try:
watcher.wait(timeout=10)
except subprocess.TimeoutExpired:
watcher.kill()
watcher.wait(timeout=5)
if load:
try:
load.close()
except ProbeFailure as exc:
errors.append({'classification': exc.classification})
# Destructive cleanup is confined to this generated namespace; never
# default stand, retained lab6 volumes, or user-supplied projects.
require(stand.project.startswith('mtusi-lab7-track2-') and str(stand.secrets).startswith(str(Path(directory))), 'unsafe_cleanup_namespace')
result = stand.compose('down', '--volumes', '--remove-orphans', timeout=180, check=False)
if result.returncode:
errors.append({'classification': 'isolated_project_cleanup_failed'})
return {'repetition': repetition, 'project': stand.project, 'passed': not errors, 'seconds': time.monotonic() - started, 'started_at': started_at,
'actions': actions, 'load': load.summary() if load else {'raw_samples': [], 'phases': {}}, 'errors': errors}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--reps', type=int, default=3)
parser.add_argument('--duration', type=float, default=30, help='seconds per baseline/recovery control phase; renewal is at least60s')
parser.add_argument('--rate', type=float, default=2, help='single-client offered messages per second')
parser.add_argument('--output', type=Path, required=True)
args = parser.parse_args()
require(args.reps >= 3 and args.duration >= 5 and 0 < args.rate <= 10, 'invalid_repeatability_parameters')
runs = []
for repetition in range(1, args.reps + 1):
with temporary_directory(prefix='mtusi-lab7-track2-private-') as directory:
Path(directory).chmod(0o700)
runs.append(run_study(args, repetition, directory))
result = {'study': 'lab7-track2', 'timestamp': dt.datetime.now(dt.timezone.utc).isoformat(), 'runs': runs,
'completed_repetitions': len(runs), 'required_repetitions': args.reps,
'passed': len(runs) == args.reps and all(run['passed'] for run in runs),
'configuration': {'offered_rate': args.rate, 'control_seconds': args.duration, 'renewal_ttl_seconds': 20,
'load_clients': 1, 'api_replicas': 2, 'trace_sampler': 'always_on', 'request_counter_sampling': 'every attempt'},
'limitations': ['Local synthetic stand; single-node Bao/PostgreSQL/S3/broker and Docker Desktop resource limits.',
'Single closed-loop client is a lifecycle continuity probe, not a capacity or open-loop benchmark.',
'Both API leaves intentionally expire: this measures total-outage fail-closed recovery, not A/B failover.',
'Fresh connections and explicit resumption attempts tested; servers with tickets disabled cannot positively resume.',
'Temporary old keys/credentials are deleted with private operator state and never written into evidence.']}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + '\n')
print(json.dumps({'study': result['study'], 'passed': result['passed'], 'completed_repetitions': len(runs),
'output': str(args.output), 'failed_repetitions': [r['repetition'] for r in runs if not r['passed']]}, sort_keys=True))
return 0 if result['passed'] else 1
if __name__ == '__main__':
try:
raise SystemExit(main())
except ProbeFailure as exc:
print(json.dumps({'passed': False, 'classification': exc.classification, 'detail': exc.detail}))
raise SystemExit(1)
except Exception as exc:
print(json.dumps({'passed': False, 'classification': 'driver_exception', 'type': type(exc).__name__}))
raise SystemExit(1)