forked from mtusi2026/mtusi2026lab7
595 lines
33 KiB
Python
Executable File
595 lines
33 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Repeatable destructive PKI study, restricted to fresh local synthetic projects.
|
|
|
|
python3 scripts/lab7_track2.py --reps 3 --duration 30 --output evidence/lab7/track2.json
|
|
|
|
Each repetition creates its own mtusi-lab7-track2-* Compose project, volumes,
|
|
loopback ports and operator state outside checkout; cleanup removes only that
|
|
project. Existing projects/volumes are never stopped or mutated. Images must
|
|
already exist: all application recreation uses --no-build. One stable HTTPS
|
|
client continuously POSTs messages; these total-expiry phases are NOT failover.
|
|
Requires securityctl bootstrap/up/issue/renew --watch/revoke/rotate-secret/
|
|
ca-rollover/audit-summary and automatic complete trust/CRL consumer reloads.
|
|
"""
|
|
import argparse
|
|
import collections
|
|
import concurrent.futures
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
sys.path.insert(0, str(ROOT / 'student-tests'))
|
|
from client import Client, Failure
|
|
from lab7_helpers import Stand, ProbeFailure, command, require, http, temporary_directory
|
|
from lab7_security import isolation
|
|
|
|
|
|
def percentile(values, fraction):
|
|
if not values:
|
|
return None
|
|
ordered = sorted(values)
|
|
return ordered[min(len(ordered) - 1, int((len(ordered) - 1) * fraction))]
|
|
|
|
|
|
def certificate(path):
|
|
result = command(['openssl', 'x509', '-in', str(path), '-noout', '-serial', '-enddate', '-issuer'])
|
|
fields = dict(line.split('=', 1) for line in result.stdout.splitlines() if '=' in line)
|
|
require('serial' in fields and 'notAfter' in fields, 'certificate_metadata_missing')
|
|
expires = dt.datetime.strptime(fields['notAfter'], '%b %d %H:%M:%S %Y %Z').replace(tzinfo=dt.timezone.utc)
|
|
return {'serial_number': fields['serial'].upper(), 'not_after': expires.isoformat(),
|
|
'expires_unix': expires.timestamp(), 'issuer': fields.get('issuer')}
|
|
|
|
|
|
def current_api_certificate(stand):
|
|
directory = stand.secrets / 'pki/api'
|
|
version = (directory / 'current-version').read_text().strip()
|
|
require(version and version not in ('.', '..') and Path(version).name == version,
|
|
'invalid_api_version_selector')
|
|
return certificate(directory / 'versions' / version / 'tls.crt')
|
|
|
|
|
|
def snapshot(stand, role, directory):
|
|
destination = Path(directory) / role
|
|
destination.mkdir(mode=0o700)
|
|
current = destination / 'current'
|
|
current.mkdir(mode=0o700)
|
|
for name in ('tls.crt', 'tls.key', 'ca.crt', 'ca.crl'):
|
|
shutil.copyfile(stand.secrets / 'pki' / role / 'current' / name, current / name)
|
|
(current / name).chmod(0o600)
|
|
return destination
|
|
|
|
|
|
def until(predicate, deadline=90, classification='recovery_deadline_exceeded'):
|
|
end = time.monotonic() + deadline
|
|
while time.monotonic() < end:
|
|
try:
|
|
result = predicate()
|
|
if result:
|
|
return result
|
|
except (ProbeFailure, Failure, OSError):
|
|
pass
|
|
time.sleep(0.2)
|
|
raise ProbeFailure(classification)
|
|
|
|
|
|
class Load:
|
|
"""A single bounded closed-loop HTTPS load client with stable actors."""
|
|
def __init__(self, base, ca, actor, chat, rate, started=None):
|
|
self.client = Client(base, str(ca), timeout=3)
|
|
self.actor, self.chat, self.rate = actor, chat, rate
|
|
self.stop = threading.Event()
|
|
self.lock = threading.Lock()
|
|
self.phase = 'baseline'
|
|
self.started = started if started is not None else time.monotonic()
|
|
self.samples = []
|
|
self.thread = threading.Thread(target=self.run, daemon=True)
|
|
|
|
def set_phase(self, phase):
|
|
with self.lock:
|
|
self.phase = phase
|
|
|
|
def run(self):
|
|
count = 0
|
|
while not self.stop.is_set():
|
|
started = time.monotonic()
|
|
with self.lock:
|
|
phase = self.phase
|
|
try:
|
|
result = self.client.message(self.actor, self.chat, 'lab7-load-' + str(count))
|
|
status, error = result['status'], None
|
|
except Failure as exc:
|
|
# Client messages omit bodies/credentials; retain only status or
|
|
# a fixed exception category, never arbitrary error text.
|
|
text = str(exc)
|
|
status = int(text.rsplit('received ', 1)[1]) if 'received ' in text and text.rsplit('received ', 1)[1].isdigit() else None
|
|
error = 'http_' + str(status) if status else 'connection_or_tls'
|
|
except Exception:
|
|
status, error = None, 'client_exception'
|
|
sample = {'at_seconds': round(started - self.started, 6), 'phase': phase,
|
|
'latency_seconds': round(time.monotonic() - started, 6), 'status': status, 'error': error}
|
|
with self.lock:
|
|
self.samples.append(sample)
|
|
count += 1
|
|
self.stop.wait(max(0, 1 / self.rate - (time.monotonic() - started)))
|
|
|
|
def close(self):
|
|
self.stop.set()
|
|
self.thread.join(timeout=8)
|
|
require(not self.thread.is_alive(), 'load_client_shutdown_timeout')
|
|
|
|
def refresh_trust(self, ca):
|
|
# Only the load thread uses self.client; pointer swap retains actor/token
|
|
# and creates a fresh CA-verified opener after trust-store publication.
|
|
self.client = Client(self.client.base, str(ca), timeout=3)
|
|
|
|
def summary(self):
|
|
with self.lock:
|
|
samples = list(self.samples)
|
|
phases = {}
|
|
for name in dict.fromkeys(s['phase'] for s in samples):
|
|
selected = [s for s in samples if s['phase'] == name]
|
|
successes = [s for s in selected if s['status'] == 201]
|
|
elapsed = max(0.001, selected[-1]['at_seconds'] + selected[-1]['latency_seconds'] - selected[0]['at_seconds'])
|
|
failures = [s for s in selected if s['error']]
|
|
phases[name] = {'requests': len(selected), 'successes': len(successes), 'errors': len(failures),
|
|
'elapsed_seconds': elapsed, 'rps': len(successes) / elapsed,
|
|
'p50_seconds': percentile([s['latency_seconds'] for s in successes], 0.50),
|
|
'p95_seconds': percentile([s['latency_seconds'] for s in successes], 0.95),
|
|
'errors_by_class': dict(collections.Counter(s['error'] for s in failures)),
|
|
'failed_span_seconds': failures[-1]['at_seconds'] - failures[0]['at_seconds'] if failures else 0}
|
|
outages = []
|
|
began = None
|
|
for sample in samples:
|
|
if sample['error'] and began is None:
|
|
began = sample
|
|
if not sample['error'] and began is not None:
|
|
outages.append({'phase_started': began['phase'], 'start_seconds': began['at_seconds'],
|
|
'recovered_seconds': sample['at_seconds'],
|
|
'observed_outage_seconds': sample['at_seconds'] - began['at_seconds'],
|
|
'recovery_request_latency_seconds': sample['latency_seconds']})
|
|
began = None
|
|
if began is not None:
|
|
outages.append({'phase_started': began['phase'], 'start_seconds': began['at_seconds'],
|
|
'recovered_seconds': None, 'observed_outage_seconds': samples[-1]['at_seconds'] - began['at_seconds']})
|
|
return {'phases': phases, 'raw_samples': samples, 'observed_outage_windows': outages,
|
|
'outage_measurement': 'Failure-to-next-success sampling windows; resolution is the actual closed-loop request cadence.'}
|
|
|
|
|
|
def free_ports(count):
|
|
sockets = []
|
|
try:
|
|
for _ in range(count):
|
|
sock = socket.socket()
|
|
sock.bind(('127.0.0.1', 0))
|
|
sockets.append(sock)
|
|
return [sock.getsockname()[1] for sock in sockets]
|
|
finally:
|
|
for sock in sockets:
|
|
sock.close()
|
|
|
|
|
|
def isolated_stand(directory, run_number):
|
|
root = Path(directory)
|
|
project = 'mtusi-lab7-track2-' + str(run_number) + '-' + uuid.uuid4().hex[:10]
|
|
ingress, s3, bao, grafana, prometheus, loki, tempo, broker, stats = free_ports(9)
|
|
env = {'OTEL_TRACES_SAMPLER': 'always_on', 'LAB7_OPERATOR_STATE': str(root / 'operator'),
|
|
'BAO_PORT': str(bao), 'GRAFANA_PORT': str(grafana), 'PUBLIC_PORT': str(ingress), 'S3_PORT': str(s3),
|
|
'PROMETHEUS_PORT': str(prometheus), 'LOKI_PORT': str(loki), 'TEMPO_PORT': str(tempo),
|
|
'BROKER_MANAGEMENT_PORT': str(broker), 'LB_STATS_PORT': str(stats)}
|
|
stand = Stand(project, root / 'secrets', 'https://localhost:' + str(ingress), env)
|
|
override = root / 'compose-study.yaml'
|
|
# Compose ports !override prevents the normal fixed loopback ports merging.
|
|
# Namespace consumers do not receive ports; publication belongs to owners.
|
|
owners = ('lb', 'seaweedfs', 'openbao', 'grafana', 'prometheus', 'loki', 'tempo', 'broker', 'otel-collector')
|
|
mappings = {'lb': [f'127.0.0.1:{ingress}:8443'], 'seaweedfs': [f'127.0.0.1:{s3}:8333'],
|
|
'openbao': [f'127.0.0.1:{bao}:8200'], 'grafana': [f'127.0.0.1:{grafana}:3000']}
|
|
lines = ['services:']
|
|
for owner in owners:
|
|
lines += [' ' + owner + ':', ' ports: !override ' + json.dumps(mappings.get(owner, []))]
|
|
for service in ('api', 'worker', 'relay'):
|
|
lines += [' ' + service + ':', ' environment:', ' OTEL_TRACES_SAMPLER: "always_on"']
|
|
override.write_text('\n'.join(lines) + '\n')
|
|
stand.overrides.append(override)
|
|
# securityctl subprocess Compose invocations must consume the same override.
|
|
stand.env['LAB7_COMPOSE_OVERRIDE'] = str(override)
|
|
return stand
|
|
|
|
|
|
def server_serial(stand):
|
|
result = stand.probe('tls', roles=('lb',), config={'host': 'api-replica', 'port': 8080, 'role': 'lb'})
|
|
require(result.get('status') == 200 and result.get('serial'), 'fresh_api_tls_control_failed')
|
|
return result['serial'].replace(':', '').upper()
|
|
|
|
|
|
def api_ready(stand, ca):
|
|
parsed = urllib.parse.urlsplit(stand.base_url)
|
|
return http('localhost', parsed.port, 'GET', '/health/ready', ca=str(ca))['status'] == 200
|
|
|
|
|
|
def retained(client, actor, chat, sentinel, attachment, content):
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
|
|
found = False
|
|
seen = set()
|
|
for _ in range(100):
|
|
page = client.call('GET', path, actor=actor, schema='MessagePage')['json']
|
|
if any(m['id'] == sentinel['id'] and m['text'] == sentinel['text'] for m in page['items']):
|
|
found = True
|
|
break
|
|
cursor = page['next_cursor']
|
|
if cursor is None:
|
|
break
|
|
require(cursor not in seen, 'persisted_history_cursor_loop')
|
|
seen.add(cursor)
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + urllib.parse.quote(cursor, safe='')
|
|
require(found, 'persisted_message_sentinel_missing')
|
|
got = client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=actor)['bytes']
|
|
require(hashlib.sha256(got).digest() == hashlib.sha256(content).digest(), 'persisted_attachment_sentinel_changed')
|
|
return {'user_id': actor['id'], 'chat_id': chat['id'], 'message_id': sentinel['id'], 'attachment_id': attachment['id'],
|
|
'attachment_sha256': hashlib.sha256(content).hexdigest()}
|
|
|
|
|
|
def resume_negative(stand, identity, mutate, control_root, expect_detail=''):
|
|
control = Path(control_root)
|
|
control.mkdir(mode=0o700)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
|
future = pool.submit(stand.probe, 'session', roles=('operator',),
|
|
config={'role': 'oldlb', 'host': 'api-replica', 'port': 8080, 'wait_seconds': 480},
|
|
extra_mounts=((identity, '/pki/oldlb'),), control_dir=control, timeout=540)
|
|
try:
|
|
until(lambda: (control / 'armed').exists() or (future.done() and future.result()), deadline=45,
|
|
classification='resumption_probe_not_armed')
|
|
mutate()
|
|
finally:
|
|
(control / 'proceed').write_text('go')
|
|
result = future.result(timeout=240)
|
|
require(all(x['classification'] == 'tls_certificate_rejected' for x in result['after']),
|
|
'old_identity_fresh_or_resumed_accepted')
|
|
if expect_detail:
|
|
require(all(expect_detail.upper() in (x.get('detail') or '').upper() for x in result['after']),
|
|
'old_identity_rejected_for_unexpected_reason', expect_detail)
|
|
return result
|
|
|
|
|
|
def run_study(args, repetition, directory):
|
|
stand = isolated_stand(directory, repetition)
|
|
ca = stand.secrets / 'pki/operator/current/ca.crt'
|
|
actions, errors = [], []
|
|
load, watcher = None, None
|
|
started = time.monotonic()
|
|
started_at = dt.datetime.now(dt.timezone.utc).isoformat()
|
|
stage = 'start'
|
|
|
|
def mark(name):
|
|
nonlocal stage
|
|
stage = name
|
|
|
|
def action(name, fn):
|
|
nonlocal stage
|
|
stage = name
|
|
before = time.monotonic()
|
|
if load:
|
|
load.set_phase(name)
|
|
result = fn()
|
|
actions.append({'phase': name, 'at_seconds': before - started, 'seconds': time.monotonic() - before, 'result': result or {}})
|
|
return result
|
|
try:
|
|
# No image build, and never bootstrap an existing project/namespace.
|
|
require(not stand.containers(), 'isolated_project_already_exists')
|
|
action('bootstrap', lambda: stand.security('bootstrap', timeout=300))
|
|
action('startup', lambda: stand.security('up', timeout=360))
|
|
stand.compose('up', '-d', '--no-build', '--scale', 'api=2', timeout=240)
|
|
until(lambda: api_ready(stand, ca), deadline=120)
|
|
client = Client(stand.base_url, str(ca), timeout=5)
|
|
alice, _, _, chat = client.bootstrap()
|
|
attachment, content = client.attachment(alice, chat)
|
|
attachment = client.wait_ready(alice, attachment, wait_seconds=120)
|
|
sentinel = client.message(alice, chat, 'lab7-persisted-' + uuid.uuid4().hex)['json']
|
|
image_ids = sorted(c['Image'] for c in stand.containers('api') + stand.containers('worker'))
|
|
action('initial_native_tls_control', lambda: {'serial_number': server_serial(stand)})
|
|
action('initial_namespace_isolation', lambda: isolation(stand))
|
|
load = Load(stand.base_url, ca, alice, chat, args.rate, started=started)
|
|
load.thread.start()
|
|
action('baseline', lambda: time.sleep(args.duration))
|
|
baseline = load.summary()['phases'].get('baseline', {})
|
|
require(baseline.get('successes', 0) > 0 and baseline.get('errors', 0) == 0, 'baseline_not_healthy')
|
|
|
|
def renewal():
|
|
nonlocal watcher
|
|
stand.security('issue', 'api', '--ttl', '20s')
|
|
original = current_api_certificate(stand)
|
|
old_expiry = original['expires_unix']
|
|
until(lambda: server_serial(stand) == original['serial_number'], deadline=10)
|
|
# Start independent cert-auth watcher at half of remaining lifetime.
|
|
time.sleep(max(0, (old_expiry - time.time()) / 2))
|
|
watcher = subprocess.Popen([sys.executable, 'scripts/securityctl.py', 'renew', 'api', '--ttl', '20s', '--watch', '--interval', '1'],
|
|
cwd=ROOT, env={**os.environ, **stand.env}, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
seen = {original['serial_number']}
|
|
records = [original]
|
|
end = time.monotonic() + max(60, args.duration)
|
|
while time.monotonic() < end:
|
|
require(watcher.poll() is None, 'leaf_renew_watcher_exited')
|
|
current = current_api_certificate(stand)
|
|
if current['serial_number'] not in seen:
|
|
# A polling timestamp is not the publication timestamp.
|
|
# Prove adoption on fresh TLS and continuity past old expiry.
|
|
until(lambda: server_serial(stand) == current['serial_number'],
|
|
deadline=5, classification='published_leaf_not_used_on_fresh_tls')
|
|
seen.add(current['serial_number'])
|
|
records.append(current)
|
|
time.sleep(0.5)
|
|
require(len(seen) >= 3 and time.time() > old_expiry, 'renewal_not_repeated_before_old_expiry')
|
|
require(api_ready(stand, ca), 'api_stopped_at_old_certificate_expiry')
|
|
return {'original_expiry': original['not_after'], 'leaf_versions': records,
|
|
'replacement_count': len(seen) - 1, 'old_expiry_survived': True}
|
|
action('leaf_renewal', renewal)
|
|
require(load.summary()['phases']['leaf_renewal']['errors'] == 0, 'renewal_load_interrupted')
|
|
|
|
def expiry():
|
|
nonlocal watcher
|
|
watcher.terminate()
|
|
try:
|
|
watcher.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
watcher.kill()
|
|
watcher.wait(timeout=5)
|
|
watcher = None
|
|
last = current_api_certificate(stand)
|
|
before = {c['Id']: c.get('RestartCount', 0) for c in stand.containers('api')}
|
|
time.sleep(max(0, last['expires_unix'] - time.time() + 1))
|
|
require(not api_ready(stand, ca), 'expired_api_remains_ready')
|
|
# The process may be in restart policy after bounded drain. A
|
|
# restart count is evidence of exit, not a fabricated state check.
|
|
def exited():
|
|
state = stand.containers('api')
|
|
return state and all(not c['State']['Running'] or c.get('RestartCount', 0) > before.get(c['Id'], 0) for c in state)
|
|
remaining = max(0.1, last['expires_unix'] + 29 - time.time())
|
|
until(exited, deadline=remaining, classification='expired_api_did_not_exit_within_shutdown_bound')
|
|
time.sleep(2)
|
|
require(not api_ready(stand, ca), 'expired_api_reopened_admission')
|
|
return {'expired_leaf': last, 'ready_closed': True, 'bounded_process_exit': True,
|
|
'shutdown_deadline_unix': last['expires_unix'] + 29, 'total_outage_not_failover': True}
|
|
action('intentional_expiry', expiry)
|
|
require(load.summary()['phases']['intentional_expiry']['errors'] > 0, 'expiry_outage_not_exercised')
|
|
|
|
def recover():
|
|
before = time.monotonic()
|
|
stand.security('issue', 'api', '--ttl', '1h')
|
|
stand.compose('up', '-d', '--no-build', '--no-deps', '--force-recreate', '--scale', 'api=2', 'api', timeout=150)
|
|
until(lambda: api_ready(stand, ca), deadline=90)
|
|
return {'recovery_seconds': time.monotonic() - before, 'retained': retained(client, alice, chat, sentinel, attachment, content)}
|
|
action('expiry_recovery', recover)
|
|
action('recovery_control', lambda: time.sleep(args.duration))
|
|
|
|
def sealed_bao():
|
|
before = time.monotonic()
|
|
stand.security('seal')
|
|
until(lambda: not api_ready(stand, ca), deadline=65,
|
|
classification='sealed_bao_did_not_close_admission')
|
|
time.sleep(2)
|
|
require(not api_ready(stand, ca), 'sealed_bao_reopened_admission')
|
|
return {'admission_closed_seconds': time.monotonic() - before,
|
|
'max_authorization_freshness_seconds': 60, 'ready_closed': True}
|
|
action('sealed_bao', sealed_bao)
|
|
require(load.summary()['phases']['sealed_bao']['errors'] > 0, 'sealed_bao_outage_not_exercised')
|
|
|
|
def unseal_recovery():
|
|
before = time.monotonic()
|
|
stand.security('unseal')
|
|
stand.compose('up', '-d', '--no-build', '--no-deps', 'api', 'worker', 'relay', timeout=150)
|
|
until(lambda: api_ready(stand, ca), deadline=90)
|
|
return {'recovery_seconds': time.monotonic() - before,
|
|
'retained': retained(client, alice, chat, sentinel, attachment, content)}
|
|
action('sealed_bao_recovery', unseal_recovery)
|
|
action('unsealed_recovery_control', lambda: time.sleep(args.duration))
|
|
require(load.summary()['phases']['unsealed_recovery_control']['errors'] == 0,
|
|
'unsealed_bao_not_recovered')
|
|
|
|
def revoke():
|
|
old = snapshot(stand, 'lb', Path(directory) / 'revoke-private')
|
|
metadata = certificate(old / 'current/tls.crt')
|
|
def mutation():
|
|
stand.security('issue', 'lb', '--ttl', '1h')
|
|
stand.security('revoke', 'lb', '--serial', metadata['serial_number'])
|
|
result = resume_negative(stand, old, mutation, Path(directory) / 'revoke-control')
|
|
until(lambda: api_ready(stand, ca), deadline=90)
|
|
return {'revoked_serial': metadata['serial_number'], 'old_identity': result, 'fresh_current_serial': server_serial(stand)}
|
|
(Path(directory) / 'revoke-private').mkdir(mode=0o700)
|
|
action('lb_leaf_rotate_revoke', revoke)
|
|
|
|
def rotate_secret():
|
|
secret_path = stand.secrets / 'backends/credentials.json'
|
|
old = json.loads(secret_path.read_text())
|
|
# Old material lives only in private temporary operator state; not
|
|
# stdout, argv, image, logs, or evidence. Only worker key is copied.
|
|
oldfile = Path(directory) / 'old-worker-credentials.json'
|
|
oldfile.write_text(json.dumps({'s3': {'worker': old['s3']['worker']}}))
|
|
oldfile.chmod(0o600)
|
|
key = 'attachments/lab7-rotation-missing-' + uuid.uuid4().hex
|
|
control = stand.probe('s3', roles=('operator',), credentials=True,
|
|
config={'single': True, 'role': 'worker', 'key': key})
|
|
require(control.get('status') == 404, 'old_s3_credential_control_not_authorized')
|
|
stand.security('rotate-secret', 'worker', timeout=180)
|
|
denied = stand.probe('s3', roles=('operator',), extra_mounts=((oldfile, '/old-credentials.json'),),
|
|
config={'credentials': '/old-credentials.json', 'single': True, 'role': 'worker', 'key': key})
|
|
require(denied.get('status') == 403, 'old_worker_s3_credential_not_403')
|
|
fresh, _ = client.attachment(alice, chat)
|
|
fresh = client.wait_ready(alice, fresh, wait_seconds=120)
|
|
oldfile.unlink()
|
|
return {'old_credential_status': 403, 'new_worker_attachment_id': fresh['id'], 'status': fresh['status']}
|
|
action('worker_s3_secret_rotation', rotate_secret)
|
|
|
|
def warm_pipeline(phase):
|
|
# Every CA phase restarts stores and both application identities.
|
|
# Retry only transport cold starts; a failed attachment is a real
|
|
# pipeline defect and must fail the run immediately.
|
|
started = time.monotonic()
|
|
deadline = started + 180
|
|
errors = []
|
|
attempts = 0
|
|
while time.monotonic() < deadline:
|
|
attempts += 1
|
|
try:
|
|
probe = Client(stand.base_url, str(ca), timeout=15)
|
|
ready, _ = probe.attachment(alice, chat)
|
|
probe.wait_ready(alice, ready, wait_seconds=60)
|
|
return {'phase': phase, 'attempts': attempts,
|
|
'seconds': round(time.monotonic() - started, 3), 'errors': errors}
|
|
except Failure as exc:
|
|
text = str(exc)
|
|
errors.append(text)
|
|
if 'attachment reached failed' in text or 'incorrect upload status' in text:
|
|
raise ProbeFailure('attachment_pipeline_failed', phase + ':' + text[:300]) from None
|
|
if time.monotonic() >= deadline:
|
|
raise ProbeFailure('attachment_pipeline_not_recovered',
|
|
phase + ':' + str(attempts) + ':' + text[:240]) from None
|
|
time.sleep(2)
|
|
raise ProbeFailure('attachment_pipeline_not_recovered', phase + ':' + str(attempts))
|
|
|
|
def rollover():
|
|
old = snapshot(stand, 'lb', Path(directory) / 'ca-private')
|
|
metadata = certificate(old / 'current/tls.crt')
|
|
warm_results = []
|
|
mark('ca_prepare')
|
|
prepare_result = stand.security('ca-rollover', '--phase', 'prepare', timeout=300)
|
|
load.refresh_trust(ca)
|
|
warm_results.append(warm_pipeline('prepare'))
|
|
mark('ca_overlap_probe')
|
|
old_control = stand.probe('tls', roles=('operator',), extra_mounts=((old, '/pki/oldlb'),),
|
|
config={'role': 'oldlb', 'host': 'api-replica', 'port': 8080, 'ca': '/pki/operator/current/ca.crt'})
|
|
require(old_control.get('status') == 200, 'overlap_old_identity_not_valid')
|
|
mark('ca_issue')
|
|
issue_result = stand.security('ca-rollover', '--phase', 'issue', timeout=360)
|
|
warm_results.append(warm_pipeline('issue'))
|
|
load.refresh_trust(ca)
|
|
mark('ca_new_replica_probe')
|
|
new_control = server_serial(stand)
|
|
again = stand.probe('tls', roles=('operator',), extra_mounts=((old, '/pki/oldlb'),),
|
|
config={'role': 'oldlb', 'host': 'api-replica', 'port': 8080, 'ca': '/pki/operator/current/ca.crt'})
|
|
require(again.get('status') == 200, 'overlap_old_identity_lost_before_retire')
|
|
mark('ca_retire')
|
|
retire_result = {}
|
|
|
|
def retire():
|
|
retire_result.update(stand.security('ca-rollover', '--phase', 'retire', timeout=360))
|
|
|
|
result = resume_negative(stand, old, retire, Path(directory) / 'ca-control',
|
|
expect_detail='UNKNOWN_CA')
|
|
load.refresh_trust(ca)
|
|
mark('ca_retire_ready')
|
|
until(lambda: api_ready(stand, ca), deadline=120)
|
|
warm_results.append(warm_pipeline('retire'))
|
|
mark('ca_attachment')
|
|
current = Client(stand.base_url, str(ca), timeout=10)
|
|
fresh, _ = current.attachment(alice, chat)
|
|
fresh = current.wait_ready(alice, fresh, wait_seconds=120)
|
|
mark('ca_backends')
|
|
bao = stand.probe('bao', roles=('api',), config={'role': 'api'})
|
|
redis = stand.probe('redis', roles=('api',), credentials=True, config={'role': 'api'})
|
|
mark('ca_telemetry')
|
|
telemetry = {}
|
|
for name, host, port, role, method, path, body in (
|
|
('api_otlp', 'otel-collector', 4318, 'api', 'POST', '/v1/traces', '{}'),
|
|
('grafana_prometheus', 'prometheus', 9090, 'grafana', 'GET', '/api/v1/query?query=1', ''),
|
|
('grafana_loki', 'loki', 3100, 'grafana', 'GET', '/loki/api/v1/query?query=vector%281%29', ''),
|
|
('prometheus_collector_scrape', 'otel-collector', 8888, 'prometheus', 'GET', '/metrics', '')):
|
|
response = stand.probe('tls', roles=(role,), config={'host': host, 'port': port, 'role': role,
|
|
'method': method, 'path': path, 'body': body, 'headers': {'Content-Type': 'application/json'}}, network='observability')
|
|
require(response.get('status') == 200, 'new_ca_telemetry_path_failed', name)
|
|
telemetry[name] = response
|
|
mark('ca_isolation')
|
|
namespace = isolation(stand)
|
|
mark('ca_retained')
|
|
return {'old_leaf_serial': metadata['serial_number'], 'overlap_old': old_control, 'overlap_new_api_serial': new_control,
|
|
'retired_old_identity': result, 'prepare': prepare_result, 'issue': issue_result,
|
|
'retire': retire_result, 'warm_pipeline': warm_results,
|
|
'new_pipeline_attachment_id': fresh['id'], 'new_bao_policy': bao,
|
|
'new_redis_cache_path': redis, 'new_telemetry_paths': telemetry, 'new_namespace_isolation': namespace,
|
|
'retained': retained(current, alice, chat, sentinel, attachment, content)}
|
|
(Path(directory) / 'ca-private').mkdir(mode=0o700)
|
|
action('ca_prepare_issue_retire', rollover)
|
|
action('final_recovery_control', lambda: time.sleep(args.duration))
|
|
require(load.summary()['phases']['final_recovery_control']['errors'] == 0, 'new_trust_paths_not_recovered')
|
|
require(sorted(c['Image'] for c in stand.containers('api') + stand.containers('worker')) == image_ids, 'study_rebuilt_or_changed_application_image')
|
|
action('bao_audit', lambda: stand.security('audit-summary'))
|
|
except ProbeFailure as exc:
|
|
errors.append({'classification': exc.classification, 'stage': stage, 'detail': exc.detail})
|
|
except Failure as exc:
|
|
# Public client errors deliberately omit credentials and query strings.
|
|
errors.append({'classification': 'public_api_contract_failure', 'stage': stage, 'detail': str(exc)})
|
|
except Exception as exc:
|
|
errors.append({'classification': 'study_exception', 'type': type(exc).__name__})
|
|
finally:
|
|
if watcher and watcher.poll() is None:
|
|
watcher.terminate()
|
|
try:
|
|
watcher.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
watcher.kill()
|
|
watcher.wait(timeout=5)
|
|
if load:
|
|
try:
|
|
load.close()
|
|
except ProbeFailure as exc:
|
|
errors.append({'classification': exc.classification})
|
|
# Destructive cleanup is confined to this generated namespace; never
|
|
# default stand, retained lab6 volumes, or user-supplied projects.
|
|
require(stand.project.startswith('mtusi-lab7-track2-') and str(stand.secrets).startswith(str(Path(directory))), 'unsafe_cleanup_namespace')
|
|
result = stand.compose('down', '--volumes', '--remove-orphans', timeout=180, check=False)
|
|
if result.returncode:
|
|
errors.append({'classification': 'isolated_project_cleanup_failed'})
|
|
return {'repetition': repetition, 'project': stand.project, 'passed': not errors, 'seconds': time.monotonic() - started, 'started_at': started_at,
|
|
'actions': actions, 'load': load.summary() if load else {'raw_samples': [], 'phases': {}}, 'errors': errors}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--reps', type=int, default=3)
|
|
parser.add_argument('--duration', type=float, default=30, help='seconds per baseline/recovery control phase; renewal is at least60s')
|
|
parser.add_argument('--rate', type=float, default=2, help='single-client offered messages per second')
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
require(args.reps >= 3 and args.duration >= 5 and 0 < args.rate <= 10, 'invalid_repeatability_parameters')
|
|
runs = []
|
|
for repetition in range(1, args.reps + 1):
|
|
with temporary_directory(prefix='mtusi-lab7-track2-private-') as directory:
|
|
Path(directory).chmod(0o700)
|
|
runs.append(run_study(args, repetition, directory))
|
|
result = {'study': 'lab7-track2', 'timestamp': dt.datetime.now(dt.timezone.utc).isoformat(), 'runs': runs,
|
|
'completed_repetitions': len(runs), 'required_repetitions': args.reps,
|
|
'passed': len(runs) == args.reps and all(run['passed'] for run in runs),
|
|
'configuration': {'offered_rate': args.rate, 'control_seconds': args.duration, 'renewal_ttl_seconds': 20,
|
|
'load_clients': 1, 'api_replicas': 2, 'trace_sampler': 'always_on', 'request_counter_sampling': 'every attempt'},
|
|
'limitations': ['Local synthetic stand; single-node Bao/PostgreSQL/S3/broker and Docker Desktop resource limits.',
|
|
'Single closed-loop client is a lifecycle continuity probe, not a capacity or open-loop benchmark.',
|
|
'Both API leaves intentionally expire: this measures total-outage fail-closed recovery, not A/B failover.',
|
|
'Fresh connections and explicit resumption attempts tested; servers with tickets disabled cannot positively resume.',
|
|
'Temporary old keys/credentials are deleted with private operator state and never written into evidence.']}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + '\n')
|
|
print(json.dumps({'study': result['study'], 'passed': result['passed'], 'completed_repetitions': len(runs),
|
|
'output': str(args.output), 'failed_repetitions': [r['repetition'] for r in runs if not r['passed']]}, sort_keys=True))
|
|
return 0 if result['passed'] else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
raise SystemExit(main())
|
|
except ProbeFailure as exc:
|
|
print(json.dumps({'passed': False, 'classification': exc.classification, 'detail': exc.detail}))
|
|
raise SystemExit(1)
|
|
except Exception as exc:
|
|
print(json.dumps({'passed': False, 'classification': 'driver_exception', 'type': type(exc).__name__}))
|
|
raise SystemExit(1)
|