forked from mtusi2026/mtusi2026lab3
lab3
This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Restart API replicas one at a time while the public load test is running."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
os.environ.setdefault('PYTHONUTF8', '1')
|
||||
|
||||
|
||||
def run(command, capture=False, timeout=None):
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=capture,
|
||||
check=False,
|
||||
timeout=timeout,
|
||||
)
|
||||
if result.returncode:
|
||||
detail = (result.stderr or result.stdout or '').strip()
|
||||
raise RuntimeError('Command failed: ' + ' '.join(command) + '\n' + detail)
|
||||
return result.stdout.strip() if capture else ''
|
||||
|
||||
|
||||
def wait_healthy(container_id, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
status = run(
|
||||
[
|
||||
'docker',
|
||||
'inspect',
|
||||
'--format',
|
||||
'{{.State.Health.Status}}',
|
||||
container_id,
|
||||
],
|
||||
capture=True,
|
||||
)
|
||||
if status == 'healthy':
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError('API replica did not become healthy after restart')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--duration', type=int, default=40)
|
||||
parser.add_argument('--concurrency', type=int, default=16)
|
||||
parser.add_argument('--output', default='evidence/rolling-restart.json')
|
||||
args = parser.parse_args()
|
||||
if args.duration < 10 or args.concurrency < 1:
|
||||
parser.error('duration must be at least 10 seconds and concurrency must be positive')
|
||||
|
||||
containers = run(
|
||||
['docker', 'compose', 'ps', '-q', 'api'], capture=True
|
||||
).splitlines()
|
||||
if len(containers) < 2:
|
||||
parser.error('At least two running API replicas are required')
|
||||
|
||||
output = (ROOT / args.output).resolve()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
load_output = output.with_name(output.stem + '-load.json')
|
||||
load_process = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / 'tests' / 'load.py'),
|
||||
'--base-url',
|
||||
args.base_url,
|
||||
'--duration',
|
||||
str(args.duration),
|
||||
'--concurrency',
|
||||
str(args.concurrency),
|
||||
'--output',
|
||||
str(load_output),
|
||||
],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
restarts = []
|
||||
try:
|
||||
time.sleep(2)
|
||||
for container_id in containers:
|
||||
instance_id = run(
|
||||
[
|
||||
'docker',
|
||||
'inspect',
|
||||
'--format',
|
||||
'{{.Config.Hostname}}',
|
||||
container_id,
|
||||
],
|
||||
capture=True,
|
||||
)
|
||||
started = time.monotonic()
|
||||
run(['docker', 'restart', '--timeout', '15', container_id], timeout=30)
|
||||
wait_healthy(container_id)
|
||||
restarts.append(
|
||||
{
|
||||
'instance_id': instance_id,
|
||||
'ready_seconds': round(time.monotonic() - started, 3),
|
||||
}
|
||||
)
|
||||
stdout, stderr = load_process.communicate(timeout=max(90, args.duration + 30))
|
||||
except Exception:
|
||||
if load_process.poll() is None:
|
||||
load_process.terminate()
|
||||
load_process.wait(timeout=5)
|
||||
raise
|
||||
if load_process.returncode:
|
||||
raise RuntimeError('Load test failed:\n' + (stderr or stdout).strip())
|
||||
|
||||
report = {
|
||||
'operation': 'serial API replica restart under load; same image, no schema change',
|
||||
'duration_seconds': args.duration,
|
||||
'concurrency': args.concurrency,
|
||||
'restarted_replicas': restarts,
|
||||
'load': json.loads(load_output.read_text(encoding='utf-8')),
|
||||
'load_raw_file': load_output.name,
|
||||
'limitations': [
|
||||
'This verifies graceful process replacement, not a new-image rolling deployment.',
|
||||
'PostgreSQL and Redis are single instances; their failure is not covered.',
|
||||
],
|
||||
}
|
||||
output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print(json.dumps(report['load'], ensure_ascii=False, indent=2))
|
||||
print('Saved serial restart evidence to ' + str(output))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (OSError, RuntimeError, TimeoutError, ValueError, json.JSONDecodeError) as error:
|
||||
print('ERROR: ' + str(error), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
Reference in New Issue
Block a user