forked from mtusi2026/mtusi2026lab6
Лабораторная 6: наблюдаемость мессенджера
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate local credentials once, never print them or overwrite existing files."""
|
||||
from pathlib import Path
|
||||
import os
|
||||
import secrets
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
folder = root / 'secrets'
|
||||
folder.mkdir(mode=0o700, exist_ok=True)
|
||||
password_file = folder / 'postgres_password'
|
||||
|
||||
def save(name, value):
|
||||
path = folder / name
|
||||
if path.exists():
|
||||
return
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(fd, 'w') as handle:
|
||||
handle.write(value + '\n')
|
||||
|
||||
save('postgres_password', secrets.token_hex(24))
|
||||
password = password_file.read_text().strip()
|
||||
save('database_url', 'postgresql://messenger:%s@postgres:5432/messenger' % password)
|
||||
save('cache_url', 'redis://redis:6379/0')
|
||||
save('cursor_secret', secrets.token_hex(32))
|
||||
save('s3_root_user', 'lab6-admin-' + secrets.token_hex(8))
|
||||
save('s3_root_password', secrets.token_hex(32))
|
||||
save('s3_access_key_id', 'lab6-api-' + secrets.token_hex(8))
|
||||
save('s3_secret_access_key', secrets.token_hex(24))
|
||||
save('rabbitmq_password', secrets.token_hex(24))
|
||||
save('grafana_password', secrets.token_hex(24))
|
||||
print('PASS: local secret files ready (existing values preserved)')
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit, bounded operator actions for this lab's Compose project only."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERVICES = ('api-a', 'api-b')
|
||||
|
||||
|
||||
def command(args, capture=False):
|
||||
result = subprocess.run(args, cwd=ROOT, check=True, text=True,
|
||||
stdout=subprocess.PIPE if capture else None)
|
||||
return result.stdout.strip() if capture else None
|
||||
|
||||
|
||||
def compose(*args, capture=False):
|
||||
return command(['docker', 'compose', '--env-file', '.env', *args], capture)
|
||||
|
||||
|
||||
def inspect(container):
|
||||
return json.loads(command(['docker', 'inspect', container], True))[0]
|
||||
|
||||
|
||||
def containers(service):
|
||||
return compose('ps', '-a', '-q', service, capture=True).splitlines()
|
||||
|
||||
|
||||
def start_services(*services):
|
||||
for service in services:
|
||||
for container in containers(service):
|
||||
command(['docker', 'start', container])
|
||||
|
||||
|
||||
def wait_healthy(service, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
ids = containers(service)
|
||||
if ids and all(inspect(c)['State'].get('Health', {}).get('Status') == 'healthy' for c in ids):
|
||||
return
|
||||
time.sleep(.5)
|
||||
raise RuntimeError('%s did not become healthy' % service)
|
||||
|
||||
|
||||
def target_service(target):
|
||||
if not target:
|
||||
raise RuntimeError('TARGET_INSTANCE_ID is required')
|
||||
for service in SERVICES:
|
||||
for container in containers(service):
|
||||
data = inspect(container)
|
||||
if data['Config']['Hostname'] == target:
|
||||
return service, container
|
||||
raise RuntimeError('Unknown API instance alias')
|
||||
|
||||
|
||||
def set_backend(service, enabled):
|
||||
# Fixed allowlist; shell receives no untrusted instance or credential input.
|
||||
if service not in SERVICES:
|
||||
raise ValueError('Unknown service')
|
||||
token = ('serve:' if enabled else 'drain:') + uuid.uuid4().hex
|
||||
path = '/tmp/control-' + service
|
||||
compose('exec', '-T', 'nginx', 'sh', '-ec',
|
||||
"printf '%s\\n' '" + token + "' > " + path + '.next; mv -f ' + path + '.next ' + path)
|
||||
deadline = time.monotonic() + 10
|
||||
expected = token + (' up' if enabled else ' down')
|
||||
while time.monotonic() < deadline:
|
||||
ack = compose('exec', '-T', 'nginx', 'sh', '-c',
|
||||
'cat /tmp/applied-' + service + ' 2>/dev/null || true', capture=True)
|
||||
if ack == expected:
|
||||
return
|
||||
time.sleep(.2)
|
||||
raise RuntimeError('Readiness controller did not acknowledge backend state for ' + service)
|
||||
|
||||
|
||||
def stop_one():
|
||||
service, container = target_service(os.environ.get('TARGET_INSTANCE_ID'))
|
||||
running = [c for name in SERVICES for c in containers(name)
|
||||
if inspect(c)['State'].get('Health', {}).get('Status') == 'healthy'
|
||||
and inspect(c)['State']['Running']]
|
||||
if len(running) < 2:
|
||||
raise RuntimeError('Restore the other replica before this experiment')
|
||||
print('Stopping API replica:', service, container[:12], flush=True)
|
||||
command(['docker', 'stop', '--time', '8', container])
|
||||
if inspect(container)['State']['Running']:
|
||||
raise RuntimeError('Target is still running')
|
||||
|
||||
|
||||
def restart():
|
||||
# Never start the one-shot migration service as part of persistence testing.
|
||||
compose('restart', 'postgres', 'redis', 's3', 'rabbitmq', *SERVICES, 'nginx', 'outbox-publisher', 'worker')
|
||||
for service in (*SERVICES, 'nginx'):
|
||||
wait_healthy(service)
|
||||
|
||||
|
||||
def rolling_update():
|
||||
# Run additive migrations once before replacing either API version.
|
||||
compose('run', '--rm', '--no-deps', 'migrate')
|
||||
for service in SERVICES:
|
||||
other = SERVICES[1] if service == SERVICES[0] else SERVICES[0]
|
||||
wait_healthy(other)
|
||||
set_backend(other, True)
|
||||
before = containers(service)
|
||||
set_backend(service, False)
|
||||
# The old nginx worker can retain the old upstream table until it exits.
|
||||
# worker_shutdown_timeout=6s bounds that drain; no new work then reaches it.
|
||||
time.sleep(6.5)
|
||||
compose('up', '-d', '--no-deps', '--force-recreate', '--no-build', service)
|
||||
wait_healthy(service)
|
||||
set_backend(service, True)
|
||||
time.sleep(2) # DNS valid=1s + readiness before moving to the next replica.
|
||||
after = containers(service)
|
||||
if before == after:
|
||||
raise RuntimeError('Replica was not replaced')
|
||||
print(json.dumps({'service': service, 'before': before, 'after': after}), flush=True)
|
||||
|
||||
|
||||
def restore():
|
||||
start_services(*SERVICES)
|
||||
for service in SERVICES:
|
||||
wait_healthy(service)
|
||||
for service in SERVICES:
|
||||
set_backend(service, True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=('stop-one', 'restart', 'restore', 'rolling-update', 'status'))
|
||||
args = parser.parse_args()
|
||||
if args.action == 'stop-one':
|
||||
stop_one()
|
||||
elif args.action == 'restart':
|
||||
restart()
|
||||
elif args.action == 'rolling-update':
|
||||
rolling_update()
|
||||
elif args.action == 'restore':
|
||||
restore()
|
||||
else:
|
||||
for service in ('postgres', 'redis', *SERVICES, 'nginx'):
|
||||
for container in containers(service):
|
||||
data = inspect(container)
|
||||
print(json.dumps({'service': service, 'id': data['Id'][:12],
|
||||
'instance': data['Config']['Hostname'],
|
||||
'running': data['State']['Running'],
|
||||
'started_at': data['State']['StartedAt'],
|
||||
'health': data['State'].get('Health', {}).get('Status'),
|
||||
'image_id': data['Image'],
|
||||
'cpu_limit': data['HostConfig']['NanoCpus'] / 1e9,
|
||||
'memory_limit_bytes': data['HostConfig']['Memory'],
|
||||
'published_ports': data['HostConfig'].get('PortBindings')}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
|
||||
exec python3 scripts/manage.py restart
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
|
||||
exec python3 scripts/manage.py rolling-update
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
|
||||
exec python3 scripts/manage.py stop-one
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Wait for the public API and local telemetry backends, with a shared deadline."""
|
||||
import time
|
||||
from urllib.error import URLError
|
||||
from urllib.request import urlopen
|
||||
|
||||
pending = {
|
||||
'api': 'http://localhost:18083/health/ready',
|
||||
'prometheus': 'http://localhost:19090/-/ready',
|
||||
'loki': 'http://localhost:13100/ready',
|
||||
'tempo': 'http://localhost:13200/ready',
|
||||
'grafana': 'http://localhost:13000/api/health',
|
||||
}
|
||||
deadline = time.monotonic() + 120
|
||||
while pending and time.monotonic() < deadline:
|
||||
for name, url in list(pending.items()):
|
||||
try:
|
||||
with urlopen(url, timeout=2) as response:
|
||||
if response.status == 200:
|
||||
del pending[name]
|
||||
except (URLError, TimeoutError, OSError):
|
||||
pass
|
||||
if pending:
|
||||
time.sleep(1)
|
||||
if pending:
|
||||
raise SystemExit('Readiness timeout: ' + ', '.join(sorted(pending)))
|
||||
print('PASS: API and telemetry backends ready')
|
||||
Reference in New Issue
Block a user