forked from mtusi2026/mtusi2026lab4
Выполнена лабораторная работа 4
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
import ast
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tokenize
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
paths = []
|
||||
for directory in ('src', 'migrations', 'infra', 'student-tests'):
|
||||
paths.extend(path for path in (ROOT / directory).rglob('*') if path.is_file() and '__pycache__' not in path.parts)
|
||||
paths.extend(ROOT / name for name in ('Dockerfile', 'compose.yaml', 'requirements.txt'))
|
||||
for directory in ('src', 'student-tests'):
|
||||
for path in (ROOT / directory).glob('*.py'):
|
||||
code = path.read_text(encoding='utf-8')
|
||||
ast.parse(code)
|
||||
assert not any(token.type == tokenize.COMMENT for token in tokenize.generate_tokens(io.StringIO(code).readline)), str(path)
|
||||
manifest = {path.relative_to(ROOT).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(paths)}
|
||||
code = "import hashlib,json; from pathlib import Path; print(json.dumps({p.as_posix():hashlib.sha256(p.read_bytes()).hexdigest() for p in Path('src').glob('*.py')}))"
|
||||
result = subprocess.check_output(['docker', 'compose', 'exec', '-T', 'api', 'python', '-c', code], cwd=ROOT, text=True)
|
||||
container = json.loads(result)
|
||||
assert container and all(manifest[name] == digest for name, digest in container.items()), 'Container sources differ'
|
||||
(ROOT / 'evidence/source-manifest.json').write_text(json.dumps({'sha256': manifest, 'container_sources_match': True}, indent=2), encoding='utf-8')
|
||||
print('PASS: application Python has no comments; running container matches final source hashes')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,33 @@
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import json
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
directory = root / 'secrets'
|
||||
directory.mkdir(exist_ok=True)
|
||||
password_file = directory / 'db_password'
|
||||
if not password_file.exists():
|
||||
password_file.write_text(secrets.token_hex(32), encoding='utf-8')
|
||||
url_file = directory / 'database_url'
|
||||
if not url_file.exists():
|
||||
url_file.write_text('postgresql://messenger:' + password_file.read_text().strip() + '@postgres:5432/messenger', encoding='utf-8')
|
||||
environment = root / '.env'
|
||||
if not environment.exists():
|
||||
environment.write_text('API_REPLICAS=2\nAPI_CPUS=0.5\nAPI_MEMORY=512m\nDB_POOL_SIZE=4\nSESSION_TTL_SECONDS=3600\n', encoding='utf-8')
|
||||
print('Local configuration prepared; existing credentials preserved.')
|
||||
for name, value in {
|
||||
'minio_root_user': 'root-' + secrets.token_hex(8),
|
||||
'minio_root_password': secrets.token_hex(32),
|
||||
's3_access_key_id': 'api-' + secrets.token_hex(8),
|
||||
's3_secret_access_key': secrets.token_hex(32),
|
||||
}.items():
|
||||
target = directory / name
|
||||
if not target.exists():
|
||||
target.write_text(value, encoding='utf-8')
|
||||
identities = []
|
||||
for name, key, password, actions in (
|
||||
('bootstrap', 'minio_root_user', 'minio_root_password', ['Admin', 'Read', 'Write', 'List']),
|
||||
('api', 's3_access_key_id', 's3_secret_access_key', ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments']),
|
||||
):
|
||||
identities.append({'name': name, 'credentials': [{'accessKey': (directory / key).read_text().strip(), 'secretKey': (directory / password).read_text().strip()}], 'actions': actions})
|
||||
(directory / 's3_config').write_text(json.dumps({'identities': identities}), encoding='utf-8')
|
||||
@@ -0,0 +1,74 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.check_output(['docker', *args], cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def containers():
|
||||
ids = docker('compose', 'ps', '-q', 'api').split()
|
||||
return json.loads(docker('inspect', *ids)) if ids else []
|
||||
|
||||
|
||||
def main():
|
||||
mode = sys.argv[1]
|
||||
if mode == 'restart':
|
||||
subprocess.run(['docker', 'compose', 'restart', 'postgres', 's3', 'api', 'lb'], cwd=ROOT, check=True)
|
||||
elif mode == 'stop':
|
||||
target = os.environ.get('TARGET_INSTANCE_ID')
|
||||
matches = [item for item in containers() if item['Config']['Hostname'] == target]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit('Target must identify exactly one running API container')
|
||||
print('Stopping API container ' + matches[0]['Id'][:12])
|
||||
docker('stop', '-t', '2', matches[0]['Id'])
|
||||
state = json.loads(docker('inspect', matches[0]['Id']))[0]
|
||||
event = {'target_instance': target, 'container_id': state['Id'][:12],
|
||||
'name': state['Name'], 'status_after': state['State']['Status'],
|
||||
'finished_at': state['State']['FinishedAt']}
|
||||
with (ROOT / 'evidence/stop-events.jsonl').open('a', encoding='utf-8') as output:
|
||||
output.write(json.dumps(event) + '\n')
|
||||
elif mode == 'rolling':
|
||||
import time
|
||||
import urllib.request
|
||||
for item in containers():
|
||||
name = item['Name'].lstrip('/')
|
||||
started = time.time()
|
||||
stages = {}
|
||||
docker('stop', '-t', '6', item['Id'])
|
||||
stages['stopped_seconds'] = round(time.time() - started, 3)
|
||||
docker('rm', item['Id'])
|
||||
stages['removed_seconds'] = round(time.time() - started, 3)
|
||||
docker('compose', 'up', '-d', '--no-deps', '--scale', 'api=2', 'api')
|
||||
stages['created_seconds'] = round(time.time() - started, 3)
|
||||
deadline = time.monotonic() + 45
|
||||
observations = []
|
||||
previous = None
|
||||
while time.monotonic() < deadline:
|
||||
active = containers()
|
||||
state = [(c['Id'][:12], c.get('State', {}).get('Health', {}).get('Status')) for c in active]
|
||||
if state != previous:
|
||||
observations.append({'elapsed_seconds': round(time.time()-started, 3), 'containers': [{'id': c['Id'][:12], 'health': c.get('State', {}).get('Health', {})} for c in active]})
|
||||
previous = state
|
||||
if len(active) == 2 and all(c.get('State', {}).get('Health', {}).get('Status') == 'healthy' for c in active):
|
||||
break
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
raise SystemExit('Replacement did not become healthy: ' + name)
|
||||
stages['healthy_seconds'] = round(time.time() - started, 3)
|
||||
with urllib.request.urlopen(os.getenv('BASE_URL', 'http://localhost:8080').rstrip('/') + '/health/ready', timeout=3) as response:
|
||||
if response.status != 200:
|
||||
raise SystemExit('LB unavailable')
|
||||
time.sleep(3)
|
||||
print(json.dumps({'event': 'rolling_replacement', 'old_name': name, 'old_container': item['Id'][:12], 'started_epoch': started, 'finished_epoch': time.time(), 'stages': stages, 'health_observations': observations, 'new_containers': [c['Id'][:12] for c in active]}), flush=True)
|
||||
else:
|
||||
raise SystemExit('Unknown operation')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
python "%~dp0operate.py" restart
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
python3 scripts/operate.py restart
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
python "%~dp0operate.py" stop
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
python3 scripts/operate.py stop
|
||||
@@ -0,0 +1,36 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
data = json.loads((ROOT / 'evidence/acceptance.json').read_text(encoding='utf-8'))
|
||||
consistency = json.loads((ROOT / 'evidence/consistency.json').read_text(encoding='utf-8'))
|
||||
lines = ['# Фактические результаты', '', '## Загрузка: 10 клиентов одновременно', '',
|
||||
'| Байт | Успех / ошибки | Время, с | RPS | p50, с | p95/p99, с | Наблюдаемый максимум RAM, MiB | Cgroup peak за жизнь API, MiB |',
|
||||
'| --- | --- | --- | --- | --- | --- | --- | --- |']
|
||||
for row in data['uploads']:
|
||||
ram = max(float(r['MemUsage'].split('MiB')[0]) for r in row['samples'])
|
||||
high = max(row.get('lifetime_cgroup_peak_bytes', {}).values(), default=0) / 1048576
|
||||
lines.append(f"| {row['size_bytes']} | {row['success']} / {row['errors']} | {row['seconds']:.3f} | {row['successful_rps']:.2f} | {row['p50']:.3f} | {row['p95']:.3f} | {ram:.2f} | {high:.2f} |")
|
||||
lines += ['', 'Профиль: одна волна на размер, без повторов HTTP. Docker stats опрашивается параллельно; время ожидания мониторинга исключено из RPS. p95 и p99 при n=10 совпадают с максимумом (nearest rank). Cgroup peak включает прогрев и все предыдущие запросы процесса; это верхняя граница памяти волны, а не её изолированный расход.', '',
|
||||
'## Reconciler: внедрённые orphan и missing', '', '| Режим | Объекты | Метаданные ready | LIST | HEAD | Удалено | Помечено missing | Время, с |', '| --- | --- | --- | --- | --- | --- | --- | --- |']
|
||||
for name, row in data['reconciler'].items():
|
||||
lines.append(f"| {name} | {row['objects']} | {row['metadata']} | {row['list_requests']} | {row['head_requests']} | {row['deleted']} | {row['marked_failed']} | {row['elapsed_seconds']:.3f} |")
|
||||
lines += ['', '## Стоимость обхода: три повтора на размер', '', '| Дополнительные объекты | Всего объектов | LIST | HEAD | Медиана, с | Минимум / максимум, с |', '| --- | --- | --- | --- | --- | --- |']
|
||||
for group in consistency['scan_cost']:
|
||||
repeats = group['repeats']
|
||||
durations = [r['elapsed_seconds'] for r in repeats]
|
||||
row = repeats[0]
|
||||
lines.append(f"| {group['extra_objects']} | {row['objects']} | {row['list_requests']} | {row['head_requests']} | {statistics.median(durations):.3f} | {min(durations):.3f} / {max(durations):.3f} |")
|
||||
lines += ['', 'Сбой INSERT после успешного PUT: число объектов до и после компенсации — ' + str(consistency['database_failure_after_put']['before_objects']) + ' и ' + str(consistency['database_failure_after_put']['after_objects']) + '. Старое вложение, используемое сообщением, сохранено; SHA-256/байты и ссылка сообщения проверены.', '',
|
||||
'## Проверки', '']
|
||||
for name in ('contract.txt', 'smoke.txt', 'acceptance.txt', 'consistency.txt', 'persistence.txt', 'failover-1.txt', 'failover-2.txt', 'smoke-final.txt'):
|
||||
lines += ['### ' + name, '', '```text', (ROOT / 'evidence' / name).read_text(encoding='utf-8').strip(), '```', '']
|
||||
(ROOT / 'evidence/RESULTS.md').write_text('\n'.join(lines), encoding='utf-8')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def call(name, args):
|
||||
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding='utf-8', env=os.environ | {'PYTHONUTF8': '1'})
|
||||
(ROOT / 'evidence' / name).write_text(result.stdout + result.stderr, encoding='utf-8')
|
||||
print(name + ': ' + str(result.returncode), flush=True)
|
||||
if result.returncode:
|
||||
print(result.stdout + result.stderr)
|
||||
raise SystemExit(result.returncode)
|
||||
|
||||
|
||||
def snapshot(name):
|
||||
ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], cwd=ROOT, text=True).split()
|
||||
rows = json.loads(subprocess.check_output(['docker', 'inspect', *ids], text=True))
|
||||
safe = [{'name': r['Name'], 'id': r['Id'][:12], 'started': r['State']['StartedAt'],
|
||||
'image': r['Config']['Image'], 'memory_limit': r['HostConfig']['Memory'],
|
||||
'cpus': r['HostConfig']['NanoCpus']/1e9} for r in rows]
|
||||
(ROOT / 'evidence' / name).write_text(json.dumps(safe, indent=2), encoding='utf-8')
|
||||
|
||||
|
||||
def ready():
|
||||
for _ in range(90):
|
||||
try:
|
||||
with urllib.request.urlopen('http://localhost:8080/health/ready', timeout=3) as response:
|
||||
if response.status == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise RuntimeError('Readiness timeout')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--full', action='store_true')
|
||||
args = parser.parse_args()
|
||||
call('contract.txt', [sys.executable, '-X', 'utf8', 'scripts/check_contract.py'])
|
||||
call('smoke.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py'])
|
||||
if not args.full:
|
||||
return
|
||||
call('acceptance.txt', [sys.executable, '-X', 'utf8', 'student-tests/acceptance.py'])
|
||||
call('consistency.txt', [sys.executable, '-X', 'utf8', 'student-tests/consistency.py'])
|
||||
call('audit.txt', [sys.executable, '-X', 'utf8', 'scripts/audit.py'])
|
||||
call('runtime-packages.txt', ['docker', 'compose', 'exec', '-T', 'api', 'pip', 'freeze'])
|
||||
snapshot('before-restart.json')
|
||||
suffix = 'cmd' if os.name == 'nt' else 'sh'
|
||||
call('persistence.txt', [sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'persistence', '--action-script', 'scripts/restart.' + suffix])
|
||||
snapshot('after-restart.json')
|
||||
for repeat in (1, 2):
|
||||
call('failover-' + str(repeat) + '.txt', [sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'failover', '--action-script', 'scripts/stop-one.' + suffix])
|
||||
call('restore-' + str(repeat) + '.txt', ['docker', 'compose', 'up', '-d', '--no-deps', 'api'])
|
||||
ready()
|
||||
call('smoke-final.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py'])
|
||||
call('summarize.txt', [sys.executable, '-X', 'utf8', 'scripts/summarize.py'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user