Выполнена лабораторная работа 4

This commit is contained in:
2026-11-16 20:06:52 +03:00
parent ec540a1a72
commit 0cd7200b1f
51 changed files with 2133 additions and 38 deletions
+32
View File
@@ -0,0 +1,32 @@
import ast
import hashlib
import io
import json
from pathlib import Path
import subprocess
import tokenize
ROOT = Path(__file__).resolve().parents[1]
def main():
paths = []
for directory in ('src', 'migrations', 'infra', 'student-tests'):
paths.extend(path for path in (ROOT / directory).rglob('*') if path.is_file() and '__pycache__' not in path.parts)
paths.extend(ROOT / name for name in ('Dockerfile', 'compose.yaml', 'requirements.txt'))
for directory in ('src', 'student-tests'):
for path in (ROOT / directory).glob('*.py'):
code = path.read_text(encoding='utf-8')
ast.parse(code)
assert not any(token.type == tokenize.COMMENT for token in tokenize.generate_tokens(io.StringIO(code).readline)), str(path)
manifest = {path.relative_to(ROOT).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(paths)}
code = "import hashlib,json; from pathlib import Path; print(json.dumps({p.as_posix():hashlib.sha256(p.read_bytes()).hexdigest() for p in Path('src').glob('*.py')}))"
result = subprocess.check_output(['docker', 'compose', 'exec', '-T', 'api', 'python', '-c', code], cwd=ROOT, text=True)
container = json.loads(result)
assert container and all(manifest[name] == digest for name, digest in container.items()), 'Container sources differ'
(ROOT / 'evidence/source-manifest.json').write_text(json.dumps({'sha256': manifest, 'container_sources_match': True}, indent=2), encoding='utf-8')
print('PASS: application Python has no comments; running container matches final source hashes')
if __name__ == '__main__':
main()
+33
View File
@@ -0,0 +1,33 @@
from pathlib import Path
import secrets
import json
root = Path(__file__).resolve().parents[1]
directory = root / 'secrets'
directory.mkdir(exist_ok=True)
password_file = directory / 'db_password'
if not password_file.exists():
password_file.write_text(secrets.token_hex(32), encoding='utf-8')
url_file = directory / 'database_url'
if not url_file.exists():
url_file.write_text('postgresql://messenger:' + password_file.read_text().strip() + '@postgres:5432/messenger', encoding='utf-8')
environment = root / '.env'
if not environment.exists():
environment.write_text('API_REPLICAS=2\nAPI_CPUS=0.5\nAPI_MEMORY=512m\nDB_POOL_SIZE=4\nSESSION_TTL_SECONDS=3600\n', encoding='utf-8')
print('Local configuration prepared; existing credentials preserved.')
for name, value in {
'minio_root_user': 'root-' + secrets.token_hex(8),
'minio_root_password': secrets.token_hex(32),
's3_access_key_id': 'api-' + secrets.token_hex(8),
's3_secret_access_key': secrets.token_hex(32),
}.items():
target = directory / name
if not target.exists():
target.write_text(value, encoding='utf-8')
identities = []
for name, key, password, actions in (
('bootstrap', 'minio_root_user', 'minio_root_password', ['Admin', 'Read', 'Write', 'List']),
('api', 's3_access_key_id', 's3_secret_access_key', ['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments']),
):
identities.append({'name': name, 'credentials': [{'accessKey': (directory / key).read_text().strip(), 'secretKey': (directory / password).read_text().strip()}], 'actions': actions})
(directory / 's3_config').write_text(json.dumps({'identities': identities}), encoding='utf-8')
+74
View File
@@ -0,0 +1,74 @@
import json
import os
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
def docker(*args):
return subprocess.check_output(['docker', *args], cwd=ROOT, text=True).strip()
def containers():
ids = docker('compose', 'ps', '-q', 'api').split()
return json.loads(docker('inspect', *ids)) if ids else []
def main():
mode = sys.argv[1]
if mode == 'restart':
subprocess.run(['docker', 'compose', 'restart', 'postgres', 's3', 'api', 'lb'], cwd=ROOT, check=True)
elif mode == 'stop':
target = os.environ.get('TARGET_INSTANCE_ID')
matches = [item for item in containers() if item['Config']['Hostname'] == target]
if len(matches) != 1:
raise SystemExit('Target must identify exactly one running API container')
print('Stopping API container ' + matches[0]['Id'][:12])
docker('stop', '-t', '2', matches[0]['Id'])
state = json.loads(docker('inspect', matches[0]['Id']))[0]
event = {'target_instance': target, 'container_id': state['Id'][:12],
'name': state['Name'], 'status_after': state['State']['Status'],
'finished_at': state['State']['FinishedAt']}
with (ROOT / 'evidence/stop-events.jsonl').open('a', encoding='utf-8') as output:
output.write(json.dumps(event) + '\n')
elif mode == 'rolling':
import time
import urllib.request
for item in containers():
name = item['Name'].lstrip('/')
started = time.time()
stages = {}
docker('stop', '-t', '6', item['Id'])
stages['stopped_seconds'] = round(time.time() - started, 3)
docker('rm', item['Id'])
stages['removed_seconds'] = round(time.time() - started, 3)
docker('compose', 'up', '-d', '--no-deps', '--scale', 'api=2', 'api')
stages['created_seconds'] = round(time.time() - started, 3)
deadline = time.monotonic() + 45
observations = []
previous = None
while time.monotonic() < deadline:
active = containers()
state = [(c['Id'][:12], c.get('State', {}).get('Health', {}).get('Status')) for c in active]
if state != previous:
observations.append({'elapsed_seconds': round(time.time()-started, 3), 'containers': [{'id': c['Id'][:12], 'health': c.get('State', {}).get('Health', {})} for c in active]})
previous = state
if len(active) == 2 and all(c.get('State', {}).get('Health', {}).get('Status') == 'healthy' for c in active):
break
time.sleep(0.5)
else:
raise SystemExit('Replacement did not become healthy: ' + name)
stages['healthy_seconds'] = round(time.time() - started, 3)
with urllib.request.urlopen(os.getenv('BASE_URL', 'http://localhost:8080').rstrip('/') + '/health/ready', timeout=3) as response:
if response.status != 200:
raise SystemExit('LB unavailable')
time.sleep(3)
print(json.dumps({'event': 'rolling_replacement', 'old_name': name, 'old_container': item['Id'][:12], 'started_epoch': started, 'finished_epoch': time.time(), 'stages': stages, 'health_observations': observations, 'new_containers': [c['Id'][:12] for c in active]}), flush=True)
else:
raise SystemExit('Unknown operation')
if __name__ == '__main__':
main()
+2
View File
@@ -0,0 +1,2 @@
@echo off
python "%~dp0operate.py" restart
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
python3 scripts/operate.py restart
+2
View File
@@ -0,0 +1,2 @@
@echo off
python "%~dp0operate.py" stop
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
python3 scripts/operate.py stop
+36
View File
@@ -0,0 +1,36 @@
import json
from pathlib import Path
import statistics
ROOT = Path(__file__).resolve().parents[1]
def main():
data = json.loads((ROOT / 'evidence/acceptance.json').read_text(encoding='utf-8'))
consistency = json.loads((ROOT / 'evidence/consistency.json').read_text(encoding='utf-8'))
lines = ['# Фактические результаты', '', '## Загрузка: 10 клиентов одновременно', '',
'| Байт | Успех / ошибки | Время, с | RPS | p50, с | p95/p99, с | Наблюдаемый максимум RAM, MiB | Cgroup peak за жизнь API, MiB |',
'| --- | --- | --- | --- | --- | --- | --- | --- |']
for row in data['uploads']:
ram = max(float(r['MemUsage'].split('MiB')[0]) for r in row['samples'])
high = max(row.get('lifetime_cgroup_peak_bytes', {}).values(), default=0) / 1048576
lines.append(f"| {row['size_bytes']} | {row['success']} / {row['errors']} | {row['seconds']:.3f} | {row['successful_rps']:.2f} | {row['p50']:.3f} | {row['p95']:.3f} | {ram:.2f} | {high:.2f} |")
lines += ['', 'Профиль: одна волна на размер, без повторов HTTP. Docker stats опрашивается параллельно; время ожидания мониторинга исключено из RPS. p95 и p99 при n=10 совпадают с максимумом (nearest rank). Cgroup peak включает прогрев и все предыдущие запросы процесса; это верхняя граница памяти волны, а не её изолированный расход.', '',
'## Reconciler: внедрённые orphan и missing', '', '| Режим | Объекты | Метаданные ready | LIST | HEAD | Удалено | Помечено missing | Время, с |', '| --- | --- | --- | --- | --- | --- | --- | --- |']
for name, row in data['reconciler'].items():
lines.append(f"| {name} | {row['objects']} | {row['metadata']} | {row['list_requests']} | {row['head_requests']} | {row['deleted']} | {row['marked_failed']} | {row['elapsed_seconds']:.3f} |")
lines += ['', '## Стоимость обхода: три повтора на размер', '', '| Дополнительные объекты | Всего объектов | LIST | HEAD | Медиана, с | Минимум / максимум, с |', '| --- | --- | --- | --- | --- | --- |']
for group in consistency['scan_cost']:
repeats = group['repeats']
durations = [r['elapsed_seconds'] for r in repeats]
row = repeats[0]
lines.append(f"| {group['extra_objects']} | {row['objects']} | {row['list_requests']} | {row['head_requests']} | {statistics.median(durations):.3f} | {min(durations):.3f} / {max(durations):.3f} |")
lines += ['', 'Сбой INSERT после успешного PUT: число объектов до и после компенсации — ' + str(consistency['database_failure_after_put']['before_objects']) + ' и ' + str(consistency['database_failure_after_put']['after_objects']) + '. Старое вложение, используемое сообщением, сохранено; SHA-256/байты и ссылка сообщения проверены.', '',
'## Проверки', '']
for name in ('contract.txt', 'smoke.txt', 'acceptance.txt', 'consistency.txt', 'persistence.txt', 'failover-1.txt', 'failover-2.txt', 'smoke-final.txt'):
lines += ['### ' + name, '', '```text', (ROOT / 'evidence' / name).read_text(encoding='utf-8').strip(), '```', '']
(ROOT / 'evidence/RESULTS.md').write_text('\n'.join(lines), encoding='utf-8')
if __name__ == '__main__':
main()
+68
View File
@@ -0,0 +1,68 @@
import argparse
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
def call(name, args):
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding='utf-8', env=os.environ | {'PYTHONUTF8': '1'})
(ROOT / 'evidence' / name).write_text(result.stdout + result.stderr, encoding='utf-8')
print(name + ': ' + str(result.returncode), flush=True)
if result.returncode:
print(result.stdout + result.stderr)
raise SystemExit(result.returncode)
def snapshot(name):
ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], cwd=ROOT, text=True).split()
rows = json.loads(subprocess.check_output(['docker', 'inspect', *ids], text=True))
safe = [{'name': r['Name'], 'id': r['Id'][:12], 'started': r['State']['StartedAt'],
'image': r['Config']['Image'], 'memory_limit': r['HostConfig']['Memory'],
'cpus': r['HostConfig']['NanoCpus']/1e9} for r in rows]
(ROOT / 'evidence' / name).write_text(json.dumps(safe, indent=2), encoding='utf-8')
def ready():
for _ in range(90):
try:
with urllib.request.urlopen('http://localhost:8080/health/ready', timeout=3) as response:
if response.status == 200:
return
except Exception:
pass
time.sleep(1)
raise RuntimeError('Readiness timeout')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--full', action='store_true')
args = parser.parse_args()
call('contract.txt', [sys.executable, '-X', 'utf8', 'scripts/check_contract.py'])
call('smoke.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py'])
if not args.full:
return
call('acceptance.txt', [sys.executable, '-X', 'utf8', 'student-tests/acceptance.py'])
call('consistency.txt', [sys.executable, '-X', 'utf8', 'student-tests/consistency.py'])
call('audit.txt', [sys.executable, '-X', 'utf8', 'scripts/audit.py'])
call('runtime-packages.txt', ['docker', 'compose', 'exec', '-T', 'api', 'pip', 'freeze'])
snapshot('before-restart.json')
suffix = 'cmd' if os.name == 'nt' else 'sh'
call('persistence.txt', [sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'persistence', '--action-script', 'scripts/restart.' + suffix])
snapshot('after-restart.json')
for repeat in (1, 2):
call('failover-' + str(repeat) + '.txt', [sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'failover', '--action-script', 'scripts/stop-one.' + suffix])
call('restore-' + str(repeat) + '.txt', ['docker', 'compose', 'up', '-d', '--no-deps', 'api'])
ready()
call('smoke-final.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py'])
call('summarize.txt', [sys.executable, '-X', 'utf8', 'scripts/summarize.py'])
if __name__ == '__main__':
main()