forked from mtusi2026/mtusi2026lab5
Реализовать версии pipeline, fencing и автоматические сценарии отказов
This commit is contained in:
@@ -0,0 +1,55 @@
|
||||
"""Аудит обязательных материалов, локальных секретов и предыдущей лабораторной."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PREVIOUS = ROOT.parent / 'vysoko-nagr 4'
|
||||
ENV = dict(os.environ, GIT_OPTIONAL_LOCKS='0')
|
||||
|
||||
|
||||
def git(root, *args):
|
||||
return subprocess.check_output(['git', '-C', str(root), *args], env=ENV).decode('utf-8').strip()
|
||||
|
||||
|
||||
def previous():
|
||||
paths = git(PREVIOUS, 'ls-files', '-z').split('\0')
|
||||
files = {p: hashlib.sha256((PREVIOUS / p).read_bytes()).hexdigest() for p in paths if p}
|
||||
return dict(head=git(PREVIOUS, 'rev-parse', 'HEAD'), branch=git(PREVIOUS, 'branch', '--show-current'),
|
||||
status=git(PREVIOUS, 'status', '--porcelain'), files=files)
|
||||
|
||||
|
||||
def main():
|
||||
import sys
|
||||
if sys.argv[1:] == ['baseline']:
|
||||
(ROOT / 'evidence/lab4-baseline.json').write_text(json.dumps(previous(), ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print('Сохранён read-only fingerprint лабораторной 4')
|
||||
return
|
||||
baseline = json.loads((ROOT / 'evidence/lab4-baseline.json').read_text(encoding='utf-8'))
|
||||
unchanged = previous() == baseline
|
||||
assert unchanged, 'Lab4 differs from baseline'
|
||||
protected = ['README.md', 'COURSE.md', 'CONTRIBUTING.md', 'MAINTAINERS.md', 'contracts', 'tests',
|
||||
'examples', 'compose.example.yaml', '.env.example', '.github']
|
||||
assert not git(ROOT, 'diff', '276dbda', '--', *protected), 'Protected lab5 material changed'
|
||||
paths = git(ROOT, 'ls-files', '-z').split('\0')
|
||||
secret_values = [p.read_bytes().strip() for p in (ROOT / 'secrets').iterdir() if p.is_file()]
|
||||
for path in paths:
|
||||
if not path:
|
||||
continue
|
||||
assert not path.startswith('secrets/') and path != '.env', 'Tracked private state'
|
||||
content = (ROOT / path).read_bytes()
|
||||
assert not any(value and value in content for value in secret_values), 'Local secret leaked in ' + path
|
||||
assert b'-----BEGIN PRIVATE KEY-----' not in content, 'Private key found'
|
||||
authors = git(ROOT, 'log', '276dbda..HEAD', '--format=%h|%an|%ae|%cn|%ce').splitlines()
|
||||
for row in authors:
|
||||
assert row.split('|')[1:] == ['Sergey Lyakhonov', 'lyserg119@gmail.com'] * 2, 'Wrong author/committer'
|
||||
report = dict(lab4_unchanged=unchanged, protected_lab5_unchanged=True, tracked_local_secrets=False,
|
||||
scanned_files=len(paths), authors=authors, head=git(ROOT, 'rev-parse', 'HEAD'))
|
||||
(ROOT / 'evidence/audit.json').write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Безопасная фиксация версий, лимитов и опубликованных портов."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def output(*args):
|
||||
return subprocess.check_output(args, cwd=ROOT, text=True, encoding='utf-8').strip()
|
||||
|
||||
|
||||
ids = output('docker', 'compose', '-p', 'lyakhonov-lab5', 'ps', '-q').split()
|
||||
containers = json.loads(output('docker', 'inspect', *ids))
|
||||
result = dict(commit=output('git', 'rev-parse', 'HEAD'), python=platform.python_version(),
|
||||
docker=json.loads(output('docker', 'version', '--format', '{{json .}}')),
|
||||
compose=output('docker', 'compose', 'version'),
|
||||
engine=json.loads(output('docker', 'info', '--format', '{"memory_bytes":{{.MemTotal}},"cpus":{{.NCPU}}}')),
|
||||
services=[])
|
||||
for row in containers:
|
||||
result['services'].append(dict(service=row['Config']['Labels']['com.docker.compose.service'], id=row['Id'],
|
||||
image=row['Config']['Image'], image_id=row['Image'], started_at=row['State']['StartedAt'],
|
||||
memory_limit=row['HostConfig']['Memory'], nanocpus=row['HostConfig']['NanoCpus'],
|
||||
ports=row['HostConfig']['PortBindings'],
|
||||
settings=[x for x in row['Config']['Env'] if x.startswith(('JOB_', 'WORKER_', 'LAB5_FAULT_', 'STAGING_RETENTION_'))]))
|
||||
(ROOT / 'evidence/environment.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print('Saved versions, IDs, resource limits, ports and non-secret experiment settings')
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Статические графики только из сохранённых фактических агрегатов."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
data = json.loads((ROOT / 'evidence/pipeline.json').read_text(encoding='utf-8'))
|
||||
points = ('before_publish', 'after_publish', 'before_effect', 'after_original', 'after_variants', 'after_commit')
|
||||
fig, ax = plt.subplots(figsize=(10, 4.5), layout='constrained')
|
||||
for index, point in enumerate(points):
|
||||
values = [row['seconds'] for row in data if row['scenario'] == point]
|
||||
ax.scatter([index] * len(values), values, s=45)
|
||||
ax.set_xticks(range(len(points)), points, rotation=15)
|
||||
ax.set_ylabel('Full scenario duration, seconds')
|
||||
ax.set_title('Actual crash scenarios: stop, upload, restart, verified downloads')
|
||||
ax.grid(axis='y', alpha=.25)
|
||||
fig.savefig(ROOT / 'evidence/faults.png', dpi=160)
|
||||
plt.close(fig)
|
||||
data = json.loads((ROOT / 'evidence/load-pipeline.json').read_text(encoding='utf-8'))
|
||||
fig, axes = plt.subplots(1, 2, figsize=(10, 4), layout='constrained')
|
||||
for index, profile in enumerate(sorted({row['profile'] for row in data})):
|
||||
rows = [row for row in data if row['profile'] == profile]
|
||||
for axis, metric in zip(axes, ('successful_rps', 'p95')):
|
||||
axis.scatter([index] * len(rows), [row[metric] for row in rows], label=profile)
|
||||
for axis, label in zip(axes, ('Completed images / second', 'Upload to ready p95, seconds')):
|
||||
axis.set_xticks(range(len({row['profile'] for row in data})), sorted({row['profile'] for row in data}))
|
||||
axis.set_ylabel(label)
|
||||
axis.grid(axis='y', alpha=.25)
|
||||
fig.suptitle('Closed-loop pipeline: 2 clients, 20 seconds, 3 repeats')
|
||||
fig.savefig(ROOT / 'evidence/load.png', dpi=160)
|
||||
plt.close(fig)
|
||||
print('Saved evidence/faults.png and evidence/load.png')
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Чистый Git-экспорт, новые секреты и отдельный временный Compose-проект."""
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import uuid
|
||||
import zipfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
name = 'lyakhonov-lab5-repro-' + uuid.uuid4().hex[:8]
|
||||
directory = Path(tempfile.mkdtemp(prefix=name + '-'))
|
||||
archive = subprocess.check_output(['git', 'archive', '--format=zip', 'HEAD'], cwd=ROOT)
|
||||
with zipfile.ZipFile(io.BytesIO(archive)) as contents:
|
||||
contents.extractall(directory)
|
||||
with socket.socket() as listener:
|
||||
listener.bind(('127.0.0.1', 0))
|
||||
port = listener.getsockname()[1]
|
||||
env = dict(os.environ, LAB5_PORT=str(port), LAB5_FAULT_INJECTION='0', JOB_TIMEOUT_SECONDS='60', JOB_LEASE_SECONDS='90')
|
||||
services = ('api', 'migrate', 'worker', 'dispatcher', 'staging-cleaner')
|
||||
images = [name + ':app', name + ':s3']
|
||||
override = {'services': {service: {'image': images[0]} for service in services}}
|
||||
override['services'].update({service: {'image': images[1]} for service in ('s3', 'bucket-init')})
|
||||
(directory / 'repro-override.json').write_text(json.dumps(override), encoding='utf-8')
|
||||
compose = ['docker', 'compose', '-p', name, '-f', 'compose.yaml', '-f', 'repro-override.json']
|
||||
events = []
|
||||
def run(args):
|
||||
result = subprocess.run(args, cwd=directory, env=env, text=True, encoding='utf-8', capture_output=True)
|
||||
# Только безопасные последние строки: ни inspect .Config.Env, ни secret files.
|
||||
events.append(dict(command=args, exit_code=result.returncode, output=(result.stdout + result.stderr)[-5000:]))
|
||||
if result.returncode:
|
||||
raise RuntimeError('reproduction failed: ' + ' '.join(args))
|
||||
try:
|
||||
run([sys.executable, '-X', 'utf8', 'scripts/bootstrap.py'])
|
||||
run([sys.executable, '-X', 'utf8', 'scripts/check_contract.py'])
|
||||
run(['docker', 'build', '-t', images[0], '.'])
|
||||
run(['docker', 'build', '-t', images[1], '-f', 'Dockerfile.s3', '.'])
|
||||
run([*compose, 'up', '-d', '--wait'])
|
||||
run([sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', f'http://127.0.0.1:{port}'])
|
||||
run([*compose, 'ps'])
|
||||
finally:
|
||||
run([*compose, 'down', '--volumes', '--remove-orphans'])
|
||||
for tag in images:
|
||||
run(['docker', 'image', 'rm', tag])
|
||||
result = dict(project=name, directory=str(directory), port=port,
|
||||
commit=subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), events=events)
|
||||
(ROOT / 'evidence/reproduction.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print('PASS: clean export, fresh secrets, isolated project; temporary Docker resources removed')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user