185 lines
9.1 KiB
Python
185 lines
9.1 KiB
Python
"""Трек 2: реальные отказы, безопасная сверка и стоимость обхода; плюс RAM upload."""
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import statistics
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
from PIL import Image
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client
|
|
from scripts.stack import command, containers, scale, save
|
|
from scripts.benchmark import percentile
|
|
|
|
|
|
def api_run(*arguments):
|
|
running = [row for row in containers('api') if row['running']]
|
|
if not running:
|
|
raise RuntimeError('No running API container')
|
|
return command(['docker', 'exec', running[0]['id'], 'python', *arguments])
|
|
|
|
|
|
def memory_stats():
|
|
output = []
|
|
for row in containers('api'):
|
|
if row['running']:
|
|
source = 'import urllib.request; print(urllib.request.urlopen("http://localhost:8080/internal/memory",timeout=5).read().decode())'
|
|
values = json.loads(command(['docker', 'exec', row['id'], 'python', '-c', source]))
|
|
output.append(dict(container=row['id'], limits=dict(cpu=row['cpu_limit'], ram=row['memory_limit_bytes']), **values))
|
|
return output
|
|
|
|
|
|
def file_checks():
|
|
result = json.loads(api_run('student-tests/storage_scenario.py', 'privacy'))
|
|
save('private_bucket.json', result)
|
|
interrupted = json.loads(api_run('student-tests/storage_scenario.py', 'interrupted'))
|
|
save('upload_interrupted.json', interrupted)
|
|
print('PASS: private bucket, limited credentials, closed console')
|
|
|
|
|
|
def storage_checks():
|
|
active = any(row['running'] for row in containers('reconciler'))
|
|
command(['docker', 'compose', 'stop', 'reconciler'])
|
|
try:
|
|
command(['docker', 'compose', 'run', '--rm', '--no-deps', 's3-bootstrap', 'deny-put'])
|
|
try:
|
|
failure = json.loads(api_run('student-tests/storage_scenario.py', 's3-failure'))
|
|
save('s3_write_failure.json', failure)
|
|
finally:
|
|
command(['docker', 'compose', 'run', '--rm', '--no-deps', 's3-bootstrap', 'allow-put'])
|
|
result = json.loads(api_run('student-tests/storage_scenario.py', 'reconcile', '--failed-chat-id', failure['chat_id']))
|
|
result['s3_write_failure'] = failure
|
|
save('reconcile_faults.json', result)
|
|
print('PASS: S3/DB faults, dry-run/apply/repeat, fresh/used/active protection')
|
|
finally:
|
|
if active:
|
|
command(['docker', 'compose', 'start', 'reconciler'])
|
|
|
|
|
|
def memory_experiment(base, repeats):
|
|
results = []
|
|
for dimension in [256, 1024]:
|
|
rng = random.Random(41)
|
|
image = Image.frombytes('RGB', (dimension, dimension), rng.randbytes(dimension*dimension*3))
|
|
output = io.BytesIO(); image.save(output, format='PNG'); image.close()
|
|
data = output.getvalue()
|
|
digest = hashlib.sha256(data).hexdigest()
|
|
for repeat in range(repeats):
|
|
print(f'Upload memory: {dimension}x{dimension}, repeat {repeat+1}/{repeats}', flush=True)
|
|
scale(2, 1, '512m', 8) # Новые процессы: high-water mark относится к этому случаю.
|
|
c = Client(base, timeout=60)
|
|
alice, bob, _, chat = c.bootstrap()
|
|
before = memory_stats()
|
|
gate = threading.Barrier(11)
|
|
def upload(_):
|
|
local = Client(base, timeout=60)
|
|
gate.wait()
|
|
start = time.monotonic()
|
|
response = local.call('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=noise.png',
|
|
201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
return response, time.monotonic()-start
|
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
|
jobs = [pool.submit(upload, index) for index in range(10)]
|
|
start = time.monotonic(); gate.wait()
|
|
batch = [job.result() for job in jobs]
|
|
elapsed = time.monotonic()-start
|
|
after = memory_stats()
|
|
instances = {response['headers']['X-Instance-Id'] for response, _ in batch}
|
|
assert len(instances) == 2, 'Memory burst did not reach both replicas'
|
|
for response, _ in batch:
|
|
att = response['json']
|
|
assert att['sha256'] == digest and att['size_bytes'] == len(data)
|
|
content = c.call('GET', '/api/v1/attachments/'+att['id']+'/content', actor=bob)['bytes']
|
|
assert content == data
|
|
values = [latency for _, latency in batch]
|
|
results.append(dict(dimension=dimension, size_bytes=len(data), concurrent_uploads=10,
|
|
repetition=repeat+1, elapsed_seconds=round(elapsed, 3), successful_uploads=10,
|
|
failed_attempts=0, burst_successful_rps=10/elapsed,
|
|
successful_latency_ms={f'p{int(p*100)}': percentile(values, p) for p in [.5, .95, .99]},
|
|
latency_samples_ms=[round(v*1000, 3) for v in values],
|
|
before=before, after=after, observed_instances=len(instances),
|
|
original_checksums_verified=True))
|
|
save('upload_memory.json', dict(runs=results, measurement='process high-water RSS and cgroup peak if available'))
|
|
return results
|
|
|
|
|
|
def cost_experiment(sizes):
|
|
active = any(row['running'] for row in containers('reconciler'))
|
|
command(['docker', 'compose', 'stop', 'reconciler'])
|
|
try:
|
|
print('Reconciler cost: datasets '+', '.join(map(str, sizes))+', three scans each', flush=True)
|
|
output = command(['docker', 'compose', 'run', '--rm', '--no-deps', '-e', 'BASE_URL=http://lb:8080',
|
|
'reconciler', 'python', 'student-tests/storage_scenario.py', 'cost',
|
|
'--sizes', *map(str, sizes)], timeout=900)
|
|
result = json.loads(output)
|
|
save('reconcile_cost.json', result)
|
|
return result['scans']
|
|
finally:
|
|
if active:
|
|
command(['docker', 'compose', 'start', 'reconciler'])
|
|
|
|
|
|
def plots(cost, uploads):
|
|
import matplotlib
|
|
matplotlib.use('Agg')
|
|
import matplotlib.pyplot as plt
|
|
sizes = sorted({row['dataset_added'] for row in cost})
|
|
seconds = [statistics.median(row['elapsed_seconds'] for row in cost if row['dataset_added']==size) for size in sizes]
|
|
fig, ax = plt.subplots(figsize=(6, 4))
|
|
ax.plot(sizes, seconds, marker='o')
|
|
ax.set(xlabel='Objects + metadata', ylabel='Median scan, seconds', title='Reconciler cost')
|
|
ax.grid(alpha=.3); fig.tight_layout()
|
|
fig.savefig(ROOT/'evidence/reconcile_cost.png', dpi=150); plt.close(fig)
|
|
dimensions = sorted({row['dimension'] for row in uploads})
|
|
peaks = [max(api['process_peak_rss_bytes'] for row in uploads if row['dimension']==d for api in row['after'])/1024**2 for d in dimensions]
|
|
fig, ax = plt.subplots(figsize=(6, 4))
|
|
ax.bar([f'{d}x{d}' for d in dimensions], peaks)
|
|
ax.set(ylabel='Peak API process RSS, MiB', title='10 concurrent uploads')
|
|
fig.tight_layout(); fig.savefig(ROOT/'evidence/upload_memory.png', dpi=150); plt.close(fig)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('--mode', choices=['privacy', 'faults', 'memory', 'cost', 'all'], default='all')
|
|
p.add_argument('--repeats', type=int, default=3)
|
|
p.add_argument('--sizes', nargs='+', type=int, default=[100, 500, 1000])
|
|
args = p.parse_args()
|
|
if args.repeats < 1 or sorted(set(args.sizes)) != args.sizes or min(args.sizes) < 1:
|
|
p.error('Positive repeats and increasing distinct dataset sizes required')
|
|
base = os.getenv('BASE_URL', 'http://localhost:8080')
|
|
save('experiment_environment.json', dict(docker=command(['docker', 'version', '--format', '{{.Server.Version}}']),
|
|
compose=command(['docker', 'compose', 'version', '--short']),
|
|
containers=containers(), client_cpus=os.cpu_count()))
|
|
if args.mode in {'privacy', 'all'}:
|
|
file_checks()
|
|
if args.mode in {'faults', 'all'}:
|
|
storage_checks()
|
|
uploads = memory_experiment(base, args.repeats) if args.mode in {'memory', 'all'} else []
|
|
cost = cost_experiment(args.sizes) if args.mode in {'cost', 'all'} else []
|
|
if uploads and cost:
|
|
plots(cost, uploads)
|
|
groups = []
|
|
for size in args.sizes:
|
|
cases = [row for row in cost if row['dataset_added'] == size]
|
|
groups.append(dict(objects=size, repetitions=len(cases),
|
|
median_seconds=statistics.median(row['elapsed_seconds'] for row in cases),
|
|
heads_per_scan=[row['counters']['head_requests'] for row in cases]))
|
|
save('experiment_summary.json', dict(reconciler_cost=groups, concurrent_uploads=10,
|
|
verified_originals=sum(row['successful_uploads'] for row in uploads),
|
|
status='PASS'))
|
|
print('Experiment completed; results saved in evidence/')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|