forked from mtusi2026/mtusi2026lab4
lab4
This commit is contained in:
@@ -0,0 +1,156 @@
|
||||
"""Один профиль нагрузки для всех серий. Только синтетические данные, без токенов в отчёте."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from collections import Counter
|
||||
import math
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, Failure
|
||||
|
||||
|
||||
def context(base):
|
||||
c = Client(base)
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
return dict(actor=alice, reader=bob, chat=chat)
|
||||
|
||||
|
||||
def percentile(values, p):
|
||||
if not values:
|
||||
return None
|
||||
return round(sorted(values)[max(0, math.ceil(p * len(values)) - 1)] * 1000, 3)
|
||||
|
||||
|
||||
def measure(base, ctx, duration, concurrency, action=None, retry_writes=False):
|
||||
gate = threading.Event()
|
||||
started = time.monotonic()
|
||||
deadline = started + duration
|
||||
action_events = []
|
||||
|
||||
def perform_action():
|
||||
nonlocal deadline
|
||||
time.sleep(2)
|
||||
start = time.monotonic()
|
||||
try:
|
||||
info = action()
|
||||
deadline = max(deadline, time.monotonic() + 5)
|
||||
action_events.append(dict(start_seconds=start-started, end_seconds=time.monotonic()-started,
|
||||
result=info))
|
||||
except Exception as exc:
|
||||
action_events.append(dict(start_seconds=start-started, error=str(exc)))
|
||||
|
||||
def worker(worker):
|
||||
c = Client(base, timeout=3)
|
||||
gate.wait()
|
||||
latencies, confirmed, bins = [], [], Counter()
|
||||
errors, instances, failures, operation_counts = Counter(), Counter(), [], Counter()
|
||||
step, pending = worker, None
|
||||
while time.monotonic() < deadline or (action_thread and action_thread.is_alive()):
|
||||
if time.monotonic() - started > 180:
|
||||
raise RuntimeError('Infrastructure action exceeded workload time limit')
|
||||
begin = time.monotonic()
|
||||
# Один и тот же цикл: 60% POST, 20% история, 20% профиль.
|
||||
operation = 'write' if pending or step % 5 < 3 else ('history' if step % 5 == 3 else 'profile')
|
||||
operation_counts[operation] += 1
|
||||
try:
|
||||
if operation == 'write':
|
||||
key, text = pending or (uuid.uuid4().hex, 'x' * 128)
|
||||
response = c.message(ctx['actor'], ctx['chat'], text, key=key)
|
||||
confirmed.append(response['json'])
|
||||
pending = None
|
||||
elif operation == 'history':
|
||||
response = c.call('GET', '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=50',
|
||||
actor=ctx['reader'], schema='MessagePage')
|
||||
else:
|
||||
response = c.call('GET', '/api/v1/users/me', actor=ctx['actor'], schema='User')
|
||||
elapsed = time.monotonic() - begin
|
||||
latencies.append(elapsed)
|
||||
instances[response['headers']['X-Instance-Id']] += 1
|
||||
bins[int(time.monotonic() - started)] += 1
|
||||
except Failure as exc:
|
||||
# Сообщения ошибки клиента не содержат credentials/body.
|
||||
errors[str(exc)] += 1
|
||||
failures.append(round(time.monotonic()-started, 3))
|
||||
if operation == 'write' and retry_writes:
|
||||
pending = (key, text)
|
||||
step += 1
|
||||
# Последний timeout мог произойти после COMMIT. Уточняем результат тем же
|
||||
# ключом вне измеряемого окна, иначе проверка ошибочно сочтёт его дубликатом.
|
||||
finished = time.monotonic()
|
||||
resolved = 0
|
||||
if pending:
|
||||
for attempt in range(10):
|
||||
try:
|
||||
confirmed.append(c.message(ctx['actor'], ctx['chat'], pending[1], key=pending[0])['json'])
|
||||
pending = None
|
||||
resolved = 1
|
||||
break
|
||||
except Failure:
|
||||
time.sleep(.5)
|
||||
if pending:
|
||||
raise RuntimeError('Final uncertain write could not be resolved')
|
||||
return dict(latencies=latencies, confirmed=confirmed, errors=errors, instances=instances,
|
||||
failures=failures, bins=bins, operation_counts=operation_counts,
|
||||
finished=finished, resolved_outside_window=resolved)
|
||||
|
||||
action_thread = threading.Thread(target=perform_action) if action else None
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
jobs = [pool.submit(worker, i) for i in range(concurrency)]
|
||||
started = time.monotonic()
|
||||
deadline = started + duration
|
||||
if action_thread:
|
||||
action_thread.start()
|
||||
gate.set()
|
||||
results = [job.result() for job in jobs]
|
||||
elapsed = max(r['finished'] for r in results) - started
|
||||
if action_thread:
|
||||
action_thread.join(timeout=120)
|
||||
if action_thread.is_alive():
|
||||
raise RuntimeError('Infrastructure action did not finish')
|
||||
if any('error' in e for e in action_events):
|
||||
raise RuntimeError(str(action_events))
|
||||
values = [x for r in results for x in r['latencies']]
|
||||
errors, instances, bins, operation_counts = Counter(), Counter(), Counter(), Counter()
|
||||
for r in results:
|
||||
errors.update(r['errors']); instances.update(r['instances'])
|
||||
bins.update(r['bins']); operation_counts.update(r['operation_counts'])
|
||||
successes, failed = len(values), sum(errors.values())
|
||||
report = dict(duration_requested=duration, elapsed_seconds=round(elapsed, 3), concurrency=concurrency,
|
||||
profile='closed-loop: 60% POST text128, 20% first50 history, 20% profile',
|
||||
retry_writes=retry_writes, successful_requests=successes, failed_attempts=failed,
|
||||
uncertain_writes_resolved_after_window=sum(r['resolved_outside_window'] for r in results),
|
||||
successful_rps=round(successes/elapsed, 3), error_ratio=failed/max(1, successes+failed),
|
||||
successful_latency_ms={f'p{int(p*100)}': percentile(values, p) for p in [.5, .95, .99]},
|
||||
instances=dict(instances), errors=dict(errors), operations=dict(operation_counts),
|
||||
successful_per_second=dict(sorted(bins.items())),
|
||||
failure_seconds=[x for r in results for x in r['failures']],
|
||||
latency_samples_ms=[round(x*1000, 3) for x in values], action_events=action_events)
|
||||
if not successes:
|
||||
raise RuntimeError('No successful requests')
|
||||
return report, [m for r in results for m in r['confirmed']]
|
||||
|
||||
|
||||
def verify_confirmations(base, ctx, confirmations):
|
||||
from urllib.parse import quote
|
||||
c = Client(base)
|
||||
path = '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=100'
|
||||
stored = {}
|
||||
while True:
|
||||
page = c.call('GET', path, actor=ctx['reader'], schema='MessagePage')['json']
|
||||
for message in page['items']:
|
||||
if message['id'] in stored:
|
||||
raise AssertionError('Duplicate ID in history')
|
||||
stored[message['id']] = message
|
||||
if page['next_cursor'] is None:
|
||||
break
|
||||
path = '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'])
|
||||
if not all(stored.get(m['id']) == m for m in confirmations):
|
||||
raise AssertionError('Confirmed message missing after infrastructure action')
|
||||
# В fault-сценарии нет seed. Каждый новый ключ должен породить одну строку.
|
||||
expected = {m['id'] for m in confirmations}
|
||||
if set(stored) - expected:
|
||||
raise AssertionError('Retry created an extra message or left an unconfirmed result')
|
||||
return dict(confirmed_messages=len(expected), stored_messages=len(stored), status='PASS')
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
endpoint="${S3_ENDPOINT:-http://s3:9000}"
|
||||
bucket="messenger-attachments"
|
||||
export MC_CONFIG_DIR=/tmp/mc-config
|
||||
attempt=0
|
||||
until mc alias set local "$endpoint" "$(cat /run/secrets/s3_root_user)" \
|
||||
"$(cat /run/secrets/s3_root_password)" >/dev/null 2>&1; do
|
||||
attempt=$((attempt + 1))
|
||||
test "$attempt" -lt 60 || exit 1
|
||||
sleep 1
|
||||
done
|
||||
case "${1:-init}" in
|
||||
init)
|
||||
mc mb --ignore-existing "local/$bucket" >/dev/null
|
||||
mc anonymous set none "local/$bucket" >/dev/null
|
||||
mc admin policy create local messenger-api /policies/s3-api-policy.json >/dev/null
|
||||
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" \
|
||||
"$(cat /run/secrets/s3_secret_access_key)" >/dev/null
|
||||
mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null
|
||||
echo 'Private bucket and limited API credentials ready'
|
||||
;;
|
||||
deny-put)
|
||||
mc admin policy create local messenger-api /policies/s3-no-put-policy.json >/dev/null
|
||||
echo 'Synthetic S3 write failure enabled'
|
||||
;;
|
||||
allow-put)
|
||||
mc admin policy create local messenger-api /policies/s3-api-policy.json >/dev/null
|
||||
echo 'Normal private bucket policy restored'
|
||||
;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Трек 2: реальные отказы, безопасная сверка и стоимость обхода; плюс RAM upload."""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import statistics
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client
|
||||
from scripts.stack import command, containers, scale, save
|
||||
from scripts.benchmark import percentile
|
||||
|
||||
|
||||
def api_run(*arguments):
|
||||
running = [row for row in containers('api') if row['running']]
|
||||
if not running:
|
||||
raise RuntimeError('No running API container')
|
||||
return command(['docker', 'exec', running[0]['id'], 'python', *arguments])
|
||||
|
||||
|
||||
def memory_stats():
|
||||
output = []
|
||||
for row in containers('api'):
|
||||
if row['running']:
|
||||
source = 'import urllib.request; print(urllib.request.urlopen("http://localhost:8080/internal/memory",timeout=5).read().decode())'
|
||||
values = json.loads(command(['docker', 'exec', row['id'], 'python', '-c', source]))
|
||||
output.append(dict(container=row['id'], limits=dict(cpu=row['cpu_limit'], ram=row['memory_limit_bytes']), **values))
|
||||
return output
|
||||
|
||||
|
||||
def file_checks():
|
||||
result = json.loads(api_run('student-tests/storage_scenario.py', 'privacy'))
|
||||
save('private_bucket.json', result)
|
||||
interrupted = json.loads(api_run('student-tests/storage_scenario.py', 'interrupted'))
|
||||
save('upload_interrupted.json', interrupted)
|
||||
print('PASS: private bucket, limited credentials, closed console')
|
||||
|
||||
|
||||
def storage_checks():
|
||||
active = any(row['running'] for row in containers('reconciler'))
|
||||
command(['docker', 'compose', 'stop', 'reconciler'])
|
||||
try:
|
||||
command(['docker', 'compose', 'run', '--rm', '--no-deps', 's3-bootstrap', 'deny-put'])
|
||||
try:
|
||||
failure = json.loads(api_run('student-tests/storage_scenario.py', 's3-failure'))
|
||||
save('s3_write_failure.json', failure)
|
||||
finally:
|
||||
command(['docker', 'compose', 'run', '--rm', '--no-deps', 's3-bootstrap', 'allow-put'])
|
||||
result = json.loads(api_run('student-tests/storage_scenario.py', 'reconcile', '--failed-chat-id', failure['chat_id']))
|
||||
result['s3_write_failure'] = failure
|
||||
save('reconcile_faults.json', result)
|
||||
print('PASS: S3/DB faults, dry-run/apply/repeat, fresh/used/active protection')
|
||||
finally:
|
||||
if active:
|
||||
command(['docker', 'compose', 'start', 'reconciler'])
|
||||
|
||||
|
||||
def memory_experiment(base, repeats):
|
||||
results = []
|
||||
for dimension in [256, 1024]:
|
||||
rng = random.Random(41)
|
||||
image = Image.frombytes('RGB', (dimension, dimension), rng.randbytes(dimension*dimension*3))
|
||||
output = io.BytesIO(); image.save(output, format='PNG'); image.close()
|
||||
data = output.getvalue()
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
for repeat in range(repeats):
|
||||
print(f'Upload memory: {dimension}x{dimension}, repeat {repeat+1}/{repeats}', flush=True)
|
||||
scale(2, 1, '512m', 8) # Новые процессы: high-water mark относится к этому случаю.
|
||||
c = Client(base, timeout=60)
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
before = memory_stats()
|
||||
gate = threading.Barrier(11)
|
||||
def upload(_):
|
||||
local = Client(base, timeout=60)
|
||||
gate.wait()
|
||||
start = time.monotonic()
|
||||
response = local.call('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=noise.png',
|
||||
201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
return response, time.monotonic()-start
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
jobs = [pool.submit(upload, index) for index in range(10)]
|
||||
start = time.monotonic(); gate.wait()
|
||||
batch = [job.result() for job in jobs]
|
||||
elapsed = time.monotonic()-start
|
||||
after = memory_stats()
|
||||
instances = {response['headers']['X-Instance-Id'] for response, _ in batch}
|
||||
assert len(instances) == 2, 'Memory burst did not reach both replicas'
|
||||
for response, _ in batch:
|
||||
att = response['json']
|
||||
assert att['sha256'] == digest and att['size_bytes'] == len(data)
|
||||
content = c.call('GET', '/api/v1/attachments/'+att['id']+'/content', actor=bob)['bytes']
|
||||
assert content == data
|
||||
values = [latency for _, latency in batch]
|
||||
results.append(dict(dimension=dimension, size_bytes=len(data), concurrent_uploads=10,
|
||||
repetition=repeat+1, elapsed_seconds=round(elapsed, 3), successful_uploads=10,
|
||||
failed_attempts=0, burst_successful_rps=10/elapsed,
|
||||
successful_latency_ms={f'p{int(p*100)}': percentile(values, p) for p in [.5, .95, .99]},
|
||||
latency_samples_ms=[round(v*1000, 3) for v in values],
|
||||
before=before, after=after, observed_instances=len(instances),
|
||||
original_checksums_verified=True))
|
||||
save('upload_memory.json', dict(runs=results, measurement='process high-water RSS and cgroup peak if available'))
|
||||
return results
|
||||
|
||||
|
||||
def cost_experiment(sizes):
|
||||
active = any(row['running'] for row in containers('reconciler'))
|
||||
command(['docker', 'compose', 'stop', 'reconciler'])
|
||||
try:
|
||||
print('Reconciler cost: datasets '+', '.join(map(str, sizes))+', three scans each', flush=True)
|
||||
output = command(['docker', 'compose', 'run', '--rm', '--no-deps', '-e', 'BASE_URL=http://lb:8080',
|
||||
'reconciler', 'python', 'student-tests/storage_scenario.py', 'cost',
|
||||
'--sizes', *map(str, sizes)], timeout=900)
|
||||
result = json.loads(output)
|
||||
save('reconcile_cost.json', result)
|
||||
return result['scans']
|
||||
finally:
|
||||
if active:
|
||||
command(['docker', 'compose', 'start', 'reconciler'])
|
||||
|
||||
|
||||
def plots(cost, uploads):
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
sizes = sorted({row['dataset_added'] for row in cost})
|
||||
seconds = [statistics.median(row['elapsed_seconds'] for row in cost if row['dataset_added']==size) for size in sizes]
|
||||
fig, ax = plt.subplots(figsize=(6, 4))
|
||||
ax.plot(sizes, seconds, marker='o')
|
||||
ax.set(xlabel='Objects + metadata', ylabel='Median scan, seconds', title='Reconciler cost')
|
||||
ax.grid(alpha=.3); fig.tight_layout()
|
||||
fig.savefig(ROOT/'evidence/reconcile_cost.png', dpi=150); plt.close(fig)
|
||||
dimensions = sorted({row['dimension'] for row in uploads})
|
||||
peaks = [max(api['process_peak_rss_bytes'] for row in uploads if row['dimension']==d for api in row['after'])/1024**2 for d in dimensions]
|
||||
fig, ax = plt.subplots(figsize=(6, 4))
|
||||
ax.bar([f'{d}x{d}' for d in dimensions], peaks)
|
||||
ax.set(ylabel='Peak API process RSS, MiB', title='10 concurrent uploads')
|
||||
fig.tight_layout(); fig.savefig(ROOT/'evidence/upload_memory.png', dpi=150); plt.close(fig)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--mode', choices=['privacy', 'faults', 'memory', 'cost', 'all'], default='all')
|
||||
p.add_argument('--repeats', type=int, default=3)
|
||||
p.add_argument('--sizes', nargs='+', type=int, default=[100, 500, 1000])
|
||||
args = p.parse_args()
|
||||
if args.repeats < 1 or sorted(set(args.sizes)) != args.sizes or min(args.sizes) < 1:
|
||||
p.error('Positive repeats and increasing distinct dataset sizes required')
|
||||
base = os.getenv('BASE_URL', 'http://localhost:8080')
|
||||
save('experiment_environment.json', dict(docker=command(['docker', 'version', '--format', '{{.Server.Version}}']),
|
||||
compose=command(['docker', 'compose', 'version', '--short']),
|
||||
containers=containers(), client_cpus=os.cpu_count()))
|
||||
if args.mode in {'privacy', 'all'}:
|
||||
file_checks()
|
||||
if args.mode in {'faults', 'all'}:
|
||||
storage_checks()
|
||||
uploads = memory_experiment(base, args.repeats) if args.mode in {'memory', 'all'} else []
|
||||
cost = cost_experiment(args.sizes) if args.mode in {'cost', 'all'} else []
|
||||
if uploads and cost:
|
||||
plots(cost, uploads)
|
||||
groups = []
|
||||
for size in args.sizes:
|
||||
cases = [row for row in cost if row['dataset_added'] == size]
|
||||
groups.append(dict(objects=size, repetitions=len(cases),
|
||||
median_seconds=statistics.median(row['elapsed_seconds'] for row in cases),
|
||||
heads_per_scan=[row['counters']['head_requests'] for row in cases]))
|
||||
save('experiment_summary.json', dict(reconciler_cost=groups, concurrent_uploads=10,
|
||||
verified_originals=sum(row['successful_uploads'] for row in uploads),
|
||||
status='PASS'))
|
||||
print('Experiment completed; results saved in evidence/')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import urllib.request
|
||||
|
||||
|
||||
host = "127.0.0.1"
|
||||
port = os.getenv("APP_PORT", "8080")
|
||||
with urllib.request.urlopen(f"http://{host}:{port}/health/ready", timeout=2) as response:
|
||||
payload = json.load(response)
|
||||
if response.status != 200 or payload != {"status": "ok"}:
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Редкие снимки Docker CPU/RAM и ожиданий PostgreSQL; overhead отражён в методике."""
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from scripts.stack import command, containers
|
||||
|
||||
|
||||
class Monitor:
|
||||
def __init__(self):
|
||||
self.samples = []
|
||||
self.stop = threading.Event()
|
||||
self.thread = threading.Thread(target=self.loop)
|
||||
|
||||
def __enter__(self):
|
||||
self.started = time.monotonic()
|
||||
self.thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.stop.set()
|
||||
self.thread.join(timeout=120)
|
||||
|
||||
def loop(self):
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
ids = [r['id'] for r in containers() if r['running'] and r['service'] != 'migrate']
|
||||
raw = command(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids])
|
||||
stats = [json.loads(line) for line in raw.splitlines()]
|
||||
query = "SELECT coalesce(json_agg(x),'[]') FROM (SELECT state,wait_event_type," \
|
||||
"wait_event,count(*) FROM pg_stat_activity WHERE backend_type='client backend' " \
|
||||
"AND pid<>pg_backend_pid() GROUP BY state,wait_event_type,wait_event) x"
|
||||
waits = json.loads(command(['docker', 'compose', 'exec', '-T', 'db',
|
||||
'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query]))
|
||||
pools = []
|
||||
for row in containers('api'):
|
||||
if row['running']:
|
||||
output = command(['docker', 'exec', row['id'], 'python', '-c',
|
||||
'import urllib.request; print(urllib.request.urlopen("http://localhost:8080/internal/pool",timeout=3).read().decode())'])
|
||||
pools.append(dict(instance=row['hostname'], stats=json.loads(output)))
|
||||
self.samples.append(dict(seconds=round(time.monotonic()-self.started, 3),
|
||||
docker=stats, postgres=waits, pools=pools))
|
||||
except Exception as exc:
|
||||
self.samples.append(dict(seconds=round(time.monotonic()-self.started, 3), error=str(exc)))
|
||||
self.stop.wait(3)
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop reconciler lb api cache db s3
|
||||
docker compose up -d --wait db cache s3
|
||||
docker compose run --rm --no-deps migrate
|
||||
docker compose run --rm --no-deps s3-bootstrap
|
||||
docker compose up -d --no-deps --wait api
|
||||
docker compose up -d --no-deps lb
|
||||
python3 scripts/stack.py wait --count 2
|
||||
docker compose up -d --no-deps reconciler
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose start api
|
||||
python3 scripts/stack.py wait --count 2
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p .secrets
|
||||
chmod 700 .secrets
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
password_file = Path('.secrets/db_password')
|
||||
if not password_file.exists():
|
||||
password_file.write_text(secrets.token_hex(24))
|
||||
password = password_file.read_text().strip()
|
||||
Path('.secrets/database_url').write_text(f'postgresql://messenger:{password}@db:5432/messenger')
|
||||
for name in ['s3_root_user', 's3_root_password', 's3_access_key_id', 's3_secret_access_key']:
|
||||
file = Path('.secrets') / name
|
||||
if not file.exists():
|
||||
file.write_text(secrets.token_hex(16 if name.endswith(('user', 'id')) else 32))
|
||||
for file in Path('.secrets').iterdir():
|
||||
file.chmod(0o644 if file.name in {'database_url', 's3_access_key_id', 's3_secret_access_key'} else 0o600)
|
||||
if not Path('.env').exists():
|
||||
Path('.env').write_text('APP_PORT=8080\nSESSION_TTL_SECONDS=3600\nDB_POOL_MAX=8\nAPI_CPU=1.0\nAPI_MEMORY=512m\n')
|
||||
print('Локальные настройки подготовлены; секреты не выводятся.')
|
||||
PY
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Небольшие операции со стендом: состояние, ожидание, масштабирование и замена."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(args, env=None, timeout=120):
|
||||
result = subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout)
|
||||
if result.returncode:
|
||||
raise RuntimeError(result.stderr[-2000:])
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def containers(service=None):
|
||||
args = ['docker', 'compose', 'ps', '-a', '-q']
|
||||
if service:
|
||||
args.append(service)
|
||||
ids = command(args).split()
|
||||
if not ids:
|
||||
return []
|
||||
# Сохраняем только нужные поля, без environment с возможными секретами.
|
||||
rows = json.loads(command(['docker', 'inspect', *ids]))
|
||||
return [dict(id=r['Id'], name=r['Name'].lstrip('/'), hostname=r['Config']['Hostname'],
|
||||
service=r['Config']['Labels']['com.docker.compose.service'],
|
||||
running=r['State']['Running'], started_at=r['State']['StartedAt'],
|
||||
cpu_limit=r['HostConfig']['NanoCpus'] / 1e9,
|
||||
memory_limit_bytes=r['HostConfig']['Memory'],
|
||||
published_ports=r['NetworkSettings']['Ports']) for r in rows]
|
||||
|
||||
|
||||
def save(name, data):
|
||||
path = ROOT / 'evidence' / name
|
||||
path.parent.mkdir(exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
|
||||
|
||||
|
||||
def wait(count, base=None):
|
||||
base = base or os.getenv('BASE_URL', 'http://localhost:8080')
|
||||
deadline = time.monotonic() + 75
|
||||
while time.monotonic() < deadline:
|
||||
observed = set()
|
||||
for _ in range(max(12, count * 8)):
|
||||
try:
|
||||
with urllib.request.urlopen(base + '/health/ready', timeout=3) as response:
|
||||
observed.add(response.headers['X-Instance-Id'])
|
||||
except (OSError, ValueError):
|
||||
break
|
||||
if len(observed) == count and None not in observed:
|
||||
return sorted(observed)
|
||||
time.sleep(.5)
|
||||
raise RuntimeError(f'Не дождались {count} готовых API через LB')
|
||||
|
||||
|
||||
def stop(instance):
|
||||
hostname = instance.split(':')[0]
|
||||
found = [r for r in containers('api') if r['hostname'] == hostname and r['running']]
|
||||
if len(found) != 1:
|
||||
raise RuntimeError('Instance alias не соответствует одной работающей API-реплике')
|
||||
before = containers()
|
||||
command(['docker', 'stop', '--time', '3', found[0]['id']])
|
||||
result = dict(target=instance, stopped_container=found[0]['id'], before=before, after=containers())
|
||||
save('last_stop.json', result)
|
||||
return result
|
||||
|
||||
|
||||
def scale(count, cpu, memory, pool):
|
||||
env = dict(os.environ, API_CPU=str(cpu), API_MEMORY=memory, DB_POOL_MAX=str(pool))
|
||||
command(['docker', 'compose', 'up', '-d', '--no-deps', '--force-recreate',
|
||||
'--scale', f'api={count}', 'api'], env)
|
||||
instances = wait(count)
|
||||
rows = containers('api')
|
||||
if len(rows) != count or not all(r['running'] for r in rows):
|
||||
raise RuntimeError('Фактическое число контейнеров отличается от запрошенного')
|
||||
for row in rows:
|
||||
if abs(row['cpu_limit'] - cpu) > .001:
|
||||
raise RuntimeError('Docker не применил CPU limit')
|
||||
if row['memory_limit_bytes'] != int(memory.rstrip('m')) * 1024 * 1024:
|
||||
raise RuntimeError('Docker не применил RAM limit')
|
||||
return dict(instances=instances, containers=rows)
|
||||
|
||||
|
||||
def rolling():
|
||||
old = [r['id'] for r in containers('api') if r['running']]
|
||||
if len(old) != 2:
|
||||
raise RuntimeError('Rolling проверяется на двух репликах обычного стенда')
|
||||
before = containers()
|
||||
command(['docker', 'compose', 'build', 'api'])
|
||||
steps = []
|
||||
for container in old:
|
||||
# Не пересоздаём одновременно старые контейнеры при изменении image.
|
||||
command(['docker', 'compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=3', 'api'])
|
||||
wait(3)
|
||||
command(['docker', 'stop', '--time', '10', container])
|
||||
command(['docker', 'rm', container])
|
||||
wait(2)
|
||||
steps.append(containers('api'))
|
||||
save('rolling_containers.json', dict(before=before, steps=steps, after=containers()))
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('action', choices=['wait', 'snapshot', 'stop', 'rolling'])
|
||||
p.add_argument('--count', type=int, default=2)
|
||||
p.add_argument('--instance')
|
||||
p.add_argument('--output', default='containers.json')
|
||||
args = p.parse_args()
|
||||
if args.action == 'wait':
|
||||
print('API ready:', ', '.join(wait(args.count)))
|
||||
elif args.action == 'snapshot':
|
||||
save(args.output, containers())
|
||||
print('Container identities and limits saved')
|
||||
elif args.action == 'stop':
|
||||
if not args.instance:
|
||||
p.error('--instance required')
|
||||
stop(args.instance)
|
||||
print('Selected API stopped')
|
||||
else:
|
||||
rolling()
|
||||
print('API replaced one at a time')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose up -d --build
|
||||
python3 scripts/stack.py wait --count 2
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
python3 scripts/stack.py stop --instance "$TARGET_INSTANCE_ID"
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
restore() {
|
||||
docker compose up -d --no-deps --force-recreate --scale api=2 api
|
||||
python3 scripts/stack.py wait --count 2
|
||||
}
|
||||
trap restore EXIT
|
||||
SESSION_TTL_SECONDS=2 docker compose up -d --no-deps --force-recreate --scale api=2 api
|
||||
python3 scripts/stack.py wait --count 2
|
||||
python3 student-tests/checks.py --expiry
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Ожидание готовности API без сторонних библиотек."""
|
||||
import os
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
url = os.getenv('BASE_URL', 'http://localhost:8080') + '/health/ready'
|
||||
for attempt in range(90):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
if response.status == 200:
|
||||
print('API ready')
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise SystemExit('API не стал готов за 90 попыток')
|
||||
Reference in New Issue
Block a user