Files
mtusi2026lab7/student-tests/security_ops.py
T

555 lines
34 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Отрицательные и эксплуатационные проверки lab7 (trust matrix, OpenBao, hardening).
tls-external внешний TLS: доверенный CA + имя — да; пустой/чужой trust store, чужое имя,
TLS 1.1 — ошибка проверки; plaintext-порта нет
mtls LB→API, worker/relay→RabbitMQ, сервисы→Collector: без сертификата, чужой CA,
доверенный CA с чужой identity — отказ; правильная identity — успех.
Каждая связь хранилищ: plaintext не отвечает, TLS с проверкой — да
acl посторонний, поддельный X-User-Id, logout, истечение сессии, лимит входа
(429 + Retry-After, общий для реплик), предел тела 413, Argon2id в БД, rehash
openbao policy allow/deny каждой сервисной identity; нет root/статического токена
в приложениях; не dev-mode; restart → sealed → unseal
hardening docker inspect всех контейнеров: user, read_only, capabilities,
no-new-privileges, privileged, host network, docker.sock, порты, лимиты
rotation ротация secret (пароль БД api, ключ S3 worker) и leaf (api, lb) под нагрузкой
без пересборки; старый пароль/сертификат отвергается
lifecycle короткий TTL токена OpenBao (renew/re-login), sealed OpenBao под нагрузкой,
рестарт worker при sealed и восстановление после unseal
argon2 замер Argon2id на контейнере API, бюджет login DoS
leaks пароль/токен/текст сообщения не попадают в Loki/Tempo/stdout (через Grafana)
Печатает JSON-итог и PASS/FAIL; секреты не печатаются.
"""
import base64
import json
import os
import socket
import ssl
import subprocess
import threading
import time
import urllib.request
import uuid
from pathlib import Path
import labutil # noqa: F401
from client import Client, Failure, arguments, check, run
from labutil import ROOT, compose, psql, sh, wait_status
from loadgen import Load
CA = str(ROOT / 'secrets' / 'pki' / 'ca.crt')
SAMPLE = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes()
PY = os.environ.get('PYTHON', 'python3')
def probe(*args, stdin=None, timeout=120):
"""secprobe в тестовом контейнере (сети front/back/telemetry, /pki — сертификаты)."""
r = compose('--profile', 'test', 'run', '--rm', '-T', 'probe', *args, input=stdin, check=False, timeout=timeout,
env={'PROBE_UID': str(os.getuid()) if hasattr(os, 'getuid') else '1001'})
try:
return json.loads(r.stdout)
except ValueError:
return {'error': (r.stderr or r.stdout)[-300:]}
def bootstrap(*args):
return sh(PY, 'scripts/bootstrap.py', *args, timeout=600).stdout.strip()
def denied(v):
return isinstance(v, str) and (v.startswith('denied') or v.startswith('server_rejected') or v.startswith('client_rejected'))
# ---------------- внешний TLS ----------------
def scenario_tls_external(c):
res = {}
out = sh(PY, 'tests/security.py', '--base-url', c.base, '--ca-file', CA, check=False).stdout.strip()
res['public_security_py'] = out.splitlines()[-1] if out else ''
check(out.endswith('NOT verify internal mTLS, OpenBao or hardening.') or 'PASS' in out, 'public security.py failed: ' + out)
def handshake(ctx, host='localhost', port=8443):
try:
with socket.create_connection(('127.0.0.1', port), timeout=5) as s:
with ctx.wrap_socket(s, server_hostname=host) as t:
return 'ok ' + t.version()
except ssl.SSLCertVerificationError as e:
return 'cert_verify_error: ' + e.verify_message
except ssl.SSLError as e:
return 'ssl_error: ' + e.reason
except OSError as e:
return 'network: ' + type(e).__name__
good = ssl.create_default_context(cafile=CA)
res['trusted_ca_localhost'] = handshake(good)
res['wrong_hostname'] = handshake(good, host='messenger.example')
rogue = ssl.create_default_context(cafile=str(ROOT / 'secrets/pki/test/rogue-ca.crt'))
res['foreign_ca_trust_store'] = handshake(rogue)
old = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
old.load_verify_locations(CA)
try:
old.minimum_version = old.maximum_version = ssl.TLSVersion.TLSv1_1
res['tls_1_1'] = handshake(old)
except (ValueError, ssl.SSLError) as e:
res['tls_1_1'] = 'client_cannot_offer: ' + str(e)[:60]
for port in (8080, 8404, 9090, 5432, 6379, 9000, 5672, 4318):
try:
socket.create_connection(('127.0.0.1', port), timeout=2).close()
res['host_port_%d' % port] = 'OPEN'
except OSError:
res['host_port_%d' % port] = 'closed'
check(res['trusted_ca_localhost'].startswith('ok TLSv1.'), 'trusted handshake failed')
check(res['wrong_hostname'].startswith('cert_verify_error'), 'hostname not verified')
check(res['foreign_ca_trust_store'].startswith('cert_verify_error'), 'foreign CA accepted')
check(not res['tls_1_1'].startswith('ok'), 'TLS 1.1 accepted')
check(all(v == 'closed' for k, v in res.items() if k.startswith('host_port')), 'unexpected host ports open')
return res
# ---------------- mTLS и TLS всех связей ----------------
def scenario_mtls(c):
p = '/pki/'
t = '/pki/test/'
lb = {
'no_client_cert': probe('tls', '-addr', 'api:8443', '-name', 'api'),
'foreign_ca_cert_cn_lb': probe('tls', '-addr', 'api:8443', '-name', 'api', '-cert', t + 'rogue-lb.crt', '-key', t + 'rogue-lb.key'),
'trusted_ca_identity_attacker': probe('tls', '-addr', 'api:8443', '-name', 'api', '-cert', t + 'attacker.crt', '-key', t + 'attacker.key'),
'trusted_ca_identity_worker': probe('tls', '-addr', 'api:8443', '-name', 'api', '-cert', p + 'worker.crt', '-key', p + 'worker.key'),
'identity_lb': probe('tls', '-addr', 'api:8443', '-name', 'api', '-cert', p + 'lb.crt', '-key', p + 'lb.key'),
'plaintext_http': probe('plain', '-addr', 'api:8443', '-send', 'GET /health/ready HTTP/1.1\\r\\nHost: api\\r\\n\\r\\n'),
}
for k in ('no_client_cert', 'foreign_ca_cert_cn_lb', 'trusted_ca_identity_attacker', 'trusted_ca_identity_worker'):
check(denied(lb[k].get('result')), 'LB→API %s not rejected: %s' % (k, lb[k]))
check(lb['identity_lb'].get('status') == 200, 'LB identity rejected: %s' % lb['identity_lb'])
broker = {
'foreign_ca': probe('amqp', '-cert', t + 'rogue-lb.crt', '-key', t + 'rogue-lb.key'),
'trusted_ca_cn_attacker': probe('amqp', '-cert', t + 'attacker.crt', '-key', t + 'attacker.key'),
'trusted_ca_cn_lb': probe('amqp', '-cert', p + 'lb.crt', '-key', p + 'lb.key'),
'worker': probe('amqp', '-cert', p + 'worker.crt', '-key', p + 'worker.key'),
'relay': probe('amqp', '-cert', p + 'relay.crt', '-key', p + 'relay.key'),
'plaintext_5672': probe('plain', '-addr', 'rabbitmq:5672', '-send', 'AMQP'),
}
for k in ('foreign_ca', 'trusted_ca_cn_attacker', 'trusted_ca_cn_lb'):
check(denied(broker[k].get('connect')), 'broker %s not rejected: %s' % (k, broker[k]))
w, r = broker['worker'], broker['relay']
check(w.get('consume attachment.process') == 'allowed' and denied(w.get('publish attachments (job)'))
and denied(w.get('declare queue evil')) and denied(w.get('delete queue attachment.process')), 'worker ACL: %s' % w)
check(r.get('publish attachments (job)') == 'allowed' and denied(r.get('consume attachment.process')), 'relay ACL: %s' % r)
otel = {
'no_client_cert': probe('otlp'),
'foreign_ca': probe('otlp', '-cert', t + 'rogue-otel.crt', '-key', t + 'rogue-otel.key'),
'main_ca_cert_lb': probe('otlp', '-cert', p + 'lb.crt', '-key', p + 'lb.key'),
'main_ca_cert_worker': probe('otlp', '-cert', p + 'worker.crt', '-key', p + 'worker.key'),
'telemetry_ca_worker': probe('otlp', '-cert', p + 'worker-otel.crt', '-key', p + 'worker-otel.key'),
'plaintext_http': probe('plain', '-addr', 'otel-collector:4318', '-send', 'POST /v1/traces HTTP/1.1\\r\\nHost: x\\r\\nContent-Length: 0\\r\\n\\r\\n'),
}
for k in ('no_client_cert', 'foreign_ca', 'main_ca_cert_lb', 'main_ca_cert_worker'):
check(otel[k].get('ingest') != 'allowed', 'collector %s accepted: %s' % (k, otel[k]))
check(otel['telemetry_ca_worker'].get('status') == 200, 'telemetry client rejected: %s' % otel['telemetry_ca_worker'])
# Хранилища и служебные связи: TLS с проверкой работает; plaintext не обслуживается.
servers = {}
for name, addr, sn, cert in (
('postgres(starttls)', None, None, None),
('redis', 'redis:6379', 'redis', None), ('s3', 's3:9000', 's3', None),
('openbao', 'openbao:8200', 'openbao', None),
('tempo', 'tempo:3200', 'tempo', 'prometheus'), ('loki', 'loki:3100', 'loki', 'prometheus'),
('prometheus', 'prometheus:9090', 'prometheus', 'grafana'),
('collector-metrics', 'otel-collector:8889', 'otel-collector', 'prometheus'),
('rabbitmq-metrics', 'rabbitmq:15691', 'rabbitmq', 'prometheus')):
if addr is None:
continue
a = ['tls', '-addr', addr, '-name', sn, '-path', '/']
if name == 'redis':
a += ['-send', 'PING\\r\\n'] # ответ -NOAUTH: TLS установлен, ACL требует вход
if cert:
a += ['-cert', p + cert + '.crt', '-key', p + cert + '.key']
r_tls = probe(*a)
r_plain = probe('plain', '-addr', addr, '-send', 'GET / HTTP/1.1\\r\\nHost: x\\r\\n\\r\\n' if name != 'redis' else 'PING\\r\\n')
entry = {'tls_verified': r_tls.get('result'), 'tls_status': r_tls.get('status'),
'plaintext_reply': r_plain.get('reply_prefix'), 'plaintext': r_plain.get('plaintext')}
if cert:
r_nocert = probe('tls', '-addr', addr, '-name', sn, '-path', '/')
entry['without_client_cert'] = r_nocert.get('result')
servers[name] = entry
rp = r_plain.get('reply_prefix') or ''
check(r_tls.get('result') == 'ok' or 'status' in r_tls, '%s TLS failed: %s' % (name, r_tls))
check('+PONG' not in rp and '200 OK' not in rp, '%s answers plaintext: %s' % (name, rp))
if cert:
check(entry['without_client_cert'] != 'ok', '%s accepts clients without certificate' % name)
# PostgreSQL: без TLS отказ pg_hba; с verify-full — подключение
pg_plain = probe('db', '-user', 'app_api', '-sslmode', 'disable', '-stdin', stdin='x')
servers['postgres'] = {'sslmode_disable': pg_plain.get('connect')}
check(denied(pg_plain.get('connect')), 'postgres accepted non-TLS connection')
return {'lb_to_api': lb, 'broker': broker, 'collector': otel, 'servers': servers}
# ---------------- ACL / сессии / вход ----------------
def scenario_acl(c):
alice, bob, eve, chat = c.bootstrap()
att = c.call('POST', '/api/v1/chats/%s/attachments?filename=a.png' % chat['id'], 202, actor=alice, raw=SAMPLE,
headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
wait_status(c, bob, att['id'], ('ready',))
res = {}
res['outsider_metadata'] = c.call('GET', '/api/v1/attachments/' + att['id'], (404,), actor=eve)['status']
res['outsider_content'] = c.call('GET', '/api/v1/attachments/%s/content' % att['id'], (404,), actor=eve)['status']
echat = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'e', 'member_ids': [eve['id']]},
schema='Chat')['json']
res['attach_foreign_file'] = c.call('POST', '/api/v1/chats/%s/messages' % echat['id'], (404,), actor=eve,
body={'text': 'x', 'attachment_ids': [att['id']]},
headers={'Idempotency-Key': uuid.uuid4().hex})['status']
res['fake_x_user_id_with_eve_token'] = c.call('GET', '/api/v1/chats/%s/messages' % chat['id'], (404,), actor=eve,
headers={'X-User-Id': alice['id']})['status']
res['x_user_id_without_token'] = c.call('GET', '/api/v1/users/me', (401,), headers={'X-User-Id': alice['id']})['status']
tok = dict(bob)
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=tok)
res['after_logout'] = c.call('GET', '/api/v1/users/me', (401,), actor=tok)['status']
# Предел тела JSON: 65 KiB → 413 (без чтения сверх лимита)
big = json.dumps({'text': 'x' * (65 * 1024)}).encode()
res['json_body_65KiB'] = c.call('POST', '/api/v1/chats/%s/messages' % chat['id'], (413,), actor=alice, raw=big,
headers={'Content-Type': 'application/json', 'Idempotency-Key': uuid.uuid4().hex})['status']
# Argon2id в БД: параметры текущего hash
h = psql("SELECT password_hash FROM users WHERE id = '%s'" % alice['id'])
res['stored_hash_params'] = h.split('$')[1] + ' ' + h.split('$')[3]
check(h.startswith('$argon2id$v=19$m=19456,t=2,p=1$'), 'unexpected hash format')
# Rehash: hash старых параметров (lab2–6: m=64 MiB, t=1, p=4) → вход → пересчитан текущими
pw = alice['registration']['password']
legacy = compose('exec', '-T', 'api', '/app/pipeline', 'argon2-hash', '-m', '65536', '-t', '1', '-p', '4', input=pw).stdout.strip()
psql("UPDATE users SET password_hash = '%s' WHERE id = '%s'" % (legacy, alice['id']))
cred = base64.b64encode(('%s:%s' % (alice['user']['username'], pw)).encode()).decode()
c.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': 'Basic ' + cred})
h2 = psql("SELECT password_hash FROM users WHERE id = '%s'" % alice['id'])
res['rehash'] = {'before': legacy.split('$')[3], 'after': h2.split('$')[3]}
check(h2.split('$')[3] == 'm=19456,t=2,p=1', 'legacy hash not rehashed')
# Лимит входа: общий для реплик счётчик в Redis; превышение → 429 + Retry-After
bad = base64.b64encode(('%s:%s' % (eve['user']['username'], 'wrong-password-xyz')).encode()).decode()
codes, retry, instances = [], None, set()
for _ in range(14):
r = c.call('POST', '/api/v1/auth/sessions', (401, 429), headers={'Authorization': 'Basic ' + bad})
codes.append(r['status'])
instances.add(r['headers'].get('X-Instance-Id'))
if r['status'] == 429:
retry = r['headers'].get('Retry-After')
res['login_rate_limit'] = {'codes': codes, 'retry_after': retry, 'served_by_replicas': len(instances)}
check(codes[:10] == [401] * 10 and codes[10] == 429 and retry, 'per-user login limit not enforced: %s' % codes)
good_eve = base64.b64encode(('%s:%s' % (eve['user']['username'], eve['registration']['password'])).encode()).decode()
res['correct_password_while_limited'] = c.call('POST', '/api/v1/auth/sessions', (429,),
headers={'Authorization': 'Basic ' + good_eve})['status']
# Истечение сессии: реплики с SESSION_TTL_SECONDS=3
compose('up', '-d', '--no-deps', '--wait', 'api', env={'SESSION_TTL_SECONDS': '3'}, timeout=240)
time.sleep(3)
short = c.user('ttl')
time.sleep(5)
res['after_ttl_expiry'] = c.call('GET', '/api/v1/users/me', (401,), actor=short)['status']
compose('up', '-d', '--no-deps', '--wait', 'api', timeout=240)
return res
# ---------------- OpenBao ----------------
def scenario_openbao(c):
p = '/pki/'
res = {}
for svc, cert in (('api', 'api'), ('worker', 'worker'), ('relay', 'relay'), ('reconciler', 'reconciler')):
r = probe('bao', '-cert', p + cert + '.crt', '-key', p + cert + '.key', '-role', svc, '-path', 'messenger/' + svc)
res[svc] = r
check(r.get('read own messenger/' + svc) == 'allowed', '%s cannot read own secret: %s' % (svc, r))
others = [k for k in r if k.startswith('read messenger/')]
check(others and all(r[k].startswith('denied') for k in others), '%s reads foreign secrets: %s' % (svc, r))
for k in ('write own secret', 'list secrets', 'write policy', 'create token with other policy', 'seal openbao'):
check(r[k].startswith('denied'), '%s allowed %s' % (svc, k))
# identity не своей роли: сертификат worker на роль api, attacker на любую
res['worker_cert_as_api_role'] = probe('bao', '-cert', p + 'worker.crt', '-key', p + 'worker.key', '-role', 'api', '-path', 'messenger/api')
res['attacker_cert'] = probe('bao', '-cert', '/pki/test/attacker.crt', '-key', '/pki/test/attacker.key', '-role', 'api', '-path', 'messenger/api')
check(res['worker_cert_as_api_role'].get('login', '').startswith('denied'), 'worker cert logged in as api')
check(res['attacker_cert'].get('login', '').startswith('denied'), 'attacker cert logged in')
# В приложениях нет токенов/ключей OpenBao и root credentials
leaks = {}
for svc in ('api', 'worker', 'relay', 'reconciler'):
cid = compose('ps', '-q', svc).stdout.split()[0]
env = sh('docker', 'inspect', '-f', '{{json .Config.Env}}', cid).stdout
mounts = sh('docker', 'exec', cid, 'ls', '/run/secrets').stdout.split()
leaks[svc] = {'env_has_token': any(x in env for x in ('BAO_TOKEN', 'VAULT_TOKEN', 'ROOT_TOKEN', 'UNSEAL')),
'secret_files': sorted(mounts)}
check(not leaks[svc]['env_has_token'], 'token in env of ' + svc)
check(all(f in ('ca.crt', 'service.crt', 'service.key', 'otel.crt', 'otel.key', 'revoked.txt') for f in mounts),
'unexpected secret files in %s: %s' % (svc, mounts))
res['app_containers'] = leaks
st = json.loads(compose('exec', '-T', '-e', 'BAO_ADDR=https://127.0.0.1:8200', '-e', 'BAO_CACERT=/run/secrets/ca.crt',
'openbao', 'bao', 'status', '-format=json', check=False).stdout)
res['status'] = {k: st.get(k) for k in ('initialized', 'sealed', 'storage_type', 't', 'n', 'version')}
check(st.get('storage_type') == 'file' and st.get('t') == 2 and st.get('n') == 3, 'openbao not file storage / shamir 2-of-3')
# Рестарт: OpenBao запечатан, оператор распечатывает
compose('restart', 'openbao')
time.sleep(3)
st2 = json.loads(compose('exec', '-T', '-e', 'BAO_ADDR=https://127.0.0.1:8200', '-e', 'BAO_CACERT=/run/secrets/ca.crt',
'openbao', 'bao', 'status', '-format=json', check=False).stdout or '{}')
res['after_restart_sealed'] = st2.get('sealed')
bootstrap('unseal')
res['after_unseal'] = probe('bao', '-cert', p + 'api.crt', '-key', p + 'api.key', '-role', 'api', '-path', 'messenger/api').get('read own messenger/api')
check(res['after_restart_sealed'] is True and res['after_unseal'] == 'allowed', 'restart/unseal flow failed')
return res
# ---------------- hardening ----------------
def scenario_hardening(c):
out = {}
for line in compose('ps', '-a', '--format', '{{.Service}} {{.ID}}').stdout.splitlines():
svc, cid = line.split()
info = json.loads(sh('docker', 'inspect', cid).stdout)[0]
hc, cfg = info['HostConfig'], info['Config']
out[svc] = {
'user': cfg.get('User') or 'root(image default)',
'read_only': hc.get('ReadonlyRootfs'), 'privileged': hc.get('Privileged'),
'cap_drop': hc.get('CapDrop'), 'cap_add': hc.get('CapAdd'),
'no_new_privileges': any('no-new-privileges' in s for s in (hc.get('SecurityOpt') or [])),
'network_mode': hc.get('NetworkMode'), 'pid_mode': hc.get('PidMode'),
'docker_sock': any('docker.sock' in (m.get('Source') or '') for m in info.get('Mounts', [])),
'memory_limit_mib': (hc.get('Memory') or 0) // 2**20, 'nano_cpus': hc.get('NanoCpus'),
'pids_limit': hc.get('PidsLimit'),
'published': sorted('%s:%s->%s' % (b['HostIp'], b['HostPort'], p) for p, bs in (hc.get('PortBindings') or {}).items() for b in bs),
}
app = ('api', 'worker', 'relay', 'reconciler', 'lb')
for s, v in out.items():
check(not v['privileged'] and v['network_mode'] != 'host' and not v['docker_sock'], 'unsafe container ' + s)
check(v['no_new_privileges'], 'no-new-privileges missing: ' + s)
for pb in v['published']:
check(pb.startswith('127.0.0.1:'), 'port published on all interfaces: %s %s' % (s, pb))
for s in app:
if s in out:
v = out[s]
check(v['read_only'] and v['cap_drop'] == ['ALL'] and not v['cap_add'], 'app hardening: %s %s' % (s, v))
check(v['memory_limit_mib'] > 0, 'no memory limit: ' + s)
for s in ('api', 'worker', 'relay', 'reconciler'):
cid = compose('ps', '-q', s).stdout.split()[0]
out[s]['process_uid'] = sh('docker', 'exec', cid, 'id', '-u').stdout.strip()
check(out[s]['process_uid'] != '0', s + ' runs as root')
# Образ приложения: нет секретов, ключей, исходников и инструмента проверки прав
files = sh('docker', 'run', '--rm', '--entrypoint', 'sh', 'mtusi2026lab7-api:local', '-c',
'find / -xdev \\( -name "*.key" -o -name "*.pem" -o -name ".env" -o -name "secprobe" -o -name "*.go" \\) 2>/dev/null '
'| grep -v -e ^/proc -e ^/usr/share/ca-certificates -e ^/etc/ssl -e ^/usr/local/go || true', check=False).stdout.split()
out['_image_sensitive_files'] = files
check(not files, 'sensitive files in app image: %s' % files)
return out
# ---------------- ротация ----------------
def scenario_rotation(c):
alice, bob, _, chat = c.bootstrap()
load = Load(c.base, CA, alice, chat, concurrency=3, timeout=10, prefix='rot').start()
res = {}
t0 = time.time()
# 1) пароль роли БД api: KV новая версия; приложения переходят без рестарта
old_pw = json.loads(compose('exec', '-T', '-e', 'BAO_ADDR=https://127.0.0.1:8200', '-e', 'BAO_CACERT=/run/secrets/ca.crt',
'-e', 'BAO_TOKEN=' + json.loads((ROOT / 'secrets/operator/bao-init.json').read_text())['root_token'],
'openbao', 'bao', 'kv', 'get', '-format=json', 'secret/messenger/api').stdout)['data']['data']['db_password']
bootstrap('rotate-db', 'api')
time.sleep(35) # SECRETS_REFRESH 30 s
# Старые соединения закрываются (как при отзыве); новые открываются с новым паролем
psql("SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE usename = 'app_api'")
time.sleep(35) # pgx health check (30 s) выбрасывает разорванные соединения
res['db_old_password'] = probe('db', '-user', 'app_api', '-stdin', stdin=old_pw).get('connect')
check(denied(res['db_old_password']), 'old db password still works')
tries = []
for _ in range(12):
try:
tries.append(c.call('GET', '/api/v1/chats', (200, 503), actor=alice)['status'])
except Failure as e:
tries.append(str(e)[-40:])
if tries[-1] == 200:
break
time.sleep(5)
res['api_after_db_rotation_attempts'] = tries
if tries[-1] != 200:
res['api_secret_events'] = [l[-220:] for l in compose('logs', '--no-color', '--since', '5m', 'api').stdout.splitlines()
if 'secrets.' in l or 'postgres' in l.lower()][-15:]
check(tries[-1] == 200, 'API did not recover after DB password rotation: %s' % res)
# 2) ключ S3 worker: загрузка после ротации обрабатывается
bootstrap('rotate-s3', 'worker')
time.sleep(35)
att = c.call('POST', '/api/v1/chats/%s/attachments?filename=r.png' % chat['id'], 202, actor=alice, raw=SAMPLE,
headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
res['upload_after_s3_rotation'] = wait_status(c, bob, att['id'], ('ready', 'failed'), timeout=150)['status']
check(res['upload_after_s3_rotation'] == 'ready', 'worker did not pick rotated S3 key')
# 3) leaf API: перевыпуск на месте, без рестарта; LB видит новый serial
before = probe('tls', '-addr', 'api:8443', '-name', 'api', '-cert', '/pki/lb.crt', '-key', '/pki/lb.key').get('server_serial')
sh('sh', 'scripts/pki.sh', 'rotate', 'api')
time.sleep(4)
after = probe('tls', '-addr', 'api:8443', '-name', 'api', '-cert', '/pki/lb.crt', '-key', '/pki/lb.key').get('server_serial')
res['api_leaf_serial'] = {'before': before, 'after': after}
check(before and after and before != after, 'api leaf not rotated live')
# 4) клиентский leaf LB: новый сертификат + reload HAProxy; старый отозван
sh('cp', 'secrets/pki/lb.crt', 'secrets/pki/test/lb-old.crt')
sh('cp', 'secrets/pki/lb.key', 'secrets/pki/test/lb-old.key')
sh('sh', 'scripts/pki.sh', 'rotate', 'lb')
compose('kill', '-s', 'USR2', 'lb') # master-worker reload, без разрыва
time.sleep(4)
res['old_lb_cert_after_revocation'] = probe('tls', '-addr', 'api:8443', '-name', 'api', '-cert', '/pki/test/lb-old.crt',
'-key', '/pki/test/lb-old.key').get('result')
res['new_lb_cert'] = probe('tls', '-addr', 'api:8443', '-name', 'api', '-cert', '/pki/lb.crt', '-key', '/pki/lb.key').get('status')
check(denied(res['old_lb_cert_after_revocation']) and res['new_lb_cert'] == 200, 'lb rotation/revocation failed')
time.sleep(5)
load.stop()
res['load_during_rotations'] = load.summary()
res['seconds'] = round(time.time() - t0)
res['images_rebuilt'] = False
check(res['load_during_rotations']['ok'] > 0, 'no successful requests during rotation')
return res
# ---------------- жизненный цикл токенов, sealed ----------------
def scenario_lifecycle(c):
alice, bob, _, chat = c.bootstrap()
res = {}
# 1) короткий TTL токена api: 20 s, максимум 45 s → renew-self, затем повторный login
bootstrap('cert-role', 'api', 'api', '20s', '45s')
t0 = time.time()
compose('restart', 'api')
compose('up', '-d', '--no-deps', '--wait', 'api', timeout=180)
time.sleep(100)
logs = compose('logs', '--no-color', '--since', '%ds' % int(time.time() - t0 + 5), 'api').stdout
res['api_openbao_logins_in_100s'] = logs.count('"event":"secrets.login"')
res['api_refresh_failures'] = logs.count('"event":"secrets.refresh_failed"')
res['api_requests_ok'] = c.call('GET', '/api/v1/chats', actor=alice)['status']
bootstrap('cert-role', 'api', 'api') # вернуть 15m / 1h
check(res['api_openbao_logins_in_100s'] >= 4, 'no re-login after token max TTL')
# 2) OpenBao запечатан: работающие процессы продолжают на кэше; новый worker стартовать не может
load = Load(c.base, CA, alice, chat, concurrency=2, timeout=10, prefix='sealed').start()
bootstrap('seal')
t_seal = time.time()
att = c.call('POST', '/api/v1/chats/%s/attachments?filename=s.png' % chat['id'], 202, actor=alice, raw=SAMPLE,
headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
res['upload_while_sealed'] = wait_status(c, bob, att['id'], ('ready', 'failed'), timeout=120)['status']
compose('restart', 'worker')
time.sleep(25)
wlogs = compose('logs', '--no-color', '--since', '30s', 'worker').stdout
res['worker_restart_while_sealed'] = 'openbao not ready' in wlogs or 'sealed' in wlogs
att2 = c.call('POST', '/api/v1/chats/%s/attachments?filename=s2.png' % chat['id'], 202, actor=alice, raw=SAMPLE,
headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
time.sleep(5)
res['upload_accepted_while_worker_blocked'] = att2['status']
bootstrap('unseal')
t_unseal = time.time()
meta = wait_status(c, bob, att2['id'], ('ready', 'failed'), timeout=180)
res['ready_seconds_after_unseal'] = round(time.time() - t_unseal, 1)
res['final_status'] = meta['status']
load.stop()
res['sealed_seconds'] = round(t_unseal - t_seal)
res['load_while_sealed'] = load.summary()
check(res['upload_while_sealed'] == 'ready', 'running worker stopped working while sealed')
check(res['worker_restart_while_sealed'], 'restarted worker did not wait for openbao')
check(meta['status'] == 'ready', 'accepted job lost while openbao sealed')
check(res['load_while_sealed']['errors'] == 0, 'API errors while openbao sealed')
return res
# ---------------- Argon2id ----------------
def scenario_argon2(c):
bench = json.loads(compose('exec', '-T', 'api', '/app/pipeline', 'argon2-bench', '-n', '10').stdout)
alice, bob, eve, chat = c.bootstrap()
# Последовательные входы (валидный пароль) — латентность входа целиком
cred = base64.b64encode(('%s:%s' % (bob['user']['username'], bob['registration']['password'])).encode()).decode()
lat = []
for _ in range(8):
t = time.time()
c.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': 'Basic ' + cred})
lat.append(round((time.time() - t) * 1000, 1))
# Всплеск входов от одного IP с разными username: первые 60 за окно проходят до Argon2id
# (401), остальные — 429 без вычисления hash; память API ограничена слотами Argon2id.
api = compose('ps', '-q', 'api').stdout.split()
import obsutil as o
mem0 = [o.mem_bytes(x, 'memory.current') for x in api]
codes = {}
lock = threading.Lock()
def attempt(i):
b = base64.b64encode(('dos_%d_%s:wrong-password-1234' % (i, uuid.uuid4().hex[:6])).encode()).decode()
try:
r = c.call('POST', '/api/v1/auth/sessions', (401, 429, 503), headers={'Authorization': 'Basic ' + b})
k = r['status']
except Failure:
k = 'error'
with lock:
codes[k] = codes.get(k, 0) + 1
t0 = time.time()
ths = [threading.Thread(target=attempt, args=(i,)) for i in range(120)]
for t in ths:
t.start()
for t in ths:
t.join()
burst_s = round(time.time() - t0, 2)
peaks = [round((o.mem_bytes(x, 'memory.peak') or 0) / 2**20, 1) for x in api]
cur = bench['results'][0]
return {'bench': bench, 'login_latency_ms_sequential': lat,
'dos_burst': {'attempts': 120, 'seconds': burst_s, 'codes': codes, 'api_mem_peak_mib': peaks,
'api_mem_before_mib': [round((m or 0) / 2**20, 1) for m in mem0]},
'budget': {'hash_ms_p50': cur['ms_p50'], 'argon2_slots_per_replica': 2, 'replicas': len(api),
'max_hashes_per_s_stand': round(2 * len(api) * 1000 / cur['ms_p50'], 1),
'argon2_memory_bound_mib': round(2 * len(api) * cur['m_kib'] / 1024, 1),
'per_ip_limit_per_min': 60, 'per_user_limit_per_min': 10}}
# ---------------- утечки в телеметрии ----------------
def scenario_leaks(c):
import obsutil as o
o.wait_grafana()
alice, bob, _, chat = c.bootstrap()
t0 = time.time() - 5
text = 'секрет-' + uuid.uuid4().hex
c.call('POST', '/api/v1/chats/%s/messages' % chat['id'], 201, actor=alice, body={'text': text},
headers={'Idempotency-Key': uuid.uuid4().hex, 'X-Request-Id': 'leak-' + uuid.uuid4().hex[:12]})
bad = base64.b64encode(('%s:%s' % (alice['user']['username'], 'Wrong-' + uuid.uuid4().hex)).encode()).decode()
c.call('POST', '/api/v1/auth/sessions', (401,), headers={'Authorization': 'Basic ' + bad})
time.sleep(20)
needles = {'password': alice['registration']['password'], 'token': alice['token'], 'message_text': text,
'basic_header': bad}
res = {}
for k, v in needles.items():
res['loki:' + k] = len(o.loki('{service_name=~"messenger-.+"} |= `%s`' % v, t0, limit=5))
traces = o.tempo_search('{resource.service.name="messenger-api"}', t0, limit=20)
blob = ''.join(json.dumps(o.tempo_trace(t['traceID'])[1]) for t in traces)
for k, v in needles.items():
res['tempo:' + k] = blob.count(v)
logs = compose('logs', '--no-color', '--since', '5m', 'api', 'worker', 'relay', 'lb').stdout
for k, v in needles.items():
res['stdout:' + k] = logs.count(v)
res['traces_scanned'] = len(traces)
check(len(traces) > 0, 'no traces to scan')
check(sum(v for k, v in res.items() if k != 'traces_scanned') == 0, 'leak: %s' % res)
return res
SCENARIOS = {'tls-external': scenario_tls_external, 'mtls': scenario_mtls, 'acl': scenario_acl,
'openbao': scenario_openbao, 'hardening': scenario_hardening, 'rotation': scenario_rotation,
'lifecycle': scenario_lifecycle, 'argon2': scenario_argon2, 'leaks': scenario_leaks}
def main():
def configure(p):
p.add_argument('scenario', choices=sorted(SCENARIOS))
p.add_argument('--output', default=None)
args = arguments('Lab7 security scenarios', configure)
c = Client(args.base_url or 'https://localhost:8443', args.ca_file or CA, timeout=30)
try:
result = SCENARIOS[args.scenario](c)
finally:
compose('up', '-d', '--no-deps', '--wait', 'api', 'worker', 'relay', 'lb', timeout=300, check=False)
text = json.dumps({'scenario': args.scenario, 'result': result}, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + '\n', encoding='utf-8')
print(text)
print('PASS: ' + args.scenario)
if __name__ == '__main__':
run(main)