forked from mtusi2026/mtusi2026lab7
lab7 completed
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Initialize operator-controlled OpenBao configuration over verified HTTPS."""
|
||||
import argparse
|
||||
import getpass
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import ssl
|
||||
import sys
|
||||
from urllib.error import HTTPError
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from app.credentials import bao_request
|
||||
|
||||
os.environ.setdefault('BAO_ADDR', 'https://localhost:8200')
|
||||
CA = ROOT / 'secrets/ca/ca.crt'
|
||||
|
||||
|
||||
def unseal(context):
|
||||
while True:
|
||||
status = bao_request('sys/seal-status', context)
|
||||
if not status['initialized']:
|
||||
raise SystemExit('Initialize OpenBao before submitting unseal shares.')
|
||||
if not status['sealed']:
|
||||
print('OpenBao is unsealed')
|
||||
return
|
||||
key = getpass.getpass(f'OpenBao share ({status["progress"]}/{status["t"]}): ')
|
||||
bao_request('sys/unseal', context, {'key': key})
|
||||
|
||||
|
||||
def configure(context):
|
||||
token = getpass.getpass('OpenBao operator token: ')
|
||||
|
||||
def request(path, payload=None):
|
||||
return bao_request(path, context, payload, token)
|
||||
|
||||
mounts = request('sys/mounts')
|
||||
if 'secret/' not in (mounts.get('data') or mounts):
|
||||
request('sys/mounts/secret', {'type': 'kv', 'options': {'version': '2'}})
|
||||
methods = request('sys/auth')
|
||||
if 'cert/' not in (methods.get('data') or methods):
|
||||
request('sys/auth/cert', {'type': 'cert'})
|
||||
for role in ('api', 'worker'):
|
||||
request('sys/policies/acl/' + role,
|
||||
{'policy': (ROOT / f'infra/openbao/{role}.hcl').read_text()})
|
||||
request('auth/cert/certs/' + role, {
|
||||
'certificate': (ROOT / f'secrets/identity/{role}/client.crt').read_text(),
|
||||
'allowed_common_names': [role],
|
||||
'token_policies': [role],
|
||||
'token_no_default_policy': True,
|
||||
'token_ttl': '120s',
|
||||
'token_explicit_max_ttl': '300s',
|
||||
})
|
||||
request('secret/data/' + role,
|
||||
{'data': json.loads((ROOT / f'secrets/operator/{role}.json').read_text())})
|
||||
print('Configured two scoped certificate roles and KV paths')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=['unseal', 'configure'])
|
||||
args = parser.parse_args()
|
||||
context = ssl.create_default_context(cafile=str(CA))
|
||||
try:
|
||||
{'unseal': unseal, 'configure': configure}[args.action](context)
|
||||
except HTTPError as exc:
|
||||
raise SystemExit(f'OpenBao rejected the operation with HTTP {exc.code}') from None
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure local Argon2id hash and verify latencies without user data."""
|
||||
from importlib.metadata import version
|
||||
import platform
|
||||
from statistics import median, quantiles
|
||||
from time import perf_counter
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
|
||||
hasher = PasswordHasher(memory_cost=19456, time_cost=2, parallelism=1)
|
||||
password = 'synthetic-benchmark-password'
|
||||
encoded = hasher.hash(password)
|
||||
print(f'Python {platform.python_version()}; {platform.machine()}; argon2-cffi {version("argon2-cffi")}')
|
||||
print('Argon2id m=19456 KiB, t=2, p=1, salt=16 B; n=100 per operation')
|
||||
for name, operation in [('hash', lambda: hasher.hash(password)),
|
||||
('verify', lambda: hasher.verify(encoded, password))]:
|
||||
for _ in range(5):
|
||||
operation()
|
||||
samples = []
|
||||
for _ in range(100):
|
||||
start = perf_counter()
|
||||
operation()
|
||||
samples.append((perf_counter() - start) * 1000)
|
||||
print(f'{name}: p50={median(samples):.2f} ms, p95={quantiles(samples, n=100)[94]:.2f} ms')
|
||||
@@ -0,0 +1,255 @@
|
||||
"""Generate local credentials, certificates and runtime configurations once."""
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BUCKET = 'messenger-attachments'
|
||||
PASSWORDS = {key: secrets.token_hex(24) for key in (
|
||||
'pg-admin', 'pg-api', 'pg-worker', 's3-admin', 's3-api', 's3-worker',
|
||||
'producer', 'consumer', 'otel-api', 'otel-worker', 'loki-write', 'tempo-write',
|
||||
'grafana', 'loki-read', 'tempo-read', 'prom-read', 'scrape')}
|
||||
S3_IDS = {'admin': 'messenger-bootstrap', 'api': 'messenger-api', 'worker': 'messenger-worker'}
|
||||
|
||||
|
||||
def write(root, path, content):
|
||||
file = root / path
|
||||
file.parent.mkdir(parents=True, exist_ok=True)
|
||||
file.write_text(content)
|
||||
# The enclosing secrets directory is 0700 on the host. Exact read-only
|
||||
# mounts also work with the image-specific UIDs inside Docker Desktop.
|
||||
file.chmod(0o444)
|
||||
|
||||
|
||||
def openssl(*args):
|
||||
subprocess.run(['openssl', *map(str, args)], check=True, stdout=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def issue(root, ca, target, name, usage, names=()):
|
||||
folder = root / target
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
key, crt = folder / f'{usage}.key', folder / f'{usage}.crt'
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
csr, config = Path(temporary) / 'leaf.csr', Path(temporary) / 'ext'
|
||||
config.write_text(
|
||||
'basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\n'
|
||||
'subjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid:always\n'
|
||||
f'extendedKeyUsage={usage}Auth\n'
|
||||
+ ('subjectAltName=' + ','.join('DNS:' + dns for dns in names) + '\n' if names else ''))
|
||||
openssl('req', '-new', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256',
|
||||
'-nodes', '-subj', '/CN=' + name, '-keyout', key, '-out', csr)
|
||||
openssl('x509', '-req', '-in', csr, '-CA', root / f'operator/{ca}.crt',
|
||||
'-CAkey', root / f'operator/{ca}.key', '-set_serial', '0x' + secrets.token_hex(16),
|
||||
'-days', '30', '-sha256', '-extfile', config, '-out', crt)
|
||||
key.chmod(0o444)
|
||||
write(root, f'{target}/{usage}.pem', crt.read_text() + key.read_text())
|
||||
|
||||
|
||||
def pki(root):
|
||||
(root / 'operator').mkdir(mode=0o700)
|
||||
for name in ('ca', 'foreign'):
|
||||
openssl('req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256',
|
||||
'-nodes', '-subj', '/CN=MTUSI lab7 ' + name, '-days', '365',
|
||||
'-addext', 'basicConstraints=critical,CA:TRUE',
|
||||
'-addext', 'keyUsage=critical,keyCertSign,cRLSign',
|
||||
'-addext', 'subjectKeyIdentifier=hash',
|
||||
'-keyout', root / f'operator/{name}.key', '-out', root / f'operator/{name}.crt')
|
||||
write(root, 'ca/ca.crt', (root / 'operator/ca.crt').read_text())
|
||||
for name in ('lb', 'api1', 'api2', 'postgres', 's3', 'broker', 'otel-collector',
|
||||
'loki', 'tempo', 'prometheus', 'grafana', 'openbao'):
|
||||
names = (name, 'localhost') if name in ('lb', 'grafana', 'openbao') else (name,)
|
||||
issue(root, 'ca', 'tls/' + name, name, 'server', names)
|
||||
for name in ('lb', 'api', 'worker'):
|
||||
issue(root, 'ca', 'identity/' + name, name, 'client')
|
||||
issue(root, 'foreign', 'identity/foreign', 'lb', 'client')
|
||||
|
||||
|
||||
COMMON = '''global
|
||||
maxconn 256
|
||||
ssl-default-bind-options ssl-min-ver TLSv1.2
|
||||
ssl-default-server-options ssl-min-ver TLSv1.2
|
||||
defaults
|
||||
mode http
|
||||
timeout connect 2s
|
||||
timeout client 35s
|
||||
timeout server 35s
|
||||
'''
|
||||
|
||||
|
||||
def frontend(port, params=''):
|
||||
return f'frontend tls\n bind :{port} ssl crt /run/tls/server.pem {params}\n'
|
||||
|
||||
|
||||
def proxy(root, name, config):
|
||||
write(root, f'proxy/{name}.cfg', COMMON + config)
|
||||
|
||||
|
||||
def local_backend(port):
|
||||
return f' default_backend local\nbackend local\n server local 127.0.0.1:{port}\n'
|
||||
|
||||
|
||||
def telemetry_proxy(root, name, port, grants, rules):
|
||||
users = ''.join(f'userlist {identity}\n user {identity} insecure-password {PASSWORDS[key]}\n'
|
||||
for identity, key in grants.items())
|
||||
proxy(root, name, users + 'frontend tls\n' + ''.join(
|
||||
f' bind :{public} ssl crt /run/tls/server.pem\n' for public in rules)
|
||||
+ ''.join(f' acl port{public} dst_port {public}\n' + statements
|
||||
for public, statements in rules.items())
|
||||
+ ' http-request deny deny_status 403\n'
|
||||
+ ''.join(f'backend port{public}\n server local 127.0.0.1:{target}\n'
|
||||
for public, target in port.items()))
|
||||
|
||||
|
||||
def grant(public, identity, method, path, exact=False):
|
||||
match = 'path' if exact else 'path_beg'
|
||||
return (f' http-request allow if port{public} {{ http_auth({identity}) }} '
|
||||
f'{{ method {method} }} {{ {match} {path} }}\n'
|
||||
f' use_backend port{public} if port{public}\n')
|
||||
|
||||
|
||||
def proxies(root):
|
||||
proxy(root, 'lb', '''resolvers docker
|
||||
nameserver embedded 127.0.0.11:53
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 5s
|
||||
''' + frontend(8443) + ''' http-request set-header X-Lab-Client-IP %[src]
|
||||
default_backend api
|
||||
backend api
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server inter 1s fall 2 rise 1 resolvers docker init-addr libc,none ssl verify required ca-file /run/ca/ca.crt crt /run/identity/client.pem
|
||||
server api1 api1:9443 check verifyhost api1 sni str(api1) check-sni api1
|
||||
server api2 api2:9443 check verifyhost api2 sni str(api2) check-sni api2
|
||||
''')
|
||||
for name in ('api1', 'api2'):
|
||||
proxy(root, name, frontend(9443, 'verify required ca-file /run/ca/ca.crt') +
|
||||
' http-request deny deny_status 403 unless { ssl_c_s_dn(CN) -m str lb }\n' +
|
||||
local_backend(8080))
|
||||
proxy(root, 'broker', frontend(5671) + ''' mode tcp
|
||||
timeout client 150s
|
||||
default_backend amqp
|
||||
backend amqp
|
||||
mode tcp
|
||||
timeout server 150s
|
||||
server local 127.0.0.1:5672
|
||||
''')
|
||||
proxy(root, 's3', frontend(9443) + f''' acl one_auth req.fhdr_cnt(authorization) eq 1
|
||||
acl admin req.fhdr(authorization) -m beg "AWS4-HMAC-SHA256 Credential={S3_IDS['admin']}/"
|
||||
acl api req.fhdr(authorization) -m beg "AWS4-HMAC-SHA256 Credential={S3_IDS['api']}/"
|
||||
acl worker req.fhdr(authorization) -m beg "AWS4-HMAC-SHA256 Credential={S3_IDS['worker']}/"
|
||||
acl bucket path /{BUCKET} /{BUCKET}/
|
||||
acl staging path_reg ^/{BUCKET}/staging/[0-9a-f-]+$
|
||||
acl final path_reg ^/{BUCKET}/final/[0-9a-f-]+/(original|thumbnail)$
|
||||
acl query_present query -m found
|
||||
http-request deny unless one_auth !query_present
|
||||
http-request allow if admin bucket {{ method HEAD PUT }}
|
||||
http-request allow if api bucket {{ method HEAD }}
|
||||
http-request allow if api staging {{ method PUT }}
|
||||
http-request allow if api final {{ method GET }}
|
||||
http-request allow if worker staging {{ method GET }}
|
||||
http-request allow if worker final {{ method PUT }}
|
||||
http-request deny
|
||||
''' + local_backend(9000))
|
||||
telemetry_proxy(root, 'otel-collector', {9443: 4318, 9444: 8889},
|
||||
{'api': 'otel-api', 'worker': 'otel-worker', 'scraper': 'scrape'},
|
||||
{9443: ''' http-request allow if port9443 { http_auth(api) } { method POST } { path /v1/logs /v1/traces /v1/metrics }
|
||||
http-request allow if port9443 { http_auth(worker) } { method POST } { path /v1/logs /v1/traces /v1/metrics }
|
||||
use_backend port9443 if port9443
|
||||
''', 9444: grant(9444, 'scraper', 'GET', '/metrics', exact=True)})
|
||||
for name, internal, write_path, read_prefix in (
|
||||
('loki', 3100, '/otlp/v1/logs', '/loki/api/v1/'),
|
||||
('tempo', 3200, '/v1/traces', '/api/')):
|
||||
ingest_port = 4318 if name == 'tempo' else internal
|
||||
telemetry_proxy(root, name, {9443: internal, 9444: ingest_port},
|
||||
{'grafana': name + '-read', 'collector': name + '-write'},
|
||||
{9443: grant(9443, 'grafana', 'GET POST', read_prefix),
|
||||
9444: grant(9444, 'collector', 'POST', write_path, exact=True)})
|
||||
telemetry_proxy(root, 'prometheus', {9443: 9090}, {'grafana': 'prom-read'},
|
||||
{9443: grant(9443, 'grafana', 'GET POST', '/api/v1/')})
|
||||
|
||||
|
||||
def settings(root):
|
||||
for name in ('pg-admin', 'pg-api', 'pg-worker', 'grafana', 'scrape'):
|
||||
write(root, 'values/' + name, PASSWORDS[name] + '\n')
|
||||
identities = []
|
||||
for role, access in S3_IDS.items():
|
||||
identities.append({'name': role, 'credentials': [
|
||||
{'accessKey': access, 'secretKey': PASSWORDS['s3-' + role]}],
|
||||
'actions': ['Admin'] if role == 'admin' else ['Read:' + BUCKET, 'Write:' + BUCKET, 'List:' + BUCKET]})
|
||||
write(root, 's3/identities.json', json.dumps({'identities': identities}))
|
||||
write(root, 's3/admin.env', f'S3_ACCESS_KEY_ID={S3_IDS["admin"]}\nS3_SECRET_ACCESS_KEY={PASSWORDS["s3-admin"]}\n')
|
||||
for role in ('api', 'worker'):
|
||||
database = f'postgresql://{role}:{PASSWORDS["pg-" + role]}@postgres:5432/messenger'
|
||||
data = {'DATABASE_URL': database + '?sslmode=verify-full&sslrootcert=/run/ca/ca.crt',
|
||||
'S3_ACCESS_KEY_ID': S3_IDS[role], 'S3_SECRET_ACCESS_KEY': PASSWORDS['s3-' + role],
|
||||
'OTEL_PASSWORD': PASSWORDS['otel-' + role]}
|
||||
if role == 'worker':
|
||||
for identity in ('producer', 'consumer'):
|
||||
data[f'BROKER_{identity.upper()}_URL'] = (
|
||||
f'amqps://{identity}:{quote(PASSWORDS[identity])}@broker:5671/messenger')
|
||||
write(root, f'operator/{role}.json', json.dumps(data))
|
||||
|
||||
|
||||
def broker(root):
|
||||
users = []
|
||||
for name in ('producer', 'consumer'):
|
||||
salt = secrets.token_bytes(4)
|
||||
digest = hashlib.sha256(salt + PASSWORDS[name].encode()).digest()
|
||||
users.append({'name': name, 'password_hash': base64.b64encode(salt + digest).decode(),
|
||||
'hashing_algorithm': 'rabbit_password_hashing_sha256', 'tags': []})
|
||||
manifest = {'users': users, 'vhosts': [{'name': 'messenger'}], 'permissions': [
|
||||
{'user': 'producer', 'vhost': 'messenger', 'configure': '^$', 'write': '^amq\\.default$', 'read': '^$'},
|
||||
{'user': 'consumer', 'vhost': 'messenger', 'configure': '^$', 'write': '^$', 'read': '^attachments$'}],
|
||||
'queues': [{'name': 'attachments', 'vhost': 'messenger', 'durable': True,
|
||||
'auto_delete': False, 'arguments': {}}]}
|
||||
write(root, 'broker/definitions.json', json.dumps(manifest))
|
||||
|
||||
|
||||
def telemetry(root):
|
||||
config = (ROOT / 'infra/otel/collector.yaml').read_text()
|
||||
for name in ('loki', 'tempo'):
|
||||
header = base64.b64encode(f'collector:{PASSWORDS[name + "-write"]}'.encode()).decode()
|
||||
config = config.replace(f'@{name.upper()}_AUTH@', header)
|
||||
write(root, 'telemetry/collector.yaml', config)
|
||||
config = (ROOT / 'infra/grafana/datasources.yaml').read_text()
|
||||
for name, key in (('loki', 'loki-read'), ('tempo', 'tempo-read'), ('prom', 'prom-read')):
|
||||
config = config.replace(f'@{name.upper()}_PASSWORD@', PASSWORDS[key])
|
||||
config = config.replace('@TLS_CA@', '\n'.join(
|
||||
' ' + line for line in (root / 'ca/ca.crt').read_text().splitlines()))
|
||||
write(root, 'telemetry/datasources.yaml', config)
|
||||
|
||||
|
||||
def main():
|
||||
os.umask(0o077)
|
||||
destination = ROOT / 'secrets'
|
||||
if destination.exists():
|
||||
if not (destination / '.ready').exists():
|
||||
raise SystemExit('Existing secrets/ has no bootstrap marker. Inspect it before retrying.')
|
||||
print('Using existing local runtime material')
|
||||
return
|
||||
temporary = Path(tempfile.mkdtemp(prefix='lab7-secrets-', dir=ROOT))
|
||||
try:
|
||||
pki(temporary)
|
||||
settings(temporary)
|
||||
broker(temporary)
|
||||
telemetry(temporary)
|
||||
proxies(temporary)
|
||||
write(temporary, '.ready', 'lab7\n')
|
||||
temporary.rename(destination)
|
||||
finally:
|
||||
if temporary.exists():
|
||||
shutil.rmtree(temporary)
|
||||
print('Created local runtime material in secrets/')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Exercise both API replica failures against the same session, data and files."""
|
||||
from pathlib import Path
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
REPLICAS = {'api1', 'api2'}
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, timeout=40)
|
||||
|
||||
|
||||
def wait(client, expected, timeout):
|
||||
observed = set()
|
||||
consecutive = 0
|
||||
required = 5 if len(expected) == 1 else 1
|
||||
start = time.monotonic()
|
||||
while time.monotonic() - start < timeout:
|
||||
try:
|
||||
response = client.call('GET', '/health/ready', schema='Health')
|
||||
replica = response['headers']['X-Instance-Id']
|
||||
except Failure:
|
||||
replica = None
|
||||
if replica in expected:
|
||||
observed.add(replica)
|
||||
consecutive += 1
|
||||
if observed == expected and consecutive >= required:
|
||||
return time.monotonic() - start
|
||||
else:
|
||||
consecutive = 0
|
||||
time.sleep(0.25)
|
||||
raise Failure('expected replicas did not become ready')
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
client = Client(args.base_url, args.ca_file or os.environ.get('CA_FILE', str(ROOT / 'secrets/ca/ca.crt')),
|
||||
timeout=2)
|
||||
wait(client, REPLICAS, 45)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
first = client.message(alice, chat, 'Проверка сохранности')['json']
|
||||
attachment, content = client.attachment(alice, chat)
|
||||
client.wait_ready(alice, attachment)
|
||||
for stopped in sorted(REPLICAS):
|
||||
other = (REPLICAS - {stopped}).pop()
|
||||
compose('stop', '-t', '2', stopped)
|
||||
try:
|
||||
elapsed = wait(client, {other}, 10)
|
||||
for _ in range(5):
|
||||
response = client.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
||||
check(response['headers']['X-Instance-Id'] == other, 'unexpected replica while one is stopped')
|
||||
message = client.message(alice, chat, 'Запись через оставшуюся реплику')['json']
|
||||
history = client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=alice,
|
||||
schema='MessagePage')['json']['items']
|
||||
check({first['id'], message['id']} <= {item['id'] for item in history}, 'message missing')
|
||||
data = client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content',
|
||||
actor=alice)['bytes']
|
||||
check(data == content, 'original file changed')
|
||||
print(f'PASS: {stopped} stopped, {other} serves existing session, messages and file. '
|
||||
f'Readiness after stop returned in {elapsed:.2f}s')
|
||||
finally:
|
||||
compose('start', stopped)
|
||||
wait(client, REPLICAS, 45)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Run additional HTTPS ACL, expiry, size and login rate checks."""
|
||||
import base64
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def expire(actor):
|
||||
digest = hashlib.sha256(actor['token'].encode()).hexdigest()
|
||||
sql = f"UPDATE sessions SET expires_at=now()-interval '1 second' WHERE token_hash='{digest}';\n"
|
||||
result = subprocess.run(
|
||||
['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-v', 'ON_ERROR_STOP=1',
|
||||
'-U', 'db_operator', '-d', 'messenger'],
|
||||
cwd=Path(__file__).resolve().parents[1], input=sql, text=True, capture_output=True)
|
||||
check(result.returncode == 0 and 'UPDATE 1' in result.stdout, 'could not expire synthetic session')
|
||||
|
||||
|
||||
def hostname(ca):
|
||||
context = ssl.create_default_context(cafile=ca)
|
||||
try:
|
||||
with urllib.request.urlopen('https://127.0.0.1:8443/health/live', context=context, timeout=5):
|
||||
pass
|
||||
except urllib.error.URLError as exc:
|
||||
check(isinstance(exc.reason, ssl.SSLCertVerificationError), 'wrong hostname failed for a non-TLS reason')
|
||||
check(exc.reason.verify_code in (62, 64), 'certificate error was not a SAN/hostname mismatch')
|
||||
else:
|
||||
check(False, 'unexpected success using IP outside certificate SAN')
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
ca = args.ca_file or os.environ.get('CA_FILE', 'secrets/ca/ca.crt')
|
||||
client = Client(args.base_url, ca)
|
||||
client.call('GET', '/health/ready', schema='Health')
|
||||
hostname(ca)
|
||||
alice, _, eve, chat = client.bootstrap()
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
attachment = client.wait_ready(alice, attachment)
|
||||
path = '/api/v1/attachments/' + attachment['id']
|
||||
spoofed = {'X-User-Id': alice['id']}
|
||||
client.call('GET', path, 404, actor=eve, headers=spoofed)
|
||||
client.call('GET', path + '/content', 404, actor=eve, headers=spoofed)
|
||||
own_chat = client.call('POST', '/api/v1/chats', 201, actor=eve,
|
||||
body={'title': 'ACL probe', 'member_ids': [eve['id']]},
|
||||
schema='Chat')['json']
|
||||
client.call('POST', '/api/v1/chats/' + own_chat['id'] + '/messages', 404, actor=eve,
|
||||
body={'text': '', 'attachment_ids': [attachment['id']]},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
client.call('POST', '/api/v1/users', 413, raw=b' ' * 65537,
|
||||
headers={'Content-Type': 'application/json'})
|
||||
upload = '/api/v1/chats/' + chat['id'] + '/attachments?filename=oversize.png'
|
||||
client.call('POST', upload, 413, actor=alice, raw=b'x' * (10 * 1024 * 1024 + 1),
|
||||
headers={'Content-Type': 'image/png'})
|
||||
client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=eve)
|
||||
client.call('GET', '/api/v1/users/me', 401, actor=eve)
|
||||
expire(alice)
|
||||
client.call('GET', '/api/v1/users/me', 401, actor=alice)
|
||||
authorization = 'Basic ' + base64.b64encode(b'unknown:synthetic-password').decode()
|
||||
for _ in range(31):
|
||||
response = client.call('POST', '/api/v1/auth/sessions', (401, 429),
|
||||
headers={'Authorization': authorization})
|
||||
if response['status'] == 429:
|
||||
wait = int(response['headers'].get('Retry-After', '0'))
|
||||
check(1 <= wait <= 60, 'invalid Retry-After')
|
||||
break
|
||||
else:
|
||||
check(False, 'login limiter never returned 429')
|
||||
print('PASS: hostname, attachment ACL/spoof, 413, logout, expired session, 429 and Retry-After')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Operator probes run on the Docker backend network; never print credentials."""
|
||||
from contextlib import contextmanager
|
||||
import http.client
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import ssl
|
||||
from urllib.error import HTTPError
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
import psycopg
|
||||
|
||||
from app.common import bucket, s3
|
||||
from app.credentials import bao_request, login
|
||||
|
||||
|
||||
def expect(condition, description):
|
||||
if not condition:
|
||||
raise SystemExit('FAIL: ' + description)
|
||||
|
||||
|
||||
def context(identity=None):
|
||||
tls = ssl.create_default_context(cafile=os.environ['LAB_CA'])
|
||||
tls.maximum_version = ssl.TLSVersion.TLSv1_2
|
||||
if identity:
|
||||
path = Path('/run/probe') / identity
|
||||
tls.load_cert_chain(path / 'client.crt', path / 'client.key')
|
||||
return tls
|
||||
|
||||
|
||||
def http_status(host, port, path, tls, method='GET', body=None):
|
||||
connection = http.client.HTTPSConnection(host, port, context=tls, timeout=5)
|
||||
try:
|
||||
connection.request(method, path, body)
|
||||
response = connection.getresponse()
|
||||
response.read()
|
||||
return response.status
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def mtls():
|
||||
for host in ('api1', 'api2'):
|
||||
expect(http_status(host, 9443, '/health/ready', context('lb')) == 200, host + ' valid LB')
|
||||
for identity in (None, 'foreign'):
|
||||
try:
|
||||
http_status(host, 9443, '/health/live', context(identity))
|
||||
except ssl.SSLCertVerificationError:
|
||||
raise SystemExit('FAIL: server identity could not be verified') from None
|
||||
except ssl.SSLError:
|
||||
continue
|
||||
raise SystemExit('FAIL: missing or foreign client cert reached HTTP')
|
||||
expect(http_status(host, 9443, '/health/live', context('api')) == 403,
|
||||
host + ' wrong client identity')
|
||||
try:
|
||||
with socket.create_connection((host, 8080), timeout=3):
|
||||
raise SystemExit('FAIL: direct API plaintext port is reachable')
|
||||
except ConnectionRefusedError:
|
||||
pass
|
||||
print('PASS:', host, 'LB accepted, missing/foreign cert and wrong role denied, port 8080 closed')
|
||||
|
||||
|
||||
@contextmanager
|
||||
def denied(description, expected=(403,)):
|
||||
try:
|
||||
yield
|
||||
except HTTPError as error:
|
||||
expect(error.code in expected, description + ': unexpected status')
|
||||
else:
|
||||
raise SystemExit('FAIL: ' + description)
|
||||
|
||||
|
||||
def secrets_access():
|
||||
for role, foreign in [('api', 'worker'), ('worker', 'api')]:
|
||||
with denied('wrong OpenBao certificate role', (400, 403)):
|
||||
login(context(foreign), role)
|
||||
tls = context(role)
|
||||
token = login(tls, role)
|
||||
result = bao_request('secret/data/' + role, tls, token=token)['data']['data']
|
||||
with denied('OpenBao foreign path'):
|
||||
bao_request('secret/data/' + foreign, tls, token=token)
|
||||
with denied('OpenBao policy write'):
|
||||
bao_request('sys/policies/acl/probe', tls, {'policy': ''}, token)
|
||||
with psycopg.connect(result['DATABASE_URL']) as database:
|
||||
encrypted = database.execute(
|
||||
'SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()').fetchone()[0]
|
||||
expect(encrypted, 'DB session encrypted')
|
||||
if role == 'worker':
|
||||
try:
|
||||
database.execute('SELECT token_hash FROM sessions LIMIT 1')
|
||||
except psycopg.errors.InsufficientPrivilege:
|
||||
database.rollback()
|
||||
else:
|
||||
raise SystemExit('FAIL: worker read API sessions')
|
||||
expected_prefix = 'final/' if role == 'api' else 'staging/'
|
||||
try:
|
||||
client = s3_with_role(result)
|
||||
client.delete_object(Bucket=bucket(), Key=expected_prefix + '00000000-0000-0000-0000-000000000000')
|
||||
except ClientError as exc:
|
||||
expect(exc.response['ResponseMetadata']['HTTPStatusCode'] == 403, 'S3 delete rejected')
|
||||
else:
|
||||
raise SystemExit('FAIL: service can delete S3 object')
|
||||
print('PASS: scoped OpenBao read for', role, '| foreign path/policy write denied | DB TLS | S3 delete denied')
|
||||
expect(http_status('otel-collector', 9443, '/v1/traces', context(), 'POST', b'') == 403,
|
||||
'unauthenticated telemetry ingest')
|
||||
print('PASS: unauthenticated telemetry denied')
|
||||
|
||||
|
||||
def s3_with_role(settings):
|
||||
for variable in ('S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY'):
|
||||
os.environ[variable] = settings[variable]
|
||||
return s3()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
mtls()
|
||||
secrets_access()
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop -t 2 api1 api2 worker
|
||||
docker compose restart -t 5 postgres seaweed rabbitmq prometheus-db loki-db tempo-db grafana
|
||||
# Unless the operator explicitly requests a seal/unseal exercise, OpenBao stays
|
||||
# available for services to authenticate again during this business-data test.
|
||||
if [ "${RESTART_OPENBAO:-0}" = 1 ]; then
|
||||
docker compose restart -t 5 openbao
|
||||
attempts=0
|
||||
until curl --silent --cacert secrets/ca/ca.crt https://localhost:8200/v1/sys/seal-status >/dev/null; do
|
||||
attempts=$((attempts+1))
|
||||
[ "$attempts" -lt 15 ] || exit 1
|
||||
sleep 1
|
||||
done
|
||||
python3 scripts/bao-operator.py unseal </dev/tty >/dev/tty 2>/dev/tty
|
||||
fi
|
||||
docker compose start api1 api2 worker
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
python3 scripts/bootstrap.py
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
case "${TARGET_INSTANCE_ID:-}" in
|
||||
api1|api2) docker compose stop "$TARGET_INSTANCE_ID" ;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user