"""Operator probes run on the Docker backend network; never print credentials.""" from contextlib import contextmanager import http.client import os from pathlib import Path import socket import ssl from urllib.error import HTTPError from botocore.exceptions import ClientError import psycopg from app.common import bucket, s3 from app.credentials import bao_request, login def expect(condition, description): if not condition: raise SystemExit('FAIL: ' + description) def context(identity=None): tls = ssl.create_default_context(cafile=os.environ['LAB_CA']) tls.maximum_version = ssl.TLSVersion.TLSv1_2 if identity: path = Path('/run/probe') / identity tls.load_cert_chain(path / 'client.crt', path / 'client.key') return tls def http_status(host, port, path, tls, method='GET', body=None): connection = http.client.HTTPSConnection(host, port, context=tls, timeout=5) try: connection.request(method, path, body) response = connection.getresponse() response.read() return response.status finally: connection.close() def mtls(): for host in ('api1', 'api2'): expect(http_status(host, 9443, '/health/ready', context('lb')) == 200, host + ' valid LB') for identity in (None, 'foreign'): try: http_status(host, 9443, '/health/live', context(identity)) except ssl.SSLCertVerificationError: raise SystemExit('FAIL: server identity could not be verified') from None except ssl.SSLError: continue raise SystemExit('FAIL: missing or foreign client cert reached HTTP') expect(http_status(host, 9443, '/health/live', context('api')) == 403, host + ' wrong client identity') try: with socket.create_connection((host, 8080), timeout=3): raise SystemExit('FAIL: direct API plaintext port is reachable') except ConnectionRefusedError: pass print('PASS:', host, 'LB accepted, missing/foreign cert and wrong role denied, port 8080 closed') @contextmanager def denied(description, expected=(403,)): try: yield except HTTPError as error: expect(error.code in expected, description + ': unexpected status') else: raise SystemExit('FAIL: ' + description) def secrets_access(): for role, foreign in [('api', 'worker'), ('worker', 'api')]: with denied('wrong OpenBao certificate role', (400, 403)): login(context(foreign), role) tls = context(role) token = login(tls, role) result = bao_request('secret/data/' + role, tls, token=token)['data']['data'] with denied('OpenBao foreign path'): bao_request('secret/data/' + foreign, tls, token=token) with denied('OpenBao policy write'): bao_request('sys/policies/acl/probe', tls, {'policy': ''}, token) with psycopg.connect(result['DATABASE_URL']) as database: encrypted = database.execute( 'SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()').fetchone()[0] expect(encrypted, 'DB session encrypted') if role == 'worker': try: database.execute('SELECT token_hash FROM sessions LIMIT 1') except psycopg.errors.InsufficientPrivilege: database.rollback() else: raise SystemExit('FAIL: worker read API sessions') expected_prefix = 'final/' if role == 'api' else 'staging/' try: client = s3_with_role(result) client.delete_object(Bucket=bucket(), Key=expected_prefix + '00000000-0000-0000-0000-000000000000') except ClientError as exc: expect(exc.response['ResponseMetadata']['HTTPStatusCode'] == 403, 'S3 delete rejected') else: raise SystemExit('FAIL: service can delete S3 object') print('PASS: scoped OpenBao read for', role, '| foreign path/policy write denied | DB TLS | S3 delete denied') expect(http_status('otel-collector', 9443, '/v1/traces', context(), 'POST', b'') == 403, 'unauthenticated telemetry ingest') print('PASS: unauthenticated telemetry denied') def s3_with_role(settings): for variable in ('S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY'): os.environ[variable] = settings[variable] return s3() if __name__ == '__main__': mtls() secrets_access()