This commit is contained in:
@@ -0,0 +1,117 @@
|
||||
"""Operator probes run on the Docker backend network; never print credentials."""
|
||||
from contextlib import contextmanager
|
||||
import http.client
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import ssl
|
||||
from urllib.error import HTTPError
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
import psycopg
|
||||
|
||||
from app.common import bucket, s3
|
||||
from app.credentials import bao_request, login
|
||||
|
||||
|
||||
def expect(condition, description):
|
||||
if not condition:
|
||||
raise SystemExit('FAIL: ' + description)
|
||||
|
||||
|
||||
def context(identity=None):
|
||||
tls = ssl.create_default_context(cafile=os.environ['LAB_CA'])
|
||||
tls.maximum_version = ssl.TLSVersion.TLSv1_2
|
||||
if identity:
|
||||
path = Path('/run/probe') / identity
|
||||
tls.load_cert_chain(path / 'client.crt', path / 'client.key')
|
||||
return tls
|
||||
|
||||
|
||||
def http_status(host, port, path, tls, method='GET', body=None):
|
||||
connection = http.client.HTTPSConnection(host, port, context=tls, timeout=5)
|
||||
try:
|
||||
connection.request(method, path, body)
|
||||
response = connection.getresponse()
|
||||
response.read()
|
||||
return response.status
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def mtls():
|
||||
for host in ('api1', 'api2'):
|
||||
expect(http_status(host, 9443, '/health/ready', context('lb')) == 200, host + ' valid LB')
|
||||
for identity in (None, 'foreign'):
|
||||
try:
|
||||
http_status(host, 9443, '/health/live', context(identity))
|
||||
except ssl.SSLCertVerificationError:
|
||||
raise SystemExit('FAIL: server identity could not be verified') from None
|
||||
except ssl.SSLError:
|
||||
continue
|
||||
raise SystemExit('FAIL: missing or foreign client cert reached HTTP')
|
||||
expect(http_status(host, 9443, '/health/live', context('api')) == 403,
|
||||
host + ' wrong client identity')
|
||||
try:
|
||||
with socket.create_connection((host, 8080), timeout=3):
|
||||
raise SystemExit('FAIL: direct API plaintext port is reachable')
|
||||
except ConnectionRefusedError:
|
||||
pass
|
||||
print('PASS:', host, 'LB accepted, missing/foreign cert and wrong role denied, port 8080 closed')
|
||||
|
||||
|
||||
@contextmanager
|
||||
def denied(description, expected=(403,)):
|
||||
try:
|
||||
yield
|
||||
except HTTPError as error:
|
||||
expect(error.code in expected, description + ': unexpected status')
|
||||
else:
|
||||
raise SystemExit('FAIL: ' + description)
|
||||
|
||||
|
||||
def secrets_access():
|
||||
for role, foreign in [('api', 'worker'), ('worker', 'api')]:
|
||||
with denied('wrong OpenBao certificate role', (400, 403)):
|
||||
login(context(foreign), role)
|
||||
tls = context(role)
|
||||
token = login(tls, role)
|
||||
result = bao_request('secret/data/' + role, tls, token=token)['data']['data']
|
||||
with denied('OpenBao foreign path'):
|
||||
bao_request('secret/data/' + foreign, tls, token=token)
|
||||
with denied('OpenBao policy write'):
|
||||
bao_request('sys/policies/acl/probe', tls, {'policy': ''}, token)
|
||||
with psycopg.connect(result['DATABASE_URL']) as database:
|
||||
encrypted = database.execute(
|
||||
'SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()').fetchone()[0]
|
||||
expect(encrypted, 'DB session encrypted')
|
||||
if role == 'worker':
|
||||
try:
|
||||
database.execute('SELECT token_hash FROM sessions LIMIT 1')
|
||||
except psycopg.errors.InsufficientPrivilege:
|
||||
database.rollback()
|
||||
else:
|
||||
raise SystemExit('FAIL: worker read API sessions')
|
||||
expected_prefix = 'final/' if role == 'api' else 'staging/'
|
||||
try:
|
||||
client = s3_with_role(result)
|
||||
client.delete_object(Bucket=bucket(), Key=expected_prefix + '00000000-0000-0000-0000-000000000000')
|
||||
except ClientError as exc:
|
||||
expect(exc.response['ResponseMetadata']['HTTPStatusCode'] == 403, 'S3 delete rejected')
|
||||
else:
|
||||
raise SystemExit('FAIL: service can delete S3 object')
|
||||
print('PASS: scoped OpenBao read for', role, '| foreign path/policy write denied | DB TLS | S3 delete denied')
|
||||
expect(http_status('otel-collector', 9443, '/v1/traces', context(), 'POST', b'') == 403,
|
||||
'unauthenticated telemetry ingest')
|
||||
print('PASS: unauthenticated telemetry denied')
|
||||
|
||||
|
||||
def s3_with_role(settings):
|
||||
for variable in ('S3_ACCESS_KEY_ID', 'S3_SECRET_ACCESS_KEY'):
|
||||
os.environ[variable] = settings[variable]
|
||||
return s3()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
mtls()
|
||||
secrets_access()
|
||||
Reference in New Issue
Block a user