Files
mtusi2026lab6/student-tests/lab6_observability.py

118 lines
6.9 KiB
Python

"""Live checks: stored signals, retry causality, privacy and bounded backend outage."""
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import subprocess
import sys
from tempfile import TemporaryDirectory
import time
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT / 'tests'), str(ROOT)]
from client import Client
from research.observations import check_dashboard, compose, get, logs, query, scalar, save, spans, sql, trace, wait
def main():
started = time.time()
result = {'utc': datetime.now(timezone.utc).isoformat(), 'checks': {}}
client = Client()
alice, bob, _, chat = client.bootstrap()
canary = 'PRIVATE-BODY-' + uuid4().hex
client.message(alice, chat, canary)
png = (ROOT / 'tests/fixtures/sample.png').read_bytes()
result['dashboard'] = get('/api/dashboards/uid/messenger-lab6')['dashboard']['uid']
assert {s['uid'] for s in get('/api/datasources')} == {'prometheus', 'loki', 'tempo'}
compose('exec', '-T', 'worker', 'python', '-c', "from pathlib import Path; Path('/tmp/pipeline-faults').mkdir(exist_ok=True)")
# Stop before accepting so a one-shot worker failure is installed deterministically.
compose('stop', '--timeout', '5', 'worker')
try:
container = compose('ps', '-aq', 'worker').stdout.strip()
response = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + canary + '.png', 202,
actor=alice, raw=png, headers={'Content-Type': 'image/png'}, schema='Attachment')
att = response['json']; trace_id = response['headers']['X-Trace-Id']
job_id = sql("SELECT id FROM attachment_jobs WHERE attachment_id='" + att['id'] + "'")
with TemporaryDirectory() as directory:
marker = Path(directory) / 'fault'; marker.write_text('transient'); marker.chmod(0o644)
subprocess.run(['docker', 'cp', str(marker), container + ':/tmp/pipeline-faults/' + job_id + '.before_process'], check=True, capture_output=True)
compose('start', 'worker')
ready = client.wait_ready(alice, att)
assert ready['status'] == 'ready'
wait(lambda: len(spans(trace(trace_id))) >= 10)
time.sleep(5)
stored = trace(trace_id)
chain = spans(stored)
names = [s['name'] for s in chain]
assert {'messenger-api', 'messenger-publisher', 'messenger-worker'} <= {s['service'] for s in chain}
assert names.count('attachments publish') >= 2 and names.count('attachments process') >= 2
assert 's3.put_object' in names and 's3.get_object' in names and 'postgresql.UPDATE' in names
identifiers = {s['spanId'] for s in chain}
assert all(not s.get('parentSpanId') or s['parentSpanId'] in identifiers for s in chain), 'Broken parent chain'
correlated = wait(lambda: (v if (v := logs('{service_name=~"messenger-api|messenger-worker"} |= "' + trace_id + '"'))['data']['result'] else None))
raw = json.dumps([stored, correlated])
assert canary not in raw and alice['token'] not in raw and alice['registration']['password'] not in raw
save('lab6-trace-retry.json', stored); save('lab6-correlated-logs.json', correlated)
result['checks']['retry_trace_and_logs'] = {'passed': True, 'trace_id': trace_id,
'request_id': response['headers']['X-Request-Id'], 'job_id': job_id, 'span_count': len(chain)}
finally:
compose('start', 'worker')
wait(lambda: query('messenger_http_requests_total'))
for name in ['messenger_http_duration_seconds_bucket','messenger_job_attempts_total',
'messenger_job_duration_seconds_bucket','messenger_queue_depth','messenger_queue_oldest_age_seconds',
'messenger_db_pool','messenger_worker_busy','messenger_worker_capacity',
'messenger_process_memory_bytes','messenger_process_cpu_time_seconds_total',
'messenger_attachment_ready_duration_seconds_bucket','messenger_ready_deadlines_total']:
assert query(name), 'Missing metric: ' + name
series = query('{__name__=~"messenger_.*"}')
forbidden = {'user_id','chat_id','message_id','attachment_id','request_id','filename','url','http_target'}
assert all(not forbidden.intersection(s['metric']) for s in series)
result['checks']['metrics_and_cardinality'] = {'passed': True, 'series': len(series)}
# Count actual missing stored log records, not just retry attempts.
lost_candidates = []
before = query('otelcol_exporter_send_failed_log_records_total')
stats = []
compose('stop', '--timeout', '5', 'loki')
outage_start = time.time()
try:
for i in range(24):
response = client.message(alice, chat, 'synthetic backend outage')
lost_candidates.append(response['headers']['X-Request-Id'])
if i % 6 == 0:
stats.append(subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',
compose('ps','-q','otel-collector').stdout.strip()],text=True).strip())
time.sleep(1.5)
finally:
compose('start', 'loki')
wait(lambda: logs('{service_name="messenger-api"}')['status'] == 'success')
time.sleep(20)
retrieved = logs('{service_name="messenger-api"} | json | event="http.request"', start=outage_start-1, limit=5000)
encoded = json.dumps(retrieved)
missing = [rid for rid in lost_candidates if rid not in encoded]
collector_id = compose('ps','-q','otel-collector').stdout.strip()
state = json.loads(subprocess.check_output(['docker','inspect',collector_id],text=True))[0]
assert state['State']['Running'] and not state['State']['OOMKilled']
assert state['HostConfig']['Memory'] == 256*1024*1024
assert missing, 'Expected finite retry timeout to cause measured telemetry loss'
client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=alice, schema='MessagePage')
result['checks']['backend_outage'] = {'passed': True,'backend':'loki', 'successful_writes':24,
'outage_seconds':round(time.time()-outage_start-20,2),'missing_request_logs':len(missing),
'tested_request_ids':lost_candidates, 'missing_request_ids':missing, 'collector_memory_samples':stats,
'collector_limit_bytes':state['HostConfig']['Memory'], 'oom_killed':False,
'failed_exports_before':before,'failed_exports_after':query('otelcol_exporter_send_failed_log_records_total')}
# Every Prometheus expression referenced by the provisioned dashboard parses and executes.
dashboard_check = check_dashboard()
save('lab6-dashboard-checks.json', dashboard_check)
result['checks']['provisioned_dashboard_queries'] = {'passed':True,'panels':len(dashboard_check['panels'])}
result['passed'] = True
result['duration_seconds'] = round(time.time()-started,2)
save('lab6-checks.json',result)
print(json.dumps(result,ensure_ascii=False,indent=2))
if __name__ == '__main__':
main()