forked from mtusi2026/mtusi2026lab6
118 lines
6.9 KiB
Python
118 lines
6.9 KiB
Python
"""Live checks: stored signals, retry causality, privacy and bounded backend outage."""
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
from tempfile import TemporaryDirectory
|
|
import time
|
|
from uuid import uuid4
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path[:0] = [str(ROOT / 'tests'), str(ROOT)]
|
|
from client import Client
|
|
from research.observations import check_dashboard, compose, get, logs, query, scalar, save, spans, sql, trace, wait
|
|
|
|
|
|
def main():
|
|
started = time.time()
|
|
result = {'utc': datetime.now(timezone.utc).isoformat(), 'checks': {}}
|
|
client = Client()
|
|
alice, bob, _, chat = client.bootstrap()
|
|
canary = 'PRIVATE-BODY-' + uuid4().hex
|
|
client.message(alice, chat, canary)
|
|
png = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
result['dashboard'] = get('/api/dashboards/uid/messenger-lab6')['dashboard']['uid']
|
|
assert {s['uid'] for s in get('/api/datasources')} == {'prometheus', 'loki', 'tempo'}
|
|
compose('exec', '-T', 'worker', 'python', '-c', "from pathlib import Path; Path('/tmp/pipeline-faults').mkdir(exist_ok=True)")
|
|
# Stop before accepting so a one-shot worker failure is installed deterministically.
|
|
compose('stop', '--timeout', '5', 'worker')
|
|
try:
|
|
container = compose('ps', '-aq', 'worker').stdout.strip()
|
|
response = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + canary + '.png', 202,
|
|
actor=alice, raw=png, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
att = response['json']; trace_id = response['headers']['X-Trace-Id']
|
|
job_id = sql("SELECT id FROM attachment_jobs WHERE attachment_id='" + att['id'] + "'")
|
|
with TemporaryDirectory() as directory:
|
|
marker = Path(directory) / 'fault'; marker.write_text('transient'); marker.chmod(0o644)
|
|
subprocess.run(['docker', 'cp', str(marker), container + ':/tmp/pipeline-faults/' + job_id + '.before_process'], check=True, capture_output=True)
|
|
compose('start', 'worker')
|
|
ready = client.wait_ready(alice, att)
|
|
assert ready['status'] == 'ready'
|
|
wait(lambda: len(spans(trace(trace_id))) >= 10)
|
|
time.sleep(5)
|
|
stored = trace(trace_id)
|
|
chain = spans(stored)
|
|
names = [s['name'] for s in chain]
|
|
assert {'messenger-api', 'messenger-publisher', 'messenger-worker'} <= {s['service'] for s in chain}
|
|
assert names.count('attachments publish') >= 2 and names.count('attachments process') >= 2
|
|
assert 's3.put_object' in names and 's3.get_object' in names and 'postgresql.UPDATE' in names
|
|
identifiers = {s['spanId'] for s in chain}
|
|
assert all(not s.get('parentSpanId') or s['parentSpanId'] in identifiers for s in chain), 'Broken parent chain'
|
|
correlated = wait(lambda: (v if (v := logs('{service_name=~"messenger-api|messenger-worker"} |= "' + trace_id + '"'))['data']['result'] else None))
|
|
raw = json.dumps([stored, correlated])
|
|
assert canary not in raw and alice['token'] not in raw and alice['registration']['password'] not in raw
|
|
save('lab6-trace-retry.json', stored); save('lab6-correlated-logs.json', correlated)
|
|
result['checks']['retry_trace_and_logs'] = {'passed': True, 'trace_id': trace_id,
|
|
'request_id': response['headers']['X-Request-Id'], 'job_id': job_id, 'span_count': len(chain)}
|
|
finally:
|
|
compose('start', 'worker')
|
|
|
|
wait(lambda: query('messenger_http_requests_total'))
|
|
for name in ['messenger_http_duration_seconds_bucket','messenger_job_attempts_total',
|
|
'messenger_job_duration_seconds_bucket','messenger_queue_depth','messenger_queue_oldest_age_seconds',
|
|
'messenger_db_pool','messenger_worker_busy','messenger_worker_capacity',
|
|
'messenger_process_memory_bytes','messenger_process_cpu_time_seconds_total',
|
|
'messenger_attachment_ready_duration_seconds_bucket','messenger_ready_deadlines_total']:
|
|
assert query(name), 'Missing metric: ' + name
|
|
series = query('{__name__=~"messenger_.*"}')
|
|
forbidden = {'user_id','chat_id','message_id','attachment_id','request_id','filename','url','http_target'}
|
|
assert all(not forbidden.intersection(s['metric']) for s in series)
|
|
result['checks']['metrics_and_cardinality'] = {'passed': True, 'series': len(series)}
|
|
|
|
# Count actual missing stored log records, not just retry attempts.
|
|
lost_candidates = []
|
|
before = query('otelcol_exporter_send_failed_log_records_total')
|
|
stats = []
|
|
compose('stop', '--timeout', '5', 'loki')
|
|
outage_start = time.time()
|
|
try:
|
|
for i in range(24):
|
|
response = client.message(alice, chat, 'synthetic backend outage')
|
|
lost_candidates.append(response['headers']['X-Request-Id'])
|
|
if i % 6 == 0:
|
|
stats.append(subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',
|
|
compose('ps','-q','otel-collector').stdout.strip()],text=True).strip())
|
|
time.sleep(1.5)
|
|
finally:
|
|
compose('start', 'loki')
|
|
wait(lambda: logs('{service_name="messenger-api"}')['status'] == 'success')
|
|
time.sleep(20)
|
|
retrieved = logs('{service_name="messenger-api"} | json | event="http.request"', start=outage_start-1, limit=5000)
|
|
encoded = json.dumps(retrieved)
|
|
missing = [rid for rid in lost_candidates if rid not in encoded]
|
|
collector_id = compose('ps','-q','otel-collector').stdout.strip()
|
|
state = json.loads(subprocess.check_output(['docker','inspect',collector_id],text=True))[0]
|
|
assert state['State']['Running'] and not state['State']['OOMKilled']
|
|
assert state['HostConfig']['Memory'] == 256*1024*1024
|
|
assert missing, 'Expected finite retry timeout to cause measured telemetry loss'
|
|
client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=alice, schema='MessagePage')
|
|
result['checks']['backend_outage'] = {'passed': True,'backend':'loki', 'successful_writes':24,
|
|
'outage_seconds':round(time.time()-outage_start-20,2),'missing_request_logs':len(missing),
|
|
'tested_request_ids':lost_candidates, 'missing_request_ids':missing, 'collector_memory_samples':stats,
|
|
'collector_limit_bytes':state['HostConfig']['Memory'], 'oom_killed':False,
|
|
'failed_exports_before':before,'failed_exports_after':query('otelcol_exporter_send_failed_log_records_total')}
|
|
# Every Prometheus expression referenced by the provisioned dashboard parses and executes.
|
|
dashboard_check = check_dashboard()
|
|
save('lab6-dashboard-checks.json', dashboard_check)
|
|
result['checks']['provisioned_dashboard_queries'] = {'passed':True,'panels':len(dashboard_check['panels'])}
|
|
result['passed'] = True
|
|
result['duration_seconds'] = round(time.time()-started,2)
|
|
save('lab6-checks.json',result)
|
|
print(json.dumps(result,ensure_ascii=False,indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|