diff --git a/Makefile b/Makefile index b7cf5f9..e78f3aa 100644 --- a/Makefile +++ b/Makefile @@ -39,8 +39,8 @@ up: bootstrap down: docker compose down lint: - $(UV) run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) - $(UV) run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) + $(UV) run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/observe.py scripts/research_slo.py $(wildcard scripts/verify_*.py) + $(UV) run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/observe.py scripts/research_slo.py $(wildcard scripts/verify_*.py) test-student: BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests verify: diff --git a/evidence/http-first.json b/evidence/http-first.json new file mode 100644 index 0000000..e14ffa9 --- /dev/null +++ b/evidence/http-first.json @@ -0,0 +1,8 @@ +{ + "source_commit": "8f3ff33b2dcb1cf3082535acf99ee4045a80400d", + "command": "BASE_URL=http://127.0.0.1:8086 uv run --locked pytest student-tests -q", + "elapsed_seconds": 417.533, + "exit_code": 0, + "summary": "112 passed in 416.52s (0:06:56)", + "result": "PASS" +} diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json index b897d8a..68a6d1d 100644 --- a/infra/grafana/dashboards/messenger.json +++ b/infra/grafana/dashboards/messenger.json @@ -679,7 +679,7 @@ }, "targets": [ { - "expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))", + "expr": "sum(rate(otelcol_exporter_send_failed_spans[1m]))", "refId": "A", "legendFormat": "spans/s", "datasource": { @@ -688,7 +688,7 @@ } }, { - "expr": "sum(rate(otelcol_exporter_send_failed_log_records_total[1m]))", + "expr": "sum(rate(otelcol_exporter_send_failed_log_records[1m]))", "refId": "B", "legendFormat": "logs/s", "datasource": { diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml index b601685..b04c4a9 100644 --- a/infra/otel/collector.yaml +++ b/infra/otel/collector.yaml @@ -15,9 +15,13 @@ processors: exporters: prometheus: endpoint: 0.0.0.0:8889 - resource_to_telemetry_conversion: - enabled: true metric_expiration: 5m + resource_constant_labels: + included: + - service.name + - service.instance.id + - service.version + - deployment.environment.name otlp_http/tempo: endpoint: http://tempo:4318 timeout: 2s @@ -44,28 +48,41 @@ extensions: health_check: endpoint: 0.0.0.0:13133 service: - extensions: [health_check] + extensions: + - health_check telemetry: logs: level: warn metrics: level: detailed readers: - - pull: - exporter: - prometheus: - host: 0.0.0.0 - port: 8888 + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 pipelines: traces: - receivers: [otlp] - processors: [memory_limiter, batch] - exporters: [otlp_http/tempo] + receivers: + - otlp + processors: + - memory_limiter + - batch + exporters: + - otlp_http/tempo logs: - receivers: [otlp] - processors: [memory_limiter, batch] - exporters: [otlp_http/loki] + receivers: + - otlp + processors: + - memory_limiter + - batch + exporters: + - otlp_http/loki metrics: - receivers: [otlp] - processors: [memory_limiter, batch] - exporters: [prometheus] + receivers: + - otlp + processors: + - memory_limiter + - batch + exporters: + - prometheus diff --git a/scripts/observe.py b/scripts/observe.py new file mode 100644 index 0000000..bc334e5 --- /dev/null +++ b/scripts/observe.py @@ -0,0 +1,134 @@ +"""Read-only queries through an internal probe, without exposing admin ports.""" + +import base64 +import json +import time +from urllib.parse import urlencode + +from scripts.stand import compose + +ENDPOINTS = { + "prometheus": "http://prometheus:9090", + "loki": "http://loki:3100", + "tempo": "http://tempo:3200", + "otel": "http://otel:13133", +} + + +def fetch(service, path, *, params=None, json_result=True): + url = ENDPOINTS[service] + path + if params: + url += "?" + urlencode(params, doseq=True) + code = ( + "import urllib.request; " + f"r=urllib.request.urlopen(urllib.request.Request({url!r},headers={{'Accept':'application/json'}}),timeout=8); " + "print(r.read().decode())" + ) + text = compose("exec", "-T", "observer", "python", "-c", code) + return json.loads(text) if json_result else text + + +def query(expression, *, at=None): + params = {"query": expression} + if at is not None: + params["time"] = str(at) + result = fetch("prometheus", "/api/v1/query", params=params) + assert result["status"] == "success" + return result["data"]["result"] + + +def scalar(expression): + result = query(expression) + return float(result[0]["value"][1]) if result else 0.0 + + +def queries(expressions): + urls = { + name: ENDPOINTS["prometheus"] + + "/api/v1/query?" + + urlencode({"query": expression}) + for name, expression in expressions.items() + } + code = ( + "import json,urllib.request\n" + f"urls={urls!r}\n" + "result={}\n" + "for name,url in urls.items():\n" + " with urllib.request.urlopen(url,timeout=8) as r: result[name]=json.load(r)['data']['result']\n" + "print(json.dumps(result))" + ) + return json.loads(compose("exec", "-T", "observer", "python", "-c", code)) + + +def logs(trace_id, *, start=None): + return fetch( + "loki", + "/loki/api/v1/query_range", + params={ + "query": '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | trace_id="' + + trace_id + + '"', + "start": str(int((start or time.time() - 3600) * 1e9)), + "end": str(time.time_ns()), + "limit": 500, + }, + ) + + +def normalize_id(value): + if not value: + return "" + if len(value) in {16, 32} and all(c in "0123456789abcdef" for c in value): + return value + return base64.b64decode(value).hex() + + +def spans(trace_id): + result = fetch("tempo", "/api/traces/" + trace_id) + flattened = [] + for batch in result.get("batches", result.get("resourceSpans", [])): + resource = { + a["key"]: next(iter(a["value"].values())) + for a in batch.get("resource", {}).get("attributes", []) + } + for scope in batch.get("scopeSpans", []): + for item in scope.get("spans", []): + flattened.append( + { + "service": resource.get("service.name"), + "version": resource.get("service.version"), + "instance": resource.get("service.instance.id"), + "name": item["name"], + "kind": item.get("kind"), + "trace_id": normalize_id(item.get("traceId")), + "span_id": normalize_id(item.get("spanId")), + "parent_span_id": normalize_id( + item.get("parentSpanId") + ), + "duration_seconds": ( + int(item["endTimeUnixNano"]) + - int(item["startTimeUnixNano"]) + ) + / 1e9, + "status": item.get("status", {}), + "attributes": { + a["key"]: next(iter(a["value"].values())) + for a in item.get("attributes", []) + }, + } + ) + return flattened + + +def wait(condition, *, timeout=90, interval=1): + end = time.monotonic() + timeout + last_error = None + while time.monotonic() < end: + try: + result = condition() + if result: + return result + except (RuntimeError, ValueError, KeyError) as exc: + last_error = type(exc).__name__ + time.sleep(interval) + raise TimeoutError("Telemetry observation not reached: " + str(last_error)) diff --git a/scripts/research_slo.py b/scripts/research_slo.py new file mode 100644 index 0000000..07d39d7 --- /dev/null +++ b/scripts/research_slo.py @@ -0,0 +1,577 @@ +"""Thirty-minute controlled SLO study with real DB/worker/backend failures.""" + +import argparse +import csv +import io +import json +import math +import os +import threading +import time +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime +from uuid import uuid4 + +import httpx +from PIL import Image + +os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml") +from scripts.observe import fetch, logs, queries, spans, wait +from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql + +ERRORS = 'haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}' +RESPONSES = 'haproxy_backend_http_responses_total{proxy="messenger"}' +SIGNALS = { + "api_error_ratio_1m": f"sum(rate({ERRORS}[1m])) / clamp_min(sum(rate({RESPONSES}[1m])),0.001)", + "queue_depth": "max(messenger_queue_depth)", + "oldest_job_age": "max(messenger_oldest_job_age_seconds)", + "worker_busy": "sum(messenger_worker_busy)", + "worker_attempts": "sum(messenger_job_attempts_total)", + "pool_used": 'sum(messenger_db_pool_used{service_name="messenger-api"})', + "pool_waiting": 'sum(messenger_db_pool_waiting{service_name="messenger-api"})', + "http_p95": "histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket[1m])))", + "slo_good": 'sum(messenger_slo_decisions_total{outcome="good"})', + "slo_bad": 'sum(messenger_slo_decisions_total{outcome="bad"})', + "collector_received_spans": "sum(otelcol_receiver_accepted_spans)", + "collector_sent_spans": "sum(otelcol_exporter_sent_spans)", + "collector_received_logs": "sum(otelcol_receiver_accepted_log_records)", + "collector_sent_logs": "sum(otelcol_exporter_sent_log_records)", + "collector_failed_spans": "sum(otelcol_exporter_send_failed_spans)", + "collector_queue": "sum(otelcol_exporter_queue_size)", + "collector_queue_capacity": "sum(otelcol_exporter_queue_capacity)", + "collector_memory": 'process_resident_memory_bytes{job="collector"}', + "app_memory": "sum(messenger_process_memory_bytes)", +} +BURN_ALERTS = {"ApiFastBurn", "ApiSlowBurn", "ReadyFastBurn", "ReadySlowBurn"} + + +def snapshot(phase): + observed = queries(SIGNALS) + values = {} + for key, rows in observed.items(): + value = float(rows[0]["value"][1]) if rows else 0.0 + values[key] = value if math.isfinite(value) else None + alerts = fetch("prometheus", "/api/v1/alerts")["data"]["alerts"] + return { + "at": time.time(), + "phase": phase, + "signals": values, + "alerts": [ + { + "name": a["labels"]["alertname"], + "state": a["state"], + "active_at": a.get("activeAt"), + } + for a in alerts + ], + } + + +def firing(samples): + return { + a["name"] + for sample in samples + for a in sample["alerts"] + if a["state"] == "firing" + } + + +class Workload: + def __init__(self, client, headers, chat): + self.client, self.headers, self.chat = client, headers, chat + self.stop = threading.Event() + self.phase = "warmup" + self.requests = [] + self.uploads = [] + self.lock = threading.Lock() + self.slots = threading.BoundedSemaphore(16) + self.pool = ThreadPoolExecutor(max_workers=16) + image = io.BytesIO() + Image.new("RGB", (320, 200), (39, 102, 157)).save(image, "PNG") + self.raw = image.getvalue() + self.threads = [ + threading.Thread(target=self.schedule_messages), + threading.Thread(target=self.schedule_uploads), + ] + + def write(self): + started, phase, trace_id = time.time(), self.phase, uuid4().hex + status, identity = 0, "" + try: + response = self.client.post( + f"/api/v1/chats/{self.chat}/messages", + headers={ + **self.headers, + "Idempotency-Key": uuid4().hex, + "traceparent": "00-" + + trace_id + + "-" + + uuid4().hex[:16] + + "-01", + }, + json={"text": "Synthetic SLO message " + uuid4().hex}, + ) + status = response.status_code + if status == 201: + identity = response.json()["id"] + except httpx.HTTPError: + pass + finally: + with self.lock: + self.requests.append( + [ + started, + phase, + status, + time.time() - started, + trace_id, + identity, + ] + ) + self.slots.release() + + def schedule_messages(self): + next_at = time.monotonic() + while not self.stop.is_set(): + if self.slots.acquire(blocking=False): + self.pool.submit(self.write) + else: + with self.lock: + self.requests.append( + [time.time(), self.phase, -1, 0, "", ""] + ) + next_at += 0.25 + self.stop.wait(max(0, next_at - time.monotonic())) + + def schedule_uploads(self): + while not self.stop.is_set(): + started, phase, trace_id = time.time(), self.phase, uuid4().hex + status, identity = 0, "" + try: + response = self.client.post( + f"/api/v1/chats/{self.chat}/attachments", + params={"filename": "synthetic-slo.png"}, + content=self.raw, + headers={ + **self.headers, + "Content-Type": "image/png", + "traceparent": "00-" + + trace_id + + "-" + + uuid4().hex[:16] + + "-01", + }, + ) + status = response.status_code + if status == 202: + identity = response.json()["id"] + except httpx.HTTPError: + pass + with self.lock: + self.uploads.append( + [started, phase, status, identity, trace_id] + ) + self.stop.wait(max(0, 5 - (time.time() - started))) + + def start(self): + for thread in self.threads: + thread.start() + + def close(self): + self.stop.set() + for thread in self.threads: + thread.join(20) + self.pool.shutdown(wait=True) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", default="evidence/slo-research-first.json") + parser.add_argument("--minimum-seconds", type=int, default=1800) + args = parser.parse_args() + output = ROOT / args.output + assert not output.exists(), "Choose a new output path" + report = { + "date": datetime.now(UTC).isoformat(), + "source_commit": command(["git", "rev-parse", "HEAD"]), + "scope": "full30m" + if args.minimum_seconds >= 1800 + else "short-development", + "profile": { + "messages_per_second": 4, + "max_inflight": 16, + "upload_every_seconds": 5, + "image_pixels": [320, 200], + }, + "complete": False, + "samples": [], + "phases": [], + } + study_started = time.monotonic() + output.parent.mkdir(parents=True, exist_ok=True) + load = None + trigger = "lab6_slo_delay_" + uuid4().hex[:8] + with httpx.Client( + base_url=BASE, + timeout=8, + trust_env=False, + limits=httpx.Limits(max_connections=24, max_keepalive_connections=24), + ) as client: + ready(timeout=90) + user, _, session = actor(client) + headers = auth(session) + response = client.post( + "/api/v1/chats", + headers=headers, + json={ + "title": "Synthetic SLO experiment", + "member_ids": [user["id"]], + }, + ) + assert response.status_code == 201 + chat = response.json()["id"] + load = Workload(client, headers, chat) + workload = load + + def save(): + output.write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n" + ) + + def hold(name, seconds): + load.phase = name + phase = { + "name": name, + "start": time.time(), + "planned_seconds": seconds, + } + report["phases"].append(phase) + deadline = time.monotonic() + seconds + start_index = len(report["samples"]) + while time.monotonic() < deadline: + report["samples"].append(snapshot(name)) + save() + time.sleep(min(5, max(0, deadline - time.monotonic()))) + phase["end"] = time.time() + phase["firing"] = sorted(firing(report["samples"][start_index:])) + return phase + + def db_delay(on): + if on: + sql( + f"CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.chat_id='{chat}'::uuid THEN PERFORM pg_sleep(3); END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {trigger}()" + ) + else: + sql( + f"DROP TRIGGER IF EXISTS {trigger} ON messages; DROP FUNCTION IF EXISTS {trigger}()" + ) + + try: + load.start() + warm = hold("warmup", 300) + assert not BURN_ALERTS.intersection(warm["firing"]), ( + "Baseline is not clean" + ) + report["baseline"] = snapshot("baseline") + for signal in ["api", "ready"]: + if signal == "api": + activate, restore = ( + lambda: db_delay(True), + lambda: db_delay(False), + ) + else: + + def activate(): + return compose("stop", "--timeout", "1", "worker") + + def restore(): + return compose( + "up", "-d", "--no-deps", "--no-recreate", "worker" + ) + + brief_started = time.time() + activate() + brief = hold(signal + "_brief", 5) + restore() + settle = hold( + signal + "_brief_recovery", 120 if signal == "api" else 60 + ) + assert not BURN_ALERTS.intersection( + brief["firing"] + settle["firing"] + ), "Brief incident triggered paging" + report.setdefault("short_spikes", []).append( + { + "signal": signal, + "action_started": brief_started, + "paging_events": 0, + } + ) + action_started = time.time() + activate() + action_finished = time.time() + degraded = hold( + signal + "_sustained", 180 if signal == "api" else 150 + ) + restore_started = time.time() + restore() + restore_finished = time.time() + recovered = hold( + signal + "_recovery", 150 if signal == "api" else 240 + ) + expected = ( + {"ApiFastBurn", "ApiSlowBurn", "ApiHighErrorRate"} + if signal == "api" + else {"ReadyFastBurn", "ReadySlowBurn", "QueueLagGrowing"} + ) + assert expected <= set(degraded["firing"]), ( + signal, + degraded["firing"], + ) + last = report["samples"][-1] + assert not expected.intersection(firing([last])), ( + "Incident alerts did not resolve" + ) + points = [ + s + for s in report["samples"] + if action_finished <= s["at"] <= recovered["end"] + ] + timings = {} + for name in sorted(expected): + first = next( + s["at"] for s in points if name in firing([s]) + ) + resolved = next( + s["at"] + for s in points + if s["at"] >= restore_finished + and name not in firing([s]) + ) + timings[name] = { + "detect_seconds": round(first - action_finished, 3), + "resolve_seconds": round( + resolved - restore_finished, 3 + ), + } + report.setdefault("incidents", []).append( + { + "signal": signal, + "action_started": action_started, + "action_finished": action_finished, + "restore_started": restore_started, + "restore_finished": restore_finished, + "timings": timings, + } + ) + if signal == "api": + candidates = [ + r + for r in load.requests + if action_finished <= r[0] <= restore_started + and r[2] >= 500 + ] + assert candidates + items = wait( + lambda identity=candidates[0][4]: spans(identity) + ) + assert any( + s["name"] == "db.postgresql" + and s["duration_seconds"] >= 2 + and s["status"].get("code") in {2, "STATUS_CODE_ERROR"} + for s in items + ) + report["database_diagnostic_trace"] = items + else: + assert ( + max( + s["signals"]["oldest_job_age"] or 0 for s in points + ) + > 20 + ) + assert ( + max(s["signals"]["queue_depth"] or 0 for s in points) + > 0 + ) + accepted = [ + r + for r in load.uploads + if action_finished <= r[0] <= restore_started + and r[2] == 202 + ] + assert accepted + report["worker_recovered_trace"] = wait( + lambda identity=accepted[0][4]: spans(identity) + ) + report["worker_recovered_attachment"] = accepted[0][3] + outage_before = snapshot("backend_before") + outage_started = time.time() + compose("stop", "--timeout", "1", "loki", "tempo") + outage = hold("backend_unavailable", 90) + points = [ + s + for s in report["samples"] + if s["phase"] == "backend_unavailable" + ] + assert max( + s["signals"]["collector_failed_spans"] or 0 for s in points + ) > (outage_before["signals"]["collector_failed_spans"] or 0) + assert all( + (s["signals"]["collector_queue"] or 0) + <= (s["signals"]["collector_queue_capacity"] or 0) + for s in points + ) + assert ( + max(s["signals"]["collector_memory"] or 0 for s in points) + < 256 * 1024**2 + ) + assert not BURN_ALERTS.intersection(outage["firing"]) + during = [ + r for r in load.requests if r[1] == "backend_unavailable" + ] + assert during and all(r[2] == 201 for r in during) + before_ready = sql( + "SELECT count(*) FROM attachments WHERE status='ready'" + ) + compose("up", "-d", "--no-deps", "--no-recreate", "loki", "tempo") + wait(lambda: fetch("loki", "/ready", json_result=False)) + wait(lambda: fetch("tempo", "/ready", json_result=False)) + hold("backend_recovery", 60) + restored = next( + r + for r in reversed(load.uploads) + if r[1] == "backend_recovery" and r[2] == 202 + ) + wait( + lambda: any( + s["name"] == "attachment.process" + for s in spans(restored[4]) + ) + ) + wait(lambda: logs(restored[4]).get("data", {}).get("result")) + report["backend_outage"] = { + "started": outage_started, + "finished": outage["end"], + "before": outage_before, + "after": snapshot("backend_after"), + "ready_count_before_backend_recovery": int(before_ready), + "successful_writes_during_outage": len(during), + "restored_trace_id": restored[4], + "queue_bounded": True, + "collector_rss_below_limit": True, + "send_failure_counter_counts_attempts_not_unique_loss": True, + } + remaining = args.minimum_seconds - ( + time.monotonic() - study_started + ) + if remaining > 0: + hold("steady_after_recovery", remaining) + load.close() + load = None + # Read through the public API: a successful SQL insert alone is not + # proof that every acknowledged message remains accessible. + seen, cursor = set(), None + while True: + page = client.get( + f"/api/v1/chats/{chat}/messages", + headers=headers, + params={ + "limit": 100, + **({"cursor": cursor} if cursor else {}), + }, + ) + page.raise_for_status() + data = page.json() + ids = [row["id"] for row in data["items"]] + assert len(ids) == len(set(ids)) and not seen.intersection(ids) + seen.update(ids) + cursor = data.get("next_cursor") + if not cursor: + break + expected_messages = { + r[5] for r in workload.requests if r[2] == 201 + } + assert seen == expected_messages + accepted_uploads = [r for r in workload.uploads if r[2] == 202] + for row in accepted_uploads: + + def completed(identity=row[3]): + response = client.get( + f"/api/v1/attachments/{identity}", headers=headers + ) + return ( + response.status_code == 200 + and response.json()["status"] == "ready" + ) + + wait(completed, timeout=120) + for row in accepted_uploads[ + :: max(1, len(accepted_uploads) // 10) + ]: + response = client.get( + f"/api/v1/attachments/{row[3]}/content", headers=headers + ) + assert ( + response.status_code == 200 + and response.content == workload.raw + ) + report["preservation"] = { + "acknowledged_messages": len(expected_messages), + "accessible_messages": len(seen), + "accepted_uploads": len(accepted_uploads), + "all_accepted_uploads_ready": True, + "sampled_originals_match": True, + } + report["rules"] = fetch("prometheus", "/api/v1/rules") + report["final_30m_sli"] = queries( + { + "api_good_ratio": f"1 - sum(increase({ERRORS}[30m])) / clamp_min(sum(increase({RESPONSES}[30m])),1)", + "ready_good_ratio": 'sum(increase(messenger_slo_decisions_total{outcome="good"}[30m])) / clamp_min(sum(increase(messenger_slo_decisions_total[30m])),1)', + } + ) + report["complete"] = True + finally: + try: + db_delay(False) + finally: + compose( + "up", + "-d", + "--no-deps", + "--no-recreate", + "worker", + "loki", + "tempo", + ) + if load is not None: + load.close() + report["elapsed_seconds"] = round( + time.monotonic() - study_started, 3 + ) + request_path = output.with_name(output.stem + "-requests.csv") + with request_path.open("w") as stream: + writer = csv.writer(stream) + writer.writerow( + [ + "timestamp", + "phase", + "status", + "seconds", + "trace_id", + "message_id", + ] + ) + writer.writerows(workload.requests) + report["requests_csv"] = request_path.name + report["request_statuses"] = dict( + Counter(str(r[2]) for r in workload.requests) + ) + report["uploads"] = workload.uploads + save() + print( + "PASS: short/sustained SLO incidents, detection/resolution and controlled false paging" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_telemetry.py b/scripts/verify_telemetry.py new file mode 100644 index 0000000..d2a8765 --- /dev/null +++ b/scripts/verify_telemetry.py @@ -0,0 +1,340 @@ +"""Actual API-to-worker causality, log correlation, provisioning and privacy.""" + +import argparse +import hashlib +import io +import json +import os +from datetime import UTC, datetime +from uuid import uuid4 + +import httpx +from PIL import Image + +os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml") +from scripts.observe import fetch, logs, query, spans, wait +from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql + + +def verify_chain(items, trace_id, *, retry): + assert items and {s["trace_id"] for s in items} == {trace_id} + by_id = {s["span_id"]: s for s in items} + producers = [s for s in items if s["kind"] == "SPAN_KIND_PRODUCER"] + consumers = [s for s in items if s["kind"] == "SPAN_KIND_CONSUMER"] + processing = [s for s in items if s["name"] == "attachment.process"] + assert producers and consumers and processing + assert {s["service"] for s in items} >= { + "messenger-api", + "messenger-relay", + "messenger-worker", + } + for consumer in consumers: + assert ( + by_id[consumer["parent_span_id"]]["kind"] == "SPAN_KIND_PRODUCER" + ) + for process in processing: + assert by_id[process["parent_span_id"]]["kind"] == "SPAN_KIND_CONSUMER" + assert any( + s["name"] == "s3.put" and s["service"] == "messenger-worker" + for s in items + ) + assert any( + s["name"] == "db.postgresql" and s["service"] == "messenger-worker" + for s in items + ) + if retry: + assert len(processing) >= 2 + assert any( + s["status"].get("code") in {2, "STATUS_CODE_ERROR"} + for s in processing + ) + return items + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", default="evidence/telemetry-first.json") + args = parser.parse_args() + output = ROOT / args.output + assert not output.exists(), "Choose a new output path" + report = { + "date": datetime.now(UTC).isoformat(), + "source_commit": command(["git", "rev-parse", "HEAD"]), + "complete": False, + } + output.parent.mkdir(parents=True, exist_ok=True) + try: + ready(timeout=90) + for service, path in [ + ("prometheus", "/-/ready"), + ("loki", "/ready"), + ("tempo", "/ready"), + ("otel", "/"), + ]: + wait(lambda s=service, p=path: fetch(s, p, json_result=False)) + report["backends_ready"] = True + buffer = io.BytesIO() + Image.new("RGB", (320, 200), (24, 110, 153)).save(buffer, "PNG") + raw = buffer.getvalue() + sentinel = "PrivacyProbe-" + uuid4().hex + with httpx.Client( + base_url=BASE, timeout=15, trust_env=False + ) as client: + user, _, session = actor(client) + headers = auth(session) + chat = client.post( + "/api/v1/chats", + headers=headers, + json={ + "title": "Synthetic telemetry", + "member_ids": [user["id"]], + }, + ).json()["id"] + message = client.post( + f"/api/v1/chats/{chat}/messages", + headers={**headers, "Idempotency-Key": uuid4().hex}, + json={"text": sentinel}, + ) + assert message.status_code == 201 + cases = [] + for retry in [False, True]: + if retry: + compose("stop", "--timeout", "1", "worker") + trace_id = uuid4().hex + trace_header = ( + "00-" + trace_id + "-" + uuid4().hex[:16] + "-01" + ) + accepted = client.post( + f"/api/v1/chats/{chat}/attachments", + params={"filename": sentinel + ".png"}, + headers={ + **headers, + "Content-Type": "image/png", + "traceparent": trace_header, + }, + content=raw, + ) + assert ( + accepted.status_code == 202 + and accepted.json()["status"] == "queued" + ) + attachment = accepted.json()["id"] + job = sql( + f"SELECT id FROM jobs WHERE attachment_id='{attachment}'::uuid" + ) + if retry: + sql( + f"INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES('{job}'::uuid,'before_effect','transient',1)" + ) + compose("up", "-d", "--no-deps", "--no-recreate", "worker") + + def finished(identity=attachment): + response = client.get( + f"/api/v1/attachments/{identity}", headers=headers + ) + return ( + response.status_code == 200 + and response.json()["status"] == "ready" + ) + + wait(finished, timeout=90) + original = client.get( + f"/api/v1/attachments/{attachment}/content", + headers=headers, + ) + assert original.content == raw + + def traced(identity=trace_id, retry_expected=retry): + items = spans(identity) + if len( + [s for s in items if s["name"] == "attachment.process"] + ) < (2 if retry_expected else 1): + return None + if any(s["kind"] == "SPAN_KIND_CONSUMER" for s in items): + # Independent export batches may arrive in a different + # order. Require the whole chain before accepting it. + try: + return verify_chain( + items, identity, retry=retry_expected + ) + except (AssertionError, KeyError): + return None + + items = wait(traced, timeout=90) + + def correlated(identity=trace_id): + result = logs(identity) + streams = result.get("data", {}).get("result", []) + services = { + x["stream"].get("service_name") for x in streams + } + return ( + result + if {"messenger-api", "messenger-worker"} <= services + else None + ) + + correlated_logs = wait(correlated, timeout=90) + span_ids = {s["span_id"] for s in items} + streams = correlated_logs["data"]["result"] + assert all( + row["stream"].get("span_id") in span_ids for row in streams + ) + assert any( + row["stream"].get("request_id") + == accepted.headers["X-Request-Id"] + for row in streams + ) + exported = json.dumps( + {"spans": items, "logs": correlated_logs} + ) + assert ( + sentinel not in exported + and session["token"] not in exported + ) + cases.append( + { + "retry": retry, + "attachment_id": attachment, + "job_id": job, + "trace_id": trace_id, + "request_id": accepted.headers["X-Request-Id"], + "original_sha256": hashlib.sha256(raw).hexdigest(), + "attempt": int( + sql( + f"SELECT attempt FROM jobs WHERE id='{job}'::uuid" + ) + ), + "spans": items, + "correlated_logs": correlated_logs, + } + ) + report["causal_uploads"] = cases + assert cases[1]["attempt"] >= 2 + report["privacy_sentinels_absent_from_traces_and_logs"] = True + names = wait( + lambda: fetch("prometheus", "/api/v1/label/__name__/values")[ + "data" + ] + ) + required = { + "messenger_http_requests_total", + "messenger_http_duration_seconds_bucket", + "messenger_job_attempts_total", + "messenger_job_duration_seconds_bucket", + "messenger_time_to_ready_seconds_bucket", + "messenger_queue_depth", + "messenger_oldest_job_age_seconds", + "messenger_db_pool_used", + "messenger_db_pool_waiting", + "messenger_worker_busy", + "messenger_worker_capacity", + "messenger_process_cpu_seconds_total", + "messenger_process_memory_bytes", + "messenger_slo_decisions_total", + } + wait( + lambda: ( + required + <= set( + fetch("prometheus", "/api/v1/label/__name__/values")[ + "data" + ] + ) + ) + ) + names = fetch("prometheus", "/api/v1/label/__name__/values")["data"] + report["metric_names"] = [ + n for n in names if n.startswith("messenger_") + ] + series = fetch( + "prometheus", + "/api/v1/series", + params={"match[]": '{__name__=~"messenger_.*"}'}, + )["data"] + forbidden = { + "user_id", + "chat_id", + "message_id", + "attachment_id", + "job_id", + "request_id", + "trace_id", + "filename", + } + assert all(not forbidden.intersection(s) for s in series) + assert all(sentinel not in json.dumps(s) for s in series) + report["metric_series_count"] = len(series) + report["metric_labels_low_cardinality"] = True + url = "http://127.0.0.1:" + os.getenv("GRAFANA_PORT", "3006") + password = (ROOT / "secrets/grafana_password").read_text().strip() + with httpx.Client( + base_url=url, auth=("admin", password), trust_env=False, timeout=15 + ) as grafana: + dashboard = grafana.get("/api/dashboards/uid/messenger-lab6") + dashboard.raise_for_status() + imported = dashboard.json()["dashboard"] + assert len(imported["panels"]) == 18 + sources = grafana.get("/api/datasources").json() + assert {s["uid"] for s in sources} == { + "prometheus", + "loki", + "tempo", + } + checks = {} + for uid in ["prometheus", "loki", "tempo"]: + r = grafana.get(f"/api/datasources/uid/{uid}/health") + r.raise_for_status() + assert r.json()["status"] == "OK", uid + checks[uid] = r.json()["status"] + result = grafana.post( + "/api/ds/query", + json={ + "queries": [ + { + "refId": "A", + "datasource": { + "uid": "prometheus", + "type": "prometheus", + }, + "expr": "sum(messenger_http_requests_total)", + "instant": True, + "format": "table", + "intervalMs": 5000, + "maxDataPoints": 10, + } + ], + "from": "now-5m", + "to": "now", + }, + ) + result.raise_for_status() + assert result.json()["results"]["A"]["frames"] + report["grafana"] = { + "dashboard_uid": imported["uid"], + "panels": len(imported["panels"]), + "datasources_health": checks, + "actual_query_frames": len( + result.json()["results"]["A"]["frames"] + ), + } + report["quantiles"] = { + str(q): query( + f"histogram_quantile({q},sum by(le)(rate(messenger_http_duration_seconds_bucket[2m])))" + ) + for q in [0.5, 0.95, 0.99] + } + assert all(report["quantiles"].values()) + report["complete"] = True + finally: + compose("up", "-d", "--no-deps", "--no-recreate", "worker") + output.write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n" + ) + print( + "PASS: causal API/publish/consumer/child/DB/S3, retry, correlated logs, privacy and Grafana" + ) + + +if __name__ == "__main__": + main() diff --git a/src/telemetry.py b/src/telemetry.py index 220f0c0..76a9749 100644 --- a/src/telemetry.py +++ b/src/telemetry.py @@ -306,6 +306,8 @@ def duration(name, seconds, attributes=None): def carrier(): result = {} _propagator.inject(result) + if len(result.get("tracestate", "")) > 512: + result.pop("tracestate", None) return result