Files
mtusi2026lab7/student-tests/test_observability.py
T
alenchha 861e7722b5 ЛР 7 — БВТ2301 — Курлова — на 4
Перенесён минимум lab6 (API без изменений), добавлена защита на границах доверия:
- внешний HTTPS; обязательный mTLS LB -> API с allowlist identity и отзывом по serial;
- TLS на всех связях: PostgreSQL verify-full, MinIO HTTPS, RabbitMQ AMQPS mTLS (EXTERNAL),
  Valkey TLS + ACL, Collector и хранилища телеметрии через mTLS с отдельными CA;
- OpenBao в server mode: cert auth, токены 15 мин, политики только на свой путь;
- роли БД, S3-политики и права брокера по принципу минимальных прав;
- Argon2id с замером, rehash, общий лимит входа, бюджет защиты от login DoS;
- hardening всех контейнеров, scan образов с разбором, модель угроз, матрица доверия;
- отрицательные тесты: TLS-аудит (135 проверок), ACL, ротации, sealed OpenBao.
2026-11-16 10:49:13 +03:00

160 lines
9.9 KiB
Python

#!/usr/bin/env python3
"""Levels 3-4: one controlled upload and one failed request are found in traces, logs and metrics; privacy checks.
1. upload with fault-transient1 (attempt 1 fails transiently, attempt 2 succeeds) -> ONE trace:
API SERVER span -> PRODUCER -> CONSUMER (retry, ERROR) -> PRODUCER -> CONSUMER (succeeded) -> S3/DB spans
2. the same trace_id in Loki for messenger-api and messenger-worker log records
3. X-Request-Id of a 500 (X-Fault: error) -> Loki log line -> its trace_id -> Tempo server span with that request_id
4. no password / token / message text / filename in logs or traces; no ids in metric labels
5. all metric names used by the dashboard and alert rules exist and have samples
python3 student-tests/test_observability.py (stack from REPORT.md must be running; FAULT_INJECTION=1)
"""
import re
import time
import uuid
import obs
from stand import Client, check, log, run, unique_name, upload, wait_status
UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
FORBIDDEN_LABELS = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url_full", "url_query",
"filename", "text", "http_target", "url_path"}
REQUIRED_METRICS = [
"http_server_request_duration_seconds_bucket", "messenger_jobs_processed_total",
"messenger_job_duration_seconds_bucket", "messenger_attachment_time_to_ready_seconds_bucket",
"messenger_queue_depth", "messenger_queue_oldest_age_seconds", "messenger_worker_busy",
"messenger_worker_capacity", "db_client_connection_count", "db_client_connection_pending_requests",
"process_cpu_time_seconds_total", "process_memory_usage_bytes", "otelcol_receiver_accepted_spans_total",
"otelcol_exporter_sent_spans_total", "otelcol_exporter_queue_size",
]
def main():
c = Client()
alice, bob, _eve, chat = c.bootstrap()
marker = "secret-text-" + uuid.uuid4().hex
sent = c.message(alice, chat, marker)
msg_rid = sent["headers"]["X-Request-Id"]
log("message sent request_id=%s (text marker kept private)" % msg_rid)
# ---------------------------------------------------------------- 1. upload -> retry -> ready, one trace
filename = unique_name("transient1")
att = upload(c, alice, chat, filename=filename)
ready, seen = wait_status(c, bob, att["id"], ("ready", "failed"), 90)
check(ready["status"] == "ready", "upload must become ready after one retry")
log("upload %s: %s" % (att["id"], " -> ".join(["queued"] + seen)))
def find():
found = obs.tempo_search('{span.messenger.attachment_id="%s"}' % att["id"])
if not found:
return None
spans = obs.tempo_trace(found[0]["traceID"])
consumers = [s for s in spans if s["kind"] == "CONSUMER"]
producers = [s for s in spans if s["kind"] == "PRODUCER"]
# Spans of one trace arrive from 3 processes with different batch delays: wait until all are in.
return (found, spans) if len(consumers) >= 2 and len(producers) >= 2 and any(
s["attrs"].get("messenger.job.outcome") == "succeeded" for s in consumers) else None
result = obs.wait_for(find, 60)
check(result, "trace of the upload with both attempts not found in Tempo")
found, spans = result
trace_ids = {t["traceID"] for t in found}
check(len(trace_ids) == 1, "upload and its jobs must be ONE trace, got %s" % trace_ids)
trace_id = found[0]["traceID"]
log("Tempo: trace %s, %d spans; tree (DB client spans omitted):" % (trace_id, len(spans)))
obs.print_tree(spans, out=lambda line: print(" " + line), skip_db=True)
by_id = {s["span_id"]: s for s in spans}
server = [s for s in spans if s["kind"] == "SERVER" and s["service"] == "messenger-api"]
producers = [s for s in spans if s["kind"] == "PRODUCER"]
consumers = [s for s in spans if s["kind"] == "CONSUMER"]
check(len(server) == 1 and server[0]["name"] == "POST /api/v1/chats/{chat_id}/attachments", "API server span")
check(len(producers) >= 2 and len(consumers) >= 2, "two publish/consume pairs (attempt 1 + retry)")
for cons in consumers:
parent = by_id.get(cons["parent_id"])
check(parent is not None and parent["kind"] == "PRODUCER", "consumer must be a child of a producer span")
check(by_id[producers[0]["parent_id"]]["kind"] == "SERVER", "first producer is a child of the upload request")
retry_parent = by_id[producers[1]["parent_id"]]
check(retry_parent["kind"] == "CONSUMER" and retry_parent["attrs"].get("messenger.job.outcome") == "retry",
"retry producer must be a child of the failed consumer")
check(consumers[0]["status"] == "ERROR" and consumers[-1]["attrs"].get("messenger.job.outcome") == "succeeded",
"attempt 1 ERROR/retry, last attempt succeeded")
worker_ops = {s["name"] for s in spans if s["service"] == "messenger-worker"}
for op in ("S3.GetObject", "S3.CopyObject", "S3.PutObject", "imageproc"):
check(op in worker_ops, "worker span %s missing" % op)
check(any(s["service"] == "messenger-worker" and s["attrs"].get("db.system") == "postgresql" for s in spans),
"worker DB spans missing")
log("PASS: W3C context API -> outbox -> broker -> worker (retry) -> S3/DB is one trace with producer/consumer links")
# ---------------------------------------------------------------- 2. logs of both services with the trace_id
lines = obs.wait_for(lambda: [l for l in obs.loki('{service_name=~"messenger-api|messenger-worker"} | trace_id="%s"'
% trace_id) if True] or None, 30)
check(lines, "no Loki records with the trace_id")
services = {l["service_name"] for l in lines}
events = [(l["service_name"].replace("messenger-", ""), l.get("event")) for l in lines]
log("Loki: %d records with trace_id=%s: %s" % (len(lines), trace_id, events))
check(services == {"messenger-api", "messenger-worker"}, "logs of both services must carry the trace_id")
check({"attachment_queued", "job_attempt_failed", "job_finished"} <= {e for _, e in events},
"expected events attachment_queued, job_attempt_failed, job_finished")
sample = lines[0]
log("Loki record fields: %s" % sorted(k for k in sample if k not in ("line",)))
# ---------------------------------------------------------------- 3. X-Request-Id of an HTTP error -> log -> trace
err = c.call("GET", "/api/v1/chats", 500, actor=alice, headers={"X-Fault": "error"})
rid = err["headers"]["X-Request-Id"]
check(err["json"]["error"]["request_id"] == rid and "Traceback" not in err["bytes"].decode(), "500 body is clean")
err_lines = obs.wait_for(lambda: obs.loki('{service_name="messenger-api"} |= "%s"' % rid, minutes=5) or None, 30)
check(err_lines, "500 request_id not found in Loki")
err_trace = err_lines[0].get("trace_id")
log("HTTP 500 X-Request-Id=%s -> Loki event=%s trace_id=%s" % (rid, err_lines[0].get("event"), err_trace))
err_spans = obs.wait_for(lambda: obs.tempo_trace(err_trace) or None, 30)
check(err_spans, "trace of the 500 not in Tempo")
srv = [s for s in err_spans if s["kind"] == "SERVER"][0]
check(srv["attrs"].get("messenger.request_id") == rid and srv["status"] == "ERROR"
and str(srv["attrs"].get("http.response.status_code")) == "500", "server span must carry request_id and ERROR")
log("Tempo: %s status=%s request_id matches; exception event recorded=%s" % (
srv["name"], srv["status"], "exception" in obs.tempo_raw(err_trace)))
msg_trace = obs.wait_for(lambda: obs.tempo_search('{span.messenger.request_id="%s"}' % msg_rid) or None, 30)
check(msg_trace, "message request trace not found by request_id")
msg_spans = obs.tempo_trace(msg_trace[0]["traceID"])
log("message POST trace %s: %s" % (msg_trace[0]["traceID"], [s["name"] for s in msg_spans]))
# ---------------------------------------------------------------- 4. privacy
secrets = {"password": alice["registration"]["password"], "token": alice["token"], "message text": marker,
"filename": filename}
raw = obs.tempo_raw(trace_id) + obs.tempo_raw(msg_trace[0]["traceID"]) + obs.tempo_raw(err_trace)
logs_raw = "\n".join(str(l) for l in obs.loki('{service_name=~"messenger-.*"}', minutes=10, limit=5000))
for what, value in secrets.items():
check(value not in raw, "%s leaked into traces" % what)
check(value not in logs_raw, "%s leaked into logs" % what)
# Image bytes: PNG signature/IHDR chunk as raw, escaped or base64 text (sample.png starts with iVBORw0KGgo).
for word in ("Authorization", "Bearer ", "filename=", "X-Amz-Signature", "iVBORw0KGgo", "IHDR", "\\x89PNG",
"\\u0089PNG"):
check(word not in raw and word not in logs_raw, "%r found in telemetry" % word)
log("PASS: password, bearer token, message text, filename, Authorization, signed URLs, image bytes absent from "
"traces and logs")
series = obs.prom('{__name__=~"messenger_.*|http_server_.*|db_client_.*|process_.*"}')
labels = set()
for metric, _ in series:
labels |= set(metric)
for name, value in metric.items():
check(not UUID_RE.search(value), "UUID-like value in metric label %s=%s" % (name, value))
check(not labels & FORBIDDEN_LABELS, "forbidden metric labels: %s" % (labels & FORBIDDEN_LABELS))
log("PASS: %d series, label names: %s" % (len(series), sorted(labels)))
routes = sorted({m.get("http_route") for m, _ in series if m.get("http_route")})
log("http_route values (templates only): %s" % routes)
# ---------------------------------------------------------------- 5. metrics referenced by dashboard/alerts
time.sleep(12) # one metric export interval
for name in REQUIRED_METRICS:
check(obs.prom(name), "metric %s has no samples" % name)
log("PASS: required metrics present: %s" % ", ".join(REQUIRED_METRICS))
print("PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy)")
if __name__ == "__main__":
run(main)