forked from mtusi2026/mtusi2026lab7
Перенесён минимум lab6 (API без изменений), добавлена защита на границах доверия: - внешний HTTPS; обязательный mTLS LB -> API с allowlist identity и отзывом по serial; - TLS на всех связях: PostgreSQL verify-full, MinIO HTTPS, RabbitMQ AMQPS mTLS (EXTERNAL), Valkey TLS + ACL, Collector и хранилища телеметрии через mTLS с отдельными CA; - OpenBao в server mode: cert auth, токены 15 мин, политики только на свой путь; - роли БД, S3-политики и права брокера по принципу минимальных прав; - Argon2id с замером, rehash, общий лимит входа, бюджет защиты от login DoS; - hardening всех контейнеров, scan образов с разбором, модель угроз, матрица доверия; - отрицательные тесты: TLS-аудит (135 проверок), ACL, ротации, sealed OpenBao.
160 lines
9.9 KiB
Python
160 lines
9.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Levels 3-4: one controlled upload and one failed request are found in traces, logs and metrics; privacy checks.
|
|
|
|
1. upload with fault-transient1 (attempt 1 fails transiently, attempt 2 succeeds) -> ONE trace:
|
|
API SERVER span -> PRODUCER -> CONSUMER (retry, ERROR) -> PRODUCER -> CONSUMER (succeeded) -> S3/DB spans
|
|
2. the same trace_id in Loki for messenger-api and messenger-worker log records
|
|
3. X-Request-Id of a 500 (X-Fault: error) -> Loki log line -> its trace_id -> Tempo server span with that request_id
|
|
4. no password / token / message text / filename in logs or traces; no ids in metric labels
|
|
5. all metric names used by the dashboard and alert rules exist and have samples
|
|
|
|
python3 student-tests/test_observability.py (stack from REPORT.md must be running; FAULT_INJECTION=1)
|
|
"""
|
|
import re
|
|
import time
|
|
import uuid
|
|
|
|
import obs
|
|
from stand import Client, check, log, run, unique_name, upload, wait_status
|
|
|
|
UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
|
|
FORBIDDEN_LABELS = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url_full", "url_query",
|
|
"filename", "text", "http_target", "url_path"}
|
|
REQUIRED_METRICS = [
|
|
"http_server_request_duration_seconds_bucket", "messenger_jobs_processed_total",
|
|
"messenger_job_duration_seconds_bucket", "messenger_attachment_time_to_ready_seconds_bucket",
|
|
"messenger_queue_depth", "messenger_queue_oldest_age_seconds", "messenger_worker_busy",
|
|
"messenger_worker_capacity", "db_client_connection_count", "db_client_connection_pending_requests",
|
|
"process_cpu_time_seconds_total", "process_memory_usage_bytes", "otelcol_receiver_accepted_spans_total",
|
|
"otelcol_exporter_sent_spans_total", "otelcol_exporter_queue_size",
|
|
]
|
|
|
|
|
|
def main():
|
|
c = Client()
|
|
alice, bob, _eve, chat = c.bootstrap()
|
|
marker = "secret-text-" + uuid.uuid4().hex
|
|
sent = c.message(alice, chat, marker)
|
|
msg_rid = sent["headers"]["X-Request-Id"]
|
|
log("message sent request_id=%s (text marker kept private)" % msg_rid)
|
|
|
|
# ---------------------------------------------------------------- 1. upload -> retry -> ready, one trace
|
|
filename = unique_name("transient1")
|
|
att = upload(c, alice, chat, filename=filename)
|
|
ready, seen = wait_status(c, bob, att["id"], ("ready", "failed"), 90)
|
|
check(ready["status"] == "ready", "upload must become ready after one retry")
|
|
log("upload %s: %s" % (att["id"], " -> ".join(["queued"] + seen)))
|
|
|
|
def find():
|
|
found = obs.tempo_search('{span.messenger.attachment_id="%s"}' % att["id"])
|
|
if not found:
|
|
return None
|
|
spans = obs.tempo_trace(found[0]["traceID"])
|
|
consumers = [s for s in spans if s["kind"] == "CONSUMER"]
|
|
producers = [s for s in spans if s["kind"] == "PRODUCER"]
|
|
# Spans of one trace arrive from 3 processes with different batch delays: wait until all are in.
|
|
return (found, spans) if len(consumers) >= 2 and len(producers) >= 2 and any(
|
|
s["attrs"].get("messenger.job.outcome") == "succeeded" for s in consumers) else None
|
|
|
|
result = obs.wait_for(find, 60)
|
|
check(result, "trace of the upload with both attempts not found in Tempo")
|
|
found, spans = result
|
|
trace_ids = {t["traceID"] for t in found}
|
|
check(len(trace_ids) == 1, "upload and its jobs must be ONE trace, got %s" % trace_ids)
|
|
trace_id = found[0]["traceID"]
|
|
log("Tempo: trace %s, %d spans; tree (DB client spans omitted):" % (trace_id, len(spans)))
|
|
obs.print_tree(spans, out=lambda line: print(" " + line), skip_db=True)
|
|
|
|
by_id = {s["span_id"]: s for s in spans}
|
|
server = [s for s in spans if s["kind"] == "SERVER" and s["service"] == "messenger-api"]
|
|
producers = [s for s in spans if s["kind"] == "PRODUCER"]
|
|
consumers = [s for s in spans if s["kind"] == "CONSUMER"]
|
|
check(len(server) == 1 and server[0]["name"] == "POST /api/v1/chats/{chat_id}/attachments", "API server span")
|
|
check(len(producers) >= 2 and len(consumers) >= 2, "two publish/consume pairs (attempt 1 + retry)")
|
|
for cons in consumers:
|
|
parent = by_id.get(cons["parent_id"])
|
|
check(parent is not None and parent["kind"] == "PRODUCER", "consumer must be a child of a producer span")
|
|
check(by_id[producers[0]["parent_id"]]["kind"] == "SERVER", "first producer is a child of the upload request")
|
|
retry_parent = by_id[producers[1]["parent_id"]]
|
|
check(retry_parent["kind"] == "CONSUMER" and retry_parent["attrs"].get("messenger.job.outcome") == "retry",
|
|
"retry producer must be a child of the failed consumer")
|
|
check(consumers[0]["status"] == "ERROR" and consumers[-1]["attrs"].get("messenger.job.outcome") == "succeeded",
|
|
"attempt 1 ERROR/retry, last attempt succeeded")
|
|
worker_ops = {s["name"] for s in spans if s["service"] == "messenger-worker"}
|
|
for op in ("S3.GetObject", "S3.CopyObject", "S3.PutObject", "imageproc"):
|
|
check(op in worker_ops, "worker span %s missing" % op)
|
|
check(any(s["service"] == "messenger-worker" and s["attrs"].get("db.system") == "postgresql" for s in spans),
|
|
"worker DB spans missing")
|
|
log("PASS: W3C context API -> outbox -> broker -> worker (retry) -> S3/DB is one trace with producer/consumer links")
|
|
|
|
# ---------------------------------------------------------------- 2. logs of both services with the trace_id
|
|
lines = obs.wait_for(lambda: [l for l in obs.loki('{service_name=~"messenger-api|messenger-worker"} | trace_id="%s"'
|
|
% trace_id) if True] or None, 30)
|
|
check(lines, "no Loki records with the trace_id")
|
|
services = {l["service_name"] for l in lines}
|
|
events = [(l["service_name"].replace("messenger-", ""), l.get("event")) for l in lines]
|
|
log("Loki: %d records with trace_id=%s: %s" % (len(lines), trace_id, events))
|
|
check(services == {"messenger-api", "messenger-worker"}, "logs of both services must carry the trace_id")
|
|
check({"attachment_queued", "job_attempt_failed", "job_finished"} <= {e for _, e in events},
|
|
"expected events attachment_queued, job_attempt_failed, job_finished")
|
|
sample = lines[0]
|
|
log("Loki record fields: %s" % sorted(k for k in sample if k not in ("line",)))
|
|
|
|
# ---------------------------------------------------------------- 3. X-Request-Id of an HTTP error -> log -> trace
|
|
err = c.call("GET", "/api/v1/chats", 500, actor=alice, headers={"X-Fault": "error"})
|
|
rid = err["headers"]["X-Request-Id"]
|
|
check(err["json"]["error"]["request_id"] == rid and "Traceback" not in err["bytes"].decode(), "500 body is clean")
|
|
err_lines = obs.wait_for(lambda: obs.loki('{service_name="messenger-api"} |= "%s"' % rid, minutes=5) or None, 30)
|
|
check(err_lines, "500 request_id not found in Loki")
|
|
err_trace = err_lines[0].get("trace_id")
|
|
log("HTTP 500 X-Request-Id=%s -> Loki event=%s trace_id=%s" % (rid, err_lines[0].get("event"), err_trace))
|
|
err_spans = obs.wait_for(lambda: obs.tempo_trace(err_trace) or None, 30)
|
|
check(err_spans, "trace of the 500 not in Tempo")
|
|
srv = [s for s in err_spans if s["kind"] == "SERVER"][0]
|
|
check(srv["attrs"].get("messenger.request_id") == rid and srv["status"] == "ERROR"
|
|
and str(srv["attrs"].get("http.response.status_code")) == "500", "server span must carry request_id and ERROR")
|
|
log("Tempo: %s status=%s request_id matches; exception event recorded=%s" % (
|
|
srv["name"], srv["status"], "exception" in obs.tempo_raw(err_trace)))
|
|
|
|
msg_trace = obs.wait_for(lambda: obs.tempo_search('{span.messenger.request_id="%s"}' % msg_rid) or None, 30)
|
|
check(msg_trace, "message request trace not found by request_id")
|
|
msg_spans = obs.tempo_trace(msg_trace[0]["traceID"])
|
|
log("message POST trace %s: %s" % (msg_trace[0]["traceID"], [s["name"] for s in msg_spans]))
|
|
|
|
# ---------------------------------------------------------------- 4. privacy
|
|
secrets = {"password": alice["registration"]["password"], "token": alice["token"], "message text": marker,
|
|
"filename": filename}
|
|
raw = obs.tempo_raw(trace_id) + obs.tempo_raw(msg_trace[0]["traceID"]) + obs.tempo_raw(err_trace)
|
|
logs_raw = "\n".join(str(l) for l in obs.loki('{service_name=~"messenger-.*"}', minutes=10, limit=5000))
|
|
for what, value in secrets.items():
|
|
check(value not in raw, "%s leaked into traces" % what)
|
|
check(value not in logs_raw, "%s leaked into logs" % what)
|
|
# Image bytes: PNG signature/IHDR chunk as raw, escaped or base64 text (sample.png starts with iVBORw0KGgo).
|
|
for word in ("Authorization", "Bearer ", "filename=", "X-Amz-Signature", "iVBORw0KGgo", "IHDR", "\\x89PNG",
|
|
"\\u0089PNG"):
|
|
check(word not in raw and word not in logs_raw, "%r found in telemetry" % word)
|
|
log("PASS: password, bearer token, message text, filename, Authorization, signed URLs, image bytes absent from "
|
|
"traces and logs")
|
|
|
|
series = obs.prom('{__name__=~"messenger_.*|http_server_.*|db_client_.*|process_.*"}')
|
|
labels = set()
|
|
for metric, _ in series:
|
|
labels |= set(metric)
|
|
for name, value in metric.items():
|
|
check(not UUID_RE.search(value), "UUID-like value in metric label %s=%s" % (name, value))
|
|
check(not labels & FORBIDDEN_LABELS, "forbidden metric labels: %s" % (labels & FORBIDDEN_LABELS))
|
|
log("PASS: %d series, label names: %s" % (len(series), sorted(labels)))
|
|
routes = sorted({m.get("http_route") for m, _ in series if m.get("http_route")})
|
|
log("http_route values (templates only): %s" % routes)
|
|
|
|
# ---------------------------------------------------------------- 5. metrics referenced by dashboard/alerts
|
|
time.sleep(12) # one metric export interval
|
|
for name in REQUIRED_METRICS:
|
|
check(obs.prom(name), "metric %s has no samples" % name)
|
|
log("PASS: required metrics present: %s" % ", ".join(REQUIRED_METRICS))
|
|
print("PASS: observability levels 3-4 (traces/logs/metrics correlation and privacy)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|