Files
mtusi2026lab7/student-tests/obs_correlation.py
T
zemiles 9cc80d2529
Application / app (pull_request) Canceled after 0s
Skeleton contracts / contracts (pull_request) Canceled after 0s
ЛР 7 — БПИ2303 — Лазарев — на 3
Перенесена реализация lab6; бизнес-API без изменений, транспорт — HTTPS.

- Внешний HTTPS (public-ca) на 127.0.0.1:8443; HSTS, TLS >= 1.2.
- Обязательное mTLS LB -> каждая API-реплика: API проверяет цепочку
  internal-ca и SAN клиента (allowlist: lb); health только на 127.0.0.1.
- TLS с проверкой сервера ко всем зависимостям: PostgreSQL verify-full
  (pg_hba только hostssl), HTTPS S3, AMQPS, OTLP/HTTPS с client cert,
  mTLS Collector -> Prometheus/Loki/Tempo, Grafana -> datasources, scrape.
- OpenBao 2.7.2 не в dev-mode: Raft на volume, init/unseal оператором,
  auth/cert с ролью и policy на сервис, TTL 15 мин + renew/re-login.
- Отдельные роли PostgreSQL с грантами по таблицам; разные ключи S3 и
  пользователи брокера выдаются из OpenBao.
- Argon2id (m=19 MiB, t=2, p=1, замер на стенде) и rehash при входе.
- Случайный X-Instance-Id; hardening контейнеров (non-root, cap_drop ALL,
  no-new-privileges, read-only, лимиты); TCP_NODELAY на сокете API.
- Модель угроз, trust matrix, отрицательные тесты TLS/mTLS, API, OpenBao,
  hardening; evidence, в т.ч. прогон из чистого clone.
2026-11-18 23:59:35 +03:00

207 lines
13 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Lab6, уровни 3 и 4: корреляция сигналов на контролируемых запросах (всё через Grafana).
python3 student-tests/obs_correlation.py --output evidence/obs-correlation.json [--retry]
1. Сообщение: по X-Trace-Id ответа найден trace в Tempo (SERVER span маршрута-шаблона + db spans),
по X-Request-Id — структурный лог event=http.request в Loki с тем же trace_id/span_id.
2. Upload -> ready: ОДИН trace связывает API (POST .../attachments, db, s3 PutObject staging,
PRODUCER «create») -> relay (PRODUCER «attachments send», parent = create) -> worker (CONSUMER
«process», parent = send; db jobs.claim, fetch/transform, s3 PutObject x4, db jobs.complete,
s3 DeleteObject). traceparent в outbox совпадает с trace. Лог worker job.completed — с тем же trace_id.
3. --retry: worker пересоздаётся с WORKER_FAULT_INJECTION=1; fault-transient-1.png — 1-я попытка
получает временную ошибку (retry_scheduled), 2-я — ready; обе попытки и повторная публикация
relay находятся в том же trace. После теста worker возвращается в обычный режим.
4. Утечки: пароль, токен, текст сообщения и filename не встречаются ни в Loki, ни в трассах;
у метрик приложений нет labels user_id/chat_id/... и значений-UUID.
"""
import argparse
import json
import re
import sys
import time
import uuid
from pathlib import Path
from lab5lib import SAMPLE_PNG, Client, Failure, Log, check, ops, recreate, wait_status
import obslib
FORBIDDEN_LABELS = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "job_id", "url",
"url_full", "url_path", "filename", "text", "trace_id", "span_id"}
UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
def by_name(spans, name, service=None):
return [s for s in spans if s["name"] == name and (service is None or s["service"] == service)]
def one(spans, name, service=None):
found = by_name(spans, name, service)
check(found, "span %r (%s) not found" % (name, service))
return found[0]
def loki_lines(query, t0, retries=15):
for _ in range(retries):
lines = obslib.loki_query(query, t0 - 30, time.time() + 5, limit=200)
if lines:
return lines
time.sleep(2)
return []
def main():
p = argparse.ArgumentParser()
p.add_argument("--base-url", default=None)
p.add_argument("--output")
p.add_argument("--retry", action="store_true", help="also prove retry chain (recreates worker)")
args = p.parse_args()
log = Log("lab6 signal correlation: request -> trace -> log, upload -> relay -> worker")
c = Client(args.base_url, timeout=30)
t0 = time.time()
alice, bob, eve, chat = c.bootstrap()
marker = "lab6-marker-" + uuid.uuid4().hex
leak_strings = {"message_text": marker, "password": alice["registration"]["password"],
"token": alice["token"]}
result = {}
# ---------------------------------------------------------------- 1. сообщение
r = c.message(alice, chat, marker)
rid, tid, inst = r["headers"]["X-Request-Id"], r["headers"].get("X-Trace-Id"), r["headers"]["X-Instance-Id"]
check(tid and re.fullmatch(r"[0-9a-f]{32}", tid), "X-Trace-Id header missing")
log.step("message sent", request_id=rid, trace_id=tid, instance=inst)
spans = obslib.wait_trace(tid, lambda s: by_name(s, "db add_message"))
server = one(spans, "POST /api/v1/chats/{chat_id}/messages", "messenger-api")
db = one(spans, "db add_message")
check(server["kind"] == "SPAN_KIND_SERVER" and db["kind"] == "SPAN_KIND_CLIENT", "span kinds")
check(server["attrs"].get("http.route") == "/api/v1/chats/{chat_id}/messages", "route template in span")
check(server["attrs"].get("http.response.status_code") in (201, "201"), "status code in span")
check(server["instance"] == inst, "span instance = X-Instance-Id")
log.step("message trace in Tempo", spans=len(spans), server_ms=server["duration_ms"],
db_spans=[s["name"] for s in spans if s["name"].startswith("db ")])
result["message_trace_tree"] = obslib.tree(spans)
lines = loki_lines('{service_name="messenger-api"} | request_id="%s"' % rid, t0)
check(lines, "API log with request_id not found in Loki")
rec = lines[-1]
check(rec["labels"].get("trace_id") == tid, "log trace_id differs from X-Trace-Id")
check(rec["labels"].get("span_id") == server["span_id"], "log span_id differs from server span")
check(rec["labels"].get("event") == "http.request", "log event")
log.step("structured API log in Loki", event=rec["labels"].get("event"), trace_id=rec["labels"].get("trace_id"),
span_id=rec["labels"].get("span_id"), severity=rec["labels"].get("severity_text"),
service_name=rec["labels"].get("service_name"), line=rec["line"][:120])
result["message"] = {"request_id": rid, "trace_id": tid, "loki_labels": sorted(rec["labels"])}
# ---------------------------------------------------------------- 2. upload -> worker
fname = "leakcheck-%s.png" % uuid.uuid4().hex[:10]
leak_strings["filename"] = fname
resp = c.call("POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], fname), 202, actor=alice,
raw=SAMPLE_PNG, headers={"Content-Type": "image/png"}, schema="Attachment")
att, utid = resp["json"], resp["headers"]["X-Trace-Id"]
ready = wait_status(c, alice, att["id"], ("ready",), timeout=60)
log.step("upload accepted and ready", attachment_id=att["id"], trace_id=utid, status=ready["status"])
need = ["POST /api/v1/chats/{chat_id}/attachments", "attachment.process.v1 create", "attachments send",
"attachments.process.v1 process", "db jobs.complete"]
spans = obslib.wait_trace(utid, lambda s: all(by_name(s, n) for n in need) and by_name(s, "s3 DeleteObject"),
timeout=60)
srv = one(spans, need[0], "messenger-api")
create = one(spans, need[1], "messenger-api")
send = one(spans, need[2], "messenger-relay")
proc = one(spans, need[3], "messenger-worker")
check(create["kind"] == "SPAN_KIND_PRODUCER" and send["kind"] == "SPAN_KIND_PRODUCER", "producer kinds")
check(proc["kind"] == "SPAN_KIND_CONSUMER", "consumer kind")
check(send["parent_id"] == create["span_id"], "relay send span is child of API create span")
check(proc["parent_id"] == send["span_id"], "worker process span is child of relay send span")
puts = [s for s in by_name(spans, "s3 PutObject") if s["service"] == "messenger-worker"]
check(len(puts) == 4, "worker wrote original + 3 variants (got %d)" % len(puts))
check(by_name(spans, "s3 PutObject", "messenger-api"), "API staging PUT span")
services = sorted({s["service"] for s in spans})
check(services == ["messenger-api", "messenger-relay", "messenger-worker"], "services in trace %s" % services)
info = ops("inspect", att["id"])
job_id = info["jobs"][0]["id"]
tps = [o.get("traceparent") for o in info["outbox"]]
check(all(tp and tp.split("-")[1] == utid for tp in tps), "outbox traceparent belongs to upload trace")
gap_ms = round((proc["start"] - srv["end"]) * 1000, 1)
log.step("one trace upload -> relay -> worker", spans=len(spans), services=services, job_id=job_id,
outbox_traceparent=tps[0], process_parent_is_send=True, send_parent_is_create=True,
worker_s3_puts=len(puts), api_to_worker_gap_ms=gap_ms,
process_ms=proc["duration_ms"], outcome=proc["attrs"].get("messenger.job.outcome"))
result["upload_trace_tree"] = obslib.tree(spans)
wl = loki_lines('{service_name="messenger-worker"} | trace_id="%s"' % utid, t0)
events = [w["labels"].get("event") for w in wl]
check("job.claimed" in events and "job.completed" in events, "worker logs for job: %s" % events)
comp = [w for w in wl if w["labels"].get("event") == "job.completed"][0]
check(comp["labels"].get("job_id") == job_id, "job_id in worker log")
log.step("worker structured logs in Loki by trace_id", events=events, job_id=comp["labels"].get("job_id"),
span_id=comp["labels"].get("span_id"), line=comp["line"][:120])
result["upload"] = {"attachment_id": att["id"], "job_id": job_id, "trace_id": utid, "worker_log_events": events}
# ---------------------------------------------------------------- 3. retry в той же трассе
if args.retry:
env = {"WORKER_FAULT_INJECTION": "1", "JOB_RETRY_BASE_SECONDS": "1", "JOB_RETRY_MAX_SECONDS": "4"}
recreate("worker", env)
try:
resp = c.call("POST", "/api/v1/chats/%s/attachments?filename=fault-transient-1.png" % chat["id"], 202,
actor=alice, raw=SAMPLE_PNG, headers={"Content-Type": "image/png"}, schema="Attachment")
ratt, rtid = resp["json"], resp["headers"]["X-Trace-Id"]
wait_status(c, alice, ratt["id"], ("ready",), timeout=90)
spans = obslib.wait_trace(rtid, lambda s: len(by_name(s, "attachments.process.v1 process")) >= 2
and by_name(s, "db jobs.complete"), timeout=60)
procs = by_name(spans, "attachments.process.v1 process")
sends = by_name(spans, "attachments send")
creates = by_name(spans, "attachment.process.v1 create")
outcomes = [s["attrs"].get("messenger.job.outcome") for s in procs]
check(len(procs) == 2 and outcomes == ["retry", "ready"], "attempt outcomes %s" % outcomes)
check(len(sends) == 2 and len(creates) == 2, "two publications (initial + retry)")
retry_create = [s for s in creates if s["service"] == "messenger-worker"]
check(retry_create and retry_create[0]["parent_id"] in {x["span_id"] for x in spans}, "retry create in worker")
check(sends[1]["parent_id"] == retry_create[0]["span_id"], "second send is child of worker retry create")
check(procs[1]["parent_id"] == sends[1]["span_id"], "second attempt consumes second publication")
log.step("retry chain in one trace", trace_id=rtid, attempts=len(procs), outcomes=outcomes,
retry_delay_s=round(procs[1]["start"] - procs[0]["end"], 2),
error_type=procs[0]["attrs"].get("error.type"))
result["retry_trace_tree"] = obslib.tree(spans)
result["retry"] = {"trace_id": rtid, "outcomes": outcomes}
finally:
recreate("worker")
log.step("worker restored without fault injection")
# ---------------------------------------------------------------- 4. утечки
time.sleep(3)
leaks = {}
for name, value in leak_strings.items():
hits = obslib.loki_query('{service_name=~"messenger-.+"} |= `%s`' % value, t0 - 30, time.time() + 5, limit=5)
leaks["loki:" + name] = len(hits)
blobs = json.dumps([obslib.tempo_trace(t) for t in (tid, utid)], ensure_ascii=False)
for name, value in leak_strings.items():
leaks["tempo:" + name] = blobs.count(value)
check(all(v == 0 for v in leaks.values()), "telemetry leak: %s" % leaks)
labels = set(obslib.prom("labels", **{"match[]": '{job=~"messenger-.+"}'})["data"])
bad_labels = sorted(labels & FORBIDDEN_LABELS)
uuid_values = {}
for lab in sorted(labels - {"__name__"}):
vals = obslib.prom("label/%s/values" % lab, **{"match[]": '{job=~"messenger-.+"}'})["data"]
u = [v for v in vals if UUID_RE.search(v)]
if u:
uuid_values[lab] = u[:3]
check(not bad_labels and not uuid_values, "high-cardinality labels: %s %s" % (bad_labels, uuid_values))
log.step("no secrets/bodies in telemetry, no ID labels in metrics", leaks=leaks, metric_labels=sorted(labels))
result["leaks"] = leaks
result["metric_labels"] = sorted(labels)
log.save(args.output, **result)
if args.output:
Path(args.output).with_suffix(".trees.txt").write_text(
"\n".join(["# message trace %s" % tid] + result["message_trace_tree"] + ["", "# upload trace %s" % utid]
+ result["upload_trace_tree"] + (["", "# retry trace %s" % result["retry"]["trace_id"]]
+ result["retry_trace_tree"] if args.retry else [])) + "\n",
encoding="utf-8")
print("PASS: correlation request->trace->log and upload->relay->worker%s" % (" incl. retry" if args.retry else ""))
return 0
if __name__ == "__main__":
try:
sys.exit(main())
except Failure as exc:
print("FAIL:", exc)
sys.exit(1)