forked from mtusi2026/mtusi2026lab7
Перенесена реализация lab6; бизнес-API без изменений, транспорт — HTTPS. - Внешний HTTPS (public-ca) на 127.0.0.1:8443; HSTS, TLS >= 1.2. - Обязательное mTLS LB -> каждая API-реплика: API проверяет цепочку internal-ca и SAN клиента (allowlist: lb); health только на 127.0.0.1. - TLS с проверкой сервера ко всем зависимостям: PostgreSQL verify-full (pg_hba только hostssl), HTTPS S3, AMQPS, OTLP/HTTPS с client cert, mTLS Collector -> Prometheus/Loki/Tempo, Grafana -> datasources, scrape. - OpenBao 2.7.2 не в dev-mode: Raft на volume, init/unseal оператором, auth/cert с ролью и policy на сервис, TTL 15 мин + renew/re-login. - Отдельные роли PostgreSQL с грантами по таблицам; разные ключи S3 и пользователи брокера выдаются из OpenBao. - Argon2id (m=19 MiB, t=2, p=1, замер на стенде) и rehash при входе. - Случайный X-Instance-Id; hardening контейнеров (non-root, cap_drop ALL, no-new-privileges, read-only, лимиты); TCP_NODELAY на сокете API. - Модель угроз, trust matrix, отрицательные тесты TLS/mTLS, API, OpenBao, hardening; evidence, в т.ч. прогон из чистого clone.
207 lines
13 KiB
Python
207 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""Lab6, уровни 3 и 4: корреляция сигналов на контролируемых запросах (всё через Grafana).
|
||
|
||
python3 student-tests/obs_correlation.py --output evidence/obs-correlation.json [--retry]
|
||
|
||
1. Сообщение: по X-Trace-Id ответа найден trace в Tempo (SERVER span маршрута-шаблона + db spans),
|
||
по X-Request-Id — структурный лог event=http.request в Loki с тем же trace_id/span_id.
|
||
2. Upload -> ready: ОДИН trace связывает API (POST .../attachments, db, s3 PutObject staging,
|
||
PRODUCER «create») -> relay (PRODUCER «attachments send», parent = create) -> worker (CONSUMER
|
||
«process», parent = send; db jobs.claim, fetch/transform, s3 PutObject x4, db jobs.complete,
|
||
s3 DeleteObject). traceparent в outbox совпадает с trace. Лог worker job.completed — с тем же trace_id.
|
||
3. --retry: worker пересоздаётся с WORKER_FAULT_INJECTION=1; fault-transient-1.png — 1-я попытка
|
||
получает временную ошибку (retry_scheduled), 2-я — ready; обе попытки и повторная публикация
|
||
relay находятся в том же trace. После теста worker возвращается в обычный режим.
|
||
4. Утечки: пароль, токен, текст сообщения и filename не встречаются ни в Loki, ни в трассах;
|
||
у метрик приложений нет labels user_id/chat_id/... и значений-UUID.
|
||
"""
|
||
import argparse
|
||
import json
|
||
import re
|
||
import sys
|
||
import time
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
from lab5lib import SAMPLE_PNG, Client, Failure, Log, check, ops, recreate, wait_status
|
||
import obslib
|
||
|
||
FORBIDDEN_LABELS = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "job_id", "url",
|
||
"url_full", "url_path", "filename", "text", "trace_id", "span_id"}
|
||
UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
|
||
|
||
|
||
def by_name(spans, name, service=None):
|
||
return [s for s in spans if s["name"] == name and (service is None or s["service"] == service)]
|
||
|
||
|
||
def one(spans, name, service=None):
|
||
found = by_name(spans, name, service)
|
||
check(found, "span %r (%s) not found" % (name, service))
|
||
return found[0]
|
||
|
||
|
||
def loki_lines(query, t0, retries=15):
|
||
for _ in range(retries):
|
||
lines = obslib.loki_query(query, t0 - 30, time.time() + 5, limit=200)
|
||
if lines:
|
||
return lines
|
||
time.sleep(2)
|
||
return []
|
||
|
||
|
||
def main():
|
||
p = argparse.ArgumentParser()
|
||
p.add_argument("--base-url", default=None)
|
||
p.add_argument("--output")
|
||
p.add_argument("--retry", action="store_true", help="also prove retry chain (recreates worker)")
|
||
args = p.parse_args()
|
||
log = Log("lab6 signal correlation: request -> trace -> log, upload -> relay -> worker")
|
||
c = Client(args.base_url, timeout=30)
|
||
t0 = time.time()
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
marker = "lab6-marker-" + uuid.uuid4().hex
|
||
leak_strings = {"message_text": marker, "password": alice["registration"]["password"],
|
||
"token": alice["token"]}
|
||
result = {}
|
||
|
||
# ---------------------------------------------------------------- 1. сообщение
|
||
r = c.message(alice, chat, marker)
|
||
rid, tid, inst = r["headers"]["X-Request-Id"], r["headers"].get("X-Trace-Id"), r["headers"]["X-Instance-Id"]
|
||
check(tid and re.fullmatch(r"[0-9a-f]{32}", tid), "X-Trace-Id header missing")
|
||
log.step("message sent", request_id=rid, trace_id=tid, instance=inst)
|
||
spans = obslib.wait_trace(tid, lambda s: by_name(s, "db add_message"))
|
||
server = one(spans, "POST /api/v1/chats/{chat_id}/messages", "messenger-api")
|
||
db = one(spans, "db add_message")
|
||
check(server["kind"] == "SPAN_KIND_SERVER" and db["kind"] == "SPAN_KIND_CLIENT", "span kinds")
|
||
check(server["attrs"].get("http.route") == "/api/v1/chats/{chat_id}/messages", "route template in span")
|
||
check(server["attrs"].get("http.response.status_code") in (201, "201"), "status code in span")
|
||
check(server["instance"] == inst, "span instance = X-Instance-Id")
|
||
log.step("message trace in Tempo", spans=len(spans), server_ms=server["duration_ms"],
|
||
db_spans=[s["name"] for s in spans if s["name"].startswith("db ")])
|
||
result["message_trace_tree"] = obslib.tree(spans)
|
||
lines = loki_lines('{service_name="messenger-api"} | request_id="%s"' % rid, t0)
|
||
check(lines, "API log with request_id not found in Loki")
|
||
rec = lines[-1]
|
||
check(rec["labels"].get("trace_id") == tid, "log trace_id differs from X-Trace-Id")
|
||
check(rec["labels"].get("span_id") == server["span_id"], "log span_id differs from server span")
|
||
check(rec["labels"].get("event") == "http.request", "log event")
|
||
log.step("structured API log in Loki", event=rec["labels"].get("event"), trace_id=rec["labels"].get("trace_id"),
|
||
span_id=rec["labels"].get("span_id"), severity=rec["labels"].get("severity_text"),
|
||
service_name=rec["labels"].get("service_name"), line=rec["line"][:120])
|
||
result["message"] = {"request_id": rid, "trace_id": tid, "loki_labels": sorted(rec["labels"])}
|
||
|
||
# ---------------------------------------------------------------- 2. upload -> worker
|
||
fname = "leakcheck-%s.png" % uuid.uuid4().hex[:10]
|
||
leak_strings["filename"] = fname
|
||
resp = c.call("POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], fname), 202, actor=alice,
|
||
raw=SAMPLE_PNG, headers={"Content-Type": "image/png"}, schema="Attachment")
|
||
att, utid = resp["json"], resp["headers"]["X-Trace-Id"]
|
||
ready = wait_status(c, alice, att["id"], ("ready",), timeout=60)
|
||
log.step("upload accepted and ready", attachment_id=att["id"], trace_id=utid, status=ready["status"])
|
||
need = ["POST /api/v1/chats/{chat_id}/attachments", "attachment.process.v1 create", "attachments send",
|
||
"attachments.process.v1 process", "db jobs.complete"]
|
||
spans = obslib.wait_trace(utid, lambda s: all(by_name(s, n) for n in need) and by_name(s, "s3 DeleteObject"),
|
||
timeout=60)
|
||
srv = one(spans, need[0], "messenger-api")
|
||
create = one(spans, need[1], "messenger-api")
|
||
send = one(spans, need[2], "messenger-relay")
|
||
proc = one(spans, need[3], "messenger-worker")
|
||
check(create["kind"] == "SPAN_KIND_PRODUCER" and send["kind"] == "SPAN_KIND_PRODUCER", "producer kinds")
|
||
check(proc["kind"] == "SPAN_KIND_CONSUMER", "consumer kind")
|
||
check(send["parent_id"] == create["span_id"], "relay send span is child of API create span")
|
||
check(proc["parent_id"] == send["span_id"], "worker process span is child of relay send span")
|
||
puts = [s for s in by_name(spans, "s3 PutObject") if s["service"] == "messenger-worker"]
|
||
check(len(puts) == 4, "worker wrote original + 3 variants (got %d)" % len(puts))
|
||
check(by_name(spans, "s3 PutObject", "messenger-api"), "API staging PUT span")
|
||
services = sorted({s["service"] for s in spans})
|
||
check(services == ["messenger-api", "messenger-relay", "messenger-worker"], "services in trace %s" % services)
|
||
info = ops("inspect", att["id"])
|
||
job_id = info["jobs"][0]["id"]
|
||
tps = [o.get("traceparent") for o in info["outbox"]]
|
||
check(all(tp and tp.split("-")[1] == utid for tp in tps), "outbox traceparent belongs to upload trace")
|
||
gap_ms = round((proc["start"] - srv["end"]) * 1000, 1)
|
||
log.step("one trace upload -> relay -> worker", spans=len(spans), services=services, job_id=job_id,
|
||
outbox_traceparent=tps[0], process_parent_is_send=True, send_parent_is_create=True,
|
||
worker_s3_puts=len(puts), api_to_worker_gap_ms=gap_ms,
|
||
process_ms=proc["duration_ms"], outcome=proc["attrs"].get("messenger.job.outcome"))
|
||
result["upload_trace_tree"] = obslib.tree(spans)
|
||
wl = loki_lines('{service_name="messenger-worker"} | trace_id="%s"' % utid, t0)
|
||
events = [w["labels"].get("event") for w in wl]
|
||
check("job.claimed" in events and "job.completed" in events, "worker logs for job: %s" % events)
|
||
comp = [w for w in wl if w["labels"].get("event") == "job.completed"][0]
|
||
check(comp["labels"].get("job_id") == job_id, "job_id in worker log")
|
||
log.step("worker structured logs in Loki by trace_id", events=events, job_id=comp["labels"].get("job_id"),
|
||
span_id=comp["labels"].get("span_id"), line=comp["line"][:120])
|
||
result["upload"] = {"attachment_id": att["id"], "job_id": job_id, "trace_id": utid, "worker_log_events": events}
|
||
|
||
# ---------------------------------------------------------------- 3. retry в той же трассе
|
||
if args.retry:
|
||
env = {"WORKER_FAULT_INJECTION": "1", "JOB_RETRY_BASE_SECONDS": "1", "JOB_RETRY_MAX_SECONDS": "4"}
|
||
recreate("worker", env)
|
||
try:
|
||
resp = c.call("POST", "/api/v1/chats/%s/attachments?filename=fault-transient-1.png" % chat["id"], 202,
|
||
actor=alice, raw=SAMPLE_PNG, headers={"Content-Type": "image/png"}, schema="Attachment")
|
||
ratt, rtid = resp["json"], resp["headers"]["X-Trace-Id"]
|
||
wait_status(c, alice, ratt["id"], ("ready",), timeout=90)
|
||
spans = obslib.wait_trace(rtid, lambda s: len(by_name(s, "attachments.process.v1 process")) >= 2
|
||
and by_name(s, "db jobs.complete"), timeout=60)
|
||
procs = by_name(spans, "attachments.process.v1 process")
|
||
sends = by_name(spans, "attachments send")
|
||
creates = by_name(spans, "attachment.process.v1 create")
|
||
outcomes = [s["attrs"].get("messenger.job.outcome") for s in procs]
|
||
check(len(procs) == 2 and outcomes == ["retry", "ready"], "attempt outcomes %s" % outcomes)
|
||
check(len(sends) == 2 and len(creates) == 2, "two publications (initial + retry)")
|
||
retry_create = [s for s in creates if s["service"] == "messenger-worker"]
|
||
check(retry_create and retry_create[0]["parent_id"] in {x["span_id"] for x in spans}, "retry create in worker")
|
||
check(sends[1]["parent_id"] == retry_create[0]["span_id"], "second send is child of worker retry create")
|
||
check(procs[1]["parent_id"] == sends[1]["span_id"], "second attempt consumes second publication")
|
||
log.step("retry chain in one trace", trace_id=rtid, attempts=len(procs), outcomes=outcomes,
|
||
retry_delay_s=round(procs[1]["start"] - procs[0]["end"], 2),
|
||
error_type=procs[0]["attrs"].get("error.type"))
|
||
result["retry_trace_tree"] = obslib.tree(spans)
|
||
result["retry"] = {"trace_id": rtid, "outcomes": outcomes}
|
||
finally:
|
||
recreate("worker")
|
||
log.step("worker restored without fault injection")
|
||
|
||
# ---------------------------------------------------------------- 4. утечки
|
||
time.sleep(3)
|
||
leaks = {}
|
||
for name, value in leak_strings.items():
|
||
hits = obslib.loki_query('{service_name=~"messenger-.+"} |= `%s`' % value, t0 - 30, time.time() + 5, limit=5)
|
||
leaks["loki:" + name] = len(hits)
|
||
blobs = json.dumps([obslib.tempo_trace(t) for t in (tid, utid)], ensure_ascii=False)
|
||
for name, value in leak_strings.items():
|
||
leaks["tempo:" + name] = blobs.count(value)
|
||
check(all(v == 0 for v in leaks.values()), "telemetry leak: %s" % leaks)
|
||
labels = set(obslib.prom("labels", **{"match[]": '{job=~"messenger-.+"}'})["data"])
|
||
bad_labels = sorted(labels & FORBIDDEN_LABELS)
|
||
uuid_values = {}
|
||
for lab in sorted(labels - {"__name__"}):
|
||
vals = obslib.prom("label/%s/values" % lab, **{"match[]": '{job=~"messenger-.+"}'})["data"]
|
||
u = [v for v in vals if UUID_RE.search(v)]
|
||
if u:
|
||
uuid_values[lab] = u[:3]
|
||
check(not bad_labels and not uuid_values, "high-cardinality labels: %s %s" % (bad_labels, uuid_values))
|
||
log.step("no secrets/bodies in telemetry, no ID labels in metrics", leaks=leaks, metric_labels=sorted(labels))
|
||
result["leaks"] = leaks
|
||
result["metric_labels"] = sorted(labels)
|
||
log.save(args.output, **result)
|
||
if args.output:
|
||
Path(args.output).with_suffix(".trees.txt").write_text(
|
||
"\n".join(["# message trace %s" % tid] + result["message_trace_tree"] + ["", "# upload trace %s" % utid]
|
||
+ result["upload_trace_tree"] + (["", "# retry trace %s" % result["retry"]["trace_id"]]
|
||
+ result["retry_trace_tree"] if args.retry else [])) + "\n",
|
||
encoding="utf-8")
|
||
print("PASS: correlation request->trace->log and upload->relay->worker%s" % (" incl. retry" if args.retry else ""))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
sys.exit(main())
|
||
except Failure as exc:
|
||
print("FAIL:", exc)
|
||
sys.exit(1)
|