forked from mtusi2026/mtusi2026lab6
docs: finish lab 6 report and reproducible evidence
This commit is contained in:
@@ -31,7 +31,7 @@ def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
||||
parser.add_argument("criterion", choices=["provisioning", "signals", "pipeline"])
|
||||
parser.add_argument("criterion", choices=["provisioning", "signals", "pipeline", "dashboard"])
|
||||
args = parser.parse_args()
|
||||
|
||||
def query(path, **params):
|
||||
@@ -51,6 +51,24 @@ def main():
|
||||
print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"]))
|
||||
return
|
||||
|
||||
if args.criterion == "dashboard":
|
||||
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
|
||||
check(len(dashboard["panels"]) >= 17, "grade 4 dashboard panels missing")
|
||||
for panel in dashboard["panels"]:
|
||||
if panel["datasource"]["type"] != "prometheus":
|
||||
continue
|
||||
for target in panel["targets"]:
|
||||
result = source("prometheus", "/api/v1/query", query=target["expr"])
|
||||
check(result["status"] == "success", "PromQL query failed")
|
||||
print(panel["title"], "—", target["expr"], "— рядов:", len(result["data"]["result"]))
|
||||
expression = ('1 - (sum(increase(messenger_http_requests_total{http_status_class="5xx"}[30m])) '
|
||||
'or 0 * sum(increase(messenger_http_requests_total[30m]))) '
|
||||
'/ sum(increase(messenger_http_requests_total[30m]))')
|
||||
result = source("prometheus", "/api/v1/query", query=expression)["data"]["result"]
|
||||
check(bool(result), "SLI series missing")
|
||||
print("SLI за 30 минут:", result[0]["value"][1], "цель: >= 0.99")
|
||||
return
|
||||
|
||||
client = Client(args.base_url)
|
||||
start = time.time_ns()
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
@@ -67,19 +85,23 @@ def main():
|
||||
start=start, end=time.time_ns())["data"]["result"]
|
||||
if logs:
|
||||
trace_id = logs[0]["stream"]["trace_id"]
|
||||
trace = source("tempo", "/api/traces/" + trace_id)
|
||||
try:
|
||||
trace = source("tempo", "/api/traces/" + trace_id)
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code != 404:
|
||||
raise
|
||||
time.sleep(2)
|
||||
continue
|
||||
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
|
||||
producer = next((span for span in spans if span["kind"] == "SPAN_KIND_PRODUCER"), None)
|
||||
consumer = next((span for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"), None)
|
||||
if producer and consumer:
|
||||
api = next(span for span in spans if span["kind"] == "SPAN_KIND_SERVER")
|
||||
check(producer["parentSpanId"] == api["spanId"], "producer parent mismatch")
|
||||
check(consumer["parentSpanId"] == producer["spanId"], "consumer parent mismatch")
|
||||
by_id = {span["spanId"]: span for span in spans}
|
||||
for consumer in (span for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"):
|
||||
producer = by_id.get(consumer.get("parentSpanId"), {})
|
||||
api = by_id.get(producer.get("parentSpanId"), {})
|
||||
children = {span["name"] for span in spans if span.get("parentSpanId") == consumer["spanId"]}
|
||||
check({"postgresql finalize attachment", "s3 read staging and write variants"} <= children,
|
||||
"worker S3 and PostgreSQL spans missing")
|
||||
print("Upload → producer → consumer → S3/PostgreSQL, trace:", trace_id)
|
||||
return
|
||||
if (producer.get("kind") == "SPAN_KIND_PRODUCER" and api.get("kind") == "SPAN_KIND_SERVER"
|
||||
and {"postgresql finalize attachment", "s3 read staging and write variants"} <= children):
|
||||
print("Upload → producer → consumer → S3/PostgreSQL, trace:", trace_id)
|
||||
return
|
||||
time.sleep(2)
|
||||
raise Failure("upload trace did not include the worker pipeline")
|
||||
|
||||
@@ -132,7 +154,9 @@ def main():
|
||||
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
|
||||
span_id_base64 = base64.b64encode(bytes.fromhex(span_id)).decode()
|
||||
matched = [span for span in spans if span["spanId"] in (span_id, span_id_base64)]
|
||||
check(len(matched) == 1, "span ID from log missing in trace")
|
||||
if not matched:
|
||||
continue
|
||||
check(len(matched) == 1, "duplicate span ID in trace")
|
||||
span = matched[0]
|
||||
check(span["kind"] in (span_kind, "SPAN_KIND_SERVER" if span_kind == 2 else "SPAN_KIND_CONSUMER"), "wrong span kind")
|
||||
check(int(span["endTimeUnixNano"]) > int(span["startTimeUnixNano"]), "span duration missing")
|
||||
|
||||
@@ -1,58 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Сбор воспроизводимых доказательств уровня 3."""
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
||||
args = parser.parse_args()
|
||||
probe = [sys.executable, "scripts/check_observability.py", "--base-url", args.base_url,
|
||||
"--grafana-url", args.grafana_url]
|
||||
checks = [
|
||||
("grade3-stand", "Стенд, тома и provisioning",
|
||||
[["docker", "compose", "config", "--quiet"],
|
||||
["docker", "compose", "ps", "--all"],
|
||||
["docker", "compose", "config", "--volumes"],
|
||||
["docker", "compose", "run", "--rm", "--no-deps", "otel-collector", "validate", "--config=/etc/otelcol-contrib/config.yaml"],
|
||||
probe + ["provisioning"]],
|
||||
"Prometheus хранит временные ряды, Loki — журналы, Tempo — трассы. Collector принимает и маршрутизирует OTLP. Grafana читает данные и загружает dashboard из JSON."),
|
||||
("grade3-smoke", "Совместимость API при включённой телеметрии",
|
||||
[["make", "check"], ["make", "test", "BASE_URL=" + args.base_url]],
|
||||
"Smoke проверяет сессии, сообщения, ACL, идемпотентность и готовность вложения с оригиналом и thumbnail."),
|
||||
("grade3-signals", "Метрики, структурные логи и трассы API и worker",
|
||||
[probe + ["signals"]],
|
||||
"Контрольный POST сопоставляется с логом по X-Request-Id. Оба сервиса сопоставляются с Tempo по trace_id и span_id. Prometheus возвращает счётчики и histogram buckets. Проверка ищет утечки синтетических значений в полученных сигналах."),
|
||||
]
|
||||
failed = False
|
||||
for name, title, commands, explanation in checks:
|
||||
lines = [f"# {title}", "", "Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.",
|
||||
"", explanation, ""]
|
||||
for command in commands:
|
||||
try:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
text=True, timeout=180)
|
||||
code, output = result.returncode, result.stdout
|
||||
except subprocess.TimeoutExpired:
|
||||
code, output = 124, "Истёк лимит времени."
|
||||
except OSError:
|
||||
code, output = 127, "Команда недоступна."
|
||||
failed |= code != 0
|
||||
output = re.sub(r"(https?://[^\s?]+)\?[^\s]+", r"\1?[redacted]", output).strip()
|
||||
lines.extend([f"Команда `{shlex.join(command)}`. Код выхода **{code}**.", "", "```text", output or "(пусто)", "```", ""])
|
||||
path = ROOT / "evidence" / (name + ".md")
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
print(path.relative_to(ROOT))
|
||||
return int(failed)
|
||||
|
||||
"""Совместимая точка входа для итогового генератора."""
|
||||
from generate_evidence import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Сбор воспроизводимых доказательств уровней 3–4."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
||||
args = parser.parse_args()
|
||||
probe = [sys.executable, "scripts/check_observability.py", "--base-url", args.base_url,
|
||||
"--grafana-url", args.grafana_url]
|
||||
incident = [sys.executable, "scripts/check_observability_incidents.py", "--base-url", args.base_url,
|
||||
"--grafana-url", args.grafana_url]
|
||||
checks = [
|
||||
("grade3-stand", "Стенд, тома и provisioning",
|
||||
[["docker", "compose", "config", "--quiet"],
|
||||
["docker", "compose", "ps", "--all"],
|
||||
["docker", "compose", "config", "--volumes"],
|
||||
["docker", "compose", "run", "--rm", "--no-deps", "otel-collector", "validate", "--config=/etc/otelcol-contrib/config.yaml"],
|
||||
probe + ["provisioning"]],
|
||||
"Prometheus хранит временные ряды, Loki — журналы, Tempo — трассы. Collector принимает и маршрутизирует OTLP. Grafana читает данные и загружает dashboard из JSON."),
|
||||
("grade3-smoke", "Совместимость API при включённой телеметрии",
|
||||
[["make", "check"], ["make", "test", "BASE_URL=" + args.base_url]],
|
||||
"Smoke проверяет сессии, сообщения, ACL, идемпотентность и готовность вложения с оригиналом и thumbnail."),
|
||||
("grade3-signals", "Метрики, структурные логи и трассы API и worker",
|
||||
[probe + ["signals"]],
|
||||
"Контрольный POST сопоставляется с логом по X-Request-Id. Оба сервиса сопоставляются с Tempo по trace_id и span_id. Prometheus возвращает счётчики и histogram buckets. Проверка ищет утечки синтетических значений в полученных сигналах."),
|
||||
("grade4-pipeline", "Сквозная трасса и повторная попытка",
|
||||
[probe + ["pipeline"], incident + ["retry"]],
|
||||
"Tempo подтверждает родительскую связь HTTP, producer и consumer, операции S3 и PostgreSQL и две попытки одной загрузки."),
|
||||
("grade4-dashboard", "Метрики диагностики и SLO",
|
||||
[probe + ["dashboard"], ["docker", "compose", "exec", "-T", "prometheus", "promtool", "check", "rules", "/etc/prometheus/alerts.yaml"]],
|
||||
"Проверяются запросы панелей, SLI за 30 минут и синтаксис двух правил."),
|
||||
("grade4-queue", "Остановка и восстановление worker",
|
||||
[incident + ["worker"]],
|
||||
"Возраст незавершённой работы вызывает firing. Вложение достигает ready после восстановления worker, алерт переходит в resolved."),
|
||||
("grade4-database", "Задержка PostgreSQL и восстановление",
|
||||
[incident + ["database"]],
|
||||
"При блокировке таблицы растёт доля 5xx и алерт срабатывает. Задержку показывает дочерний спан PostgreSQL. После снятия блокировки алерт переходит в resolved."),
|
||||
("grade4-telemetry", "Недоступность Tempo",
|
||||
[incident + ["telemetry"]],
|
||||
"Проверяются сообщения и готовность вложения при остановленном Tempo, снимок памяти Collector и число отброшенных спанов по его журналу."),
|
||||
("grade4-load", "Короткая нагрузка на API",
|
||||
[["lscpu"], ["free", "-h"], ["docker", "compose", "version"],
|
||||
[sys.executable, "tests/load.py", "--base-url", args.base_url, "--duration", "30", "--concurrency", "4"],
|
||||
["docker", "compose", "stats", "--no-stream"]],
|
||||
"Один closed-loop прогон POST сообщений с четырьмя клиентами, отдельно от инцидентов. Снимок ресурсов сделан сразу после прогона."),
|
||||
]
|
||||
output_root = ROOT / "evidence/criteria"
|
||||
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ")
|
||||
run = output_root / "runs" / stamp
|
||||
run.mkdir(parents=True)
|
||||
summary = {}
|
||||
failed = False
|
||||
for name, title, commands, explanation in checks:
|
||||
lines = [f"# {title}", "", "Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.",
|
||||
"", explanation, ""]
|
||||
results = []
|
||||
summary[name] = results
|
||||
for index, command in enumerate(commands, 1):
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
text=True, timeout=240)
|
||||
code, output = result.returncode, result.stdout
|
||||
except subprocess.TimeoutExpired:
|
||||
code, output = 124, "Истёк лимит времени."
|
||||
except OSError:
|
||||
code, output = 127, "Команда недоступна."
|
||||
failed |= code != 0
|
||||
output = re.sub(r"(https?://[^\s?]+)\?[^\s]+", r"\1?[redacted]", output).strip()
|
||||
output = "\n".join(line.rstrip() for line in output.splitlines())
|
||||
raw = f"{name}-{index}.txt"
|
||||
(run / raw).write_text(output + "\n", encoding="utf-8")
|
||||
seconds = round(time.monotonic() - started, 2)
|
||||
results.append(dict(command=shlex.join(command), exit_code=code, seconds=seconds, output=raw))
|
||||
lines.extend([f"Команда `{shlex.join(command)}`. Код выхода **{code}**. Время {seconds} с.",
|
||||
"", f"[Вывод](runs/{stamp}/{raw})", "", "````text", output or "(пусто)", "````", ""])
|
||||
path = output_root / (name + ".md")
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
print(path.relative_to(ROOT), flush=True)
|
||||
index = ["# Доказательства критериев", "", f"[Сводка прогона {stamp}](runs/{stamp}/summary.json)", ""]
|
||||
for name, results in summary.items():
|
||||
state = "PASS" if all(result["exit_code"] == 0 for result in results) else "FAIL"
|
||||
index.append(f"- [{name}]({name}.md) — {state}")
|
||||
(output_root / "README.md").write_text("\n".join(index) + "\n", encoding="utf-8")
|
||||
(run / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return int(failed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user