This commit is contained in:
k00kzaAntonovaIra
2026-11-22 19:55:50 +03:00
parent 470d9d6f1e
commit 2d10a9b788
186 changed files with 15732 additions and 425 deletions
+1
View File
@@ -0,0 +1 @@
"""Команды управления учебным стендом."""
+59 -35
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Offline structural checks; full schema validation is a separate target."""
import ast
import json
from pathlib import Path
@@ -7,48 +8,71 @@ import re
ROOT = Path(__file__).resolve().parents[1]
def main():
lab = json.loads((ROOT/'lab.json').read_text())['number']
spec = json.loads((ROOT/'contracts/openapi.json').read_text())
assert spec['openapi']=='3.1.0'
assert spec['info']['version']=='1.%d.0'%lab
lab = json.loads((ROOT / "lab.json").read_text(encoding="utf-8"))["number"]
spec = json.loads((ROOT / "contracts/openapi.json").read_text(encoding="utf-8"))
assert spec["openapi"] == "3.1.0"
assert spec["info"]["version"] == "1.%d.0" % lab
operation_ids = set()
def walk(value):
if isinstance(value,dict):
if '$ref' in value:
pointer = value['$ref']
assert pointer.startswith('#/'), 'unexpected external reference'
if isinstance(value, dict):
if "$ref" in value:
pointer = value["$ref"]
assert pointer.startswith("#/"), "unexpected external reference"
target = spec
for part in pointer[2:].split('/'):
target = target[part.replace('~1','/').replace('~0','~')]
for part in pointer[2:].split("/"):
target = target[part.replace("~1", "/").replace("~0", "~")]
for v in value.values():
walk(v)
elif isinstance(value,list):
elif isinstance(value, list):
for v in value:
walk(v)
walk(spec)
for route,methods in spec['paths'].items():
placeholders = set(re.findall(r'\{([^}]+)\}',route))
for method,op in methods.items():
assert op['operationId'] not in operation_ids, 'duplicate operationId'
operation_ids.add(op['operationId'])
params = {p['name'] for p in op.get('parameters',[]) if p['in']=='path' and p.get('required')}
assert placeholders==params, 'path parameter mismatch: '+route
for response in op['responses'].values():
assert 'X-Request-Id' in response['headers']
if lab>=3:
assert 'X-Instance-Id' in response['headers']
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
ast.parse(file.read_text(),filename=str(file))
for file in ROOT.rglob('*.json'):
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
continue
json.loads(file.read_text())
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
assert (ROOT/required).is_file(), 'missing '+required
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
if __name__=='__main__':
walk(spec)
for route, methods in spec["paths"].items():
placeholders = set(re.findall(r"\{([^}]+)\}", route))
for method, op in methods.items():
assert op["operationId"] not in operation_ids, "duplicate operationId"
operation_ids.add(op["operationId"])
params = {
p["name"]
for p in op.get("parameters", [])
if p["in"] == "path" and p.get("required")
}
assert placeholders == params, "path parameter mismatch: " + route
for response in op["responses"].values():
assert "X-Request-Id" in response["headers"]
if lab >= 3:
assert "X-Instance-Id" in response["headers"]
assert ("/api/v1/auth/sessions" in spec["paths"]) == (lab >= 2)
assert ("/api/v1/attachments/{attachment_id}" in spec["paths"]) == (lab >= 4)
for file in list((ROOT / "tests").glob("*.py")) + list(
(ROOT / "scripts").glob("*.py")
):
ast.parse(file.read_text(encoding="utf-8"), filename=str(file))
for file in ROOT.rglob("*.json"):
if any(
part in {".git", ".venv", "node_modules", "vendor", "output"}
for part in file.parts
):
continue
json.loads(file.read_text(encoding="utf-8"))
for required in (
"README.md",
"COURSE.md",
"CONTRIBUTING.md",
"REPORT.md",
"contracts/README.md",
"tests/smoke.py",
"tests/README.md",
):
assert (ROOT / required).is_file(), "missing " + required
print(
"PASS: offline skeleton structure, local refs and Python syntax; application NOT tested."
)
if __name__ == "__main__":
main()
+431
View File
@@ -0,0 +1,431 @@
"""Fault/recovery очереди на явно выбранном Compose-стенде; только синтетические данные."""
import argparse
from datetime import datetime
import hashlib
from io import BytesIO
import json
from pathlib import Path
import subprocess
import time
from uuid import UUID, uuid4
from PIL import Image
from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compose
from scripts.deployment import ComposeProject
from tests.client import Client, check
PROBE = """
import asyncio, hashlib, json
from uuid import UUID
from sqlalchemy import select
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.models.job import AttachmentJob
from app.models.attachment import Attachment
from app.storage.s3 import create_s3_storage
async def main():
settings=Settings(); engine=create_db_engine(settings); storage=create_s3_storage(settings.s3)
try:
async with create_session_factory(engine)() as session:
job, att=(await session.execute(select(AttachmentJob,Attachment).join(Attachment,Attachment.id==AttachmentJob.attachment_id).where(Attachment.id==UUID(ATTACHMENT_ID)))).one()
result=dict(job_id=str(job.id),status=att.status,attempt=job.attempt,published_attempt=job.published_attempt,limit=job.attempt_limit,updated_at=att.updated_at,next_attempt_at=job.next_attempt_at,last_error_code=job.last_error_code,permanent=job.last_error_permanent)
objects=[]
for prefix in ('originals/'+UUID(ATTACHMENT_ID).hex+'/', 'variants/'+UUID(ATTACHMENT_ID).hex+'/'):
entries=await asyncio.to_thread(lambda: list(storage._client.list_objects(storage.settings.bucket,prefix=prefix,recursive=True)))
for obj in entries:
stream=await storage.open_object(obj.object_name); checksum=hashlib.sha256()
try:
async for chunk in stream: checksum.update(chunk)
finally: await stream.aclose()
objects.append(dict(name=obj.object_name.split('/')[-1],size=obj.size,sha256=checksum.hexdigest(),last_modified=obj.last_modified))
result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
finally: await storage.aclose();await engine.dispose()
asyncio.run(main())
"""
def probe(attachment_id):
code = "ATTACHMENT_ID = " + repr(str(UUID(attachment_id))) + "\n" + PROBE
result = subprocess.run(
COMPOSE + ["exec", "-T", "api", "python", "-"],
input=code,
encoding="utf-8",
capture_output=True,
cwd=ROOT,
timeout=30,
)
check(result.returncode == 0, "Probe failed; diagnostic output hidden")
return json.loads(result.stdout)
def admin(*arguments, expected=0):
result = compose(
"run",
"--rm",
"--no-deps",
"worker",
"python",
"-m",
"app.core.job_admin",
*arguments,
)
check(result.returncode == expected, "Unexpected operator command exit status")
return json.loads(result.stdout)
def verify_result(client, actor, attachment, original):
ready = client.wait_ready(actor, attachment, wait_seconds=180)
check(ready["id"] == attachment["id"], "Attachment identity changed")
path = "/api/v1/attachments/" + ready["id"] + "/content"
check(
client.call("GET", path, actor=actor)["bytes"] == original, "Original changed"
)
check(
{v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"},
"Incomplete pipeline",
)
for variant in ready["variants"]:
data = client.call("GET", path + "?variant=" + variant["name"], actor=actor)[
"bytes"
]
check(
hashlib.sha256(data).hexdigest() == variant["sha256"], "Checksum mismatch"
)
with Image.open(BytesIO(data)) as image:
check(
image.size == (variant["width"], variant["height"]),
"Metadata dimensions mismatch",
)
check(
image.format == ("WEBP" if variant["name"] == "optimized" else "JPEG"),
"Wrong format",
)
check(
not image.getexif()
and not image.info.get("icc_profile")
and not image.info.get("xmp"),
"Metadata leaked",
)
if variant["name"] == "crop":
check(image.size == (128, 128), "Wrong crop size")
snapshot = probe(ready["id"])
check(
len(snapshot["objects"]) == 4,
"Expected exactly one original and three variants",
)
return snapshot
def crash(client, actor, chat, mode):
name = "messenger-worker-fault-" + uuid4().hex
try:
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
result = compose(
"run",
"--rm",
"--no-deps",
"--name",
name,
"worker",
"python",
"-m",
"app.core.job_worker",
"--" + mode,
attachment["id"],
)
phase = (
"after_object:optimized" if mode == "crash-after-object" else "before_ack"
)
marker = json.dumps({"fault": phase, "attachment_id": attachment["id"]})
check(
result.returncode == 137 and marker in result.stdout,
"Targeted crash not observed",
)
before = probe(attachment["id"])
check(
before["status"]
== ("processing" if mode == "crash-after-object" else "ready"),
"Unexpected crash state",
)
check(len(before["objects"]) == 4, "Crash must occur after all four PUTs")
finally:
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
if mode == "crash-before-ack":
# Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
wait_queue_empty()
after = probe(attachment["id"])
check(after == before, "Ready duplicate rewrote result")
else:
check(
after["attempt"] == before["attempt"] + 1,
"Lease did not recover the interrupted attempt",
)
check(
[(x["name"], x["sha256"]) for x in before["objects"]]
== [(x["name"], x["sha256"]) for x in after["objects"]],
"Replay changed object content",
)
return dict(
scenario=mode,
attachment_id=attachment["id"],
exit_code=137,
before=before,
after=after,
)
def retry_replay(client, actor, chat, permanent, on_queued=None):
name = "messenger-retry-fault-" + uuid4().hex
samples = []
configuration = ComposeProject().config["services"]["worker"]["environment"]
retry_base = float(configuration["JOB_RETRY_SECONDS"])
retry_max = float(configuration["JOB_RETRY_MAX_SECONDS"])
try:
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
if on_queued is not None:
on_queued(attachment)
mode = "fail-permanent" if permanent else "fail-transient"
require_compose(
"run",
"-d",
"--no-deps",
"--name",
name,
"worker",
"python",
"-m",
"app.core.job_worker",
"--" + mode,
attachment["id"],
)
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
sample = probe(attachment["id"])
if (
sample["attempt"]
and sample["status"] in {"queued", "failed"}
and (not samples or samples[-1]["attempt"] != sample["attempt"])
):
samples.append(sample)
if sample["status"] == "failed":
break
time.sleep(0.5)
check(sample["status"] == "failed", "Retry never reached failed")
check(
sample["attempt"] == (1 if permanent else sample["limit"]),
"Incorrect attempt budget",
)
check(sample["permanent"] == permanent, "Incorrect error classification")
if not permanent:
check(len(samples) == sample["limit"], "Missed retry samples")
for entry in samples:
delay = (
datetime.fromisoformat(entry["next_attempt_at"])
- datetime.fromisoformat(entry["updated_at"])
).total_seconds()
ceiling = min(retry_max, retry_base * 2 ** (entry["attempt"] - 1))
check(
ceiling / 2 <= delay <= ceiling,
"Retry delay outside configured jitter bounds",
)
client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=actor,
expected=409,
)
job_id = sample["job_id"]
request_id = str(uuid4())
listing = admin("list-failed")
check(
any(row["job_id"] == job_id for row in listing),
"Job missing from failed registry",
)
subprocess.run(
["docker", "rm", "-f", name], capture_output=True, timeout=30, check=True
)
first = admin(
"replay",
job_id,
"--request-id",
request_id,
"--actor",
"verification",
"--reason",
"dependency_restored",
)
check(first["status"] == "queued", "Replay did not queue job")
finally:
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
repeat = admin(
"replay",
job_id,
"--request-id",
request_id,
"--actor",
"verification",
"--reason",
"dependency_restored",
)
check(repeat["already_applied"], "Replay request id not idempotent")
refused = admin(
"replay",
job_id,
"--request-id",
str(uuid4()),
"--actor",
"verification",
"--reason",
"manual_review",
expected=2,
)
check(refused["error_code"] == "job_not_failed", "Successful job was replayed")
check(probe(attachment["id"]) == after, "Operator replay altered ready job")
check(len(admin("history", job_id)) == 1, "Incorrect audit count")
return dict(
scenario=mode,
attachment_id=attachment["id"],
samples=samples,
after=after,
replay_idempotent=True,
)
def queue_state():
output = require_compose(
"exec",
"-T",
"broker",
"rabbitmqctl",
"list_queues",
"--formatter=json",
"name",
"messages_ready",
"messages_unacknowledged",
"consumers",
"durable",
)
return next(row for row in json.loads(output) if row["name"] == "attachment-jobs")
def wait_queue_empty():
deadline = time.monotonic() + 20
while time.monotonic() < deadline:
state = queue_state()
if state["messages_ready"] == 0 and state["messages_unacknowledged"] == 0:
return state
time.sleep(0.5)
check(False, "RabbitMQ did not acknowledge all completed deliveries")
def persistence(client, actor, chat):
try:
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
client.message(actor, chat, "", attachments=[attachment["id"]])
client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=actor,
expected=409,
)
time.sleep(2)
queued_before = queue_state()
check(
queued_before["messages_ready"] >= 1 and queued_before["durable"],
"No durable queued message before restart",
)
result = subprocess.run(
[str(ROOT / "scripts/restart.sh")],
cwd=ROOT,
capture_output=True,
timeout=180,
)
check(result.returncode == 0, "Restart hook failed")
check(
"worker"
not in require_compose(
"ps", "--status", "running", "--services"
).splitlines(),
"Restart started deliberately stopped worker",
)
queued_after = queue_state()
check(
queued_after["messages_ready"] >= 1 and queued_after["durable"],
"Broker lost queued delivery across restart",
)
client.message(actor, chat, "session and chat survived broker/API restart")
finally:
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
return dict(
scenario="worker_stopped_restart_all",
attachment_id=attachment["id"],
queue_before=queued_before,
queue_after=queued_after,
after=after,
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--ca-file")
parser.add_argument(
"--scenario",
choices=[
"all",
"persistence",
"crash-after-object",
"crash-before-ack",
"fail-transient",
"fail-permanent",
],
default="all",
)
parser.add_argument("--output", default="evidence/jobs.json")
args = parser.parse_args()
client = Client(args.base_url, args.ca_file, timeout=40)
aliases = {r.instance_id for r in ComposeProject().replicas() if r.running}
check(
client.call("GET", "/health/live", schema="Health")["headers"].get(
"X-Instance-Id"
)
in aliases,
"Wrong Compose project for BASE_URL",
)
actor, _, _, chat = client.bootstrap()
results = []
operations = {
"persistence": persistence,
"crash-after-object": lambda *x: crash(*x, "crash-after-object"),
"crash-before-ack": lambda *x: crash(*x, "crash-before-ack"),
"fail-transient": lambda *x: retry_replay(*x, False),
"fail-permanent": lambda *x: retry_replay(*x, True),
}
selected = (
operations.values() if args.scenario == "all" else [operations[args.scenario]]
)
for operation in selected:
result = operation(client, actor, chat)
results.append(result)
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text(
json.dumps(dict(status="in_progress", results=results), indent=2) + "\n"
)
print("PASS: " + result["scenario"], flush=True)
Path(args.output).write_text(
json.dumps(dict(status="passed", results=results), indent=2) + "\n"
)
if __name__ == "__main__":
main()
+728
View File
@@ -0,0 +1,728 @@
"""Проверка наблюдаемости через реальные запросы Grafana к трём backend.
--incidents явно разрешает ограниченные отказы на выбранном Compose-проекте.
В evidence сохраняются только агрегаты и выбранные технические поля.
"""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
import math
import subprocess
import threading
from datetime import datetime, timezone
import json
from pathlib import Path
import time
import urllib.parse
import urllib.request
from uuid import uuid4
from scripts.deployment import ComposeProject
from tests.client import Client, Failure, check
ROOT = Path(__file__).resolve().parents[1]
def eventually(operation, message, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
value = operation()
if value:
return value
except (OSError, ValueError, KeyError, AssertionError):
pass
time.sleep(1)
raise AssertionError(message)
class Grafana:
def __init__(self, base):
self.base = base.rstrip("/")
secret = (
(ROOT / "secrets/grafana_admin_password")
.read_text(encoding="utf-8")
.strip()
)
self.authorization = (
"Basic " + base64.b64encode(("admin:" + secret).encode()).decode()
)
def get(self, path, **params):
url = self.base + path
if params:
url += "?" + urllib.parse.urlencode(params)
request = urllib.request.Request(
url,
headers={"Authorization": self.authorization, "Accept": "application/json"},
)
with urllib.request.urlopen(request, timeout=10) as response:
return json.load(response)
def proxy(self, source, path, **params):
return self.get("/api/datasources/proxy/uid/" + source + path, **params)
def query(self, expression):
value = self.proxy("prometheus", "/api/v1/query", query=expression)
check(value["status"] == "success", "Prometheus query failed")
return value["data"]["result"]
def logs(self, selector, start):
value = self.proxy(
"loki",
"/loki/api/v1/query_range",
query=selector,
start=start,
end=time.time_ns(),
limit=1000,
direction="forward",
)
check(value["status"] == "success", "Loki query failed")
return [
json.loads(line)
for stream in value["data"]["result"]
for _, line, *_ in stream["values"]
]
def trace(self, trace_id):
return self.proxy("tempo", "/api/traces/" + trace_id)
def identifier_hex(value, byte_length):
if len(value) == byte_length * 2 and all(
c in "0123456789abcdefABCDEF" for c in value
):
return value.lower()
return base64.b64decode(value, validate=True).hex()
def spans_in(value):
found = []
if isinstance(value, dict):
for key, item in value.items():
if key == "spans":
found.extend(item)
else:
found.extend(spans_in(item))
elif isinstance(value, list):
for item in value:
found.extend(spans_in(item))
return found
def correlation(
grafana, service, start, *, trace_id=None, attachment_id=None, request_id=None
):
selector = '{service_name="' + service + '"} | json'
if service == "messenger-api":
selector += ' | event="http.request"'
if trace_id:
selector += ' | trace_id="' + trace_id + '"'
if attachment_id:
selector += ' | attachment_id="' + attachment_id + '"'
if request_id:
selector += ' | request_id="' + request_id + '"'
records = eventually(
lambda: grafana.logs(selector, start), "No correlated log for " + service
)
record = records[0]
trace_id = record["trace_id"]
def matching_trace():
candidate = grafana.trace(trace_id)
return (
candidate
if any(
identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
for span in spans_in(candidate)
)
else None
)
trace = eventually(matching_trace, "Matching span is missing in Tempo")
spans = [
span
for span in spans_in(trace)
if identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
]
# Tempo 2.9 JSON использует protobuf bytes (Base64); Loki хранит hex.
check(
any(
identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
for span in spans
),
"Log span_id not found in real trace",
)
check(
all(
int(span["endTimeUnixNano"]) > int(span["startTimeUnixNano"])
for span in spans
),
"Trace has no real duration",
)
return {
"trace_id": trace_id,
"span_id": record["span_id"],
"log": record,
"spans": [
{
key: span.get(key)
for key in (
"name",
"kind",
"spanId",
"startTimeUnixNano",
"endTimeUnixNano",
"status",
)
}
for span in spans
],
}, trace
def run_load(args):
check(
1 <= args.duration <= 300
and 1 <= args.rate <= 40
and 1 <= args.concurrency <= 16,
"Load profile outside bounds",
)
client = Client(args.base_url)
actor, _, _, chat = client.bootstrap()
project = ComposeProject()
container_ids = project.run("ps", "--quiet").splitlines()
samples = []
stop = threading.Event()
def sample():
while not stop.is_set():
result = subprocess.run(
[
"docker",
"stats",
"--no-stream",
"--format",
"{{json .}}",
*container_ids,
],
capture_output=True,
encoding="utf-8",
timeout=15,
)
if result.returncode == 0:
samples.append(
{
"elapsed_seconds": round(time.monotonic() - started, 3),
"containers": [
{
k: row[k]
for k in ("Name", "CPUPerc", "MemUsage", "MemPerc")
}
for row in (
json.loads(line) for line in result.stdout.splitlines()
)
],
}
)
stop.wait(3)
def write(index):
local = Client(args.base_url)
before = time.monotonic()
try:
local.message(actor, chat, "synthetic-observability-" + str(index))
return time.monotonic() - before
except Failure:
return None
started = time.monotonic()
sampler = threading.Thread(target=sample)
sampler.start()
try:
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
futures = []
for index in range(args.duration * args.rate):
time.sleep(max(0, started + index / args.rate - time.monotonic()))
futures.append(pool.submit(write, index))
results = [future.result() for future in futures]
elapsed = time.monotonic() - started
finally:
stop.set()
sampler.join()
latencies = sorted(value for value in results if value is not None)
failed = len(results) - len(latencies)
def percentile(q):
return (
round(latencies[max(0, math.ceil(q * len(latencies)) - 1)] * 1000, 3)
if latencies
else None
)
report = {
"date_utc": datetime.now(timezone.utc).isoformat(),
"workload": "synthetic POST messages; fixed arrival rate; no retries",
"duration_seconds": args.duration,
"elapsed_seconds": round(elapsed, 3),
"arrival_rate": args.rate,
"concurrency_limit": args.concurrency,
"successful_requests": len(latencies),
"failed_requests": failed,
"error_ratio": failed / len(results),
"successful_rps": round(len(latencies) / elapsed, 3),
"successful_latency_ms": {
name: percentile(q)
for name, q in [("p50", 0.5), ("p95", 0.95), ("p99", 0.99)]
},
"dataset": {"users": 3, "chats": 1, "messages": len(latencies)},
"resource_samples": samples,
"limits": "Latency covers successes from worker start; client scheduling is not included. Small controlled load, not a capacity benchmark.",
}
check(bool(latencies) and failed == 0, "Controlled load contains errors")
return report
def metric(grafana, expression):
rows = grafana.query(expression)
return float(rows[0]["value"][1]) if rows else None
def alert_firing(grafana, name):
return bool(grafana.query(f'ALERTS{{alertname="{name}",alertstate="firing"}}'))
def trace_chain(grafana, trace_id, *, retry=False):
def inspect():
spans = spans_in(grafana.trace(trace_id))
by_id = {identifier_hex(span["spanId"], 8): span for span in spans}
workers = [
span
for span in spans
if span["name"] == "attachment.process.v1"
and span.get("kind") in ("SPAN_KIND_CONSUMER", 5)
]
for worker in workers:
publisher = by_id.get(identifier_hex(worker.get("parentSpanId", ""), 8), {})
if publisher.get("kind") not in ("SPAN_KIND_PRODUCER", 4):
return None
upload = by_id.get(identifier_hex(publisher.get("parentSpanId", ""), 8), {})
if upload.get("name") != "POST /api/v1/chats/{chat_id}/attachments":
return None
names = {span["name"] for span in spans}
if (
not workers
or not {"s3.read_staging", "s3.write_result", "postgres.commit_ready"}
<= names
):
return None
if retry:
outcomes = set()
for span in workers:
attrs = {row["key"]: row["value"] for row in span.get("attributes", [])}
outcomes.add(
(
int(attrs["job.attempt"]["intValue"]),
attrs.get("outcome", {}).get("stringValue"),
)
)
if not {(1, "error"), (2, "error")} <= outcomes or not any(
n > 2 and outcome == "success" for n, outcome in outcomes
):
return None
return [
{
"name": span["name"],
"kind": span.get("kind"),
"span_id": identifier_hex(span["spanId"], 8),
"parent_span_id": identifier_hex(span.get("parentSpanId", ""), 8),
"job": {
entry["key"]: entry["value"]
for entry in span.get("attributes", [])
if entry["key"] in {"job.attempt", "outcome"}
},
}
for span in spans
]
return eventually(inspect, "Missing upload -> producer -> consumer -> S3/DB chain")
def telemetry_outage(project, client, grafana, actor, chat):
failed = 'sum({__name__=~"otelcol_exporter_send_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
refused = 'sum({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
baseline = [metric(grafana, expr) for expr in (failed, refused)]
rss = 'sum({__name__=~"otelcol_process_memory_rss(_bytes)?"})'
samples = []
successful = 0
try:
project.run("stop", "loki")
start = time.monotonic()
for second in range(51):
for index in range(4):
client.message(actor, chat, f"outage-{second}-{index}")
successful += 1
if second % 5 == 0:
samples.append(
{
"seconds": round(time.monotonic() - start, 2),
"rss_bytes": metric(grafana, rss),
"queued_batches": metric(
grafana,
'sum(otelcol_exporter_queue_size{exporter="otlphttp/loki"})',
),
"capacity_batches": metric(
grafana,
'sum(otelcol_exporter_queue_capacity{exporter="otlphttp/loki"})',
),
}
)
time.sleep(max(0, start + second + 1 - time.monotonic()))
losses = [
metric(grafana, expr) - count
for expr, count in zip((failed, refused), baseline)
]
check(sum(losses) > 0, "No measured Collector log loss during prolonged outage")
check(
all(
0 <= row["queued_batches"] <= row["capacity_batches"] == 256
for row in samples
),
"Unexpected queue bound",
)
check(
max(row["rss_bytes"] for row in samples) < 256 * 1024**2,
"Collector crossed container memory limit",
)
finally:
project.run("start", "loki")
eventually(
lambda: grafana.proxy("loki", "/loki/api/v1/labels").get("status") == "success",
"Loki did not recover",
)
restored = client.message(actor, chat, "telemetry-backend-restored")
correlation(
grafana,
"messenger-api",
time.time_ns() - 30 * 10**9,
trace_id=restored["headers"]["X-Trace-Id"],
)
return {
"scenario": "loki_unavailable",
"successful_messages": successful,
"dropped_after_export_failure": losses[0],
"dropped_at_enqueue": losses[1],
"samples": samples,
"log_export_recovered": True,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--grafana-url", default="http://localhost:13000")
parser.add_argument("--incidents", action="store_true")
parser.add_argument("--restart-telemetry", action="store_true")
parser.add_argument("--output", default="evidence/observability.json")
parser.add_argument("--duration", type=int, default=10)
parser.add_argument("--rate", type=int, default=10)
parser.add_argument("--concurrency", type=int, default=2)
args = parser.parse_args()
load = run_load(args)
start = time.time_ns()
project = ComposeProject()
client = Client(args.base_url, timeout=40)
aliases = {r.instance_id for r in project.replicas() if r.running}
check(len(aliases) >= 2, "Need two real API containers")
check(
client.call("GET", "/health/live")["headers"].get("X-Instance-Id") in aliases,
"Wrong Compose project for BASE_URL",
)
grafana = Grafana(args.grafana_url)
eventually(
lambda: grafana.get("/api/health").get("database") == "ok",
"Grafana is not ready",
)
sources = grafana.get("/api/datasources")
check(
{"prometheus", "loki", "tempo"} <= {source["uid"] for source in sources},
"Missing provisioned datasource",
)
dashboard = grafana.get("/api/dashboards/uid/messenger-overview")["dashboard"]
check(len(dashboard["panels"]) >= 8, "Dashboard is missing panels")
actor, bob, eve, chat = client.bootstrap()
sentinel = "private-content-" + uuid4().hex
response = client.message(actor, chat, sentinel)
controlled_trace = response["headers"].get("X-Trace-Id")
check(bool(controlled_trace), "No API trace identifier")
attachment, _ = client.attachment(actor, chat)
client.wait_ready(actor, attachment)
api, api_trace = correlation(
grafana,
"messenger-api",
start,
trace_id=controlled_trace,
request_id=response["headers"]["X-Request-Id"],
)
worker, worker_trace = correlation(
grafana, "messenger-worker", start, attachment_id=attachment["id"]
)
accepted_chain = trace_chain(grafana, worker["trace_id"])
check(
api["log"]["request_id"] == response["headers"]["X-Request-Id"],
"Request id correlation lost",
)
check(
any(span["kind"] in ("SPAN_KIND_SERVER", 2) for span in api["spans"]),
"No server span",
)
check(
any(span["kind"] in ("SPAN_KIND_CONSUMER", 5) for span in worker["spans"]),
"No consumer span",
)
processed = eventually(
lambda: grafana.logs(
'{service_name="messenger-worker"} | json | event="job.processed" | trace_id="'
+ worker["trace_id"]
+ '"',
start,
),
"No worker result log",
)
check(processed[0]["outcome"] == "success", "Controlled worker did not succeed")
metrics = {}
expressions = {
"http_requests": "sum(messenger_http_requests_total)",
"http_duration": "sum(messenger_http_duration_seconds_count)",
"job_success": 'sum(messenger_job_attempts_total{outcome="success"})',
"job_duration": "sum(messenger_job_duration_seconds_count)",
}
for name, expression in expressions.items():
metrics[name] = eventually(
lambda expression=expression: grafana.query(expression),
"Missing metric: " + expression,
)
check(float(metrics[name][0]["value"][1]) > 0, "Empty metric: " + name)
# Все PromQL из dashboard должны давать численные данные после traffic.
for panel in dashboard["panels"]:
if panel["datasource"]["uid"] == "prometheus":
eventually(
lambda panel=panel: all(
grafana.query(target["expr"]) for target in panel["targets"]
),
"Empty dashboard panel: " + panel["title"],
)
forbidden_labels = {
"user_id",
"chat_id",
"message_id",
"attachment_id",
"request_id",
"filename",
"url",
"http_target",
"http_url",
}
series = grafana.query('{__name__=~"messenger_.*"}')
check(
not any(forbidden_labels & set(row["metric"]) for row in series),
"High cardinality metric label",
)
report = {
"date_utc": datetime.now(timezone.utc).isoformat(),
"project": project.name,
"load": load,
"api_replicas": len(aliases),
"dashboard_uid": dashboard["uid"],
"api_correlation": api,
"worker_correlation": worker,
"metrics": metrics,
"upload_trace": {"trace_id": worker["trace_id"], "chain": accepted_chain},
"incidents": [],
}
if args.incidents:
def fail_request():
return client.call(
"POST",
"/api/v1/users",
expected=503,
body={
"username": "fault_" + uuid4().hex[:12],
"display_name": "Synthetic",
"password": "not-exported-" + uuid4().hex,
},
)
before = time.monotonic()
try:
project.run("stop", "postgres")
failed = fail_request()
error_trace = failed["headers"].get("X-Trace-Id")
error, db_trace = correlation(
grafana,
"messenger-api",
start,
trace_id=error_trace,
request_id=failed["headers"]["X-Request-Id"],
)
diagnostics = eventually(
lambda: grafana.logs(
'{service_name="messenger-api"} | json | event="dependency.unavailable" | trace_id="'
+ error_trace
+ '"',
start,
),
"No dependency diagnostic log",
)
checkout = next(
span
for span in spans_in(db_trace)
if span["name"] == "postgres.checkout"
and span.get("status", {}).get("code") in ("STATUS_CODE_ERROR", 2)
)
def db_alert():
for _ in range(4):
fail_request()
return alert_firing(grafana, "MessengerHighErrorRate")
eventually(db_alert, "Error rate alert did not fire", timeout=75)
report["incidents"].append(
{
"scenario": "postgres_unavailable",
"http_status": 503,
"correlation": error,
"diagnostic": diagnostics[0],
"db_span": checkout["name"],
"alert": "MessengerHighErrorRate",
"firing_after_seconds": round(time.monotonic() - before, 3),
}
)
finally:
project.run("start", "postgres")
eventually(lambda: _ready(client), "API did not recover after PostgreSQL")
eventually(
lambda: not alert_firing(grafana, "MessengerHighErrorRate"),
"Error rate alert not resolved",
)
report["incidents"][-1]["resolved_after_seconds"] = round(
time.monotonic() - before, 3
)
from scripts.check_jobs import retry_replay
queue = {}
def observe_queue(_):
observed = time.monotonic()
eventually(
lambda: alert_firing(grafana, "MessengerQueueLag"),
"Queue lag alert did not fire",
)
queue.update(
depth=metric(grafana, "max(messenger_queue_depth_ratio)"),
oldest_seconds=metric(grafana, "max(messenger_queue_oldest_seconds)"),
firing_after_seconds=round(time.monotonic() - observed, 3),
)
incident = retry_replay(
client, actor, chat, permanent=False, on_queued=observe_queue
)
retried_worker, retry_trace = correlation(
grafana, "messenger-worker", start, attachment_id=incident["attachment_id"]
)
chain = trace_chain(grafana, retried_worker["trace_id"], retry=True)
eventually(
lambda: not alert_firing(grafana, "MessengerQueueLag")
and metric(grafana, "max(messenger_queue_depth_ratio)") == 0,
"Backlog did not drain",
)
queue.update(resolved=True, final_depth=0)
report["incidents"].append(
{
"scenario": "worker_stopped_then_transient_retry",
"alert": "MessengerQueueLag",
"queue": queue,
"recovered_status": incident["after"]["status"],
"trace_id": retried_worker["trace_id"],
"chain": chain,
}
)
report["incidents"].append(
telemetry_outage(project, client, grafana, actor, chat)
)
else:
retry_trace = {}
# Не сохраняем API bodies или SDK exceptions; проверяем реальные выбранные сигналы.
logs = grafana.logs('{service_name=~"messenger-api|messenger-worker"}', start)
exported = json.dumps([logs, api_trace, worker_trace, retry_trace])
private = [
sentinel,
*(person["token"] for person in (actor, bob, eve)),
*(person["registration"]["password"] for person in (actor, bob, eve)),
]
private.extend(
path.read_text(encoding="utf-8").strip()
for path in (ROOT / "secrets").iterdir()
if path.is_file()
)
check(
not any(secret and secret in exported for secret in private),
"Private value found in telemetry",
)
check(
all(
record.get("trace_id")
and record.get("span_id")
and record.get("timestamp")
and record.get("severity")
for record in logs
),
"Incomplete structured logs",
)
report["privacy_checked_logs"] = len(logs)
report["checks"] = {
"three_backends": True,
"dashboard_queries": True,
"server_consumer_traces": True,
"log_trace_correlation": True,
"low_cardinality": True,
"privacy": True,
}
if args.restart_telemetry:
historical_time = time.time()
expression = "sum(messenger_http_requests_total)"
before = grafana.proxy(
"prometheus", "/api/v1/query", query=expression, time=historical_time
)["data"]["result"]
project.run("restart", "prometheus", "loki", "tempo", "grafana")
after = eventually(
lambda: grafana.proxy(
"prometheus", "/api/v1/query", query=expression, time=historical_time
)["data"]["result"],
"Prometheus history lost",
)
check(before == after, "Historical metric changed after storage restart")
correlation(grafana, "messenger-api", start, trace_id=api["trace_id"])
correlation(grafana, "messenger-worker", start, trace_id=worker["trace_id"])
report["checks"]["history_survives_restart"] = True
destination = ROOT / args.output
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
print(
"PASS: telemetry, dashboard, privacy, correlations"
+ ("; alerts and outage recovery verified" if args.incidents else "")
)
def _ready(client):
return client.call("GET", "/health/ready", expected=[200, 503])["status"] == 200
if __name__ == "__main__":
main()
+147
View File
@@ -0,0 +1,147 @@
"""Сценарий COMMIT → SIGKILL до publish → ready без повторного upload.
Требуется уже работающий отдельный стенд. Скрипт не выполняется при запуске API.
Публичные tests не изменяет, secrets/tokens не печатает и evidence сам не сохраняет.
"""
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
from uuid import uuid4
from scripts.deployment import ComposeProject
from tests.client import Client, check
ROOT = Path(__file__).resolve().parents[1]
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess:
return subprocess.run(
COMPOSE + list(arguments),
cwd=ROOT,
encoding="utf-8",
capture_output=True,
timeout=timeout,
)
def require_compose(*arguments: str) -> str:
result = compose(*arguments)
check(result.returncode == 0, "Compose command failed; output hidden")
return result.stdout
def main() -> None:
parser = argparse.ArgumentParser(
description="Отказ publisher после COMMIT перед RabbitMQ publish"
)
parser.add_argument("--base-url", required=True)
parser.add_argument("--ca-file")
args = parser.parse_args()
running = set(
require_compose("ps", "--status", "running", "--services").splitlines()
)
check(
{"api", "postgres", "minio", "broker", "publisher", "worker"} <= running,
"Start the application stack before this scenario",
)
client = Client(args.base_url, args.ca_file, timeout=40)
project = ComposeProject()
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
live = client.call("GET", "/health/live", schema="Health")
check(
live["headers"].get("X-Instance-Id") in aliases,
"BASE_URL does not belong to the selected Compose project",
)
alice, _, _, chat = client.bootstrap()
attachment = None
fault_container = "messenger-outbox-fault-" + uuid4().hex
try:
require_compose("stop", "publisher")
attachment, original = client.attachment(alice, chat)
message = client.message(alice, chat, "", attachments=[attachment["id"]])[
"json"
]
client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}/content",
expected=409,
actor=alice,
)
killed = compose(
"run",
"--rm",
"--no-deps",
"--name",
fault_container,
"publisher",
"python",
"-m",
"app.core.outbox_publisher",
"--crash-before-publish",
attachment["id"],
)
marker = json.dumps(
{"fault": "before_publish", "attachment_id": attachment["id"]}
)
check(
killed.returncode == 137 and marker in killed.stdout,
"Expected targeted SIGKILL before publish was not observed",
)
pending = client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}",
actor=alice,
schema="Attachment",
)["json"]
check(pending["status"] == "queued", "Crash must leave accepted job queued")
finally:
try:
subprocess.run(
["docker", "rm", "-f", fault_container], capture_output=True, timeout=30
)
finally:
require_compose("start", "publisher")
ready = client.wait_ready(alice, attachment, wait_seconds=60)
check(ready["id"] == attachment["id"], "Recovered a different attachment")
check(
{variant["name"] for variant in ready["variants"]}
== {"thumbnail", "crop", "optimized"},
"Missing required variants",
)
path = f"/api/v1/attachments/{attachment['id']}/content"
check(
client.call("GET", path, actor=alice)["bytes"] == original,
"Original changed after recovery",
)
for variant in ready["variants"]:
downloaded = client.call(
"GET", path + "?variant=" + variant["name"], actor=alice
)["bytes"]
check(
len(downloaded) == variant["size_bytes"]
and hashlib.sha256(downloaded).hexdigest() == variant["sha256"],
"Recovered variant checksum mismatch",
)
print(
json.dumps(
{
"scenario": "commit_before_publish",
"attachment_id": ready["id"],
"message_id": message["id"],
"publisher_exit_code": killed.returncode,
"status_after_crash": pending["status"],
"status_after_restart": ready["status"],
"variants": [variant["name"] for variant in ready["variants"]],
"reuploaded": False,
}
)
)
if __name__ == "__main__":
main()
+152
View File
@@ -0,0 +1,152 @@
"""Реальный MinIO: повтор bootstrap, private bucket, ограниченные права и отказ."""
import json
import subprocess
import time
from scripts.deployment import ComposeProject
from tests.client import Client, arguments, check, run
PROBE = r"""
import asyncio, json, sys
from pathlib import Path
from uuid import UUID
from urllib.request import Request, urlopen
from urllib.error import HTTPError, URLError
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.storage.s3 import create_s3_storage
from app.models.attachment import Attachment
from minio.error import S3Error
async def main():
settings = Settings()
engine = create_db_engine(settings)
storage = create_s3_storage(settings.s3)
try:
async with create_session_factory(engine)() as session:
record = await session.get(Attachment, UUID(json.load(sys.stdin)['attachment_id']))
key = record.object_key
results = {}
for method in ('GET', 'PUT'):
request = Request(settings.s3.endpoint + '/' + settings.s3.bucket + '/' + key,
method=method, data=b'x' if method == 'PUT' else None)
try:
with urlopen(request, timeout=3) as response:
results[method] = response.status
except HTTPError as error:
results[method] = error.code
error.close()
try:
storage._client.get_bucket_policy(settings.s3.bucket)
results['admin_denied'] = False
except S3Error as error:
results['admin_denied'] = error.code == 'AccessDenied'
try:
buckets = storage._client.list_buckets()
results['other_buckets_denied'] = all(bucket.name == settings.s3.bucket for bucket in buckets)
except S3Error as error:
results['other_buckets_denied'] = error.code == 'AccessDenied'
try:
with urlopen(settings.s3.endpoint.replace(':9000', ':9001'), timeout=1):
results['console_unreachable'] = False
except HTTPError as error:
results['console_unreachable'] = False
error.close()
except URLError:
results['console_unreachable'] = True
results['temp_originals_absent'] = not any(Path('/tmp').glob('messenger-upload-*'))
results['root_files_absent'] = not any(Path('/run/secrets', name).exists()
for name in ('minio_root_user', 'minio_root_password'))
print(json.dumps(results))
finally:
await storage.aclose()
await engine.dispose()
asyncio.run(main())
"""
def main():
def configure(parser):
parser.add_argument(
"--with-outage",
action="store_true",
help="временно остановить MinIO выбранного проекта",
)
args = arguments("MinIO private bucket and failure checks", configure)
project = ComposeProject()
client = Client(args.base_url, args.ca_file, timeout=40)
alice, _, _, chat = client.bootstrap()
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
live = client.call("GET", "/health/live", schema="Health")
check(
live["headers"].get("X-Instance-Id") in aliases,
"BASE_URL does not belong to the selected Compose project",
)
attachment, original = client.attachment(alice, chat)
attachment = client.wait_ready(alice, attachment)
for _ in range(2):
project.run("run", "--rm", "--no-deps", "minio-bootstrap", timeout=100)
replicas = project.replicas()
check(bool(replicas), "API not running")
result = subprocess.run(
["docker", "exec", "-i", replicas[0].id, "python", "-c", PROBE],
input=json.dumps({"attachment_id": attachment["id"]}),
encoding="utf-8",
capture_output=True,
timeout=20,
)
check(result.returncode == 0, "private bucket probe failed (details hidden)")
statuses = json.loads(result.stdout)
check(statuses["GET"] == statuses["PUT"] == 403, "anonymous S3 access allowed")
check(
statuses["admin_denied"]
and statuses["other_buckets_denied"]
and statuses["root_files_absent"],
"API has excessive access",
)
check(statuses["console_unreachable"], "console reachable from API network")
check(statuses["temp_originals_absent"], "API retained temporary originals")
check(
not project.config["services"]["minio"].get("ports"),
"MinIO publishes host ports",
)
env = project.config["services"]["minio"]["environment"]
check(env["MINIO_BROWSER"] == "off", "browser enabled")
check(
"127.0.0.1:9001" in project.config["services"]["minio"]["command"],
"console is not bound to loopback",
)
path = "/api/v1/attachments/" + attachment["id"] + "/content"
check(
client.call("GET", path, actor=alice)["bytes"] == original,
"bootstrap changed original",
)
if args.with_outage:
try:
project.run("stop", "minio")
client.call("GET", path, expected=503, actor=alice)
finally:
project.run("start", "minio")
deadline = time.monotonic() + 60
while True:
try:
client.call("GET", "/health/ready", schema="Health")
break
except Exception:
if time.monotonic() >= deadline:
raise
time.sleep(0.5)
check(
client.call("GET", path, actor=alice)["bytes"] == original,
"original lost after MinIO restart",
)
print(
"PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console"
+ (", MinIO outage/recovery" if args.with_outage else "")
)
if __name__ == "__main__":
run(main)
+201
View File
@@ -0,0 +1,201 @@
"""Последовательная проверка лабораторной 6 в отдельном Docker-окружении."""
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import subprocess
import tarfile
import time
ROOT = Path("/workspace")
EVIDENCE = ROOT / "evidence"
COMPOSE = ["docker", "compose", "--env-file", ".env"]
BASE = ["--base-url", "http://127.0.0.1:28080"]
def step(report, name, command, *, timeout=600, env=None, required=True):
print("Запуск " + name, flush=True)
started = time.monotonic()
log = name + ".txt"
with (EVIDENCE / log).open("w", encoding="utf-8", newline="\n") as output:
output.write("$ " + " ".join(command) + "\n")
output.flush()
result = subprocess.run(
command,
cwd=ROOT,
stdout=output,
stderr=subprocess.STDOUT,
timeout=timeout,
env={**os.environ, **(env or {})},
)
report["checks"].append(
{
"name": name,
"exit_code": result.returncode,
"duration_seconds": round(time.monotonic() - started, 3),
"log": log,
}
)
if result.returncode:
print("Ошибка " + name, flush=True)
if required:
result.check_returncode()
else:
print("Готово " + name, flush=True)
def main():
with tarfile.open("/tmp/source.tar") as archive:
archive.extractall(ROOT, filter="data")
os.chdir(ROOT)
EVIDENCE.mkdir(exist_ok=True)
deadline = time.monotonic() + 120
while subprocess.run(["docker", "info"], capture_output=True).returncode:
if time.monotonic() > deadline:
raise RuntimeError("Проверочный Docker не запустился")
time.sleep(1)
os.environ.update(
COMPOSE_PROJECT_NAME="messenger-test",
COMPOSE_FILE="compose.yaml:compose.test.yaml",
LB_HOST_PORT="28080",
GRAFANA_HOST_PORT="23000",
DOCKER_TESTS="1",
TEST_USE_MIGRATIONS="1",
TEST_REDIS_HOST="127.0.0.1",
TEST_REDIS_PORT="26379",
TEST_REDIS_DB="15",
)
integration_only = os.environ.get("INTEGRATION_ONLY") == "1"
report = {
"status": "failed",
"profile": "integration" if integration_only else "full",
"started_at": datetime.now(timezone.utc).isoformat(),
"checks": [],
}
try:
step(report, "00_init", ["python", "scripts/init_local.py"])
step(report, "01_contract", ["python", "scripts/check_contract.py"])
step(report, "02_openapi", ["python", "scripts/validate_openapi.py"])
step(
report,
"03_import_images",
["docker", "image", "load", "--input", "/tmp/images.tar"],
)
step(
report,
"03_startup",
COMPOSE + ["up", "-d", "--no-build", "--pull", "never"],
timeout=3600,
)
step(
report,
"03_readiness",
COMPOSE
+ [
"up",
"-d",
"--no-deps",
"--wait",
"--wait-timeout",
"240",
"nginx",
"grafana",
],
)
if not integration_only:
password = (ROOT / "secrets/db_password").read_text(encoding="utf-8")
test_env = {
"OTEL_ENABLED": "false",
"TEST_DATABASE_URL": (
f"postgresql+asyncpg://messenger:{password}@127.0.0.1:25432/messenger_test"
),
"TEST_REDIS_PASSWORD": (ROOT / "secrets/redis_password").read_text(
encoding="utf-8"
),
}
step(
report,
"04_test_database",
COMPOSE
+ [
"exec",
"-T",
"postgres",
"createdb",
"-U",
"messenger",
"messenger_test",
],
)
step(
report,
"05_pytest",
["python", "-m", "pytest", "-q", "--junitxml=evidence/pytest.xml"],
timeout=1800,
env=test_env,
required=False,
)
step(report, "06_smoke", ["python", "tests/smoke.py", *BASE], required=False)
step(
report,
"07_observability",
[
"python",
"-m",
"scripts.check_observability",
*BASE,
"--grafana-url",
"http://127.0.0.1:23000",
"--incidents",
"--restart-telemetry",
],
timeout=900,
required=False,
)
for number, mode, hook in (
("08", "persistence", "restart"),
("09", "failover", "stop-one"),
):
step(
report,
f"{number}_{mode}",
[
"python",
"tests/resilience.py",
"--mode",
mode,
"--action-script",
f"scripts/{hook}.sh",
*BASE,
],
required=False,
)
if all(check["exit_code"] == 0 for check in report["checks"]):
report["status"] = "passed"
finally:
try:
step(
report,
"10_cleanup",
COMPOSE + ["down", "--volumes", "--remove-orphans"],
timeout=180,
)
except (subprocess.SubprocessError, OSError):
report["status"] = "failed"
raise
finally:
report["finished_at"] = datetime.now(timezone.utc).isoformat()
(EVIDENCE / "verification.json").write_text(
json.dumps(report, indent=2, ensure_ascii=False) + "\n",
encoding="utf-8",
)
if report["status"] != "passed":
raise SystemExit(1)
if __name__ == "__main__":
main()
+68
View File
@@ -0,0 +1,68 @@
"""Общие операции с репликами текущего Compose-проекта."""
from dataclasses import dataclass
import json
import os
from pathlib import Path
import subprocess
ROOT = Path(__file__).resolve().parents[1]
@dataclass(frozen=True)
class Replica:
id: str
image: str
instance_id: str
running: bool
health: str | None
class ComposeProject:
def __init__(self) -> None:
self.command = ["docker", "compose", "--env-file", ".env"]
self.config = json.loads(self.run("config", "--format", "json"))
self.name = self.config["name"]
def docker(self, *args: str, timeout: int = 90, env=None) -> str:
result = subprocess.run(
["docker", *args],
cwd=ROOT,
encoding="utf-8",
capture_output=True,
timeout=timeout,
env={**os.environ, **(env or {})},
)
if result.returncode:
raise RuntimeError(result.stderr.strip() or result.stdout.strip())
return result.stdout.strip()
def run(self, *args: str, timeout: int = 90, env=None) -> str:
return self.docker(*self.command[1:], *args, timeout=timeout, env=env)
def replicas(self) -> list[Replica]:
ids = self.run("ps", "--all", "--quiet", "api").splitlines()
if not ids:
return []
template = (
'{"id":{{json .Id}},"image":{{json .Image}},'
'"hostname":{{json .Config.Hostname}},"running":{{json .State.Running}},'
'"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}}'
)
rows = self.docker("inspect", "--format", template, *ids).splitlines()
configured = str(
self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID")
or ""
).strip()
return [
Replica(
row["id"],
row["image"],
configured or row["hostname"],
row["running"],
row["health"],
)
for row in map(json.loads, rows)
]
+39
View File
@@ -0,0 +1,39 @@
"""Создаёт только отсутствующие локальные настройки и файловые секреты."""
import os
from pathlib import Path
import secrets
def main() -> None:
root = Path(__file__).resolve().parents[1]
destination = root / ".env"
if not destination.exists():
with destination.open("x", encoding="utf-8", newline="\n") as target:
target.write((root / ".env.example").read_text(encoding="utf-8"))
destination.chmod(0o600)
directory = root / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
for name in (
"db_password",
"redis_password",
"broker_password",
"minio_root_user",
"minio_root_password",
"s3_access_key_id",
"s3_secret_access_key",
"grafana_admin_password",
):
path = directory / name
try:
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444)
except FileExistsError:
continue
with os.fdopen(descriptor, "w", encoding="utf-8") as target:
target.write(secrets.token_urlsafe(24))
print("Создан файл " + str(path.relative_to(root)))
print("Настройки готовы; существующие значения сохранены")
if __name__ == "__main__":
main()
+36
View File
@@ -0,0 +1,36 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
compose() {
docker compose --env-file .env "$@"
}
wait_healthy() {
for service in "$@"; do
attempts=0
until states=$(compose ps --format '{{.Health}}' "$service") \
&& [ -n "$states" ] \
&& [ -z "$(printf '%s\n' "$states" | grep -v '^healthy$')" ]; do
attempts=$((attempts + 1))
if [ "$attempts" -ge 60 ]; then
echo "$service не стал healthy за 60 секунд" >&2
exit 1
fi
sleep 1
done
done
}
worker_running=$(compose ps --status running --quiet worker)
compose stop worker publisher cleanup
compose restart postgres redis minio broker
wait_healthy postgres redis minio broker
compose restart api
wait_healthy api
compose restart nginx
wait_healthy nginx
compose start publisher cleanup
if [ -n "$worker_running" ]; then
compose start worker
fi
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
*[!a-zA-Z0-9_.:-]*|'') exit 1 ;;
esac
target=""
for container in $(docker compose --env-file .env ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container")
configured=$(docker inspect --format '{{range .Config.Env}}{{if eq (index (split . "=") 0) "INSTANCE_ID"}}{{println .}}{{end}}{{end}}' "$container" \
| sed -n 's/^INSTANCE_ID=//p' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
if [ "$TARGET_INSTANCE_ID" = "$hostname" ] || [ "$TARGET_INSTANCE_ID" = "$configured" ]; then
if [ -n "$target" ]; then
echo 'Instance id matches multiple API containers' >&2
exit 1
fi
target="$container"
fi
done
if [ -z "$target" ]; then
echo 'Target API instance not found in this Compose project' >&2
exit 1
fi
echo "Stopping API container $target" >&2
docker stop --timeout 8 "$target" >/dev/null
+258
View File
@@ -0,0 +1,258 @@
"""Запускает проверки в одноразовом Docker-окружении и сохраняет evidence."""
import argparse
from datetime import datetime, timezone
import io
import json
import os
from pathlib import Path
import subprocess
import tarfile
import tempfile
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
EXCLUDED = {
".git",
".agents",
".aws",
".codex",
".env",
"secrets",
"evidence",
".venv",
"venv",
"__pycache__",
".pytest_cache",
".github",
"docs",
}
TEXT_SUFFIXES = {".py", ".sh", ".conf", ".yaml", ".yml", ".json", ".txt", ".md", ".ini"}
def docker(*args, **kwargs):
kwargs.setdefault("check", True)
return subprocess.run(["docker", *args], cwd=ROOT, **kwargs)
def archive_source(destination):
with tarfile.open(destination, "w") as archive:
for path in sorted(ROOT.rglob("*")):
relative = path.relative_to(ROOT)
if (
not path.is_file()
or path.is_symlink()
or any(part in EXCLUDED for part in relative.parts)
or path.name.startswith(".env.")
and path.name != ".env.example"
or path.suffix in {".zip", ".log", ".pem", ".key"}
):
continue
content = path.read_bytes()
if path.suffix in TEXT_SUFFIXES:
content = content.replace(b"\r\n", b"\n")
entry = tarfile.TarInfo(relative.as_posix())
entry.size = len(content)
entry.mode = 0o755 if path.suffix == ".sh" else 0o644
archive.addfile(entry, io.BytesIO(content))
def main():
parser = argparse.ArgumentParser(description=__doc__)
mode = parser.add_mutually_exclusive_group()
mode.add_argument("--format", action="store_true", help="применить Black и выйти")
mode.add_argument(
"--integration-only",
action="store_true",
help="запустить проверки стенда и телеметрии без pytest",
)
args = parser.parse_args()
project = "messenger-check-" + uuid4().hex[:12]
image = project + ":tools"
image_env = {
"API_IMAGE": project + ":api",
"MINIO_IMAGE": project + ":server",
"MINIO_BOOTSTRAP_IMAGE": project + ":bootstrap",
}
engine = project + "-engine"
runner = project + "-runner"
workspace = project + "-workspace"
data = project + "-data"
network = project + "-network"
output = ROOT / "evidence" / "lab6"
if args.integration_only:
output /= "integration"
output.mkdir(parents=True, exist_ok=True)
summary = {
"status": "failed",
"project": project,
"profile": "integration" if args.integration_only else "full",
"started_at": datetime.now(timezone.utc).isoformat(),
}
try:
docker("build", "--tag", image, "--file", "infra/testing/Dockerfile", ".")
if args.format:
docker(
"run",
"--rm",
"--volume",
str(ROOT) + ":/workspace",
image,
"python3",
"-m",
"black",
"app",
"infra",
"migrations",
"scripts",
"student-tests",
"tests",
)
summary["status"] = "formatted"
return
compose = ("compose", "--env-file", ".env.example")
env = {**os.environ, **image_env}
config = json.loads(
docker(
*compose, "config", "--format", "json", env=env, capture_output=True
).stdout
)
dependencies = sorted(
{
service["image"]
for service in config["services"].values()
if "build" not in service
}
)
docker(
*compose,
"pull",
"--ignore-buildable",
"--policy",
"missing",
env=env,
timeout=3600,
)
docker(
*compose, "build", "api", "minio", "minio-bootstrap", env=env, timeout=3600
)
docker("network", "create", network, stdout=subprocess.DEVNULL)
for volume in (workspace, data):
docker("volume", "create", volume, stdout=subprocess.DEVNULL)
docker(
"run",
"-d",
"--privileged",
"--name",
engine,
"--network",
network,
"--env",
"DOCKER_TLS_CERTDIR=",
"--volume",
workspace + ":/workspace",
"--volume",
data + ":/var/lib/docker",
"docker:29.1.3-dind",
"dockerd",
"--host=unix:///var/run/docker.sock",
"--host=tcp://127.0.0.1:2375",
"--tls=false",
stdout=subprocess.DEVNULL,
)
docker(
"create",
"--name",
runner,
"--network",
"container:" + engine,
"--env",
"DOCKER_HOST=tcp://127.0.0.1:2375",
*[
arg
for key, value in image_env.items()
for arg in ("--env", f"{key}={value}")
],
"--env",
f"INTEGRATION_ONLY={int(args.integration_only)}",
"--volume",
workspace + ":/workspace",
image,
stdout=subprocess.DEVNULL,
)
with tempfile.TemporaryDirectory(prefix="messenger-check-") as directory:
archive = Path(directory) / "source.tar"
archive_source(archive)
docker("cp", str(archive), runner + ":/tmp/source.tar")
image_archive = Path(directory) / "images.tar"
docker(
"image",
"save",
"--output",
str(image_archive),
*image_env.values(),
*dependencies,
)
docker("cp", str(image_archive), runner + ":/tmp/images.tar")
docker("start", "--attach", runner, check=False, timeout=7200)
result = subprocess.run(
["docker", "wait", runner],
cwd=ROOT,
text=True,
encoding="utf-8",
capture_output=True,
timeout=7200,
)
result.check_returncode()
docker("cp", runner + ":/workspace/evidence/.", str(output))
if int(result.stdout.strip()) != 0:
raise RuntimeError(
f"Проверки завершились ошибкой. См. {output.relative_to(ROOT)}."
)
summary["status"] = "passed"
finally:
cleanup = []
for command in (
["rm", "--force", runner, engine],
["volume", "rm", workspace, data],
["network", "rm", network],
["image", "rm", image, *image_env.values()],
):
result = subprocess.run(
["docker", *command], cwd=ROOT, encoding="utf-8", capture_output=True
)
cleanup.append(
{
"command": ["docker", *command],
"returncode": result.returncode,
"output": result.stdout.strip() + result.stderr.strip(),
}
)
remaining = {}
for kind in ("container", "volume", "network", "image"):
result = subprocess.run(
["docker", kind, "ls", "--format", "{{json .}}"],
cwd=ROOT,
encoding="utf-8",
capture_output=True,
)
remaining[kind] = [
line for line in result.stdout.splitlines() if project in line
]
if result.returncode:
remaining[kind].append("resource audit failed")
summary["cleanup"] = cleanup
summary["remaining_resources"] = remaining
summary["finished_at"] = datetime.now(timezone.utc).isoformat()
if any(remaining.values()):
summary["status"] = "failed"
(output / "run.json").write_text(
json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
)
if summary["status"] == "failed":
raise SystemExit(1)
print("PASS: проверки завершены, временное окружение удалено")
if __name__ == "__main__":
main()
+15 -10
View File
@@ -1,23 +1,28 @@
#!/usr/bin/env python3
"""Full OpenAPI and JSON Schema checks; install requirements-dev.txt first."""
import json
from pathlib import Path
from openapi_spec_validator import validate
from jsonschema import Draft202012Validator
root = Path(__file__).resolve().parents[1]
spec = json.loads((root/'contracts/openapi.json').read_text())
spec = json.loads((root / "contracts/openapi.json").read_text(encoding="utf-8"))
validate(spec)
from referencing import Registry, Resource
# The base URI resolves local #/components references inside examples.
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource))
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name}
resource = dict(spec, **{"$schema": "https://json-schema.org/draft/2020-12/schema"})
registry = Registry().with_resource(
"urn:messenger:openapi", Resource.from_contents(resource)
)
for name, example in json.loads(
(root / "contracts/examples.json").read_text(encoding="utf-8")
).items():
schema = {"$ref": "urn:messenger:openapi#/components/schemas/" + name}
Draft202012Validator(schema, registry=registry).validate(example)
for path in (root/'contracts').glob('*.schema.json'):
schema = json.loads(path.read_text())
for path in (root / "contracts").glob("*.schema.json"):
schema = json.loads(path.read_text(encoding="utf-8"))
Draft202012Validator.check_schema(schema)
for example in schema.get('examples',[]):
for example in schema.get("examples", []):
Draft202012Validator(schema).validate(example)
print('PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.')
print("PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.")