solution
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Команды управления учебным стендом."""
|
||||
+59
-35
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline structural checks; full schema validation is a separate target."""
|
||||
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
@@ -7,48 +8,71 @@ import re
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
lab = json.loads((ROOT/'lab.json').read_text())['number']
|
||||
spec = json.loads((ROOT/'contracts/openapi.json').read_text())
|
||||
assert spec['openapi']=='3.1.0'
|
||||
assert spec['info']['version']=='1.%d.0'%lab
|
||||
lab = json.loads((ROOT / "lab.json").read_text(encoding="utf-8"))["number"]
|
||||
spec = json.loads((ROOT / "contracts/openapi.json").read_text(encoding="utf-8"))
|
||||
assert spec["openapi"] == "3.1.0"
|
||||
assert spec["info"]["version"] == "1.%d.0" % lab
|
||||
operation_ids = set()
|
||||
|
||||
def walk(value):
|
||||
if isinstance(value,dict):
|
||||
if '$ref' in value:
|
||||
pointer = value['$ref']
|
||||
assert pointer.startswith('#/'), 'unexpected external reference'
|
||||
if isinstance(value, dict):
|
||||
if "$ref" in value:
|
||||
pointer = value["$ref"]
|
||||
assert pointer.startswith("#/"), "unexpected external reference"
|
||||
target = spec
|
||||
for part in pointer[2:].split('/'):
|
||||
target = target[part.replace('~1','/').replace('~0','~')]
|
||||
for part in pointer[2:].split("/"):
|
||||
target = target[part.replace("~1", "/").replace("~0", "~")]
|
||||
for v in value.values():
|
||||
walk(v)
|
||||
elif isinstance(value,list):
|
||||
elif isinstance(value, list):
|
||||
for v in value:
|
||||
walk(v)
|
||||
walk(spec)
|
||||
for route,methods in spec['paths'].items():
|
||||
placeholders = set(re.findall(r'\{([^}]+)\}',route))
|
||||
for method,op in methods.items():
|
||||
assert op['operationId'] not in operation_ids, 'duplicate operationId'
|
||||
operation_ids.add(op['operationId'])
|
||||
params = {p['name'] for p in op.get('parameters',[]) if p['in']=='path' and p.get('required')}
|
||||
assert placeholders==params, 'path parameter mismatch: '+route
|
||||
for response in op['responses'].values():
|
||||
assert 'X-Request-Id' in response['headers']
|
||||
if lab>=3:
|
||||
assert 'X-Instance-Id' in response['headers']
|
||||
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
|
||||
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
|
||||
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
|
||||
ast.parse(file.read_text(),filename=str(file))
|
||||
for file in ROOT.rglob('*.json'):
|
||||
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
|
||||
continue
|
||||
json.loads(file.read_text())
|
||||
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
|
||||
assert (ROOT/required).is_file(), 'missing '+required
|
||||
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
|
||||
|
||||
if __name__=='__main__':
|
||||
walk(spec)
|
||||
for route, methods in spec["paths"].items():
|
||||
placeholders = set(re.findall(r"\{([^}]+)\}", route))
|
||||
for method, op in methods.items():
|
||||
assert op["operationId"] not in operation_ids, "duplicate operationId"
|
||||
operation_ids.add(op["operationId"])
|
||||
params = {
|
||||
p["name"]
|
||||
for p in op.get("parameters", [])
|
||||
if p["in"] == "path" and p.get("required")
|
||||
}
|
||||
assert placeholders == params, "path parameter mismatch: " + route
|
||||
for response in op["responses"].values():
|
||||
assert "X-Request-Id" in response["headers"]
|
||||
if lab >= 3:
|
||||
assert "X-Instance-Id" in response["headers"]
|
||||
assert ("/api/v1/auth/sessions" in spec["paths"]) == (lab >= 2)
|
||||
assert ("/api/v1/attachments/{attachment_id}" in spec["paths"]) == (lab >= 4)
|
||||
for file in list((ROOT / "tests").glob("*.py")) + list(
|
||||
(ROOT / "scripts").glob("*.py")
|
||||
):
|
||||
ast.parse(file.read_text(encoding="utf-8"), filename=str(file))
|
||||
for file in ROOT.rglob("*.json"):
|
||||
if any(
|
||||
part in {".git", ".venv", "node_modules", "vendor", "output"}
|
||||
for part in file.parts
|
||||
):
|
||||
continue
|
||||
json.loads(file.read_text(encoding="utf-8"))
|
||||
for required in (
|
||||
"README.md",
|
||||
"COURSE.md",
|
||||
"CONTRIBUTING.md",
|
||||
"REPORT.md",
|
||||
"contracts/README.md",
|
||||
"tests/smoke.py",
|
||||
"tests/README.md",
|
||||
):
|
||||
assert (ROOT / required).is_file(), "missing " + required
|
||||
print(
|
||||
"PASS: offline skeleton structure, local refs and Python syntax; application NOT tested."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,431 @@
|
||||
"""Fault/recovery очереди на явно выбранном Compose-стенде; только синтетические данные."""
|
||||
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compose
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, check
|
||||
|
||||
PROBE = """
|
||||
import asyncio, hashlib, json
|
||||
from uuid import UUID
|
||||
from sqlalchemy import select
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.models.job import AttachmentJob
|
||||
from app.models.attachment import Attachment
|
||||
from app.storage.s3 import create_s3_storage
|
||||
async def main():
|
||||
settings=Settings(); engine=create_db_engine(settings); storage=create_s3_storage(settings.s3)
|
||||
try:
|
||||
async with create_session_factory(engine)() as session:
|
||||
job, att=(await session.execute(select(AttachmentJob,Attachment).join(Attachment,Attachment.id==AttachmentJob.attachment_id).where(Attachment.id==UUID(ATTACHMENT_ID)))).one()
|
||||
result=dict(job_id=str(job.id),status=att.status,attempt=job.attempt,published_attempt=job.published_attempt,limit=job.attempt_limit,updated_at=att.updated_at,next_attempt_at=job.next_attempt_at,last_error_code=job.last_error_code,permanent=job.last_error_permanent)
|
||||
objects=[]
|
||||
for prefix in ('originals/'+UUID(ATTACHMENT_ID).hex+'/', 'variants/'+UUID(ATTACHMENT_ID).hex+'/'):
|
||||
entries=await asyncio.to_thread(lambda: list(storage._client.list_objects(storage.settings.bucket,prefix=prefix,recursive=True)))
|
||||
for obj in entries:
|
||||
stream=await storage.open_object(obj.object_name); checksum=hashlib.sha256()
|
||||
try:
|
||||
async for chunk in stream: checksum.update(chunk)
|
||||
finally: await stream.aclose()
|
||||
objects.append(dict(name=obj.object_name.split('/')[-1],size=obj.size,sha256=checksum.hexdigest(),last_modified=obj.last_modified))
|
||||
result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
|
||||
finally: await storage.aclose();await engine.dispose()
|
||||
asyncio.run(main())
|
||||
"""
|
||||
|
||||
|
||||
def probe(attachment_id):
|
||||
code = "ATTACHMENT_ID = " + repr(str(UUID(attachment_id))) + "\n" + PROBE
|
||||
result = subprocess.run(
|
||||
COMPOSE + ["exec", "-T", "api", "python", "-"],
|
||||
input=code,
|
||||
encoding="utf-8",
|
||||
capture_output=True,
|
||||
cwd=ROOT,
|
||||
timeout=30,
|
||||
)
|
||||
check(result.returncode == 0, "Probe failed; diagnostic output hidden")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def admin(*arguments, expected=0):
|
||||
result = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"app.core.job_admin",
|
||||
*arguments,
|
||||
)
|
||||
check(result.returncode == expected, "Unexpected operator command exit status")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def verify_result(client, actor, attachment, original):
|
||||
ready = client.wait_ready(actor, attachment, wait_seconds=180)
|
||||
check(ready["id"] == attachment["id"], "Attachment identity changed")
|
||||
path = "/api/v1/attachments/" + ready["id"] + "/content"
|
||||
check(
|
||||
client.call("GET", path, actor=actor)["bytes"] == original, "Original changed"
|
||||
)
|
||||
check(
|
||||
{v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"},
|
||||
"Incomplete pipeline",
|
||||
)
|
||||
for variant in ready["variants"]:
|
||||
data = client.call("GET", path + "?variant=" + variant["name"], actor=actor)[
|
||||
"bytes"
|
||||
]
|
||||
check(
|
||||
hashlib.sha256(data).hexdigest() == variant["sha256"], "Checksum mismatch"
|
||||
)
|
||||
with Image.open(BytesIO(data)) as image:
|
||||
check(
|
||||
image.size == (variant["width"], variant["height"]),
|
||||
"Metadata dimensions mismatch",
|
||||
)
|
||||
check(
|
||||
image.format == ("WEBP" if variant["name"] == "optimized" else "JPEG"),
|
||||
"Wrong format",
|
||||
)
|
||||
check(
|
||||
not image.getexif()
|
||||
and not image.info.get("icc_profile")
|
||||
and not image.info.get("xmp"),
|
||||
"Metadata leaked",
|
||||
)
|
||||
if variant["name"] == "crop":
|
||||
check(image.size == (128, 128), "Wrong crop size")
|
||||
snapshot = probe(ready["id"])
|
||||
check(
|
||||
len(snapshot["objects"]) == 4,
|
||||
"Expected exactly one original and three variants",
|
||||
)
|
||||
return snapshot
|
||||
|
||||
|
||||
def crash(client, actor, chat, mode):
|
||||
name = "messenger-worker-fault-" + uuid4().hex
|
||||
try:
|
||||
require_compose("stop", "worker")
|
||||
attachment, original = client.attachment(actor, chat)
|
||||
result = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"--name",
|
||||
name,
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"app.core.job_worker",
|
||||
"--" + mode,
|
||||
attachment["id"],
|
||||
)
|
||||
phase = (
|
||||
"after_object:optimized" if mode == "crash-after-object" else "before_ack"
|
||||
)
|
||||
marker = json.dumps({"fault": phase, "attachment_id": attachment["id"]})
|
||||
check(
|
||||
result.returncode == 137 and marker in result.stdout,
|
||||
"Targeted crash not observed",
|
||||
)
|
||||
before = probe(attachment["id"])
|
||||
check(
|
||||
before["status"]
|
||||
== ("processing" if mode == "crash-after-object" else "ready"),
|
||||
"Unexpected crash state",
|
||||
)
|
||||
check(len(before["objects"]) == 4, "Crash must occur after all four PUTs")
|
||||
finally:
|
||||
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
|
||||
require_compose("start", "worker")
|
||||
after = verify_result(client, actor, attachment, original)
|
||||
if mode == "crash-before-ack":
|
||||
# Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
|
||||
wait_queue_empty()
|
||||
after = probe(attachment["id"])
|
||||
check(after == before, "Ready duplicate rewrote result")
|
||||
else:
|
||||
check(
|
||||
after["attempt"] == before["attempt"] + 1,
|
||||
"Lease did not recover the interrupted attempt",
|
||||
)
|
||||
check(
|
||||
[(x["name"], x["sha256"]) for x in before["objects"]]
|
||||
== [(x["name"], x["sha256"]) for x in after["objects"]],
|
||||
"Replay changed object content",
|
||||
)
|
||||
return dict(
|
||||
scenario=mode,
|
||||
attachment_id=attachment["id"],
|
||||
exit_code=137,
|
||||
before=before,
|
||||
after=after,
|
||||
)
|
||||
|
||||
|
||||
def retry_replay(client, actor, chat, permanent, on_queued=None):
|
||||
name = "messenger-retry-fault-" + uuid4().hex
|
||||
samples = []
|
||||
configuration = ComposeProject().config["services"]["worker"]["environment"]
|
||||
retry_base = float(configuration["JOB_RETRY_SECONDS"])
|
||||
retry_max = float(configuration["JOB_RETRY_MAX_SECONDS"])
|
||||
try:
|
||||
require_compose("stop", "worker")
|
||||
attachment, original = client.attachment(actor, chat)
|
||||
if on_queued is not None:
|
||||
on_queued(attachment)
|
||||
mode = "fail-permanent" if permanent else "fail-transient"
|
||||
require_compose(
|
||||
"run",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--name",
|
||||
name,
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"app.core.job_worker",
|
||||
"--" + mode,
|
||||
attachment["id"],
|
||||
)
|
||||
deadline = time.monotonic() + 180
|
||||
while time.monotonic() < deadline:
|
||||
sample = probe(attachment["id"])
|
||||
if (
|
||||
sample["attempt"]
|
||||
and sample["status"] in {"queued", "failed"}
|
||||
and (not samples or samples[-1]["attempt"] != sample["attempt"])
|
||||
):
|
||||
samples.append(sample)
|
||||
if sample["status"] == "failed":
|
||||
break
|
||||
time.sleep(0.5)
|
||||
check(sample["status"] == "failed", "Retry never reached failed")
|
||||
check(
|
||||
sample["attempt"] == (1 if permanent else sample["limit"]),
|
||||
"Incorrect attempt budget",
|
||||
)
|
||||
check(sample["permanent"] == permanent, "Incorrect error classification")
|
||||
if not permanent:
|
||||
check(len(samples) == sample["limit"], "Missed retry samples")
|
||||
for entry in samples:
|
||||
delay = (
|
||||
datetime.fromisoformat(entry["next_attempt_at"])
|
||||
- datetime.fromisoformat(entry["updated_at"])
|
||||
).total_seconds()
|
||||
ceiling = min(retry_max, retry_base * 2 ** (entry["attempt"] - 1))
|
||||
check(
|
||||
ceiling / 2 <= delay <= ceiling,
|
||||
"Retry delay outside configured jitter bounds",
|
||||
)
|
||||
client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=actor,
|
||||
expected=409,
|
||||
)
|
||||
job_id = sample["job_id"]
|
||||
request_id = str(uuid4())
|
||||
listing = admin("list-failed")
|
||||
check(
|
||||
any(row["job_id"] == job_id for row in listing),
|
||||
"Job missing from failed registry",
|
||||
)
|
||||
subprocess.run(
|
||||
["docker", "rm", "-f", name], capture_output=True, timeout=30, check=True
|
||||
)
|
||||
first = admin(
|
||||
"replay",
|
||||
job_id,
|
||||
"--request-id",
|
||||
request_id,
|
||||
"--actor",
|
||||
"verification",
|
||||
"--reason",
|
||||
"dependency_restored",
|
||||
)
|
||||
check(first["status"] == "queued", "Replay did not queue job")
|
||||
finally:
|
||||
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
|
||||
require_compose("start", "worker")
|
||||
after = verify_result(client, actor, attachment, original)
|
||||
repeat = admin(
|
||||
"replay",
|
||||
job_id,
|
||||
"--request-id",
|
||||
request_id,
|
||||
"--actor",
|
||||
"verification",
|
||||
"--reason",
|
||||
"dependency_restored",
|
||||
)
|
||||
check(repeat["already_applied"], "Replay request id not idempotent")
|
||||
refused = admin(
|
||||
"replay",
|
||||
job_id,
|
||||
"--request-id",
|
||||
str(uuid4()),
|
||||
"--actor",
|
||||
"verification",
|
||||
"--reason",
|
||||
"manual_review",
|
||||
expected=2,
|
||||
)
|
||||
check(refused["error_code"] == "job_not_failed", "Successful job was replayed")
|
||||
check(probe(attachment["id"]) == after, "Operator replay altered ready job")
|
||||
check(len(admin("history", job_id)) == 1, "Incorrect audit count")
|
||||
return dict(
|
||||
scenario=mode,
|
||||
attachment_id=attachment["id"],
|
||||
samples=samples,
|
||||
after=after,
|
||||
replay_idempotent=True,
|
||||
)
|
||||
|
||||
|
||||
def queue_state():
|
||||
output = require_compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"broker",
|
||||
"rabbitmqctl",
|
||||
"list_queues",
|
||||
"--formatter=json",
|
||||
"name",
|
||||
"messages_ready",
|
||||
"messages_unacknowledged",
|
||||
"consumers",
|
||||
"durable",
|
||||
)
|
||||
return next(row for row in json.loads(output) if row["name"] == "attachment-jobs")
|
||||
|
||||
|
||||
def wait_queue_empty():
|
||||
deadline = time.monotonic() + 20
|
||||
while time.monotonic() < deadline:
|
||||
state = queue_state()
|
||||
if state["messages_ready"] == 0 and state["messages_unacknowledged"] == 0:
|
||||
return state
|
||||
time.sleep(0.5)
|
||||
check(False, "RabbitMQ did not acknowledge all completed deliveries")
|
||||
|
||||
|
||||
def persistence(client, actor, chat):
|
||||
try:
|
||||
require_compose("stop", "worker")
|
||||
attachment, original = client.attachment(actor, chat)
|
||||
client.message(actor, chat, "", attachments=[attachment["id"]])
|
||||
client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=actor,
|
||||
expected=409,
|
||||
)
|
||||
time.sleep(2)
|
||||
queued_before = queue_state()
|
||||
check(
|
||||
queued_before["messages_ready"] >= 1 and queued_before["durable"],
|
||||
"No durable queued message before restart",
|
||||
)
|
||||
result = subprocess.run(
|
||||
[str(ROOT / "scripts/restart.sh")],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
timeout=180,
|
||||
)
|
||||
check(result.returncode == 0, "Restart hook failed")
|
||||
check(
|
||||
"worker"
|
||||
not in require_compose(
|
||||
"ps", "--status", "running", "--services"
|
||||
).splitlines(),
|
||||
"Restart started deliberately stopped worker",
|
||||
)
|
||||
queued_after = queue_state()
|
||||
check(
|
||||
queued_after["messages_ready"] >= 1 and queued_after["durable"],
|
||||
"Broker lost queued delivery across restart",
|
||||
)
|
||||
client.message(actor, chat, "session and chat survived broker/API restart")
|
||||
finally:
|
||||
require_compose("start", "worker")
|
||||
after = verify_result(client, actor, attachment, original)
|
||||
return dict(
|
||||
scenario="worker_stopped_restart_all",
|
||||
attachment_id=attachment["id"],
|
||||
queue_before=queued_before,
|
||||
queue_after=queued_after,
|
||||
after=after,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--ca-file")
|
||||
parser.add_argument(
|
||||
"--scenario",
|
||||
choices=[
|
||||
"all",
|
||||
"persistence",
|
||||
"crash-after-object",
|
||||
"crash-before-ack",
|
||||
"fail-transient",
|
||||
"fail-permanent",
|
||||
],
|
||||
default="all",
|
||||
)
|
||||
parser.add_argument("--output", default="evidence/jobs.json")
|
||||
args = parser.parse_args()
|
||||
client = Client(args.base_url, args.ca_file, timeout=40)
|
||||
aliases = {r.instance_id for r in ComposeProject().replicas() if r.running}
|
||||
check(
|
||||
client.call("GET", "/health/live", schema="Health")["headers"].get(
|
||||
"X-Instance-Id"
|
||||
)
|
||||
in aliases,
|
||||
"Wrong Compose project for BASE_URL",
|
||||
)
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
results = []
|
||||
operations = {
|
||||
"persistence": persistence,
|
||||
"crash-after-object": lambda *x: crash(*x, "crash-after-object"),
|
||||
"crash-before-ack": lambda *x: crash(*x, "crash-before-ack"),
|
||||
"fail-transient": lambda *x: retry_replay(*x, False),
|
||||
"fail-permanent": lambda *x: retry_replay(*x, True),
|
||||
}
|
||||
selected = (
|
||||
operations.values() if args.scenario == "all" else [operations[args.scenario]]
|
||||
)
|
||||
for operation in selected:
|
||||
result = operation(client, actor, chat)
|
||||
results.append(result)
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.output).write_text(
|
||||
json.dumps(dict(status="in_progress", results=results), indent=2) + "\n"
|
||||
)
|
||||
print("PASS: " + result["scenario"], flush=True)
|
||||
Path(args.output).write_text(
|
||||
json.dumps(dict(status="passed", results=results), indent=2) + "\n"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,728 @@
|
||||
"""Проверка наблюдаемости через реальные запросы Grafana к трём backend.
|
||||
|
||||
--incidents явно разрешает ограниченные отказы на выбранном Compose-проекте.
|
||||
В evidence сохраняются только агрегаты и выбранные технические поля.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import math
|
||||
import subprocess
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from uuid import uuid4
|
||||
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, Failure, check
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def eventually(operation, message, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
value = operation()
|
||||
if value:
|
||||
return value
|
||||
except (OSError, ValueError, KeyError, AssertionError):
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise AssertionError(message)
|
||||
|
||||
|
||||
class Grafana:
|
||||
def __init__(self, base):
|
||||
self.base = base.rstrip("/")
|
||||
secret = (
|
||||
(ROOT / "secrets/grafana_admin_password")
|
||||
.read_text(encoding="utf-8")
|
||||
.strip()
|
||||
)
|
||||
self.authorization = (
|
||||
"Basic " + base64.b64encode(("admin:" + secret).encode()).decode()
|
||||
)
|
||||
|
||||
def get(self, path, **params):
|
||||
url = self.base + path
|
||||
if params:
|
||||
url += "?" + urllib.parse.urlencode(params)
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
headers={"Authorization": self.authorization, "Accept": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
return json.load(response)
|
||||
|
||||
def proxy(self, source, path, **params):
|
||||
return self.get("/api/datasources/proxy/uid/" + source + path, **params)
|
||||
|
||||
def query(self, expression):
|
||||
value = self.proxy("prometheus", "/api/v1/query", query=expression)
|
||||
check(value["status"] == "success", "Prometheus query failed")
|
||||
return value["data"]["result"]
|
||||
|
||||
def logs(self, selector, start):
|
||||
value = self.proxy(
|
||||
"loki",
|
||||
"/loki/api/v1/query_range",
|
||||
query=selector,
|
||||
start=start,
|
||||
end=time.time_ns(),
|
||||
limit=1000,
|
||||
direction="forward",
|
||||
)
|
||||
check(value["status"] == "success", "Loki query failed")
|
||||
return [
|
||||
json.loads(line)
|
||||
for stream in value["data"]["result"]
|
||||
for _, line, *_ in stream["values"]
|
||||
]
|
||||
|
||||
def trace(self, trace_id):
|
||||
return self.proxy("tempo", "/api/traces/" + trace_id)
|
||||
|
||||
|
||||
def identifier_hex(value, byte_length):
|
||||
if len(value) == byte_length * 2 and all(
|
||||
c in "0123456789abcdefABCDEF" for c in value
|
||||
):
|
||||
return value.lower()
|
||||
return base64.b64decode(value, validate=True).hex()
|
||||
|
||||
|
||||
def spans_in(value):
|
||||
found = []
|
||||
if isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
if key == "spans":
|
||||
found.extend(item)
|
||||
else:
|
||||
found.extend(spans_in(item))
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
found.extend(spans_in(item))
|
||||
return found
|
||||
|
||||
|
||||
def correlation(
|
||||
grafana, service, start, *, trace_id=None, attachment_id=None, request_id=None
|
||||
):
|
||||
selector = '{service_name="' + service + '"} | json'
|
||||
if service == "messenger-api":
|
||||
selector += ' | event="http.request"'
|
||||
if trace_id:
|
||||
selector += ' | trace_id="' + trace_id + '"'
|
||||
if attachment_id:
|
||||
selector += ' | attachment_id="' + attachment_id + '"'
|
||||
if request_id:
|
||||
selector += ' | request_id="' + request_id + '"'
|
||||
records = eventually(
|
||||
lambda: grafana.logs(selector, start), "No correlated log for " + service
|
||||
)
|
||||
record = records[0]
|
||||
trace_id = record["trace_id"]
|
||||
|
||||
def matching_trace():
|
||||
candidate = grafana.trace(trace_id)
|
||||
return (
|
||||
candidate
|
||||
if any(
|
||||
identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
|
||||
for span in spans_in(candidate)
|
||||
)
|
||||
else None
|
||||
)
|
||||
|
||||
trace = eventually(matching_trace, "Matching span is missing in Tempo")
|
||||
spans = [
|
||||
span
|
||||
for span in spans_in(trace)
|
||||
if identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
|
||||
]
|
||||
# Tempo 2.9 JSON использует protobuf bytes (Base64); Loki хранит hex.
|
||||
check(
|
||||
any(
|
||||
identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
|
||||
for span in spans
|
||||
),
|
||||
"Log span_id not found in real trace",
|
||||
)
|
||||
check(
|
||||
all(
|
||||
int(span["endTimeUnixNano"]) > int(span["startTimeUnixNano"])
|
||||
for span in spans
|
||||
),
|
||||
"Trace has no real duration",
|
||||
)
|
||||
return {
|
||||
"trace_id": trace_id,
|
||||
"span_id": record["span_id"],
|
||||
"log": record,
|
||||
"spans": [
|
||||
{
|
||||
key: span.get(key)
|
||||
for key in (
|
||||
"name",
|
||||
"kind",
|
||||
"spanId",
|
||||
"startTimeUnixNano",
|
||||
"endTimeUnixNano",
|
||||
"status",
|
||||
)
|
||||
}
|
||||
for span in spans
|
||||
],
|
||||
}, trace
|
||||
|
||||
|
||||
def run_load(args):
|
||||
check(
|
||||
1 <= args.duration <= 300
|
||||
and 1 <= args.rate <= 40
|
||||
and 1 <= args.concurrency <= 16,
|
||||
"Load profile outside bounds",
|
||||
)
|
||||
client = Client(args.base_url)
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
project = ComposeProject()
|
||||
container_ids = project.run("ps", "--quiet").splitlines()
|
||||
samples = []
|
||||
stop = threading.Event()
|
||||
|
||||
def sample():
|
||||
while not stop.is_set():
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"stats",
|
||||
"--no-stream",
|
||||
"--format",
|
||||
"{{json .}}",
|
||||
*container_ids,
|
||||
],
|
||||
capture_output=True,
|
||||
encoding="utf-8",
|
||||
timeout=15,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
samples.append(
|
||||
{
|
||||
"elapsed_seconds": round(time.monotonic() - started, 3),
|
||||
"containers": [
|
||||
{
|
||||
k: row[k]
|
||||
for k in ("Name", "CPUPerc", "MemUsage", "MemPerc")
|
||||
}
|
||||
for row in (
|
||||
json.loads(line) for line in result.stdout.splitlines()
|
||||
)
|
||||
],
|
||||
}
|
||||
)
|
||||
stop.wait(3)
|
||||
|
||||
def write(index):
|
||||
local = Client(args.base_url)
|
||||
before = time.monotonic()
|
||||
try:
|
||||
local.message(actor, chat, "synthetic-observability-" + str(index))
|
||||
return time.monotonic() - before
|
||||
except Failure:
|
||||
return None
|
||||
|
||||
started = time.monotonic()
|
||||
sampler = threading.Thread(target=sample)
|
||||
sampler.start()
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
futures = []
|
||||
for index in range(args.duration * args.rate):
|
||||
time.sleep(max(0, started + index / args.rate - time.monotonic()))
|
||||
futures.append(pool.submit(write, index))
|
||||
results = [future.result() for future in futures]
|
||||
elapsed = time.monotonic() - started
|
||||
finally:
|
||||
stop.set()
|
||||
sampler.join()
|
||||
latencies = sorted(value for value in results if value is not None)
|
||||
failed = len(results) - len(latencies)
|
||||
|
||||
def percentile(q):
|
||||
return (
|
||||
round(latencies[max(0, math.ceil(q * len(latencies)) - 1)] * 1000, 3)
|
||||
if latencies
|
||||
else None
|
||||
)
|
||||
|
||||
report = {
|
||||
"date_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"workload": "synthetic POST messages; fixed arrival rate; no retries",
|
||||
"duration_seconds": args.duration,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"arrival_rate": args.rate,
|
||||
"concurrency_limit": args.concurrency,
|
||||
"successful_requests": len(latencies),
|
||||
"failed_requests": failed,
|
||||
"error_ratio": failed / len(results),
|
||||
"successful_rps": round(len(latencies) / elapsed, 3),
|
||||
"successful_latency_ms": {
|
||||
name: percentile(q)
|
||||
for name, q in [("p50", 0.5), ("p95", 0.95), ("p99", 0.99)]
|
||||
},
|
||||
"dataset": {"users": 3, "chats": 1, "messages": len(latencies)},
|
||||
"resource_samples": samples,
|
||||
"limits": "Latency covers successes from worker start; client scheduling is not included. Small controlled load, not a capacity benchmark.",
|
||||
}
|
||||
check(bool(latencies) and failed == 0, "Controlled load contains errors")
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def metric(grafana, expression):
|
||||
rows = grafana.query(expression)
|
||||
return float(rows[0]["value"][1]) if rows else None
|
||||
|
||||
|
||||
def alert_firing(grafana, name):
|
||||
return bool(grafana.query(f'ALERTS{{alertname="{name}",alertstate="firing"}}'))
|
||||
|
||||
|
||||
def trace_chain(grafana, trace_id, *, retry=False):
|
||||
def inspect():
|
||||
spans = spans_in(grafana.trace(trace_id))
|
||||
by_id = {identifier_hex(span["spanId"], 8): span for span in spans}
|
||||
workers = [
|
||||
span
|
||||
for span in spans
|
||||
if span["name"] == "attachment.process.v1"
|
||||
and span.get("kind") in ("SPAN_KIND_CONSUMER", 5)
|
||||
]
|
||||
for worker in workers:
|
||||
publisher = by_id.get(identifier_hex(worker.get("parentSpanId", ""), 8), {})
|
||||
if publisher.get("kind") not in ("SPAN_KIND_PRODUCER", 4):
|
||||
return None
|
||||
upload = by_id.get(identifier_hex(publisher.get("parentSpanId", ""), 8), {})
|
||||
if upload.get("name") != "POST /api/v1/chats/{chat_id}/attachments":
|
||||
return None
|
||||
names = {span["name"] for span in spans}
|
||||
if (
|
||||
not workers
|
||||
or not {"s3.read_staging", "s3.write_result", "postgres.commit_ready"}
|
||||
<= names
|
||||
):
|
||||
return None
|
||||
if retry:
|
||||
outcomes = set()
|
||||
for span in workers:
|
||||
attrs = {row["key"]: row["value"] for row in span.get("attributes", [])}
|
||||
outcomes.add(
|
||||
(
|
||||
int(attrs["job.attempt"]["intValue"]),
|
||||
attrs.get("outcome", {}).get("stringValue"),
|
||||
)
|
||||
)
|
||||
if not {(1, "error"), (2, "error")} <= outcomes or not any(
|
||||
n > 2 and outcome == "success" for n, outcome in outcomes
|
||||
):
|
||||
return None
|
||||
return [
|
||||
{
|
||||
"name": span["name"],
|
||||
"kind": span.get("kind"),
|
||||
"span_id": identifier_hex(span["spanId"], 8),
|
||||
"parent_span_id": identifier_hex(span.get("parentSpanId", ""), 8),
|
||||
"job": {
|
||||
entry["key"]: entry["value"]
|
||||
for entry in span.get("attributes", [])
|
||||
if entry["key"] in {"job.attempt", "outcome"}
|
||||
},
|
||||
}
|
||||
for span in spans
|
||||
]
|
||||
|
||||
return eventually(inspect, "Missing upload -> producer -> consumer -> S3/DB chain")
|
||||
|
||||
|
||||
def telemetry_outage(project, client, grafana, actor, chat):
|
||||
failed = 'sum({__name__=~"otelcol_exporter_send_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
|
||||
refused = 'sum({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
|
||||
baseline = [metric(grafana, expr) for expr in (failed, refused)]
|
||||
rss = 'sum({__name__=~"otelcol_process_memory_rss(_bytes)?"})'
|
||||
samples = []
|
||||
successful = 0
|
||||
try:
|
||||
project.run("stop", "loki")
|
||||
start = time.monotonic()
|
||||
for second in range(51):
|
||||
for index in range(4):
|
||||
client.message(actor, chat, f"outage-{second}-{index}")
|
||||
successful += 1
|
||||
if second % 5 == 0:
|
||||
samples.append(
|
||||
{
|
||||
"seconds": round(time.monotonic() - start, 2),
|
||||
"rss_bytes": metric(grafana, rss),
|
||||
"queued_batches": metric(
|
||||
grafana,
|
||||
'sum(otelcol_exporter_queue_size{exporter="otlphttp/loki"})',
|
||||
),
|
||||
"capacity_batches": metric(
|
||||
grafana,
|
||||
'sum(otelcol_exporter_queue_capacity{exporter="otlphttp/loki"})',
|
||||
),
|
||||
}
|
||||
)
|
||||
time.sleep(max(0, start + second + 1 - time.monotonic()))
|
||||
losses = [
|
||||
metric(grafana, expr) - count
|
||||
for expr, count in zip((failed, refused), baseline)
|
||||
]
|
||||
check(sum(losses) > 0, "No measured Collector log loss during prolonged outage")
|
||||
check(
|
||||
all(
|
||||
0 <= row["queued_batches"] <= row["capacity_batches"] == 256
|
||||
for row in samples
|
||||
),
|
||||
"Unexpected queue bound",
|
||||
)
|
||||
check(
|
||||
max(row["rss_bytes"] for row in samples) < 256 * 1024**2,
|
||||
"Collector crossed container memory limit",
|
||||
)
|
||||
finally:
|
||||
project.run("start", "loki")
|
||||
eventually(
|
||||
lambda: grafana.proxy("loki", "/loki/api/v1/labels").get("status") == "success",
|
||||
"Loki did not recover",
|
||||
)
|
||||
restored = client.message(actor, chat, "telemetry-backend-restored")
|
||||
correlation(
|
||||
grafana,
|
||||
"messenger-api",
|
||||
time.time_ns() - 30 * 10**9,
|
||||
trace_id=restored["headers"]["X-Trace-Id"],
|
||||
)
|
||||
return {
|
||||
"scenario": "loki_unavailable",
|
||||
"successful_messages": successful,
|
||||
"dropped_after_export_failure": losses[0],
|
||||
"dropped_at_enqueue": losses[1],
|
||||
"samples": samples,
|
||||
"log_export_recovered": True,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--grafana-url", default="http://localhost:13000")
|
||||
parser.add_argument("--incidents", action="store_true")
|
||||
parser.add_argument("--restart-telemetry", action="store_true")
|
||||
parser.add_argument("--output", default="evidence/observability.json")
|
||||
parser.add_argument("--duration", type=int, default=10)
|
||||
parser.add_argument("--rate", type=int, default=10)
|
||||
parser.add_argument("--concurrency", type=int, default=2)
|
||||
args = parser.parse_args()
|
||||
load = run_load(args)
|
||||
start = time.time_ns()
|
||||
project = ComposeProject()
|
||||
client = Client(args.base_url, timeout=40)
|
||||
aliases = {r.instance_id for r in project.replicas() if r.running}
|
||||
check(len(aliases) >= 2, "Need two real API containers")
|
||||
check(
|
||||
client.call("GET", "/health/live")["headers"].get("X-Instance-Id") in aliases,
|
||||
"Wrong Compose project for BASE_URL",
|
||||
)
|
||||
grafana = Grafana(args.grafana_url)
|
||||
eventually(
|
||||
lambda: grafana.get("/api/health").get("database") == "ok",
|
||||
"Grafana is not ready",
|
||||
)
|
||||
sources = grafana.get("/api/datasources")
|
||||
check(
|
||||
{"prometheus", "loki", "tempo"} <= {source["uid"] for source in sources},
|
||||
"Missing provisioned datasource",
|
||||
)
|
||||
dashboard = grafana.get("/api/dashboards/uid/messenger-overview")["dashboard"]
|
||||
check(len(dashboard["panels"]) >= 8, "Dashboard is missing panels")
|
||||
actor, bob, eve, chat = client.bootstrap()
|
||||
sentinel = "private-content-" + uuid4().hex
|
||||
response = client.message(actor, chat, sentinel)
|
||||
controlled_trace = response["headers"].get("X-Trace-Id")
|
||||
check(bool(controlled_trace), "No API trace identifier")
|
||||
attachment, _ = client.attachment(actor, chat)
|
||||
client.wait_ready(actor, attachment)
|
||||
api, api_trace = correlation(
|
||||
grafana,
|
||||
"messenger-api",
|
||||
start,
|
||||
trace_id=controlled_trace,
|
||||
request_id=response["headers"]["X-Request-Id"],
|
||||
)
|
||||
worker, worker_trace = correlation(
|
||||
grafana, "messenger-worker", start, attachment_id=attachment["id"]
|
||||
)
|
||||
accepted_chain = trace_chain(grafana, worker["trace_id"])
|
||||
check(
|
||||
api["log"]["request_id"] == response["headers"]["X-Request-Id"],
|
||||
"Request id correlation lost",
|
||||
)
|
||||
check(
|
||||
any(span["kind"] in ("SPAN_KIND_SERVER", 2) for span in api["spans"]),
|
||||
"No server span",
|
||||
)
|
||||
check(
|
||||
any(span["kind"] in ("SPAN_KIND_CONSUMER", 5) for span in worker["spans"]),
|
||||
"No consumer span",
|
||||
)
|
||||
processed = eventually(
|
||||
lambda: grafana.logs(
|
||||
'{service_name="messenger-worker"} | json | event="job.processed" | trace_id="'
|
||||
+ worker["trace_id"]
|
||||
+ '"',
|
||||
start,
|
||||
),
|
||||
"No worker result log",
|
||||
)
|
||||
check(processed[0]["outcome"] == "success", "Controlled worker did not succeed")
|
||||
metrics = {}
|
||||
expressions = {
|
||||
"http_requests": "sum(messenger_http_requests_total)",
|
||||
"http_duration": "sum(messenger_http_duration_seconds_count)",
|
||||
"job_success": 'sum(messenger_job_attempts_total{outcome="success"})',
|
||||
"job_duration": "sum(messenger_job_duration_seconds_count)",
|
||||
}
|
||||
for name, expression in expressions.items():
|
||||
metrics[name] = eventually(
|
||||
lambda expression=expression: grafana.query(expression),
|
||||
"Missing metric: " + expression,
|
||||
)
|
||||
check(float(metrics[name][0]["value"][1]) > 0, "Empty metric: " + name)
|
||||
# Все PromQL из dashboard должны давать численные данные после traffic.
|
||||
for panel in dashboard["panels"]:
|
||||
if panel["datasource"]["uid"] == "prometheus":
|
||||
eventually(
|
||||
lambda panel=panel: all(
|
||||
grafana.query(target["expr"]) for target in panel["targets"]
|
||||
),
|
||||
"Empty dashboard panel: " + panel["title"],
|
||||
)
|
||||
forbidden_labels = {
|
||||
"user_id",
|
||||
"chat_id",
|
||||
"message_id",
|
||||
"attachment_id",
|
||||
"request_id",
|
||||
"filename",
|
||||
"url",
|
||||
"http_target",
|
||||
"http_url",
|
||||
}
|
||||
series = grafana.query('{__name__=~"messenger_.*"}')
|
||||
check(
|
||||
not any(forbidden_labels & set(row["metric"]) for row in series),
|
||||
"High cardinality metric label",
|
||||
)
|
||||
report = {
|
||||
"date_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"project": project.name,
|
||||
"load": load,
|
||||
"api_replicas": len(aliases),
|
||||
"dashboard_uid": dashboard["uid"],
|
||||
"api_correlation": api,
|
||||
"worker_correlation": worker,
|
||||
"metrics": metrics,
|
||||
"upload_trace": {"trace_id": worker["trace_id"], "chain": accepted_chain},
|
||||
"incidents": [],
|
||||
}
|
||||
if args.incidents:
|
||||
|
||||
def fail_request():
|
||||
return client.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
expected=503,
|
||||
body={
|
||||
"username": "fault_" + uuid4().hex[:12],
|
||||
"display_name": "Synthetic",
|
||||
"password": "not-exported-" + uuid4().hex,
|
||||
},
|
||||
)
|
||||
|
||||
before = time.monotonic()
|
||||
try:
|
||||
project.run("stop", "postgres")
|
||||
failed = fail_request()
|
||||
error_trace = failed["headers"].get("X-Trace-Id")
|
||||
error, db_trace = correlation(
|
||||
grafana,
|
||||
"messenger-api",
|
||||
start,
|
||||
trace_id=error_trace,
|
||||
request_id=failed["headers"]["X-Request-Id"],
|
||||
)
|
||||
diagnostics = eventually(
|
||||
lambda: grafana.logs(
|
||||
'{service_name="messenger-api"} | json | event="dependency.unavailable" | trace_id="'
|
||||
+ error_trace
|
||||
+ '"',
|
||||
start,
|
||||
),
|
||||
"No dependency diagnostic log",
|
||||
)
|
||||
checkout = next(
|
||||
span
|
||||
for span in spans_in(db_trace)
|
||||
if span["name"] == "postgres.checkout"
|
||||
and span.get("status", {}).get("code") in ("STATUS_CODE_ERROR", 2)
|
||||
)
|
||||
|
||||
def db_alert():
|
||||
for _ in range(4):
|
||||
fail_request()
|
||||
return alert_firing(grafana, "MessengerHighErrorRate")
|
||||
|
||||
eventually(db_alert, "Error rate alert did not fire", timeout=75)
|
||||
report["incidents"].append(
|
||||
{
|
||||
"scenario": "postgres_unavailable",
|
||||
"http_status": 503,
|
||||
"correlation": error,
|
||||
"diagnostic": diagnostics[0],
|
||||
"db_span": checkout["name"],
|
||||
"alert": "MessengerHighErrorRate",
|
||||
"firing_after_seconds": round(time.monotonic() - before, 3),
|
||||
}
|
||||
)
|
||||
finally:
|
||||
project.run("start", "postgres")
|
||||
eventually(lambda: _ready(client), "API did not recover after PostgreSQL")
|
||||
eventually(
|
||||
lambda: not alert_firing(grafana, "MessengerHighErrorRate"),
|
||||
"Error rate alert not resolved",
|
||||
)
|
||||
report["incidents"][-1]["resolved_after_seconds"] = round(
|
||||
time.monotonic() - before, 3
|
||||
)
|
||||
|
||||
from scripts.check_jobs import retry_replay
|
||||
|
||||
queue = {}
|
||||
|
||||
def observe_queue(_):
|
||||
observed = time.monotonic()
|
||||
eventually(
|
||||
lambda: alert_firing(grafana, "MessengerQueueLag"),
|
||||
"Queue lag alert did not fire",
|
||||
)
|
||||
queue.update(
|
||||
depth=metric(grafana, "max(messenger_queue_depth_ratio)"),
|
||||
oldest_seconds=metric(grafana, "max(messenger_queue_oldest_seconds)"),
|
||||
firing_after_seconds=round(time.monotonic() - observed, 3),
|
||||
)
|
||||
|
||||
incident = retry_replay(
|
||||
client, actor, chat, permanent=False, on_queued=observe_queue
|
||||
)
|
||||
retried_worker, retry_trace = correlation(
|
||||
grafana, "messenger-worker", start, attachment_id=incident["attachment_id"]
|
||||
)
|
||||
chain = trace_chain(grafana, retried_worker["trace_id"], retry=True)
|
||||
eventually(
|
||||
lambda: not alert_firing(grafana, "MessengerQueueLag")
|
||||
and metric(grafana, "max(messenger_queue_depth_ratio)") == 0,
|
||||
"Backlog did not drain",
|
||||
)
|
||||
queue.update(resolved=True, final_depth=0)
|
||||
report["incidents"].append(
|
||||
{
|
||||
"scenario": "worker_stopped_then_transient_retry",
|
||||
"alert": "MessengerQueueLag",
|
||||
"queue": queue,
|
||||
"recovered_status": incident["after"]["status"],
|
||||
"trace_id": retried_worker["trace_id"],
|
||||
"chain": chain,
|
||||
}
|
||||
)
|
||||
report["incidents"].append(
|
||||
telemetry_outage(project, client, grafana, actor, chat)
|
||||
)
|
||||
else:
|
||||
retry_trace = {}
|
||||
# Не сохраняем API bodies или SDK exceptions; проверяем реальные выбранные сигналы.
|
||||
logs = grafana.logs('{service_name=~"messenger-api|messenger-worker"}', start)
|
||||
exported = json.dumps([logs, api_trace, worker_trace, retry_trace])
|
||||
private = [
|
||||
sentinel,
|
||||
*(person["token"] for person in (actor, bob, eve)),
|
||||
*(person["registration"]["password"] for person in (actor, bob, eve)),
|
||||
]
|
||||
private.extend(
|
||||
path.read_text(encoding="utf-8").strip()
|
||||
for path in (ROOT / "secrets").iterdir()
|
||||
if path.is_file()
|
||||
)
|
||||
check(
|
||||
not any(secret and secret in exported for secret in private),
|
||||
"Private value found in telemetry",
|
||||
)
|
||||
check(
|
||||
all(
|
||||
record.get("trace_id")
|
||||
and record.get("span_id")
|
||||
and record.get("timestamp")
|
||||
and record.get("severity")
|
||||
for record in logs
|
||||
),
|
||||
"Incomplete structured logs",
|
||||
)
|
||||
report["privacy_checked_logs"] = len(logs)
|
||||
report["checks"] = {
|
||||
"three_backends": True,
|
||||
"dashboard_queries": True,
|
||||
"server_consumer_traces": True,
|
||||
"log_trace_correlation": True,
|
||||
"low_cardinality": True,
|
||||
"privacy": True,
|
||||
}
|
||||
if args.restart_telemetry:
|
||||
historical_time = time.time()
|
||||
expression = "sum(messenger_http_requests_total)"
|
||||
before = grafana.proxy(
|
||||
"prometheus", "/api/v1/query", query=expression, time=historical_time
|
||||
)["data"]["result"]
|
||||
project.run("restart", "prometheus", "loki", "tempo", "grafana")
|
||||
after = eventually(
|
||||
lambda: grafana.proxy(
|
||||
"prometheus", "/api/v1/query", query=expression, time=historical_time
|
||||
)["data"]["result"],
|
||||
"Prometheus history lost",
|
||||
)
|
||||
check(before == after, "Historical metric changed after storage restart")
|
||||
correlation(grafana, "messenger-api", start, trace_id=api["trace_id"])
|
||||
correlation(grafana, "messenger-worker", start, trace_id=worker["trace_id"])
|
||||
report["checks"]["history_survives_restart"] = True
|
||||
destination = ROOT / args.output
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
print(
|
||||
"PASS: telemetry, dashboard, privacy, correlations"
|
||||
+ ("; alerts and outage recovery verified" if args.incidents else "")
|
||||
)
|
||||
|
||||
|
||||
def _ready(client):
|
||||
return client.call("GET", "/health/ready", expected=[200, 503])["status"] == 200
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Сценарий COMMIT → SIGKILL до publish → ready без повторного upload.
|
||||
|
||||
Требуется уже работающий отдельный стенд. Скрипт не выполняется при запуске API.
|
||||
Публичные tests не изменяет, secrets/tokens не печатает и evidence сам не сохраняет.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from uuid import uuid4
|
||||
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, check
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
COMPOSE + list(arguments),
|
||||
cwd=ROOT,
|
||||
encoding="utf-8",
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def require_compose(*arguments: str) -> str:
|
||||
result = compose(*arguments)
|
||||
check(result.returncode == 0, "Compose command failed; output hidden")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Отказ publisher после COMMIT перед RabbitMQ publish"
|
||||
)
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--ca-file")
|
||||
args = parser.parse_args()
|
||||
running = set(
|
||||
require_compose("ps", "--status", "running", "--services").splitlines()
|
||||
)
|
||||
check(
|
||||
{"api", "postgres", "minio", "broker", "publisher", "worker"} <= running,
|
||||
"Start the application stack before this scenario",
|
||||
)
|
||||
client = Client(args.base_url, args.ca_file, timeout=40)
|
||||
project = ComposeProject()
|
||||
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
|
||||
live = client.call("GET", "/health/live", schema="Health")
|
||||
check(
|
||||
live["headers"].get("X-Instance-Id") in aliases,
|
||||
"BASE_URL does not belong to the selected Compose project",
|
||||
)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
attachment = None
|
||||
fault_container = "messenger-outbox-fault-" + uuid4().hex
|
||||
try:
|
||||
require_compose("stop", "publisher")
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
message = client.message(alice, chat, "", attachments=[attachment["id"]])[
|
||||
"json"
|
||||
]
|
||||
client.call(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment['id']}/content",
|
||||
expected=409,
|
||||
actor=alice,
|
||||
)
|
||||
killed = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"--name",
|
||||
fault_container,
|
||||
"publisher",
|
||||
"python",
|
||||
"-m",
|
||||
"app.core.outbox_publisher",
|
||||
"--crash-before-publish",
|
||||
attachment["id"],
|
||||
)
|
||||
marker = json.dumps(
|
||||
{"fault": "before_publish", "attachment_id": attachment["id"]}
|
||||
)
|
||||
check(
|
||||
killed.returncode == 137 and marker in killed.stdout,
|
||||
"Expected targeted SIGKILL before publish was not observed",
|
||||
)
|
||||
pending = client.call(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment['id']}",
|
||||
actor=alice,
|
||||
schema="Attachment",
|
||||
)["json"]
|
||||
check(pending["status"] == "queued", "Crash must leave accepted job queued")
|
||||
finally:
|
||||
|
||||
try:
|
||||
subprocess.run(
|
||||
["docker", "rm", "-f", fault_container], capture_output=True, timeout=30
|
||||
)
|
||||
finally:
|
||||
require_compose("start", "publisher")
|
||||
ready = client.wait_ready(alice, attachment, wait_seconds=60)
|
||||
check(ready["id"] == attachment["id"], "Recovered a different attachment")
|
||||
check(
|
||||
{variant["name"] for variant in ready["variants"]}
|
||||
== {"thumbnail", "crop", "optimized"},
|
||||
"Missing required variants",
|
||||
)
|
||||
path = f"/api/v1/attachments/{attachment['id']}/content"
|
||||
check(
|
||||
client.call("GET", path, actor=alice)["bytes"] == original,
|
||||
"Original changed after recovery",
|
||||
)
|
||||
for variant in ready["variants"]:
|
||||
downloaded = client.call(
|
||||
"GET", path + "?variant=" + variant["name"], actor=alice
|
||||
)["bytes"]
|
||||
check(
|
||||
len(downloaded) == variant["size_bytes"]
|
||||
and hashlib.sha256(downloaded).hexdigest() == variant["sha256"],
|
||||
"Recovered variant checksum mismatch",
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"scenario": "commit_before_publish",
|
||||
"attachment_id": ready["id"],
|
||||
"message_id": message["id"],
|
||||
"publisher_exit_code": killed.returncode,
|
||||
"status_after_crash": pending["status"],
|
||||
"status_after_restart": ready["status"],
|
||||
"variants": [variant["name"] for variant in ready["variants"]],
|
||||
"reuploaded": False,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Реальный MinIO: повтор bootstrap, private bucket, ограниченные права и отказ."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, arguments, check, run
|
||||
|
||||
PROBE = r"""
|
||||
import asyncio, json, sys
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.error import HTTPError, URLError
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.storage.s3 import create_s3_storage
|
||||
from app.models.attachment import Attachment
|
||||
from minio.error import S3Error
|
||||
async def main():
|
||||
settings = Settings()
|
||||
engine = create_db_engine(settings)
|
||||
storage = create_s3_storage(settings.s3)
|
||||
try:
|
||||
async with create_session_factory(engine)() as session:
|
||||
record = await session.get(Attachment, UUID(json.load(sys.stdin)['attachment_id']))
|
||||
key = record.object_key
|
||||
results = {}
|
||||
for method in ('GET', 'PUT'):
|
||||
request = Request(settings.s3.endpoint + '/' + settings.s3.bucket + '/' + key,
|
||||
method=method, data=b'x' if method == 'PUT' else None)
|
||||
try:
|
||||
with urlopen(request, timeout=3) as response:
|
||||
results[method] = response.status
|
||||
except HTTPError as error:
|
||||
results[method] = error.code
|
||||
error.close()
|
||||
try:
|
||||
storage._client.get_bucket_policy(settings.s3.bucket)
|
||||
results['admin_denied'] = False
|
||||
except S3Error as error:
|
||||
results['admin_denied'] = error.code == 'AccessDenied'
|
||||
try:
|
||||
buckets = storage._client.list_buckets()
|
||||
results['other_buckets_denied'] = all(bucket.name == settings.s3.bucket for bucket in buckets)
|
||||
except S3Error as error:
|
||||
results['other_buckets_denied'] = error.code == 'AccessDenied'
|
||||
try:
|
||||
with urlopen(settings.s3.endpoint.replace(':9000', ':9001'), timeout=1):
|
||||
results['console_unreachable'] = False
|
||||
except HTTPError as error:
|
||||
results['console_unreachable'] = False
|
||||
error.close()
|
||||
except URLError:
|
||||
results['console_unreachable'] = True
|
||||
results['temp_originals_absent'] = not any(Path('/tmp').glob('messenger-upload-*'))
|
||||
results['root_files_absent'] = not any(Path('/run/secrets', name).exists()
|
||||
for name in ('minio_root_user', 'minio_root_password'))
|
||||
print(json.dumps(results))
|
||||
finally:
|
||||
await storage.aclose()
|
||||
await engine.dispose()
|
||||
asyncio.run(main())
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument(
|
||||
"--with-outage",
|
||||
action="store_true",
|
||||
help="временно остановить MinIO выбранного проекта",
|
||||
)
|
||||
|
||||
args = arguments("MinIO private bucket and failure checks", configure)
|
||||
project = ComposeProject()
|
||||
client = Client(args.base_url, args.ca_file, timeout=40)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
|
||||
live = client.call("GET", "/health/live", schema="Health")
|
||||
check(
|
||||
live["headers"].get("X-Instance-Id") in aliases,
|
||||
"BASE_URL does not belong to the selected Compose project",
|
||||
)
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
attachment = client.wait_ready(alice, attachment)
|
||||
|
||||
for _ in range(2):
|
||||
project.run("run", "--rm", "--no-deps", "minio-bootstrap", timeout=100)
|
||||
replicas = project.replicas()
|
||||
check(bool(replicas), "API not running")
|
||||
result = subprocess.run(
|
||||
["docker", "exec", "-i", replicas[0].id, "python", "-c", PROBE],
|
||||
input=json.dumps({"attachment_id": attachment["id"]}),
|
||||
encoding="utf-8",
|
||||
capture_output=True,
|
||||
timeout=20,
|
||||
)
|
||||
check(result.returncode == 0, "private bucket probe failed (details hidden)")
|
||||
statuses = json.loads(result.stdout)
|
||||
check(statuses["GET"] == statuses["PUT"] == 403, "anonymous S3 access allowed")
|
||||
check(
|
||||
statuses["admin_denied"]
|
||||
and statuses["other_buckets_denied"]
|
||||
and statuses["root_files_absent"],
|
||||
"API has excessive access",
|
||||
)
|
||||
check(statuses["console_unreachable"], "console reachable from API network")
|
||||
check(statuses["temp_originals_absent"], "API retained temporary originals")
|
||||
check(
|
||||
not project.config["services"]["minio"].get("ports"),
|
||||
"MinIO publishes host ports",
|
||||
)
|
||||
env = project.config["services"]["minio"]["environment"]
|
||||
check(env["MINIO_BROWSER"] == "off", "browser enabled")
|
||||
check(
|
||||
"127.0.0.1:9001" in project.config["services"]["minio"]["command"],
|
||||
"console is not bound to loopback",
|
||||
)
|
||||
path = "/api/v1/attachments/" + attachment["id"] + "/content"
|
||||
check(
|
||||
client.call("GET", path, actor=alice)["bytes"] == original,
|
||||
"bootstrap changed original",
|
||||
)
|
||||
if args.with_outage:
|
||||
try:
|
||||
project.run("stop", "minio")
|
||||
client.call("GET", path, expected=503, actor=alice)
|
||||
finally:
|
||||
project.run("start", "minio")
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except Exception:
|
||||
if time.monotonic() >= deadline:
|
||||
raise
|
||||
time.sleep(0.5)
|
||||
check(
|
||||
client.call("GET", path, actor=alice)["bytes"] == original,
|
||||
"original lost after MinIO restart",
|
||||
)
|
||||
print(
|
||||
"PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console"
|
||||
+ (", MinIO outage/recovery" if args.with_outage else "")
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,201 @@
|
||||
"""Последовательная проверка лабораторной 6 в отдельном Docker-окружении."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tarfile
|
||||
import time
|
||||
|
||||
ROOT = Path("/workspace")
|
||||
EVIDENCE = ROOT / "evidence"
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
BASE = ["--base-url", "http://127.0.0.1:28080"]
|
||||
|
||||
|
||||
def step(report, name, command, *, timeout=600, env=None, required=True):
|
||||
print("Запуск " + name, flush=True)
|
||||
started = time.monotonic()
|
||||
log = name + ".txt"
|
||||
with (EVIDENCE / log).open("w", encoding="utf-8", newline="\n") as output:
|
||||
output.write("$ " + " ".join(command) + "\n")
|
||||
output.flush()
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
stdout=output,
|
||||
stderr=subprocess.STDOUT,
|
||||
timeout=timeout,
|
||||
env={**os.environ, **(env or {})},
|
||||
)
|
||||
report["checks"].append(
|
||||
{
|
||||
"name": name,
|
||||
"exit_code": result.returncode,
|
||||
"duration_seconds": round(time.monotonic() - started, 3),
|
||||
"log": log,
|
||||
}
|
||||
)
|
||||
if result.returncode:
|
||||
print("Ошибка " + name, flush=True)
|
||||
if required:
|
||||
result.check_returncode()
|
||||
else:
|
||||
print("Готово " + name, flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
with tarfile.open("/tmp/source.tar") as archive:
|
||||
archive.extractall(ROOT, filter="data")
|
||||
os.chdir(ROOT)
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
deadline = time.monotonic() + 120
|
||||
while subprocess.run(["docker", "info"], capture_output=True).returncode:
|
||||
if time.monotonic() > deadline:
|
||||
raise RuntimeError("Проверочный Docker не запустился")
|
||||
time.sleep(1)
|
||||
|
||||
os.environ.update(
|
||||
COMPOSE_PROJECT_NAME="messenger-test",
|
||||
COMPOSE_FILE="compose.yaml:compose.test.yaml",
|
||||
LB_HOST_PORT="28080",
|
||||
GRAFANA_HOST_PORT="23000",
|
||||
DOCKER_TESTS="1",
|
||||
TEST_USE_MIGRATIONS="1",
|
||||
TEST_REDIS_HOST="127.0.0.1",
|
||||
TEST_REDIS_PORT="26379",
|
||||
TEST_REDIS_DB="15",
|
||||
)
|
||||
integration_only = os.environ.get("INTEGRATION_ONLY") == "1"
|
||||
report = {
|
||||
"status": "failed",
|
||||
"profile": "integration" if integration_only else "full",
|
||||
"started_at": datetime.now(timezone.utc).isoformat(),
|
||||
"checks": [],
|
||||
}
|
||||
|
||||
try:
|
||||
step(report, "00_init", ["python", "scripts/init_local.py"])
|
||||
step(report, "01_contract", ["python", "scripts/check_contract.py"])
|
||||
step(report, "02_openapi", ["python", "scripts/validate_openapi.py"])
|
||||
step(
|
||||
report,
|
||||
"03_import_images",
|
||||
["docker", "image", "load", "--input", "/tmp/images.tar"],
|
||||
)
|
||||
step(
|
||||
report,
|
||||
"03_startup",
|
||||
COMPOSE + ["up", "-d", "--no-build", "--pull", "never"],
|
||||
timeout=3600,
|
||||
)
|
||||
step(
|
||||
report,
|
||||
"03_readiness",
|
||||
COMPOSE
|
||||
+ [
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--wait",
|
||||
"--wait-timeout",
|
||||
"240",
|
||||
"nginx",
|
||||
"grafana",
|
||||
],
|
||||
)
|
||||
if not integration_only:
|
||||
password = (ROOT / "secrets/db_password").read_text(encoding="utf-8")
|
||||
test_env = {
|
||||
"OTEL_ENABLED": "false",
|
||||
"TEST_DATABASE_URL": (
|
||||
f"postgresql+asyncpg://messenger:{password}@127.0.0.1:25432/messenger_test"
|
||||
),
|
||||
"TEST_REDIS_PASSWORD": (ROOT / "secrets/redis_password").read_text(
|
||||
encoding="utf-8"
|
||||
),
|
||||
}
|
||||
step(
|
||||
report,
|
||||
"04_test_database",
|
||||
COMPOSE
|
||||
+ [
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"createdb",
|
||||
"-U",
|
||||
"messenger",
|
||||
"messenger_test",
|
||||
],
|
||||
)
|
||||
step(
|
||||
report,
|
||||
"05_pytest",
|
||||
["python", "-m", "pytest", "-q", "--junitxml=evidence/pytest.xml"],
|
||||
timeout=1800,
|
||||
env=test_env,
|
||||
required=False,
|
||||
)
|
||||
step(report, "06_smoke", ["python", "tests/smoke.py", *BASE], required=False)
|
||||
step(
|
||||
report,
|
||||
"07_observability",
|
||||
[
|
||||
"python",
|
||||
"-m",
|
||||
"scripts.check_observability",
|
||||
*BASE,
|
||||
"--grafana-url",
|
||||
"http://127.0.0.1:23000",
|
||||
"--incidents",
|
||||
"--restart-telemetry",
|
||||
],
|
||||
timeout=900,
|
||||
required=False,
|
||||
)
|
||||
for number, mode, hook in (
|
||||
("08", "persistence", "restart"),
|
||||
("09", "failover", "stop-one"),
|
||||
):
|
||||
step(
|
||||
report,
|
||||
f"{number}_{mode}",
|
||||
[
|
||||
"python",
|
||||
"tests/resilience.py",
|
||||
"--mode",
|
||||
mode,
|
||||
"--action-script",
|
||||
f"scripts/{hook}.sh",
|
||||
*BASE,
|
||||
],
|
||||
required=False,
|
||||
)
|
||||
if all(check["exit_code"] == 0 for check in report["checks"]):
|
||||
report["status"] = "passed"
|
||||
finally:
|
||||
try:
|
||||
step(
|
||||
report,
|
||||
"10_cleanup",
|
||||
COMPOSE + ["down", "--volumes", "--remove-orphans"],
|
||||
timeout=180,
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
report["status"] = "failed"
|
||||
raise
|
||||
finally:
|
||||
report["finished_at"] = datetime.now(timezone.utc).isoformat()
|
||||
(EVIDENCE / "verification.json").write_text(
|
||||
json.dumps(report, indent=2, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
if report["status"] != "passed":
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Общие операции с репликами текущего Compose-проекта."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Replica:
|
||||
|
||||
id: str
|
||||
image: str
|
||||
instance_id: str
|
||||
running: bool
|
||||
health: str | None
|
||||
|
||||
|
||||
class ComposeProject:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.command = ["docker", "compose", "--env-file", ".env"]
|
||||
self.config = json.loads(self.run("config", "--format", "json"))
|
||||
self.name = self.config["name"]
|
||||
|
||||
def docker(self, *args: str, timeout: int = 90, env=None) -> str:
|
||||
result = subprocess.run(
|
||||
["docker", *args],
|
||||
cwd=ROOT,
|
||||
encoding="utf-8",
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
env={**os.environ, **(env or {})},
|
||||
)
|
||||
if result.returncode:
|
||||
raise RuntimeError(result.stderr.strip() or result.stdout.strip())
|
||||
return result.stdout.strip()
|
||||
|
||||
def run(self, *args: str, timeout: int = 90, env=None) -> str:
|
||||
return self.docker(*self.command[1:], *args, timeout=timeout, env=env)
|
||||
|
||||
def replicas(self) -> list[Replica]:
|
||||
ids = self.run("ps", "--all", "--quiet", "api").splitlines()
|
||||
if not ids:
|
||||
return []
|
||||
template = (
|
||||
'{"id":{{json .Id}},"image":{{json .Image}},'
|
||||
'"hostname":{{json .Config.Hostname}},"running":{{json .State.Running}},'
|
||||
'"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}}'
|
||||
)
|
||||
rows = self.docker("inspect", "--format", template, *ids).splitlines()
|
||||
configured = str(
|
||||
self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID")
|
||||
or ""
|
||||
).strip()
|
||||
return [
|
||||
Replica(
|
||||
row["id"],
|
||||
row["image"],
|
||||
configured or row["hostname"],
|
||||
row["running"],
|
||||
row["health"],
|
||||
)
|
||||
for row in map(json.loads, rows)
|
||||
]
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Создаёт только отсутствующие локальные настройки и файловые секреты."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
|
||||
def main() -> None:
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
destination = root / ".env"
|
||||
if not destination.exists():
|
||||
with destination.open("x", encoding="utf-8", newline="\n") as target:
|
||||
target.write((root / ".env.example").read_text(encoding="utf-8"))
|
||||
destination.chmod(0o600)
|
||||
directory = root / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
for name in (
|
||||
"db_password",
|
||||
"redis_password",
|
||||
"broker_password",
|
||||
"minio_root_user",
|
||||
"minio_root_password",
|
||||
"s3_access_key_id",
|
||||
"s3_secret_access_key",
|
||||
"grafana_admin_password",
|
||||
):
|
||||
path = directory / name
|
||||
try:
|
||||
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444)
|
||||
except FileExistsError:
|
||||
continue
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as target:
|
||||
target.write(secrets.token_urlsafe(24))
|
||||
print("Создан файл " + str(path.relative_to(root)))
|
||||
print("Настройки готовы; существующие значения сохранены")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
compose() {
|
||||
docker compose --env-file .env "$@"
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
for service in "$@"; do
|
||||
attempts=0
|
||||
until states=$(compose ps --format '{{.Health}}' "$service") \
|
||||
&& [ -n "$states" ] \
|
||||
&& [ -z "$(printf '%s\n' "$states" | grep -v '^healthy$')" ]; do
|
||||
attempts=$((attempts + 1))
|
||||
if [ "$attempts" -ge 60 ]; then
|
||||
echo "$service не стал healthy за 60 секунд" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
worker_running=$(compose ps --status running --quiet worker)
|
||||
compose stop worker publisher cleanup
|
||||
compose restart postgres redis minio broker
|
||||
wait_healthy postgres redis minio broker
|
||||
compose restart api
|
||||
wait_healthy api
|
||||
compose restart nginx
|
||||
wait_healthy nginx
|
||||
compose start publisher cleanup
|
||||
if [ -n "$worker_running" ]; then
|
||||
compose start worker
|
||||
fi
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!a-zA-Z0-9_.:-]*|'') exit 1 ;;
|
||||
esac
|
||||
|
||||
target=""
|
||||
for container in $(docker compose --env-file .env ps -q api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container")
|
||||
configured=$(docker inspect --format '{{range .Config.Env}}{{if eq (index (split . "=") 0) "INSTANCE_ID"}}{{println .}}{{end}}{{end}}' "$container" \
|
||||
| sed -n 's/^INSTANCE_ID=//p' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
|
||||
if [ "$TARGET_INSTANCE_ID" = "$hostname" ] || [ "$TARGET_INSTANCE_ID" = "$configured" ]; then
|
||||
if [ -n "$target" ]; then
|
||||
echo 'Instance id matches multiple API containers' >&2
|
||||
exit 1
|
||||
fi
|
||||
target="$container"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$target" ]; then
|
||||
echo 'Target API instance not found in this Compose project' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Stopping API container $target" >&2
|
||||
docker stop --timeout 8 "$target" >/dev/null
|
||||
+258
@@ -0,0 +1,258 @@
|
||||
"""Запускает проверки в одноразовом Docker-окружении и сохраняет evidence."""
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
from uuid import uuid4
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EXCLUDED = {
|
||||
".git",
|
||||
".agents",
|
||||
".aws",
|
||||
".codex",
|
||||
".env",
|
||||
"secrets",
|
||||
"evidence",
|
||||
".venv",
|
||||
"venv",
|
||||
"__pycache__",
|
||||
".pytest_cache",
|
||||
".github",
|
||||
"docs",
|
||||
}
|
||||
TEXT_SUFFIXES = {".py", ".sh", ".conf", ".yaml", ".yml", ".json", ".txt", ".md", ".ini"}
|
||||
|
||||
|
||||
def docker(*args, **kwargs):
|
||||
kwargs.setdefault("check", True)
|
||||
return subprocess.run(["docker", *args], cwd=ROOT, **kwargs)
|
||||
|
||||
|
||||
def archive_source(destination):
|
||||
with tarfile.open(destination, "w") as archive:
|
||||
for path in sorted(ROOT.rglob("*")):
|
||||
relative = path.relative_to(ROOT)
|
||||
if (
|
||||
not path.is_file()
|
||||
or path.is_symlink()
|
||||
or any(part in EXCLUDED for part in relative.parts)
|
||||
or path.name.startswith(".env.")
|
||||
and path.name != ".env.example"
|
||||
or path.suffix in {".zip", ".log", ".pem", ".key"}
|
||||
):
|
||||
continue
|
||||
content = path.read_bytes()
|
||||
if path.suffix in TEXT_SUFFIXES:
|
||||
content = content.replace(b"\r\n", b"\n")
|
||||
entry = tarfile.TarInfo(relative.as_posix())
|
||||
entry.size = len(content)
|
||||
entry.mode = 0o755 if path.suffix == ".sh" else 0o644
|
||||
archive.addfile(entry, io.BytesIO(content))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument("--format", action="store_true", help="применить Black и выйти")
|
||||
mode.add_argument(
|
||||
"--integration-only",
|
||||
action="store_true",
|
||||
help="запустить проверки стенда и телеметрии без pytest",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
project = "messenger-check-" + uuid4().hex[:12]
|
||||
image = project + ":tools"
|
||||
image_env = {
|
||||
"API_IMAGE": project + ":api",
|
||||
"MINIO_IMAGE": project + ":server",
|
||||
"MINIO_BOOTSTRAP_IMAGE": project + ":bootstrap",
|
||||
}
|
||||
engine = project + "-engine"
|
||||
runner = project + "-runner"
|
||||
workspace = project + "-workspace"
|
||||
data = project + "-data"
|
||||
network = project + "-network"
|
||||
output = ROOT / "evidence" / "lab6"
|
||||
if args.integration_only:
|
||||
output /= "integration"
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
summary = {
|
||||
"status": "failed",
|
||||
"project": project,
|
||||
"profile": "integration" if args.integration_only else "full",
|
||||
"started_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
try:
|
||||
docker("build", "--tag", image, "--file", "infra/testing/Dockerfile", ".")
|
||||
if args.format:
|
||||
docker(
|
||||
"run",
|
||||
"--rm",
|
||||
"--volume",
|
||||
str(ROOT) + ":/workspace",
|
||||
image,
|
||||
"python3",
|
||||
"-m",
|
||||
"black",
|
||||
"app",
|
||||
"infra",
|
||||
"migrations",
|
||||
"scripts",
|
||||
"student-tests",
|
||||
"tests",
|
||||
)
|
||||
summary["status"] = "formatted"
|
||||
return
|
||||
compose = ("compose", "--env-file", ".env.example")
|
||||
env = {**os.environ, **image_env}
|
||||
config = json.loads(
|
||||
docker(
|
||||
*compose, "config", "--format", "json", env=env, capture_output=True
|
||||
).stdout
|
||||
)
|
||||
dependencies = sorted(
|
||||
{
|
||||
service["image"]
|
||||
for service in config["services"].values()
|
||||
if "build" not in service
|
||||
}
|
||||
)
|
||||
docker(
|
||||
*compose,
|
||||
"pull",
|
||||
"--ignore-buildable",
|
||||
"--policy",
|
||||
"missing",
|
||||
env=env,
|
||||
timeout=3600,
|
||||
)
|
||||
docker(
|
||||
*compose, "build", "api", "minio", "minio-bootstrap", env=env, timeout=3600
|
||||
)
|
||||
docker("network", "create", network, stdout=subprocess.DEVNULL)
|
||||
for volume in (workspace, data):
|
||||
docker("volume", "create", volume, stdout=subprocess.DEVNULL)
|
||||
docker(
|
||||
"run",
|
||||
"-d",
|
||||
"--privileged",
|
||||
"--name",
|
||||
engine,
|
||||
"--network",
|
||||
network,
|
||||
"--env",
|
||||
"DOCKER_TLS_CERTDIR=",
|
||||
"--volume",
|
||||
workspace + ":/workspace",
|
||||
"--volume",
|
||||
data + ":/var/lib/docker",
|
||||
"docker:29.1.3-dind",
|
||||
"dockerd",
|
||||
"--host=unix:///var/run/docker.sock",
|
||||
"--host=tcp://127.0.0.1:2375",
|
||||
"--tls=false",
|
||||
stdout=subprocess.DEVNULL,
|
||||
)
|
||||
docker(
|
||||
"create",
|
||||
"--name",
|
||||
runner,
|
||||
"--network",
|
||||
"container:" + engine,
|
||||
"--env",
|
||||
"DOCKER_HOST=tcp://127.0.0.1:2375",
|
||||
*[
|
||||
arg
|
||||
for key, value in image_env.items()
|
||||
for arg in ("--env", f"{key}={value}")
|
||||
],
|
||||
"--env",
|
||||
f"INTEGRATION_ONLY={int(args.integration_only)}",
|
||||
"--volume",
|
||||
workspace + ":/workspace",
|
||||
image,
|
||||
stdout=subprocess.DEVNULL,
|
||||
)
|
||||
with tempfile.TemporaryDirectory(prefix="messenger-check-") as directory:
|
||||
archive = Path(directory) / "source.tar"
|
||||
archive_source(archive)
|
||||
docker("cp", str(archive), runner + ":/tmp/source.tar")
|
||||
image_archive = Path(directory) / "images.tar"
|
||||
docker(
|
||||
"image",
|
||||
"save",
|
||||
"--output",
|
||||
str(image_archive),
|
||||
*image_env.values(),
|
||||
*dependencies,
|
||||
)
|
||||
docker("cp", str(image_archive), runner + ":/tmp/images.tar")
|
||||
docker("start", "--attach", runner, check=False, timeout=7200)
|
||||
result = subprocess.run(
|
||||
["docker", "wait", runner],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
capture_output=True,
|
||||
timeout=7200,
|
||||
)
|
||||
result.check_returncode()
|
||||
docker("cp", runner + ":/workspace/evidence/.", str(output))
|
||||
if int(result.stdout.strip()) != 0:
|
||||
raise RuntimeError(
|
||||
f"Проверки завершились ошибкой. См. {output.relative_to(ROOT)}."
|
||||
)
|
||||
summary["status"] = "passed"
|
||||
finally:
|
||||
cleanup = []
|
||||
for command in (
|
||||
["rm", "--force", runner, engine],
|
||||
["volume", "rm", workspace, data],
|
||||
["network", "rm", network],
|
||||
["image", "rm", image, *image_env.values()],
|
||||
):
|
||||
result = subprocess.run(
|
||||
["docker", *command], cwd=ROOT, encoding="utf-8", capture_output=True
|
||||
)
|
||||
cleanup.append(
|
||||
{
|
||||
"command": ["docker", *command],
|
||||
"returncode": result.returncode,
|
||||
"output": result.stdout.strip() + result.stderr.strip(),
|
||||
}
|
||||
)
|
||||
remaining = {}
|
||||
for kind in ("container", "volume", "network", "image"):
|
||||
result = subprocess.run(
|
||||
["docker", kind, "ls", "--format", "{{json .}}"],
|
||||
cwd=ROOT,
|
||||
encoding="utf-8",
|
||||
capture_output=True,
|
||||
)
|
||||
remaining[kind] = [
|
||||
line for line in result.stdout.splitlines() if project in line
|
||||
]
|
||||
if result.returncode:
|
||||
remaining[kind].append("resource audit failed")
|
||||
summary["cleanup"] = cleanup
|
||||
summary["remaining_resources"] = remaining
|
||||
summary["finished_at"] = datetime.now(timezone.utc).isoformat()
|
||||
if any(remaining.values()):
|
||||
summary["status"] = "failed"
|
||||
(output / "run.json").write_text(
|
||||
json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
|
||||
)
|
||||
if summary["status"] == "failed":
|
||||
raise SystemExit(1)
|
||||
print("PASS: проверки завершены, временное окружение удалено")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+15
-10
@@ -1,23 +1,28 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Full OpenAPI and JSON Schema checks; install requirements-dev.txt first."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from openapi_spec_validator import validate
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
spec = json.loads((root/'contracts/openapi.json').read_text())
|
||||
spec = json.loads((root / "contracts/openapi.json").read_text(encoding="utf-8"))
|
||||
validate(spec)
|
||||
from referencing import Registry, Resource
|
||||
# The base URI resolves local #/components references inside examples.
|
||||
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
|
||||
registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource))
|
||||
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
|
||||
schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name}
|
||||
|
||||
resource = dict(spec, **{"$schema": "https://json-schema.org/draft/2020-12/schema"})
|
||||
registry = Registry().with_resource(
|
||||
"urn:messenger:openapi", Resource.from_contents(resource)
|
||||
)
|
||||
for name, example in json.loads(
|
||||
(root / "contracts/examples.json").read_text(encoding="utf-8")
|
||||
).items():
|
||||
schema = {"$ref": "urn:messenger:openapi#/components/schemas/" + name}
|
||||
Draft202012Validator(schema, registry=registry).validate(example)
|
||||
for path in (root/'contracts').glob('*.schema.json'):
|
||||
schema = json.loads(path.read_text())
|
||||
for path in (root / "contracts").glob("*.schema.json"):
|
||||
schema = json.loads(path.read_text(encoding="utf-8"))
|
||||
Draft202012Validator.check_schema(schema)
|
||||
for example in schema.get('examples',[]):
|
||||
for example in schema.get("examples", []):
|
||||
Draft202012Validator(schema).validate(example)
|
||||
print('PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.')
|
||||
print("PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.")
|
||||
|
||||
Reference in New Issue
Block a user