forked from mtusi2026/mtusi2026lab4
356 lines
14 KiB
Python
356 lines
14 KiB
Python
"""Проверяет инфраструктуру, SIGTERM и обновление реального стенда."""
|
|
|
|
import asyncio
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
from threading import Event
|
|
import time
|
|
from uuid import uuid4
|
|
|
|
import asyncpg
|
|
|
|
from scripts.deployment import ComposeProject, ROOT, deployment_lock
|
|
from scripts.failover_all import exercise, failure_kind, history
|
|
from scripts.rolling_update import update
|
|
from tests.client import Client, Failure, arguments, check, run
|
|
|
|
PROBE = """import json, sys, urllib.request, urllib.error
|
|
payload = json.load(sys.stdin)
|
|
results = []
|
|
for item in payload:
|
|
data = json.dumps(item['body']).encode() if 'body' in item else None
|
|
request = urllib.request.Request(item['url'], data=data, headers=item.get('headers', {}), method=item.get('method', 'GET'))
|
|
try:
|
|
response = urllib.request.urlopen(request, timeout=7)
|
|
except urllib.error.HTTPError as error:
|
|
response = error
|
|
with response:
|
|
results.append({'status': response.status, 'body': json.loads(response.read())})
|
|
print(json.dumps(results))
|
|
"""
|
|
|
|
|
|
def probe(project, runner, requests):
|
|
"""Обращается к внутреннему API, передавая токен только через stdin."""
|
|
result = subprocess.run(
|
|
["docker", "exec", "-i", runner, "python", "-c", PROBE],
|
|
input=json.dumps(requests),
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=10,
|
|
)
|
|
check(result.returncode == 0, "internal HTTP probe failed")
|
|
return json.loads(result.stdout)
|
|
|
|
|
|
def audit(project):
|
|
"""Проверяет фактические лимиты, порты и область доступа к секретам."""
|
|
allowed = {
|
|
"postgres": {"db_password"},
|
|
"redis": {"redis_password"},
|
|
"migrate": {"db_password"},
|
|
"api": {"db_password", "redis_password", "s3_access_key_id", "s3_secret_access_key"},
|
|
"minio": {"minio_root_user", "minio_root_password"},
|
|
"minio-bootstrap": {"minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"},
|
|
"cleanup": {"db_password", "s3_access_key_id", "s3_secret_access_key"},
|
|
"nginx": set(),
|
|
}
|
|
ids = project.run("ps", "--all", "--quiet").splitlines()
|
|
containers = json.loads(project.docker("inspect", *ids))
|
|
for item in containers:
|
|
service = item["Config"]["Labels"]["com.docker.compose.service"]
|
|
check(service in allowed, "unexpected service")
|
|
host = item["HostConfig"]
|
|
check(
|
|
host["Memory"] > 0 and host["NanoCpus"] > 0,
|
|
service + ": missing CPU/RAM limit",
|
|
)
|
|
env_names = {entry.split("=", 1)[0] for entry in item["Config"]["Env"]}
|
|
check(
|
|
not env_names
|
|
& {
|
|
"DB_PASSWORD",
|
|
"REDIS_PASSWORD",
|
|
"POSTGRES_PASSWORD",
|
|
"REDISCLI_AUTH",
|
|
"DATABASE_URL",
|
|
"MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY",
|
|
},
|
|
service + ": password in environment",
|
|
)
|
|
secrets = {
|
|
Path(mount["Destination"]).name
|
|
for mount in item["Mounts"]
|
|
if mount["Destination"].startswith("/run/secrets/")
|
|
}
|
|
check(secrets == allowed[service], service + ": incorrect secret scope")
|
|
if service in {"api", "minio"}:
|
|
check(not host["PortBindings"], service + " publishes ports")
|
|
for mount in item["Mounts"]:
|
|
if mount["Destination"].startswith("/run/secrets/"):
|
|
check(not mount["RW"], "writable secret mount")
|
|
check(
|
|
"container_name" not in project.config["services"][service],
|
|
"fixed container name",
|
|
)
|
|
check(
|
|
len(containers) == len(allowed) - 1 + len(project.replicas()),
|
|
"unexpected container count",
|
|
)
|
|
print(
|
|
"PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords",
|
|
flush=True,
|
|
)
|
|
|
|
|
|
async def graceful(project, base_url):
|
|
"""Удерживает реальный POST в БД и завершает его после SIGTERM."""
|
|
replicas = project.replicas()
|
|
target, runner = replicas[:2]
|
|
client = Client(base_url)
|
|
actor, _, _, chat = await asyncio.to_thread(client.bootstrap)
|
|
path = f"/api/v1/chats/{chat['id']}/messages"
|
|
key = uuid4().hex
|
|
raw = json.dumps([actor["id"], "POST", path, key]).encode()
|
|
lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True)
|
|
database = project.config["services"]["postgres"]
|
|
env = database["environment"]
|
|
conn = await asyncpg.connect(
|
|
host="127.0.0.1",
|
|
port=int(database["ports"][0]["published"]),
|
|
database=env["POSTGRES_DB"],
|
|
user=env["POSTGRES_USER"],
|
|
password=(ROOT / "secrets/db_password").read_text().strip(),
|
|
timeout=3,
|
|
)
|
|
network = project.config['networks']['default']['name']
|
|
address_template = '{{with index .NetworkSettings.Networks ' + json.dumps(network) + '}}{{.IPAddress}}{{end}}'
|
|
address = project.docker(
|
|
"inspect",
|
|
"--format",
|
|
address_template,
|
|
target.id,
|
|
)
|
|
check(bool(address), 'API has no address in the application network')
|
|
port = project.config["services"]["api"]["environment"]["APP_PORT"]
|
|
direct = f"http://{address}:{port}"
|
|
started = time.monotonic()
|
|
task = None
|
|
try:
|
|
async with conn.transaction():
|
|
await conn.execute("SELECT pg_advisory_xact_lock($1)", lock_id)
|
|
task = asyncio.create_task(
|
|
asyncio.to_thread(
|
|
probe,
|
|
project,
|
|
runner.id,
|
|
[
|
|
{
|
|
"url": direct + path,
|
|
"method": "POST",
|
|
"body": {"text": "graceful-" + key},
|
|
"headers": {
|
|
"Authorization": "Bearer " + actor["token"],
|
|
"Idempotency-Key": key,
|
|
"Content-Type": "application/json",
|
|
},
|
|
}
|
|
],
|
|
)
|
|
)
|
|
deadline = time.monotonic() + 3
|
|
while not await conn.fetchval(
|
|
"SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE wait_event = 'advisory' AND datname = current_database())"
|
|
):
|
|
check(time.monotonic() < deadline, "POST did not reach database lock")
|
|
await asyncio.sleep(0.02)
|
|
started = time.monotonic()
|
|
await asyncio.to_thread(
|
|
project.docker, "kill", "--signal", "SIGTERM", target.id
|
|
)
|
|
results = await asyncio.to_thread(
|
|
probe,
|
|
project,
|
|
runner.id,
|
|
[
|
|
{"url": direct + "/health/ready"},
|
|
{"url": direct + "/health/live"},
|
|
{"url": direct + "/api/v1/users/me"},
|
|
],
|
|
)
|
|
check(
|
|
[item["status"] for item in results] == [503, 200, 503],
|
|
"incorrect drain readiness/liveness/new work",
|
|
)
|
|
await asyncio.sleep(max(0, 1.2 - (time.monotonic() - started)))
|
|
result = (await task)[0]
|
|
check(result["status"] == 201, "in-flight POST failed during SIGTERM")
|
|
await asyncio.to_thread(project.docker, "wait", target.id, timeout=10)
|
|
state = json.loads(
|
|
project.docker("inspect", "--format", "{{json .State}}", target.id)
|
|
)
|
|
check(
|
|
not state["OOMKilled"] and state["ExitCode"] != 137,
|
|
"forced shutdown instead of graceful completion",
|
|
)
|
|
check(time.monotonic() - started < 8, "shutdown exceeded Docker grace budget")
|
|
items = await asyncio.to_thread(history, client, actor, chat)
|
|
check(
|
|
result["body"]["id"] in {item["id"] for item in items},
|
|
"acknowledged in-flight message lost",
|
|
)
|
|
print(
|
|
f"PASS SIGTERM: readiness 503, live 200, new work 503, in-flight POST 201; exit in {time.monotonic() - started:.3f}s",
|
|
flush=True,
|
|
)
|
|
finally:
|
|
await conn.close()
|
|
if task:
|
|
await asyncio.gather(task, return_exceptions=True)
|
|
if any(item.id == target.id and item.running for item in project.replicas()):
|
|
await asyncio.to_thread(project.docker, "stop", target.id)
|
|
await asyncio.to_thread(project.docker, "start", target.id)
|
|
await asyncio.to_thread(project.wait_healthy, {item.id for item in replicas})
|
|
await asyncio.to_thread(project.wait_routed, target.instance_id, base_url)
|
|
|
|
|
|
class ObservedProject(ComposeProject):
|
|
"""Подсчитывает миграции и проверяет готовность перед заменой."""
|
|
|
|
def __init__(self):
|
|
self.migrations = 0
|
|
self.replacements = 0
|
|
self.replacement_started = []
|
|
super().__init__()
|
|
|
|
def run(self, *args, **kwargs):
|
|
if args[:6] == ("run", "--rm", "--no-deps", "--pull", "never", "migrate"):
|
|
self.migrations += 1
|
|
return super().run(*args, **kwargs)
|
|
|
|
def docker(self, *args, **kwargs):
|
|
if args[0] == "stop":
|
|
healthy = [
|
|
item
|
|
for item in self.replicas()
|
|
if item.id != args[1] and item.running and item.health == "healthy"
|
|
]
|
|
check(len(healthy) >= 2, "rolling update reduces healthy capacity")
|
|
self.replacements += 1
|
|
self.replacement_started.append(time.monotonic())
|
|
return super().docker(*args, **kwargs)
|
|
|
|
|
|
def rolling(base_url):
|
|
"""Заменяет образ под постоянной записью с проверкой сохранности."""
|
|
project = ObservedProject()
|
|
before = project.replicas()
|
|
client = Client(base_url)
|
|
actor, _, _, chat = client.bootstrap()
|
|
stop = Event()
|
|
results, errors = [], []
|
|
|
|
def worker():
|
|
local = Client(base_url, timeout=2)
|
|
key = uuid4().hex
|
|
while not stop.is_set():
|
|
started = time.monotonic()
|
|
try:
|
|
result = local.message(actor, chat, "rolling-" + key, key=key)
|
|
results.append(
|
|
(started, time.monotonic(), result["json"]["id"], result["elapsed"])
|
|
)
|
|
key = uuid4().hex
|
|
except Failure as error:
|
|
errors.append(failure_kind(error))
|
|
stop.wait(0.08)
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
jobs = [pool.submit(worker) for _ in range(2)]
|
|
try:
|
|
time.sleep(1)
|
|
started = time.monotonic()
|
|
update(project, base_url, revision="verification-" + uuid4().hex[:12])
|
|
finished = time.monotonic()
|
|
time.sleep(1)
|
|
finally:
|
|
stop.set()
|
|
for job in jobs:
|
|
job.result(timeout=5)
|
|
after = project.replicas()
|
|
check(
|
|
project.migrations == 1 and project.replacements == len(before),
|
|
"migration/replacement count incorrect",
|
|
)
|
|
check(
|
|
{item.image for item in before}.isdisjoint({item.image for item in after}),
|
|
"new image was not deployed",
|
|
)
|
|
check(
|
|
{item.id for item in before}.isdisjoint({item.id for item in after}),
|
|
"old containers remain",
|
|
)
|
|
writes = [row for row in results if started <= row[0] <= finished]
|
|
check(bool(writes), "no writes during update")
|
|
check(
|
|
any(project.replacement_started[0] <= row[0] <= finished for row in writes),
|
|
"no writes during container replacement",
|
|
)
|
|
items = history(client, actor, chat)
|
|
check(
|
|
{row[2] for row in results} <= {item["id"] for item in items},
|
|
"rolling update lost confirmed messages",
|
|
)
|
|
check(
|
|
len(items) == len({item["text"] for item in items}),
|
|
"rolling update created duplicate messages",
|
|
)
|
|
latencies = sorted(row[3] for row in writes)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"scenario": "rolling update",
|
|
"replacements": project.replacements,
|
|
"migration_runs": project.migrations,
|
|
"successful_writes": len(writes),
|
|
"failed_attempts": len(errors),
|
|
"errors_by_kind": dict(Counter(errors)),
|
|
"elapsed_seconds": round(finished - started, 3),
|
|
"successful_rps": round(len(writes) / (finished - started), 3),
|
|
"p95_seconds": round(
|
|
latencies[min(len(latencies) - 1, int(len(latencies) * 0.95))], 3
|
|
),
|
|
}
|
|
),
|
|
flush=True,
|
|
)
|
|
|
|
|
|
def main():
|
|
"""Выполняет проверки только на отдельном тестовом проекте."""
|
|
|
|
def configure(parser):
|
|
parser.add_argument(
|
|
"--phase", choices=("all", "update-and-failover"), default="all"
|
|
)
|
|
|
|
args = arguments("Grade 4 integration checks", configure)
|
|
project = ComposeProject()
|
|
check(project.name.endswith("-test"), "use isolated test Compose project")
|
|
with deployment_lock(project):
|
|
audit(project)
|
|
if args.phase == "all":
|
|
asyncio.run(graceful(project, args.base_url))
|
|
rolling(args.base_url)
|
|
args.duration, args.concurrency = 10, 2
|
|
exercise(project, args)
|
|
audit(project)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|