prepared skeleton for lab5
This commit is contained in:
@@ -0,0 +1,355 @@
|
||||
"""Проверяет инфраструктуру, SIGTERM и обновление реального стенда."""
|
||||
|
||||
import asyncio
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from threading import Event
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
import asyncpg
|
||||
|
||||
from scripts.deployment import ComposeProject, ROOT, deployment_lock
|
||||
from scripts.failover_all import exercise, failure_kind, history
|
||||
from scripts.rolling_update import update
|
||||
from tests.client import Client, Failure, arguments, check, run
|
||||
|
||||
PROBE = """import json, sys, urllib.request, urllib.error
|
||||
payload = json.load(sys.stdin)
|
||||
results = []
|
||||
for item in payload:
|
||||
data = json.dumps(item['body']).encode() if 'body' in item else None
|
||||
request = urllib.request.Request(item['url'], data=data, headers=item.get('headers', {}), method=item.get('method', 'GET'))
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=7)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
results.append({'status': response.status, 'body': json.loads(response.read())})
|
||||
print(json.dumps(results))
|
||||
"""
|
||||
|
||||
|
||||
def probe(project, runner, requests):
|
||||
"""Обращается к внутреннему API, передавая токен только через stdin."""
|
||||
result = subprocess.run(
|
||||
["docker", "exec", "-i", runner, "python", "-c", PROBE],
|
||||
input=json.dumps(requests),
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
)
|
||||
check(result.returncode == 0, "internal HTTP probe failed")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def audit(project):
|
||||
"""Проверяет фактические лимиты, порты и область доступа к секретам."""
|
||||
allowed = {
|
||||
"postgres": {"db_password"},
|
||||
"redis": {"redis_password"},
|
||||
"migrate": {"db_password"},
|
||||
"api": {"db_password", "redis_password", "s3_access_key_id", "s3_secret_access_key"},
|
||||
"minio": {"minio_root_user", "minio_root_password"},
|
||||
"minio-bootstrap": {"minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"},
|
||||
"cleanup": {"db_password", "s3_access_key_id", "s3_secret_access_key"},
|
||||
"nginx": set(),
|
||||
}
|
||||
ids = project.run("ps", "--all", "--quiet").splitlines()
|
||||
containers = json.loads(project.docker("inspect", *ids))
|
||||
for item in containers:
|
||||
service = item["Config"]["Labels"]["com.docker.compose.service"]
|
||||
check(service in allowed, "unexpected service")
|
||||
host = item["HostConfig"]
|
||||
check(
|
||||
host["Memory"] > 0 and host["NanoCpus"] > 0,
|
||||
service + ": missing CPU/RAM limit",
|
||||
)
|
||||
env_names = {entry.split("=", 1)[0] for entry in item["Config"]["Env"]}
|
||||
check(
|
||||
not env_names
|
||||
& {
|
||||
"DB_PASSWORD",
|
||||
"REDIS_PASSWORD",
|
||||
"POSTGRES_PASSWORD",
|
||||
"REDISCLI_AUTH",
|
||||
"DATABASE_URL",
|
||||
"MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY",
|
||||
},
|
||||
service + ": password in environment",
|
||||
)
|
||||
secrets = {
|
||||
Path(mount["Destination"]).name
|
||||
for mount in item["Mounts"]
|
||||
if mount["Destination"].startswith("/run/secrets/")
|
||||
}
|
||||
check(secrets == allowed[service], service + ": incorrect secret scope")
|
||||
if service in {"api", "minio"}:
|
||||
check(not host["PortBindings"], service + " publishes ports")
|
||||
for mount in item["Mounts"]:
|
||||
if mount["Destination"].startswith("/run/secrets/"):
|
||||
check(not mount["RW"], "writable secret mount")
|
||||
check(
|
||||
"container_name" not in project.config["services"][service],
|
||||
"fixed container name",
|
||||
)
|
||||
check(
|
||||
len(containers) == len(allowed) - 1 + len(project.replicas()),
|
||||
"unexpected container count",
|
||||
)
|
||||
print(
|
||||
"PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
async def graceful(project, base_url):
|
||||
"""Удерживает реальный POST в БД и завершает его после SIGTERM."""
|
||||
replicas = project.replicas()
|
||||
target, runner = replicas[:2]
|
||||
client = Client(base_url)
|
||||
actor, _, _, chat = await asyncio.to_thread(client.bootstrap)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
key = uuid4().hex
|
||||
raw = json.dumps([actor["id"], "POST", path, key]).encode()
|
||||
lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True)
|
||||
database = project.config["services"]["postgres"]
|
||||
env = database["environment"]
|
||||
conn = await asyncpg.connect(
|
||||
host="127.0.0.1",
|
||||
port=int(database["ports"][0]["published"]),
|
||||
database=env["POSTGRES_DB"],
|
||||
user=env["POSTGRES_USER"],
|
||||
password=(ROOT / "secrets/db_password").read_text().strip(),
|
||||
timeout=3,
|
||||
)
|
||||
network = project.config['networks']['default']['name']
|
||||
address_template = '{{with index .NetworkSettings.Networks ' + json.dumps(network) + '}}{{.IPAddress}}{{end}}'
|
||||
address = project.docker(
|
||||
"inspect",
|
||||
"--format",
|
||||
address_template,
|
||||
target.id,
|
||||
)
|
||||
check(bool(address), 'API has no address in the application network')
|
||||
port = project.config["services"]["api"]["environment"]["APP_PORT"]
|
||||
direct = f"http://{address}:{port}"
|
||||
started = time.monotonic()
|
||||
task = None
|
||||
try:
|
||||
async with conn.transaction():
|
||||
await conn.execute("SELECT pg_advisory_xact_lock($1)", lock_id)
|
||||
task = asyncio.create_task(
|
||||
asyncio.to_thread(
|
||||
probe,
|
||||
project,
|
||||
runner.id,
|
||||
[
|
||||
{
|
||||
"url": direct + path,
|
||||
"method": "POST",
|
||||
"body": {"text": "graceful-" + key},
|
||||
"headers": {
|
||||
"Authorization": "Bearer " + actor["token"],
|
||||
"Idempotency-Key": key,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
)
|
||||
deadline = time.monotonic() + 3
|
||||
while not await conn.fetchval(
|
||||
"SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE wait_event = 'advisory' AND datname = current_database())"
|
||||
):
|
||||
check(time.monotonic() < deadline, "POST did not reach database lock")
|
||||
await asyncio.sleep(0.02)
|
||||
started = time.monotonic()
|
||||
await asyncio.to_thread(
|
||||
project.docker, "kill", "--signal", "SIGTERM", target.id
|
||||
)
|
||||
results = await asyncio.to_thread(
|
||||
probe,
|
||||
project,
|
||||
runner.id,
|
||||
[
|
||||
{"url": direct + "/health/ready"},
|
||||
{"url": direct + "/health/live"},
|
||||
{"url": direct + "/api/v1/users/me"},
|
||||
],
|
||||
)
|
||||
check(
|
||||
[item["status"] for item in results] == [503, 200, 503],
|
||||
"incorrect drain readiness/liveness/new work",
|
||||
)
|
||||
await asyncio.sleep(max(0, 1.2 - (time.monotonic() - started)))
|
||||
result = (await task)[0]
|
||||
check(result["status"] == 201, "in-flight POST failed during SIGTERM")
|
||||
await asyncio.to_thread(project.docker, "wait", target.id, timeout=10)
|
||||
state = json.loads(
|
||||
project.docker("inspect", "--format", "{{json .State}}", target.id)
|
||||
)
|
||||
check(
|
||||
not state["OOMKilled"] and state["ExitCode"] != 137,
|
||||
"forced shutdown instead of graceful completion",
|
||||
)
|
||||
check(time.monotonic() - started < 8, "shutdown exceeded Docker grace budget")
|
||||
items = await asyncio.to_thread(history, client, actor, chat)
|
||||
check(
|
||||
result["body"]["id"] in {item["id"] for item in items},
|
||||
"acknowledged in-flight message lost",
|
||||
)
|
||||
print(
|
||||
f"PASS SIGTERM: readiness 503, live 200, new work 503, in-flight POST 201; exit in {time.monotonic() - started:.3f}s",
|
||||
flush=True,
|
||||
)
|
||||
finally:
|
||||
await conn.close()
|
||||
if task:
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
if any(item.id == target.id and item.running for item in project.replicas()):
|
||||
await asyncio.to_thread(project.docker, "stop", target.id)
|
||||
await asyncio.to_thread(project.docker, "start", target.id)
|
||||
await asyncio.to_thread(project.wait_healthy, {item.id for item in replicas})
|
||||
await asyncio.to_thread(project.wait_routed, target.instance_id, base_url)
|
||||
|
||||
|
||||
class ObservedProject(ComposeProject):
|
||||
"""Подсчитывает миграции и проверяет готовность перед заменой."""
|
||||
|
||||
def __init__(self):
|
||||
self.migrations = 0
|
||||
self.replacements = 0
|
||||
self.replacement_started = []
|
||||
super().__init__()
|
||||
|
||||
def run(self, *args, **kwargs):
|
||||
if args[:6] == ("run", "--rm", "--no-deps", "--pull", "never", "migrate"):
|
||||
self.migrations += 1
|
||||
return super().run(*args, **kwargs)
|
||||
|
||||
def docker(self, *args, **kwargs):
|
||||
if args[0] == "stop":
|
||||
healthy = [
|
||||
item
|
||||
for item in self.replicas()
|
||||
if item.id != args[1] and item.running and item.health == "healthy"
|
||||
]
|
||||
check(len(healthy) >= 2, "rolling update reduces healthy capacity")
|
||||
self.replacements += 1
|
||||
self.replacement_started.append(time.monotonic())
|
||||
return super().docker(*args, **kwargs)
|
||||
|
||||
|
||||
def rolling(base_url):
|
||||
"""Заменяет образ под постоянной записью с проверкой сохранности."""
|
||||
project = ObservedProject()
|
||||
before = project.replicas()
|
||||
client = Client(base_url)
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
stop = Event()
|
||||
results, errors = [], []
|
||||
|
||||
def worker():
|
||||
local = Client(base_url, timeout=2)
|
||||
key = uuid4().hex
|
||||
while not stop.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = local.message(actor, chat, "rolling-" + key, key=key)
|
||||
results.append(
|
||||
(started, time.monotonic(), result["json"]["id"], result["elapsed"])
|
||||
)
|
||||
key = uuid4().hex
|
||||
except Failure as error:
|
||||
errors.append(failure_kind(error))
|
||||
stop.wait(0.08)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
jobs = [pool.submit(worker) for _ in range(2)]
|
||||
try:
|
||||
time.sleep(1)
|
||||
started = time.monotonic()
|
||||
update(project, base_url, revision="verification-" + uuid4().hex[:12])
|
||||
finished = time.monotonic()
|
||||
time.sleep(1)
|
||||
finally:
|
||||
stop.set()
|
||||
for job in jobs:
|
||||
job.result(timeout=5)
|
||||
after = project.replicas()
|
||||
check(
|
||||
project.migrations == 1 and project.replacements == len(before),
|
||||
"migration/replacement count incorrect",
|
||||
)
|
||||
check(
|
||||
{item.image for item in before}.isdisjoint({item.image for item in after}),
|
||||
"new image was not deployed",
|
||||
)
|
||||
check(
|
||||
{item.id for item in before}.isdisjoint({item.id for item in after}),
|
||||
"old containers remain",
|
||||
)
|
||||
writes = [row for row in results if started <= row[0] <= finished]
|
||||
check(bool(writes), "no writes during update")
|
||||
check(
|
||||
any(project.replacement_started[0] <= row[0] <= finished for row in writes),
|
||||
"no writes during container replacement",
|
||||
)
|
||||
items = history(client, actor, chat)
|
||||
check(
|
||||
{row[2] for row in results} <= {item["id"] for item in items},
|
||||
"rolling update lost confirmed messages",
|
||||
)
|
||||
check(
|
||||
len(items) == len({item["text"] for item in items}),
|
||||
"rolling update created duplicate messages",
|
||||
)
|
||||
latencies = sorted(row[3] for row in writes)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"scenario": "rolling update",
|
||||
"replacements": project.replacements,
|
||||
"migration_runs": project.migrations,
|
||||
"successful_writes": len(writes),
|
||||
"failed_attempts": len(errors),
|
||||
"errors_by_kind": dict(Counter(errors)),
|
||||
"elapsed_seconds": round(finished - started, 3),
|
||||
"successful_rps": round(len(writes) / (finished - started), 3),
|
||||
"p95_seconds": round(
|
||||
latencies[min(len(latencies) - 1, int(len(latencies) * 0.95))], 3
|
||||
),
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
"""Выполняет проверки только на отдельном тестовом проекте."""
|
||||
|
||||
def configure(parser):
|
||||
parser.add_argument(
|
||||
"--phase", choices=("all", "update-and-failover"), default="all"
|
||||
)
|
||||
|
||||
args = arguments("Grade 4 integration checks", configure)
|
||||
project = ComposeProject()
|
||||
check(project.name.endswith("-test"), "use isolated test Compose project")
|
||||
with deployment_lock(project):
|
||||
audit(project)
|
||||
if args.phase == "all":
|
||||
asyncio.run(graceful(project, args.base_url))
|
||||
rolling(args.base_url)
|
||||
args.duration, args.concurrency = 10, 2
|
||||
exercise(project, args)
|
||||
audit(project)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Reference in New Issue
Block a user