pure version of lab5
This commit is contained in:
@@ -1,355 +0,0 @@
|
||||
"""Проверяет инфраструктуру, SIGTERM и обновление реального стенда."""
|
||||
|
||||
import asyncio
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from threading import Event
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
import asyncpg
|
||||
|
||||
from scripts.deployment import ComposeProject, ROOT, deployment_lock
|
||||
from scripts.failover_all import exercise, failure_kind, history
|
||||
from scripts.rolling_update import update
|
||||
from tests.client import Client, Failure, arguments, check, run
|
||||
|
||||
PROBE = """import json, sys, urllib.request, urllib.error
|
||||
payload = json.load(sys.stdin)
|
||||
results = []
|
||||
for item in payload:
|
||||
data = json.dumps(item['body']).encode() if 'body' in item else None
|
||||
request = urllib.request.Request(item['url'], data=data, headers=item.get('headers', {}), method=item.get('method', 'GET'))
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=7)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
results.append({'status': response.status, 'body': json.loads(response.read())})
|
||||
print(json.dumps(results))
|
||||
"""
|
||||
|
||||
|
||||
def probe(project, runner, requests):
|
||||
"""Обращается к внутреннему API, передавая токен только через stdin."""
|
||||
result = subprocess.run(
|
||||
["docker", "exec", "-i", runner, "python", "-c", PROBE],
|
||||
input=json.dumps(requests),
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
)
|
||||
check(result.returncode == 0, "internal HTTP probe failed")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def audit(project):
|
||||
"""Проверяет фактические лимиты, порты и область доступа к секретам."""
|
||||
allowed = {
|
||||
"postgres": {"db_password"},
|
||||
"redis": {"redis_password"},
|
||||
"migrate": {"db_password"},
|
||||
"api": {"db_password", "redis_password", "s3_access_key_id", "s3_secret_access_key"},
|
||||
"minio": {"minio_root_user", "minio_root_password"},
|
||||
"minio-bootstrap": {"minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"},
|
||||
"cleanup": {"db_password", "s3_access_key_id", "s3_secret_access_key"},
|
||||
"nginx": set(),
|
||||
}
|
||||
ids = project.run("ps", "--all", "--quiet").splitlines()
|
||||
containers = json.loads(project.docker("inspect", *ids))
|
||||
for item in containers:
|
||||
service = item["Config"]["Labels"]["com.docker.compose.service"]
|
||||
check(service in allowed, "unexpected service")
|
||||
host = item["HostConfig"]
|
||||
check(
|
||||
host["Memory"] > 0 and host["NanoCpus"] > 0,
|
||||
service + ": missing CPU/RAM limit",
|
||||
)
|
||||
env_names = {entry.split("=", 1)[0] for entry in item["Config"]["Env"]}
|
||||
check(
|
||||
not env_names
|
||||
& {
|
||||
"DB_PASSWORD",
|
||||
"REDIS_PASSWORD",
|
||||
"POSTGRES_PASSWORD",
|
||||
"REDISCLI_AUTH",
|
||||
"DATABASE_URL",
|
||||
"MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY",
|
||||
},
|
||||
service + ": password in environment",
|
||||
)
|
||||
secrets = {
|
||||
Path(mount["Destination"]).name
|
||||
for mount in item["Mounts"]
|
||||
if mount["Destination"].startswith("/run/secrets/")
|
||||
}
|
||||
check(secrets == allowed[service], service + ": incorrect secret scope")
|
||||
if service in {"api", "minio"}:
|
||||
check(not host["PortBindings"], service + " publishes ports")
|
||||
for mount in item["Mounts"]:
|
||||
if mount["Destination"].startswith("/run/secrets/"):
|
||||
check(not mount["RW"], "writable secret mount")
|
||||
check(
|
||||
"container_name" not in project.config["services"][service],
|
||||
"fixed container name",
|
||||
)
|
||||
check(
|
||||
len(containers) == len(allowed) - 1 + len(project.replicas()),
|
||||
"unexpected container count",
|
||||
)
|
||||
print(
|
||||
"PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
async def graceful(project, base_url):
|
||||
"""Удерживает реальный POST в БД и завершает его после SIGTERM."""
|
||||
replicas = project.replicas()
|
||||
target, runner = replicas[:2]
|
||||
client = Client(base_url)
|
||||
actor, _, _, chat = await asyncio.to_thread(client.bootstrap)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
key = uuid4().hex
|
||||
raw = json.dumps([actor["id"], "POST", path, key]).encode()
|
||||
lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True)
|
||||
database = project.config["services"]["postgres"]
|
||||
env = database["environment"]
|
||||
conn = await asyncpg.connect(
|
||||
host="127.0.0.1",
|
||||
port=int(database["ports"][0]["published"]),
|
||||
database=env["POSTGRES_DB"],
|
||||
user=env["POSTGRES_USER"],
|
||||
password=(ROOT / "secrets/db_password").read_text().strip(),
|
||||
timeout=3,
|
||||
)
|
||||
network = project.config['networks']['default']['name']
|
||||
address_template = '{{with index .NetworkSettings.Networks ' + json.dumps(network) + '}}{{.IPAddress}}{{end}}'
|
||||
address = project.docker(
|
||||
"inspect",
|
||||
"--format",
|
||||
address_template,
|
||||
target.id,
|
||||
)
|
||||
check(bool(address), 'API has no address in the application network')
|
||||
port = project.config["services"]["api"]["environment"]["APP_PORT"]
|
||||
direct = f"http://{address}:{port}"
|
||||
started = time.monotonic()
|
||||
task = None
|
||||
try:
|
||||
async with conn.transaction():
|
||||
await conn.execute("SELECT pg_advisory_xact_lock($1)", lock_id)
|
||||
task = asyncio.create_task(
|
||||
asyncio.to_thread(
|
||||
probe,
|
||||
project,
|
||||
runner.id,
|
||||
[
|
||||
{
|
||||
"url": direct + path,
|
||||
"method": "POST",
|
||||
"body": {"text": "graceful-" + key},
|
||||
"headers": {
|
||||
"Authorization": "Bearer " + actor["token"],
|
||||
"Idempotency-Key": key,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
)
|
||||
deadline = time.monotonic() + 3
|
||||
while not await conn.fetchval(
|
||||
"SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE wait_event = 'advisory' AND datname = current_database())"
|
||||
):
|
||||
check(time.monotonic() < deadline, "POST did not reach database lock")
|
||||
await asyncio.sleep(0.02)
|
||||
started = time.monotonic()
|
||||
await asyncio.to_thread(
|
||||
project.docker, "kill", "--signal", "SIGTERM", target.id
|
||||
)
|
||||
results = await asyncio.to_thread(
|
||||
probe,
|
||||
project,
|
||||
runner.id,
|
||||
[
|
||||
{"url": direct + "/health/ready"},
|
||||
{"url": direct + "/health/live"},
|
||||
{"url": direct + "/api/v1/users/me"},
|
||||
],
|
||||
)
|
||||
check(
|
||||
[item["status"] for item in results] == [503, 200, 503],
|
||||
"incorrect drain readiness/liveness/new work",
|
||||
)
|
||||
await asyncio.sleep(max(0, 1.2 - (time.monotonic() - started)))
|
||||
result = (await task)[0]
|
||||
check(result["status"] == 201, "in-flight POST failed during SIGTERM")
|
||||
await asyncio.to_thread(project.docker, "wait", target.id, timeout=10)
|
||||
state = json.loads(
|
||||
project.docker("inspect", "--format", "{{json .State}}", target.id)
|
||||
)
|
||||
check(
|
||||
not state["OOMKilled"] and state["ExitCode"] != 137,
|
||||
"forced shutdown instead of graceful completion",
|
||||
)
|
||||
check(time.monotonic() - started < 8, "shutdown exceeded Docker grace budget")
|
||||
items = await asyncio.to_thread(history, client, actor, chat)
|
||||
check(
|
||||
result["body"]["id"] in {item["id"] for item in items},
|
||||
"acknowledged in-flight message lost",
|
||||
)
|
||||
print(
|
||||
f"PASS SIGTERM: readiness 503, live 200, new work 503, in-flight POST 201; exit in {time.monotonic() - started:.3f}s",
|
||||
flush=True,
|
||||
)
|
||||
finally:
|
||||
await conn.close()
|
||||
if task:
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
if any(item.id == target.id and item.running for item in project.replicas()):
|
||||
await asyncio.to_thread(project.docker, "stop", target.id)
|
||||
await asyncio.to_thread(project.docker, "start", target.id)
|
||||
await asyncio.to_thread(project.wait_healthy, {item.id for item in replicas})
|
||||
await asyncio.to_thread(project.wait_routed, target.instance_id, base_url)
|
||||
|
||||
|
||||
class ObservedProject(ComposeProject):
|
||||
"""Подсчитывает миграции и проверяет готовность перед заменой."""
|
||||
|
||||
def __init__(self):
|
||||
self.migrations = 0
|
||||
self.replacements = 0
|
||||
self.replacement_started = []
|
||||
super().__init__()
|
||||
|
||||
def run(self, *args, **kwargs):
|
||||
if args[:6] == ("run", "--rm", "--no-deps", "--pull", "never", "migrate"):
|
||||
self.migrations += 1
|
||||
return super().run(*args, **kwargs)
|
||||
|
||||
def docker(self, *args, **kwargs):
|
||||
if args[0] == "stop":
|
||||
healthy = [
|
||||
item
|
||||
for item in self.replicas()
|
||||
if item.id != args[1] and item.running and item.health == "healthy"
|
||||
]
|
||||
check(len(healthy) >= 2, "rolling update reduces healthy capacity")
|
||||
self.replacements += 1
|
||||
self.replacement_started.append(time.monotonic())
|
||||
return super().docker(*args, **kwargs)
|
||||
|
||||
|
||||
def rolling(base_url):
|
||||
"""Заменяет образ под постоянной записью с проверкой сохранности."""
|
||||
project = ObservedProject()
|
||||
before = project.replicas()
|
||||
client = Client(base_url)
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
stop = Event()
|
||||
results, errors = [], []
|
||||
|
||||
def worker():
|
||||
local = Client(base_url, timeout=2)
|
||||
key = uuid4().hex
|
||||
while not stop.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = local.message(actor, chat, "rolling-" + key, key=key)
|
||||
results.append(
|
||||
(started, time.monotonic(), result["json"]["id"], result["elapsed"])
|
||||
)
|
||||
key = uuid4().hex
|
||||
except Failure as error:
|
||||
errors.append(failure_kind(error))
|
||||
stop.wait(0.08)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
jobs = [pool.submit(worker) for _ in range(2)]
|
||||
try:
|
||||
time.sleep(1)
|
||||
started = time.monotonic()
|
||||
update(project, base_url, revision="verification-" + uuid4().hex[:12])
|
||||
finished = time.monotonic()
|
||||
time.sleep(1)
|
||||
finally:
|
||||
stop.set()
|
||||
for job in jobs:
|
||||
job.result(timeout=5)
|
||||
after = project.replicas()
|
||||
check(
|
||||
project.migrations == 1 and project.replacements == len(before),
|
||||
"migration/replacement count incorrect",
|
||||
)
|
||||
check(
|
||||
{item.image for item in before}.isdisjoint({item.image for item in after}),
|
||||
"new image was not deployed",
|
||||
)
|
||||
check(
|
||||
{item.id for item in before}.isdisjoint({item.id for item in after}),
|
||||
"old containers remain",
|
||||
)
|
||||
writes = [row for row in results if started <= row[0] <= finished]
|
||||
check(bool(writes), "no writes during update")
|
||||
check(
|
||||
any(project.replacement_started[0] <= row[0] <= finished for row in writes),
|
||||
"no writes during container replacement",
|
||||
)
|
||||
items = history(client, actor, chat)
|
||||
check(
|
||||
{row[2] for row in results} <= {item["id"] for item in items},
|
||||
"rolling update lost confirmed messages",
|
||||
)
|
||||
check(
|
||||
len(items) == len({item["text"] for item in items}),
|
||||
"rolling update created duplicate messages",
|
||||
)
|
||||
latencies = sorted(row[3] for row in writes)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"scenario": "rolling update",
|
||||
"replacements": project.replacements,
|
||||
"migration_runs": project.migrations,
|
||||
"successful_writes": len(writes),
|
||||
"failed_attempts": len(errors),
|
||||
"errors_by_kind": dict(Counter(errors)),
|
||||
"elapsed_seconds": round(finished - started, 3),
|
||||
"successful_rps": round(len(writes) / (finished - started), 3),
|
||||
"p95_seconds": round(
|
||||
latencies[min(len(latencies) - 1, int(len(latencies) * 0.95))], 3
|
||||
),
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
"""Выполняет проверки только на отдельном тестовом проекте."""
|
||||
|
||||
def configure(parser):
|
||||
parser.add_argument(
|
||||
"--phase", choices=("all", "update-and-failover"), default="all"
|
||||
)
|
||||
|
||||
args = arguments("Grade 4 integration checks", configure)
|
||||
project = ComposeProject()
|
||||
check(project.name.endswith("-test"), "use isolated test Compose project")
|
||||
with deployment_lock(project):
|
||||
audit(project)
|
||||
if args.phase == "all":
|
||||
asyncio.run(graceful(project, args.base_url))
|
||||
rolling(args.base_url)
|
||||
args.duration, args.concurrency = 10, 2
|
||||
exercise(project, args)
|
||||
audit(project)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -1,39 +0,0 @@
|
||||
"""Проверяет конкурентные повторы через балансировщик."""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import uuid4
|
||||
|
||||
from tests.client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Отправляет один ключ параллельно через разные реплики."""
|
||||
args = arguments("Concurrent idempotency through load balancer")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
key = uuid4().hex
|
||||
body = "parallel-" + key
|
||||
|
||||
def send(_):
|
||||
return Client(args.base_url, args.ca_file).message(alice, chat, body, key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(send, range(24)))
|
||||
ids = {result["json"]["id"] for result in results}
|
||||
replicas = {result["headers"]["X-Instance-Id"] for result in results}
|
||||
check(len(ids) == 1, "parallel retries created duplicate messages")
|
||||
check(len(replicas) >= 2, "parallel retries did not reach two API replicas")
|
||||
history = client.call(
|
||||
"GET",
|
||||
f"/api/v1/chats/{chat['id']}/messages?limit=100",
|
||||
actor=alice,
|
||||
schema="MessagePage",
|
||||
)["json"]["items"]
|
||||
check(
|
||||
len(history) == 1 and history[0] == results[0]["json"], "history has duplicates"
|
||||
)
|
||||
print(f"PASS: 24 concurrent replays, {len(replicas)} replicas, one message")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Fault/recovery lab5 на явно выбранном Compose-стенде; только синтетические данные."""
|
||||
"""Fault/recovery очереди на явно выбранном Compose-стенде; только синтетические данные."""
|
||||
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
@@ -79,7 +79,7 @@ def verify_result(client, actor, attachment, original):
|
||||
|
||||
|
||||
def crash(client, actor, chat, mode):
|
||||
name='lab5-worker-fault-'+uuid4().hex
|
||||
name='messenger-worker-fault-'+uuid4().hex
|
||||
try:
|
||||
require_compose('stop','worker')
|
||||
attachment,original=client.attachment(actor,chat)
|
||||
@@ -106,7 +106,7 @@ def crash(client, actor, chat, mode):
|
||||
|
||||
|
||||
def retry_replay(client,actor,chat,permanent):
|
||||
name='lab5-retry-fault-'+uuid4().hex
|
||||
name='messenger-retry-fault-'+uuid4().hex
|
||||
samples=[]
|
||||
configuration=ComposeProject().config['services']['worker']['environment']
|
||||
retry_base=float(configuration['JOB_RETRY_SECONDS'])
|
||||
@@ -189,10 +189,11 @@ def persistence(client,actor,chat):
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url',required=True)
|
||||
parser.add_argument('--ca-file')
|
||||
parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all')
|
||||
parser.add_argument('--output',default='evidence/lab5-jobs.json')
|
||||
parser.add_argument('--output',default='evidence/jobs.json')
|
||||
args=parser.parse_args()
|
||||
client=Client(args.base_url,timeout=40)
|
||||
client=Client(args.base_url,args.ca_file,timeout=40)
|
||||
aliases={r.instance_id for r in ComposeProject().replicas() if r.running}
|
||||
check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL')
|
||||
actor,_,_,chat=client.bootstrap()
|
||||
@@ -25,7 +25,7 @@ def check(condition: bool, message: str) -> None:
|
||||
class Stand:
|
||||
def __init__(self, directory: str) -> None:
|
||||
self.directory = Path(directory)
|
||||
self.project = "mtusi-lab3-nginx-check-" + uuid4().hex[:12]
|
||||
self.project = "messenger-nginx-check-" + uuid4().hex[:12]
|
||||
self.base_url = ""
|
||||
backend = self.directory / "backend.conf"
|
||||
backend.write_text(
|
||||
@@ -246,7 +246,7 @@ def run_checks(stand: Stand) -> None:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="mtusi-lab3-nginx-check-") as directory:
|
||||
with tempfile.TemporaryDirectory(prefix="messenger-nginx-check-") as directory:
|
||||
stand = Stand(directory)
|
||||
try:
|
||||
run_checks(stand)
|
||||
@@ -254,7 +254,7 @@ def main() -> None:
|
||||
print(stand.compose("logs", "--no-color", "--tail", "20"), flush=True)
|
||||
raise
|
||||
finally:
|
||||
stand.compose("down", "--remove-orphans", "--timeout", "1")
|
||||
stand.compose("down", "--volumes", "--remove-orphans", "--timeout", "1")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -34,7 +34,7 @@ def main() -> None:
|
||||
parser.add_argument("--ca-file")
|
||||
args = parser.parse_args()
|
||||
running = set(require_compose("ps", "--status", "running", "--services").splitlines())
|
||||
check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the lab5 stack before this scenario")
|
||||
check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the application stack before this scenario")
|
||||
client = Client(args.base_url, args.ca_file, timeout=40)
|
||||
project = ComposeProject()
|
||||
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
|
||||
@@ -42,7 +42,7 @@ def main() -> None:
|
||||
check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project")
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
attachment = None
|
||||
fault_container = "lab5-outbox-fault-" + uuid4().hex
|
||||
fault_container = "messenger-outbox-fault-" + uuid4().hex
|
||||
try:
|
||||
require_compose("stop", "publisher")
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
|
||||
+4
-20
@@ -1,8 +1,6 @@
|
||||
"""Реальный MinIO: повтор bootstrap, private bucket, ограниченные права и отказ."""
|
||||
|
||||
import json
|
||||
import base64
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
@@ -70,12 +68,15 @@ asyncio.run(main())
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--with-outage', action='store_true', help='временно остановить MinIO выбранного проекта')
|
||||
parser.add_argument('--with-faults', action='store_true', help='проверить компенсацию и очистку на реальных S3/БД')
|
||||
args = arguments('MinIO private bucket and failure checks', configure)
|
||||
project = ComposeProject()
|
||||
client = Client(args.base_url, args.ca_file, timeout=40)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
|
||||
live = client.call('GET', '/health/live', schema='Health')
|
||||
check(live['headers'].get('X-Instance-Id') in aliases, 'BASE_URL does not belong to the selected Compose project')
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
attachment = client.wait_ready(alice, attachment)
|
||||
# Повторный bootstrap не должен очистить bucket или сменить используемые credentials.
|
||||
for _ in range(2):
|
||||
project.run('run', '--rm', '--no-deps', 'minio-bootstrap', timeout=100)
|
||||
@@ -96,21 +97,6 @@ def main():
|
||||
check('127.0.0.1:9001' in project.config['services']['minio']['command'], 'console is not bound to loopback')
|
||||
path = '/api/v1/attachments/' + attachment['id'] + '/content'
|
||||
check(client.call('GET', path, actor=alice)['bytes'] == original, 'bootstrap changed original')
|
||||
if args.with_faults:
|
||||
project.run('stop', 'cleanup')
|
||||
try:
|
||||
fault_script = Path(__file__).with_name('s3_faults.py').read_text()
|
||||
result = subprocess.run(
|
||||
['docker', 'exec', '-i', replicas[0].id, 'python', '-c', fault_script],
|
||||
input=json.dumps({'user': alice['user'], 'chat_id': chat['id'],
|
||||
'image': base64.b64encode(original).decode()}),
|
||||
text=True, capture_output=True, timeout=60,
|
||||
)
|
||||
check(result.returncode == 0, 'real S3/database compensation probe failed (details hidden)')
|
||||
check(all(json.loads(result.stdout).values()), 'real S3/database invariants failed')
|
||||
finally:
|
||||
project.run('start', 'cleanup')
|
||||
check(client.call('GET', path, actor=alice)['bytes'] == original, 'cleanup damaged confirmed original')
|
||||
if args.with_outage:
|
||||
try:
|
||||
project.run('stop', 'minio')
|
||||
@@ -129,8 +115,6 @@ def main():
|
||||
check(client.call('GET', path, actor=alice)['bytes'] == original, 'original lost after MinIO restart')
|
||||
print('PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console'
|
||||
+ (', MinIO outage/recovery' if args.with_outage else ''))
|
||||
if args.with_faults:
|
||||
print('PASS: real S3 PUT/database failure, compensation, durable journal, grace period, dry-run, repeated cleanup')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -1,204 +0,0 @@
|
||||
"""Повторяемый прогон lab4; публикует только успешные обезличенные evidence."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
from importlib.metadata import version
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
from tempfile import TemporaryDirectory
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROJECT = 'mtusi-messenger-lab5-test'
|
||||
EVIDENCE = ROOT / 'evidence'
|
||||
ENVIRONMENT = {
|
||||
**os.environ,
|
||||
'COMPOSE_PROJECT_NAME': PROJECT,
|
||||
'COMPOSE_FILE': 'compose.yaml:compose.test.yaml',
|
||||
'LB_HOST_PORT': '28080',
|
||||
}
|
||||
COMPOSE = ['docker', 'compose', '--env-file', '.env']
|
||||
NONSECRET_SETTINGS = {
|
||||
'APP_PORT', 'API_REPLICAS', 'MAX_MESSAGES', 'MAX_IN_FLIGHT',
|
||||
'DB_POOL_SIZE', 'DB_MAX_OVERFLOW', 'SESSION_TTL_SECONDS',
|
||||
'IDEMPOTENCY_TTL_SECONDS', 'SHUTDOWN_DRAIN_SECONDS',
|
||||
'GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS', 'READINESS_TIMEOUT_SECONDS',
|
||||
'S3_ENDPOINT', 'S3_BUCKET', 'S3_REGION', 'S3_CONNECT_TIMEOUT_SECONDS',
|
||||
'S3_READ_TIMEOUT_SECONDS', 'S3_UPLOAD_TIMEOUT_SECONDS',
|
||||
'S3_DOWNLOAD_TIMEOUT_SECONDS', 'S3_IMAGE_TIMEOUT_SECONDS',
|
||||
'S3_MAX_UPLOAD_BYTES', 'S3_MAX_IMAGE_PIXELS', 'S3_CHUNK_BYTES',
|
||||
'S3_MAX_CONCURRENT_UPLOADS', 'S3_CLEANUP_GRACE_SECONDS',
|
||||
'S3_CLEANUP_INTERVAL_SECONDS',
|
||||
}
|
||||
SNAPSHOT_FORMAT = '{"id":{{json .Id}},"name":{{json .Name}},"service":{{json (index .Config.Labels "com.docker.compose.service")}},"image_id":{{json .Image}},"status":{{json .State.Status}},"started_at":{{json .State.StartedAt}},"pid":{{json .State.Pid}},"exit_code":{{json .State.ExitCode}},"oom_killed":{{json .State.OOMKilled}}}'
|
||||
IMAGE_FORMAT = '{"id":{{json .Id}},"repo_digests":{{json .RepoDigests}},"architecture":{{json .Architecture}},"os":{{json .Os}}}'
|
||||
|
||||
|
||||
def timestamp():
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def invoke(command):
|
||||
return subprocess.run(command, cwd=ROOT, env=ENVIRONMENT, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
|
||||
|
||||
def read_command(command):
|
||||
result = invoke(command)
|
||||
if result.returncode:
|
||||
raise RuntimeError('Не выполнена команда: ' + shlex.join(command))
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def source_manifest():
|
||||
names = {
|
||||
'.dockerignore', '.env.example',
|
||||
'Dockerfile', 'Makefile', 'alembic.ini', 'compose.yaml',
|
||||
'compose.test.yaml', 'pytest.ini', 'requirements.txt', 'requirements-dev.txt', 'lab.json',
|
||||
}
|
||||
for directory in ('app', 'migrations', 'student-tests', 'scripts', 'infra', 'contracts', 'tests'):
|
||||
names.update(str(path.relative_to(ROOT)) for path in (ROOT / directory).rglob('*')
|
||||
if path.is_file() and '__pycache__' not in path.parts
|
||||
and path.suffix != '.md' and not path.name.startswith('.'))
|
||||
return ''.join(f'{hashlib.sha256((ROOT / name).read_bytes()).hexdigest()} {name}\n' for name in sorted(names))
|
||||
|
||||
|
||||
def sanitize(text, secrets, staging):
|
||||
clean = text.replace(str(ROOT), '.').replace(str(staging), '<temporary>')
|
||||
for secret in secrets:
|
||||
clean = clean.replace(secret, '<redacted>')
|
||||
clean = re.sub(r'(?i)\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+', r'\1 <redacted>', clean)
|
||||
clean = re.sub(r"(?i)([\"'](?:token|password|secret_access_key)[\"']\s*:\s*[\"'])[^\"']*", r'\1<redacted>', clean)
|
||||
return clean
|
||||
|
||||
|
||||
def container_snapshot(images):
|
||||
ids = read_command(['docker', 'ps', '-aq', '--filter', f'label=com.docker.compose.project={PROJECT}']).splitlines()
|
||||
rows = []
|
||||
if ids:
|
||||
output = read_command(['docker', 'inspect', '--format', SNAPSHOT_FORMAT, *ids])
|
||||
rows = [json.loads(line) for line in output.splitlines()]
|
||||
for row in rows:
|
||||
image = row['image_id']
|
||||
if image not in images:
|
||||
images[image] = json.loads(read_command(['docker', 'image', 'inspect', '--format', IMAGE_FORMAT, image]))
|
||||
return rows
|
||||
|
||||
|
||||
def remaining_resources():
|
||||
return {resource: read_command(['docker', resource, 'ls', '-aq' if resource == 'container' else '-q', '--filter',
|
||||
f'label=com.docker.compose.project={PROJECT}']).splitlines()
|
||||
for resource in ('container', 'network', 'volume')}
|
||||
|
||||
|
||||
def extract_metrics(output, upload_file):
|
||||
rows = [json.loads(line) for line in output.splitlines() if line.startswith('{') and line.endswith('}')]
|
||||
pytest = re.search(r'(\d+) passed(?:, [^\n]*?)? in ([\d.]+)s', output)
|
||||
if pytest is None:
|
||||
raise RuntimeError('В выводе нет успешного итогового результата pytest')
|
||||
return {
|
||||
'pytest': {'passed': int(pytest[1]), 'elapsed_seconds': float(pytest[2])},
|
||||
'uploads': json.loads(upload_file.read_text()),
|
||||
'public_failover': [row for row in rows if 'observed_instances_before' in row],
|
||||
'rolling_update': [row for row in rows if row.get('scenario') == 'rolling update'],
|
||||
'replica_failures': [row for row in rows if 'first_recovery_seconds' in row and 'instance' in row],
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
if any(remaining_resources().values()):
|
||||
raise RuntimeError('Тестовый проект уже существует; сначала завершите его отдельно')
|
||||
started = timestamp()
|
||||
manifest = source_manifest()
|
||||
secrets = [path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file()]
|
||||
secrets = [value for value in secrets if value]
|
||||
snapshots, images, commands, combined, nginx_output = {}, {}, [], [], ''
|
||||
info = json.loads(read_command(['docker', 'info', '--format', '{{json .}}']))
|
||||
config = json.loads(read_command([*COMPOSE, 'config', '--format', 'json']))
|
||||
services = {name: {
|
||||
'image': service.get('image'),
|
||||
'limits': service.get('deploy', {}).get('resources', {}).get('limits', {}),
|
||||
'settings': {key: value for key, value in service.get('environment', {}).items() if key in NONSECRET_SETTINGS},
|
||||
'ports': service.get('ports', []), 'networks': list(service.get('networks', {})),
|
||||
'secret_names': [entry['source'] for entry in service.get('secrets', [])],
|
||||
'tmpfs': service.get('tmpfs', []),
|
||||
} for name, service in config['services'].items()}
|
||||
with TemporaryDirectory(prefix='lab4-evidence-') as folder:
|
||||
staging = Path(folder)
|
||||
upload_file = staging / 'uploads.json'
|
||||
|
||||
def execute(command):
|
||||
begin = timestamp()
|
||||
result = invoke(command)
|
||||
output = sanitize(result.stdout, secrets, staging)
|
||||
display = sanitize(shlex.join(command), secrets, staging)
|
||||
commands.append({'command': display, 'started_at_utc': begin,
|
||||
'finished_at_utc': timestamp(), 'exit_code': result.returncode})
|
||||
print(f'{"PASS" if result.returncode == 0 else "FAIL"}: {display}', flush=True)
|
||||
if result.returncode:
|
||||
print(output[-5000:], flush=True)
|
||||
raise RuntimeError('Прогон остановлен; неуспешные evidence не опубликованы')
|
||||
return f'$ {display}\n{output}\n'
|
||||
|
||||
cleanup_output = ''
|
||||
try:
|
||||
combined.append(execute(['make', 'check', 'validate', 'test-stand-up', f'TEST_PROJECT={PROJECT}']))
|
||||
snapshots['started'] = container_snapshot(images)
|
||||
combined.append(execute(['make', 'test-docker', f'TEST_PROJECT={PROJECT}', 'PYTEST_ARGS=--tb=short']))
|
||||
combined.append(execute(['make', 'test-stand-smoke', 'test-stand-idempotency', f'TEST_PROJECT={PROJECT}']))
|
||||
snapshots['before_restart'] = container_snapshot(images)
|
||||
combined.append(execute(['make', 'test-stand-persistence', f'TEST_PROJECT={PROJECT}']))
|
||||
snapshots['after_restart'] = container_snapshot(images)
|
||||
combined.append(execute(['make', 'test-stand-failover', f'TEST_PROJECT={PROJECT}']))
|
||||
snapshots['after_failover'] = container_snapshot(images)
|
||||
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_s3', '--base-url', 'http://localhost:28080', '--with-outage', '--with-faults']))
|
||||
combined.append(execute(['.venv/bin/python', '-m', 'scripts.upload_load', '--base-url', 'http://localhost:28080', '--concurrency', '10', '--output', str(upload_file)]))
|
||||
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_grade4', '--base-url', 'http://localhost:28080']))
|
||||
snapshots['after_update_and_failures'] = container_snapshot(images)
|
||||
nginx_output = execute(['make', 'test-nginx'])
|
||||
finally:
|
||||
cleanup_output = execute([*COMPOSE, 'down', '--volumes', '--remove-orphans'])
|
||||
remaining = remaining_resources()
|
||||
if any(remaining.values()):
|
||||
raise RuntimeError('После очистки остались ресурсы тестового проекта')
|
||||
if source_manifest() != manifest:
|
||||
raise RuntimeError('Во время прогона изменились проверяемые исходники')
|
||||
output = ''.join(combined)
|
||||
metrics = extract_metrics(output, upload_file)
|
||||
run = {
|
||||
'started_at_utc': started, 'finished_at_utc': timestamp(),
|
||||
'started_at_moscow': datetime.fromisoformat(started).astimezone(ZoneInfo('Europe/Moscow')).isoformat(),
|
||||
'base_commit': read_command(['git', 'rev-parse', 'HEAD']),
|
||||
'worktree_dirty': bool(read_command(['git', 'status', '--porcelain'])),
|
||||
'source_manifest_sha256': hashlib.sha256(manifest.encode()).hexdigest(),
|
||||
'project': PROJECT, 'runtime': {'python': platform.python_version(),
|
||||
'packages': {name: version(name) for name in ('fastapi', 'SQLAlchemy', 'pydantic', 'minio', 'Pillow', 'pytest')},
|
||||
'compose': read_command(['docker', 'compose', 'version'])},
|
||||
'docker_host': {key: info[key] for key in ('ServerVersion', 'OperatingSystem', 'Architecture', 'NCPU', 'MemTotal', 'CgroupVersion')},
|
||||
'services': services, 'container_snapshots': snapshots, 'images': images,
|
||||
'commands': commands, 'remaining_test_resources': remaining,
|
||||
'minio_source_release': 'RELEASE.2025-10-15T17-29-55Z',
|
||||
'mc_source_release': 'RELEASE.2025-08-13T08-35-41Z',
|
||||
}
|
||||
artifacts = {
|
||||
'test-output.txt': output, 'nginx-output.txt': nginx_output,
|
||||
'cleanup.txt': cleanup_output, 'source-sha256.txt': manifest,
|
||||
'metrics.json': json.dumps(metrics, ensure_ascii=False, indent=2) + '\n',
|
||||
'run.json': json.dumps(run, ensure_ascii=False, indent=2) + '\n',
|
||||
}
|
||||
for name, content in artifacts.items():
|
||||
if any(secret in content for secret in secrets):
|
||||
raise RuntimeError('Обнаружено значение секрета; evidence не опубликованы')
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
for name, content in artifacts.items():
|
||||
(EVIDENCE / name).write_text(content)
|
||||
print('Evidence опубликованы; тестовые ресурсы удалены.', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,17 +1,10 @@
|
||||
"""Общие операции с репликами текущего Compose-проекта."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import ssl
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import urlopen
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
@@ -65,43 +58,3 @@ class ComposeProject:
|
||||
Replica(row["id"], row["image"], configured or row["hostname"], row["running"], row["health"])
|
||||
for row in map(json.loads, rows)
|
||||
]
|
||||
|
||||
def wait_healthy(self, ids: set[str], timeout: float = 40) -> None:
|
||||
"""Ожидает готовность конкретных реплик."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
current = {replica.id: replica for replica in self.replicas()}
|
||||
if not ids <= current.keys():
|
||||
raise RuntimeError("Контейнер API исчез во время развёртывания")
|
||||
if all(current[id].running and current[id].health == "healthy" for id in ids):
|
||||
return
|
||||
if any(not current[id].running for id in ids):
|
||||
raise RuntimeError("Контейнер API завершился до готовности")
|
||||
time.sleep(0.25)
|
||||
raise TimeoutError("API не стал healthy за отведённое время")
|
||||
|
||||
def wait_routed(self, instance_id: str, base_url: str, ca_file=None) -> None:
|
||||
"""Ожидает ответа реплики через балансировщик."""
|
||||
context = ssl.create_default_context(cafile=ca_file)
|
||||
deadline = time.monotonic() + 40
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urlopen(base_url.rstrip("/") + "/health/live", timeout=3, context=context) as response:
|
||||
if response.status == 200 and response.headers.get("X-Instance-Id") == instance_id:
|
||||
return
|
||||
except (HTTPError, URLError, TimeoutError, OSError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise TimeoutError("Реплика не появилась в ответах балансировщика")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def deployment_lock(project: ComposeProject):
|
||||
"""Исключает параллельные изменения одного стенда."""
|
||||
path = Path(tempfile.gettempdir()) / f"{project.name}-deployment.lock"
|
||||
with path.open("a") as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
raise RuntimeError("Другой процесс уже изменяет этот Compose-проект") from None
|
||||
yield
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
BASE_URL="${BASE_URL:-http://localhost:8080}"
|
||||
TTL="${TTL:?укажите TTL, с которым запущен API (SESSION_TTL_SECONDS)}"
|
||||
API="$BASE_URL/api/v1"
|
||||
USERNAME="expiry_$(date +%s)"
|
||||
PASSWORD="expiry-password-123"
|
||||
|
||||
code() {
|
||||
curl -s -o /dev/null -w '%{http_code}' "$@"
|
||||
}
|
||||
|
||||
curl -sf -o /dev/null -X POST "$API/users" -H 'Content-Type: application/json' \
|
||||
-d "{\"username\":\"$USERNAME\",\"display_name\":\"Expiry\",\"password\":\"$PASSWORD\"}"
|
||||
SESSION=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD")
|
||||
TOKEN=$(printf '%s' "$SESSION" | python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])')
|
||||
EXPIRES_AT=$(printf '%s' "$SESSION" | python3 -c 'import json, sys; print(json.load(sys.stdin)["expires_at"])')
|
||||
AUTH="Authorization: Bearer $TOKEN"
|
||||
|
||||
echo "SESSION_TTL_SECONDS=$TTL, login at $(date -u +%H:%M:%S)Z, expires_at=$EXPIRES_AT"
|
||||
echo "GET /users/me right after login -> $(code "$API/users/me" -H "$AUTH")"
|
||||
sleep $((TTL + 1))
|
||||
printf "%-35s -> %s\n" "GET /users/me after $((TTL + 1)) s" "$(code "$API/users/me" -H "$AUTH")"
|
||||
echo "DELETE /auth/sessions/current after -> $(code -X DELETE "$API/auth/sessions/current" -H "$AUTH")"
|
||||
NEW_TOKEN=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD" \
|
||||
| python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])')
|
||||
echo "GET /users/me with new login token -> $(code "$API/users/me" -H "Authorization: Bearer $NEW_TOKEN")"
|
||||
@@ -1,132 +0,0 @@
|
||||
"""EXPLAIN (ANALYZE, BUFFERS) запроса истории на синтетических данных.
|
||||
|
||||
Данные создаются в одной транзакции и откатываются в конце: база не меняется.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncConnection
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine
|
||||
|
||||
TARGET_MESSAGES = 10_000
|
||||
NOISE_MESSAGES = 40_000
|
||||
PAGE = 51
|
||||
|
||||
HISTORY_FIRST_PAGE = """
|
||||
SELECT * FROM messages
|
||||
WHERE chat_id = :chat_id
|
||||
ORDER BY created_at, id
|
||||
LIMIT :page
|
||||
"""
|
||||
|
||||
HISTORY_AFTER_CURSOR = """
|
||||
SELECT * FROM messages
|
||||
WHERE chat_id = :chat_id AND (created_at, id) > (:after_created_at, :after_id)
|
||||
ORDER BY created_at, id
|
||||
LIMIT :page
|
||||
"""
|
||||
|
||||
|
||||
async def seed(connection: AsyncConnection) -> dict[str, object]:
|
||||
user_id, target_id, noise_id = uuid4(), uuid4(), uuid4()
|
||||
await connection.execute(
|
||||
text(
|
||||
"INSERT INTO users (id, username, display_name, password_hash) "
|
||||
"VALUES (:id, :username, 'Explain', 'x')"
|
||||
),
|
||||
{"id": user_id, "username": "explain_" + uuid4().hex[:8]},
|
||||
)
|
||||
await connection.execute(
|
||||
text("INSERT INTO chats (id, title) VALUES (:a, 'target'), (:b, 'noise')"),
|
||||
{"a": target_id, "b": noise_id},
|
||||
)
|
||||
await connection.execute(
|
||||
text("INSERT INTO chat_members (chat_id, user_id) VALUES (:a, :u), (:b, :u)"),
|
||||
{"a": target_id, "b": noise_id, "u": user_id},
|
||||
)
|
||||
for chat_id, count in ((target_id, TARGET_MESSAGES), (noise_id, NOISE_MESSAGES)):
|
||||
await connection.execute(
|
||||
text(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) "
|
||||
"SELECT gen_random_uuid(), :chat_id, :user_id, 'message ' || g, "
|
||||
"now() - make_interval(secs => :count - g) "
|
||||
"FROM generate_series(1, :count) AS g"
|
||||
),
|
||||
{"chat_id": chat_id, "user_id": user_id, "count": count},
|
||||
)
|
||||
await connection.execute(text("ANALYZE messages"))
|
||||
|
||||
middle = (
|
||||
await connection.execute(
|
||||
text(
|
||||
"SELECT created_at, id FROM messages WHERE chat_id = :c "
|
||||
"ORDER BY created_at, id OFFSET :o LIMIT 1"
|
||||
),
|
||||
{"c": target_id, "o": TARGET_MESSAGES // 2},
|
||||
)
|
||||
).one()
|
||||
return {
|
||||
"chat_id": target_id,
|
||||
"after_created_at": middle.created_at,
|
||||
"after_id": middle.id,
|
||||
"page": PAGE,
|
||||
}
|
||||
|
||||
|
||||
async def explain(
|
||||
connection: AsyncConnection, title: str, query: str, params: dict[str, object]
|
||||
) -> None:
|
||||
rows = await connection.execute(
|
||||
text(f"EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING ON) {query}"), params
|
||||
)
|
||||
print(f"== {title}")
|
||||
print(query.strip())
|
||||
print()
|
||||
for row in rows:
|
||||
print(row[0])
|
||||
print()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
engine = create_db_engine(Settings()) # type: ignore[call-arg]
|
||||
async with engine.connect() as connection:
|
||||
transaction = await connection.begin()
|
||||
params = await seed(connection)
|
||||
total = (
|
||||
await connection.execute(text("SELECT count(*) FROM messages"))
|
||||
).scalar_one()
|
||||
print(
|
||||
f"messages in table: {total}; target chat: {TARGET_MESSAGES}; noise chat: {NOISE_MESSAGES}"
|
||||
)
|
||||
print()
|
||||
|
||||
first_page = {"chat_id": params["chat_id"], "page": PAGE}
|
||||
await explain(
|
||||
connection, "first page (limit+1 = 51)", HISTORY_FIRST_PAGE, first_page
|
||||
)
|
||||
await explain(
|
||||
connection,
|
||||
"page after cursor in the middle of history",
|
||||
HISTORY_AFTER_CURSOR,
|
||||
params,
|
||||
)
|
||||
|
||||
await connection.execute(text("SET LOCAL enable_indexscan = off"))
|
||||
await connection.execute(text("SET LOCAL enable_bitmapscan = off"))
|
||||
await explain(
|
||||
connection,
|
||||
"same page with index scans disabled (for comparison)",
|
||||
HISTORY_AFTER_CURSOR,
|
||||
params,
|
||||
)
|
||||
|
||||
await transaction.rollback()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,148 +0,0 @@
|
||||
"""Сценарий поочерёдного отказа всех API-реплик под записью."""
|
||||
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from threading import Event, Lock
|
||||
import time
|
||||
from urllib.parse import quote
|
||||
from uuid import uuid4
|
||||
|
||||
from scripts.deployment import ComposeProject, ROOT, deployment_lock
|
||||
from tests.client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def failure_kind(error: Failure) -> str:
|
||||
"""Различает HTTP-отказы, ошибки соединения и проверки контракта."""
|
||||
match = re.search(r"received (\d{3})$", str(error))
|
||||
if match:
|
||||
return "http_" + match.group(1)
|
||||
return "connection" if "connection/TLS failure" in str(error) else "contract"
|
||||
|
||||
|
||||
def snapshot(project: ComposeProject, phase: str, target: str) -> list:
|
||||
"""Печатает состояние API без окружения и секретов."""
|
||||
replicas = project.replicas()
|
||||
print(json.dumps({
|
||||
"scenario": "replica_failure", "phase": phase, "target": target,
|
||||
"replicas": [{"instance": item.instance_id, "running": item.running, "health": item.health} for item in replicas],
|
||||
}), flush=True)
|
||||
return replicas
|
||||
|
||||
|
||||
def history(client: Client, actor: dict, chat: dict) -> list[dict]:
|
||||
"""Читает всю историю без повторов курсора."""
|
||||
path = f"/api/v1/chats/{chat['id']}/messages?limit=100"
|
||||
items = []
|
||||
cursors = set()
|
||||
for _ in range(1000):
|
||||
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
||||
items.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
return items
|
||||
check(cursor not in cursors, "cursor cycle")
|
||||
cursors.add(cursor)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages?limit=100&cursor={quote(cursor, safe='')}"
|
||||
raise Failure("history exceeded safety bound")
|
||||
|
||||
|
||||
def exercise(project: ComposeProject, args) -> None:
|
||||
"""Останавливает каждую реплику и восстанавливает её перед следующей."""
|
||||
replicas = project.replicas()
|
||||
check(len(replicas) >= 2, "at least two API replicas are required")
|
||||
project.wait_healthy({item.id for item in replicas})
|
||||
client = Client(args.base_url, args.ca_file, timeout=2)
|
||||
for item in replicas:
|
||||
project.wait_routed(item.instance_id, client.base, args.ca_file)
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
confirmed = set()
|
||||
|
||||
for target in replicas:
|
||||
snapshot(project, "before", target.instance_id)
|
||||
stop = Event()
|
||||
lock = Lock()
|
||||
successes = []
|
||||
errors = []
|
||||
|
||||
def worker():
|
||||
local = Client(client.base, args.ca_file, timeout=2)
|
||||
key = uuid4().hex
|
||||
while not stop.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = local.message(actor, chat, "failover-" + key, key=key)
|
||||
with lock:
|
||||
successes.append((started, time.monotonic(), result["json"]["id"]))
|
||||
key = uuid4().hex
|
||||
except Failure as error:
|
||||
with lock:
|
||||
errors.append(failure_kind(error))
|
||||
time.sleep(0.1)
|
||||
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
jobs = [pool.submit(worker) for _ in range(args.concurrency)]
|
||||
try:
|
||||
time.sleep(2)
|
||||
stop_started = time.monotonic()
|
||||
subprocess.run(
|
||||
[str(ROOT / "scripts/stop-one.sh")], cwd=ROOT,
|
||||
env={**os.environ, "TARGET_INSTANCE_ID": target.instance_id},
|
||||
check=True, timeout=10, capture_output=True,
|
||||
)
|
||||
stop_finished = time.monotonic()
|
||||
stopped = snapshot(project, "stopped", target.instance_id)
|
||||
check(any(item.id == target.id and not item.running for item in stopped), "target API was not stopped")
|
||||
check(any(item.id != target.id and item.running and item.health == "healthy" for item in stopped), "no healthy surviving API")
|
||||
time.sleep(args.duration)
|
||||
finally:
|
||||
stop.set()
|
||||
try:
|
||||
for job in jobs:
|
||||
job.result(timeout=5)
|
||||
finally:
|
||||
project.docker("start", target.id)
|
||||
project.wait_healthy({item.id for item in replicas})
|
||||
project.wait_routed(target.instance_id, client.base, args.ca_file)
|
||||
snapshot(project, "restored", target.instance_id)
|
||||
|
||||
elapsed = max(end for _, end, _ in successes) - started if successes else 0
|
||||
recovered = [end for begin, end, _ in successes if begin >= stop_finished]
|
||||
check(bool(recovered), "no confirmed writes after failure")
|
||||
recovery = min(recovered) - stop_started
|
||||
check(recovery <= 10, "writes did not recover within 10 seconds")
|
||||
confirmed.update(id for _, _, id in successes)
|
||||
items = history(client, actor, chat)
|
||||
check(confirmed <= {item["id"] for item in items}, "confirmed messages were lost")
|
||||
texts = [item["text"] for item in items]
|
||||
check(len(texts) == len(set(texts)), "uncertain retry created duplicates")
|
||||
print(json.dumps({
|
||||
"instance": target.instance_id,
|
||||
"successful_writes": len(successes),
|
||||
"failed_attempts": len(errors),
|
||||
"errors_by_kind": dict(Counter(errors)),
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"successful_rps": round(len(successes) / elapsed, 3) if elapsed else 0,
|
||||
"first_recovery_seconds": round(recovery, 3),
|
||||
}), flush=True)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Готовит параметры эксперимента отказа реплик."""
|
||||
def configure(parser):
|
||||
parser.add_argument("--duration", type=float, default=15)
|
||||
parser.add_argument("--concurrency", type=int, default=2)
|
||||
|
||||
args = arguments("Fail each API replica under write load", configure)
|
||||
check(1 <= args.duration <= 120 and 1 <= args.concurrency <= 8, "invalid workload")
|
||||
project = ComposeProject()
|
||||
with deployment_lock(project):
|
||||
exercise(project, args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -1,64 +0,0 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
BASE_URL="${BASE_URL:-http://localhost:8080}"
|
||||
API="$BASE_URL/api/v1"
|
||||
USERNAME="demo_$(date +%s)"
|
||||
PASSWORD="demo-password-123"
|
||||
|
||||
compose() {
|
||||
docker compose --env-file .env "$@"
|
||||
}
|
||||
|
||||
code() {
|
||||
curl -s -o /dev/null -w '%{http_code} (%{time_total}s)' "$@"
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
attempts=0
|
||||
until [ "$(compose ps --format '{{.Health}}' "$1")" = "healthy" ]; do
|
||||
attempts=$((attempts + 1))
|
||||
[ "$attempts" -lt 60 ] || { echo "$1 не стал healthy" >&2; exit 1; }
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
curl -sf -o /dev/null -X POST "$API/users" -H 'Content-Type: application/json' \
|
||||
-d "{\"username\":\"$USERNAME\",\"display_name\":\"Demo\",\"password\":\"$PASSWORD\"}"
|
||||
TOKEN=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD" \
|
||||
| python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])')
|
||||
AUTH="Authorization: Bearer $TOKEN"
|
||||
|
||||
echo "== baseline"
|
||||
echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")"
|
||||
|
||||
echo
|
||||
echo "== redis stopped"
|
||||
compose stop redis >/dev/null 2>&1
|
||||
echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")"
|
||||
echo "POST /auth/sessions (Basic) -> $(code -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD")"
|
||||
echo "GET /users/me (no token) -> $(code "$API/users/me")"
|
||||
echo "GET /health/ready -> $(code "$BASE_URL/health/ready")"
|
||||
echo "GET /health/live -> $(code "$BASE_URL/health/live")"
|
||||
compose start redis >/dev/null 2>&1
|
||||
wait_healthy redis
|
||||
echo "-- redis started again (AOF restored)"
|
||||
echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")"
|
||||
|
||||
echo
|
||||
echo "== postgres stopped"
|
||||
compose stop postgres >/dev/null 2>&1
|
||||
echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")"
|
||||
echo "GET /api/v1/chats (Bearer) -> $(code "$API/chats" -H "$AUTH")"
|
||||
echo "GET /health/ready -> $(code "$BASE_URL/health/ready")"
|
||||
echo "GET /health/live -> $(code "$BASE_URL/health/live")"
|
||||
compose start postgres >/dev/null 2>&1
|
||||
wait_healthy postgres
|
||||
echo "-- postgres started again"
|
||||
echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")"
|
||||
|
||||
echo
|
||||
echo "== logout"
|
||||
echo "DELETE /auth/sessions/current -> $(code -X DELETE "$API/auth/sessions/current" -H "$AUTH")"
|
||||
echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")"
|
||||
@@ -3,7 +3,7 @@ set -eu
|
||||
# TODO: copy to restart.sh and implement using YOUR compose/stack services.
|
||||
# Restart API and ALL used state stores (DB, sessions, S3, broker as applicable).
|
||||
# Preserve persistent volumes and schema/data; never down -v / flush / re-seed.
|
||||
# Allow operator unseal if required in lab7; do not print credentials.
|
||||
# Do not print credentials.
|
||||
# Exit 0 only after the intended action; test waits for API readiness separately.
|
||||
printf '%s
|
||||
' 'TODO: implement restart.sh for your infrastructure' >&2
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
"""Поочерёдно заменяет API с одной дополнительной репликой."""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
from scripts.deployment import ComposeProject, deployment_lock
|
||||
|
||||
|
||||
def update(project: ComposeProject, base_url: str, ca_file=None, revision: str | None = None) -> None:
|
||||
"""Собирает образ, применяет миграции и заменяет реплики."""
|
||||
original = project.replicas()
|
||||
if len(original) < 2 or any(not item.running or item.health != "healthy" for item in original):
|
||||
raise RuntimeError("До обновления нужны минимум две здоровые API-реплики")
|
||||
old_images = {item.image for item in original}
|
||||
if len(old_images) != 1:
|
||||
raise RuntimeError("Сначала восстановите единый образ работающих реплик")
|
||||
old_image = old_images.pop()
|
||||
profile = project.docker(
|
||||
"image", "inspect", old_image,
|
||||
"--format", '{{index .Config.Labels "io.mtusi.messenger.secrets"}}',
|
||||
)
|
||||
if profile != "file-v1":
|
||||
raise RuntimeError("Сначала выполните первоначальный переход на файловые секреты через make up")
|
||||
count = len(original)
|
||||
print("Сборка нового образа API", flush=True)
|
||||
build_args = ("--build-arg", f"API_BUILD_REVISION={revision}") if revision else ()
|
||||
project.run("build", *build_args, "api", timeout=300)
|
||||
tag = project.config["services"]["api"]["image"]
|
||||
new_image = project.docker("image", "inspect", tag, "--format", "{{.Id}}")
|
||||
print("Однократное применение совместимых миграций", flush=True)
|
||||
project.run("run", "--rm", "--no-deps", "--pull", "never", "migrate")
|
||||
if old_image == new_image:
|
||||
print("Образ не изменился; замена контейнеров не требуется")
|
||||
return
|
||||
|
||||
try:
|
||||
for old in original:
|
||||
project.run(
|
||||
"up", "-d", "--no-deps", "--no-build", "--pull", "never",
|
||||
"--no-recreate", "--scale", f"api={count + 1}", "api",
|
||||
)
|
||||
current = project.replicas()
|
||||
additions = [item for item in current if item.id not in {replica.id for replica in original} and item.image == new_image]
|
||||
if not additions:
|
||||
raise RuntimeError("Новая реплика не создана")
|
||||
project.wait_healthy({item.id for item in additions})
|
||||
for added in additions:
|
||||
project.wait_routed(added.instance_id, base_url, ca_file)
|
||||
survivors = {item.id for item in project.replicas() if item.id != old.id}
|
||||
project.wait_healthy(survivors)
|
||||
project.docker("stop", old.id)
|
||||
project.docker("rm", old.id)
|
||||
print(f"Заменена реплика {old.instance_id}", flush=True)
|
||||
final = project.replicas()
|
||||
if len(final) != count or any(item.image != new_image for item in final):
|
||||
raise RuntimeError("Состав реплик после обновления не соответствует плану")
|
||||
project.wait_healthy({item.id for item in final})
|
||||
except Exception:
|
||||
print("Обновление прервано; восстанавливается прежний образ без отката БД", file=sys.stderr)
|
||||
project.run(
|
||||
"up", "-d", "--no-deps", "--no-build", "--pull", "never",
|
||||
"--force-recreate", "--scale", f"api={count}", "api",
|
||||
env={"API_IMAGE": old_image},
|
||||
)
|
||||
project.wait_healthy({item.id for item in project.replicas()})
|
||||
raise
|
||||
|
||||
dangling = project.docker("image", "ls", "--quiet", "--no-trunc", "--filter", "dangling=true").splitlines()
|
||||
if old_image in dangling:
|
||||
references = project.docker("ps", "--all", "--quiet", "--filter", f"ancestor={old_image}")
|
||||
if not references:
|
||||
try:
|
||||
project.docker("image", "rm", old_image)
|
||||
except RuntimeError:
|
||||
print("Прежний образ не удалён: состояние Docker изменилось", file=sys.stderr)
|
||||
print(f"Готово: {count} здоровых реплик нового образа")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Запускает обновление с блокировкой проекта."""
|
||||
parser = argparse.ArgumentParser(description="Rolling update of API replicas")
|
||||
parser.add_argument("--base-url", default="http://localhost:8080")
|
||||
parser.add_argument("--ca-file", default=None)
|
||||
parser.add_argument("--revision", default=None)
|
||||
args = parser.parse_args()
|
||||
project = ComposeProject()
|
||||
with deployment_lock(project):
|
||||
update(project, args.base_url or "http://localhost:8080", args.ca_file, args.revision)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,108 +0,0 @@
|
||||
"""Исполняется внутри API-контейнера: реальные S3/БД и управляемые отказы."""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import json
|
||||
import sys
|
||||
from uuid import UUID
|
||||
|
||||
from minio.error import S3Error
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.domain import attachments as domain
|
||||
from app.errors.exceptions import unavailable
|
||||
from app.models.attachment import UploadIntent
|
||||
from app.schemas.user import User
|
||||
from app.storage import attachments as repository
|
||||
from app.storage.s3 import create_s3_storage
|
||||
|
||||
|
||||
async def object_exists(storage, key):
|
||||
exists = True
|
||||
try:
|
||||
await storage.stat_object(key)
|
||||
except S3Error as error:
|
||||
if error.code not in {'NoSuchKey', 'NoSuchObject'}:
|
||||
raise
|
||||
exists = False
|
||||
return exists
|
||||
|
||||
|
||||
async def main():
|
||||
payload = json.load(sys.stdin)
|
||||
settings = Settings()
|
||||
engine = create_db_engine(settings)
|
||||
factory = create_session_factory(engine)
|
||||
storage = create_s3_storage(settings.s3)
|
||||
user = User.model_validate(payload['user'])
|
||||
chat_id = UUID(payload['chat_id'])
|
||||
original = base64.b64decode(payload['image'])
|
||||
keys = []
|
||||
create_record = repository.create_attachment
|
||||
put_file, delete_object = storage.put_file, storage.delete_object
|
||||
|
||||
async def capture_put(key, path, **metadata):
|
||||
keys.append(key)
|
||||
await put_file(key, path, **metadata)
|
||||
|
||||
async def fail_metadata(*args, **kwargs):
|
||||
raise OperationalError('injected metadata failure after real S3 PUT', {}, None)
|
||||
|
||||
async def fail_delete(*args):
|
||||
raise unavailable('injected compensation failure')
|
||||
|
||||
async def chunks():
|
||||
yield original
|
||||
|
||||
async def attempt():
|
||||
raised = False
|
||||
async with factory() as session:
|
||||
try:
|
||||
await domain.upload_attachment(session, storage, user, chat_id,
|
||||
filename='fault.png', content_type='image/png', chunks=chunks())
|
||||
except OperationalError:
|
||||
raised = True
|
||||
assert raised, 'injected database failure was not raised'
|
||||
|
||||
storage.put_file = capture_put
|
||||
repository.create_attachment = fail_metadata
|
||||
try:
|
||||
await attempt()
|
||||
assert not await object_exists(storage, keys[-1]), 'compensation left original'
|
||||
async with factory() as session:
|
||||
intent = await session.scalar(select(UploadIntent).where(UploadIntent.object_key == keys[-1]))
|
||||
assert intent is None, 'compensated upload left intent'
|
||||
storage.delete_object = fail_delete
|
||||
await attempt()
|
||||
key = keys[-1]
|
||||
assert await object_exists(storage, key), 'fault did not leave real S3 object'
|
||||
storage.delete_object = delete_object
|
||||
async with factory() as session:
|
||||
result = await domain.reconcile_attachments(session, storage, dry_run=False)
|
||||
assert result.deleted == 0, 'fresh object deleted before grace period'
|
||||
intent = await session.scalar(select(UploadIntent).where(UploadIntent.object_key == key))
|
||||
assert intent is not None, 'durable intent missing'
|
||||
intent.created_at = datetime.now(timezone.utc) - timedelta(days=2)
|
||||
await session.commit()
|
||||
dry_run = await domain.reconcile_attachments(session, storage)
|
||||
assert dry_run.candidates == 1 and dry_run.deleted == 0
|
||||
assert await object_exists(storage, key), 'dry-run removed original'
|
||||
applied = await domain.reconcile_attachments(session, storage, dry_run=False)
|
||||
assert applied.deleted == 1 and not await object_exists(storage, key)
|
||||
repeated = await domain.reconcile_attachments(session, storage, dry_run=False)
|
||||
assert repeated.examined == repeated.deleted == 0
|
||||
print(json.dumps({'compensation': True, 'durable_intent': True, 'grace_period': True,
|
||||
'dry_run': True, 'cleanup': True, 'idempotent_cleanup': True}))
|
||||
finally:
|
||||
repository.create_attachment = create_record
|
||||
storage.put_file, storage.delete_object = put_file, delete_object
|
||||
await storage.aclose()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -1,113 +0,0 @@
|
||||
"""Повторяемые параллельные upload и память cgroup API, без записи токенов."""
|
||||
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
import json
|
||||
from pathlib import Path
|
||||
import random
|
||||
import subprocess
|
||||
from threading import Barrier
|
||||
import time
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, Failure, arguments, check, run
|
||||
|
||||
MONITOR = r'''
|
||||
import json, pathlib, sys, threading
|
||||
root = pathlib.Path('/sys/fs/cgroup')
|
||||
def current():
|
||||
return int((root / 'memory.current').read_text())
|
||||
def peak():
|
||||
path = root / 'memory.peak'
|
||||
return int(path.read_text()) if path.exists() else None
|
||||
stop = threading.Event()
|
||||
threading.Thread(target=lambda: (sys.stdin.read(), stop.set()), daemon=True).start()
|
||||
initial, before = current(), peak()
|
||||
sampled = initial
|
||||
print('ready', flush=True)
|
||||
while not stop.wait(0.02):
|
||||
sampled = max(sampled, current())
|
||||
print(json.dumps({'initial_bytes': initial, 'sampled_peak_bytes': sampled,
|
||||
'cgroup_peak_before_bytes': before, 'cgroup_peak_after_bytes': peak(), 'interval_ms': 20}), flush=True)
|
||||
'''
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--concurrency', type=int, default=10)
|
||||
parser.add_argument('--rounds', type=int, default=3)
|
||||
parser.add_argument('--output', type=Path)
|
||||
args = arguments('Concurrent attachment uploads and API memory', configure)
|
||||
check(10 <= args.concurrency <= 32 and 1 <= args.rounds <= 10, 'use concurrency 10..32 and rounds 1..10')
|
||||
project = ComposeProject()
|
||||
client = Client(args.base_url, args.ca_file, timeout=65)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
random_bytes = random.Random(42).randbytes(1500 * 1500 * 3)
|
||||
image = Image.frombytes('RGB', (1500, 1500), random_bytes)
|
||||
buffer = BytesIO()
|
||||
image.save(buffer, format='PNG')
|
||||
data = buffer.getvalue()
|
||||
check(len(data) <= 10485760, 'fixture exceeds upload limit')
|
||||
monitors = []
|
||||
measurements = {}
|
||||
rows = []
|
||||
started = time.monotonic()
|
||||
try:
|
||||
for replica in project.replicas():
|
||||
process = subprocess.Popen(['docker', 'exec', '-i', replica.id, 'python', '-c', MONITOR],
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
monitors.append((replica.instance_id, process))
|
||||
check(process.stdout.readline().strip() == 'ready', 'memory monitor failed')
|
||||
check(bool(monitors), 'API not running')
|
||||
for _ in range(args.rounds):
|
||||
barrier = Barrier(args.concurrency)
|
||||
def upload(index):
|
||||
local = Client(args.base_url, args.ca_file, timeout=65)
|
||||
barrier.wait(timeout=15)
|
||||
request_started = time.monotonic()
|
||||
try:
|
||||
return local.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=load.png",
|
||||
expected=[201, 400, 401, 404, 409, 413, 415, 429, 500, 502, 503, 504], actor=alice, raw=data,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
except Failure:
|
||||
# При раннем отказе сервер может закрыть ещё отправляемое тело.
|
||||
return {'status': 'failed_request', 'elapsed': time.monotonic() - request_started}
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
rows.extend(pool.map(upload, range(args.concurrency)))
|
||||
finally:
|
||||
elapsed = time.monotonic() - started
|
||||
for alias, process in monitors:
|
||||
output, _ = process.communicate(input='', timeout=10)
|
||||
check(process.returncode == 0, 'memory monitor failed')
|
||||
measurements[alias] = json.loads(output)
|
||||
succeeded = [row for row in rows if row['status'] == 201]
|
||||
check(bool(succeeded), 'all uploads were rejected')
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
for row in succeeded:
|
||||
check(row['json']['sha256'] == digest, 'upload checksum differs')
|
||||
original = client.call('GET', '/api/v1/attachments/' + row['json']['id'] + '/content', actor=alice)['bytes']
|
||||
check(hashlib.sha256(original).hexdigest() == digest, 'download checksum differs')
|
||||
latencies = sorted(row['elapsed'] for row in rows)
|
||||
result = {
|
||||
'concurrency': args.concurrency, 'rounds': args.rounds, 'attempts': len(rows),
|
||||
'fixture': {'format': 'PNG', 'width': 1500, 'height': 1500, 'size_bytes': len(data), 'sha256': digest},
|
||||
'statuses': dict(Counter(str(row['status']) for row in rows)),
|
||||
'elapsed_seconds': round(elapsed, 3), 'successful_uploads': len(succeeded),
|
||||
'p95_seconds': round(latencies[min(len(latencies) - 1, int(len(latencies) * .95))], 3),
|
||||
'memory': measurements, 'api_memory_limit': project.config['services']['api']['deploy']['resources']['limits']['memory'],
|
||||
'notes': '20 ms samples include monitor and decoder processes; cgroup peak is cumulative since container creation; 429 and failed requests counted separately; no upload retries',
|
||||
}
|
||||
encoded = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(encoded + '\n')
|
||||
print(encoded)
|
||||
check(all(row['status'] in {201, 429} for row in rows), 'unexpected HTTP/transport errors during upload experiment; inspect status counts')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -11,9 +11,9 @@ validate(spec)
|
||||
from referencing import Registry, Resource
|
||||
# The base URI resolves local #/components references inside examples.
|
||||
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
|
||||
registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource))
|
||||
registry = Registry().with_resource('urn:messenger:openapi', Resource.from_contents(resource))
|
||||
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
|
||||
schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name}
|
||||
schema = {'$ref': 'urn:messenger:openapi#/components/schemas/' + name}
|
||||
Draft202012Validator(schema, registry=registry).validate(example)
|
||||
for path in (root/'contracts').glob('*.schema.json'):
|
||||
schema = json.loads(path.read_text())
|
||||
|
||||
Reference in New Issue
Block a user