pure version of lab5

This commit is contained in:
2026-11-10 07:44:19 +03:00
parent 2b9fb3f375
commit 7cd02da51d
31 changed files with 295 additions and 1866 deletions
-355
View File
@@ -1,355 +0,0 @@
"""Проверяет инфраструктуру, SIGTERM и обновление реального стенда."""
import asyncio
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path
import subprocess
from threading import Event
import time
from uuid import uuid4
import asyncpg
from scripts.deployment import ComposeProject, ROOT, deployment_lock
from scripts.failover_all import exercise, failure_kind, history
from scripts.rolling_update import update
from tests.client import Client, Failure, arguments, check, run
PROBE = """import json, sys, urllib.request, urllib.error
payload = json.load(sys.stdin)
results = []
for item in payload:
data = json.dumps(item['body']).encode() if 'body' in item else None
request = urllib.request.Request(item['url'], data=data, headers=item.get('headers', {}), method=item.get('method', 'GET'))
try:
response = urllib.request.urlopen(request, timeout=7)
except urllib.error.HTTPError as error:
response = error
with response:
results.append({'status': response.status, 'body': json.loads(response.read())})
print(json.dumps(results))
"""
def probe(project, runner, requests):
"""Обращается к внутреннему API, передавая токен только через stdin."""
result = subprocess.run(
["docker", "exec", "-i", runner, "python", "-c", PROBE],
input=json.dumps(requests),
text=True,
capture_output=True,
timeout=10,
)
check(result.returncode == 0, "internal HTTP probe failed")
return json.loads(result.stdout)
def audit(project):
"""Проверяет фактические лимиты, порты и область доступа к секретам."""
allowed = {
"postgres": {"db_password"},
"redis": {"redis_password"},
"migrate": {"db_password"},
"api": {"db_password", "redis_password", "s3_access_key_id", "s3_secret_access_key"},
"minio": {"minio_root_user", "minio_root_password"},
"minio-bootstrap": {"minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"},
"cleanup": {"db_password", "s3_access_key_id", "s3_secret_access_key"},
"nginx": set(),
}
ids = project.run("ps", "--all", "--quiet").splitlines()
containers = json.loads(project.docker("inspect", *ids))
for item in containers:
service = item["Config"]["Labels"]["com.docker.compose.service"]
check(service in allowed, "unexpected service")
host = item["HostConfig"]
check(
host["Memory"] > 0 and host["NanoCpus"] > 0,
service + ": missing CPU/RAM limit",
)
env_names = {entry.split("=", 1)[0] for entry in item["Config"]["Env"]}
check(
not env_names
& {
"DB_PASSWORD",
"REDIS_PASSWORD",
"POSTGRES_PASSWORD",
"REDISCLI_AUTH",
"DATABASE_URL",
"MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY",
},
service + ": password in environment",
)
secrets = {
Path(mount["Destination"]).name
for mount in item["Mounts"]
if mount["Destination"].startswith("/run/secrets/")
}
check(secrets == allowed[service], service + ": incorrect secret scope")
if service in {"api", "minio"}:
check(not host["PortBindings"], service + " publishes ports")
for mount in item["Mounts"]:
if mount["Destination"].startswith("/run/secrets/"):
check(not mount["RW"], "writable secret mount")
check(
"container_name" not in project.config["services"][service],
"fixed container name",
)
check(
len(containers) == len(allowed) - 1 + len(project.replicas()),
"unexpected container count",
)
print(
"PASS infrastructure: CPU/RAM, private API ports, scoped read-only file secrets, no environment passwords",
flush=True,
)
async def graceful(project, base_url):
"""Удерживает реальный POST в БД и завершает его после SIGTERM."""
replicas = project.replicas()
target, runner = replicas[:2]
client = Client(base_url)
actor, _, _, chat = await asyncio.to_thread(client.bootstrap)
path = f"/api/v1/chats/{chat['id']}/messages"
key = uuid4().hex
raw = json.dumps([actor["id"], "POST", path, key]).encode()
lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True)
database = project.config["services"]["postgres"]
env = database["environment"]
conn = await asyncpg.connect(
host="127.0.0.1",
port=int(database["ports"][0]["published"]),
database=env["POSTGRES_DB"],
user=env["POSTGRES_USER"],
password=(ROOT / "secrets/db_password").read_text().strip(),
timeout=3,
)
network = project.config['networks']['default']['name']
address_template = '{{with index .NetworkSettings.Networks ' + json.dumps(network) + '}}{{.IPAddress}}{{end}}'
address = project.docker(
"inspect",
"--format",
address_template,
target.id,
)
check(bool(address), 'API has no address in the application network')
port = project.config["services"]["api"]["environment"]["APP_PORT"]
direct = f"http://{address}:{port}"
started = time.monotonic()
task = None
try:
async with conn.transaction():
await conn.execute("SELECT pg_advisory_xact_lock($1)", lock_id)
task = asyncio.create_task(
asyncio.to_thread(
probe,
project,
runner.id,
[
{
"url": direct + path,
"method": "POST",
"body": {"text": "graceful-" + key},
"headers": {
"Authorization": "Bearer " + actor["token"],
"Idempotency-Key": key,
"Content-Type": "application/json",
},
}
],
)
)
deadline = time.monotonic() + 3
while not await conn.fetchval(
"SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE wait_event = 'advisory' AND datname = current_database())"
):
check(time.monotonic() < deadline, "POST did not reach database lock")
await asyncio.sleep(0.02)
started = time.monotonic()
await asyncio.to_thread(
project.docker, "kill", "--signal", "SIGTERM", target.id
)
results = await asyncio.to_thread(
probe,
project,
runner.id,
[
{"url": direct + "/health/ready"},
{"url": direct + "/health/live"},
{"url": direct + "/api/v1/users/me"},
],
)
check(
[item["status"] for item in results] == [503, 200, 503],
"incorrect drain readiness/liveness/new work",
)
await asyncio.sleep(max(0, 1.2 - (time.monotonic() - started)))
result = (await task)[0]
check(result["status"] == 201, "in-flight POST failed during SIGTERM")
await asyncio.to_thread(project.docker, "wait", target.id, timeout=10)
state = json.loads(
project.docker("inspect", "--format", "{{json .State}}", target.id)
)
check(
not state["OOMKilled"] and state["ExitCode"] != 137,
"forced shutdown instead of graceful completion",
)
check(time.monotonic() - started < 8, "shutdown exceeded Docker grace budget")
items = await asyncio.to_thread(history, client, actor, chat)
check(
result["body"]["id"] in {item["id"] for item in items},
"acknowledged in-flight message lost",
)
print(
f"PASS SIGTERM: readiness 503, live 200, new work 503, in-flight POST 201; exit in {time.monotonic() - started:.3f}s",
flush=True,
)
finally:
await conn.close()
if task:
await asyncio.gather(task, return_exceptions=True)
if any(item.id == target.id and item.running for item in project.replicas()):
await asyncio.to_thread(project.docker, "stop", target.id)
await asyncio.to_thread(project.docker, "start", target.id)
await asyncio.to_thread(project.wait_healthy, {item.id for item in replicas})
await asyncio.to_thread(project.wait_routed, target.instance_id, base_url)
class ObservedProject(ComposeProject):
"""Подсчитывает миграции и проверяет готовность перед заменой."""
def __init__(self):
self.migrations = 0
self.replacements = 0
self.replacement_started = []
super().__init__()
def run(self, *args, **kwargs):
if args[:6] == ("run", "--rm", "--no-deps", "--pull", "never", "migrate"):
self.migrations += 1
return super().run(*args, **kwargs)
def docker(self, *args, **kwargs):
if args[0] == "stop":
healthy = [
item
for item in self.replicas()
if item.id != args[1] and item.running and item.health == "healthy"
]
check(len(healthy) >= 2, "rolling update reduces healthy capacity")
self.replacements += 1
self.replacement_started.append(time.monotonic())
return super().docker(*args, **kwargs)
def rolling(base_url):
"""Заменяет образ под постоянной записью с проверкой сохранности."""
project = ObservedProject()
before = project.replicas()
client = Client(base_url)
actor, _, _, chat = client.bootstrap()
stop = Event()
results, errors = [], []
def worker():
local = Client(base_url, timeout=2)
key = uuid4().hex
while not stop.is_set():
started = time.monotonic()
try:
result = local.message(actor, chat, "rolling-" + key, key=key)
results.append(
(started, time.monotonic(), result["json"]["id"], result["elapsed"])
)
key = uuid4().hex
except Failure as error:
errors.append(failure_kind(error))
stop.wait(0.08)
with ThreadPoolExecutor(max_workers=2) as pool:
jobs = [pool.submit(worker) for _ in range(2)]
try:
time.sleep(1)
started = time.monotonic()
update(project, base_url, revision="verification-" + uuid4().hex[:12])
finished = time.monotonic()
time.sleep(1)
finally:
stop.set()
for job in jobs:
job.result(timeout=5)
after = project.replicas()
check(
project.migrations == 1 and project.replacements == len(before),
"migration/replacement count incorrect",
)
check(
{item.image for item in before}.isdisjoint({item.image for item in after}),
"new image was not deployed",
)
check(
{item.id for item in before}.isdisjoint({item.id for item in after}),
"old containers remain",
)
writes = [row for row in results if started <= row[0] <= finished]
check(bool(writes), "no writes during update")
check(
any(project.replacement_started[0] <= row[0] <= finished for row in writes),
"no writes during container replacement",
)
items = history(client, actor, chat)
check(
{row[2] for row in results} <= {item["id"] for item in items},
"rolling update lost confirmed messages",
)
check(
len(items) == len({item["text"] for item in items}),
"rolling update created duplicate messages",
)
latencies = sorted(row[3] for row in writes)
print(
json.dumps(
{
"scenario": "rolling update",
"replacements": project.replacements,
"migration_runs": project.migrations,
"successful_writes": len(writes),
"failed_attempts": len(errors),
"errors_by_kind": dict(Counter(errors)),
"elapsed_seconds": round(finished - started, 3),
"successful_rps": round(len(writes) / (finished - started), 3),
"p95_seconds": round(
latencies[min(len(latencies) - 1, int(len(latencies) * 0.95))], 3
),
}
),
flush=True,
)
def main():
"""Выполняет проверки только на отдельном тестовом проекте."""
def configure(parser):
parser.add_argument(
"--phase", choices=("all", "update-and-failover"), default="all"
)
args = arguments("Grade 4 integration checks", configure)
project = ComposeProject()
check(project.name.endswith("-test"), "use isolated test Compose project")
with deployment_lock(project):
audit(project)
if args.phase == "all":
asyncio.run(graceful(project, args.base_url))
rolling(args.base_url)
args.duration, args.concurrency = 10, 2
exercise(project, args)
audit(project)
if __name__ == "__main__":
run(main)
-39
View File
@@ -1,39 +0,0 @@
"""Проверяет конкурентные повторы через балансировщик."""
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
from tests.client import Client, arguments, check, run
def main() -> None:
"""Отправляет один ключ параллельно через разные реплики."""
args = arguments("Concurrent idempotency through load balancer")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
key = uuid4().hex
body = "parallel-" + key
def send(_):
return Client(args.base_url, args.ca_file).message(alice, chat, body, key=key)
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(send, range(24)))
ids = {result["json"]["id"] for result in results}
replicas = {result["headers"]["X-Instance-Id"] for result in results}
check(len(ids) == 1, "parallel retries created duplicate messages")
check(len(replicas) >= 2, "parallel retries did not reach two API replicas")
history = client.call(
"GET",
f"/api/v1/chats/{chat['id']}/messages?limit=100",
actor=alice,
schema="MessagePage",
)["json"]["items"]
check(
len(history) == 1 and history[0] == results[0]["json"], "history has duplicates"
)
print(f"PASS: 24 concurrent replays, {len(replicas)} replicas, one message")
if __name__ == "__main__":
run(main)
@@ -1,4 +1,4 @@
"""Fault/recovery lab5 на явно выбранном Compose-стенде; только синтетические данные."""
"""Fault/recovery очереди на явно выбранном Compose-стенде; только синтетические данные."""
import argparse
from datetime import datetime
@@ -79,7 +79,7 @@ def verify_result(client, actor, attachment, original):
def crash(client, actor, chat, mode):
name='lab5-worker-fault-'+uuid4().hex
name='messenger-worker-fault-'+uuid4().hex
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
@@ -106,7 +106,7 @@ def crash(client, actor, chat, mode):
def retry_replay(client,actor,chat,permanent):
name='lab5-retry-fault-'+uuid4().hex
name='messenger-retry-fault-'+uuid4().hex
samples=[]
configuration=ComposeProject().config['services']['worker']['environment']
retry_base=float(configuration['JOB_RETRY_SECONDS'])
@@ -189,10 +189,11 @@ def persistence(client,actor,chat):
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--base-url',required=True)
parser.add_argument('--ca-file')
parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all')
parser.add_argument('--output',default='evidence/lab5-jobs.json')
parser.add_argument('--output',default='evidence/jobs.json')
args=parser.parse_args()
client=Client(args.base_url,timeout=40)
client=Client(args.base_url,args.ca_file,timeout=40)
aliases={r.instance_id for r in ComposeProject().replicas() if r.running}
check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL')
actor,_,_,chat=client.bootstrap()
+3 -3
View File
@@ -25,7 +25,7 @@ def check(condition: bool, message: str) -> None:
class Stand:
def __init__(self, directory: str) -> None:
self.directory = Path(directory)
self.project = "mtusi-lab3-nginx-check-" + uuid4().hex[:12]
self.project = "messenger-nginx-check-" + uuid4().hex[:12]
self.base_url = ""
backend = self.directory / "backend.conf"
backend.write_text(
@@ -246,7 +246,7 @@ def run_checks(stand: Stand) -> None:
def main() -> None:
with tempfile.TemporaryDirectory(prefix="mtusi-lab3-nginx-check-") as directory:
with tempfile.TemporaryDirectory(prefix="messenger-nginx-check-") as directory:
stand = Stand(directory)
try:
run_checks(stand)
@@ -254,7 +254,7 @@ def main() -> None:
print(stand.compose("logs", "--no-color", "--tail", "20"), flush=True)
raise
finally:
stand.compose("down", "--remove-orphans", "--timeout", "1")
stand.compose("down", "--volumes", "--remove-orphans", "--timeout", "1")
if __name__ == "__main__":
+2 -2
View File
@@ -34,7 +34,7 @@ def main() -> None:
parser.add_argument("--ca-file")
args = parser.parse_args()
running = set(require_compose("ps", "--status", "running", "--services").splitlines())
check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the lab5 stack before this scenario")
check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the application stack before this scenario")
client = Client(args.base_url, args.ca_file, timeout=40)
project = ComposeProject()
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
@@ -42,7 +42,7 @@ def main() -> None:
check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project")
alice, _, _, chat = client.bootstrap()
attachment = None
fault_container = "lab5-outbox-fault-" + uuid4().hex
fault_container = "messenger-outbox-fault-" + uuid4().hex
try:
require_compose("stop", "publisher")
attachment, original = client.attachment(alice, chat)
+4 -20
View File
@@ -1,8 +1,6 @@
"""Реальный MinIO: повтор bootstrap, private bucket, ограниченные права и отказ."""
import json
import base64
from pathlib import Path
import subprocess
import time
@@ -70,12 +68,15 @@ asyncio.run(main())
def main():
def configure(parser):
parser.add_argument('--with-outage', action='store_true', help='временно остановить MinIO выбранного проекта')
parser.add_argument('--with-faults', action='store_true', help='проверить компенсацию и очистку на реальных S3/БД')
args = arguments('MinIO private bucket and failure checks', configure)
project = ComposeProject()
client = Client(args.base_url, args.ca_file, timeout=40)
alice, _, _, chat = client.bootstrap()
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
live = client.call('GET', '/health/live', schema='Health')
check(live['headers'].get('X-Instance-Id') in aliases, 'BASE_URL does not belong to the selected Compose project')
attachment, original = client.attachment(alice, chat)
attachment = client.wait_ready(alice, attachment)
# Повторный bootstrap не должен очистить bucket или сменить используемые credentials.
for _ in range(2):
project.run('run', '--rm', '--no-deps', 'minio-bootstrap', timeout=100)
@@ -96,21 +97,6 @@ def main():
check('127.0.0.1:9001' in project.config['services']['minio']['command'], 'console is not bound to loopback')
path = '/api/v1/attachments/' + attachment['id'] + '/content'
check(client.call('GET', path, actor=alice)['bytes'] == original, 'bootstrap changed original')
if args.with_faults:
project.run('stop', 'cleanup')
try:
fault_script = Path(__file__).with_name('s3_faults.py').read_text()
result = subprocess.run(
['docker', 'exec', '-i', replicas[0].id, 'python', '-c', fault_script],
input=json.dumps({'user': alice['user'], 'chat_id': chat['id'],
'image': base64.b64encode(original).decode()}),
text=True, capture_output=True, timeout=60,
)
check(result.returncode == 0, 'real S3/database compensation probe failed (details hidden)')
check(all(json.loads(result.stdout).values()), 'real S3/database invariants failed')
finally:
project.run('start', 'cleanup')
check(client.call('GET', path, actor=alice)['bytes'] == original, 'cleanup damaged confirmed original')
if args.with_outage:
try:
project.run('stop', 'minio')
@@ -129,8 +115,6 @@ def main():
check(client.call('GET', path, actor=alice)['bytes'] == original, 'original lost after MinIO restart')
print('PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console'
+ (', MinIO outage/recovery' if args.with_outage else ''))
if args.with_faults:
print('PASS: real S3 PUT/database failure, compensation, durable journal, grace period, dry-run, repeated cleanup')
if __name__ == '__main__':
-204
View File
@@ -1,204 +0,0 @@
"""Повторяемый прогон lab4; публикует только успешные обезличенные evidence."""
from datetime import datetime, timezone
import hashlib
from importlib.metadata import version
import json
import os
from pathlib import Path
import platform
import re
import shlex
import subprocess
from tempfile import TemporaryDirectory
from zoneinfo import ZoneInfo
ROOT = Path(__file__).resolve().parents[1]
PROJECT = 'mtusi-messenger-lab5-test'
EVIDENCE = ROOT / 'evidence'
ENVIRONMENT = {
**os.environ,
'COMPOSE_PROJECT_NAME': PROJECT,
'COMPOSE_FILE': 'compose.yaml:compose.test.yaml',
'LB_HOST_PORT': '28080',
}
COMPOSE = ['docker', 'compose', '--env-file', '.env']
NONSECRET_SETTINGS = {
'APP_PORT', 'API_REPLICAS', 'MAX_MESSAGES', 'MAX_IN_FLIGHT',
'DB_POOL_SIZE', 'DB_MAX_OVERFLOW', 'SESSION_TTL_SECONDS',
'IDEMPOTENCY_TTL_SECONDS', 'SHUTDOWN_DRAIN_SECONDS',
'GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS', 'READINESS_TIMEOUT_SECONDS',
'S3_ENDPOINT', 'S3_BUCKET', 'S3_REGION', 'S3_CONNECT_TIMEOUT_SECONDS',
'S3_READ_TIMEOUT_SECONDS', 'S3_UPLOAD_TIMEOUT_SECONDS',
'S3_DOWNLOAD_TIMEOUT_SECONDS', 'S3_IMAGE_TIMEOUT_SECONDS',
'S3_MAX_UPLOAD_BYTES', 'S3_MAX_IMAGE_PIXELS', 'S3_CHUNK_BYTES',
'S3_MAX_CONCURRENT_UPLOADS', 'S3_CLEANUP_GRACE_SECONDS',
'S3_CLEANUP_INTERVAL_SECONDS',
}
SNAPSHOT_FORMAT = '{"id":{{json .Id}},"name":{{json .Name}},"service":{{json (index .Config.Labels "com.docker.compose.service")}},"image_id":{{json .Image}},"status":{{json .State.Status}},"started_at":{{json .State.StartedAt}},"pid":{{json .State.Pid}},"exit_code":{{json .State.ExitCode}},"oom_killed":{{json .State.OOMKilled}}}'
IMAGE_FORMAT = '{"id":{{json .Id}},"repo_digests":{{json .RepoDigests}},"architecture":{{json .Architecture}},"os":{{json .Os}}}'
def timestamp():
return datetime.now(timezone.utc).isoformat()
def invoke(command):
return subprocess.run(command, cwd=ROOT, env=ENVIRONMENT, text=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
def read_command(command):
result = invoke(command)
if result.returncode:
raise RuntimeError('Не выполнена команда: ' + shlex.join(command))
return result.stdout.strip()
def source_manifest():
names = {
'.dockerignore', '.env.example',
'Dockerfile', 'Makefile', 'alembic.ini', 'compose.yaml',
'compose.test.yaml', 'pytest.ini', 'requirements.txt', 'requirements-dev.txt', 'lab.json',
}
for directory in ('app', 'migrations', 'student-tests', 'scripts', 'infra', 'contracts', 'tests'):
names.update(str(path.relative_to(ROOT)) for path in (ROOT / directory).rglob('*')
if path.is_file() and '__pycache__' not in path.parts
and path.suffix != '.md' and not path.name.startswith('.'))
return ''.join(f'{hashlib.sha256((ROOT / name).read_bytes()).hexdigest()} {name}\n' for name in sorted(names))
def sanitize(text, secrets, staging):
clean = text.replace(str(ROOT), '.').replace(str(staging), '<temporary>')
for secret in secrets:
clean = clean.replace(secret, '<redacted>')
clean = re.sub(r'(?i)\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+', r'\1 <redacted>', clean)
clean = re.sub(r"(?i)([\"'](?:token|password|secret_access_key)[\"']\s*:\s*[\"'])[^\"']*", r'\1<redacted>', clean)
return clean
def container_snapshot(images):
ids = read_command(['docker', 'ps', '-aq', '--filter', f'label=com.docker.compose.project={PROJECT}']).splitlines()
rows = []
if ids:
output = read_command(['docker', 'inspect', '--format', SNAPSHOT_FORMAT, *ids])
rows = [json.loads(line) for line in output.splitlines()]
for row in rows:
image = row['image_id']
if image not in images:
images[image] = json.loads(read_command(['docker', 'image', 'inspect', '--format', IMAGE_FORMAT, image]))
return rows
def remaining_resources():
return {resource: read_command(['docker', resource, 'ls', '-aq' if resource == 'container' else '-q', '--filter',
f'label=com.docker.compose.project={PROJECT}']).splitlines()
for resource in ('container', 'network', 'volume')}
def extract_metrics(output, upload_file):
rows = [json.loads(line) for line in output.splitlines() if line.startswith('{') and line.endswith('}')]
pytest = re.search(r'(\d+) passed(?:, [^\n]*?)? in ([\d.]+)s', output)
if pytest is None:
raise RuntimeError('В выводе нет успешного итогового результата pytest')
return {
'pytest': {'passed': int(pytest[1]), 'elapsed_seconds': float(pytest[2])},
'uploads': json.loads(upload_file.read_text()),
'public_failover': [row for row in rows if 'observed_instances_before' in row],
'rolling_update': [row for row in rows if row.get('scenario') == 'rolling update'],
'replica_failures': [row for row in rows if 'first_recovery_seconds' in row and 'instance' in row],
}
def main():
if any(remaining_resources().values()):
raise RuntimeError('Тестовый проект уже существует; сначала завершите его отдельно')
started = timestamp()
manifest = source_manifest()
secrets = [path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file()]
secrets = [value for value in secrets if value]
snapshots, images, commands, combined, nginx_output = {}, {}, [], [], ''
info = json.loads(read_command(['docker', 'info', '--format', '{{json .}}']))
config = json.loads(read_command([*COMPOSE, 'config', '--format', 'json']))
services = {name: {
'image': service.get('image'),
'limits': service.get('deploy', {}).get('resources', {}).get('limits', {}),
'settings': {key: value for key, value in service.get('environment', {}).items() if key in NONSECRET_SETTINGS},
'ports': service.get('ports', []), 'networks': list(service.get('networks', {})),
'secret_names': [entry['source'] for entry in service.get('secrets', [])],
'tmpfs': service.get('tmpfs', []),
} for name, service in config['services'].items()}
with TemporaryDirectory(prefix='lab4-evidence-') as folder:
staging = Path(folder)
upload_file = staging / 'uploads.json'
def execute(command):
begin = timestamp()
result = invoke(command)
output = sanitize(result.stdout, secrets, staging)
display = sanitize(shlex.join(command), secrets, staging)
commands.append({'command': display, 'started_at_utc': begin,
'finished_at_utc': timestamp(), 'exit_code': result.returncode})
print(f'{"PASS" if result.returncode == 0 else "FAIL"}: {display}', flush=True)
if result.returncode:
print(output[-5000:], flush=True)
raise RuntimeError('Прогон остановлен; неуспешные evidence не опубликованы')
return f'$ {display}\n{output}\n'
cleanup_output = ''
try:
combined.append(execute(['make', 'check', 'validate', 'test-stand-up', f'TEST_PROJECT={PROJECT}']))
snapshots['started'] = container_snapshot(images)
combined.append(execute(['make', 'test-docker', f'TEST_PROJECT={PROJECT}', 'PYTEST_ARGS=--tb=short']))
combined.append(execute(['make', 'test-stand-smoke', 'test-stand-idempotency', f'TEST_PROJECT={PROJECT}']))
snapshots['before_restart'] = container_snapshot(images)
combined.append(execute(['make', 'test-stand-persistence', f'TEST_PROJECT={PROJECT}']))
snapshots['after_restart'] = container_snapshot(images)
combined.append(execute(['make', 'test-stand-failover', f'TEST_PROJECT={PROJECT}']))
snapshots['after_failover'] = container_snapshot(images)
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_s3', '--base-url', 'http://localhost:28080', '--with-outage', '--with-faults']))
combined.append(execute(['.venv/bin/python', '-m', 'scripts.upload_load', '--base-url', 'http://localhost:28080', '--concurrency', '10', '--output', str(upload_file)]))
combined.append(execute(['.venv/bin/python', '-m', 'scripts.check_grade4', '--base-url', 'http://localhost:28080']))
snapshots['after_update_and_failures'] = container_snapshot(images)
nginx_output = execute(['make', 'test-nginx'])
finally:
cleanup_output = execute([*COMPOSE, 'down', '--volumes', '--remove-orphans'])
remaining = remaining_resources()
if any(remaining.values()):
raise RuntimeError('После очистки остались ресурсы тестового проекта')
if source_manifest() != manifest:
raise RuntimeError('Во время прогона изменились проверяемые исходники')
output = ''.join(combined)
metrics = extract_metrics(output, upload_file)
run = {
'started_at_utc': started, 'finished_at_utc': timestamp(),
'started_at_moscow': datetime.fromisoformat(started).astimezone(ZoneInfo('Europe/Moscow')).isoformat(),
'base_commit': read_command(['git', 'rev-parse', 'HEAD']),
'worktree_dirty': bool(read_command(['git', 'status', '--porcelain'])),
'source_manifest_sha256': hashlib.sha256(manifest.encode()).hexdigest(),
'project': PROJECT, 'runtime': {'python': platform.python_version(),
'packages': {name: version(name) for name in ('fastapi', 'SQLAlchemy', 'pydantic', 'minio', 'Pillow', 'pytest')},
'compose': read_command(['docker', 'compose', 'version'])},
'docker_host': {key: info[key] for key in ('ServerVersion', 'OperatingSystem', 'Architecture', 'NCPU', 'MemTotal', 'CgroupVersion')},
'services': services, 'container_snapshots': snapshots, 'images': images,
'commands': commands, 'remaining_test_resources': remaining,
'minio_source_release': 'RELEASE.2025-10-15T17-29-55Z',
'mc_source_release': 'RELEASE.2025-08-13T08-35-41Z',
}
artifacts = {
'test-output.txt': output, 'nginx-output.txt': nginx_output,
'cleanup.txt': cleanup_output, 'source-sha256.txt': manifest,
'metrics.json': json.dumps(metrics, ensure_ascii=False, indent=2) + '\n',
'run.json': json.dumps(run, ensure_ascii=False, indent=2) + '\n',
}
for name, content in artifacts.items():
if any(secret in content for secret in secrets):
raise RuntimeError('Обнаружено значение секрета; evidence не опубликованы')
EVIDENCE.mkdir(exist_ok=True)
for name, content in artifacts.items():
(EVIDENCE / name).write_text(content)
print('Evidence опубликованы; тестовые ресурсы удалены.', flush=True)
if __name__ == '__main__':
main()
-47
View File
@@ -1,17 +1,10 @@
"""Общие операции с репликами текущего Compose-проекта."""
from contextlib import contextmanager
from dataclasses import dataclass
import fcntl
import json
import os
from pathlib import Path
import ssl
import subprocess
import tempfile
import time
from urllib.error import HTTPError, URLError
from urllib.request import urlopen
ROOT = Path(__file__).resolve().parents[1]
@@ -65,43 +58,3 @@ class ComposeProject:
Replica(row["id"], row["image"], configured or row["hostname"], row["running"], row["health"])
for row in map(json.loads, rows)
]
def wait_healthy(self, ids: set[str], timeout: float = 40) -> None:
"""Ожидает готовность конкретных реплик."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
current = {replica.id: replica for replica in self.replicas()}
if not ids <= current.keys():
raise RuntimeError("Контейнер API исчез во время развёртывания")
if all(current[id].running and current[id].health == "healthy" for id in ids):
return
if any(not current[id].running for id in ids):
raise RuntimeError("Контейнер API завершился до готовности")
time.sleep(0.25)
raise TimeoutError("API не стал healthy за отведённое время")
def wait_routed(self, instance_id: str, base_url: str, ca_file=None) -> None:
"""Ожидает ответа реплики через балансировщик."""
context = ssl.create_default_context(cafile=ca_file)
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
try:
with urlopen(base_url.rstrip("/") + "/health/live", timeout=3, context=context) as response:
if response.status == 200 and response.headers.get("X-Instance-Id") == instance_id:
return
except (HTTPError, URLError, TimeoutError, OSError):
pass
time.sleep(0.1)
raise TimeoutError("Реплика не появилась в ответах балансировщика")
@contextmanager
def deployment_lock(project: ComposeProject):
"""Исключает параллельные изменения одного стенда."""
path = Path(tempfile.gettempdir()) / f"{project.name}-deployment.lock"
with path.open("a") as lock:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise RuntimeError("Другой процесс уже изменяет этот Compose-проект") from None
yield
-28
View File
@@ -1,28 +0,0 @@
#!/bin/sh
set -eu
BASE_URL="${BASE_URL:-http://localhost:8080}"
TTL="${TTL:?укажите TTL, с которым запущен API (SESSION_TTL_SECONDS)}"
API="$BASE_URL/api/v1"
USERNAME="expiry_$(date +%s)"
PASSWORD="expiry-password-123"
code() {
curl -s -o /dev/null -w '%{http_code}' "$@"
}
curl -sf -o /dev/null -X POST "$API/users" -H 'Content-Type: application/json' \
-d "{\"username\":\"$USERNAME\",\"display_name\":\"Expiry\",\"password\":\"$PASSWORD\"}"
SESSION=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD")
TOKEN=$(printf '%s' "$SESSION" | python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])')
EXPIRES_AT=$(printf '%s' "$SESSION" | python3 -c 'import json, sys; print(json.load(sys.stdin)["expires_at"])')
AUTH="Authorization: Bearer $TOKEN"
echo "SESSION_TTL_SECONDS=$TTL, login at $(date -u +%H:%M:%S)Z, expires_at=$EXPIRES_AT"
echo "GET /users/me right after login -> $(code "$API/users/me" -H "$AUTH")"
sleep $((TTL + 1))
printf "%-35s -> %s\n" "GET /users/me after $((TTL + 1)) s" "$(code "$API/users/me" -H "$AUTH")"
echo "DELETE /auth/sessions/current after -> $(code -X DELETE "$API/auth/sessions/current" -H "$AUTH")"
NEW_TOKEN=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD" \
| python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])')
echo "GET /users/me with new login token -> $(code "$API/users/me" -H "Authorization: Bearer $NEW_TOKEN")"
-132
View File
@@ -1,132 +0,0 @@
"""EXPLAIN (ANALYZE, BUFFERS) запроса истории на синтетических данных.
Данные создаются в одной транзакции и откатываются в конце: база не меняется.
"""
import asyncio
from uuid import uuid4
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncConnection
from app.core.config import Settings
from app.core.database import create_db_engine
TARGET_MESSAGES = 10_000
NOISE_MESSAGES = 40_000
PAGE = 51
HISTORY_FIRST_PAGE = """
SELECT * FROM messages
WHERE chat_id = :chat_id
ORDER BY created_at, id
LIMIT :page
"""
HISTORY_AFTER_CURSOR = """
SELECT * FROM messages
WHERE chat_id = :chat_id AND (created_at, id) > (:after_created_at, :after_id)
ORDER BY created_at, id
LIMIT :page
"""
async def seed(connection: AsyncConnection) -> dict[str, object]:
user_id, target_id, noise_id = uuid4(), uuid4(), uuid4()
await connection.execute(
text(
"INSERT INTO users (id, username, display_name, password_hash) "
"VALUES (:id, :username, 'Explain', 'x')"
),
{"id": user_id, "username": "explain_" + uuid4().hex[:8]},
)
await connection.execute(
text("INSERT INTO chats (id, title) VALUES (:a, 'target'), (:b, 'noise')"),
{"a": target_id, "b": noise_id},
)
await connection.execute(
text("INSERT INTO chat_members (chat_id, user_id) VALUES (:a, :u), (:b, :u)"),
{"a": target_id, "b": noise_id, "u": user_id},
)
for chat_id, count in ((target_id, TARGET_MESSAGES), (noise_id, NOISE_MESSAGES)):
await connection.execute(
text(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) "
"SELECT gen_random_uuid(), :chat_id, :user_id, 'message ' || g, "
"now() - make_interval(secs => :count - g) "
"FROM generate_series(1, :count) AS g"
),
{"chat_id": chat_id, "user_id": user_id, "count": count},
)
await connection.execute(text("ANALYZE messages"))
middle = (
await connection.execute(
text(
"SELECT created_at, id FROM messages WHERE chat_id = :c "
"ORDER BY created_at, id OFFSET :o LIMIT 1"
),
{"c": target_id, "o": TARGET_MESSAGES // 2},
)
).one()
return {
"chat_id": target_id,
"after_created_at": middle.created_at,
"after_id": middle.id,
"page": PAGE,
}
async def explain(
connection: AsyncConnection, title: str, query: str, params: dict[str, object]
) -> None:
rows = await connection.execute(
text(f"EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING ON) {query}"), params
)
print(f"== {title}")
print(query.strip())
print()
for row in rows:
print(row[0])
print()
async def main() -> None:
engine = create_db_engine(Settings()) # type: ignore[call-arg]
async with engine.connect() as connection:
transaction = await connection.begin()
params = await seed(connection)
total = (
await connection.execute(text("SELECT count(*) FROM messages"))
).scalar_one()
print(
f"messages in table: {total}; target chat: {TARGET_MESSAGES}; noise chat: {NOISE_MESSAGES}"
)
print()
first_page = {"chat_id": params["chat_id"], "page": PAGE}
await explain(
connection, "first page (limit+1 = 51)", HISTORY_FIRST_PAGE, first_page
)
await explain(
connection,
"page after cursor in the middle of history",
HISTORY_AFTER_CURSOR,
params,
)
await connection.execute(text("SET LOCAL enable_indexscan = off"))
await connection.execute(text("SET LOCAL enable_bitmapscan = off"))
await explain(
connection,
"same page with index scans disabled (for comparison)",
HISTORY_AFTER_CURSOR,
params,
)
await transaction.rollback()
await engine.dispose()
if __name__ == "__main__":
asyncio.run(main())
-148
View File
@@ -1,148 +0,0 @@
"""Сценарий поочерёдного отказа всех API-реплик под записью."""
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import json
import os
import re
import subprocess
from threading import Event, Lock
import time
from urllib.parse import quote
from uuid import uuid4
from scripts.deployment import ComposeProject, ROOT, deployment_lock
from tests.client import Client, Failure, arguments, check, run
def failure_kind(error: Failure) -> str:
"""Различает HTTP-отказы, ошибки соединения и проверки контракта."""
match = re.search(r"received (\d{3})$", str(error))
if match:
return "http_" + match.group(1)
return "connection" if "connection/TLS failure" in str(error) else "contract"
def snapshot(project: ComposeProject, phase: str, target: str) -> list:
"""Печатает состояние API без окружения и секретов."""
replicas = project.replicas()
print(json.dumps({
"scenario": "replica_failure", "phase": phase, "target": target,
"replicas": [{"instance": item.instance_id, "running": item.running, "health": item.health} for item in replicas],
}), flush=True)
return replicas
def history(client: Client, actor: dict, chat: dict) -> list[dict]:
"""Читает всю историю без повторов курсора."""
path = f"/api/v1/chats/{chat['id']}/messages?limit=100"
items = []
cursors = set()
for _ in range(1000):
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
items.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return items
check(cursor not in cursors, "cursor cycle")
cursors.add(cursor)
path = f"/api/v1/chats/{chat['id']}/messages?limit=100&cursor={quote(cursor, safe='')}"
raise Failure("history exceeded safety bound")
def exercise(project: ComposeProject, args) -> None:
"""Останавливает каждую реплику и восстанавливает её перед следующей."""
replicas = project.replicas()
check(len(replicas) >= 2, "at least two API replicas are required")
project.wait_healthy({item.id for item in replicas})
client = Client(args.base_url, args.ca_file, timeout=2)
for item in replicas:
project.wait_routed(item.instance_id, client.base, args.ca_file)
actor, _, _, chat = client.bootstrap()
confirmed = set()
for target in replicas:
snapshot(project, "before", target.instance_id)
stop = Event()
lock = Lock()
successes = []
errors = []
def worker():
local = Client(client.base, args.ca_file, timeout=2)
key = uuid4().hex
while not stop.is_set():
started = time.monotonic()
try:
result = local.message(actor, chat, "failover-" + key, key=key)
with lock:
successes.append((started, time.monotonic(), result["json"]["id"]))
key = uuid4().hex
except Failure as error:
with lock:
errors.append(failure_kind(error))
time.sleep(0.1)
started = time.monotonic()
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
jobs = [pool.submit(worker) for _ in range(args.concurrency)]
try:
time.sleep(2)
stop_started = time.monotonic()
subprocess.run(
[str(ROOT / "scripts/stop-one.sh")], cwd=ROOT,
env={**os.environ, "TARGET_INSTANCE_ID": target.instance_id},
check=True, timeout=10, capture_output=True,
)
stop_finished = time.monotonic()
stopped = snapshot(project, "stopped", target.instance_id)
check(any(item.id == target.id and not item.running for item in stopped), "target API was not stopped")
check(any(item.id != target.id and item.running and item.health == "healthy" for item in stopped), "no healthy surviving API")
time.sleep(args.duration)
finally:
stop.set()
try:
for job in jobs:
job.result(timeout=5)
finally:
project.docker("start", target.id)
project.wait_healthy({item.id for item in replicas})
project.wait_routed(target.instance_id, client.base, args.ca_file)
snapshot(project, "restored", target.instance_id)
elapsed = max(end for _, end, _ in successes) - started if successes else 0
recovered = [end for begin, end, _ in successes if begin >= stop_finished]
check(bool(recovered), "no confirmed writes after failure")
recovery = min(recovered) - stop_started
check(recovery <= 10, "writes did not recover within 10 seconds")
confirmed.update(id for _, _, id in successes)
items = history(client, actor, chat)
check(confirmed <= {item["id"] for item in items}, "confirmed messages were lost")
texts = [item["text"] for item in items]
check(len(texts) == len(set(texts)), "uncertain retry created duplicates")
print(json.dumps({
"instance": target.instance_id,
"successful_writes": len(successes),
"failed_attempts": len(errors),
"errors_by_kind": dict(Counter(errors)),
"elapsed_seconds": round(elapsed, 3),
"successful_rps": round(len(successes) / elapsed, 3) if elapsed else 0,
"first_recovery_seconds": round(recovery, 3),
}), flush=True)
def main() -> None:
"""Готовит параметры эксперимента отказа реплик."""
def configure(parser):
parser.add_argument("--duration", type=float, default=15)
parser.add_argument("--concurrency", type=int, default=2)
args = arguments("Fail each API replica under write load", configure)
check(1 <= args.duration <= 120 and 1 <= args.concurrency <= 8, "invalid workload")
project = ComposeProject()
with deployment_lock(project):
exercise(project, args)
if __name__ == "__main__":
run(main)
-64
View File
@@ -1,64 +0,0 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
BASE_URL="${BASE_URL:-http://localhost:8080}"
API="$BASE_URL/api/v1"
USERNAME="demo_$(date +%s)"
PASSWORD="demo-password-123"
compose() {
docker compose --env-file .env "$@"
}
code() {
curl -s -o /dev/null -w '%{http_code} (%{time_total}s)' "$@"
}
wait_healthy() {
attempts=0
until [ "$(compose ps --format '{{.Health}}' "$1")" = "healthy" ]; do
attempts=$((attempts + 1))
[ "$attempts" -lt 60 ] || { echo "$1 не стал healthy" >&2; exit 1; }
sleep 1
done
}
curl -sf -o /dev/null -X POST "$API/users" -H 'Content-Type: application/json' \
-d "{\"username\":\"$USERNAME\",\"display_name\":\"Demo\",\"password\":\"$PASSWORD\"}"
TOKEN=$(curl -sf -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD" \
| python3 -c 'import json, sys; print(json.load(sys.stdin)["token"])')
AUTH="Authorization: Bearer $TOKEN"
echo "== baseline"
echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")"
echo
echo "== redis stopped"
compose stop redis >/dev/null 2>&1
echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")"
echo "POST /auth/sessions (Basic) -> $(code -X POST "$API/auth/sessions" -u "$USERNAME:$PASSWORD")"
echo "GET /users/me (no token) -> $(code "$API/users/me")"
echo "GET /health/ready -> $(code "$BASE_URL/health/ready")"
echo "GET /health/live -> $(code "$BASE_URL/health/live")"
compose start redis >/dev/null 2>&1
wait_healthy redis
echo "-- redis started again (AOF restored)"
echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")"
echo
echo "== postgres stopped"
compose stop postgres >/dev/null 2>&1
echo "GET /users/me (Bearer) -> $(code "$API/users/me" -H "$AUTH")"
echo "GET /api/v1/chats (Bearer) -> $(code "$API/chats" -H "$AUTH")"
echo "GET /health/ready -> $(code "$BASE_URL/health/ready")"
echo "GET /health/live -> $(code "$BASE_URL/health/live")"
compose start postgres >/dev/null 2>&1
wait_healthy postgres
echo "-- postgres started again"
echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")"
echo
echo "== logout"
echo "DELETE /auth/sessions/current -> $(code -X DELETE "$API/auth/sessions/current" -H "$AUTH")"
echo "GET /users/me (same token) -> $(code "$API/users/me" -H "$AUTH")"
+1 -1
View File
@@ -3,7 +3,7 @@ set -eu
# TODO: copy to restart.sh and implement using YOUR compose/stack services.
# Restart API and ALL used state stores (DB, sessions, S3, broker as applicable).
# Preserve persistent volumes and schema/data; never down -v / flush / re-seed.
# Allow operator unseal if required in lab7; do not print credentials.
# Do not print credentials.
# Exit 0 only after the intended action; test waits for API readiness separately.
printf '%s
' 'TODO: implement restart.sh for your infrastructure' >&2
-92
View File
@@ -1,92 +0,0 @@
"""Поочерёдно заменяет API с одной дополнительной репликой."""
import argparse
import sys
from scripts.deployment import ComposeProject, deployment_lock
def update(project: ComposeProject, base_url: str, ca_file=None, revision: str | None = None) -> None:
"""Собирает образ, применяет миграции и заменяет реплики."""
original = project.replicas()
if len(original) < 2 or any(not item.running or item.health != "healthy" for item in original):
raise RuntimeError("До обновления нужны минимум две здоровые API-реплики")
old_images = {item.image for item in original}
if len(old_images) != 1:
raise RuntimeError("Сначала восстановите единый образ работающих реплик")
old_image = old_images.pop()
profile = project.docker(
"image", "inspect", old_image,
"--format", '{{index .Config.Labels "io.mtusi.messenger.secrets"}}',
)
if profile != "file-v1":
raise RuntimeError("Сначала выполните первоначальный переход на файловые секреты через make up")
count = len(original)
print("Сборка нового образа API", flush=True)
build_args = ("--build-arg", f"API_BUILD_REVISION={revision}") if revision else ()
project.run("build", *build_args, "api", timeout=300)
tag = project.config["services"]["api"]["image"]
new_image = project.docker("image", "inspect", tag, "--format", "{{.Id}}")
print("Однократное применение совместимых миграций", flush=True)
project.run("run", "--rm", "--no-deps", "--pull", "never", "migrate")
if old_image == new_image:
print("Образ не изменился; замена контейнеров не требуется")
return
try:
for old in original:
project.run(
"up", "-d", "--no-deps", "--no-build", "--pull", "never",
"--no-recreate", "--scale", f"api={count + 1}", "api",
)
current = project.replicas()
additions = [item for item in current if item.id not in {replica.id for replica in original} and item.image == new_image]
if not additions:
raise RuntimeError("Новая реплика не создана")
project.wait_healthy({item.id for item in additions})
for added in additions:
project.wait_routed(added.instance_id, base_url, ca_file)
survivors = {item.id for item in project.replicas() if item.id != old.id}
project.wait_healthy(survivors)
project.docker("stop", old.id)
project.docker("rm", old.id)
print(f"Заменена реплика {old.instance_id}", flush=True)
final = project.replicas()
if len(final) != count or any(item.image != new_image for item in final):
raise RuntimeError("Состав реплик после обновления не соответствует плану")
project.wait_healthy({item.id for item in final})
except Exception:
print("Обновление прервано; восстанавливается прежний образ без отката БД", file=sys.stderr)
project.run(
"up", "-d", "--no-deps", "--no-build", "--pull", "never",
"--force-recreate", "--scale", f"api={count}", "api",
env={"API_IMAGE": old_image},
)
project.wait_healthy({item.id for item in project.replicas()})
raise
dangling = project.docker("image", "ls", "--quiet", "--no-trunc", "--filter", "dangling=true").splitlines()
if old_image in dangling:
references = project.docker("ps", "--all", "--quiet", "--filter", f"ancestor={old_image}")
if not references:
try:
project.docker("image", "rm", old_image)
except RuntimeError:
print("Прежний образ не удалён: состояние Docker изменилось", file=sys.stderr)
print(f"Готово: {count} здоровых реплик нового образа")
def main() -> None:
"""Запускает обновление с блокировкой проекта."""
parser = argparse.ArgumentParser(description="Rolling update of API replicas")
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--ca-file", default=None)
parser.add_argument("--revision", default=None)
args = parser.parse_args()
project = ComposeProject()
with deployment_lock(project):
update(project, args.base_url or "http://localhost:8080", args.ca_file, args.revision)
if __name__ == "__main__":
main()
-108
View File
@@ -1,108 +0,0 @@
"""Исполняется внутри API-контейнера: реальные S3/БД и управляемые отказы."""
import asyncio
import base64
from datetime import datetime, timedelta, timezone
import json
import sys
from uuid import UUID
from minio.error import S3Error
from sqlalchemy import select
from sqlalchemy.exc import OperationalError
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.domain import attachments as domain
from app.errors.exceptions import unavailable
from app.models.attachment import UploadIntent
from app.schemas.user import User
from app.storage import attachments as repository
from app.storage.s3 import create_s3_storage
async def object_exists(storage, key):
exists = True
try:
await storage.stat_object(key)
except S3Error as error:
if error.code not in {'NoSuchKey', 'NoSuchObject'}:
raise
exists = False
return exists
async def main():
payload = json.load(sys.stdin)
settings = Settings()
engine = create_db_engine(settings)
factory = create_session_factory(engine)
storage = create_s3_storage(settings.s3)
user = User.model_validate(payload['user'])
chat_id = UUID(payload['chat_id'])
original = base64.b64decode(payload['image'])
keys = []
create_record = repository.create_attachment
put_file, delete_object = storage.put_file, storage.delete_object
async def capture_put(key, path, **metadata):
keys.append(key)
await put_file(key, path, **metadata)
async def fail_metadata(*args, **kwargs):
raise OperationalError('injected metadata failure after real S3 PUT', {}, None)
async def fail_delete(*args):
raise unavailable('injected compensation failure')
async def chunks():
yield original
async def attempt():
raised = False
async with factory() as session:
try:
await domain.upload_attachment(session, storage, user, chat_id,
filename='fault.png', content_type='image/png', chunks=chunks())
except OperationalError:
raised = True
assert raised, 'injected database failure was not raised'
storage.put_file = capture_put
repository.create_attachment = fail_metadata
try:
await attempt()
assert not await object_exists(storage, keys[-1]), 'compensation left original'
async with factory() as session:
intent = await session.scalar(select(UploadIntent).where(UploadIntent.object_key == keys[-1]))
assert intent is None, 'compensated upload left intent'
storage.delete_object = fail_delete
await attempt()
key = keys[-1]
assert await object_exists(storage, key), 'fault did not leave real S3 object'
storage.delete_object = delete_object
async with factory() as session:
result = await domain.reconcile_attachments(session, storage, dry_run=False)
assert result.deleted == 0, 'fresh object deleted before grace period'
intent = await session.scalar(select(UploadIntent).where(UploadIntent.object_key == key))
assert intent is not None, 'durable intent missing'
intent.created_at = datetime.now(timezone.utc) - timedelta(days=2)
await session.commit()
dry_run = await domain.reconcile_attachments(session, storage)
assert dry_run.candidates == 1 and dry_run.deleted == 0
assert await object_exists(storage, key), 'dry-run removed original'
applied = await domain.reconcile_attachments(session, storage, dry_run=False)
assert applied.deleted == 1 and not await object_exists(storage, key)
repeated = await domain.reconcile_attachments(session, storage, dry_run=False)
assert repeated.examined == repeated.deleted == 0
print(json.dumps({'compensation': True, 'durable_intent': True, 'grace_period': True,
'dry_run': True, 'cleanup': True, 'idempotent_cleanup': True}))
finally:
repository.create_attachment = create_record
storage.put_file, storage.delete_object = put_file, delete_object
await storage.aclose()
await engine.dispose()
if __name__ == '__main__':
asyncio.run(main())
-113
View File
@@ -1,113 +0,0 @@
"""Повторяемые параллельные upload и память cgroup API, без записи токенов."""
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import hashlib
from io import BytesIO
import json
from pathlib import Path
import random
import subprocess
from threading import Barrier
import time
from PIL import Image
from scripts.deployment import ComposeProject
from tests.client import Client, Failure, arguments, check, run
MONITOR = r'''
import json, pathlib, sys, threading
root = pathlib.Path('/sys/fs/cgroup')
def current():
return int((root / 'memory.current').read_text())
def peak():
path = root / 'memory.peak'
return int(path.read_text()) if path.exists() else None
stop = threading.Event()
threading.Thread(target=lambda: (sys.stdin.read(), stop.set()), daemon=True).start()
initial, before = current(), peak()
sampled = initial
print('ready', flush=True)
while not stop.wait(0.02):
sampled = max(sampled, current())
print(json.dumps({'initial_bytes': initial, 'sampled_peak_bytes': sampled,
'cgroup_peak_before_bytes': before, 'cgroup_peak_after_bytes': peak(), 'interval_ms': 20}), flush=True)
'''
def main():
def configure(parser):
parser.add_argument('--concurrency', type=int, default=10)
parser.add_argument('--rounds', type=int, default=3)
parser.add_argument('--output', type=Path)
args = arguments('Concurrent attachment uploads and API memory', configure)
check(10 <= args.concurrency <= 32 and 1 <= args.rounds <= 10, 'use concurrency 10..32 and rounds 1..10')
project = ComposeProject()
client = Client(args.base_url, args.ca_file, timeout=65)
alice, _, _, chat = client.bootstrap()
random_bytes = random.Random(42).randbytes(1500 * 1500 * 3)
image = Image.frombytes('RGB', (1500, 1500), random_bytes)
buffer = BytesIO()
image.save(buffer, format='PNG')
data = buffer.getvalue()
check(len(data) <= 10485760, 'fixture exceeds upload limit')
monitors = []
measurements = {}
rows = []
started = time.monotonic()
try:
for replica in project.replicas():
process = subprocess.Popen(['docker', 'exec', '-i', replica.id, 'python', '-c', MONITOR],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
monitors.append((replica.instance_id, process))
check(process.stdout.readline().strip() == 'ready', 'memory monitor failed')
check(bool(monitors), 'API not running')
for _ in range(args.rounds):
barrier = Barrier(args.concurrency)
def upload(index):
local = Client(args.base_url, args.ca_file, timeout=65)
barrier.wait(timeout=15)
request_started = time.monotonic()
try:
return local.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=load.png",
expected=[201, 400, 401, 404, 409, 413, 415, 429, 500, 502, 503, 504], actor=alice, raw=data,
headers={'Content-Type': 'image/png'}, schema='Attachment')
except Failure:
# При раннем отказе сервер может закрыть ещё отправляемое тело.
return {'status': 'failed_request', 'elapsed': time.monotonic() - request_started}
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
rows.extend(pool.map(upload, range(args.concurrency)))
finally:
elapsed = time.monotonic() - started
for alias, process in monitors:
output, _ = process.communicate(input='', timeout=10)
check(process.returncode == 0, 'memory monitor failed')
measurements[alias] = json.loads(output)
succeeded = [row for row in rows if row['status'] == 201]
check(bool(succeeded), 'all uploads were rejected')
digest = hashlib.sha256(data).hexdigest()
for row in succeeded:
check(row['json']['sha256'] == digest, 'upload checksum differs')
original = client.call('GET', '/api/v1/attachments/' + row['json']['id'] + '/content', actor=alice)['bytes']
check(hashlib.sha256(original).hexdigest() == digest, 'download checksum differs')
latencies = sorted(row['elapsed'] for row in rows)
result = {
'concurrency': args.concurrency, 'rounds': args.rounds, 'attempts': len(rows),
'fixture': {'format': 'PNG', 'width': 1500, 'height': 1500, 'size_bytes': len(data), 'sha256': digest},
'statuses': dict(Counter(str(row['status']) for row in rows)),
'elapsed_seconds': round(elapsed, 3), 'successful_uploads': len(succeeded),
'p95_seconds': round(latencies[min(len(latencies) - 1, int(len(latencies) * .95))], 3),
'memory': measurements, 'api_memory_limit': project.config['services']['api']['deploy']['resources']['limits']['memory'],
'notes': '20 ms samples include monitor and decoder processes; cgroup peak is cumulative since container creation; 429 and failed requests counted separately; no upload retries',
}
encoded = json.dumps(result, ensure_ascii=False, indent=2)
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(encoded + '\n')
print(encoded)
check(all(row['status'] in {201, 429} for row in rows), 'unexpected HTTP/transport errors during upload experiment; inspect status counts')
if __name__ == '__main__':
run(main)
+2 -2
View File
@@ -11,9 +11,9 @@ validate(spec)
from referencing import Registry, Resource
# The base URI resolves local #/components references inside examples.
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource))
registry = Registry().with_resource('urn:messenger:openapi', Resource.from_contents(resource))
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name}
schema = {'$ref': 'urn:messenger:openapi#/components/schemas/' + name}
Draft202012Validator(schema, registry=registry).validate(example)
for path in (root/'contracts').glob('*.schema.json'):
schema = json.loads(path.read_text())