Files
mtusi2026lab4/scripts/lab4_evidence.py
T
2026-11-06 13:57:29 +03:00

390 lines
19 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Повторяемая проверка lab4: реальные отказы, приватный MinIO и 10 uploads с RAM."""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path
import platform
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
from urllib.parse import quote
from evidence_utils import MemorySampler, exec_json, png_fixture, record_command, save_json
from replicas import COMPOSE, ROOT, SERVICES, compose, inventory, wait_balanced
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check
from resilience import history
def upload(c, actor, chat, content, filename="sample.png"):
response = c.call(
"POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=" + quote(filename),
201, actor=actor, raw=content, headers={"Content-Type": "image/png"}, schema="Attachment",
)
att = response["json"]
check(att["status"] == "ready" and att["error_code"] is None, "upload must finish ready")
check(att["variants"] == [], "lab4 must return the original without variants")
check(att["size_bytes"] == len(content), "uploaded size mismatch")
check(att["sha256"] == hashlib.sha256(content).hexdigest(), "uploaded checksum mismatch")
check(att["chat_id"] == chat["id"] and att["owner_id"] == actor["id"], "upload ownership mismatch")
return response
def download(c, actor, att, content):
metadata = c.call("GET", "/api/v1/attachments/" + att["id"], actor=actor, schema="Attachment")
check(metadata["json"] == att, "attachment metadata changed")
response = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=actor)
check(response["bytes"] == content, "download differs byte-for-byte from original")
check(response["headers"].get_content_type() == "image/png", "download Content-Type mismatch")
return response
def attachment_contract(c, alice, bob, eve, chat, content):
response = upload(c, alice, chat, content)
att = response["json"]
attached = c.message(alice, chat, "", attachments=[att["id"]])["json"]
check(attached["attachment_ids"] == [att["id"]], "attachment-only message lost attachment")
check(attached in history(c, bob, chat), "second member cannot read attached message")
download(c, bob, att, content)
for suffix in ("", "/content"):
c.call("GET", "/api/v1/attachments/" + att["id"] + suffix, 404, actor=eve)
return {
"attachment_id": att["id"], "upload_instance": response["headers"]["X-Instance-Id"],
"sha256": att["sha256"], "size_bytes": len(content), "download_equal": True,
"attachment_only_message_visible": True, "outsider_metadata_status": 404,
"outsider_content_status": 404,
}, att, response["headers"]["X-Instance-Id"]
def uploader_failover(c, bob, att, content, target):
"""Остановить именно процесс, вернувший 201 на binary upload."""
before = inventory()
matches = [row for row in before if row.get("instance_id") == target]
check(len(matches) == 1, "upload must identify one current API process")
service = matches[0]["service"]
started = time.monotonic()
try:
env = dict(os.environ, TARGET_INSTANCE_ID=target, PYTHONUTF8="1")
result = subprocess.run(
[sys.executable, str(ROOT / "scripts/stop_one.py")], cwd=ROOT, env=env,
capture_output=True, timeout=15,
)
check(result.returncode == 0, "could not stop uploading API")
stopped = next(row for row in inventory() if row["service"] == service)
check(stopped["state"] == "exited" and stopped["pid"] == 0, "upload process still alive")
deadline = time.monotonic() + 10
while True:
try:
response = download(c, bob, att, content)
break
except Failure:
check(time.monotonic() < deadline, "download failed to recover after uploader stop")
time.sleep(0.2)
survivor = response["headers"]["X-Instance-Id"]
check(survivor != target, "download came from the stopped uploader")
return {
"upload_instance": target, "stopped_service": service,
"stopped_state": stopped["state"], "stopped_pid": stopped["pid"],
"download_instance": survivor, "sha256": hashlib.sha256(response["bytes"]).hexdigest(),
"download_equal": True, "seconds_from_stop_start": round(time.monotonic() - started, 3),
}
finally:
compose("up", "-d", "--no-deps", service)
wait_balanced(timeout=60)
def persistence(c, bob, chat, att, content, out):
before = inventory()
record_command(out, "04_restart_command", [sys.executable, "scripts/restart.py"], timeout=120)
after = inventory()
check({r["instance_id"] for r in before}.isdisjoint(r["instance_id"] for r in after),
"restart must replace both API processes")
response = download(c, bob, att, content)
check(any(att["id"] in message["attachment_ids"] for message in history(c, bob, chat)),
"attached message lost after restart")
return {
"before": before, "after": after, "existing_session_works": True,
"message_preserved": True, "metadata_equal": True, "download_equal": True,
"sha256": hashlib.sha256(response["bytes"]).hexdigest(),
"restart_scope": "all default Compose services, force recreated; named volumes retained",
}
DIRECT_PROBES = r'''
import base64, json, urllib.error, urllib.request, sys
p = json.load(sys.stdin)
result = {}
for name, method, path, body in (
('live', 'GET', '/health/live', None),
('ready', 'GET', '/health/ready', None),
('upload', 'POST', '/api/v1/chats/' + p['chat_id'] + '/attachments?filename=outage.png', base64.b64decode(p['png'])),
('content', 'GET', '/api/v1/attachments/' + p['attachment_id'] + '/content', None),
):
headers = {'Authorization': 'Bearer ' + p['token'], 'Content-Type': 'image/png'}
request = urllib.request.Request('http://127.0.0.1:8080' + path, method=method, headers=headers, data=body)
try:
response = urllib.request.urlopen(request, timeout=30)
except urllib.error.HTTPError as error:
response = error
with response:
data = response.read(16 * 1024 * 1024)
result[name] = {'status': response.status, 'instance_id': response.headers.get('X-Instance-Id'),
'request_id_present': bool(response.headers.get('X-Request-Id'))}
if response.status >= 400:
error = json.loads(data)['error']
result[name]['error_code'] = error['code']
result[name]['request_id_matches'] = error['request_id'] == response.headers.get('X-Request-Id')
print(json.dumps(result))
'''
def s3_outage(c, alice, bob, chat, att, content):
results = {}
public = None
try:
compose("stop", "--timeout", "10", "minio")
for service in SERVICES:
results[service] = exec_json(service, DIRECT_PROBES, {
"token": alice["token"], "chat_id": chat["id"], "attachment_id": att["id"],
"png": base64.b64encode(content).decode("ascii"),
}, timeout=120)
for name, status in (("live", 200), ("ready", 503), ("upload", 503), ("content", 503)):
actual = results[service][name]
check(actual["status"] == status, service + "/" + name + ": " + str(actual))
check(actual["request_id_present"] and actual["instance_id"], "missing API trace headers")
if status == 503:
check(actual["error_code"] == "unavailable" and actual["request_id_matches"],
"S3 outage must return a structured 503")
try:
reply = urllib.request.urlopen(c.base + "/health/ready", timeout=10)
except urllib.error.HTTPError as error:
reply = error
with reply:
payload = json.loads(reply.read())
public = {"status": reply.status, "error_code": payload["error"]["code"],
"request_id_matches": payload["error"]["request_id"] == reply.headers.get("X-Request-Id")}
check(public["status"] == 503 and public["request_id_matches"], "public readiness must return structured 503")
finally:
compose("start", "minio")
wait_balanced(timeout=60)
download(c, bob, att, content)
return {"direct_api_probes": results, "public_readiness": public, "restored_download_equal": True,
"method": "probe each API directly inside its container because LB excludes unready backends"}
PRIVATE_BUCKET = r'''
import asyncio, io, json, os, sys, uuid, urllib.error, urllib.request
import urllib.error
import urllib.request
from urllib.parse import quote, urlsplit
from minio import Minio
from minio.error import S3Error
from sqlalchemy import text
from app.config import Settings
from app.db import create_engine
p = json.load(sys.stdin)
async def key():
engine = create_engine(Settings.from_env())
try:
async with engine.connect() as connection:
return (await connection.execute(text('SELECT object_key FROM attachments WHERE id=:id'),
{'id': uuid.UUID(p['attachment_id'])})).scalar_one()
finally:
await engine.dispose()
object_key = asyncio.run(key())
endpoint = os.environ['S3_ENDPOINT_URL'].rstrip('/')
bucket = os.environ['S3_BUCKET']
url = endpoint + '/' + quote(bucket, safe='') + '/' + quote(object_key, safe='/')
try:
response = urllib.request.urlopen(url, timeout=5)
except urllib.error.HTTPError as error:
response = error
with response:
anonymous_status = response.status
parsed = urlsplit(endpoint)
client = Minio(parsed.netloc, access_key=os.environ['S3_ACCESS_KEY'], secret_key=os.environ['S3_SECRET_KEY'],
secure=parsed.scheme == 'https', region=os.environ.get('S3_REGION', 'us-east-1'))
new_bucket = 'forbidden-' + uuid.uuid4().hex
outside_key = 'forbidden/' + uuid.uuid4().hex
operations = {
'list_foreign_bucket': lambda: list(client.list_objects(new_bucket, recursive=True)),
'make_bucket': lambda: client.make_bucket(new_bucket),
'put_outside_prefix': lambda: client.put_object(bucket, outside_key, io.BytesIO(b'x'), 1),
}
# MinIO deliberately filters ListBuckets to buckets permitted by bucket-level IAM.
# Verify the names rather than incorrectly requiring HTTP 403 for this API.
visible_buckets = [item.name for item in client.list_buckets()]
outcomes = {}
for name, operation in operations.items():
try:
operation()
outcomes[name] = 'UNEXPECTED_SUCCESS'
except S3Error as error:
outcomes[name] = error.code
except Exception as error:
outcomes[name] = type(error).__name__
# Clean synthetic probes if an incorrect policy unexpectedly allowed them.
for action in (
(lambda: client.remove_bucket(new_bucket)) if outcomes['make_bucket'] == 'UNEXPECTED_SUCCESS' else None,
(lambda: client.remove_object(bucket, outside_key)) if outcomes['put_outside_prefix'] == 'UNEXPECTED_SUCCESS' else None,
):
if action:
try: action()
except Exception: pass
print(json.dumps({'anonymous_get_status': anonymous_status, 'restricted_operations': outcomes,
'visible_buckets': visible_buckets, 'visible_buckets_limited': set(visible_buckets) <= {bucket},
'root_credentials_absent_in_api': not any(name in os.environ for name in ('MINIO_ROOT_USER', 'MINIO_ROOT_PASSWORD')),
'server_assigned_original_prefix': object_key.startswith('originals/')}))
'''
def private_bucket(att, out):
# The same bootstrap is executed twice against an already populated bucket.
for number in (1, 2):
record_command(out, "06_bootstrap_repeat_" + str(number),
COMPOSE + ["run", "--rm", "--no-deps", "minio-init"], timeout=120)
result = exec_json("api-a", PRIVATE_BUCKET, {"attachment_id": att["id"]})
check(result["anonymous_get_status"] == 403, "private bucket anonymous GET: " + str(result))
check(all(value == "AccessDenied" for value in result["restricted_operations"].values()),
"API S3 privilege violation: " + str(result))
check(result["visible_buckets_limited"], "API can see an unrelated bucket")
check(result["root_credentials_absent_in_api"], "API must not receive root S3 credentials")
check(result["server_assigned_original_prefix"], "unexpected server object key prefix")
result["bootstrap_repeated_successfully"] = 2
return result
def concurrent_uploads(base, alice, bob, chat, concurrency):
content = png_fixture()
check(len(content) >= 2 * 1024 * 1024, "experiment fixture must be at least 2 MiB")
barrier = threading.Barrier(concurrency, timeout=30)
samplers = []
def send(number):
client = Client(base, timeout=90)
barrier.wait()
started = time.monotonic()
try:
response = upload(client, alice, chat, content, "parallel-%02d.png" % number)
return {"ok": True, "started": started, "finished": time.monotonic(),
"attachment": response["json"], "instance": response["headers"]["X-Instance-Id"]}
except Exception as error:
return {"ok": False, "started": started, "finished": time.monotonic(),
"error": str(error) if isinstance(error, Failure) else type(error).__name__}
try:
for service in SERVICES:
samplers.append(MemorySampler(service).start())
with ThreadPoolExecutor(max_workers=concurrency) as pool:
responses = list(pool.map(send, range(concurrency)))
finally:
for sampler in samplers:
sampler.stop()
good = [r for r in responses if r["ok"]]
elapsed = max(r["finished"] for r in responses) - min(r["started"] for r in responses)
memory = [sampler.report() for sampler in samplers]
result = {
"concurrency": concurrency, "barrier_synchronized": True, "requests": len(responses),
"successes": len(good), "failures": len(responses) - len(good),
"fixture_bytes": len(content), "fixture_dimensions": [1024, 683],
"fixture_sha256": hashlib.sha256(content).hexdigest(),
"elapsed_seconds": round(elapsed, 3), "successful_uploads_per_second": round(len(good) / elapsed, 3),
"successful_mib_per_second": round(len(good) * len(content) / (1024 * 1024 * elapsed), 3),
"instances": sorted({r["instance"] for r in good}), "memory": memory,
"latency_seconds": [round(r["finished"] - r["started"], 3) for r in responses],
"errors": [r["error"] for r in responses if not r["ok"]],
}
# Preserve measured results even when the experiment finds a failure.
result["all_downloads_equal"] = False
if len(good) != concurrency:
return result
check(len({r["attachment"]["id"] for r in good}) == concurrency, "uploads reused attachment IDs")
check(len(result["instances"]) == len(SERVICES), "concurrent uploads did not visit both API replicas")
for response in good:
download(Client(base, timeout=30), bob, response["attachment"], content)
result["all_downloads_equal"] = True
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--output", default="evidence/lab4")
parser.add_argument("--concurrency", type=int, default=10)
parser.add_argument("--skip-cluster", action="store_true", help="Run only lab4 experiments; record omitted cluster coverage")
parser.add_argument("--skip-student-tests", action="store_true", help="Repeat infrastructure experiments without rerunning the already completed suite; record NOT_RUN")
args = parser.parse_args()
check(10 <= args.concurrency <= 32, "concurrency must be 10..32")
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
save_json(out / "result.json", {"status": "RUNNING"})
os.environ["BASE_URL"] = args.base_url
def stage(name, action):
print("Running " + name, flush=True)
started = time.monotonic()
result = action()
save_json(out / (name + ".json"), result)
print("Completed " + name + " in %.2f seconds" % (time.monotonic() - started), flush=True)
return result
try:
save_json(out / "00_environment.json", {
"at_utc": datetime.now(timezone.utc).isoformat(), "platform": platform.platform(),
"python": platform.python_version(), "host_cpu_count": os.cpu_count(),
"cluster_checks": "NOT_RUN" if args.skip_cluster else "included",
"synthetic_data_only": True, "host_dependencies": "Python standard library + Docker Compose",
})
record_command(out, "00_docker", ["docker", "version"])
record_command(out, "00_compose_config", COMPOSE + ["config", "--quiet"])
if not args.skip_cluster:
print("Running preserved cluster checks", flush=True)
record_command(out, "01_cluster", [
sys.executable, "scripts/cluster_evidence.py", "--base-url", args.base_url,
"--output", str(out / "cluster"),
], timeout=900)
wait_balanced(timeout=60)
record_command(out, "01_public_smoke", [sys.executable, "tests/smoke.py", "--base-url", args.base_url])
c = Client(args.base_url, timeout=15)
alice, bob, eve, chat = c.bootstrap()
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
contract, att, target = attachment_contract(c, alice, bob, eve, chat, content)
save_json(out / "02_attachment_contract.json", contract)
stage("03_uploader_failover", lambda: uploader_failover(c, bob, att, content, target))
stage("04_full_restart", lambda: persistence(c, bob, chat, att, content, out))
stage("05_s3_outage", lambda: s3_outage(c, alice, bob, chat, att, content))
stage("06_private_bucket", lambda: private_bucket(att, out))
result = stage("07_concurrent_uploads", lambda: concurrent_uploads(
args.base_url, alice, bob, chat, args.concurrency
))
check(result["failures"] == 0 and result["all_downloads_equal"], "concurrent uploads failed; see measured results")
record_command(out, "08_cleanup_dry_run", COMPOSE + ["exec", "-T", "api-a", "python", "-m", "app.cleanup", "--dry-run"])
if args.skip_student_tests:
save_json(out / "09_student_tests.json", {"status": "NOT_RUN", "reason": "explicit --skip-student-tests; consult full run"})
else:
record_command(out, "09_student_tests", COMPOSE + ["--profile", "test", "run", "--rm", "--build", "tests"], timeout=900)
save_json(out / "10_final_processes.json", inventory())
save_json(out / "result.json", {"status": "PASS", "at_utc": datetime.now(timezone.utc).isoformat(),
"cluster_checks": "NOT_RUN" if args.skip_cluster else "PASS",
"student_tests": "NOT_RUN" if args.skip_student_tests else "PASS"})
print("PASS: lab4 evidence saved to " + str(out), flush=True)
except BaseException as error:
save_json(out / "result.json", {"status": "FAIL", "at_utc": datetime.now(timezone.utc).isoformat(),
"error_type": type(error).__name__,
"note": "Inspect completed files; an incomplete run does not prove acceptance."})
raise
if __name__ == "__main__":
main()