390 lines
19 KiB
Python
390 lines
19 KiB
Python
#!/usr/bin/env python3
|
||
"""Повторяемая проверка lab4: реальные отказы, приватный MinIO и 10 uploads с RAM."""
|
||
|
||
import argparse
|
||
import base64
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from datetime import datetime, timezone
|
||
import hashlib
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import platform
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from urllib.parse import quote
|
||
|
||
from evidence_utils import MemorySampler, exec_json, png_fixture, record_command, save_json
|
||
from replicas import COMPOSE, ROOT, SERVICES, compose, inventory, wait_balanced
|
||
|
||
sys.path.insert(0, str(ROOT / "tests"))
|
||
from client import Client, Failure, check
|
||
from resilience import history
|
||
|
||
|
||
def upload(c, actor, chat, content, filename="sample.png"):
|
||
response = c.call(
|
||
"POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=" + quote(filename),
|
||
201, actor=actor, raw=content, headers={"Content-Type": "image/png"}, schema="Attachment",
|
||
)
|
||
att = response["json"]
|
||
check(att["status"] == "ready" and att["error_code"] is None, "upload must finish ready")
|
||
check(att["variants"] == [], "lab4 must return the original without variants")
|
||
check(att["size_bytes"] == len(content), "uploaded size mismatch")
|
||
check(att["sha256"] == hashlib.sha256(content).hexdigest(), "uploaded checksum mismatch")
|
||
check(att["chat_id"] == chat["id"] and att["owner_id"] == actor["id"], "upload ownership mismatch")
|
||
return response
|
||
|
||
|
||
def download(c, actor, att, content):
|
||
metadata = c.call("GET", "/api/v1/attachments/" + att["id"], actor=actor, schema="Attachment")
|
||
check(metadata["json"] == att, "attachment metadata changed")
|
||
response = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=actor)
|
||
check(response["bytes"] == content, "download differs byte-for-byte from original")
|
||
check(response["headers"].get_content_type() == "image/png", "download Content-Type mismatch")
|
||
return response
|
||
|
||
|
||
def attachment_contract(c, alice, bob, eve, chat, content):
|
||
response = upload(c, alice, chat, content)
|
||
att = response["json"]
|
||
attached = c.message(alice, chat, "", attachments=[att["id"]])["json"]
|
||
check(attached["attachment_ids"] == [att["id"]], "attachment-only message lost attachment")
|
||
check(attached in history(c, bob, chat), "second member cannot read attached message")
|
||
download(c, bob, att, content)
|
||
for suffix in ("", "/content"):
|
||
c.call("GET", "/api/v1/attachments/" + att["id"] + suffix, 404, actor=eve)
|
||
return {
|
||
"attachment_id": att["id"], "upload_instance": response["headers"]["X-Instance-Id"],
|
||
"sha256": att["sha256"], "size_bytes": len(content), "download_equal": True,
|
||
"attachment_only_message_visible": True, "outsider_metadata_status": 404,
|
||
"outsider_content_status": 404,
|
||
}, att, response["headers"]["X-Instance-Id"]
|
||
|
||
|
||
def uploader_failover(c, bob, att, content, target):
|
||
"""Остановить именно процесс, вернувший 201 на binary upload."""
|
||
before = inventory()
|
||
matches = [row for row in before if row.get("instance_id") == target]
|
||
check(len(matches) == 1, "upload must identify one current API process")
|
||
service = matches[0]["service"]
|
||
started = time.monotonic()
|
||
try:
|
||
env = dict(os.environ, TARGET_INSTANCE_ID=target, PYTHONUTF8="1")
|
||
result = subprocess.run(
|
||
[sys.executable, str(ROOT / "scripts/stop_one.py")], cwd=ROOT, env=env,
|
||
capture_output=True, timeout=15,
|
||
)
|
||
check(result.returncode == 0, "could not stop uploading API")
|
||
stopped = next(row for row in inventory() if row["service"] == service)
|
||
check(stopped["state"] == "exited" and stopped["pid"] == 0, "upload process still alive")
|
||
deadline = time.monotonic() + 10
|
||
while True:
|
||
try:
|
||
response = download(c, bob, att, content)
|
||
break
|
||
except Failure:
|
||
check(time.monotonic() < deadline, "download failed to recover after uploader stop")
|
||
time.sleep(0.2)
|
||
survivor = response["headers"]["X-Instance-Id"]
|
||
check(survivor != target, "download came from the stopped uploader")
|
||
return {
|
||
"upload_instance": target, "stopped_service": service,
|
||
"stopped_state": stopped["state"], "stopped_pid": stopped["pid"],
|
||
"download_instance": survivor, "sha256": hashlib.sha256(response["bytes"]).hexdigest(),
|
||
"download_equal": True, "seconds_from_stop_start": round(time.monotonic() - started, 3),
|
||
}
|
||
finally:
|
||
compose("up", "-d", "--no-deps", service)
|
||
wait_balanced(timeout=60)
|
||
|
||
|
||
def persistence(c, bob, chat, att, content, out):
|
||
before = inventory()
|
||
record_command(out, "04_restart_command", [sys.executable, "scripts/restart.py"], timeout=120)
|
||
after = inventory()
|
||
check({r["instance_id"] for r in before}.isdisjoint(r["instance_id"] for r in after),
|
||
"restart must replace both API processes")
|
||
response = download(c, bob, att, content)
|
||
check(any(att["id"] in message["attachment_ids"] for message in history(c, bob, chat)),
|
||
"attached message lost after restart")
|
||
return {
|
||
"before": before, "after": after, "existing_session_works": True,
|
||
"message_preserved": True, "metadata_equal": True, "download_equal": True,
|
||
"sha256": hashlib.sha256(response["bytes"]).hexdigest(),
|
||
"restart_scope": "all default Compose services, force recreated; named volumes retained",
|
||
}
|
||
|
||
|
||
DIRECT_PROBES = r'''
|
||
import base64, json, urllib.error, urllib.request, sys
|
||
p = json.load(sys.stdin)
|
||
result = {}
|
||
for name, method, path, body in (
|
||
('live', 'GET', '/health/live', None),
|
||
('ready', 'GET', '/health/ready', None),
|
||
('upload', 'POST', '/api/v1/chats/' + p['chat_id'] + '/attachments?filename=outage.png', base64.b64decode(p['png'])),
|
||
('content', 'GET', '/api/v1/attachments/' + p['attachment_id'] + '/content', None),
|
||
):
|
||
headers = {'Authorization': 'Bearer ' + p['token'], 'Content-Type': 'image/png'}
|
||
request = urllib.request.Request('http://127.0.0.1:8080' + path, method=method, headers=headers, data=body)
|
||
try:
|
||
response = urllib.request.urlopen(request, timeout=30)
|
||
except urllib.error.HTTPError as error:
|
||
response = error
|
||
with response:
|
||
data = response.read(16 * 1024 * 1024)
|
||
result[name] = {'status': response.status, 'instance_id': response.headers.get('X-Instance-Id'),
|
||
'request_id_present': bool(response.headers.get('X-Request-Id'))}
|
||
if response.status >= 400:
|
||
error = json.loads(data)['error']
|
||
result[name]['error_code'] = error['code']
|
||
result[name]['request_id_matches'] = error['request_id'] == response.headers.get('X-Request-Id')
|
||
print(json.dumps(result))
|
||
'''
|
||
|
||
|
||
def s3_outage(c, alice, bob, chat, att, content):
|
||
results = {}
|
||
public = None
|
||
try:
|
||
compose("stop", "--timeout", "10", "minio")
|
||
for service in SERVICES:
|
||
results[service] = exec_json(service, DIRECT_PROBES, {
|
||
"token": alice["token"], "chat_id": chat["id"], "attachment_id": att["id"],
|
||
"png": base64.b64encode(content).decode("ascii"),
|
||
}, timeout=120)
|
||
for name, status in (("live", 200), ("ready", 503), ("upload", 503), ("content", 503)):
|
||
actual = results[service][name]
|
||
check(actual["status"] == status, service + "/" + name + ": " + str(actual))
|
||
check(actual["request_id_present"] and actual["instance_id"], "missing API trace headers")
|
||
if status == 503:
|
||
check(actual["error_code"] == "unavailable" and actual["request_id_matches"],
|
||
"S3 outage must return a structured 503")
|
||
try:
|
||
reply = urllib.request.urlopen(c.base + "/health/ready", timeout=10)
|
||
except urllib.error.HTTPError as error:
|
||
reply = error
|
||
with reply:
|
||
payload = json.loads(reply.read())
|
||
public = {"status": reply.status, "error_code": payload["error"]["code"],
|
||
"request_id_matches": payload["error"]["request_id"] == reply.headers.get("X-Request-Id")}
|
||
check(public["status"] == 503 and public["request_id_matches"], "public readiness must return structured 503")
|
||
finally:
|
||
compose("start", "minio")
|
||
wait_balanced(timeout=60)
|
||
download(c, bob, att, content)
|
||
return {"direct_api_probes": results, "public_readiness": public, "restored_download_equal": True,
|
||
"method": "probe each API directly inside its container because LB excludes unready backends"}
|
||
|
||
|
||
PRIVATE_BUCKET = r'''
|
||
import asyncio, io, json, os, sys, uuid, urllib.error, urllib.request
|
||
import urllib.error
|
||
import urllib.request
|
||
from urllib.parse import quote, urlsplit
|
||
from minio import Minio
|
||
from minio.error import S3Error
|
||
from sqlalchemy import text
|
||
from app.config import Settings
|
||
from app.db import create_engine
|
||
p = json.load(sys.stdin)
|
||
async def key():
|
||
engine = create_engine(Settings.from_env())
|
||
try:
|
||
async with engine.connect() as connection:
|
||
return (await connection.execute(text('SELECT object_key FROM attachments WHERE id=:id'),
|
||
{'id': uuid.UUID(p['attachment_id'])})).scalar_one()
|
||
finally:
|
||
await engine.dispose()
|
||
object_key = asyncio.run(key())
|
||
endpoint = os.environ['S3_ENDPOINT_URL'].rstrip('/')
|
||
bucket = os.environ['S3_BUCKET']
|
||
url = endpoint + '/' + quote(bucket, safe='') + '/' + quote(object_key, safe='/')
|
||
try:
|
||
response = urllib.request.urlopen(url, timeout=5)
|
||
except urllib.error.HTTPError as error:
|
||
response = error
|
||
with response:
|
||
anonymous_status = response.status
|
||
parsed = urlsplit(endpoint)
|
||
client = Minio(parsed.netloc, access_key=os.environ['S3_ACCESS_KEY'], secret_key=os.environ['S3_SECRET_KEY'],
|
||
secure=parsed.scheme == 'https', region=os.environ.get('S3_REGION', 'us-east-1'))
|
||
new_bucket = 'forbidden-' + uuid.uuid4().hex
|
||
outside_key = 'forbidden/' + uuid.uuid4().hex
|
||
operations = {
|
||
'list_foreign_bucket': lambda: list(client.list_objects(new_bucket, recursive=True)),
|
||
'make_bucket': lambda: client.make_bucket(new_bucket),
|
||
'put_outside_prefix': lambda: client.put_object(bucket, outside_key, io.BytesIO(b'x'), 1),
|
||
}
|
||
# MinIO deliberately filters ListBuckets to buckets permitted by bucket-level IAM.
|
||
# Verify the names rather than incorrectly requiring HTTP 403 for this API.
|
||
visible_buckets = [item.name for item in client.list_buckets()]
|
||
outcomes = {}
|
||
for name, operation in operations.items():
|
||
try:
|
||
operation()
|
||
outcomes[name] = 'UNEXPECTED_SUCCESS'
|
||
except S3Error as error:
|
||
outcomes[name] = error.code
|
||
except Exception as error:
|
||
outcomes[name] = type(error).__name__
|
||
# Clean synthetic probes if an incorrect policy unexpectedly allowed them.
|
||
for action in (
|
||
(lambda: client.remove_bucket(new_bucket)) if outcomes['make_bucket'] == 'UNEXPECTED_SUCCESS' else None,
|
||
(lambda: client.remove_object(bucket, outside_key)) if outcomes['put_outside_prefix'] == 'UNEXPECTED_SUCCESS' else None,
|
||
):
|
||
if action:
|
||
try: action()
|
||
except Exception: pass
|
||
print(json.dumps({'anonymous_get_status': anonymous_status, 'restricted_operations': outcomes,
|
||
'visible_buckets': visible_buckets, 'visible_buckets_limited': set(visible_buckets) <= {bucket},
|
||
'root_credentials_absent_in_api': not any(name in os.environ for name in ('MINIO_ROOT_USER', 'MINIO_ROOT_PASSWORD')),
|
||
'server_assigned_original_prefix': object_key.startswith('originals/')}))
|
||
'''
|
||
|
||
|
||
def private_bucket(att, out):
|
||
# The same bootstrap is executed twice against an already populated bucket.
|
||
for number in (1, 2):
|
||
record_command(out, "06_bootstrap_repeat_" + str(number),
|
||
COMPOSE + ["run", "--rm", "--no-deps", "minio-init"], timeout=120)
|
||
result = exec_json("api-a", PRIVATE_BUCKET, {"attachment_id": att["id"]})
|
||
check(result["anonymous_get_status"] == 403, "private bucket anonymous GET: " + str(result))
|
||
check(all(value == "AccessDenied" for value in result["restricted_operations"].values()),
|
||
"API S3 privilege violation: " + str(result))
|
||
check(result["visible_buckets_limited"], "API can see an unrelated bucket")
|
||
check(result["root_credentials_absent_in_api"], "API must not receive root S3 credentials")
|
||
check(result["server_assigned_original_prefix"], "unexpected server object key prefix")
|
||
result["bootstrap_repeated_successfully"] = 2
|
||
return result
|
||
|
||
|
||
def concurrent_uploads(base, alice, bob, chat, concurrency):
|
||
content = png_fixture()
|
||
check(len(content) >= 2 * 1024 * 1024, "experiment fixture must be at least 2 MiB")
|
||
barrier = threading.Barrier(concurrency, timeout=30)
|
||
samplers = []
|
||
|
||
def send(number):
|
||
client = Client(base, timeout=90)
|
||
barrier.wait()
|
||
started = time.monotonic()
|
||
try:
|
||
response = upload(client, alice, chat, content, "parallel-%02d.png" % number)
|
||
return {"ok": True, "started": started, "finished": time.monotonic(),
|
||
"attachment": response["json"], "instance": response["headers"]["X-Instance-Id"]}
|
||
except Exception as error:
|
||
return {"ok": False, "started": started, "finished": time.monotonic(),
|
||
"error": str(error) if isinstance(error, Failure) else type(error).__name__}
|
||
|
||
try:
|
||
for service in SERVICES:
|
||
samplers.append(MemorySampler(service).start())
|
||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||
responses = list(pool.map(send, range(concurrency)))
|
||
finally:
|
||
for sampler in samplers:
|
||
sampler.stop()
|
||
good = [r for r in responses if r["ok"]]
|
||
elapsed = max(r["finished"] for r in responses) - min(r["started"] for r in responses)
|
||
memory = [sampler.report() for sampler in samplers]
|
||
result = {
|
||
"concurrency": concurrency, "barrier_synchronized": True, "requests": len(responses),
|
||
"successes": len(good), "failures": len(responses) - len(good),
|
||
"fixture_bytes": len(content), "fixture_dimensions": [1024, 683],
|
||
"fixture_sha256": hashlib.sha256(content).hexdigest(),
|
||
"elapsed_seconds": round(elapsed, 3), "successful_uploads_per_second": round(len(good) / elapsed, 3),
|
||
"successful_mib_per_second": round(len(good) * len(content) / (1024 * 1024 * elapsed), 3),
|
||
"instances": sorted({r["instance"] for r in good}), "memory": memory,
|
||
"latency_seconds": [round(r["finished"] - r["started"], 3) for r in responses],
|
||
"errors": [r["error"] for r in responses if not r["ok"]],
|
||
}
|
||
# Preserve measured results even when the experiment finds a failure.
|
||
result["all_downloads_equal"] = False
|
||
if len(good) != concurrency:
|
||
return result
|
||
check(len({r["attachment"]["id"] for r in good}) == concurrency, "uploads reused attachment IDs")
|
||
check(len(result["instances"]) == len(SERVICES), "concurrent uploads did not visit both API replicas")
|
||
for response in good:
|
||
download(Client(base, timeout=30), bob, response["attachment"], content)
|
||
result["all_downloads_equal"] = True
|
||
return result
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument("--base-url", default="http://localhost:8080")
|
||
parser.add_argument("--output", default="evidence/lab4")
|
||
parser.add_argument("--concurrency", type=int, default=10)
|
||
parser.add_argument("--skip-cluster", action="store_true", help="Run only lab4 experiments; record omitted cluster coverage")
|
||
parser.add_argument("--skip-student-tests", action="store_true", help="Repeat infrastructure experiments without rerunning the already completed suite; record NOT_RUN")
|
||
args = parser.parse_args()
|
||
check(10 <= args.concurrency <= 32, "concurrency must be 10..32")
|
||
out = ROOT / args.output
|
||
out.mkdir(parents=True, exist_ok=True)
|
||
save_json(out / "result.json", {"status": "RUNNING"})
|
||
os.environ["BASE_URL"] = args.base_url
|
||
|
||
def stage(name, action):
|
||
print("Running " + name, flush=True)
|
||
started = time.monotonic()
|
||
result = action()
|
||
save_json(out / (name + ".json"), result)
|
||
print("Completed " + name + " in %.2f seconds" % (time.monotonic() - started), flush=True)
|
||
return result
|
||
|
||
try:
|
||
save_json(out / "00_environment.json", {
|
||
"at_utc": datetime.now(timezone.utc).isoformat(), "platform": platform.platform(),
|
||
"python": platform.python_version(), "host_cpu_count": os.cpu_count(),
|
||
"cluster_checks": "NOT_RUN" if args.skip_cluster else "included",
|
||
"synthetic_data_only": True, "host_dependencies": "Python standard library + Docker Compose",
|
||
})
|
||
record_command(out, "00_docker", ["docker", "version"])
|
||
record_command(out, "00_compose_config", COMPOSE + ["config", "--quiet"])
|
||
if not args.skip_cluster:
|
||
print("Running preserved cluster checks", flush=True)
|
||
record_command(out, "01_cluster", [
|
||
sys.executable, "scripts/cluster_evidence.py", "--base-url", args.base_url,
|
||
"--output", str(out / "cluster"),
|
||
], timeout=900)
|
||
wait_balanced(timeout=60)
|
||
record_command(out, "01_public_smoke", [sys.executable, "tests/smoke.py", "--base-url", args.base_url])
|
||
c = Client(args.base_url, timeout=15)
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||
contract, att, target = attachment_contract(c, alice, bob, eve, chat, content)
|
||
save_json(out / "02_attachment_contract.json", contract)
|
||
stage("03_uploader_failover", lambda: uploader_failover(c, bob, att, content, target))
|
||
stage("04_full_restart", lambda: persistence(c, bob, chat, att, content, out))
|
||
stage("05_s3_outage", lambda: s3_outage(c, alice, bob, chat, att, content))
|
||
stage("06_private_bucket", lambda: private_bucket(att, out))
|
||
result = stage("07_concurrent_uploads", lambda: concurrent_uploads(
|
||
args.base_url, alice, bob, chat, args.concurrency
|
||
))
|
||
check(result["failures"] == 0 and result["all_downloads_equal"], "concurrent uploads failed; see measured results")
|
||
record_command(out, "08_cleanup_dry_run", COMPOSE + ["exec", "-T", "api-a", "python", "-m", "app.cleanup", "--dry-run"])
|
||
if args.skip_student_tests:
|
||
save_json(out / "09_student_tests.json", {"status": "NOT_RUN", "reason": "explicit --skip-student-tests; consult full run"})
|
||
else:
|
||
record_command(out, "09_student_tests", COMPOSE + ["--profile", "test", "run", "--rm", "--build", "tests"], timeout=900)
|
||
save_json(out / "10_final_processes.json", inventory())
|
||
save_json(out / "result.json", {"status": "PASS", "at_utc": datetime.now(timezone.utc).isoformat(),
|
||
"cluster_checks": "NOT_RUN" if args.skip_cluster else "PASS",
|
||
"student_tests": "NOT_RUN" if args.skip_student_tests else "PASS"})
|
||
print("PASS: lab4 evidence saved to " + str(out), flush=True)
|
||
except BaseException as error:
|
||
save_json(out / "result.json", {"status": "FAIL", "at_utc": datetime.now(timezone.utc).isoformat(),
|
||
"error_type": type(error).__name__,
|
||
"note": "Inspect completed files; an incomplete run does not prove acceptance."})
|
||
raise
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|