ЛР4 на 3

Co-authored-by: Югай Артур Рустамович <f1ekserr@yandex.ru>
This commit is contained in:
2026-11-16 22:00:33 +03:00
co-authored by Югай Артур Рустамович
parent ec540a1a72
commit 0c5c19bb09
44 changed files with 3722 additions and 42 deletions
+677
View File
@@ -0,0 +1,677 @@
import argparse
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path
import platform
import secrets
import shutil
import socket
import subprocess
import sys
import time
from urllib.error import URLError
from urllib.request import ProxyHandler, build_opener
import venv
ROOT = Path(__file__).resolve().parents[1]
LOG = []
LOCAL_HTTP = build_opener(ProxyHandler({}))
def settings():
path = ROOT / ".env"
result = {}
if path.exists():
for line in path.read_text(encoding="utf-8-sig").splitlines():
key, separator, value = line.partition("=")
if separator and not key.strip().startswith("#"):
result[key.strip()] = value.strip()
return result
def environment(args):
result = os.environ.copy()
result.update(PYTHONUTF8="1", PYTHONIOENCODING="utf-8", PYTHONDONTWRITEBYTECODE="1",
COMPOSE_PROJECT_NAME=args.project, HTTP_PORT=str(args.port),
SESSION_TTL_SECONDS=str(args.ttl), LAB4_PROJECT=args.project,
LAB4_PORT=str(args.port), LAB4_TTL=str(args.ttl))
for key in ("NO_PROXY", "no_proxy"):
result[key] = ",".join(filter(None, [result.get(key), "localhost", "127.0.0.1"]))
return result
def say(value):
line = str(value).replace(str(ROOT), ".")
print(line, flush=True)
LOG.append(line)
def run(args, command, quiet=False):
command = [str(item) for item in command]
if not quiet:
say("$ " + subprocess.list2cmdline(command))
with subprocess.Popen(command, cwd=ROOT, env=environment(args), stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace") as process:
lines = []
for line in process.stdout:
lines.append(line)
if not quiet:
say(line.rstrip())
code = process.wait()
if code:
raise RuntimeError(f"{command[0]} exited with code {code}")
return "".join(lines)
def compose(args, *command, quiet=False):
if args.project != "kuznetsov-lab4" and not args.project.startswith("kuznetsov-lab4-"):
raise RuntimeError("This runner only manages Kuznetsov lab4 projects")
return run(args, ["docker", "compose", "--project-name", args.project, "--env-file", ".env", *command], quiet)
def interpreter():
return ROOT / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
def evidence(name, value):
path = ROOT / "evidence" / name
path.parent.mkdir(exist_ok=True)
content = json.dumps(value, ensure_ascii=False, indent=2) if not isinstance(value, str) else value
path.write_text(content.rstrip() + "\n", encoding="utf-8", newline="\n")
def setup(args):
if not (ROOT / ".env").exists():
shutil.copyfile(ROOT / ".env.example", ROOT / ".env")
folder = ROOT / "secrets"
folder.mkdir(exist_ok=True)
password_file = folder / "postgres_password"
if not password_file.exists():
password_file.write_text(secrets.token_hex(32) + "\n", encoding="ascii")
password = password_file.read_text(encoding="ascii").strip()
values = {"database_url": f"postgresql://messenger:{password}@db:5432/messenger",
"cursor_key": secrets.token_hex(32)}
for name, value in values.items():
path = folder / name
if not path.exists():
path.write_text(value + "\n", encoding="ascii")
if os.name != "nt":
path.chmod(0o644)
if os.name != "nt":
folder.chmod(0o700)
password_file.chmod(0o644)
for name in ("s3_access_key_id", "s3_secret_access_key", "s3_admin_access_key_id", "s3_admin_secret_access_key"):
path = folder / name
if not path.exists():
path.write_text(secrets.token_hex(32) + "\n", encoding="ascii")
if os.name != "nt":
path.chmod(0o644)
config = folder / "s3_config.json"
if not config.exists():
def key(name):
return (folder / name).read_text(encoding="ascii").strip()
bucket = settings().get("S3_BUCKET", "kuznetsov-lab4-attachments")
identities = [
{"name": "bootstrap", "credentials": [{"accessKey": key("s3_admin_access_key_id"),
"secretKey": key("s3_admin_secret_access_key")}], "actions": ["Admin", "Read", "Write", "List"]},
{"name": "messenger", "credentials": [{"accessKey": key("s3_access_key_id"),
"secretKey": key("s3_secret_access_key")}],
"actions": [action + ":" + bucket for action in ("Read", "Write", "List")]},
]
config.write_text(json.dumps({"identities": identities}) + "\n", encoding="ascii")
if os.name != "nt":
config.chmod(0o644)
if not interpreter().exists():
venv.EnvBuilder(with_pip=True).create(ROOT / ".venv")
lockfiles = [ROOT / "requirements.lock", ROOT / "requirements-validation.lock"]
fingerprint = hashlib.sha256(b"".join(path.read_bytes() for path in lockfiles)).hexdigest()
marker = ROOT / ".venv/.lab-dependencies"
if not marker.exists() or marker.read_text(encoding="ascii") != fingerprint:
run(args, [interpreter(), "-X", "utf8", "-m", "pip", "install", "--disable-pip-version-check",
"--no-input", "--progress-bar", "off", "--quiet", "-r", lockfiles[0], "-r", lockfiles[1]])
marker.write_text(fingerprint, encoding="ascii")
say("Environment ready; existing settings and secrets preserved")
def base_url(args):
return f"http://127.0.0.1:{args.port}"
def wait_storage(args, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
container = compose(args, "ps", "--all", "--quiet", "s3-storage", quiet=True).strip()
if container:
status = run(args, ["docker", "inspect", "--format", "{{.State.Health.Status}}", container], quiet=True).strip()
if status == "healthy":
return
time.sleep(0.5)
raise RuntimeError("S3 storage readiness timeout")
def wait_ready(args, timeout=90):
wait_storage(args, timeout)
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
with LOCAL_HTTP.open(base_url(args) + "/health/ready", timeout=4) as response:
if response.status == 200 and json.load(response) == {"status": "ok"}:
return
except (URLError, OSError, ValueError):
pass
time.sleep(0.3)
raise RuntimeError("API readiness timeout")
def build(args):
compose(args, "config", "--quiet")
compose(args, "build", "api1")
compose(args, "run", "--rm", "--no-deps", "lb", "haproxy", "-c", "-f",
"/usr/local/etc/haproxy/haproxy.cfg")
def start(args):
current = compose(args, "ps", "--all", "--quiet", "lb", quiet=True).strip()
if not current:
with socket.socket() as probe:
try:
probe.bind(("127.0.0.1", args.port))
except OSError:
raise RuntimeError(f"Port {args.port} is busy; stop its container or use --port") from None
compose(args, "up", "-d", "--no-build", "db", "migrate", "s3-storage", "s3", "bootstrap", "api1", "api2", "lb")
wait_ready(args)
wait_replicas(args)
say("Two API replicas ready through LB: " + base_url(args))
def states(args):
services = ("api1", "api2", "db", "lb", "s3", "s3-storage")
containers = compose(args, "ps", "--all", "--quiet", *services, quiet=True).split()
if len(set(containers)) != len(services):
raise RuntimeError("Missing runtime container")
result = {}
for data in json.loads(run(args, ["docker", "inspect", *containers], quiet=True)):
labels = data["Config"].get("Labels", {})
service = labels.get("com.docker.compose.service")
if labels.get("com.docker.compose.project") != args.project or service not in services or service in result:
raise RuntimeError("Unexpected container ownership")
result[service] = {
"id": data["Id"], "image": data["Image"], "started_at": data["State"]["StartedAt"],
"running": data["State"]["Running"], "user": data["Config"]["User"],
"volumes": [mount["Name"] for mount in data["Mounts"] if mount["Type"] == "volume"],
}
return result
def restart(args):
before = states(args)
if not all(item["running"] for item in before.values()):
raise RuntimeError("All runtime containers must be running before restart")
compose(args, "restart", "--timeout", "10", "db", "s3-storage", "s3", "api1", "api2", "lb")
wait_storage(args)
compose(args, "run", "--rm", "--no-deps", "bootstrap")
after = states(args)
for service in before:
if before[service]["started_at"] == after[service]["started_at"]:
raise RuntimeError("Container was not restarted: " + service)
if before[service]["volumes"] != after[service]["volumes"]:
raise RuntimeError("Volume changed during restart")
evidence("restart.json", {"project": args.project, "before": before, "after": after})
say("PASS: both APIs, PostgreSQL, S3 and LB restarted, volumes preserved")
def contract_tests(args):
run(args, [interpreter(), "-X", "utf8", "scripts/check_contract.py"])
run(args, [interpreter(), "-X", "utf8", "scripts/validate_openapi.py"])
compose(args, "config", "--quiet")
def api_tests(args):
wait_ready(args)
run(args, [interpreter(), "-X", "utf8", "tests/smoke.py", "--base-url", base_url(args)])
os.environ["BASE_URL"] = base_url(args)
run(args, [interpreter(), "-X", "utf8", "-m", "unittest", "discover", "-s", "student-tests", "-v"])
def client(args):
sys.path.insert(0, str(ROOT / "tests"))
from client import Client
for key in ("NO_PROXY", "no_proxy"):
os.environ[key] = environment(args)[key]
return Client(base_url(args))
def database_stats(args):
sql = """SELECT json_build_object(
'users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats),
'members',(SELECT count(*) FROM chat_members),'messages',(SELECT count(*) FROM messages),
'sessions',(SELECT count(*) FROM sessions),'migrations',(SELECT count(*) FROM schema_migrations),
'idempotency_keys',(SELECT count(*) FROM idempotency_keys),
'attachments',(SELECT count(*) FROM attachments),'message_attachments',(SELECT count(*) FROM message_attachments),
'expiry_fingerprint',(SELECT md5(COALESCE(string_agg(user_id::text || created_at::text || expires_at::text,
',' ORDER BY token_hash),'')) FROM sessions),
'passwords_are_argon2id',(SELECT bool_and(password_hash LIKE '$argon2id$%') FROM users),
'session_hashes_only',(SELECT bool_and(token_hash ~ '^[0-9a-f]{64}$') FROM sessions))"""
result = compose(args, "exec", "-T", "db", "psql", "-U", "messenger", "-d", "messenger",
"-At", "-c", sql, quiet=True)
return json.loads(result)
def restart_check(args):
from urllib.parse import urlencode
c = client(args)
alice, bob, eve, chat = c.bootstrap()
path = "/api/v1/chats/" + chat["id"] + "/messages"
replay_key = secrets.token_hex(16)
original = c.message(alice, chat, "first", key=replay_key)["json"]
last = c.message(bob, chat, "second")["json"]
page = c.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")["json"]
revoked = c.user("revoked")
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=revoked)
hook = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh")
run(args, [interpreter(), "-X", "utf8", "tests/resilience.py", "--mode", "persistence",
"--action-script", hook, "--base-url", base_url(args)])
continued = c.call("GET", path + "?" + urlencode({"cursor": page["next_cursor"]}),
actor=alice, schema="MessagePage")["json"]
if continued != {"items": [last], "next_cursor": None}:
raise RuntimeError("Old cursor did not survive restart")
c.call("GET", "/api/v1/users/me", 401, actor=revoked)
c.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve)
before = database_stats(args)
compose(args, "run", "--rm", "--no-deps", "migrate")
compose(args, "run", "--rm", "--no-deps", "migrate")
after = database_stats(args)
if before != after or after["migrations"] != 3:
raise RuntimeError("Migration replay changed existing data")
if not after["passwords_are_argon2id"] or not after["session_hashes_only"]:
raise RuntimeError("Unexpected credential representation in DB")
repeated = c.message(alice, chat, "first", key=replay_key)["json"]
if repeated != original:
raise RuntimeError("Idempotency result was not preserved")
result = {"idempotency_after_restart": "PASS", "public_persistence": "PASS", "cursor_after_restart": "PASS",
"logout_after_restart": "PASS", "migration_replay": "PASS", "database": after}
evidence("persistence.json", result)
say("PASS: old session/cursor, durable logout and repeatable migrations")
return result
def ttl_check(args):
test = argparse.Namespace(project=args.project + "-ttl", port=args.test_port, ttl=2,
test_port=args.test_port, action="start")
try:
start(test)
c = client(test)
actor = c.user("expiry")
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
before = database_stats(test)["expiry_fingerprint"]
compose(test, "restart", "--timeout", "3", "api1", "api2")
wait_ready(test)
after = database_stats(test)["expiry_fingerprint"]
if before != after:
raise RuntimeError("Session expiry changed during restart")
time.sleep(2.2)
c.call("GET", "/api/v1/users/me", 401, actor=actor)
registration = actor["registration"]
import base64
basic = base64.b64encode((registration["username"] + ":" + registration["password"]).encode()).decode()
c.call("POST", "/api/v1/auth/sessions", 201, headers={"Authorization": "Basic " + basic}, schema="Session")
say("PASS: TTL=2, expired token rejected after API restart; fresh login works")
result = {"ttl_seconds": 2, "expiry_unchanged_on_restart": True, "before_restart": 200,
"after_expiry_and_restart": 401, "new_login": 201}
evidence("expiry.json", result)
return result
finally:
compose(test, "stop", "--timeout", "10", "lb", "api1", "api2", "s3", "s3-storage", "db")
def dependency_check(args):
c = client(args)
actor, bob, eve, chat = c.bootstrap()
attachment, original = c.attachment(actor, chat)
content_path = "/api/v1/attachments/" + attachment["id"] + "/content"
result = {}
for service in ("db", "s3"):
try:
compose(args, "stop", "--timeout", "10", service)
c.call("GET", "/health/live", schema="Health")
outage_response(args, "/health/ready")
outage_response(args, content_path, actor["token"])
finally:
compose(args, "start", service)
wait_ready(args)
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
if c.call("GET", content_path, actor=bob)["bytes"] != original:
raise RuntimeError("Original changed after dependency recovery")
result[service] = {"live": 200, "ready": 503, "download": 503, "recovered": 200}
say("PASS: " + service + " outage returns bounded errors; session and file work after recovery")
return result
def private_s3_check(args):
code = (
"import os,json,boto3; from botocore import UNSIGNED; "
"from botocore.config import Config; from botocore.exceptions import ClientError\n"
"c=boto3.client('s3',endpoint_url=os.environ['S3_ENDPOINT'],"
"config=Config(signature_version=UNSIGNED,connect_timeout=2,read_timeout=5,proxies={}))\n"
"try:\n r=c.get_object(Bucket=os.environ['S3_BUCKET'],Key='system/readiness'); r['Body'].close(); status=200\n"
"except ClientError as e:\n status=e.response['ResponseMetadata']['HTTPStatusCode']\n"
"finally:\n c.close()\n"
"print(json.dumps({'anonymous_get':status}))"
)
result = json.loads(compose(args, "exec", "-T", "api1", "python", "-c", code, quiet=True))
if result["anonymous_get"] != 403:
raise RuntimeError("S3 bucket is publicly readable")
evidence("private-s3.json", result)
say("PASS: unsigned direct S3 GET denied")
return result
def verify(args):
summary = {"date_utc": datetime.now(timezone.utc).isoformat(), "platform": platform.platform(),
"source_commit": run(args, ["git", "rev-parse", "HEAD"], quiet=True).strip(),
"worktree_dirty_at_start": bool(run(args, ["git", "status", "--porcelain"], quiet=True).strip()),
"project": args.project, "status": "FAIL"}
try:
setup(args)
contract_tests(args)
build(args)
start(args)
api_tests(args)
summary["private_s3"] = private_s3_check(args)
summary["persistence"] = restart_check(args)
summary["failover"] = failover_check(args)
summary["attachments_failover"] = attachment_failover_check(args)
summary["expiry"] = ttl_check(args)
summary["dependency"] = dependency_check(args)
run(args, [interpreter(), "-X", "utf8", "tests/smoke.py", "--base-url", base_url(args)])
summary["containers"] = states(args)
summary["docker"] = run(args, ["docker", "version", "--format", "{{.Server.Version}}"], quiet=True).strip()
summary["resources"] = json.loads(run(
args, ["docker", "info", "--format", '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}'], quiet=True))
summary["status"] = "PASS"
say("PASS: full lab4 verification")
finally:
evidence("verification.txt", "\n".join(LOG))
evidence("summary.json", summary)
def instance_map(args):
current = states(args)
result = {}
for service in ("api1", "api2"):
item = current[service]
if not item["running"]:
continue
code = (
"import urllib.request; "
"o=urllib.request.build_opener(urllib.request.ProxyHandler({})); "
"r=o.open('http://127.0.0.1:8080/health/live',timeout=1); "
"print(r.headers['X-Instance-Id'])"
)
alias = run(args, ["docker", "exec", item["id"], "python", "-c", code], quiet=True).strip()
import re
if not re.fullmatch(r"[0-9a-f]{32}", alias):
raise RuntimeError("Invalid API instance alias")
result[service] = {"instance_id": alias, "container_id": item["id"]}
return result
def wait_replicas(args):
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
mapping = instance_map(args)
if len(mapping) == 2:
c = client(args)
seen = {c.call("GET", "/health/ready", schema="Health")["headers"]["X-Instance-Id"]
for _ in range(8)}
if seen == {item["instance_id"] for item in mapping.values()}:
return mapping
except (RuntimeError, OSError, AssertionError):
pass
time.sleep(0.3)
raise RuntimeError("Two healthy API replicas were not observed through LB")
def restore(args):
compose(args, "start", "api1", "api2")
wait_ready(args)
return wait_replicas(args)
def stop_one(args):
target = os.environ.get("TARGET_INSTANCE_ID", "")
before = instance_map(args)
matches = [(name, item) for name, item in before.items() if item["instance_id"] == target]
if len(before) != 2 or len(matches) != 1:
raise RuntimeError("Target must match exactly one of two running API replicas")
service, item = matches[0]
started = time.monotonic()
run(args, ["docker", "stop", "--timeout", "3", item["container_id"]])
after = states(args)
if after[service]["running"] or not after["api1" if service == "api2" else "api2"]["running"]:
raise RuntimeError("Unexpected API state after stop")
elapsed = time.monotonic() - started
if elapsed >= 10:
raise RuntimeError("Stopping the selected API exceeded 10 seconds")
evidence("stop-one.json", {"service": service, "target_instance": target, "before": before,
"after": after, "stop_seconds": round(elapsed, 3)})
say("Stopped only " + service)
def failover_check(args):
wait_replicas(args)
hook = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh")
try:
output = run(args, [interpreter(), "-X", "utf8", "tests/resilience.py", "--mode", "failover",
"--action-script", hook, "--base-url", base_url(args)])
first = json.loads((ROOT / "evidence/stop-one.json").read_text(encoding="utf-8"))
first["metrics"] = next(json.loads(line) for line in output.splitlines()
if line.startswith("{") and "first_recovery_seconds" in line)
if first["after"][first["service"]]["running"]:
raise RuntimeError("Public hook did not leave the target stopped")
finally:
restore(args)
other_service = "api2" if first["service"] == "api1" else "api1"
mapping = instance_map(args)
target = mapping[other_service]["instance_id"]
c = client(args)
c.timeout = 2
alice, bob, eve, chat = c.bootstrap()
for _ in range(10):
replay_key = secrets.token_hex(16)
replay_text = "before-failure-" + replay_key
response = c.message(alice, chat, replay_text, key=replay_key)
if response["headers"]["X-Instance-Id"] == target:
original = response["json"]
break
else:
raise RuntimeError("Second API was not reached through LB")
env = environment(args)
env["TARGET_INSTANCE_ID"] = target
started = time.monotonic()
process = subprocess.Popen([str(hook)], cwd=ROOT, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
from client import Failure
from resilience import history
confirmations, after_instances = [], []
failures = 0
recovered = None
key = secrets.token_hex(16)
text = "during-failure-" + key
try:
while time.monotonic() - started < 15:
code = process.poll()
if code not in (None, 0):
raise RuntimeError("Second stop hook failed")
if code is None and time.monotonic() - started >= 10:
raise RuntimeError("Second stop hook exceeded 10 seconds")
try:
response = c.message(alice, chat, text, key=key)
confirmations.append(response["json"])
if code == 0:
if recovered is None:
recovered = time.monotonic() - started
after_instances.append(response["headers"]["X-Instance-Id"])
key = secrets.token_hex(16)
text = "during-failure-" + key
except Failure:
failures += 1
time.sleep(0.1)
if process.poll() != 0 or recovered is None or recovered > 10:
raise RuntimeError("Second API failure did not recover within 10 seconds")
if len(after_instances) < 5 or target in after_instances[-5:]:
raise RuntimeError("Stopped API still serves requests")
replayed = c.message(alice, chat, replay_text, key=replay_key)
if replayed["json"] != original or replayed["headers"]["X-Instance-Id"] == target:
raise RuntimeError("Message replay through the survivor changed the result")
messages = history(c, bob, chat)
ids = [item["id"] for item in messages]
texts = [item["text"] for item in messages]
if len(ids) != len(set(ids)) or len(texts) != len(set(texts)):
raise RuntimeError("Duplicates appeared during failover")
if any(item not in messages for item in [original, *confirmations]):
raise RuntimeError("A confirmed message was lost")
second = json.loads((ROOT / "evidence/stop-one.json").read_text(encoding="utf-8"))
if second["service"] != other_service:
raise RuntimeError("Wrong second API was stopped")
second["metrics"] = {"successful_writes": len(confirmations), "failed_attempts": failures,
"first_recovery_seconds": round(recovered, 3),
"replay_on_survivor": "PASS"}
result = {"public": first, "second_replica": second, "status": "PASS"}
evidence("failover.json", result)
say("PASS: both physical API replicas stopped separately; no lost messages or duplicates")
return result
finally:
if process.poll() is None:
process.terminate()
process.wait(timeout=5)
restore(args)
def outage_response(args, path, token=None):
from urllib.error import HTTPError
from urllib.request import Request
from client import shape
headers = {"Authorization": "Bearer " + token} if token else {}
try:
response = LOCAL_HTTP.open(Request(base_url(args) + path, headers=headers), timeout=10)
except HTTPError as error:
response = error
with response:
payload = json.load(response)
if response.status != 503 or not response.headers.get("X-Request-Id"):
raise RuntimeError("Unexpected dependency failure response")
shape(payload, "Error")
if payload["error"]["request_id"] != response.headers["X-Request-Id"]:
raise RuntimeError("LB error request ID mismatch")
def attachment_failover_check(args):
import hashlib
results = []
raw = (ROOT / "tests/fixtures/sample.png").read_bytes()
try:
for service in ("api1", "api2"):
mapping = wait_replicas(args)
c = client(args)
alice, bob, eve, chat = c.bootstrap()
target = mapping[service]["instance_id"]
for _ in range(8):
uploaded = c.call(
"POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=failover.png", 201,
actor=alice, raw=raw, headers={"Content-Type": "image/png"}, schema="Attachment",
)
if uploaded["headers"]["X-Instance-Id"] == target:
break
else:
raise RuntimeError("Selected upload replica was not observed")
attachment = uploaded["json"]
linked = c.message(alice, chat, "", attachments=[attachment["id"]])["json"]
hook = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh")
previous = os.environ.get("TARGET_INSTANCE_ID")
os.environ["TARGET_INSTANCE_ID"] = target
started = time.monotonic()
try:
run(args, [str(hook)])
finally:
if previous is None:
os.environ.pop("TARGET_INSTANCE_ID", None)
else:
os.environ["TARGET_INSTANCE_ID"] = previous
from client import Client
reads = Client(base_url(args), timeout=2)
errors = 0
while True:
try:
downloaded = reads.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)
history_response = reads.call(
"GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage",
)
break
except (AssertionError, OSError) as error:
if isinstance(error, AssertionError) and not any(
text in str(error) for text in ("received 502", "received 503", "connection/TLS failure")
):
raise
errors += 1
if time.monotonic() - started >= 10:
raise RuntimeError("File and history did not recover within 10 seconds") from None
time.sleep(0.1)
recovered = time.monotonic() - started
if downloaded["bytes"] != raw or hashlib.sha256(raw).hexdigest() != attachment["sha256"]:
raise RuntimeError("Original file changed after API failure")
if downloaded["headers"]["X-Instance-Id"] == target or recovered > 10:
raise RuntimeError("Upload replica was not replaced")
history = history_response["json"]["items"]
if history_response["headers"]["X-Instance-Id"] == target:
raise RuntimeError("Stopped API returned history")
if history != [linked]:
raise RuntimeError("Attached message changed during failover")
results.append({"stopped_service": service, "upload_instance": target,
"download_instance": downloaded["headers"]["X-Instance-Id"],
"recovery_seconds": round(recovered, 3), "transitional_errors": errors,
"same_original": True, "message_preserved": True})
restore(args)
result = {"status": "PASS", "cases": results}
evidence("attachment-failover.json", result)
say("PASS: stopped each uploader; other API downloaded the same original")
return result
finally:
restore(args)
def main():
saved = settings()
parser = argparse.ArgumentParser()
parser.add_argument("action", choices=["setup", "build", "start", "test", "restart", "stop-one",
"restart-check", "failover-check", "restore", "stop", "verify"])
parser.add_argument("--project", default=os.environ.get("LAB4_PROJECT", saved.get("COMPOSE_PROJECT_NAME", "kuznetsov-lab4")))
parser.add_argument("--port", type=int, default=int(os.environ.get("LAB4_PORT", saved.get("HTTP_PORT", "18104"))))
parser.add_argument("--ttl", type=int, default=int(os.environ.get("LAB4_TTL", saved.get("SESSION_TTL_SECONDS", "3600"))))
parser.add_argument("--test-port", type=int, default=18114)
args = parser.parse_args()
if args.project != "kuznetsov-lab4" and not args.project.startswith("kuznetsov-lab4-"):
parser.error("Use a dedicated Kuznetsov lab4 project")
if not 1 <= args.port <= 65535 or not 1 <= args.test_port <= 65534 or not 1 <= args.ttl <= 86400:
parser.error("Invalid port or session TTL")
if args.action in ("setup", "build", "start", "test", "restart-check", "failover-check"):
setup(args)
actions = {"setup": lambda item: None, "build": build, "start": start,
"test": lambda item: (contract_tests(item), api_tests(item)),
"restart": restart, "restart-check": restart_check, "verify": verify,
"stop-one": stop_one, "restore": restore, "failover-check": failover_check,
"stop": lambda item: compose(item, "stop", "--timeout", "10", "lb", "api1", "api2", "s3", "s3-storage", "db")}
actions[args.action](args)
if __name__ == "__main__":
try:
main()
except (RuntimeError, OSError, ValueError) as error:
print("FAIL: " + str(error), file=sys.stderr)
raise SystemExit(1)
+3
View File
@@ -0,0 +1,3 @@
@echo off
python "%~dp0lab.py" restart
exit /b %errorlevel%
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
TASK_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
exec python3 "$TASK_DIR/lab.py" restart
+3
View File
@@ -0,0 +1,3 @@
@echo off
python "%~dp0lab.py" stop-one
exit /b %errorlevel%
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?missing TARGET_INSTANCE_ID}"
TASK_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
exec python3 "$TASK_DIR/lab.py" stop-one