"""Воспроизведение замечаний к lab3. Только текущий локальный Compose-стенд. Временные переименования, pause и замены контейнеров восстанавливаются в finally. Секреты не печатаются. Все созданные пользователи и сообщения синтетические. """ import argparse import base64 import http.client import json import os import subprocess import sys import threading import time import uuid from concurrent.futures import ThreadPoolExecutor from urllib.parse import urlsplit from replicas import ( ROOT, compose, containers, describe, docker, instance_id, wait_ready, wait_visible, ) sys.path.insert(0, str(ROOT / "tests")) from client import Client EVIDENCE = ROOT / "evidence" def sql(query): return subprocess.check_output( [ "docker", "compose", "exec", "-T", "postgres", "sh", "-c", 'exec psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At', ], cwd=ROOT, input=query, text=True, timeout=15, ).strip() def wire(method, path, actor=None, body=None, timeout=7): url = urlsplit(os.environ.get("BASE_URL", "http://localhost:8080")) headers = {"Content-Type": "application/json"} if actor: headers["Authorization"] = "Bearer " + actor["token"] conn = http.client.HTTPConnection(url.hostname, url.port, timeout=timeout) before = time.monotonic() try: conn.request( method, path, body=None if body is None else json.dumps(body).encode(), headers=headers, ) response = conn.getresponse() response.read() return { "status": response.status, "seconds": round(time.monotonic() - before, 3), "instance": response.getheader("X-Instance-Id"), } finally: conn.close() def direct_ready(cid): code = """import urllib.request,urllib.error try: r=urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=6) except urllib.error.HTTPError as e: r=e with r: print(r.status) """ return int(docker("exec", cid, "python", "-c", code)) def restored(): for cid in containers(): wait_visible(wait_ready(cid)) def contract(): client = Client() alice, bob, eve, chat = client.bootstrap() path = "/api/v1/chats/" + chat["id"] + "/messages" for endpoint in ["/api/v1/chats", path]: for query in [ "cursor=W10", "cursor=bnVsbA", "limit=abc", "limit=0", "limit=101", ]: client.call("GET", endpoint + "?" + query, 400, actor=alice) # Unicode Basic password including colon and no body on login. registration = { "username": "audit_utf_" + uuid.uuid4().hex[:12], "display_name": "Unicode", "password": "пароль:😀с-пробелом 123", } client.call("POST", "/api/v1/users", 201, body=registration, schema="User") basic = ( "Basic " + base64.b64encode( (registration["username"] + ":" + registration["password"]).encode() ).decode() ) client.call( "POST", "/api/v1/auth/sessions", 201, headers={"Authorization": basic}, schema="Session", ) client.call( "POST", "/api/v1/auth/sessions", 400, raw=b"{}", headers={"Authorization": basic}, ) client.call( "POST", "/api/v1/chats", 400, actor=alice, body={ "title": "Unknown member", "member_ids": [alice["id"], str(uuid.uuid4())], }, ) # Empty page limit=7 is warmed before the successful write. assert not client.call("GET", path + "?limit=7", actor=alice, schema="MessagePage")[ "json" ]["items"] key = uuid.uuid4().hex barrier = threading.Barrier(16) clients = [Client() for _ in range(16)] message = "Начало\u0000конец 👋" def send(local): barrier.wait() return local.message(alice, chat, message, key=key) with ThreadPoolExecutor(max_workers=16) as pool: responses = list(pool.map(send, clients)) original = responses[0]["json"] assert all(r["json"] == original for r in responses) observed = {r["headers"]["X-Instance-Id"] for r in responses} assert len(observed) >= 2 assert int(sql(f"SELECT count(*) FROM messages WHERE chat_id='{chat['id']}'")) == 1 assert ( int(sql(f"SELECT count(*) FROM message_requests WHERE chat_id='{chat['id']}'")) == 1 ) assert ( int( sql( f"SELECT position(decode('00','hex') IN text) FROM messages WHERE id='{original['id']}'" ) ) > 0 ) for limit in [1, 7, 10, 20, 50, 100]: result = client.call( "GET", path + "?limit=" + str(limit), actor=bob, schema="MessagePage" )["json"] assert result["items"] == [original], "Stale history or lost U+0000" client.call( "POST", path, 409, actor=alice, body={"text": "changed"}, headers={"Idempotency-Key": key}, ) chat2 = client.call( "POST", "/api/v1/chats", 201, actor=alice, body={"title": "Other chat", "member_ids": [alice["id"]]}, schema="Chat", )["json"] client.message(alice, chat2, message, key=key) client.message(bob, chat, message, key=key) for bad in ["", "x" * 129, "bad key", "bad/key"]: client.call( "POST", path, 400, actor=alice, body={"text": "bad key"}, headers={"Idempotency-Key": bad}, ) # Bind cursor to actual chat; limit=1 produces a nonempty cursor. page = client.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")[ "json" ] client.call( "GET", "/api/v1/chats/" + chat2["id"] + "/messages?cursor=" + page["next_cursor"], 400, actor=alice, ) large = client.call( "POST", path, 413, actor=alice, raw=b'{"text":"valid"}' + b" " * 70000, headers={ "Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex, }, ) assert large["headers"]["X-Instance-Id"] and large["headers"]["X-Request-Id"] # A raw U+0000 is illegal JSON; escaped \u0000 is valid and must round-trip. client.call( "POST", path, 400, actor=alice, raw=b'{"text":"bad\x00json"}', headers={ "Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex, }, ) client.call( "POST", path, 404, actor=eve, body={"text": message}, headers={"Idempotency-Key": key}, ) # Exercise boundary on Unicode code points, not UTF-8 byte count. long_message = client.message(alice, chat, "\u0000" * 4000)["json"] assert long_message["text"] == "\u0000" * 4000 replay = client.message(alice, chat, message, key=key) assert replay["json"] == original return { "result": "PASS", "simultaneous_requests": 16, "observed_instances": sorted(observed), "database_rows_for_first_key": 1, "headers_413": "present", "nul_roundtrip": "201, replay and history; 4000 code points supported", "checks": [ "invalid cursor types/limit", "UTF-8 Basic", "login body rejected", "unknown member 400", "key scope/format", "foreign cursor", "fresh history limit=7 and other limits", "outsider hidden", "raw invalid JSON rejected", ], } def readiness(): client = Client() actor = client.user("audit_ready") evidence = [] for table in ["users", "chats", "chat_members", "messages", "message_requests"]: hidden = "audit_hidden_" + table try: sql(f"ALTER TABLE {table} RENAME TO {hidden}") statuses = {cid[:12]: direct_ready(cid) for cid in containers()} assert set(statuses.values()) == {503}, statuses deadline = time.monotonic() + 8 while wire("GET", "/api/v1/users/me", actor)["status"] != 503: assert time.monotonic() < deadline, "LB still routes to unready API" time.sleep(0.2) results = [] for _ in range(8): result = wire("GET", "/api/v1/users/me", actor) assert result["status"] == 503, result results.append(result) registration = wire( "POST", "/api/v1/users", body={ "username": "blocked_" + uuid.uuid4().hex[:12], "display_name": "Must not route", "password": "valid-audit-password-123", }, ) assert registration["status"] == 503, registration evidence.append( { "missing_table": table, "direct_readiness": statuses, "business_requests": results, "registration": registration, } ) finally: sql(f"ALTER TABLE IF EXISTS {hidden} RENAME TO {table}") restored() return {"result": "PASS", "experiments": evidence} def frozen_stores(): client = Client() actor = client.user("audit_pause") client.call("GET", "/api/v1/users/me", actor=actor, schema="User") evidence = [] for service in ["postgres", "redis"]: try: began = time.monotonic() compose("pause", service) transition = wire("GET", "/health/ready") assert transition["status"] == 503, transition assert transition["seconds"] < 7 # Allow active checks to exclude all API, then test the LB route, # including registration that itself does not depend on Redis. time.sleep(4) result = wire("GET", "/api/v1/users/me", actor) registration = wire( "POST", "/api/v1/users", body={ "username": "paused_" + uuid.uuid4().hex[:12], "display_name": "Paused store", "password": "valid-audit-password-123", }, ) assert result["status"] == registration["status"] == 503, ( result, registration, ) evidence.append( { "service": service, "initial_ready_request": transition, "business": result, "registration": registration, "elapsed_seconds": round(time.monotonic() - began, 3), } ) finally: compose("unpause", service) restored() client.call("GET", "/api/v1/users/me", actor=actor, schema="User") return {"result": "PASS", "experiments": evidence} def changing_ips(): client = Client() actor = client.user("audit_dns") chat = client.call( "POST", "/api/v1/chats", 201, actor=actor, body={"title": "Forced new IPs", "member_ids": [actor["id"]]}, schema="Chat", )["json"] originals = containers() lb = compose("ps", "-q", "lb") lb_started = describe(lb)["StartedAt"] holders = [] events = [] occupied = [] try: for old in originals: networks = json.loads( docker("inspect", "--format", "{{json .NetworkSettings.Networks}}", old) ) assert len(networks) == 1 network, state = next(iter(networks.items())) ip = state["IPAddress"] alias = instance_id(old) docker("stop", "--time", "10", old) docker("rm", old) holder = "lab3-audit-ip-" + uuid.uuid4().hex[:10] docker( "run", "-d", "--name", holder, "--network", network, "--ip", ip, "--cpus", ".05", "--memory", "32m", "--read-only", "--cap-drop", "ALL", "--entrypoint", "python", "mtusi-messenger:lab3", "-c", "import time; time.sleep(900)", ) holders.append(holder) occupied.append(ip) before = set(containers()) compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=2", "api") added = set(containers()) - before assert len(added) == 1 new = added.pop() new_alias = wait_ready(new) new_ip = docker( "inspect", "--format", "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", new, ) assert new_ip not in occupied began = time.monotonic() wait_visible(new_alias, timeout=10) message = client.message(actor, chat, "DNS " + new_alias)["json"] page = client.call( "GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=actor, schema="MessagePage", )["json"] assert message in page["items"] events.append( { "old_container": old[:12], "old_ip": ip, "old_instance": alias, "new_container": new[:12], "new_ip": new_ip, "new_instance": new_alias, "discovery_after_ready_seconds": round(time.monotonic() - began, 3), } ) observed = { client.call("GET", "/api/v1/users/me", actor=actor, schema="User")[ "headers" ]["X-Instance-Id"] for _ in range(40) } assert observed == {e["new_instance"] for e in events}, observed assert describe(lb)["StartedAt"] == lb_started, "HAProxy was restarted" return { "result": "PASS", "events": events, "observed_instances_after": sorted(observed), "lb_restarted": False, "old_ips_kept_occupied_during_test": occupied, } finally: for holder in holders: docker("rm", "-f", holder) compose("up", "-d", "--no-deps", "--wait", "api") restored() def limits_and_evidence(): limits = {} for service in ["api", "postgres", "redis", "lb", "migration"]: limits[service] = [] for cid in compose("ps", "--all", "-q", service).split(): state = json.loads( docker("inspect", "--format", "{{json .HostConfig}}", cid) ) record = { "container": cid[:12], "cpu": state["NanoCpus"] / 1e9, "memory_bytes": state["Memory"], } assert record["cpu"] > 0 and record["memory_bytes"] > 0 if service == "api": assert not state["PortBindings"] limits[service].append(record) rows = json.loads((EVIDENCE / "lab3-research/results.json").read_text()) for series in ["fixed", "growing"]: for count in [1, 2, 4]: assert any( r["job"]["series"] == series and r["job"]["replicas"] == count for r in rows ) fixed = [r for r in rows if r["job"]["series"] == "fixed"] assert {r["job"]["replicas"] * r["job"]["cpus"] for r in fixed} == {2.0} assert {r["job"]["replicas"] * int(r["job"]["memory"][:-1]) for r in fixed} == { 1024 } assert any(r["job"]["series"] == "improved-db" for r in rows) for name in ["results.json", "summary.csv", "scaling.png", "bottleneck.png"]: assert (EVIDENCE / "lab3-research" / name).stat().st_size > 0 # The rolling command explicitly invokes Python and needs no executable bit. assert "$(PYTHON) scripts/rolling-update.py" in (ROOT / "Makefile").read_text() for file in ["scripts/restart.sh", "scripts/stop-one.sh"]: assert (ROOT / file).stat().st_mode & 0o111 return { "result": "PASS", "limits": limits, "fixed_api_budget": {"cpu": 2, "memory_mib": 1024}, "research_runs": len(rows), "rolling_command": "python3 scripts/rolling-update.py", "shell_hooks_executable": True, } def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--phase", choices=["all", "contract", "infrastructure"], default="all" ) args = parser.parse_args() checks = [("contract", contract)] if args.phase == "contract" else [] if args.phase == "all": checks.append(("contract", contract)) if args.phase in ["all", "infrastructure"]: checks += [ ("readiness", readiness), ("frozen-stores", frozen_stores), ("changing-ips", changing_ips), ("limits-evidence", limits_and_evidence), ] summary = {} for name, check in checks: print("Checking " + name, flush=True) result = check() (EVIDENCE / ("lab3-audit-" + name + ".json")).write_text( json.dumps(result, ensure_ascii=False, indent=2) + "\n" ) summary[name] = "PASS" print(name + " PASS", flush=True) (EVIDENCE / ("lab3-audit-summary-" + args.phase + ".json")).write_text( json.dumps(summary, indent=2) + "\n" ) if __name__ == "__main__": main()