forked from mtusi2026/mtusi2026lab3
551 lines
18 KiB
Python
551 lines
18 KiB
Python
"""Воспроизведение замечаний к lab3. Только текущий локальный Compose-стенд.
|
|
Временные переименования, pause и замены контейнеров восстанавливаются в finally.
|
|
Секреты не печатаются. Все созданные пользователи и сообщения синтетические.
|
|
"""
|
|
|
|
import argparse
|
|
import base64
|
|
import http.client
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from urllib.parse import urlsplit
|
|
|
|
from replicas import (
|
|
ROOT,
|
|
compose,
|
|
containers,
|
|
describe,
|
|
docker,
|
|
instance_id,
|
|
wait_ready,
|
|
wait_visible,
|
|
)
|
|
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client
|
|
|
|
EVIDENCE = ROOT / "evidence"
|
|
|
|
|
|
def sql(query):
|
|
return subprocess.check_output(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"sh",
|
|
"-c",
|
|
'exec psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At',
|
|
],
|
|
cwd=ROOT,
|
|
input=query,
|
|
text=True,
|
|
timeout=15,
|
|
).strip()
|
|
|
|
|
|
def wire(method, path, actor=None, body=None, timeout=7):
|
|
url = urlsplit(os.environ.get("BASE_URL", "http://localhost:8080"))
|
|
headers = {"Content-Type": "application/json"}
|
|
if actor:
|
|
headers["Authorization"] = "Bearer " + actor["token"]
|
|
conn = http.client.HTTPConnection(url.hostname, url.port, timeout=timeout)
|
|
before = time.monotonic()
|
|
try:
|
|
conn.request(
|
|
method,
|
|
path,
|
|
body=None if body is None else json.dumps(body).encode(),
|
|
headers=headers,
|
|
)
|
|
response = conn.getresponse()
|
|
response.read()
|
|
return {
|
|
"status": response.status,
|
|
"seconds": round(time.monotonic() - before, 3),
|
|
"instance": response.getheader("X-Instance-Id"),
|
|
}
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def direct_ready(cid):
|
|
code = """import urllib.request,urllib.error
|
|
try: r=urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=6)
|
|
except urllib.error.HTTPError as e: r=e
|
|
with r: print(r.status)
|
|
"""
|
|
return int(docker("exec", cid, "python", "-c", code))
|
|
|
|
|
|
def restored():
|
|
for cid in containers():
|
|
wait_visible(wait_ready(cid))
|
|
|
|
|
|
def contract():
|
|
client = Client()
|
|
alice, bob, eve, chat = client.bootstrap()
|
|
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
|
for endpoint in ["/api/v1/chats", path]:
|
|
for query in [
|
|
"cursor=W10",
|
|
"cursor=bnVsbA",
|
|
"limit=abc",
|
|
"limit=0",
|
|
"limit=101",
|
|
]:
|
|
client.call("GET", endpoint + "?" + query, 400, actor=alice)
|
|
# Unicode Basic password including colon and no body on login.
|
|
registration = {
|
|
"username": "audit_utf_" + uuid.uuid4().hex[:12],
|
|
"display_name": "Unicode",
|
|
"password": "пароль:😀с-пробелом 123",
|
|
}
|
|
client.call("POST", "/api/v1/users", 201, body=registration, schema="User")
|
|
basic = (
|
|
"Basic "
|
|
+ base64.b64encode(
|
|
(registration["username"] + ":" + registration["password"]).encode()
|
|
).decode()
|
|
)
|
|
client.call(
|
|
"POST",
|
|
"/api/v1/auth/sessions",
|
|
201,
|
|
headers={"Authorization": basic},
|
|
schema="Session",
|
|
)
|
|
client.call(
|
|
"POST",
|
|
"/api/v1/auth/sessions",
|
|
400,
|
|
raw=b"{}",
|
|
headers={"Authorization": basic},
|
|
)
|
|
client.call(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
400,
|
|
actor=alice,
|
|
body={
|
|
"title": "Unknown member",
|
|
"member_ids": [alice["id"], str(uuid.uuid4())],
|
|
},
|
|
)
|
|
# Empty page limit=7 is warmed before the successful write.
|
|
assert not client.call("GET", path + "?limit=7", actor=alice, schema="MessagePage")[
|
|
"json"
|
|
]["items"]
|
|
key = uuid.uuid4().hex
|
|
barrier = threading.Barrier(16)
|
|
clients = [Client() for _ in range(16)]
|
|
message = "Начало\u0000конец 👋"
|
|
|
|
def send(local):
|
|
barrier.wait()
|
|
return local.message(alice, chat, message, key=key)
|
|
|
|
with ThreadPoolExecutor(max_workers=16) as pool:
|
|
responses = list(pool.map(send, clients))
|
|
original = responses[0]["json"]
|
|
assert all(r["json"] == original for r in responses)
|
|
observed = {r["headers"]["X-Instance-Id"] for r in responses}
|
|
assert len(observed) >= 2
|
|
assert int(sql(f"SELECT count(*) FROM messages WHERE chat_id='{chat['id']}'")) == 1
|
|
assert (
|
|
int(sql(f"SELECT count(*) FROM message_requests WHERE chat_id='{chat['id']}'"))
|
|
== 1
|
|
)
|
|
assert (
|
|
int(
|
|
sql(
|
|
f"SELECT position(decode('00','hex') IN text) FROM messages WHERE id='{original['id']}'"
|
|
)
|
|
)
|
|
> 0
|
|
)
|
|
for limit in [1, 7, 10, 20, 50, 100]:
|
|
result = client.call(
|
|
"GET", path + "?limit=" + str(limit), actor=bob, schema="MessagePage"
|
|
)["json"]
|
|
assert result["items"] == [original], "Stale history or lost U+0000"
|
|
client.call(
|
|
"POST",
|
|
path,
|
|
409,
|
|
actor=alice,
|
|
body={"text": "changed"},
|
|
headers={"Idempotency-Key": key},
|
|
)
|
|
chat2 = client.call(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
201,
|
|
actor=alice,
|
|
body={"title": "Other chat", "member_ids": [alice["id"]]},
|
|
schema="Chat",
|
|
)["json"]
|
|
client.message(alice, chat2, message, key=key)
|
|
client.message(bob, chat, message, key=key)
|
|
for bad in ["", "x" * 129, "bad key", "bad/key"]:
|
|
client.call(
|
|
"POST",
|
|
path,
|
|
400,
|
|
actor=alice,
|
|
body={"text": "bad key"},
|
|
headers={"Idempotency-Key": bad},
|
|
)
|
|
# Bind cursor to actual chat; limit=1 produces a nonempty cursor.
|
|
page = client.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")[
|
|
"json"
|
|
]
|
|
client.call(
|
|
"GET",
|
|
"/api/v1/chats/" + chat2["id"] + "/messages?cursor=" + page["next_cursor"],
|
|
400,
|
|
actor=alice,
|
|
)
|
|
large = client.call(
|
|
"POST",
|
|
path,
|
|
413,
|
|
actor=alice,
|
|
raw=b'{"text":"valid"}' + b" " * 70000,
|
|
headers={
|
|
"Content-Type": "application/json",
|
|
"Idempotency-Key": uuid.uuid4().hex,
|
|
},
|
|
)
|
|
assert large["headers"]["X-Instance-Id"] and large["headers"]["X-Request-Id"]
|
|
# A raw U+0000 is illegal JSON; escaped \u0000 is valid and must round-trip.
|
|
client.call(
|
|
"POST",
|
|
path,
|
|
400,
|
|
actor=alice,
|
|
raw=b'{"text":"bad\x00json"}',
|
|
headers={
|
|
"Content-Type": "application/json",
|
|
"Idempotency-Key": uuid.uuid4().hex,
|
|
},
|
|
)
|
|
client.call(
|
|
"POST",
|
|
path,
|
|
404,
|
|
actor=eve,
|
|
body={"text": message},
|
|
headers={"Idempotency-Key": key},
|
|
)
|
|
# Exercise boundary on Unicode code points, not UTF-8 byte count.
|
|
long_message = client.message(alice, chat, "\u0000" * 4000)["json"]
|
|
assert long_message["text"] == "\u0000" * 4000
|
|
replay = client.message(alice, chat, message, key=key)
|
|
assert replay["json"] == original
|
|
return {
|
|
"result": "PASS",
|
|
"simultaneous_requests": 16,
|
|
"observed_instances": sorted(observed),
|
|
"database_rows_for_first_key": 1,
|
|
"headers_413": "present",
|
|
"nul_roundtrip": "201, replay and history; 4000 code points supported",
|
|
"checks": [
|
|
"invalid cursor types/limit",
|
|
"UTF-8 Basic",
|
|
"login body rejected",
|
|
"unknown member 400",
|
|
"key scope/format",
|
|
"foreign cursor",
|
|
"fresh history limit=7 and other limits",
|
|
"outsider hidden",
|
|
"raw invalid JSON rejected",
|
|
],
|
|
}
|
|
|
|
|
|
def readiness():
|
|
client = Client()
|
|
actor = client.user("audit_ready")
|
|
evidence = []
|
|
for table in ["users", "chats", "chat_members", "messages", "message_requests"]:
|
|
hidden = "audit_hidden_" + table
|
|
try:
|
|
sql(f"ALTER TABLE {table} RENAME TO {hidden}")
|
|
statuses = {cid[:12]: direct_ready(cid) for cid in containers()}
|
|
assert set(statuses.values()) == {503}, statuses
|
|
deadline = time.monotonic() + 8
|
|
while wire("GET", "/api/v1/users/me", actor)["status"] != 503:
|
|
assert time.monotonic() < deadline, "LB still routes to unready API"
|
|
time.sleep(0.2)
|
|
results = []
|
|
for _ in range(8):
|
|
result = wire("GET", "/api/v1/users/me", actor)
|
|
assert result["status"] == 503, result
|
|
results.append(result)
|
|
registration = wire(
|
|
"POST",
|
|
"/api/v1/users",
|
|
body={
|
|
"username": "blocked_" + uuid.uuid4().hex[:12],
|
|
"display_name": "Must not route",
|
|
"password": "valid-audit-password-123",
|
|
},
|
|
)
|
|
assert registration["status"] == 503, registration
|
|
evidence.append(
|
|
{
|
|
"missing_table": table,
|
|
"direct_readiness": statuses,
|
|
"business_requests": results,
|
|
"registration": registration,
|
|
}
|
|
)
|
|
finally:
|
|
sql(f"ALTER TABLE IF EXISTS {hidden} RENAME TO {table}")
|
|
restored()
|
|
return {"result": "PASS", "experiments": evidence}
|
|
|
|
|
|
def frozen_stores():
|
|
client = Client()
|
|
actor = client.user("audit_pause")
|
|
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
evidence = []
|
|
for service in ["postgres", "redis"]:
|
|
try:
|
|
began = time.monotonic()
|
|
compose("pause", service)
|
|
transition = wire("GET", "/health/ready")
|
|
assert transition["status"] == 503, transition
|
|
assert transition["seconds"] < 7
|
|
# Allow active checks to exclude all API, then test the LB route,
|
|
# including registration that itself does not depend on Redis.
|
|
time.sleep(4)
|
|
result = wire("GET", "/api/v1/users/me", actor)
|
|
registration = wire(
|
|
"POST",
|
|
"/api/v1/users",
|
|
body={
|
|
"username": "paused_" + uuid.uuid4().hex[:12],
|
|
"display_name": "Paused store",
|
|
"password": "valid-audit-password-123",
|
|
},
|
|
)
|
|
assert result["status"] == registration["status"] == 503, (
|
|
result,
|
|
registration,
|
|
)
|
|
evidence.append(
|
|
{
|
|
"service": service,
|
|
"initial_ready_request": transition,
|
|
"business": result,
|
|
"registration": registration,
|
|
"elapsed_seconds": round(time.monotonic() - began, 3),
|
|
}
|
|
)
|
|
finally:
|
|
compose("unpause", service)
|
|
restored()
|
|
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
return {"result": "PASS", "experiments": evidence}
|
|
|
|
|
|
def changing_ips():
|
|
client = Client()
|
|
actor = client.user("audit_dns")
|
|
chat = client.call(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
201,
|
|
actor=actor,
|
|
body={"title": "Forced new IPs", "member_ids": [actor["id"]]},
|
|
schema="Chat",
|
|
)["json"]
|
|
originals = containers()
|
|
lb = compose("ps", "-q", "lb")
|
|
lb_started = describe(lb)["StartedAt"]
|
|
holders = []
|
|
events = []
|
|
occupied = []
|
|
try:
|
|
for old in originals:
|
|
networks = json.loads(
|
|
docker("inspect", "--format", "{{json .NetworkSettings.Networks}}", old)
|
|
)
|
|
assert len(networks) == 1
|
|
network, state = next(iter(networks.items()))
|
|
ip = state["IPAddress"]
|
|
alias = instance_id(old)
|
|
docker("stop", "--time", "10", old)
|
|
docker("rm", old)
|
|
holder = "lab3-audit-ip-" + uuid.uuid4().hex[:10]
|
|
docker(
|
|
"run",
|
|
"-d",
|
|
"--name",
|
|
holder,
|
|
"--network",
|
|
network,
|
|
"--ip",
|
|
ip,
|
|
"--cpus",
|
|
".05",
|
|
"--memory",
|
|
"32m",
|
|
"--read-only",
|
|
"--cap-drop",
|
|
"ALL",
|
|
"--entrypoint",
|
|
"python",
|
|
"mtusi-messenger:lab3",
|
|
"-c",
|
|
"import time; time.sleep(900)",
|
|
)
|
|
holders.append(holder)
|
|
occupied.append(ip)
|
|
before = set(containers())
|
|
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=2", "api")
|
|
added = set(containers()) - before
|
|
assert len(added) == 1
|
|
new = added.pop()
|
|
new_alias = wait_ready(new)
|
|
new_ip = docker(
|
|
"inspect",
|
|
"--format",
|
|
"{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}",
|
|
new,
|
|
)
|
|
assert new_ip not in occupied
|
|
began = time.monotonic()
|
|
wait_visible(new_alias, timeout=10)
|
|
message = client.message(actor, chat, "DNS " + new_alias)["json"]
|
|
page = client.call(
|
|
"GET",
|
|
"/api/v1/chats/" + chat["id"] + "/messages",
|
|
actor=actor,
|
|
schema="MessagePage",
|
|
)["json"]
|
|
assert message in page["items"]
|
|
events.append(
|
|
{
|
|
"old_container": old[:12],
|
|
"old_ip": ip,
|
|
"old_instance": alias,
|
|
"new_container": new[:12],
|
|
"new_ip": new_ip,
|
|
"new_instance": new_alias,
|
|
"discovery_after_ready_seconds": round(time.monotonic() - began, 3),
|
|
}
|
|
)
|
|
observed = {
|
|
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")[
|
|
"headers"
|
|
]["X-Instance-Id"]
|
|
for _ in range(40)
|
|
}
|
|
assert observed == {e["new_instance"] for e in events}, observed
|
|
assert describe(lb)["StartedAt"] == lb_started, "HAProxy was restarted"
|
|
return {
|
|
"result": "PASS",
|
|
"events": events,
|
|
"observed_instances_after": sorted(observed),
|
|
"lb_restarted": False,
|
|
"old_ips_kept_occupied_during_test": occupied,
|
|
}
|
|
finally:
|
|
for holder in holders:
|
|
docker("rm", "-f", holder)
|
|
compose("up", "-d", "--no-deps", "--wait", "api")
|
|
restored()
|
|
|
|
|
|
def limits_and_evidence():
|
|
limits = {}
|
|
for service in ["api", "postgres", "redis", "lb", "migration"]:
|
|
limits[service] = []
|
|
for cid in compose("ps", "--all", "-q", service).split():
|
|
state = json.loads(
|
|
docker("inspect", "--format", "{{json .HostConfig}}", cid)
|
|
)
|
|
record = {
|
|
"container": cid[:12],
|
|
"cpu": state["NanoCpus"] / 1e9,
|
|
"memory_bytes": state["Memory"],
|
|
}
|
|
assert record["cpu"] > 0 and record["memory_bytes"] > 0
|
|
if service == "api":
|
|
assert not state["PortBindings"]
|
|
limits[service].append(record)
|
|
rows = json.loads((EVIDENCE / "lab3-research/results.json").read_text())
|
|
for series in ["fixed", "growing"]:
|
|
for count in [1, 2, 4]:
|
|
assert any(
|
|
r["job"]["series"] == series and r["job"]["replicas"] == count
|
|
for r in rows
|
|
)
|
|
fixed = [r for r in rows if r["job"]["series"] == "fixed"]
|
|
assert {r["job"]["replicas"] * r["job"]["cpus"] for r in fixed} == {2.0}
|
|
assert {r["job"]["replicas"] * int(r["job"]["memory"][:-1]) for r in fixed} == {
|
|
1024
|
|
}
|
|
assert any(r["job"]["series"] == "improved-db" for r in rows)
|
|
for name in ["results.json", "summary.csv", "scaling.png", "bottleneck.png"]:
|
|
assert (EVIDENCE / "lab3-research" / name).stat().st_size > 0
|
|
# The rolling command explicitly invokes Python and needs no executable bit.
|
|
assert "$(PYTHON) scripts/rolling-update.py" in (ROOT / "Makefile").read_text()
|
|
for file in ["scripts/restart.sh", "scripts/stop-one.sh"]:
|
|
assert (ROOT / file).stat().st_mode & 0o111
|
|
return {
|
|
"result": "PASS",
|
|
"limits": limits,
|
|
"fixed_api_budget": {"cpu": 2, "memory_mib": 1024},
|
|
"research_runs": len(rows),
|
|
"rolling_command": "python3 scripts/rolling-update.py",
|
|
"shell_hooks_executable": True,
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--phase", choices=["all", "contract", "infrastructure"], default="all"
|
|
)
|
|
args = parser.parse_args()
|
|
checks = [("contract", contract)] if args.phase == "contract" else []
|
|
if args.phase == "all":
|
|
checks.append(("contract", contract))
|
|
if args.phase in ["all", "infrastructure"]:
|
|
checks += [
|
|
("readiness", readiness),
|
|
("frozen-stores", frozen_stores),
|
|
("changing-ips", changing_ips),
|
|
("limits-evidence", limits_and_evidence),
|
|
]
|
|
summary = {}
|
|
for name, check in checks:
|
|
print("Checking " + name, flush=True)
|
|
result = check()
|
|
(EVIDENCE / ("lab3-audit-" + name + ".json")).write_text(
|
|
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
summary[name] = "PASS"
|
|
print(name + " PASS", flush=True)
|
|
(EVIDENCE / ("lab3-audit-summary-" + args.phase + ".json")).write_text(
|
|
json.dumps(summary, indent=2) + "\n"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|