forked from mtusi2026/mtusi2026lab3
готово
This commit is contained in:
@@ -0,0 +1,61 @@
|
||||
"""Проверка реально установленных Python-пакетов API через публичную базу OSV."""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import ssl
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
def fetch(url, data=None):
|
||||
request = Request(url, data=None if data is None else json.dumps(data).encode(),
|
||||
headers={'Content-Type': 'application/json'})
|
||||
with urlopen(request, timeout=30, context=TLS_CONTEXT) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', default='evidence/lab2-production-dependencies.json')
|
||||
parser.add_argument('--ca-file', help='Дополнительный доверенный CA bundle; TLS-проверка остаётся включённой')
|
||||
args = parser.parse_args()
|
||||
global TLS_CONTEXT
|
||||
TLS_CONTEXT = ssl.create_default_context(cafile=args.ca_file)
|
||||
raw = subprocess.check_output(['docker', 'compose', 'exec', '-T', 'api', 'python', '-c',
|
||||
'import importlib.metadata as m,json; print(json.dumps(sorted({d.metadata["Name"]:d.version for d in m.distributions()}.items())))'],
|
||||
text=True, timeout=20)
|
||||
packages = json.loads(raw)
|
||||
queries = [{'package': {'ecosystem': 'PyPI', 'name': name}, 'version': version}
|
||||
for name, version in packages]
|
||||
pending = list(enumerate(queries))
|
||||
vulnerabilities = {i: set() for i in range(len(packages))}
|
||||
while pending:
|
||||
results = fetch('https://api.osv.dev/v1/querybatch', {'queries': [q for _, q in pending]})['results']
|
||||
if len(results) != len(pending):
|
||||
raise RuntimeError('Неполный ответ OSV')
|
||||
next_pending = []
|
||||
for (index, query), result in zip(pending, results):
|
||||
vulnerabilities[index].update(v['id'] for v in result.get('vulns', []))
|
||||
if result.get('next_page_token'):
|
||||
next_pending.append((index, {**query, 'page_token': result['next_page_token']}))
|
||||
pending = next_pending
|
||||
ids = sorted(set().union(*vulnerabilities.values()))
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
records = dict(zip(ids, pool.map(lambda i: fetch('https://api.osv.dev/v1/vulns/' + i), ids)))
|
||||
findings = [{'name': packages[i][0], 'version': packages[i][1],
|
||||
'vulnerabilities': [records[v] for v in sorted(found)]}
|
||||
for i, found in vulnerabilities.items() if found]
|
||||
report = {'checked_at_utc': datetime.now(timezone.utc).isoformat(),
|
||||
'source': 'https://api.osv.dev/v1/querybatch', 'packages': packages, 'findings': findings,
|
||||
'scope': 'Python packages in running API; excludes OS/container image vulnerabilities'}
|
||||
Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
|
||||
print('OSV: проверено пакетов:', len(packages), '; пакетов с известными advisory:', len(findings))
|
||||
for finding in findings:
|
||||
print(finding['name'], finding['version'], ', '.join(v['id'] for v in finding['vulnerabilities']))
|
||||
raise SystemExit(bool(findings))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,550 @@
|
||||
"""Воспроизведение замечаний к lab3. Только текущий локальный Compose-стенд.
|
||||
Временные переименования, pause и замены контейнеров восстанавливаются в finally.
|
||||
Секреты не печатаются. Все созданные пользователи и сообщения синтетические.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from replicas import (
|
||||
ROOT,
|
||||
compose,
|
||||
containers,
|
||||
describe,
|
||||
docker,
|
||||
instance_id,
|
||||
wait_ready,
|
||||
wait_visible,
|
||||
)
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client
|
||||
|
||||
EVIDENCE = ROOT / "evidence"
|
||||
|
||||
|
||||
def sql(query):
|
||||
return subprocess.check_output(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"sh",
|
||||
"-c",
|
||||
'exec psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At',
|
||||
],
|
||||
cwd=ROOT,
|
||||
input=query,
|
||||
text=True,
|
||||
timeout=15,
|
||||
).strip()
|
||||
|
||||
|
||||
def wire(method, path, actor=None, body=None, timeout=7):
|
||||
url = urlsplit(os.environ.get("BASE_URL", "http://localhost:8080"))
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if actor:
|
||||
headers["Authorization"] = "Bearer " + actor["token"]
|
||||
conn = http.client.HTTPConnection(url.hostname, url.port, timeout=timeout)
|
||||
before = time.monotonic()
|
||||
try:
|
||||
conn.request(
|
||||
method,
|
||||
path,
|
||||
body=None if body is None else json.dumps(body).encode(),
|
||||
headers=headers,
|
||||
)
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
return {
|
||||
"status": response.status,
|
||||
"seconds": round(time.monotonic() - before, 3),
|
||||
"instance": response.getheader("X-Instance-Id"),
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def direct_ready(cid):
|
||||
code = """import urllib.request,urllib.error
|
||||
try: r=urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=6)
|
||||
except urllib.error.HTTPError as e: r=e
|
||||
with r: print(r.status)
|
||||
"""
|
||||
return int(docker("exec", cid, "python", "-c", code))
|
||||
|
||||
|
||||
def restored():
|
||||
for cid in containers():
|
||||
wait_visible(wait_ready(cid))
|
||||
|
||||
|
||||
def contract():
|
||||
client = Client()
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
for endpoint in ["/api/v1/chats", path]:
|
||||
for query in [
|
||||
"cursor=W10",
|
||||
"cursor=bnVsbA",
|
||||
"limit=abc",
|
||||
"limit=0",
|
||||
"limit=101",
|
||||
]:
|
||||
client.call("GET", endpoint + "?" + query, 400, actor=alice)
|
||||
# Unicode Basic password including colon and no body on login.
|
||||
registration = {
|
||||
"username": "audit_utf_" + uuid.uuid4().hex[:12],
|
||||
"display_name": "Unicode",
|
||||
"password": "пароль:😀с-пробелом 123",
|
||||
}
|
||||
client.call("POST", "/api/v1/users", 201, body=registration, schema="User")
|
||||
basic = (
|
||||
"Basic "
|
||||
+ base64.b64encode(
|
||||
(registration["username"] + ":" + registration["password"]).encode()
|
||||
).decode()
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
201,
|
||||
headers={"Authorization": basic},
|
||||
schema="Session",
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
400,
|
||||
raw=b"{}",
|
||||
headers={"Authorization": basic},
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
400,
|
||||
actor=alice,
|
||||
body={
|
||||
"title": "Unknown member",
|
||||
"member_ids": [alice["id"], str(uuid.uuid4())],
|
||||
},
|
||||
)
|
||||
# Empty page limit=7 is warmed before the successful write.
|
||||
assert not client.call("GET", path + "?limit=7", actor=alice, schema="MessagePage")[
|
||||
"json"
|
||||
]["items"]
|
||||
key = uuid.uuid4().hex
|
||||
barrier = threading.Barrier(16)
|
||||
clients = [Client() for _ in range(16)]
|
||||
message = "Начало\u0000конец 👋"
|
||||
|
||||
def send(local):
|
||||
barrier.wait()
|
||||
return local.message(alice, chat, message, key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||
responses = list(pool.map(send, clients))
|
||||
original = responses[0]["json"]
|
||||
assert all(r["json"] == original for r in responses)
|
||||
observed = {r["headers"]["X-Instance-Id"] for r in responses}
|
||||
assert len(observed) >= 2
|
||||
assert int(sql(f"SELECT count(*) FROM messages WHERE chat_id='{chat['id']}'")) == 1
|
||||
assert (
|
||||
int(sql(f"SELECT count(*) FROM message_requests WHERE chat_id='{chat['id']}'"))
|
||||
== 1
|
||||
)
|
||||
assert (
|
||||
int(
|
||||
sql(
|
||||
f"SELECT position(decode('00','hex') IN text) FROM messages WHERE id='{original['id']}'"
|
||||
)
|
||||
)
|
||||
> 0
|
||||
)
|
||||
for limit in [1, 7, 10, 20, 50, 100]:
|
||||
result = client.call(
|
||||
"GET", path + "?limit=" + str(limit), actor=bob, schema="MessagePage"
|
||||
)["json"]
|
||||
assert result["items"] == [original], "Stale history or lost U+0000"
|
||||
client.call(
|
||||
"POST",
|
||||
path,
|
||||
409,
|
||||
actor=alice,
|
||||
body={"text": "changed"},
|
||||
headers={"Idempotency-Key": key},
|
||||
)
|
||||
chat2 = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "Other chat", "member_ids": [alice["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
client.message(alice, chat2, message, key=key)
|
||||
client.message(bob, chat, message, key=key)
|
||||
for bad in ["", "x" * 129, "bad key", "bad/key"]:
|
||||
client.call(
|
||||
"POST",
|
||||
path,
|
||||
400,
|
||||
actor=alice,
|
||||
body={"text": "bad key"},
|
||||
headers={"Idempotency-Key": bad},
|
||||
)
|
||||
# Bind cursor to actual chat; limit=1 produces a nonempty cursor.
|
||||
page = client.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")[
|
||||
"json"
|
||||
]
|
||||
client.call(
|
||||
"GET",
|
||||
"/api/v1/chats/" + chat2["id"] + "/messages?cursor=" + page["next_cursor"],
|
||||
400,
|
||||
actor=alice,
|
||||
)
|
||||
large = client.call(
|
||||
"POST",
|
||||
path,
|
||||
413,
|
||||
actor=alice,
|
||||
raw=b'{"text":"valid"}' + b" " * 70000,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Idempotency-Key": uuid.uuid4().hex,
|
||||
},
|
||||
)
|
||||
assert large["headers"]["X-Instance-Id"] and large["headers"]["X-Request-Id"]
|
||||
# A raw U+0000 is illegal JSON; escaped \u0000 is valid and must round-trip.
|
||||
client.call(
|
||||
"POST",
|
||||
path,
|
||||
400,
|
||||
actor=alice,
|
||||
raw=b'{"text":"bad\x00json"}',
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Idempotency-Key": uuid.uuid4().hex,
|
||||
},
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
path,
|
||||
404,
|
||||
actor=eve,
|
||||
body={"text": message},
|
||||
headers={"Idempotency-Key": key},
|
||||
)
|
||||
# Exercise boundary on Unicode code points, not UTF-8 byte count.
|
||||
long_message = client.message(alice, chat, "\u0000" * 4000)["json"]
|
||||
assert long_message["text"] == "\u0000" * 4000
|
||||
replay = client.message(alice, chat, message, key=key)
|
||||
assert replay["json"] == original
|
||||
return {
|
||||
"result": "PASS",
|
||||
"simultaneous_requests": 16,
|
||||
"observed_instances": sorted(observed),
|
||||
"database_rows_for_first_key": 1,
|
||||
"headers_413": "present",
|
||||
"nul_roundtrip": "201, replay and history; 4000 code points supported",
|
||||
"checks": [
|
||||
"invalid cursor types/limit",
|
||||
"UTF-8 Basic",
|
||||
"login body rejected",
|
||||
"unknown member 400",
|
||||
"key scope/format",
|
||||
"foreign cursor",
|
||||
"fresh history limit=7 and other limits",
|
||||
"outsider hidden",
|
||||
"raw invalid JSON rejected",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def readiness():
|
||||
client = Client()
|
||||
actor = client.user("audit_ready")
|
||||
evidence = []
|
||||
for table in ["users", "chats", "chat_members", "messages", "message_requests"]:
|
||||
hidden = "audit_hidden_" + table
|
||||
try:
|
||||
sql(f"ALTER TABLE {table} RENAME TO {hidden}")
|
||||
statuses = {cid[:12]: direct_ready(cid) for cid in containers()}
|
||||
assert set(statuses.values()) == {503}, statuses
|
||||
deadline = time.monotonic() + 8
|
||||
while wire("GET", "/api/v1/users/me", actor)["status"] != 503:
|
||||
assert time.monotonic() < deadline, "LB still routes to unready API"
|
||||
time.sleep(0.2)
|
||||
results = []
|
||||
for _ in range(8):
|
||||
result = wire("GET", "/api/v1/users/me", actor)
|
||||
assert result["status"] == 503, result
|
||||
results.append(result)
|
||||
registration = wire(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
body={
|
||||
"username": "blocked_" + uuid.uuid4().hex[:12],
|
||||
"display_name": "Must not route",
|
||||
"password": "valid-audit-password-123",
|
||||
},
|
||||
)
|
||||
assert registration["status"] == 503, registration
|
||||
evidence.append(
|
||||
{
|
||||
"missing_table": table,
|
||||
"direct_readiness": statuses,
|
||||
"business_requests": results,
|
||||
"registration": registration,
|
||||
}
|
||||
)
|
||||
finally:
|
||||
sql(f"ALTER TABLE IF EXISTS {hidden} RENAME TO {table}")
|
||||
restored()
|
||||
return {"result": "PASS", "experiments": evidence}
|
||||
|
||||
|
||||
def frozen_stores():
|
||||
client = Client()
|
||||
actor = client.user("audit_pause")
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
evidence = []
|
||||
for service in ["postgres", "redis"]:
|
||||
try:
|
||||
began = time.monotonic()
|
||||
compose("pause", service)
|
||||
transition = wire("GET", "/health/ready")
|
||||
assert transition["status"] == 503, transition
|
||||
assert transition["seconds"] < 7
|
||||
# Allow active checks to exclude all API, then test the LB route,
|
||||
# including registration that itself does not depend on Redis.
|
||||
time.sleep(4)
|
||||
result = wire("GET", "/api/v1/users/me", actor)
|
||||
registration = wire(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
body={
|
||||
"username": "paused_" + uuid.uuid4().hex[:12],
|
||||
"display_name": "Paused store",
|
||||
"password": "valid-audit-password-123",
|
||||
},
|
||||
)
|
||||
assert result["status"] == registration["status"] == 503, (
|
||||
result,
|
||||
registration,
|
||||
)
|
||||
evidence.append(
|
||||
{
|
||||
"service": service,
|
||||
"initial_ready_request": transition,
|
||||
"business": result,
|
||||
"registration": registration,
|
||||
"elapsed_seconds": round(time.monotonic() - began, 3),
|
||||
}
|
||||
)
|
||||
finally:
|
||||
compose("unpause", service)
|
||||
restored()
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
return {"result": "PASS", "experiments": evidence}
|
||||
|
||||
|
||||
def changing_ips():
|
||||
client = Client()
|
||||
actor = client.user("audit_dns")
|
||||
chat = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=actor,
|
||||
body={"title": "Forced new IPs", "member_ids": [actor["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
originals = containers()
|
||||
lb = compose("ps", "-q", "lb")
|
||||
lb_started = describe(lb)["StartedAt"]
|
||||
holders = []
|
||||
events = []
|
||||
occupied = []
|
||||
try:
|
||||
for old in originals:
|
||||
networks = json.loads(
|
||||
docker("inspect", "--format", "{{json .NetworkSettings.Networks}}", old)
|
||||
)
|
||||
assert len(networks) == 1
|
||||
network, state = next(iter(networks.items()))
|
||||
ip = state["IPAddress"]
|
||||
alias = instance_id(old)
|
||||
docker("stop", "--time", "10", old)
|
||||
docker("rm", old)
|
||||
holder = "lab3-audit-ip-" + uuid.uuid4().hex[:10]
|
||||
docker(
|
||||
"run",
|
||||
"-d",
|
||||
"--name",
|
||||
holder,
|
||||
"--network",
|
||||
network,
|
||||
"--ip",
|
||||
ip,
|
||||
"--cpus",
|
||||
".05",
|
||||
"--memory",
|
||||
"32m",
|
||||
"--read-only",
|
||||
"--cap-drop",
|
||||
"ALL",
|
||||
"--entrypoint",
|
||||
"python",
|
||||
"mtusi-messenger:lab3",
|
||||
"-c",
|
||||
"import time; time.sleep(900)",
|
||||
)
|
||||
holders.append(holder)
|
||||
occupied.append(ip)
|
||||
before = set(containers())
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=2", "api")
|
||||
added = set(containers()) - before
|
||||
assert len(added) == 1
|
||||
new = added.pop()
|
||||
new_alias = wait_ready(new)
|
||||
new_ip = docker(
|
||||
"inspect",
|
||||
"--format",
|
||||
"{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}",
|
||||
new,
|
||||
)
|
||||
assert new_ip not in occupied
|
||||
began = time.monotonic()
|
||||
wait_visible(new_alias, timeout=10)
|
||||
message = client.message(actor, chat, "DNS " + new_alias)["json"]
|
||||
page = client.call(
|
||||
"GET",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||
actor=actor,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
assert message in page["items"]
|
||||
events.append(
|
||||
{
|
||||
"old_container": old[:12],
|
||||
"old_ip": ip,
|
||||
"old_instance": alias,
|
||||
"new_container": new[:12],
|
||||
"new_ip": new_ip,
|
||||
"new_instance": new_alias,
|
||||
"discovery_after_ready_seconds": round(time.monotonic() - began, 3),
|
||||
}
|
||||
)
|
||||
observed = {
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")[
|
||||
"headers"
|
||||
]["X-Instance-Id"]
|
||||
for _ in range(40)
|
||||
}
|
||||
assert observed == {e["new_instance"] for e in events}, observed
|
||||
assert describe(lb)["StartedAt"] == lb_started, "HAProxy was restarted"
|
||||
return {
|
||||
"result": "PASS",
|
||||
"events": events,
|
||||
"observed_instances_after": sorted(observed),
|
||||
"lb_restarted": False,
|
||||
"old_ips_kept_occupied_during_test": occupied,
|
||||
}
|
||||
finally:
|
||||
for holder in holders:
|
||||
docker("rm", "-f", holder)
|
||||
compose("up", "-d", "--no-deps", "--wait", "api")
|
||||
restored()
|
||||
|
||||
|
||||
def limits_and_evidence():
|
||||
limits = {}
|
||||
for service in ["api", "postgres", "redis", "lb", "migration"]:
|
||||
limits[service] = []
|
||||
for cid in compose("ps", "--all", "-q", service).split():
|
||||
state = json.loads(
|
||||
docker("inspect", "--format", "{{json .HostConfig}}", cid)
|
||||
)
|
||||
record = {
|
||||
"container": cid[:12],
|
||||
"cpu": state["NanoCpus"] / 1e9,
|
||||
"memory_bytes": state["Memory"],
|
||||
}
|
||||
assert record["cpu"] > 0 and record["memory_bytes"] > 0
|
||||
if service == "api":
|
||||
assert not state["PortBindings"]
|
||||
limits[service].append(record)
|
||||
rows = json.loads((EVIDENCE / "lab3-research/results.json").read_text())
|
||||
for series in ["fixed", "growing"]:
|
||||
for count in [1, 2, 4]:
|
||||
assert any(
|
||||
r["job"]["series"] == series and r["job"]["replicas"] == count
|
||||
for r in rows
|
||||
)
|
||||
fixed = [r for r in rows if r["job"]["series"] == "fixed"]
|
||||
assert {r["job"]["replicas"] * r["job"]["cpus"] for r in fixed} == {2.0}
|
||||
assert {r["job"]["replicas"] * int(r["job"]["memory"][:-1]) for r in fixed} == {
|
||||
1024
|
||||
}
|
||||
assert any(r["job"]["series"] == "improved-db" for r in rows)
|
||||
for name in ["results.json", "summary.csv", "scaling.png", "bottleneck.png"]:
|
||||
assert (EVIDENCE / "lab3-research" / name).stat().st_size > 0
|
||||
# The rolling command explicitly invokes Python and needs no executable bit.
|
||||
assert "$(PYTHON) scripts/rolling-update.py" in (ROOT / "Makefile").read_text()
|
||||
for file in ["scripts/restart.sh", "scripts/stop-one.sh"]:
|
||||
assert (ROOT / file).stat().st_mode & 0o111
|
||||
return {
|
||||
"result": "PASS",
|
||||
"limits": limits,
|
||||
"fixed_api_budget": {"cpu": 2, "memory_mib": 1024},
|
||||
"research_runs": len(rows),
|
||||
"rolling_command": "python3 scripts/rolling-update.py",
|
||||
"shell_hooks_executable": True,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--phase", choices=["all", "contract", "infrastructure"], default="all"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
checks = [("contract", contract)] if args.phase == "contract" else []
|
||||
if args.phase == "all":
|
||||
checks.append(("contract", contract))
|
||||
if args.phase in ["all", "infrastructure"]:
|
||||
checks += [
|
||||
("readiness", readiness),
|
||||
("frozen-stores", frozen_stores),
|
||||
("changing-ips", changing_ips),
|
||||
("limits-evidence", limits_and_evidence),
|
||||
]
|
||||
summary = {}
|
||||
for name, check in checks:
|
||||
print("Checking " + name, flush=True)
|
||||
result = check()
|
||||
(EVIDENCE / ("lab3-audit-" + name + ".json")).write_text(
|
||||
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
summary[name] = "PASS"
|
||||
print(name + " PASS", flush=True)
|
||||
(EVIDENCE / ("lab3-audit-summary-" + args.phase + ".json")).write_text(
|
||||
json.dumps(summary, indent=2) + "\n"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,167 @@
|
||||
"""Повторяемый closed-loop: постоянные HTTP-соединения, 80% POST / 20% GET.
|
||||
Не измеряет open-loop SLA. Ошибки и задержки всех попыток учитываются отдельно.
|
||||
"""
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from runtime_config import base_url
|
||||
|
||||
os.environ.setdefault("BASE_URL", base_url())
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client
|
||||
|
||||
|
||||
def fixture():
|
||||
c = Client(os.environ.get("BASE_URL"))
|
||||
actor = c.user("bench")
|
||||
chat = c.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=actor,
|
||||
body={"title": "Synthetic benchmark", "member_ids": [actor["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
for n in range(20):
|
||||
c.message(actor, chat, "seed-%d" % n)
|
||||
return actor, chat
|
||||
|
||||
|
||||
def measure(actor, chat, duration=15, concurrency=16, event=None):
|
||||
url = urlsplit(os.environ.get("BASE_URL", "http://localhost:8080"))
|
||||
assert url.scheme == "http", "Local isolated HTTP benchmark only"
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
barrier = threading.Barrier(concurrency + 1)
|
||||
start = [None]
|
||||
prefix = uuid.uuid4().hex
|
||||
|
||||
def worker(index):
|
||||
connection = http.client.HTTPConnection(url.hostname, url.port or 80, timeout=5)
|
||||
headers = {
|
||||
"Authorization": "Bearer " + actor["token"],
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
latencies, successes, failures, instances, seconds, confirmations = (
|
||||
[],
|
||||
Counter(),
|
||||
Counter(),
|
||||
set(),
|
||||
{},
|
||||
[],
|
||||
)
|
||||
completed = []
|
||||
counter = 0
|
||||
barrier.wait()
|
||||
while time.monotonic() < start[0] + duration:
|
||||
write = counter % 5 != 4
|
||||
key = "%s-%d-%d" % (prefix, index, counter)
|
||||
body = json.dumps({"text": "load-" + key}) if write else None
|
||||
before = time.monotonic()
|
||||
status = 0
|
||||
try:
|
||||
connection.request(
|
||||
"POST" if write else "GET",
|
||||
path if write else path + "?limit=20",
|
||||
body=body,
|
||||
headers={**headers, "Idempotency-Key": key},
|
||||
)
|
||||
response = connection.getresponse()
|
||||
data = response.read()
|
||||
status = response.status
|
||||
if status == (201 if write else 200):
|
||||
parsed = json.loads(data)
|
||||
successes["write" if write else "read"] += 1
|
||||
instances.add(response.getheader("X-Instance-Id"))
|
||||
completed.append((time.monotonic(), "write" if write else "read"))
|
||||
if write:
|
||||
confirmations.append((key, parsed["id"]))
|
||||
else:
|
||||
failures[str(status)] += 1
|
||||
except (OSError, http.client.HTTPException, ValueError):
|
||||
failures["connection"] += 1
|
||||
connection.close()
|
||||
elapsed = time.monotonic() - before
|
||||
latencies.append((elapsed, status == (201 if write else 200)))
|
||||
bucket = int(before - start[0])
|
||||
record = seconds.setdefault(bucket, {"success": 0, "error": 0})
|
||||
record["success" if status == (201 if write else 200) else "error"] += 1
|
||||
counter += 1
|
||||
connection.close()
|
||||
return (
|
||||
latencies,
|
||||
successes,
|
||||
failures,
|
||||
instances,
|
||||
seconds,
|
||||
confirmations,
|
||||
completed,
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
futures = [pool.submit(worker, i) for i in range(concurrency)]
|
||||
start[0] = time.monotonic()
|
||||
barrier.wait()
|
||||
results = [f.result() for f in futures]
|
||||
elapsed = time.monotonic() - start[0]
|
||||
latencies = sorted(v for r in results for v, _ in r[0])
|
||||
successful = sorted(v for r in results for v, ok in r[0] if ok)
|
||||
successes, failures, instances, seconds = Counter(), Counter(), set(), {}
|
||||
for _, ok, bad, seen, buckets, _, _ in results:
|
||||
successes.update(ok)
|
||||
failures.update(bad)
|
||||
instances.update(seen)
|
||||
for b, counts in buckets.items():
|
||||
dest = seconds.setdefault(b, {"success": 0, "error": 0})
|
||||
for k, v in counts.items():
|
||||
dest[k] += v
|
||||
|
||||
def percentiles(values):
|
||||
return {
|
||||
name: round(values[max(0, math.ceil(p * len(values)) - 1)] * 1000, 3)
|
||||
if values
|
||||
else None
|
||||
for name, p in [("p50", 0.5), ("p95", 0.95), ("p99", 0.99)]
|
||||
}
|
||||
|
||||
report = {
|
||||
"duration_seconds": duration,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"concurrency": concurrency,
|
||||
"successes": dict(successes),
|
||||
"errors": dict(failures),
|
||||
"rps": round(sum(successes.values()) / elapsed, 3),
|
||||
"all_attempt_latency_ms": percentiles(latencies),
|
||||
"successful_latency_ms": percentiles(successful),
|
||||
"instances": sorted(instances),
|
||||
"seconds": seconds,
|
||||
"profile": "closed-loop, keepalive, 80% unique POST/20% GET first 20, no retries",
|
||||
}
|
||||
if event:
|
||||
report["first_success_after_action_seconds"] = {
|
||||
kind: round(
|
||||
min(
|
||||
(
|
||||
at - event[0]
|
||||
for r in results
|
||||
for at, method in r[6]
|
||||
if method == kind and at >= event[0]
|
||||
),
|
||||
default=duration,
|
||||
),
|
||||
3,
|
||||
)
|
||||
for kind in ["read", "write"]
|
||||
}
|
||||
return report, [x for r in results for x in r[5]]
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Проверка реализации в новом клоне: контракт, restart и отказ одной API."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from replicas import ROOT, compose, containers, describe, wait_ready, wait_visible
|
||||
|
||||
|
||||
def check(name, *command):
|
||||
result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True)
|
||||
(ROOT / "evidence" / ("lab3-transfer-" + name + ".txt")).write_text(
|
||||
result.stdout + result.stderr
|
||||
)
|
||||
print(name, "PASS" if result.returncode == 0 else "FAIL", flush=True)
|
||||
if result.returncode:
|
||||
raise RuntimeError((result.stdout + result.stderr)[-2000:])
|
||||
|
||||
|
||||
check(
|
||||
"persistence",
|
||||
sys.executable,
|
||||
"tests/resilience.py",
|
||||
"--mode",
|
||||
"persistence",
|
||||
"--action-script",
|
||||
"scripts/restart.sh",
|
||||
)
|
||||
try:
|
||||
check(
|
||||
"failover",
|
||||
sys.executable,
|
||||
"tests/resilience.py",
|
||||
"--mode",
|
||||
"failover",
|
||||
"--action-script",
|
||||
"scripts/stop-one.sh",
|
||||
)
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "--wait", "api")
|
||||
for cid in containers():
|
||||
wait_visible(wait_ready(cid))
|
||||
check("final-smoke", sys.executable, "tests/smoke.py")
|
||||
logs = compose("logs", "--no-color", "migration")
|
||||
(ROOT / "evidence/lab3-transfer-migrations.txt").write_text(logs + "\n")
|
||||
proof = {
|
||||
"project": "mtusi2026lab3",
|
||||
"schema_revision": "006",
|
||||
"api_instances": len(containers()),
|
||||
"api_states": {
|
||||
cid[:12]: {
|
||||
"status": describe(cid)["Status"],
|
||||
"health": describe(cid).get("Health", {}).get("Status"),
|
||||
}
|
||||
for cid in containers()
|
||||
},
|
||||
"public_sha256": {
|
||||
name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest()
|
||||
for name in [
|
||||
"contracts/openapi.json",
|
||||
"tests/client.py",
|
||||
"tests/smoke.py",
|
||||
"tests/resilience.py",
|
||||
"tests/load.py",
|
||||
]
|
||||
},
|
||||
}
|
||||
assert proof["api_instances"] == 2
|
||||
(ROOT / "evidence/lab3-transfer-runtime.json").write_text(
|
||||
json.dumps(proof, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print("PASS: transferred application retains data/session and survives API failover", flush=True)
|
||||
@@ -0,0 +1,344 @@
|
||||
"""Дополнительный adversarial-аудит: неоднозначность, Unicode, гонки, rollback.
|
||||
Только синтетические данные. Временный DB-trigger действует на один случайный
|
||||
ключ и удаляется в finally. Повреждённый кэш восстанавливается из PostgreSQL.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import urlencode, urlsplit
|
||||
|
||||
from replicas import ROOT
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client
|
||||
|
||||
|
||||
def sql(query):
|
||||
return subprocess.check_output(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"sh",
|
||||
"-c",
|
||||
'exec psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At',
|
||||
],
|
||||
cwd=ROOT,
|
||||
input=query,
|
||||
text=True,
|
||||
timeout=15,
|
||||
).strip()
|
||||
|
||||
|
||||
def redis(*args, raw=None):
|
||||
command = ["docker", "compose", "exec", "-T", "redis", "redis-cli", "--raw"]
|
||||
if raw is not None:
|
||||
command += ["-x"]
|
||||
return subprocess.check_output(
|
||||
command + list(args), cwd=ROOT, input=raw, timeout=10
|
||||
).rstrip(b"\n")
|
||||
|
||||
|
||||
def duplicate_headers(path, actor, pairs):
|
||||
body = b'{"text":"ambiguous headers"}'
|
||||
conn = http.client.HTTPConnection(
|
||||
urlsplit(os.environ.get("BASE_URL", "http://localhost:8080")).hostname,
|
||||
urlsplit(os.environ.get("BASE_URL", "http://localhost:8080")).port or 80,
|
||||
timeout=5,
|
||||
)
|
||||
try:
|
||||
conn.putrequest("POST", path)
|
||||
for name, value in [
|
||||
("Authorization", "Bearer " + actor["token"]),
|
||||
("Content-Length", str(len(body))),
|
||||
*pairs,
|
||||
]:
|
||||
conn.putheader(name, value)
|
||||
conn.endheaders(body)
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
assert response.status == 400, response.status
|
||||
assert response.getheader("X-Instance-Id") and response.getheader(
|
||||
"X-Request-Id"
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def main():
|
||||
client = Client()
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
duplicate_headers(
|
||||
path,
|
||||
alice,
|
||||
[
|
||||
("Content-Type", "application/json"),
|
||||
("Idempotency-Key", uuid.uuid4().hex),
|
||||
("Idempotency-Key", uuid.uuid4().hex),
|
||||
],
|
||||
)
|
||||
duplicate_headers(
|
||||
path,
|
||||
alice,
|
||||
[
|
||||
("Content-Type", "application/json"),
|
||||
("Content-Type", "text/plain"),
|
||||
("Idempotency-Key", uuid.uuid4().hex),
|
||||
],
|
||||
)
|
||||
for endpoint in ["/api/v1/chats", path]:
|
||||
for val in ["1.0", "1e0", "+1", " 1 ", "١", "true", "-1", "101"]:
|
||||
client.call(
|
||||
"GET", endpoint + "?" + urlencode({"limit": val}), 400, actor=alice
|
||||
)
|
||||
for query in ["limit=1&limit=100", "cursor=W10&cursor=bnVsbA", "extra=1"]:
|
||||
client.call("GET", endpoint + "?" + query, 400, actor=alice)
|
||||
# Every Unicode scalar permitted by the contract is stored unchanged.
|
||||
special = "Имя\u0000😀"
|
||||
registration = {
|
||||
"username": "edge_" + uuid.uuid4().hex[:16],
|
||||
"display_name": special,
|
||||
"password": " пароль:\u0000😀с пробелами ",
|
||||
}
|
||||
registered = client.call(
|
||||
"POST", "/api/v1/users", 201, body=registration, schema="User"
|
||||
)["json"]
|
||||
assert registered["display_name"] == special
|
||||
basic = (
|
||||
"Basic "
|
||||
+ base64.b64encode(
|
||||
(registration["username"] + ":" + registration["password"]).encode()
|
||||
).decode()
|
||||
)
|
||||
session = client.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
201,
|
||||
headers={"Authorization": basic},
|
||||
schema="Session",
|
||||
)["json"]
|
||||
assert session["user"] == registered
|
||||
special_actor = {"id": registered["id"], "token": session["token"]}
|
||||
assert (
|
||||
client.call("GET", "/api/v1/users/me", actor=special_actor, schema="User")[
|
||||
"json"
|
||||
]
|
||||
== registered
|
||||
)
|
||||
title = "Чат\u0000😀"
|
||||
special_chat = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=special_actor,
|
||||
body={"title": title, "member_ids": [registered["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
assert special_chat["title"] == title
|
||||
assert (
|
||||
client.call(
|
||||
"GET",
|
||||
"/api/v1/chats/" + special_chat["id"],
|
||||
actor=special_actor,
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
== special_chat
|
||||
)
|
||||
maximum_name = {
|
||||
**registration,
|
||||
"username": "max_" + uuid.uuid4().hex[:16],
|
||||
"display_name": "😀" * 100,
|
||||
}
|
||||
assert (
|
||||
client.call("POST", "/api/v1/users", 201, body=maximum_name, schema="User")[
|
||||
"json"
|
||||
]["display_name"]
|
||||
== "😀" * 100
|
||||
)
|
||||
max_chat = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "😀" * 100, "member_ids": [alice["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
assert max_chat["title"] == "😀" * 100
|
||||
for value in ["\u0000", "😀" * 4000, "e\u0301", "\u200b", " Пробелы\nсохраняются "]:
|
||||
assert client.message(alice, chat, value)["json"]["text"] == value
|
||||
for value in ["😀" * 4001, "\u2003" * 4, None, 123, True, [], {}]:
|
||||
client.call(
|
||||
"POST",
|
||||
path,
|
||||
400,
|
||||
actor=alice,
|
||||
body={"text": value},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
same_key = uuid.uuid4().hex
|
||||
client.message(alice, chat, "é", key=same_key)
|
||||
client.call(
|
||||
"POST",
|
||||
path,
|
||||
409,
|
||||
actor=alice,
|
||||
body={"text": "e\u0301"},
|
||||
headers={"Idempotency-Key": same_key},
|
||||
)
|
||||
# Conflicting simultaneous content must commit exactly one of the bodies.
|
||||
race_chat = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "Conflicting race", "member_ids": [alice["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
race_path = "/api/v1/chats/" + race_chat["id"] + "/messages"
|
||||
race_key = uuid.uuid4().hex
|
||||
barrier = threading.Barrier(16)
|
||||
locals_ = [Client() for _ in range(16)]
|
||||
|
||||
def contender(i):
|
||||
barrier.wait()
|
||||
return locals_[i].call(
|
||||
"POST",
|
||||
race_path,
|
||||
[201, 409],
|
||||
actor=alice,
|
||||
body={"text": "A" if i % 2 else "B"},
|
||||
headers={"Idempotency-Key": race_key},
|
||||
schema="Message",
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||
results = list(pool.map(contender, range(16)))
|
||||
winners = [r["json"] for r in results if r["status"] == 201]
|
||||
assert len(winners) == 8 and all(m == winners[0] for m in winners)
|
||||
assert (
|
||||
int(sql(f"SELECT count(*) FROM messages WHERE chat_id='{race_chat['id']}'"))
|
||||
== 1
|
||||
)
|
||||
# Inject a failure exactly between the INSERT message and saving its result.
|
||||
fail_key = uuid.uuid4().hex
|
||||
trigger = "audit_fail_" + uuid.uuid4().hex[:10]
|
||||
fault_chat = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "Atomicity fault", "member_ids": [alice["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
fault_path = "/api/v1/chats/" + fault_chat["id"] + "/messages"
|
||||
try:
|
||||
sql(
|
||||
f"CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected audit failure'; END $$; CREATE TRIGGER {trigger} BEFORE UPDATE ON message_requests FOR EACH ROW WHEN (NEW.key='{fail_key}' AND NEW.message_id IS NOT NULL) EXECUTE FUNCTION {trigger}();"
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
fault_path,
|
||||
503,
|
||||
actor=alice,
|
||||
body={"text": "Retry after rollback"},
|
||||
headers={"Idempotency-Key": fail_key},
|
||||
)
|
||||
assert (
|
||||
int(
|
||||
sql(f"SELECT count(*) FROM messages WHERE chat_id='{fault_chat['id']}'")
|
||||
)
|
||||
== 0
|
||||
)
|
||||
assert (
|
||||
int(
|
||||
sql(
|
||||
f"SELECT count(*) FROM message_requests WHERE chat_id='{fault_chat['id']}'"
|
||||
)
|
||||
)
|
||||
== 0
|
||||
)
|
||||
finally:
|
||||
sql(
|
||||
f"DROP TRIGGER IF EXISTS {trigger} ON message_requests; DROP FUNCTION IF EXISTS {trigger}();"
|
||||
)
|
||||
client.message(alice, fault_chat, "Retry after rollback", key=fail_key)
|
||||
assert (
|
||||
int(sql(f"SELECT count(*) FROM messages WHERE chat_id='{fault_chat['id']}'"))
|
||||
== 1
|
||||
)
|
||||
# Corrupt-but-parseable cached values must not break the public API.
|
||||
profile_key = "user:v1:" + alice["id"]
|
||||
for field, value in [
|
||||
("display_name", "bad\ud800"),
|
||||
("created_at", "0001-01-01T00:00:00+23:59"),
|
||||
("created_at", "2026-11-06T12:00:00"),
|
||||
]:
|
||||
try:
|
||||
redis(
|
||||
"SET",
|
||||
profile_key,
|
||||
raw=json.dumps({**alice["user"], field: value}).encode(),
|
||||
)
|
||||
assert (
|
||||
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")[
|
||||
"json"
|
||||
]
|
||||
== alice["user"]
|
||||
)
|
||||
assert 0 < int(redis("TTL", profile_key)) <= 60
|
||||
finally:
|
||||
redis("DEL", profile_key)
|
||||
# Reuse one TCP connection: routing must not pin this user to one API.
|
||||
conn = http.client.HTTPConnection(
|
||||
urlsplit(os.environ.get("BASE_URL", "http://localhost:8080")).hostname,
|
||||
urlsplit(os.environ.get("BASE_URL", "http://localhost:8080")).port or 80,
|
||||
timeout=5,
|
||||
)
|
||||
instances = set()
|
||||
try:
|
||||
for _ in range(8):
|
||||
conn.request(
|
||||
"GET",
|
||||
"/api/v1/users/me",
|
||||
headers={"Authorization": "Bearer " + alice["token"]},
|
||||
)
|
||||
r = conn.getresponse()
|
||||
assert r.status == 200
|
||||
r.read()
|
||||
instances.add(r.getheader("X-Instance-Id"))
|
||||
finally:
|
||||
conn.close()
|
||||
assert len(instances) >= 2
|
||||
proof = {
|
||||
"result": "PASS",
|
||||
"checks": [
|
||||
"duplicate Idempotency-Key and Content-Type rejected",
|
||||
"strict integer and repeated query parameters",
|
||||
"NUL/emoji user and chat Unicode roundtrip",
|
||||
"100 code point names and 4000 code point messages",
|
||||
"no text trim or silent Unicode normalization",
|
||||
"16 conflicting simultaneous requests: 8 replayed winners, 8 conflicts, one DB row",
|
||||
"DB failure after message INSERT rolls back message and key; same key succeeds after recovery",
|
||||
"invalid cached Unicode/timezone falls back to DB with TTL",
|
||||
"one TCP connection reaches both API",
|
||||
],
|
||||
"keepalive_instances": sorted(instances),
|
||||
}
|
||||
(ROOT / "evidence/lab3-edge-audit.json").write_text(
|
||||
json.dumps(proof, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(json.dumps(proof, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Финальные регрессии и безопасный снимок инфраструктуры без Config.Env."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from replicas import compose, containers, docker
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEST = ROOT / "evidence"
|
||||
|
||||
|
||||
def check(name, command, stdin=None):
|
||||
result = subprocess.run(command, cwd=ROOT, input=stdin, text=True, capture_output=True)
|
||||
(DEST / ("lab3-final-" + name + ".txt")).write_text(result.stdout + result.stderr)
|
||||
print(name, "PASS" if result.returncode == 0 else "FAIL", flush=True)
|
||||
if result.returncode:
|
||||
raise RuntimeError((result.stdout + result.stderr)[-3000:])
|
||||
|
||||
|
||||
for name in ["smoke", "lab3", "regressions", "dependencies"]:
|
||||
check(name, [sys.executable, "tests/" + name + ".py"])
|
||||
for name in [
|
||||
"database",
|
||||
"passwords",
|
||||
"settings",
|
||||
"callbacks",
|
||||
"idempotency_db",
|
||||
"readiness",
|
||||
"error_headers",
|
||||
]:
|
||||
check(
|
||||
name,
|
||||
["docker", "exec", "-i", containers()[0], "python"],
|
||||
(ROOT / "tests" / (name + ".py")).read_text(),
|
||||
)
|
||||
|
||||
snapshot = {}
|
||||
all_ids = []
|
||||
for service in ["api", "lb", "postgres", "redis", "migration"]:
|
||||
ids = compose("ps", "--all", "-q", service).split()
|
||||
snapshot[service] = []
|
||||
for cid in ids:
|
||||
# Only explicitly selected infrastructure fields; credentials are excluded.
|
||||
obj = json.loads(docker("inspect", "--format", "{{json .}}", cid))
|
||||
state = obj["State"]
|
||||
record = {
|
||||
"id": cid[:12],
|
||||
"name": obj["Name"],
|
||||
"image_id": obj["Image"],
|
||||
"status": state["Status"],
|
||||
"exit_code": state["ExitCode"],
|
||||
"cpu_limit": obj["HostConfig"]["NanoCpus"] / 1e9,
|
||||
"memory_limit_bytes": obj["HostConfig"]["Memory"],
|
||||
"ports": obj["NetworkSettings"]["Ports"],
|
||||
"read_only_root": obj["HostConfig"]["ReadonlyRootfs"],
|
||||
}
|
||||
snapshot[service].append(record)
|
||||
assert record["cpu_limit"] > 0 and record["memory_limit_bytes"] > 0
|
||||
if service == "api":
|
||||
assert not any(record["ports"].values()), "API port is published"
|
||||
if state["Running"]:
|
||||
all_ids.append(cid)
|
||||
assert len(snapshot["api"]) == 2
|
||||
snapshot["post_test_usage"] = [
|
||||
json.loads(line)
|
||||
for line in docker("stats", "--no-stream", "--format", "{{json .}}", *all_ids).splitlines()
|
||||
]
|
||||
snapshot["note"] = (
|
||||
"CPU/memory usage snapshot after tests, not a peak load measurement; cgroup CPU deltas under load are in research results."
|
||||
)
|
||||
(DEST / "lab3-infrastructure.json").write_text(
|
||||
json.dumps(snapshot, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print("PASS: 2 API, internal ports, CPU/RAM limits on all containers", flush=True)
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Runtime API доступен только на loopback внутри контейнера HAProxy."""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from replicas import ROOT, compose, docker
|
||||
|
||||
|
||||
def runtime(command):
|
||||
container = compose("ps", "-q", "lb")
|
||||
return subprocess.check_output(
|
||||
["docker", "exec", "-i", container, "nc", "-w", "1", "127.0.0.1", "9999"],
|
||||
input=command + "\n",
|
||||
text=True,
|
||||
cwd=ROOT,
|
||||
timeout=5,
|
||||
).strip()
|
||||
|
||||
|
||||
def drain(container):
|
||||
address = docker(
|
||||
"inspect", "--format", "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", container
|
||||
)
|
||||
lines = runtime("show servers state api_pool").splitlines()
|
||||
header = next(line.lstrip("# ").split() for line in lines if line.startswith("#"))
|
||||
rows = [
|
||||
dict(zip(header, line.split()))
|
||||
for line in lines
|
||||
if line and not line.startswith("#") and line != "1"
|
||||
]
|
||||
names = [r["srv_name"] for r in rows if r.get("srv_addr") == address]
|
||||
if not names or any(re.fullmatch(r"api[0-9]+", n) is None for n in names):
|
||||
raise RuntimeError("Cannot map container to HAProxy backend")
|
||||
for name in names:
|
||||
result = runtime("set server api_pool/" + name + " state drain")
|
||||
if result:
|
||||
raise RuntimeError(result)
|
||||
deadline = time.monotonic() + 10
|
||||
while True:
|
||||
rows = list(csv.DictReader(io.StringIO(runtime("show stat").lstrip("# "))))
|
||||
active = sum(
|
||||
int(r["scur"]) for r in rows if r["pxname"] == "api_pool" and r["svname"] in names
|
||||
)
|
||||
if active == 0:
|
||||
return names
|
||||
if time.monotonic() > deadline:
|
||||
raise RuntimeError("Backend did not drain within 10s")
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def release(names):
|
||||
# Старый процесс уже остановлен. Слот возвращаем под контроль health checks,
|
||||
# но не объявляем здоровым принудительно: новая реплика должна пройти ready.
|
||||
for name in names:
|
||||
for setting in ["health down", "state ready"]:
|
||||
result = runtime("set server api_pool/" + name + " " + setting)
|
||||
if result:
|
||||
raise RuntimeError(result)
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Графики только из сохранённых исходных результатов; значения не сглаживаются."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
os.environ.setdefault("MPLCONFIGDIR", "/tmp/mtusi-lab3-matplotlib")
|
||||
os.environ.setdefault("XDG_CACHE_HOME", "/tmp/mtusi-lab3-cache")
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEST = ROOT / "evidence/lab3-research"
|
||||
rows = json.loads((DEST / "results.json").read_text())
|
||||
groups = defaultdict(list)
|
||||
for row in rows:
|
||||
job = row["job"]
|
||||
groups[(job["series"], job["replicas"], job["concurrency"])].append(row)
|
||||
plt.rcParams.update(
|
||||
{
|
||||
"font.family": "DejaVu Sans",
|
||||
"font.size": 11,
|
||||
"axes.spines.top": False,
|
||||
"axes.spines.right": False,
|
||||
"figure.dpi": 140,
|
||||
}
|
||||
)
|
||||
colors = {"fixed": "#315de0", "growing": "#139879"}
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 4.3), layout="constrained")
|
||||
for series, label in [
|
||||
("fixed", "Постоянные 2 CPU / 1 GiB на все API"),
|
||||
("growing", "1 CPU / 256 MiB на каждую API"),
|
||||
]:
|
||||
values = [[r["report"]["rps"] for r in groups[(series, n, 32)]] for n in [1, 2, 4]]
|
||||
axes[0].errorbar(
|
||||
[1, 2, 4],
|
||||
[statistics.mean(v) for v in values],
|
||||
yerr=[statistics.stdev(v) for v in values],
|
||||
marker="o",
|
||||
capsize=5,
|
||||
label=label,
|
||||
color=colors[series],
|
||||
)
|
||||
latency = [
|
||||
[r["report"]["all_attempt_latency_ms"]["p95"] for r in groups[(series, n, 32)]]
|
||||
for n in [1, 2, 4]
|
||||
]
|
||||
axes[1].plot(
|
||||
[1, 2, 4],
|
||||
[statistics.mean(v) for v in latency],
|
||||
marker="o",
|
||||
color=colors[series],
|
||||
label=label,
|
||||
)
|
||||
for ax in axes:
|
||||
ax.set_xlabel("Количество API-реплик")
|
||||
ax.set_xticks([1, 2, 4])
|
||||
ax.grid(alpha=0.2)
|
||||
axes[0].set_ylabel("Успешных запросов / с")
|
||||
axes[0].set_title("Масштабирование: среднее ± SD, 2 повтора")
|
||||
axes[1].set_ylabel("p95 всех попыток, мс")
|
||||
axes[1].set_title("Задержка при 32 клиентах")
|
||||
axes[0].legend(fontsize=8, loc="best")
|
||||
fig.suptitle("PostgreSQL: 0.25 CPU; профиль 80% POST / 20% GET", fontsize=12)
|
||||
fig.savefig(DEST / "scaling.png")
|
||||
fig.savefig(DEST / "scaling.svg")
|
||||
plt.close(fig)
|
||||
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 4.2), layout="constrained")
|
||||
concurrencies = [8, 32, 64]
|
||||
measurements = [
|
||||
groups[("growing", 4, 32)] if c == 32 else groups[("saturation", 4, c)] for c in concurrencies
|
||||
]
|
||||
axes[0].plot(
|
||||
concurrencies,
|
||||
[statistics.mean(r["report"]["rps"] for r in rs) for rs in measurements],
|
||||
marker="o",
|
||||
color="#315de0",
|
||||
)
|
||||
axes[0].set_xlabel("Параллельные клиенты")
|
||||
axes[0].set_ylabel("Успешных запросов / с")
|
||||
axes[0].set_title("4 API: рост очереди не даёт роста RPS")
|
||||
axes[0].grid(alpha=0.2)
|
||||
base = groups[("growing", 4, 32)]
|
||||
improved = groups[("improved-db", 4, 32)]
|
||||
means = [statistics.mean(r["report"]["rps"] for r in rs) for rs in [base, improved]]
|
||||
axes[1].bar(["БД: 0.25 CPU", "БД: 1 CPU"], means, color=["#778392", "#139879"], width=0.55)
|
||||
for i, m in enumerate(means):
|
||||
axes[1].text(i, m + max(means) * 0.02, f"{m:.1f}", ha="center")
|
||||
axes[1].set_ylim(0, max(means) * 1.18)
|
||||
axes[1].set_ylabel("Успешных запросов / с")
|
||||
axes[1].set_title("Проверка улучшения: 4 API, 32 клиента")
|
||||
fig.savefig(DEST / "bottleneck.png")
|
||||
fig.savefig(DEST / "bottleneck.svg")
|
||||
plt.close(fig)
|
||||
print("Saved scaling.png/svg and bottleneck.png/svg")
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Управление только API-контейнерами текущего Compose-проекта."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from runtime_config import base_url
|
||||
|
||||
os.environ.setdefault("BASE_URL", base_url())
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.check_output(["docker", *args], cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return docker("compose", "--env-file", os.environ.get("ENV_FILE", ".env"), *args)
|
||||
|
||||
|
||||
def containers():
|
||||
return compose("ps", "--all", "-q", "api").split()
|
||||
|
||||
|
||||
def describe(container):
|
||||
# Никогда не выводим Config.Env: там credentials.
|
||||
return json.loads(docker("inspect", "--format", "{{json .State}}", container))
|
||||
|
||||
|
||||
def instance_id(container):
|
||||
code = "import urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2); print(r.headers['X-Instance-Id'])"
|
||||
return docker("exec", container, "python", "-c", code)
|
||||
|
||||
|
||||
def wait_ready(container, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
state = describe(container)
|
||||
if state.get("Health", {}).get("Status") == "healthy":
|
||||
return instance_id(container)
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError("API container did not become healthy: " + container[:12])
|
||||
|
||||
|
||||
def wait_visible(instance, timeout=30):
|
||||
base = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/")
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(base + "/health/ready", timeout=2) as response:
|
||||
if response.headers.get("X-Instance-Id") == instance:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("New replica was not discovered by HAProxy")
|
||||
@@ -0,0 +1,294 @@
|
||||
"""Трек 1: фиксированные/растущие ресурсы, насыщение PostgreSQL, улучшение.
|
||||
Учебные данные добавляются; volumes не удаляются. По завершении восстанавливает
|
||||
обычную конфигурацию. Сводки не содержат сессионных токенов.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from benchmark import fixture, measure
|
||||
from replicas import compose, containers, docker, wait_ready, wait_visible
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEST = ROOT / "evidence/lab3-research"
|
||||
|
||||
|
||||
def cpu_stats():
|
||||
result = {}
|
||||
for service in ["api", "postgres", "redis", "lb"]:
|
||||
for cid in compose("ps", "-q", service).split():
|
||||
raw = docker("exec", cid, "cat", "/sys/fs/cgroup/cpu.stat")
|
||||
memory = docker(
|
||||
"exec", cid, "cat", "/sys/fs/cgroup/memory.current", "/sys/fs/cgroup/memory.peak"
|
||||
).split()
|
||||
result[cid] = {
|
||||
"service": service,
|
||||
"memory_current_bytes": int(memory[0]),
|
||||
"memory_peak_bytes": int(memory[1]),
|
||||
**{line.split()[0]: int(line.split()[1]) for line in raw.splitlines()},
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def pg_stats():
|
||||
# Не выводим env/password и не меняем данные.
|
||||
query = "SELECT json_build_object('commits',xact_commit,'rollbacks',xact_rollback,'blocks_read',blks_read,'blocks_hit',blks_hit,'temp_bytes',temp_bytes) FROM pg_stat_database WHERE datname=current_database()"
|
||||
result = subprocess.check_output(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
os.environ.get("ENV_FILE", ".env"),
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"sh",
|
||||
"-c",
|
||||
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At',
|
||||
],
|
||||
input=query,
|
||||
text=True,
|
||||
cwd=ROOT,
|
||||
)
|
||||
return json.loads(result)
|
||||
|
||||
|
||||
def configure(replicas, cpus, memory, db_cpus, pool=10):
|
||||
os.environ.update(
|
||||
API_REPLICAS=str(replicas),
|
||||
API_CPUS=str(cpus),
|
||||
API_MEMORY=memory,
|
||||
POSTGRES_CPUS=str(db_cpus),
|
||||
DATABASE_POOL_SIZE=str(pool),
|
||||
LOG_LEVEL="WARNING",
|
||||
)
|
||||
compose("up", "-d", "--no-build", "--wait")
|
||||
ids = containers()
|
||||
assert len(ids) == replicas
|
||||
for cid in ids:
|
||||
wait_visible(wait_ready(cid))
|
||||
|
||||
|
||||
def main():
|
||||
global DEST
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--full",
|
||||
action="store_true",
|
||||
help="Полная матрица 1/2/4; без флага короткая проверка ресурса БД",
|
||||
)
|
||||
parser.add_argument("--duration", type=int, default=12)
|
||||
parser.add_argument("--repeats", type=int, default=2)
|
||||
parser.add_argument("--concurrency", type=int, default=32)
|
||||
args = parser.parse_args()
|
||||
if not args.full:
|
||||
DEST = ROOT / "evidence/lab3-control"
|
||||
args.duration = min(args.duration, 8)
|
||||
args.repeats = 1
|
||||
DEST.mkdir(parents=True, exist_ok=True)
|
||||
machine = {
|
||||
"host": platform.platform(),
|
||||
"architecture": platform.machine(),
|
||||
"docker": json.loads(docker("info", "--format", "{{json .}}")),
|
||||
}
|
||||
# Keep only hardware/runtime fields. Docker Info also has registry/user info.
|
||||
machine["docker"] = {
|
||||
k: machine["docker"].get(k)
|
||||
for k in [
|
||||
"ServerVersion",
|
||||
"NCPU",
|
||||
"MemTotal",
|
||||
"KernelVersion",
|
||||
"OperatingSystem",
|
||||
"Architecture",
|
||||
]
|
||||
}
|
||||
machine["application"] = {
|
||||
"schema_revision": "006",
|
||||
"message_storage": "UTF-8 bytea",
|
||||
"source_sha256": {
|
||||
str(path.relative_to(ROOT)): hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
for path in sorted((ROOT / "src").rglob("*.py"))
|
||||
},
|
||||
"configuration_sha256": {
|
||||
name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest()
|
||||
for name in ["compose.yaml", "infra/haproxy.cfg", "scripts/benchmark.py"]
|
||||
},
|
||||
}
|
||||
machine["method"] = {
|
||||
"warmup_seconds": 3,
|
||||
"repeats": args.repeats,
|
||||
"duration_seconds": args.duration,
|
||||
"concurrency": args.concurrency,
|
||||
"profile": "80% POST unique keys, 20% GET first page limit20, 20 seed messages; fresh chat per run",
|
||||
"generator": "Python http.client, one persistent connection per thread, closed-loop, no retries",
|
||||
"limits": "PostgreSQL 0.25 CPU/512MiB baseline, LB 0.5CPU/128MiB, Redis 0.25CPU/128MiB",
|
||||
"randomization_seed": 20261106,
|
||||
}
|
||||
(DEST / "method.json").write_text(json.dumps(machine, ensure_ascii=False, indent=2) + "\n")
|
||||
jobs = []
|
||||
for repeat in range(args.repeats):
|
||||
for count in [1, 2, 4]:
|
||||
jobs += [
|
||||
dict(
|
||||
series="fixed",
|
||||
replicas=count,
|
||||
cpus=2 / count,
|
||||
memory=f"{1024 // count}m",
|
||||
db=0.25,
|
||||
concurrency=args.concurrency,
|
||||
repeat=repeat,
|
||||
),
|
||||
dict(
|
||||
series="growing",
|
||||
replicas=count,
|
||||
cpus=1,
|
||||
memory="256m",
|
||||
db=0.25,
|
||||
concurrency=args.concurrency,
|
||||
repeat=repeat,
|
||||
),
|
||||
]
|
||||
random.Random(20261106).shuffle(jobs)
|
||||
# Additional load sweep establishes a plateau; the final series tests a
|
||||
# hypothesis that allocating more CPU to PostgreSQL raises throughput.
|
||||
for repeat in range(args.repeats):
|
||||
for concurrency in [8, 64]:
|
||||
jobs.append(
|
||||
dict(
|
||||
series="saturation",
|
||||
replicas=4,
|
||||
cpus=1,
|
||||
memory="256m",
|
||||
db=0.25,
|
||||
concurrency=concurrency,
|
||||
repeat=repeat,
|
||||
)
|
||||
)
|
||||
jobs.append(
|
||||
dict(
|
||||
series="improved-db",
|
||||
replicas=4,
|
||||
cpus=1,
|
||||
memory="256m",
|
||||
db=1,
|
||||
concurrency=args.concurrency,
|
||||
repeat=repeat,
|
||||
)
|
||||
)
|
||||
if not args.full:
|
||||
jobs = [
|
||||
job
|
||||
for job in jobs
|
||||
if job["replicas"] == 4
|
||||
and job["series"] in {"growing", "improved-db"}
|
||||
and job["concurrency"] == args.concurrency
|
||||
]
|
||||
results = []
|
||||
try:
|
||||
for index, job in enumerate(jobs):
|
||||
print(f"[{index + 1}/{len(jobs)}] {job}", flush=True)
|
||||
configure(job["replicas"], job["cpus"], job["memory"], job["db"])
|
||||
actor, chat = fixture()
|
||||
measure(actor, chat, 3, job["concurrency"])
|
||||
before = cpu_stats()
|
||||
pg_before = pg_stats()
|
||||
began = time.monotonic()
|
||||
report, _ = measure(actor, chat, args.duration, job["concurrency"])
|
||||
elapsed = time.monotonic() - began
|
||||
pg_after = pg_stats()
|
||||
after = cpu_stats()
|
||||
cpu = {}
|
||||
for cid, end in after.items():
|
||||
if cid not in before:
|
||||
continue
|
||||
delta = {k: end[k] - before[cid][k] for k in end if k != "service"}
|
||||
cpu[cid[:12]] = {
|
||||
"service": end["service"],
|
||||
"memory_current_mib": round(end["memory_current_bytes"] / 1048576, 2),
|
||||
"memory_peak_lifetime_mib": round(end["memory_peak_bytes"] / 1048576, 2),
|
||||
"cpu_cores_used": round(delta["usage_usec"] / 1e6 / elapsed, 3),
|
||||
"throttled_seconds": round(delta.get("throttled_usec", 0) / 1e6, 3),
|
||||
"throttled_period_ratio": round(
|
||||
delta.get("nr_throttled", 0) / max(1, delta.get("nr_periods", 0)), 3
|
||||
),
|
||||
}
|
||||
row = {
|
||||
"job": job,
|
||||
"report": report,
|
||||
"cpu": cpu,
|
||||
"postgres_delta": {k: pg_after[k] - pg_before[k] for k in pg_before},
|
||||
}
|
||||
results.append(row)
|
||||
(DEST / f"run-{index + 1:02}.json").write_text(
|
||||
json.dumps(row, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(
|
||||
"RPS",
|
||||
report["rps"],
|
||||
"errors",
|
||||
report["errors"],
|
||||
"p95",
|
||||
report["all_attempt_latency_ms"]["p95"],
|
||||
flush=True,
|
||||
)
|
||||
(DEST / "results.json").write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n")
|
||||
with (DEST / "summary.csv").open("w") as f:
|
||||
fields = [
|
||||
"series",
|
||||
"replicas",
|
||||
"cpus",
|
||||
"memory",
|
||||
"db",
|
||||
"concurrency",
|
||||
"repeat",
|
||||
"rps",
|
||||
"p95_ms",
|
||||
"errors",
|
||||
"postgres_cores",
|
||||
]
|
||||
writer = csv.DictWriter(f, fieldnames=fields)
|
||||
writer.writeheader()
|
||||
for row in results:
|
||||
writer.writerow(
|
||||
{
|
||||
**row["job"],
|
||||
"rps": row["report"]["rps"],
|
||||
"p95_ms": row["report"]["all_attempt_latency_ms"]["p95"],
|
||||
"errors": sum(row["report"]["errors"].values()),
|
||||
"postgres_cores": sum(
|
||||
c["cpu_cores_used"]
|
||||
for c in row["cpu"].values()
|
||||
if c["service"] == "postgres"
|
||||
),
|
||||
}
|
||||
)
|
||||
print(
|
||||
"PASS: research series complete; inspect results before drawing conclusions", flush=True
|
||||
)
|
||||
finally:
|
||||
for key in [
|
||||
"API_REPLICAS",
|
||||
"API_CPUS",
|
||||
"API_MEMORY",
|
||||
"POSTGRES_CPUS",
|
||||
"DATABASE_POOL_SIZE",
|
||||
"LOG_LEVEL",
|
||||
]:
|
||||
os.environ.pop(key, None)
|
||||
compose("up", "-d", "--no-build", "--wait")
|
||||
for cid in containers():
|
||||
wait_visible(wait_ready(cid))
|
||||
print("Default two-replica configuration restored", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# --volumes не используется: PostgreSQL и сессии должны сохраниться.
|
||||
docker compose --env-file "${ENV_FILE:-.env}" restart postgres redis
|
||||
docker compose --env-file "${ENV_FILE:-.env}" up -d --wait --force-recreate api
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Полная проверка перед сдачей; инфраструктурные этапы строго последовательно."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--research", action="store_true", help="Повторить 18 измерений и построить графики"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
steps = [
|
||||
("audit-run", "scripts/audit-lab3.py"),
|
||||
("edge-audit", "scripts/edge-audit-lab3.py"),
|
||||
("verify", "scripts/verify-lab3.py"),
|
||||
("final-checks", "scripts/final-checks-lab3.py"),
|
||||
("clean-start", "scripts/test-clean-lab3.py"),
|
||||
]
|
||||
if args.research:
|
||||
steps += [
|
||||
("research-run", "scripts/research-lab3.py"),
|
||||
("research-plots", "scripts/plot-lab3.py"),
|
||||
]
|
||||
summary = {}
|
||||
for name, script in steps:
|
||||
print("Starting " + name, flush=True)
|
||||
with (ROOT / "evidence" / ("lab3-" + name + ".txt")).open("w") as output:
|
||||
result = subprocess.run(
|
||||
[sys.executable, script] + (["--full"] if name == "research-run" else []),
|
||||
cwd=ROOT,
|
||||
stdout=output,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
summary[name] = "PASS" if result.returncode == 0 else "FAIL"
|
||||
(ROOT / "evidence/lab3-review-summary.json").write_text(json.dumps(summary, indent=2) + "\n")
|
||||
print(name + " " + summary[name], flush=True)
|
||||
if result.returncode:
|
||||
raise SystemExit(result.returncode)
|
||||
print("PASS: complete pre-submission review", flush=True)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Compose rolling replacement с одной временной репликой и ожиданием LB."""
|
||||
|
||||
import json
|
||||
import time
|
||||
|
||||
from haproxy import drain, release
|
||||
from replicas import compose, containers, describe, docker, wait_ready, wait_visible
|
||||
|
||||
old = [c for c in containers() if describe(c)["Running"]]
|
||||
if len(old) < 2 or len(old) != len(containers()):
|
||||
raise SystemExit("Restore all API replicas first; need >=2")
|
||||
count = len(old)
|
||||
events = []
|
||||
# Build separately before starting this script. --no-recreate keeps old instances
|
||||
# alive even when the image has changed. Each new container uses the current image.
|
||||
for index, previous in enumerate(old):
|
||||
before = set(containers())
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={count + 1}", "api")
|
||||
added = set(containers()) - before
|
||||
assert len(added) == 1, "Expected one surge replica"
|
||||
replacement = added.pop()
|
||||
new_instance = wait_ready(replacement)
|
||||
wait_visible(new_instance)
|
||||
drained_slots = drain(previous)
|
||||
started = time.monotonic()
|
||||
docker("stop", "--time", "10", previous)
|
||||
state = describe(previous)
|
||||
assert state["ExitCode"] == 0, "API did not exit gracefully"
|
||||
docker("rm", previous)
|
||||
release(drained_slots)
|
||||
event = {
|
||||
"old_container": previous[:12],
|
||||
"new_container": replacement[:12],
|
||||
"new_instance": new_instance,
|
||||
"old_exit_code": state["ExitCode"],
|
||||
"drained_slots": drained_slots,
|
||||
"stop_seconds": round(time.monotonic() - started, 3),
|
||||
}
|
||||
events.append(event)
|
||||
assert len(containers()) == count
|
||||
print(
|
||||
json.dumps({"mode": "rolling replacement, one surge, same schema", "events": events}, indent=2)
|
||||
)
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Внешний адрес локального стенда: BASE_URL → APP_PORT → выбранный .env."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def base_url() -> str:
|
||||
if os.environ.get("BASE_URL"):
|
||||
return os.environ["BASE_URL"].rstrip("/")
|
||||
port = os.environ.get("APP_PORT")
|
||||
if not port:
|
||||
path = Path(os.environ.get("ENV_FILE", str(ROOT / ".env")))
|
||||
if not path.is_absolute():
|
||||
path = ROOT / path
|
||||
if path.is_file():
|
||||
for line in path.read_text().splitlines():
|
||||
key, separator, value = line.partition("=")
|
||||
if separator and key.strip() == "APP_PORT":
|
||||
port = value.split("#", 1)[0].strip().strip("\"'")
|
||||
break
|
||||
port = port or "8080"
|
||||
if not port.isascii() or not port.isdigit() or not 1 <= int(port) <= 65535:
|
||||
raise ValueError("APP_PORT должен быть номером порта от 1 до 65535")
|
||||
return "http://localhost:" + str(int(port))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(base_url())
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Сопоставить X-Instance-Id реальному контейнеру и оставить его остановленным."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
from replicas import containers, describe, docker, instance_id
|
||||
|
||||
target = os.environ["TARGET_INSTANCE_ID"]
|
||||
running = [c for c in containers() if describe(c)["Running"]]
|
||||
if len(running) < 2:
|
||||
raise SystemExit("Need at least two running API containers")
|
||||
for container in running:
|
||||
if instance_id(container) == target:
|
||||
before = describe(container)
|
||||
started = time.monotonic()
|
||||
docker("stop", "--time", "10", container)
|
||||
after = describe(container)
|
||||
assert not after["Running"]
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"container": container[:12],
|
||||
"instance_id": target,
|
||||
"started_at": before["StartedAt"],
|
||||
"finished_at": after["FinishedAt"],
|
||||
"stop_seconds": round(time.monotonic() - started, 3),
|
||||
"exit_code": after["ExitCode"],
|
||||
}
|
||||
)
|
||||
)
|
||||
break
|
||||
else:
|
||||
raise SystemExit("Requested instance not found in this Compose project")
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec "${PYTHON:-python3}" scripts/stop-one.py
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Запуск на пустых volumes в отдельном временном Compose-проекте.
|
||||
Текущий стенд и его volumes не удаляются. Секрет создаётся в TemporaryDirectory.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
project = "lab3-audit-clean-" + uuid.uuid4().hex[:8]
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
with TemporaryDirectory(prefix="lab3-clean-") as folder:
|
||||
env_file = Path(folder) / ".env"
|
||||
env_file.write_text(
|
||||
"POSTGRES_PASSWORD=" + secrets.token_hex(24) + "\nAPP_PORT=" + str(port) + "\n"
|
||||
)
|
||||
env = {
|
||||
**os.environ,
|
||||
"COMPOSE_PROJECT_NAME": project,
|
||||
"ENV_FILE": str(env_file),
|
||||
"APP_PORT": str(port),
|
||||
}
|
||||
command = ["docker", "compose", "--env-file", str(env_file)]
|
||||
|
||||
def run(*args):
|
||||
return subprocess.run(args, cwd=ROOT, env=env, text=True, check=True, capture_output=True)
|
||||
|
||||
try:
|
||||
run(*command, "up", "-d", "--no-build", "--wait")
|
||||
version = run(
|
||||
*command,
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"sh",
|
||||
"-c",
|
||||
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "SELECT version_num FROM alembic_version"',
|
||||
).stdout.strip()
|
||||
assert version == "006", version
|
||||
result = run(sys.executable, "tests/smoke.py", "--base-url", f"http://127.0.0.1:{port}")
|
||||
proof = {
|
||||
"result": "PASS",
|
||||
"project": project,
|
||||
"schema_revision": version,
|
||||
"public_smoke": result.stdout.strip(),
|
||||
"empty_volumes": True,
|
||||
"temporary_project_removed": True,
|
||||
}
|
||||
except subprocess.CalledProcessError as error:
|
||||
print((error.stdout + error.stderr)[-3000:])
|
||||
raise
|
||||
finally:
|
||||
run(*command, "down", "--volumes")
|
||||
(ROOT / "evidence/lab3-clean-start.json").write_text(
|
||||
json.dumps(proof, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(json.dumps(proof, ensure_ascii=False, indent=2))
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Сохраняет обычный TTL и восстанавливает API даже при падении проверки.
|
||||
restore() { docker compose up -d --wait --force-recreate api; }
|
||||
trap restore EXIT
|
||||
SESSION_TTL_SECONDS=2 docker compose up -d --wait --force-recreate api
|
||||
"${PYTHON:-python3}" tests/expiry.py
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Исходный persistence-сценарий плюс доказательство рестарта всех трёх сервисов."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from runtime_config import base_url
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def states():
|
||||
result = {}
|
||||
for service in ("api", "postgres", "redis"):
|
||||
ids = subprocess.check_output(
|
||||
["docker", "compose", "ps", "-q", service], cwd=ROOT, text=True
|
||||
).split()
|
||||
if not ids:
|
||||
raise RuntimeError("Не запущен сервис " + service)
|
||||
result[service] = {}
|
||||
for container in ids:
|
||||
state = json.loads(
|
||||
subprocess.check_output(
|
||||
["docker", "inspect", "--format", "{{json .State}}", container],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
)
|
||||
)
|
||||
result[service][container] = {
|
||||
"StartedAt": state["StartedAt"],
|
||||
"Pid": state["Pid"],
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default=base_url())
|
||||
parser.add_argument("--action-script", required=True)
|
||||
parser.add_argument("--ca-file")
|
||||
args = parser.parse_args()
|
||||
before = states()
|
||||
print("ДО: " + json.dumps(before), flush=True)
|
||||
command = [
|
||||
sys.executable,
|
||||
str(ROOT / "tests/resilience.py"),
|
||||
"--mode",
|
||||
"persistence",
|
||||
"--action-script",
|
||||
args.action_script,
|
||||
"--base-url",
|
||||
args.base_url,
|
||||
]
|
||||
if args.ca_file:
|
||||
command += ["--ca-file", args.ca_file]
|
||||
subprocess.run(command, cwd=ROOT, check=True)
|
||||
after = states()
|
||||
print("ПОСЛЕ: " + json.dumps(after), flush=True)
|
||||
for service in before:
|
||||
if before[service] == after[service]:
|
||||
raise AssertionError("Сервис не перезапущен: " + service)
|
||||
print(
|
||||
"PASS: StartedAt изменился у API, PostgreSQL и Redis; прежний токен и данные сохранены"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Полная проверка lab3. Использует текущий учебный стенд, volumes сохраняет."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from benchmark import Client, fixture, measure
|
||||
from replicas import compose, containers, describe, docker, instance_id, wait_ready, wait_visible
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / "evidence"
|
||||
|
||||
|
||||
def run_test(name, *args, stdin=None):
|
||||
result = subprocess.run(args, cwd=ROOT, input=stdin, text=True, capture_output=True)
|
||||
(EVIDENCE / name).write_text(result.stdout + result.stderr)
|
||||
print(name, "PASS" if result.returncode == 0 else "FAIL", flush=True)
|
||||
if result.returncode:
|
||||
raise RuntimeError((result.stdout + result.stderr)[-3000:])
|
||||
|
||||
|
||||
def main():
|
||||
py = sys.executable
|
||||
for filename in ["smoke", "lab3", "extra", "regressions", "production"]:
|
||||
run_test("lab3-" + filename + ".txt", py, "tests/" + filename + ".py")
|
||||
run_test(
|
||||
"lab3-db-idempotency.txt",
|
||||
"docker",
|
||||
"exec",
|
||||
"-i",
|
||||
containers()[0],
|
||||
"python",
|
||||
stdin=(ROOT / "tests/idempotency_db.py").read_text(),
|
||||
)
|
||||
run_test(
|
||||
"lab3-public-failover.txt",
|
||||
py,
|
||||
"tests/resilience.py",
|
||||
"--mode",
|
||||
"failover",
|
||||
"--action-script",
|
||||
"scripts/stop-one.sh",
|
||||
)
|
||||
compose("up", "-d", "--no-deps", "--wait", "api")
|
||||
for c in containers():
|
||||
wait_visible(wait_ready(c))
|
||||
events = []
|
||||
actor, chat = fixture()
|
||||
initial = containers()
|
||||
for target in initial:
|
||||
alias = instance_id(target)
|
||||
print("Failure test", target[:12], alias, flush=True)
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
event = []
|
||||
load = pool.submit(measure, actor, chat, 12, 8, event)
|
||||
time.sleep(3)
|
||||
event.append(time.monotonic())
|
||||
docker("kill", "--signal", "KILL", target)
|
||||
report, confirmations = load.result()
|
||||
assert not describe(target)["Running"]
|
||||
assert report["successes"].get("read") and report["successes"].get("write")
|
||||
assert all(report["seconds"][b]["success"] > 0 for b in range(5, 11)), (
|
||||
"No service after detection"
|
||||
)
|
||||
c = Client()
|
||||
# Each acknowledged message is read back with the same idempotency key.
|
||||
for key, mid in confirmations:
|
||||
response = c.message(actor, chat, "load-" + key, key=key)
|
||||
assert response["json"]["id"] == mid
|
||||
assert response["headers"]["X-Instance-Id"] != alias
|
||||
report.update(
|
||||
{
|
||||
"stopped_container": target[:12],
|
||||
"stopped_instance": alias,
|
||||
"exit_code": describe(target)["ExitCode"],
|
||||
"all_acknowledged_messages_replayed": len(confirmations),
|
||||
"failure_started_at_load_second": 3,
|
||||
"service_after_second_5": "reads/writes continue",
|
||||
"note": "hard SIGKILL, no automatic retries in load, replay confirmations after failure",
|
||||
}
|
||||
)
|
||||
events.append(report)
|
||||
docker("start", target)
|
||||
wait_visible(wait_ready(target))
|
||||
(EVIDENCE / "lab3-failures.json").write_text(
|
||||
json.dumps(events, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print("Both replicas survived sequential hard-failure tests", flush=True)
|
||||
actor, chat = fixture()
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
load = pool.submit(measure, actor, chat, 40, 8)
|
||||
time.sleep(3)
|
||||
run_test("lab3-rolling.txt", py, "scripts/rolling-update.py")
|
||||
report, confirmations = load.result()
|
||||
(EVIDENCE / "lab3-rolling-load.json").write_text(json.dumps(report, indent=2) + "\n")
|
||||
assert report["successes"].get("read") and report["successes"].get("write")
|
||||
# Zero errors are not required by rubric; any transitional errors are reported.
|
||||
client = Client()
|
||||
for key, mid in confirmations:
|
||||
assert client.message(actor, chat, "load-" + key, key=key)["json"]["id"] == mid
|
||||
before = {c: describe(c)["StartedAt"] for c in containers()}
|
||||
storage_before = {
|
||||
s: describe(compose("ps", "-q", s))["StartedAt"] for s in ["postgres", "redis"]
|
||||
}
|
||||
run_test(
|
||||
"lab3-persistence.txt",
|
||||
py,
|
||||
"tests/resilience.py",
|
||||
"--mode",
|
||||
"persistence",
|
||||
"--action-script",
|
||||
"scripts/restart.sh",
|
||||
)
|
||||
after = {c: describe(c)["StartedAt"] for c in containers()}
|
||||
storage_after = {
|
||||
s: describe(compose("ps", "-q", s))["StartedAt"] for s in ["postgres", "redis"]
|
||||
}
|
||||
assert set(before) != set(after) or before != after
|
||||
assert all(storage_before[s] != storage_after[s] for s in storage_before)
|
||||
proof = {
|
||||
"api_before": before,
|
||||
"api_after": after,
|
||||
"storage_before": storage_before,
|
||||
"storage_after": storage_after,
|
||||
}
|
||||
(EVIDENCE / "lab3-restart-proof.json").write_text(json.dumps(proof, indent=2) + "\n")
|
||||
run_test("lab3-final-smoke.txt", py, "tests/smoke.py")
|
||||
print("PASS: all lab3 acceptance checks", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user