forked from mtusi2026/mtusi2026lab4
lab4: s3 attachments, private bucket, healthchecks, tests and report
This commit is contained in:
@@ -8,8 +8,8 @@ import re
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
def main():
|
||||
lab = json.loads((ROOT/'lab.json').read_text())['number']
|
||||
spec = json.loads((ROOT/'contracts/openapi.json').read_text())
|
||||
lab = json.loads((ROOT/'lab.json').read_text(encoding='utf-8'))['number']
|
||||
spec = json.loads((ROOT/'contracts/openapi.json').read_text(encoding='utf-8'))
|
||||
assert spec['openapi']=='3.1.0'
|
||||
assert spec['info']['version']=='1.%d.0'%lab
|
||||
operation_ids = set()
|
||||
@@ -41,11 +41,11 @@ def main():
|
||||
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
|
||||
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
|
||||
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
|
||||
ast.parse(file.read_text(),filename=str(file))
|
||||
ast.parse(file.read_text(encoding='utf-8'),filename=str(file))
|
||||
for file in ROOT.rglob('*.json'):
|
||||
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
|
||||
continue
|
||||
json.loads(file.read_text())
|
||||
json.loads(file.read_text(encoding='utf-8'))
|
||||
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
|
||||
assert (ROOT/required).is_file(), 'missing '+required
|
||||
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Convenience runner. Does not fabricate evidence or control Docker itself."""
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
for command in (
|
||||
[sys.executable, str(ROOT / "tests/smoke.py")],
|
||||
[sys.executable, str(ROOT / "student-tests/run.py")],
|
||||
[sys.executable, str(ROOT / "student-tests/test_attachments.py")],
|
||||
):
|
||||
print("RUN:", " ".join(command), flush=True)
|
||||
subprocess.run(command, cwd=ROOT, check=True)
|
||||
print("PASS: public smoke + lab3 concurrency + grade4 negative cases; run extra resilience and memory experiment separately")
|
||||
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stops each API separately and replaces both under a small closed-loop load."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, arguments, check, run
|
||||
from resilience import history
|
||||
import ops
|
||||
|
||||
|
||||
def exercise(c, name, action):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
initial_key = uuid.uuid4().hex
|
||||
initial = c.message(alice, chat, "До воздействия", key=initial_key)["json"]
|
||||
observed = {c.call("GET", "/api/v1/users/me", actor=alice)["headers"]["X-Instance-Id"]
|
||||
for _ in range(20)}
|
||||
check(len(observed) >= 2, "need two actual instances before " + name)
|
||||
stop = threading.Event()
|
||||
barrier = threading.Barrier(5)
|
||||
|
||||
def worker(number):
|
||||
local = Client(c.base, timeout=2)
|
||||
writes, cycles, errors = [], [], 0
|
||||
barrier.wait()
|
||||
key = uuid.uuid4().hex
|
||||
body = "worker-%d-%s" % (number, key)
|
||||
while not stop.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = local.message(alice, chat, body, key=key)
|
||||
writes.append(response["json"])
|
||||
# Read with the original session while another backend may serve it.
|
||||
local.call("GET", "/api/v1/chats/" + chat["id"], actor=bob, schema="Chat")
|
||||
finished = time.monotonic()
|
||||
cycles.append((finished, finished - started,
|
||||
response["headers"]["X-Instance-Id"]))
|
||||
key = uuid.uuid4().hex
|
||||
body = "worker-%d-%s" % (number, key)
|
||||
except Failure:
|
||||
errors += 1
|
||||
# An uncertain write keeps its original key and body.
|
||||
time.sleep(0.05)
|
||||
return writes, cycles, errors
|
||||
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
futures = [pool.submit(worker, n) for n in range(4)]
|
||||
barrier.wait()
|
||||
try:
|
||||
time.sleep(1)
|
||||
action_started = time.monotonic()
|
||||
phases = action()
|
||||
action_finished = time.monotonic()
|
||||
time.sleep(7)
|
||||
finally:
|
||||
stop.set()
|
||||
results = [f.result() for f in futures]
|
||||
elapsed = time.monotonic() - started
|
||||
writes = [msg for messages, _, _ in results for msg in messages]
|
||||
cycles = sorted(item for _, values, _ in results for item in values)
|
||||
errors = sum(n for _, _, n in results)
|
||||
# An in-flight POST can finish before the stop and its paired GET after it.
|
||||
# Only cycles started after the action count as traffic to a surviving API.
|
||||
recovered = [t for t, duration, _ in cycles if t - duration >= action_finished]
|
||||
check(bool(recovered), "no reads/writes after " + name)
|
||||
recovery = recovered[0] - action_started
|
||||
check(recovery <= 10 if name.startswith("stop-") else True,
|
||||
"failover exceeded 10 seconds")
|
||||
times = [started] + [t for t, _, _ in cycles] + [time.monotonic()]
|
||||
max_gap = max(b - a for a, b in zip(times, times[1:]))
|
||||
check(max_gap <= 10, "no successful read/write cycle for more than 10 seconds")
|
||||
during = [t for t, _, _ in cycles if action_started <= t <= action_finished]
|
||||
if name == "rolling-update":
|
||||
check(bool(during), "no successful traffic during rolling replacement")
|
||||
for phase in phases or []:
|
||||
check(any(phase["started"] <= t <= phase["finished"] for t in during),
|
||||
"no successful traffic while replacing " + phase["service"])
|
||||
stored = history(c, bob, chat)
|
||||
ids = {m["id"] for m in stored}
|
||||
check(len(ids) == len(stored), "duplicate message IDs")
|
||||
check(len({m["text"] for m in stored}) == len(stored), "retry created duplicate text")
|
||||
by_id = {m["id"]: m for m in stored}
|
||||
check(all(by_id.get(m["id"]) == m for m in [initial] + writes), "confirmed data lost")
|
||||
check(c.message(alice, chat, initial["text"], key=initial_key)["json"] == initial,
|
||||
"pre-action idempotency result lost")
|
||||
latency = sorted(d * 1000 for _, d, _ in cycles)
|
||||
def percentile(p):
|
||||
return round(latency[max(0, math.ceil(p * len(latency)) - 1)], 3)
|
||||
result = {"scenario": name, "concurrency": 4, "elapsed_seconds": round(elapsed, 3),
|
||||
"successful_read_write_cycles": len(cycles), "failed_attempts": errors,
|
||||
"successful_http_requests": len(writes) + len(cycles),
|
||||
"successful_rps": round((len(writes) + len(cycles)) / elapsed, 3),
|
||||
"successful_cycles_per_second": round(len(cycles) / elapsed, 3),
|
||||
"successful_cycles_during_action": len(during),
|
||||
"max_success_gap_seconds": round(max_gap, 3),
|
||||
"first_success_after_action_seconds": round(recovery, 3),
|
||||
"latency_ms": {"p50": percentile(.5), "p95": percentile(.95), "p99": percentile(.99)},
|
||||
"instances_before": sorted(observed),
|
||||
"instances_after": sorted({i for t, duration, i in cycles
|
||||
if t - duration >= action_finished}),
|
||||
"confirmed_messages_preserved": len({m["id"] for m in [initial] + writes}),
|
||||
"note": "Closed-loop: one POST + one GET per cycle. RPS counts confirmed HTTP responses, including partial cycles. Latency covers successful cycles only. Recovery includes action duration."}
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2), flush=True)
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Lab3 grade4 Compose experiments")
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
ops.BASE_URL = c.base
|
||||
dest = ROOT / "evidence"
|
||||
dest.mkdir(exist_ok=True)
|
||||
http_args = ["--base-url", c.base]
|
||||
if args.ca_file:
|
||||
http_args += ["--ca-file", args.ca_file]
|
||||
|
||||
def checked(script, filename, *extra):
|
||||
result = subprocess.run([sys.executable, str(ROOT / script), *extra], cwd=ROOT,
|
||||
text=True, encoding="utf-8", stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
env=dict(os.environ, PYTHONIOENCODING="utf-8"))
|
||||
(dest / filename).write_text(result.stdout, encoding="utf-8")
|
||||
print(result.stdout, end="", flush=True)
|
||||
check(result.returncode == 0, script + " failed; see evidence/" + filename)
|
||||
|
||||
ops.restore()
|
||||
checked("scripts/check_contract.py", "check.txt")
|
||||
checked("tests/smoke.py", "smoke.txt", *http_args)
|
||||
checked("student-tests/run.py", "student-tests.txt", *http_args)
|
||||
results = []
|
||||
processes = []
|
||||
for service in ops.SERVICES:
|
||||
ops.restore()
|
||||
before = ops.state(service)
|
||||
try:
|
||||
result = exercise(c, "stop-" + service, lambda: ops.stop_one(service))
|
||||
after = ops.state(service)
|
||||
check(after["status"] == "exited", "stopped API restarted unexpectedly")
|
||||
check(all(i.split(":", 1)[0] != before["hostname"] for i in result["instances_after"]),
|
||||
"stopped replica still handles requests")
|
||||
results.append(result)
|
||||
processes.append({"action": "stop-" + service, "before": before, "after": after})
|
||||
finally:
|
||||
ops.restore()
|
||||
before = [ops.state(s) for s in ops.SERVICES]
|
||||
results.append(exercise(c, "rolling-update", ops.rolling_update))
|
||||
after = [ops.state(s) for s in ops.SERVICES]
|
||||
check(all(a["container"] != b["container"] for a, b in zip(before, after)),
|
||||
"both API containers must be replaced")
|
||||
processes.append({"action": "rolling-update", "before": before, "after": after})
|
||||
(dest / "grade4.json").write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
(dest / "processes.json").write_text(json.dumps(processes, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
suffix = ".cmd" if os.name == "nt" else ".sh"
|
||||
try:
|
||||
checked("tests/resilience.py", "failover.txt", "--mode", "failover",
|
||||
"--action-script", str(ROOT / ("scripts/stop-one" + suffix)), *http_args)
|
||||
finally:
|
||||
ops.restore()
|
||||
checked("tests/resilience.py", "persistence.txt", "--mode", "persistence",
|
||||
"--action-script", str(ROOT / ("scripts/restart" + suffix)), *http_args)
|
||||
print("PASS: both replica failures, rolling replacement and confirmed data")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,71 @@
|
||||
"""10 simultaneous real uploads + Docker RSS observations (NO invented metrics)."""
|
||||
import concurrent.futures
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from statistics import median
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check, arguments, run
|
||||
|
||||
|
||||
def mib(value):
|
||||
value = value.strip()
|
||||
for suffix, multiplier in (("GiB",1024),("MiB",1),("KiB",1/1024),("B",1/1048576)):
|
||||
if value.endswith(suffix):
|
||||
return float(value[:-len(suffix)]) * multiplier
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Concurrent uploads + observed API container memory')
|
||||
client = Client(args.base_url, args.ca_file, timeout=30)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
file = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
results, memory = [], []
|
||||
stop = threading.Event()
|
||||
def observe():
|
||||
while not stop.is_set():
|
||||
r = subprocess.run(['docker','stats','--no-stream','--format','{{.Name}}|{{.MemUsage}}'],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
if r.returncode == 0:
|
||||
for line in r.stdout.splitlines():
|
||||
if '|api1-' in line or '|api2-' in line:
|
||||
pass
|
||||
parts = line.split('|')
|
||||
if len(parts)==2 and ('api1-' in parts[0] or 'api2-' in parts[0]):
|
||||
amount = mib(parts[1].split('/')[0])
|
||||
if amount is not None:
|
||||
memory.append({'container': parts[0], 'mem_mib': round(amount,3)})
|
||||
stop.wait(0.2)
|
||||
monitor = threading.Thread(target=observe, daemon=True)
|
||||
monitor.start()
|
||||
try:
|
||||
def worker(i):
|
||||
local = Client(args.base_url,args.ca_file,timeout=30)
|
||||
start=time.perf_counter()
|
||||
resp=local.call('POST', f'/api/v1/chats/{chat["id"]}/attachments?filename=run{i}.png',
|
||||
201, actor=alice, raw=file, headers={'Content-Type':'image/png'}, schema='Attachment')
|
||||
return {'elapsed_ms':round((time.perf_counter()-start)*1000,2), 'instance': resp['headers'].get('X-Instance-Id')}
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
|
||||
futures=[executor.submit(worker,i) for i in range(10)]
|
||||
results=[f.result() for f in concurrent.futures.as_completed(futures)]
|
||||
finally:
|
||||
stop.set(); monitor.join(timeout=12)
|
||||
output={'concurrency':10,'file_bytes':len(file),'completed':len(results),
|
||||
'median_latency_ms':round(median(x['elapsed_ms'] for x in results),2),
|
||||
'api_peak_mem_mib_observed':max([x['mem_mib'] for x in memory],default=None),
|
||||
'memory_samples':len(memory),'instances':sorted(set(x['instance'] for x in results))}
|
||||
target=ROOT / 'evidence' / 'uploads-10.json'
|
||||
target.write_text(json.dumps(output,ensure_ascii=False,indent=2),encoding='utf-8')
|
||||
check(len(results)==10, 'not all uploads completed')
|
||||
print('PASS: 10 parallel uploads; metrics written:',target)
|
||||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
||||
if not memory:
|
||||
print('WARNING: no Docker memory samples; run docker stats separately for a valid grade-4 experiment')
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator actions for the local Compose stand; volumes are kept."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERVICES = ("api1", "api2")
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/")
|
||||
|
||||
|
||||
def compose(*args, capture=False):
|
||||
result = subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True,
|
||||
text=True, stdout=subprocess.PIPE if capture else None)
|
||||
return result.stdout.strip() if capture else None
|
||||
|
||||
|
||||
def state(service):
|
||||
cid = compose("ps", "--all", "--quiet", service, capture=True)
|
||||
if not cid:
|
||||
raise RuntimeError("No container for " + service)
|
||||
result = subprocess.check_output(
|
||||
["docker", "inspect", "--format",
|
||||
'{{json .Config.Hostname}} {{json .State.Status}}', cid], text=True)
|
||||
hostname, status = result.strip().split()
|
||||
return {"service": service, "container": cid[:12],
|
||||
"hostname": json.loads(hostname), "status": json.loads(status)}
|
||||
|
||||
|
||||
def wait_instances(count=2, timeout=40):
|
||||
deadline = time.monotonic() + timeout
|
||||
seen = set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=2) as r:
|
||||
instance = r.headers.get("X-Instance-Id")
|
||||
if instance:
|
||||
seen.add(instance)
|
||||
if len(seen) >= count:
|
||||
return seen
|
||||
except (OSError, urllib.error.URLError):
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
raise RuntimeError("Not enough ready API instances through LB")
|
||||
|
||||
|
||||
def stop_one(service=None):
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "").split(":", 1)[0]
|
||||
if target:
|
||||
service = next((s for s in SERVICES if state(s)["hostname"] == target), None)
|
||||
if not service:
|
||||
raise RuntimeError("TARGET_INSTANCE_ID is not an API container")
|
||||
service = service or "api1"
|
||||
if service not in SERVICES:
|
||||
raise RuntimeError("Only API replicas may be stopped")
|
||||
compose("kill", "--signal", "SIGKILL", service)
|
||||
current = state(service)
|
||||
if current["status"] != "exited":
|
||||
raise RuntimeError("API did not stop")
|
||||
print(json.dumps(current))
|
||||
|
||||
|
||||
def restore():
|
||||
compose("up", "-d", "--no-deps", "--no-build", "--wait", "--wait-timeout", "40", *SERVICES)
|
||||
wait_instances()
|
||||
|
||||
|
||||
def restart():
|
||||
compose("down")
|
||||
compose("up", "-d", "--no-build", "--wait", "--wait-timeout", "60")
|
||||
wait_instances()
|
||||
|
||||
|
||||
def rolling_update():
|
||||
phases = []
|
||||
compose("build", "api1")
|
||||
# This migration only adds tables and is compatible with the old API.
|
||||
compose("run", "--rm", "--no-deps", "migrate")
|
||||
for service in SERVICES:
|
||||
before = state(service)
|
||||
started = time.monotonic()
|
||||
compose("up", "-d", "--no-deps", "--no-build", "--force-recreate",
|
||||
"--wait", "--wait-timeout", "40", service)
|
||||
after = state(service)
|
||||
if before["container"] == after["container"]:
|
||||
raise RuntimeError("Container was not replaced")
|
||||
seen = wait_instances()
|
||||
if not any(i.split(":", 1)[0] == after["hostname"] for i in seen):
|
||||
raise RuntimeError("LB has not discovered the replacement")
|
||||
phases.append({"service": service, "started": started, "finished": time.monotonic()})
|
||||
print(json.dumps({"replaced": service, "before": before, "after": after}))
|
||||
return phases
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", choices=["restart", "stop-one", "restore", "rolling-update"])
|
||||
parser.add_argument("--service", choices=SERVICES)
|
||||
args = parser.parse_args()
|
||||
if args.action == "stop-one":
|
||||
stop_one(args.service)
|
||||
else:
|
||||
{"restart": restart, "restore": restore, "rolling-update": rolling_update}[args.action]()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
python scripts\ops.py restart
|
||||
exit /b %errorlevel%
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/ops.py restart
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
python scripts\ops.py stop-one
|
||||
exit /b %errorlevel%
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/ops.py stop-one
|
||||
@@ -6,17 +6,17 @@ from openapi_spec_validator import validate
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
spec = json.loads((root/'contracts/openapi.json').read_text())
|
||||
spec = json.loads((root/'contracts/openapi.json').read_text(encoding='utf-8'))
|
||||
validate(spec)
|
||||
from referencing import Registry, Resource
|
||||
# The base URI resolves local #/components references inside examples.
|
||||
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
|
||||
registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource))
|
||||
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
|
||||
for name, example in json.loads((root/'contracts/examples.json').read_text(encoding='utf-8')).items():
|
||||
schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name}
|
||||
Draft202012Validator(schema, registry=registry).validate(example)
|
||||
for path in (root/'contracts').glob('*.schema.json'):
|
||||
schema = json.loads(path.read_text())
|
||||
schema = json.loads(path.read_text(encoding='utf-8'))
|
||||
Draft202012Validator.check_schema(schema)
|
||||
for example in schema.get('examples',[]):
|
||||
Draft202012Validator(schema).validate(example)
|
||||
|
||||
Reference in New Issue
Block a user