lab4: s3 attachments, private bucket, healthchecks, tests and report

This commit is contained in:
2026-11-03 00:35:32 +03:00
parent ec540a1a72
commit dfadf3eb7c
61 changed files with 2329 additions and 133 deletions
+4 -4
View File
@@ -8,8 +8,8 @@ import re
ROOT = Path(__file__).resolve().parents[1]
def main():
lab = json.loads((ROOT/'lab.json').read_text())['number']
spec = json.loads((ROOT/'contracts/openapi.json').read_text())
lab = json.loads((ROOT/'lab.json').read_text(encoding='utf-8'))['number']
spec = json.loads((ROOT/'contracts/openapi.json').read_text(encoding='utf-8'))
assert spec['openapi']=='3.1.0'
assert spec['info']['version']=='1.%d.0'%lab
operation_ids = set()
@@ -41,11 +41,11 @@ def main():
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
ast.parse(file.read_text(),filename=str(file))
ast.parse(file.read_text(encoding='utf-8'),filename=str(file))
for file in ROOT.rglob('*.json'):
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
continue
json.loads(file.read_text())
json.loads(file.read_text(encoding='utf-8'))
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
assert (ROOT/required).is_file(), 'missing '+required
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
+13
View File
@@ -0,0 +1,13 @@
"""Convenience runner. Does not fabricate evidence or control Docker itself."""
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
for command in (
[sys.executable, str(ROOT / "tests/smoke.py")],
[sys.executable, str(ROOT / "student-tests/run.py")],
[sys.executable, str(ROOT / "student-tests/test_attachments.py")],
):
print("RUN:", " ".join(command), flush=True)
subprocess.run(command, cwd=ROOT, check=True)
print("PASS: public smoke + lab3 concurrency + grade4 negative cases; run extra resilience and memory experiment separately")
+175
View File
@@ -0,0 +1,175 @@
#!/usr/bin/env python3
"""Stops each API separately and replaces both under a small closed-loop load."""
from concurrent.futures import ThreadPoolExecutor
import json
import math
import os
from pathlib import Path
import sys
import subprocess
import threading
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, arguments, check, run
from resilience import history
import ops
def exercise(c, name, action):
alice, bob, eve, chat = c.bootstrap()
initial_key = uuid.uuid4().hex
initial = c.message(alice, chat, "До воздействия", key=initial_key)["json"]
observed = {c.call("GET", "/api/v1/users/me", actor=alice)["headers"]["X-Instance-Id"]
for _ in range(20)}
check(len(observed) >= 2, "need two actual instances before " + name)
stop = threading.Event()
barrier = threading.Barrier(5)
def worker(number):
local = Client(c.base, timeout=2)
writes, cycles, errors = [], [], 0
barrier.wait()
key = uuid.uuid4().hex
body = "worker-%d-%s" % (number, key)
while not stop.is_set():
started = time.monotonic()
try:
response = local.message(alice, chat, body, key=key)
writes.append(response["json"])
# Read with the original session while another backend may serve it.
local.call("GET", "/api/v1/chats/" + chat["id"], actor=bob, schema="Chat")
finished = time.monotonic()
cycles.append((finished, finished - started,
response["headers"]["X-Instance-Id"]))
key = uuid.uuid4().hex
body = "worker-%d-%s" % (number, key)
except Failure:
errors += 1
# An uncertain write keeps its original key and body.
time.sleep(0.05)
return writes, cycles, errors
started = time.monotonic()
with ThreadPoolExecutor(max_workers=4) as pool:
futures = [pool.submit(worker, n) for n in range(4)]
barrier.wait()
try:
time.sleep(1)
action_started = time.monotonic()
phases = action()
action_finished = time.monotonic()
time.sleep(7)
finally:
stop.set()
results = [f.result() for f in futures]
elapsed = time.monotonic() - started
writes = [msg for messages, _, _ in results for msg in messages]
cycles = sorted(item for _, values, _ in results for item in values)
errors = sum(n for _, _, n in results)
# An in-flight POST can finish before the stop and its paired GET after it.
# Only cycles started after the action count as traffic to a surviving API.
recovered = [t for t, duration, _ in cycles if t - duration >= action_finished]
check(bool(recovered), "no reads/writes after " + name)
recovery = recovered[0] - action_started
check(recovery <= 10 if name.startswith("stop-") else True,
"failover exceeded 10 seconds")
times = [started] + [t for t, _, _ in cycles] + [time.monotonic()]
max_gap = max(b - a for a, b in zip(times, times[1:]))
check(max_gap <= 10, "no successful read/write cycle for more than 10 seconds")
during = [t for t, _, _ in cycles if action_started <= t <= action_finished]
if name == "rolling-update":
check(bool(during), "no successful traffic during rolling replacement")
for phase in phases or []:
check(any(phase["started"] <= t <= phase["finished"] for t in during),
"no successful traffic while replacing " + phase["service"])
stored = history(c, bob, chat)
ids = {m["id"] for m in stored}
check(len(ids) == len(stored), "duplicate message IDs")
check(len({m["text"] for m in stored}) == len(stored), "retry created duplicate text")
by_id = {m["id"]: m for m in stored}
check(all(by_id.get(m["id"]) == m for m in [initial] + writes), "confirmed data lost")
check(c.message(alice, chat, initial["text"], key=initial_key)["json"] == initial,
"pre-action idempotency result lost")
latency = sorted(d * 1000 for _, d, _ in cycles)
def percentile(p):
return round(latency[max(0, math.ceil(p * len(latency)) - 1)], 3)
result = {"scenario": name, "concurrency": 4, "elapsed_seconds": round(elapsed, 3),
"successful_read_write_cycles": len(cycles), "failed_attempts": errors,
"successful_http_requests": len(writes) + len(cycles),
"successful_rps": round((len(writes) + len(cycles)) / elapsed, 3),
"successful_cycles_per_second": round(len(cycles) / elapsed, 3),
"successful_cycles_during_action": len(during),
"max_success_gap_seconds": round(max_gap, 3),
"first_success_after_action_seconds": round(recovery, 3),
"latency_ms": {"p50": percentile(.5), "p95": percentile(.95), "p99": percentile(.99)},
"instances_before": sorted(observed),
"instances_after": sorted({i for t, duration, i in cycles
if t - duration >= action_finished}),
"confirmed_messages_preserved": len({m["id"] for m in [initial] + writes}),
"note": "Closed-loop: one POST + one GET per cycle. RPS counts confirmed HTTP responses, including partial cycles. Latency covers successful cycles only. Recovery includes action duration."}
print(json.dumps(result, ensure_ascii=False, indent=2), flush=True)
return result
def main():
args = arguments("Lab3 grade4 Compose experiments")
c = Client(args.base_url, args.ca_file)
ops.BASE_URL = c.base
dest = ROOT / "evidence"
dest.mkdir(exist_ok=True)
http_args = ["--base-url", c.base]
if args.ca_file:
http_args += ["--ca-file", args.ca_file]
def checked(script, filename, *extra):
result = subprocess.run([sys.executable, str(ROOT / script), *extra], cwd=ROOT,
text=True, encoding="utf-8", stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=dict(os.environ, PYTHONIOENCODING="utf-8"))
(dest / filename).write_text(result.stdout, encoding="utf-8")
print(result.stdout, end="", flush=True)
check(result.returncode == 0, script + " failed; see evidence/" + filename)
ops.restore()
checked("scripts/check_contract.py", "check.txt")
checked("tests/smoke.py", "smoke.txt", *http_args)
checked("student-tests/run.py", "student-tests.txt", *http_args)
results = []
processes = []
for service in ops.SERVICES:
ops.restore()
before = ops.state(service)
try:
result = exercise(c, "stop-" + service, lambda: ops.stop_one(service))
after = ops.state(service)
check(after["status"] == "exited", "stopped API restarted unexpectedly")
check(all(i.split(":", 1)[0] != before["hostname"] for i in result["instances_after"]),
"stopped replica still handles requests")
results.append(result)
processes.append({"action": "stop-" + service, "before": before, "after": after})
finally:
ops.restore()
before = [ops.state(s) for s in ops.SERVICES]
results.append(exercise(c, "rolling-update", ops.rolling_update))
after = [ops.state(s) for s in ops.SERVICES]
check(all(a["container"] != b["container"] for a, b in zip(before, after)),
"both API containers must be replaced")
processes.append({"action": "rolling-update", "before": before, "after": after})
(dest / "grade4.json").write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
(dest / "processes.json").write_text(json.dumps(processes, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
suffix = ".cmd" if os.name == "nt" else ".sh"
try:
checked("tests/resilience.py", "failover.txt", "--mode", "failover",
"--action-script", str(ROOT / ("scripts/stop-one" + suffix)), *http_args)
finally:
ops.restore()
checked("tests/resilience.py", "persistence.txt", "--mode", "persistence",
"--action-script", str(ROOT / ("scripts/restart" + suffix)), *http_args)
print("PASS: both replica failures, rolling replacement and confirmed data")
if __name__ == "__main__":
run(main)
+71
View File
@@ -0,0 +1,71 @@
"""10 simultaneous real uploads + Docker RSS observations (NO invented metrics)."""
import concurrent.futures
import json
import subprocess
import sys
import threading
import time
from pathlib import Path
from statistics import median
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check, arguments, run
def mib(value):
value = value.strip()
for suffix, multiplier in (("GiB",1024),("MiB",1),("KiB",1/1024),("B",1/1048576)):
if value.endswith(suffix):
return float(value[:-len(suffix)]) * multiplier
return None
def main():
args = arguments('Concurrent uploads + observed API container memory')
client = Client(args.base_url, args.ca_file, timeout=30)
alice, bob, eve, chat = client.bootstrap()
file = (ROOT / 'tests/fixtures/sample.png').read_bytes()
results, memory = [], []
stop = threading.Event()
def observe():
while not stop.is_set():
r = subprocess.run(['docker','stats','--no-stream','--format','{{.Name}}|{{.MemUsage}}'],
capture_output=True, text=True, timeout=10)
if r.returncode == 0:
for line in r.stdout.splitlines():
if '|api1-' in line or '|api2-' in line:
pass
parts = line.split('|')
if len(parts)==2 and ('api1-' in parts[0] or 'api2-' in parts[0]):
amount = mib(parts[1].split('/')[0])
if amount is not None:
memory.append({'container': parts[0], 'mem_mib': round(amount,3)})
stop.wait(0.2)
monitor = threading.Thread(target=observe, daemon=True)
monitor.start()
try:
def worker(i):
local = Client(args.base_url,args.ca_file,timeout=30)
start=time.perf_counter()
resp=local.call('POST', f'/api/v1/chats/{chat["id"]}/attachments?filename=run{i}.png',
201, actor=alice, raw=file, headers={'Content-Type':'image/png'}, schema='Attachment')
return {'elapsed_ms':round((time.perf_counter()-start)*1000,2), 'instance': resp['headers'].get('X-Instance-Id')}
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
futures=[executor.submit(worker,i) for i in range(10)]
results=[f.result() for f in concurrent.futures.as_completed(futures)]
finally:
stop.set(); monitor.join(timeout=12)
output={'concurrency':10,'file_bytes':len(file),'completed':len(results),
'median_latency_ms':round(median(x['elapsed_ms'] for x in results),2),
'api_peak_mem_mib_observed':max([x['mem_mib'] for x in memory],default=None),
'memory_samples':len(memory),'instances':sorted(set(x['instance'] for x in results))}
target=ROOT / 'evidence' / 'uploads-10.json'
target.write_text(json.dumps(output,ensure_ascii=False,indent=2),encoding='utf-8')
check(len(results)==10, 'not all uploads completed')
print('PASS: 10 parallel uploads; metrics written:',target)
print(json.dumps(output, ensure_ascii=False, indent=2))
if not memory:
print('WARNING: no Docker memory samples; run docker stats separately for a valid grade-4 experiment')
if __name__ == '__main__':
run(main)
+112
View File
@@ -0,0 +1,112 @@
#!/usr/bin/env python3
"""Operator actions for the local Compose stand; volumes are kept."""
import argparse
import json
import os
from pathlib import Path
import subprocess
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
SERVICES = ("api1", "api2")
BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/")
def compose(*args, capture=False):
result = subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True,
text=True, stdout=subprocess.PIPE if capture else None)
return result.stdout.strip() if capture else None
def state(service):
cid = compose("ps", "--all", "--quiet", service, capture=True)
if not cid:
raise RuntimeError("No container for " + service)
result = subprocess.check_output(
["docker", "inspect", "--format",
'{{json .Config.Hostname}} {{json .State.Status}}', cid], text=True)
hostname, status = result.strip().split()
return {"service": service, "container": cid[:12],
"hostname": json.loads(hostname), "status": json.loads(status)}
def wait_instances(count=2, timeout=40):
deadline = time.monotonic() + timeout
seen = set()
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=2) as r:
instance = r.headers.get("X-Instance-Id")
if instance:
seen.add(instance)
if len(seen) >= count:
return seen
except (OSError, urllib.error.URLError):
pass
time.sleep(0.2)
raise RuntimeError("Not enough ready API instances through LB")
def stop_one(service=None):
target = os.environ.get("TARGET_INSTANCE_ID", "").split(":", 1)[0]
if target:
service = next((s for s in SERVICES if state(s)["hostname"] == target), None)
if not service:
raise RuntimeError("TARGET_INSTANCE_ID is not an API container")
service = service or "api1"
if service not in SERVICES:
raise RuntimeError("Only API replicas may be stopped")
compose("kill", "--signal", "SIGKILL", service)
current = state(service)
if current["status"] != "exited":
raise RuntimeError("API did not stop")
print(json.dumps(current))
def restore():
compose("up", "-d", "--no-deps", "--no-build", "--wait", "--wait-timeout", "40", *SERVICES)
wait_instances()
def restart():
compose("down")
compose("up", "-d", "--no-build", "--wait", "--wait-timeout", "60")
wait_instances()
def rolling_update():
phases = []
compose("build", "api1")
# This migration only adds tables and is compatible with the old API.
compose("run", "--rm", "--no-deps", "migrate")
for service in SERVICES:
before = state(service)
started = time.monotonic()
compose("up", "-d", "--no-deps", "--no-build", "--force-recreate",
"--wait", "--wait-timeout", "40", service)
after = state(service)
if before["container"] == after["container"]:
raise RuntimeError("Container was not replaced")
seen = wait_instances()
if not any(i.split(":", 1)[0] == after["hostname"] for i in seen):
raise RuntimeError("LB has not discovered the replacement")
phases.append({"service": service, "started": started, "finished": time.monotonic()})
print(json.dumps({"replaced": service, "before": before, "after": after}))
return phases
def main():
parser = argparse.ArgumentParser()
parser.add_argument("action", choices=["restart", "stop-one", "restore", "rolling-update"])
parser.add_argument("--service", choices=SERVICES)
args = parser.parse_args()
if args.action == "stop-one":
stop_one(args.service)
else:
{"restart": restart, "restore": restore, "rolling-update": rolling_update}[args.action]()
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
@echo off
cd /d "%~dp0.."
python scripts\ops.py restart
exit /b %errorlevel%
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/ops.py restart
+4
View File
@@ -0,0 +1,4 @@
@echo off
cd /d "%~dp0.."
python scripts\ops.py stop-one
exit /b %errorlevel%
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/ops.py stop-one
+3 -3
View File
@@ -6,17 +6,17 @@ from openapi_spec_validator import validate
from jsonschema import Draft202012Validator
root = Path(__file__).resolve().parents[1]
spec = json.loads((root/'contracts/openapi.json').read_text())
spec = json.loads((root/'contracts/openapi.json').read_text(encoding='utf-8'))
validate(spec)
from referencing import Registry, Resource
# The base URI resolves local #/components references inside examples.
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource))
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
for name, example in json.loads((root/'contracts/examples.json').read_text(encoding='utf-8')).items():
schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name}
Draft202012Validator(schema, registry=registry).validate(example)
for path in (root/'contracts').glob('*.schema.json'):
schema = json.loads(path.read_text())
schema = json.loads(path.read_text(encoding='utf-8'))
Draft202012Validator.check_schema(schema)
for example in schema.get('examples',[]):
Draft202012Validator(schema).validate(example)