lab4: S3-вложения, presigned upload и проверки

This commit is contained in:
Vladislav Kunetskiy
2026-10-21 22:42:48 +03:00
parent d342b2a227
commit 49c2fe3071
79 changed files with 12266 additions and 910 deletions
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""Fail when the SeaweedFS identities and the API environment disagree.
The stand keeps synthetic credentials in two places on purpose: the identities
file is mounted into SeaweedFS, while the API receives the same values through
the environment. This check keeps the copies from drifting.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
IDENTITIES = ROOT / "infra/s3/s3.json"
def parse_env(path: Path) -> dict[str, str]:
values: dict[str, str] = {}
for raw in path.read_text().splitlines():
line = raw.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
values[key.strip()] = value.strip().strip('"').strip("'")
return values
def main() -> int:
identities = json.loads(IDENTITIES.read_text())
by_name = {item["name"]: item for item in identities.get("identities", [])}
source = ROOT / ".env"
if not source.is_file():
source = ROOT / ".env.example"
env = parse_env(source)
expected = {
"admin": ("S3_ADMIN_ACCESS_KEY", "S3_ADMIN_SECRET_ACCESS_KEY"),
"app": ("S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"),
}
failures: list[str] = []
for name, (key_var, secret_var) in expected.items():
identity = by_name.get(name)
if identity is None:
failures.append(f"{IDENTITIES}: identity {name!r} is missing")
continue
credentials = identity.get("credentials") or []
if not credentials:
failures.append(f"{IDENTITIES}: identity {name!r} has no credentials")
continue
access_key = credentials[0].get("accessKey")
secret_key = credentials[0].get("secretKey")
if env.get(key_var) != access_key:
failures.append(f"{source.name}: {key_var} does not match identity {name!r}")
if env.get(secret_var) != secret_key:
failures.append(f"{source.name}: {secret_var} does not match identity {name!r}")
if failures:
print("FAIL: S3 credentials are inconsistent")
for failure in failures:
print(" -", failure)
return 1
print(f"PASS: S3 identities match {source.name} (admin + app)")
return 0
if __name__ == "__main__":
sys.exit(main())
+147
View File
@@ -0,0 +1,147 @@
#!/usr/bin/env python3
"""Measure API memory under concurrent 9 MiB uploads (lab4 level 4).
Run against a live stand. Writes raw CSV samples and a summary under
evidence/lab4/memory/. The primary metric is cgroup anon memory: memory.current
also includes page cache from the temporary spool files.
python3 scripts/lab4_memory_experiment.py --base-url http://localhost:8080
"""
from __future__ import annotations
import argparse
import csv
import os
import struct
import sys
import threading
import time
import zlib
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client # noqa: E402
OUTPUT = ROOT / 'evidence/lab4/memory'
def png_of_size(width: int, height: int, seed: int) -> bytes:
"""Build a valid, poorly compressible PNG (random RGB scanlines)."""
rng = __import__('random').Random(seed)
raw = bytearray()
for _ in range(height):
raw.append(0) # filter: None
raw.extend(rng.randbytes(width * 3))
def chunk(kind: bytes, data: bytes) -> bytes:
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xFFFFFFFF)
ihdr = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) + chunk(b'IDAT', zlib.compress(bytes(raw), 6)) + chunk(b'IEND', b'')
def containers(service: str) -> list[str]:
output = os.popen(f'docker compose ps -q {service}').read().split()
if not output:
raise SystemExit(f'no running containers for service {service}')
return output
def read_number(container: str, path: str) -> int:
output = os.popen(f'docker exec {container} cat {path} 2>/dev/null').read().strip()
return int(output) if output.isdigit() else 0
def memory_snapshot() -> dict[str, int]:
api = containers('api')
anon = current = 0
for container in api:
stats = os.popen(f'docker exec {container} cat /sys/fs/cgroup/memory.stat 2>/dev/null').read()
for line in stats.splitlines():
if line.startswith('anon '):
anon += int(line.split()[1])
current += read_number(container, '/sys/fs/cgroup/memory.current')
return {'anon': anon, 'current': current}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--concurrency', type=int, default=10)
parser.add_argument('--width', type=int, default=2100)
parser.add_argument('--height', type=int, default=1500)
parser.add_argument('--rounds', type=int, default=3)
args = parser.parse_args()
content = png_of_size(args.width, args.height, 42)
if len(content) > 10 * 1024 * 1024:
raise SystemExit(f'generated PNG is {len(content)} bytes, above the 10 MiB contract limit')
client = Client(args.base_url, timeout=120)
alice, _, _, chat = client.bootstrap()
OUTPUT.mkdir(parents=True, exist_ok=True)
samples: list[dict[str, int | float | str]] = []
stop = threading.Event()
def sample(phase: str = 'during') -> None:
started = time.monotonic()
while not stop.is_set():
snapshot = memory_snapshot()
samples.append({'elapsed': round(time.monotonic() - started, 3), 'phase': phase, **snapshot})
time.sleep(0.2)
baseline = memory_snapshot()
sampler = threading.Thread(target=sample, daemon=True)
sampler.start()
def upload(_: int) -> None:
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=memory.png'
client.call(
'POST', path, 201, actor=alice, raw=content,
headers={'Content-Type': 'image/png'}, schema='Attachment',
)
started = time.monotonic()
for _ in range(args.rounds):
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
list(pool.map(upload, range(args.concurrency)))
elapsed = time.monotonic() - started
stop.set()
sampler.join(timeout=2)
after = memory_snapshot()
peak_anon = max((int(item['anon']) for item in samples), default=after['anon'])
peak_current = max((int(item['current']) for item in samples), default=after['current'])
csv_path = OUTPUT / 'samples.csv'
with csv_path.open('w', newline='') as handle:
writer = csv.DictWriter(handle, fieldnames=['elapsed', 'phase', 'anon', 'current'])
writer.writeheader()
writer.writerows(samples)
summary = OUTPUT / 'summary.txt'
summary.write_text(
'Lab4 memory experiment\n'
f'concurrency: {args.concurrency}\n'
f'file_size_bytes: {len(content)}\n'
f'uploads: {args.concurrency * args.rounds}\n'
f'rounds: {args.rounds}\n'
f'elapsed_seconds: {elapsed:.3f}\n'
f'baseline_anon_bytes: {baseline["anon"]}\n'
f'peak_anon_bytes: {peak_anon}\n'
f'anon_delta_bytes: {peak_anon - baseline["anon"]}\n'
f'peak_memory_current_bytes: {peak_current}\n'
'note: memory.current includes page cache from temporary spool files; anon is the primary metric\n'
)
print(summary.read_text())
print('CSV:', csv_path)
if __name__ == '__main__':
main()
+179
View File
@@ -0,0 +1,179 @@
#!/usr/bin/env python3
"""Compare proxy upload with presigned direct upload (lab4 track 1).
Measures client-facing LB traffic, API traffic, API anon memory and latency for
two file sizes. Run against a live stand from the repository root.
python3 scripts/lab4_track1.py --base-url http://localhost:8080
"""
from __future__ import annotations
import argparse
import csv
import os
import statistics
import sys
import threading
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'student-tests'))
from client import Client # noqa: E402
from presign_flow import complete, presign, put # noqa: E402
from lab4_memory_experiment import png_of_size # noqa: E402
OUTPUT = ROOT / 'evidence/lab4/track1'
def containers(service: str) -> list[str]:
ids = os.popen(f'docker compose ps -q {service}').read().split()
if not ids:
raise SystemExit(f'no running containers for service {service}')
return ids
def read_net(service: str) -> tuple[int, int]:
rx = tx = 0
for container in containers(service):
for direction, path in (('rx', 'rx_bytes'), ('tx', 'tx_bytes')):
value = os.popen(
f'docker exec {container} cat /sys/class/net/eth0/statistics/{path} 2>/dev/null'
).read().strip()
if value.isdigit():
if direction == 'rx':
rx += int(value)
else:
tx += int(value)
return rx, tx
def read_anon() -> int:
total = 0
for container in containers('api'):
stats = os.popen(f'docker exec {container} cat /sys/fs/cgroup/memory.stat 2>/dev/null').read()
for line in stats.splitlines():
if line.startswith('anon '):
total += int(line.split()[1])
return total
class Sampler:
def __init__(self) -> None:
self.stop = threading.Event()
self.peak = 0
self.thread = threading.Thread(target=self.run, daemon=True)
def run(self) -> None:
while not self.stop.is_set():
self.peak = max(self.peak, read_anon())
time.sleep(0.1)
def __enter__(self) -> 'Sampler':
self.thread.start()
return self
def __exit__(self, *_args: object) -> None:
self.stop.set()
self.thread.join(timeout=2)
def proxy_upload(client: Client, actor: dict, chat: dict, content: bytes) -> None:
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=track1.png'
client.call(
'POST', path, 201, actor=actor, raw=content,
headers={'Content-Type': 'image/png'}, schema='Attachment',
)
def presign_upload(client: Client, actor: dict, chat: dict, content: bytes) -> None:
prepared = presign(client, actor, chat, content)
status = put(prepared['upload_url'], content)
if status != 200:
raise SystemExit(f'presigned PUT failed with {status}')
complete(client, actor, prepared['attachment_id'], prepared['sha256'])
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--iterations', type=int, default=5)
args = parser.parse_args()
client = Client(args.base_url, timeout=180)
alice, _, _, chat = client.bootstrap()
files = {
'~1MiB': png_of_size(700, 500, 11),
'~9MiB': png_of_size(2100, 1500, 12),
}
OUTPUT.mkdir(parents=True, exist_ok=True)
rows: list[dict[str, object]] = []
for label, content in files.items():
for flow, operation in (('proxy', proxy_upload), ('presign', presign_upload)):
for iteration in range(args.iterations):
lb_before, lb_tx_before = read_net('lb')
api_before, api_tx_before = read_net('api')
started = time.monotonic()
with Sampler() as sampler:
operation(client, alice, chat, content)
latency = time.monotonic() - started
lb_after, lb_tx_after = read_net('lb')
api_after, api_tx_after = read_net('api')
rows.append({
'size': label,
'bytes': len(content),
'flow': flow,
'iteration': iteration,
'latency_s': round(latency, 4),
'lb_rx_bytes': lb_after - lb_before,
'lb_tx_bytes': lb_tx_after - lb_tx_before,
'api_rx_bytes': api_after - api_before,
'api_tx_bytes': api_tx_after - api_tx_before,
'api_anon_peak': sampler.peak,
})
csv_path = OUTPUT / 'results.csv'
with csv_path.open('w', newline='') as handle:
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
lines = ['Lab4 track 1: proxy vs presigned upload', '']
lines.append('| size | flow | median latency, s | p95 latency, s | LB rx, bytes | API rx, bytes | API anon peak, bytes |')
lines.append('|---|---|---:|---:|---:|---:|---:|')
for label in files:
for flow in ('proxy', 'presign'):
group = [row for row in rows if row['size'] == label and row['flow'] == flow]
latencies = sorted(float(row['latency_s']) for row in group)
median = statistics.median(latencies)
p95 = latencies[min(len(latencies) - 1, int(round(0.95 * (len(latencies) - 1))))]
lb_rx = sum(int(row['lb_rx_bytes']) for row in group) / len(group)
api_rx = sum(int(row['api_rx_bytes']) for row in group) / len(group)
peak = max(int(row['api_anon_peak']) for row in group)
lines.append(
f'| {label} | {flow} | {median:.3f} | {p95:.3f} | {lb_rx:.0f} | {api_rx:.0f} | {peak} |'
)
lines += [
'',
'Note: presigned upload keeps the file body off the API request path and off the load balancer.',
'Completion still streams the object from S3 once to verify its SHA-256, so api_rx stays close to',
'the file size; the saving is in the client-facing LB leg and in not accepting/buffering the body.',
'',
]
summary = OUTPUT / 'summary.txt'
summary.write_text('\n'.join(lines))
print(summary.read_text())
print('CSV:', csv_path)
if __name__ == '__main__':
main()
+2 -2
View File
@@ -24,9 +24,9 @@ if [ -f .env ]; then
fi
OUT=${OUT:-evidence/lb-readiness.txt}
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab3}
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab4}
NET=${PROJECT}_default
IMAGE=${IMAGE:-mtusi-messenger:lab3}
IMAGE=${IMAGE:-mtusi-messenger:lab4}
STATS=${STATS:-http://127.0.0.1:8404/stats;csv}
BASE_URL=${BASE_URL:-http://localhost:8080}
CANDIDATE=lb-readiness-candidate
+4 -4
View File
@@ -1,6 +1,6 @@
#!/bin/sh
set -eu
# Restart hook: bounce API replicas and every state store of the lab3 stand.
# Restart hook: bounce API replicas and every state store of the lab4 stand.
#
# scripts/restart.sh
#
@@ -31,9 +31,9 @@ compose() {
# 1. State stores: restart the running ones, then make sure they are healthy.
# Volumes are attached as before, nothing is dropped or re-seeded.
compose up -d --wait postgres redis
compose restart postgres redis
compose up -d --wait postgres redis
compose up -d --wait postgres redis seaweedfs
compose restart postgres redis seaweedfs
compose up -d --wait postgres redis seaweedfs
# 2. API replicas: restart in place (same containers, same addresses) and bring
# back any replica that was stopped by the fault-injection hook.
+2 -2
View File
@@ -25,8 +25,8 @@ DURATION=${DURATION:-60}
CONCURRENCY=${CONCURRENCY:-8}
BASE_URL=${BASE_URL:-http://localhost:8080}
OUT=${OUT:-evidence/rolling-update}
IMAGE=${IMAGE:-mtusi-messenger:lab3}
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab3}
IMAGE=${IMAGE:-mtusi-messenger:lab4}
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab4}
NET=${PROJECT}_default
STATS=${STATS:-http://127.0.0.1:8404/stats;csv}
PREFIX=lb-rollout
+3 -3
View File
@@ -1,6 +1,6 @@
#!/bin/sh
set -eu
# Fault-injection hook: stop exactly ONE api replica of the lab3 stand.
# Fault-injection hook: stop exactly ONE api replica of the lab4 stand.
#
# TARGET_INSTANCE_ID=<hostname|container name|short id> scripts/stop-one.sh
#
@@ -13,7 +13,7 @@ set -eu
# No credentials are printed.
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab3}
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab4}
candidates=$(docker ps -q \
--filter "label=com.docker.compose.project=${PROJECT}" \
@@ -26,7 +26,7 @@ fi
# Matches the requested alias against every reasonable identifier of a replica:
# container hostname (= X-Instance-Id), full/short container id, full container
# name (mtusi-messenger-lab3-api-1) and the service-local suffix (api-1).
# name (mtusi-messenger-lab4-api-1) and the service-local suffix (api-1).
match_replica() {
target=$1
full_id=$2