lab4: S3-вложения, presigned upload и проверки
This commit is contained in:
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail when the SeaweedFS identities and the API environment disagree.
|
||||
|
||||
The stand keeps synthetic credentials in two places on purpose: the identities
|
||||
file is mounted into SeaweedFS, while the API receives the same values through
|
||||
the environment. This check keeps the copies from drifting.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
IDENTITIES = ROOT / "infra/s3/s3.json"
|
||||
|
||||
|
||||
def parse_env(path: Path) -> dict[str, str]:
|
||||
values: dict[str, str] = {}
|
||||
for raw in path.read_text().splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
values[key.strip()] = value.strip().strip('"').strip("'")
|
||||
return values
|
||||
|
||||
|
||||
def main() -> int:
|
||||
identities = json.loads(IDENTITIES.read_text())
|
||||
by_name = {item["name"]: item for item in identities.get("identities", [])}
|
||||
|
||||
source = ROOT / ".env"
|
||||
if not source.is_file():
|
||||
source = ROOT / ".env.example"
|
||||
env = parse_env(source)
|
||||
|
||||
expected = {
|
||||
"admin": ("S3_ADMIN_ACCESS_KEY", "S3_ADMIN_SECRET_ACCESS_KEY"),
|
||||
"app": ("S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"),
|
||||
}
|
||||
|
||||
failures: list[str] = []
|
||||
for name, (key_var, secret_var) in expected.items():
|
||||
identity = by_name.get(name)
|
||||
if identity is None:
|
||||
failures.append(f"{IDENTITIES}: identity {name!r} is missing")
|
||||
continue
|
||||
credentials = identity.get("credentials") or []
|
||||
if not credentials:
|
||||
failures.append(f"{IDENTITIES}: identity {name!r} has no credentials")
|
||||
continue
|
||||
access_key = credentials[0].get("accessKey")
|
||||
secret_key = credentials[0].get("secretKey")
|
||||
if env.get(key_var) != access_key:
|
||||
failures.append(f"{source.name}: {key_var} does not match identity {name!r}")
|
||||
if env.get(secret_var) != secret_key:
|
||||
failures.append(f"{source.name}: {secret_var} does not match identity {name!r}")
|
||||
|
||||
if failures:
|
||||
print("FAIL: S3 credentials are inconsistent")
|
||||
for failure in failures:
|
||||
print(" -", failure)
|
||||
return 1
|
||||
|
||||
print(f"PASS: S3 identities match {source.name} (admin + app)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+147
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure API memory under concurrent 9 MiB uploads (lab4 level 4).
|
||||
|
||||
Run against a live stand. Writes raw CSV samples and a summary under
|
||||
evidence/lab4/memory/. The primary metric is cgroup anon memory: memory.current
|
||||
also includes page cache from the temporary spool files.
|
||||
|
||||
python3 scripts/lab4_memory_experiment.py --base-url http://localhost:8080
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
|
||||
from client import Client # noqa: E402
|
||||
|
||||
OUTPUT = ROOT / 'evidence/lab4/memory'
|
||||
|
||||
|
||||
def png_of_size(width: int, height: int, seed: int) -> bytes:
|
||||
"""Build a valid, poorly compressible PNG (random RGB scanlines)."""
|
||||
rng = __import__('random').Random(seed)
|
||||
raw = bytearray()
|
||||
for _ in range(height):
|
||||
raw.append(0) # filter: None
|
||||
raw.extend(rng.randbytes(width * 3))
|
||||
|
||||
def chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
|
||||
ihdr = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
|
||||
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) + chunk(b'IDAT', zlib.compress(bytes(raw), 6)) + chunk(b'IEND', b'')
|
||||
|
||||
|
||||
def containers(service: str) -> list[str]:
|
||||
output = os.popen(f'docker compose ps -q {service}').read().split()
|
||||
if not output:
|
||||
raise SystemExit(f'no running containers for service {service}')
|
||||
return output
|
||||
|
||||
|
||||
def read_number(container: str, path: str) -> int:
|
||||
output = os.popen(f'docker exec {container} cat {path} 2>/dev/null').read().strip()
|
||||
return int(output) if output.isdigit() else 0
|
||||
|
||||
|
||||
def memory_snapshot() -> dict[str, int]:
|
||||
api = containers('api')
|
||||
anon = current = 0
|
||||
for container in api:
|
||||
stats = os.popen(f'docker exec {container} cat /sys/fs/cgroup/memory.stat 2>/dev/null').read()
|
||||
for line in stats.splitlines():
|
||||
if line.startswith('anon '):
|
||||
anon += int(line.split()[1])
|
||||
current += read_number(container, '/sys/fs/cgroup/memory.current')
|
||||
return {'anon': anon, 'current': current}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--concurrency', type=int, default=10)
|
||||
parser.add_argument('--width', type=int, default=2100)
|
||||
parser.add_argument('--height', type=int, default=1500)
|
||||
parser.add_argument('--rounds', type=int, default=3)
|
||||
args = parser.parse_args()
|
||||
|
||||
content = png_of_size(args.width, args.height, 42)
|
||||
if len(content) > 10 * 1024 * 1024:
|
||||
raise SystemExit(f'generated PNG is {len(content)} bytes, above the 10 MiB contract limit')
|
||||
|
||||
client = Client(args.base_url, timeout=120)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
|
||||
OUTPUT.mkdir(parents=True, exist_ok=True)
|
||||
samples: list[dict[str, int | float | str]] = []
|
||||
stop = threading.Event()
|
||||
|
||||
def sample(phase: str = 'during') -> None:
|
||||
started = time.monotonic()
|
||||
while not stop.is_set():
|
||||
snapshot = memory_snapshot()
|
||||
samples.append({'elapsed': round(time.monotonic() - started, 3), 'phase': phase, **snapshot})
|
||||
time.sleep(0.2)
|
||||
|
||||
baseline = memory_snapshot()
|
||||
sampler = threading.Thread(target=sample, daemon=True)
|
||||
sampler.start()
|
||||
|
||||
def upload(_: int) -> None:
|
||||
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=memory.png'
|
||||
client.call(
|
||||
'POST', path, 201, actor=alice, raw=content,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment',
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
for _ in range(args.rounds):
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
list(pool.map(upload, range(args.concurrency)))
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
stop.set()
|
||||
sampler.join(timeout=2)
|
||||
after = memory_snapshot()
|
||||
|
||||
peak_anon = max((int(item['anon']) for item in samples), default=after['anon'])
|
||||
peak_current = max((int(item['current']) for item in samples), default=after['current'])
|
||||
|
||||
csv_path = OUTPUT / 'samples.csv'
|
||||
with csv_path.open('w', newline='') as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=['elapsed', 'phase', 'anon', 'current'])
|
||||
writer.writeheader()
|
||||
writer.writerows(samples)
|
||||
|
||||
summary = OUTPUT / 'summary.txt'
|
||||
summary.write_text(
|
||||
'Lab4 memory experiment\n'
|
||||
f'concurrency: {args.concurrency}\n'
|
||||
f'file_size_bytes: {len(content)}\n'
|
||||
f'uploads: {args.concurrency * args.rounds}\n'
|
||||
f'rounds: {args.rounds}\n'
|
||||
f'elapsed_seconds: {elapsed:.3f}\n'
|
||||
f'baseline_anon_bytes: {baseline["anon"]}\n'
|
||||
f'peak_anon_bytes: {peak_anon}\n'
|
||||
f'anon_delta_bytes: {peak_anon - baseline["anon"]}\n'
|
||||
f'peak_memory_current_bytes: {peak_current}\n'
|
||||
'note: memory.current includes page cache from temporary spool files; anon is the primary metric\n'
|
||||
)
|
||||
|
||||
print(summary.read_text())
|
||||
print('CSV:', csv_path)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+179
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare proxy upload with presigned direct upload (lab4 track 1).
|
||||
|
||||
Measures client-facing LB traffic, API traffic, API anon memory and latency for
|
||||
two file sizes. Run against a live stand from the repository root.
|
||||
|
||||
python3 scripts/lab4_track1.py --base-url http://localhost:8080
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
sys.path.insert(0, str(ROOT / 'student-tests'))
|
||||
|
||||
from client import Client # noqa: E402
|
||||
from presign_flow import complete, presign, put # noqa: E402
|
||||
from lab4_memory_experiment import png_of_size # noqa: E402
|
||||
|
||||
OUTPUT = ROOT / 'evidence/lab4/track1'
|
||||
|
||||
|
||||
def containers(service: str) -> list[str]:
|
||||
ids = os.popen(f'docker compose ps -q {service}').read().split()
|
||||
if not ids:
|
||||
raise SystemExit(f'no running containers for service {service}')
|
||||
return ids
|
||||
|
||||
|
||||
def read_net(service: str) -> tuple[int, int]:
|
||||
rx = tx = 0
|
||||
for container in containers(service):
|
||||
for direction, path in (('rx', 'rx_bytes'), ('tx', 'tx_bytes')):
|
||||
value = os.popen(
|
||||
f'docker exec {container} cat /sys/class/net/eth0/statistics/{path} 2>/dev/null'
|
||||
).read().strip()
|
||||
if value.isdigit():
|
||||
if direction == 'rx':
|
||||
rx += int(value)
|
||||
else:
|
||||
tx += int(value)
|
||||
return rx, tx
|
||||
|
||||
|
||||
def read_anon() -> int:
|
||||
total = 0
|
||||
for container in containers('api'):
|
||||
stats = os.popen(f'docker exec {container} cat /sys/fs/cgroup/memory.stat 2>/dev/null').read()
|
||||
for line in stats.splitlines():
|
||||
if line.startswith('anon '):
|
||||
total += int(line.split()[1])
|
||||
return total
|
||||
|
||||
|
||||
class Sampler:
|
||||
def __init__(self) -> None:
|
||||
self.stop = threading.Event()
|
||||
self.peak = 0
|
||||
self.thread = threading.Thread(target=self.run, daemon=True)
|
||||
|
||||
def run(self) -> None:
|
||||
while not self.stop.is_set():
|
||||
self.peak = max(self.peak, read_anon())
|
||||
time.sleep(0.1)
|
||||
|
||||
def __enter__(self) -> 'Sampler':
|
||||
self.thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: object) -> None:
|
||||
self.stop.set()
|
||||
self.thread.join(timeout=2)
|
||||
|
||||
|
||||
def proxy_upload(client: Client, actor: dict, chat: dict, content: bytes) -> None:
|
||||
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=track1.png'
|
||||
client.call(
|
||||
'POST', path, 201, actor=actor, raw=content,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment',
|
||||
)
|
||||
|
||||
|
||||
def presign_upload(client: Client, actor: dict, chat: dict, content: bytes) -> None:
|
||||
prepared = presign(client, actor, chat, content)
|
||||
status = put(prepared['upload_url'], content)
|
||||
if status != 200:
|
||||
raise SystemExit(f'presigned PUT failed with {status}')
|
||||
complete(client, actor, prepared['attachment_id'], prepared['sha256'])
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--iterations', type=int, default=5)
|
||||
args = parser.parse_args()
|
||||
|
||||
client = Client(args.base_url, timeout=180)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
|
||||
files = {
|
||||
'~1MiB': png_of_size(700, 500, 11),
|
||||
'~9MiB': png_of_size(2100, 1500, 12),
|
||||
}
|
||||
|
||||
OUTPUT.mkdir(parents=True, exist_ok=True)
|
||||
rows: list[dict[str, object]] = []
|
||||
|
||||
for label, content in files.items():
|
||||
for flow, operation in (('proxy', proxy_upload), ('presign', presign_upload)):
|
||||
for iteration in range(args.iterations):
|
||||
lb_before, lb_tx_before = read_net('lb')
|
||||
api_before, api_tx_before = read_net('api')
|
||||
started = time.monotonic()
|
||||
|
||||
with Sampler() as sampler:
|
||||
operation(client, alice, chat, content)
|
||||
|
||||
latency = time.monotonic() - started
|
||||
lb_after, lb_tx_after = read_net('lb')
|
||||
api_after, api_tx_after = read_net('api')
|
||||
|
||||
rows.append({
|
||||
'size': label,
|
||||
'bytes': len(content),
|
||||
'flow': flow,
|
||||
'iteration': iteration,
|
||||
'latency_s': round(latency, 4),
|
||||
'lb_rx_bytes': lb_after - lb_before,
|
||||
'lb_tx_bytes': lb_tx_after - lb_tx_before,
|
||||
'api_rx_bytes': api_after - api_before,
|
||||
'api_tx_bytes': api_tx_after - api_tx_before,
|
||||
'api_anon_peak': sampler.peak,
|
||||
})
|
||||
|
||||
csv_path = OUTPUT / 'results.csv'
|
||||
with csv_path.open('w', newline='') as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
lines = ['Lab4 track 1: proxy vs presigned upload', '']
|
||||
lines.append('| size | flow | median latency, s | p95 latency, s | LB rx, bytes | API rx, bytes | API anon peak, bytes |')
|
||||
lines.append('|---|---|---:|---:|---:|---:|---:|')
|
||||
for label in files:
|
||||
for flow in ('proxy', 'presign'):
|
||||
group = [row for row in rows if row['size'] == label and row['flow'] == flow]
|
||||
latencies = sorted(float(row['latency_s']) for row in group)
|
||||
median = statistics.median(latencies)
|
||||
p95 = latencies[min(len(latencies) - 1, int(round(0.95 * (len(latencies) - 1))))]
|
||||
lb_rx = sum(int(row['lb_rx_bytes']) for row in group) / len(group)
|
||||
api_rx = sum(int(row['api_rx_bytes']) for row in group) / len(group)
|
||||
peak = max(int(row['api_anon_peak']) for row in group)
|
||||
lines.append(
|
||||
f'| {label} | {flow} | {median:.3f} | {p95:.3f} | {lb_rx:.0f} | {api_rx:.0f} | {peak} |'
|
||||
)
|
||||
|
||||
lines += [
|
||||
'',
|
||||
'Note: presigned upload keeps the file body off the API request path and off the load balancer.',
|
||||
'Completion still streams the object from S3 once to verify its SHA-256, so api_rx stays close to',
|
||||
'the file size; the saving is in the client-facing LB leg and in not accepting/buffering the body.',
|
||||
'',
|
||||
]
|
||||
summary = OUTPUT / 'summary.txt'
|
||||
summary.write_text('\n'.join(lines))
|
||||
print(summary.read_text())
|
||||
print('CSV:', csv_path)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -24,9 +24,9 @@ if [ -f .env ]; then
|
||||
fi
|
||||
|
||||
OUT=${OUT:-evidence/lb-readiness.txt}
|
||||
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab3}
|
||||
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab4}
|
||||
NET=${PROJECT}_default
|
||||
IMAGE=${IMAGE:-mtusi-messenger:lab3}
|
||||
IMAGE=${IMAGE:-mtusi-messenger:lab4}
|
||||
STATS=${STATS:-http://127.0.0.1:8404/stats;csv}
|
||||
BASE_URL=${BASE_URL:-http://localhost:8080}
|
||||
CANDIDATE=lb-readiness-candidate
|
||||
|
||||
+4
-4
@@ -1,6 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Restart hook: bounce API replicas and every state store of the lab3 stand.
|
||||
# Restart hook: bounce API replicas and every state store of the lab4 stand.
|
||||
#
|
||||
# scripts/restart.sh
|
||||
#
|
||||
@@ -31,9 +31,9 @@ compose() {
|
||||
|
||||
# 1. State stores: restart the running ones, then make sure they are healthy.
|
||||
# Volumes are attached as before, nothing is dropped or re-seeded.
|
||||
compose up -d --wait postgres redis
|
||||
compose restart postgres redis
|
||||
compose up -d --wait postgres redis
|
||||
compose up -d --wait postgres redis seaweedfs
|
||||
compose restart postgres redis seaweedfs
|
||||
compose up -d --wait postgres redis seaweedfs
|
||||
|
||||
# 2. API replicas: restart in place (same containers, same addresses) and bring
|
||||
# back any replica that was stopped by the fault-injection hook.
|
||||
|
||||
@@ -25,8 +25,8 @@ DURATION=${DURATION:-60}
|
||||
CONCURRENCY=${CONCURRENCY:-8}
|
||||
BASE_URL=${BASE_URL:-http://localhost:8080}
|
||||
OUT=${OUT:-evidence/rolling-update}
|
||||
IMAGE=${IMAGE:-mtusi-messenger:lab3}
|
||||
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab3}
|
||||
IMAGE=${IMAGE:-mtusi-messenger:lab4}
|
||||
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab4}
|
||||
NET=${PROJECT}_default
|
||||
STATS=${STATS:-http://127.0.0.1:8404/stats;csv}
|
||||
PREFIX=lb-rollout
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Fault-injection hook: stop exactly ONE api replica of the lab3 stand.
|
||||
# Fault-injection hook: stop exactly ONE api replica of the lab4 stand.
|
||||
#
|
||||
# TARGET_INSTANCE_ID=<hostname|container name|short id> scripts/stop-one.sh
|
||||
#
|
||||
@@ -13,7 +13,7 @@ set -eu
|
||||
# No credentials are printed.
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab3}
|
||||
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab4}
|
||||
|
||||
candidates=$(docker ps -q \
|
||||
--filter "label=com.docker.compose.project=${PROJECT}" \
|
||||
@@ -26,7 +26,7 @@ fi
|
||||
|
||||
# Matches the requested alias against every reasonable identifier of a replica:
|
||||
# container hostname (= X-Instance-Id), full/short container id, full container
|
||||
# name (mtusi-messenger-lab3-api-1) and the service-local suffix (api-1).
|
||||
# name (mtusi-messenger-lab4-api-1) and the service-local suffix (api-1).
|
||||
match_replica() {
|
||||
target=$1
|
||||
full_id=$2
|
||||
|
||||
Reference in New Issue
Block a user