lab4: прямая загрузка, скрипты экспериментов и проверок

This commit is contained in:
2026-10-22 22:39:25 +03:00
parent 7de3012b46
commit 39a6df8c09
12 changed files with 1574 additions and 2 deletions
+293
View File
@@ -0,0 +1,293 @@
#!/usr/bin/env python3
"""Upload benchmark: API traffic and peak RSS per upload for proxy vs direct mode.
attachments_bench.py run --mode proxy --size-bytes 1048576 --concurrency 10 \
--uploads-per-worker 5 --repeat 3 --output evidence/lab4/bench/proxy-1048576-c10.json
attachments_bench.py summarize evidence/lab4/bench/*.json > evidence/lab4/summary.md
Run from the repository root with the stand up. Tokens, signed URLs and secrets
are never written to the output.
"""
import argparse
import base64
import hashlib
import http.client
import json
import os
import statistics
import subprocess
import sys
import threading
import time
import urllib.parse
import uuid
from datetime import datetime, timezone
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from pngkit import noise_png # noqa: E402
def sh(*args, check=True):
return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip()
def read_env(path=".env"):
values = {}
with open(path) as handle:
for line in handle:
line = line.strip()
if line and not line.startswith("#") and "=" in line:
key, _, value = line.partition("=")
values[key] = value
return values
def request(host, port, method, path, headers=None, body=None):
connection = http.client.HTTPConnection(host, port, timeout=120)
try:
connection.request(method, path, body=body, headers=headers or {})
response = connection.getresponse()
payload = response.read()
return response.status, payload
finally:
connection.close()
def api_json(env, method, path, token=None, body=None, basic=None):
headers = {}
data = None
if body is not None:
headers["content-type"] = "application/json"
data = json.dumps(body).encode()
if token:
headers["authorization"] = f"Bearer {token}"
if basic:
headers["authorization"] = "Basic " + base64.b64encode(basic.encode()).decode()
port = int(env.get("APP_PUBLIC_PORT", "8080"))
status, payload = request("127.0.0.1", port, method, path, headers, data)
return status, json.loads(payload) if payload else None
def containers():
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "api").split()
return sorted(ids)
NODE_METRICS = (
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics'+process.argv[1],"
"{method:process.argv[1]?'POST':'GET',headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})"
".then(r=>r.text()).then(console.log)"
)
def metrics(container):
return json.loads(sh("docker", "exec", container, "node", "-e", NODE_METRICS, ""))
def reset_metrics(container):
sh("docker", "exec", container, "node", "-e", NODE_METRICS, "/reset")
def net_bytes(container):
for line in sh("docker", "exec", container, "cat", "/proc/net/dev").splitlines():
if line.strip().startswith("eth0:"):
fields = line.split(":", 1)[1].split()
return int(fields[0]), int(fields[8])
raise RuntimeError("eth0 not found")
def percentile(values, fraction):
ordered = sorted(values)
if not ordered:
return 0.0
return ordered[min(len(ordered) - 1, max(0, int(round(fraction * len(ordered) + 0.5)) - 1))]
def one_upload(env, mode, token, chat_id, data, seed):
digest = hashlib.sha256(data).hexdigest()
filename = f"bench-{seed}.png"
port = int(env.get("APP_PUBLIC_PORT", "8080"))
started = time.perf_counter()
if mode == "proxy":
status, payload = request(
"127.0.0.1", port, "POST",
f"/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(filename)}",
{"authorization": f"Bearer {token}", "content-type": "image/png"}, data,
)
ok = status == 201
attachment = json.loads(payload) if ok else None
else:
status, ticket = api_json(
env, "POST", f"/api/v1/chats/{chat_id}/attachments/direct-uploads", token,
{"filename": filename, "content_type": "image/png", "size_bytes": len(data), "sha256": digest},
)
ok = status == 201
attachment = None
if ok:
target = urllib.parse.urlsplit(ticket["url"])
put_status, _ = request(
target.hostname, target.port, "PUT", target.path + "?" + target.query,
{**ticket["headers"]}, data,
)
ok = put_status == 200
if ok:
status, attachment = api_json(
env, "POST", f"/api/v1/attachments/{ticket['attachment_id']}/complete", token
)
ok = status == 200
elapsed_ms = (time.perf_counter() - started) * 1000
if ok and (attachment is None or attachment["sha256"] != digest):
ok = False
return ok, elapsed_ms
def run_once(env, args, token, chat_id, ids):
for container in ids:
reset_metrics(container)
time.sleep(0.2)
baseline = {c: metrics(c) for c in ids}
net_before = {c: net_bytes(c) for c in ids}
latencies, failures = [], []
lock = threading.Lock()
def worker(index):
for step in range(args.uploads_per_worker):
seed = index * 1000 + step
data = noise_png(args.size_bytes, seed)
ok, elapsed = one_upload(env, args.mode, token, chat_id, data, seed)
with lock:
(latencies if ok else failures).append(elapsed)
started = time.perf_counter()
threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
duration = time.perf_counter() - started
time.sleep(0.2)
after = {c: metrics(c) for c in ids}
net_after = {c: net_bytes(c) for c in ids}
replicas = []
for c in ids:
base_rss = baseline[c]["memory_bytes"]["rss"]
peak = after[c]["memory_peak_bytes"]
replicas.append({
"instance_id": after[c]["instance_id"],
"delta_rx_bytes": net_after[c][0] - net_before[c][0],
"delta_tx_bytes": net_after[c][1] - net_before[c][1],
"baseline_rss_bytes": base_rss,
"peak_memory_bytes": peak,
"delta_rss_bytes": peak["rss"] - base_rss,
"counters": {k: v for k, v in after[c]["counters"].items() if k.startswith("attachments.")},
})
total = args.concurrency * args.uploads_per_worker
return {
"uploads": total,
"succeeded": len(latencies),
"failed": len(failures),
"duration_s": round(duration, 3),
"latency_ms": {
"p50": round(percentile(latencies, 0.50), 1),
"p95": round(percentile(latencies, 0.95), 1),
"p99": round(percentile(latencies, 0.99), 1),
},
"replicas": replicas,
"total_delta_rx_bytes": sum(r["delta_rx_bytes"] for r in replicas),
"total_delta_tx_bytes": sum(r["delta_tx_bytes"] for r in replicas),
"rx_bytes_per_upload": round(sum(r["delta_rx_bytes"] for r in replicas) / total),
"tx_bytes_per_upload": round(sum(r["delta_tx_bytes"] for r in replicas) / total),
"max_delta_rss_bytes": max(r["delta_rss_bytes"] for r in replicas),
}
def command_run(args):
env = read_env()
ids = containers()
if not ids:
sys.exit("no api containers found")
name = f"bench_{uuid.uuid4().hex[:10]}"
password = "bench:" + uuid.uuid4().hex
status, user = api_json(env, "POST", "/api/v1/users",
body={"username": name, "display_name": "bench", "password": password})
assert status == 201, status
status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}")
assert status == 201, status
token = session["token"]
status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]})
assert status == 201, status
# Warm-up: one upload per replica path, not measured.
for seed in range(len(ids) * 2):
one_upload(env, args.mode, token, chat["id"], noise_png(args.size_bytes, 900000 + seed), seed)
limits = []
for c in ids:
limits.append(sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", c))
repeats = [run_once(env, args, token, chat["id"], ids) for _ in range(args.repeat)]
mem_total = next(int(l.split()[1]) * 1024 for l in open("/proc/meminfo") if l.startswith("MemTotal"))
result = {
"date": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"git": sh("git", "rev-parse", "--short", "HEAD", check=False),
"params": {
"mode": args.mode, "size_bytes": args.size_bytes, "concurrency": args.concurrency,
"uploads_per_worker": args.uploads_per_worker, "repeat": args.repeat,
},
"host": {"nproc": os.cpu_count(), "mem_total_bytes": mem_total},
"api_container_limits_nanocpus_memory": limits,
"repeats": repeats,
}
os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
with open(args.output, "w") as handle:
json.dump(result, handle, indent=2)
handle.write("\n")
for index, item in enumerate(repeats):
print(f"repeat {index + 1}: ok={item['succeeded']}/{item['uploads']} "
f"rx/upload={item['rx_bytes_per_upload']} tx/upload={item['tx_bytes_per_upload']} "
f"maxΔrss={item['max_delta_rss_bytes']} p50={item['latency_ms']['p50']}ms")
def mib(value):
return f"{value / 1048576:.2f}"
def command_summarize(args):
rows = []
for path in args.paths:
with open(path) as handle:
data = json.load(handle)
p = data["params"]
reps = data["repeats"]
med = lambda key: statistics.median(r[key] for r in reps) # noqa: E731
rows.append((p["mode"], p["size_bytes"], p["concurrency"], reps, med, data))
rows.sort(key=lambda r: (r[1], r[0], r[2]))
print("| mode | size, MiB | C | ok | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | p50, ms | p95, ms |")
print("|---|---|---|---|---|---|---|---|---|")
for mode, size, conc, reps, med, _data in rows:
ok = sum(r["succeeded"] for r in reps)
total = sum(r["uploads"] for r in reps)
p50 = statistics.median(r["latency_ms"]["p50"] for r in reps)
p95 = statistics.median(r["latency_ms"]["p95"] for r in reps)
print(f"| {mode} | {mib(size)} | {conc} | {ok}/{total} | {mib(med('rx_bytes_per_upload'))} | "
f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {p50:.0f} | {p95:.0f} |")
def main():
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(dest="command", required=True)
run = sub.add_parser("run")
run.add_argument("--mode", choices=["proxy", "direct"], required=True)
run.add_argument("--size-bytes", type=int, required=True)
run.add_argument("--concurrency", type=int, default=10)
run.add_argument("--uploads-per-worker", type=int, default=5)
run.add_argument("--repeat", type=int, default=3)
run.add_argument("--output", required=True)
summarize = sub.add_parser("summarize")
summarize.add_argument("paths", nargs="+")
args = parser.parse_args()
{"run": command_run, "summarize": command_summarize}[args.command](args)
if __name__ == "__main__":
main()
+54
View File
@@ -0,0 +1,54 @@
#!/bin/sh
# Проверка приватности S3 вне API: анонимный доступ, права отдельных credentials
# API, закрытая консоль. Секреты не печатаются.
set -u
cd "$(dirname "$0")/../.."
env_file="${COMPOSE_ENV_FILE:-.env}"
# shellcheck disable=SC1090
set -a; . "./$env_file"; set +a
endpoint="${S3_PUBLIC_ENDPOINT:-http://localhost:9000}"
bucket="${S3_BUCKET:-messenger-attachments}"
compose="docker compose --env-file $env_file"
fail=0
report() { # status name
if [ "$1" = ok ]; then echo "PASS $2"; else echo "FAIL $2"; fail=1; fi
}
key=$($compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -At \
-c "SELECT object_key FROM attachments WHERE status = 'ready' ORDER BY created_at DESC LIMIT 1")
if [ -z "$key" ]; then
echo "FAIL no ready attachment to check; upload one first (make test)"
exit 1
fi
code=$(curl -s -o /dev/null -w '%{http_code}' "$endpoint/$bucket/$key")
[ "$code" = 403 ] && report ok "anonymous GET of a ready object -> 403" || report fail "anonymous GET of a ready object -> $code (want 403)"
code=$(curl -s -o /dev/null -w '%{http_code}' "$endpoint/$bucket/")
[ "$code" = 403 ] && report ok "anonymous list of the bucket -> 403" || report fail "anonymous list of the bucket -> $code (want 403)"
code=$(curl -s -o /dev/null -w '%{http_code}' -X PUT --data-binary x "$endpoint/$bucket/attachments/anonymous-write")
[ "$code" = 403 ] && report ok "anonymous PUT -> 403" || report fail "anonymous PUT -> $code (want 403)"
# The API credentials only, inside a one-off container (secrets come from files).
api_mc() {
$compose run --rm --no-deps -T --entrypoint /bin/sh s3-init -c '
IFS= read -r k < /run/secrets/s3_api_access_key_id
IFS= read -r s < /run/secrets/s3_api_secret_access_key
export MC_CONFIG_DIR=/tmp/.mc MC_HOST_api="http://$k:$s@s3:9000"
exec mc "$@"' sh "$@" >/dev/null 2>&1
}
api_mc mb api/forbidden-bucket && report fail "API credentials can create buckets" || report ok "API credentials cannot create buckets"
api_mc admin info api && report fail "API credentials can use admin API" || report ok "API credentials cannot use admin API"
api_mc stat "api/$bucket/$key" && report ok "API credentials can read attachments/*" || report fail "API credentials cannot read attachments/*"
api_mc stat "api/$bucket/outside-prefix" ; rc=$?
api_mc cp /etc/hostname "api/$bucket/outside-prefix" && report fail "API credentials can write outside attachments/" || report ok "API credentials cannot write outside attachments/"
: "$rc"
port=$($compose port s3 9001 2>/dev/null || true)
[ -z "$port" ] && report ok "console port 9001 is not published" || report fail "console port 9001 is published ($port)"
if curl -s -o /dev/null --max-time 3 http://localhost:9001; then report fail "console reachable on localhost:9001"; else report ok "console not reachable on localhost:9001"; fi
exit "$fail"
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# Сверка bucket и БД (только чтение): orphans — объект без строки, missing —
# ready-строка без объекта. Код выхода 1, если что-то найдено.
set -eu
cd "$(dirname "$0")/../.."
env_file="${COMPOSE_ENV_FILE:-.env}"
# shellcheck disable=SC1090
set -a; . "./$env_file"; set +a
bucket="${S3_BUCKET:-messenger-attachments}"
compose="docker compose --env-file $env_file"
objects=$(mktemp)
rows=$(mktemp)
trap 'rm -f "$objects" "$rows"' EXIT
$compose run --rm --no-deps -T --entrypoint /bin/sh s3-init -c '
IFS= read -r u < /run/secrets/minio_root_user
IFS= read -r p < /run/secrets/minio_root_password
export MC_CONFIG_DIR=/tmp/.mc MC_HOST_stand="http://$u:$p@s3:9000"
exec mc ls --recursive "stand/$S3_BUCKET"' 2>/dev/null \
| sed -n 's/.* \(attachments\/.*\)$/\1/p' | sort > "$objects"
$compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -At \
-c "SELECT object_key, status FROM attachments" | sort > "$rows"
echo "bucket: $bucket"
echo "objects: $(wc -l < "$objects" | tr -d ' ')"
echo "rows by status:"
cut -d'|' -f2 "$rows" | sort | uniq -c | sed 's/^ */ /'
orphans=$(cut -d'|' -f1 "$rows" | sort | comm -13 - "$objects")
missing=$(grep '|ready$' "$rows" | cut -d'|' -f1 | sort | comm -23 - "$objects")
echo "orphans: $(printf '%s' "$orphans" | grep -c . || true)"
[ -z "$orphans" ] || printf '%s\n' "$orphans" | sed 's/^/ orphan /'
echo "missing: $(printf '%s' "$missing" | grep -c . || true)"
[ -z "$missing" ] || printf '%s\n' "$missing" | sed 's/^/ missing /'
[ -z "$orphans$missing" ]
+50
View File
@@ -0,0 +1,50 @@
"""Synthetic PNG helpers (stdlib only); same semantics as app/test/support/png.ts."""
import random
import struct
import zlib
SIGNATURE = b"\x89PNG\r\n\x1a\n"
NOISE_WIDTH = 1024
NOISE_ROW = 1 + NOISE_WIDTH * 3
def chunk(kind: bytes, data: bytes) -> bytes:
crc = zlib.crc32(kind + data) & 0xFFFFFFFF
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", crc)
def build_png(width: int, height: int, idat: bytes) -> bytes:
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return SIGNATURE + chunk(b"IHDR", ihdr) + chunk(b"IDAT", idat) + chunk(b"IEND", b"")
def pad_png(data: bytes, size: int) -> bytes:
"""Insert a private ancillary chunk before IEND to reach exactly `size` bytes."""
extra = size - len(data)
if extra == 0:
return data
if extra < 12:
raise ValueError("padding needs at least 12 bytes")
return data[:-12] + chunk(b"prVt", bytes(extra - 12)) + data[-12:]
def _noise_of_height(height: int, rng: random.Random) -> bytes:
raw = bytearray(rng.randbytes(NOISE_ROW * height))
for row in range(height):
raw[row * NOISE_ROW] = 0 # filter type "none"
compressor = zlib.compressobj(0)
return build_png(NOISE_WIDTH, height, compressor.compress(bytes(raw)) + compressor.flush())
def noise_png(size: int, seed: int = 0) -> bytes:
"""Incompressible, structurally valid PNG of exactly `size` bytes."""
rng = random.Random(seed)
height = max(1, size // NOISE_ROW)
while True:
png = _noise_of_height(height, rng)
if len(png) > size or 0 < size - len(png) < 12:
height -= 1
if height < 1:
raise ValueError("size is too small")
continue
return pad_png(png, size)
+23
View File
@@ -0,0 +1,23 @@
#!/bin/sh
# Experiment matrix: sizes x {proxy,direct} at C=10, plus proxy 9.5 MiB at C=20.
# Needs a running stand (make stand-up) and `make init-env` credentials.
set -eu
cd "$(dirname "$0")/../.."
out=evidence/lab4/bench
mkdir -p "$out"
bench() {
mode=$1 size=$2 conc=$3
python3 scripts/lab4/attachments_bench.py run --mode "$mode" --size-bytes "$size" \
--concurrency "$conc" --uploads-per-worker 5 --repeat 3 \
--output "$out/$mode-$size-c$conc.json"
}
for size in 131072 1048576 9961472; do
for mode in proxy direct; do
echo "== $mode size=$size c=10"
bench "$mode" "$size" 10
done
done
echo "== proxy size=9961472 c=20"
bench proxy 9961472 20
python3 scripts/lab4/attachments_bench.py summarize "$out"/*.json > evidence/lab4/summary.md
cat evidence/lab4/summary.md