lab5: базовый коммит из lab4 (abba248)

Перенесено дерево lab4 целиком: NestJS-приложение с вложениями и S3,
Dockerfile, compose.yaml, infra/{haproxy,minio}, стендовые скрипты,
scripts/lab4, evidence/lab4 и contracts/extensions.openapi.json.

Задание lab5 оставлено своим: README, REPORT, COURSE, lab.json,
contracts/{openapi,infrastructure,jobs.schema,attachment-states,examples},
tests, compose.example.yaml, .github/pull_request_template.md.

Адаптации при переносе:
- docker-проект mtusi-lab4 -> mtusi-lab5 и тег образа mtusi-messenger:lab5,
  чтобы тома и контейнеры не пересекались с lab4 (scripts/stop-one.sh:
  LAB_PROJECT вместо LAB4_PROJECT);
- Makefile: цели lab5 сохранены, добавлены стендовые (stand-up/down,
  wait-ready, init-env, test-e2e, test-dependencies, s3-security,
  consistency, bench);
- .env.example: реальные переменные приложения и стенда lab4 (без них
  compose.yaml не поднимается) + блок lab5 (BROKER_URL_FILE, WORKER_*).
This commit is contained in:
2026-11-04 23:44:11 +03:00
parent 276dbdad8a
commit 1982bdb690
153 changed files with 21742 additions and 11 deletions
+22
View File
@@ -0,0 +1,22 @@
#!/bin/sh
# Black-box набор приложения (app/test/*.e2e-spec.ts) против поднятого стенда.
# Требует loopback-портов PostgreSQL, Redis и MinIO из compose.yaml.
set -eu
env_file="${COMPOSE_ENV_FILE:-.env}"
compose_dir="$(pwd)"
# shellcheck disable=SC1090
set -a
. "./$env_file"
set +a
export API_BASE_URL="${API_BASE_URL:-http://127.0.0.1:${APP_PUBLIC_PORT:-8080}}"
export DATABASE_URL="postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in $env_file}@127.0.0.1:${POSTGRES_PUBLISHED_PORT:-5432}/${POSTGRES_DB:-messenger}"
export REDIS_URL="redis://127.0.0.1:${REDIS_PUBLISHED_PORT:-6379}"
export E2E_S3_PUBLIC_ENDPOINT="${S3_PUBLIC_ENDPOINT:-http://localhost:9000}"
export E2E_S3_BUCKET="${S3_BUCKET:-messenger-attachments}"
export E2E_COMPOSE_DIR="$compose_dir"
export E2E_COMPOSE_ENV_FILE="$env_file"
cd "$(dirname "$0")/../app"
exec npm run test:e2e -- "$@"
+40
View File
@@ -0,0 +1,40 @@
#!/bin/sh
# Идемпотентно создаёт .env и secrets/* для локального стенда. Значения не печатает.
set -eu
cd "$(dirname "$0")/.."
if [ ! -f .env ]; then
(
umask 077
pg=$(openssl rand -hex 24)
mt=$(openssl rand -hex 16)
sed \
-e "s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=${pg}|" \
-e "s|^DATABASE_URL=.*|DATABASE_URL=postgresql://messenger:${pg}@postgres:5432/messenger|" \
-e "s|^METRICS_TOKEN=.*|METRICS_TOKEN=${mt}|" \
.env.example > .env
)
echo "created .env"
else
echo "kept .env"
fi
mkdir -p secrets
chmod 700 secrets
write_secret() {
name=$1
value=$2
if [ -s "secrets/$name" ]; then
echo "kept secrets/$name"
return
fi
(umask 133; printf '%s' "$value" > "secrets/$name")
chmod 0644 "secrets/$name"
echo "created secrets/$name"
}
write_secret minio_root_user "root-$(openssl rand -hex 6)"
write_secret minio_root_password "$(openssl rand -hex 24)"
write_secret s3_api_access_key_id "api-$(openssl rand -hex 8)"
write_secret s3_api_secret_access_key "$(openssl rand -hex 24)"
+340
View File
@@ -0,0 +1,340 @@
#!/usr/bin/env python3
"""Upload benchmark: API traffic and peak RSS per upload for proxy vs direct mode.
attachments_bench.py run --mode proxy --size-bytes 1048576 --concurrency 10 \
--uploads-per-worker 5 --repeat 3 --output evidence/lab4/bench/proxy-1048576-c10.json
attachments_bench.py summarize evidence/lab4/bench/*.json > evidence/lab4/summary.md
Run from the repository root with the stand up. Tokens, signed URLs and secrets
are never written to the output.
"""
import argparse
import base64
import hashlib
import http.client
import json
import os
import statistics
import subprocess
import sys
import threading
import time
import urllib.parse
import uuid
from datetime import datetime, timezone
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from pngkit import noise_png # noqa: E402
def sh(*args, check=True):
return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip()
def read_env(path=".env"):
values = {}
with open(path) as handle:
for line in handle:
line = line.strip()
if line and not line.startswith("#") and "=" in line:
key, _, value = line.partition("=")
values[key] = value
return values
def request(host, port, method, path, headers=None, body=None):
connection = http.client.HTTPConnection(host, port, timeout=120)
try:
connection.request(method, path, body=body, headers=headers or {})
response = connection.getresponse()
payload = response.read()
return response.status, payload
finally:
connection.close()
def api_json(env, method, path, token=None, body=None, basic=None):
headers = {}
data = None
if body is not None:
headers["content-type"] = "application/json"
data = json.dumps(body).encode()
if token:
headers["authorization"] = f"Bearer {token}"
if basic:
headers["authorization"] = "Basic " + base64.b64encode(basic.encode()).decode()
port = int(env.get("APP_PUBLIC_PORT", "8080"))
status, payload = request("127.0.0.1", port, method, path, headers, data)
return status, json.loads(payload) if payload else None
def containers():
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "api").split()
return sorted(ids)
NODE_METRICS = (
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics'+process.argv[1],"
"{method:process.argv[1]?'POST':'GET',headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})"
".then(r=>r.text()).then(console.log)"
)
def metrics(container):
return json.loads(sh("docker", "exec", container, "node", "-e", NODE_METRICS, ""))
def reset_metrics(container):
sh("docker", "exec", container, "node", "-e", NODE_METRICS, "/reset")
def net_bytes(container):
for line in sh("docker", "exec", container, "cat", "/proc/net/dev").splitlines():
if line.strip().startswith("eth0:"):
fields = line.split(":", 1)[1].split()
return int(fields[0]), int(fields[8])
raise RuntimeError("eth0 not found")
def percentile(values, fraction):
ordered = sorted(values)
if not ordered:
return 0.0
return ordered[min(len(ordered) - 1, max(0, int(round(fraction * len(ordered) + 0.5)) - 1))]
def one_upload(env, mode, token, chat_id, data, seed):
"""Returns (ok, elapsed_ms, outcome); outcome names the failing step and status, e.g. "complete:429"."""
digest = hashlib.sha256(data).hexdigest()
filename = f"bench-{seed}.png"
port = int(env.get("APP_PUBLIC_PORT", "8080"))
started = time.perf_counter()
outcome, attachment = "ok", None
try:
if mode == "proxy":
status, body = request(
"127.0.0.1", port, "POST",
f"/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(filename)}",
{"authorization": f"Bearer {token}", "content-type": "image/png"}, data,
)
if status == 201:
attachment = json.loads(body)
else:
outcome = f"upload:{status}"
else:
headers = {"authorization": f"Bearer {token}", "content-type": "application/json"}
status, body = request(
"127.0.0.1", port, "POST", f"/api/v1/chats/{chat_id}/attachments/direct-uploads", headers,
json.dumps({"filename": filename, "content_type": "image/png",
"size_bytes": len(data), "sha256": digest}).encode(),
)
if status != 201:
outcome = f"init:{status}"
else:
ticket = json.loads(body)
target = urllib.parse.urlsplit(ticket["url"])
put_status, _ = request(
target.hostname, target.port, "PUT", target.path + "?" + target.query,
{**ticket["headers"]}, data,
)
if put_status != 200:
outcome = f"put:{put_status}"
else:
status, body = request(
"127.0.0.1", port, "POST", f"/api/v1/attachments/{ticket['attachment_id']}/complete",
{"authorization": f"Bearer {token}"},
)
if status == 200:
attachment = json.loads(body)
else:
outcome = f"complete:{status}"
except (OSError, http.client.HTTPException, ValueError) as error:
outcome = f"error:{type(error).__name__}"
elapsed_ms = (time.perf_counter() - started) * 1000
if outcome == "ok" and (attachment is None or attachment["sha256"] != digest):
outcome = "checksum_mismatch"
return outcome == "ok", elapsed_ms, outcome
def payload(args, seed):
if args.image:
with open(args.image, "rb") as handle:
return handle.read()
return noise_png(args.size_bytes, seed)
def run_once(env, args, actors, ids):
for container in ids:
reset_metrics(container)
time.sleep(0.2)
baseline = {c: metrics(c) for c in ids}
net_before = {c: net_bytes(c) for c in ids}
latencies, failures = [], []
outcomes = {}
lock = threading.Lock()
def worker(index):
for step in range(args.uploads_per_worker):
seed = index * 1000 + step
data = payload(args, seed)
token, chat_id = actors[index % len(actors)]
ok, elapsed, outcome = one_upload(env, args.mode, token, chat_id, data, seed)
with lock:
(latencies if ok else failures).append(elapsed)
outcomes[outcome] = outcomes.get(outcome, 0) + 1
started = time.perf_counter()
threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
duration = time.perf_counter() - started
time.sleep(0.2)
after = {c: metrics(c) for c in ids}
net_after = {c: net_bytes(c) for c in ids}
replicas = []
for c in ids:
base_rss = baseline[c]["memory_bytes"]["rss"]
peak = after[c]["memory_peak_bytes"]
replicas.append({
"instance_id": after[c]["instance_id"],
"delta_rx_bytes": net_after[c][0] - net_before[c][0],
"delta_tx_bytes": net_after[c][1] - net_before[c][1],
"baseline_rss_bytes": base_rss,
"peak_memory_bytes": peak,
"delta_rss_bytes": peak["rss"] - base_rss,
"counters": {k: v for k, v in after[c]["counters"].items() if k.startswith("attachments.")},
})
total = args.concurrency * args.uploads_per_worker
return {
"uploads": total,
"succeeded": len(latencies),
"failed": len(failures),
"outcomes": outcomes,
"duration_s": round(duration, 3),
"latency_ms": {
"p50": round(percentile(latencies, 0.50), 1),
"p95": round(percentile(latencies, 0.95), 1),
"p99": round(percentile(latencies, 0.99), 1),
},
"replicas": replicas,
"total_delta_rx_bytes": sum(r["delta_rx_bytes"] for r in replicas),
"total_delta_tx_bytes": sum(r["delta_tx_bytes"] for r in replicas),
"rx_bytes_per_upload": round(sum(r["delta_rx_bytes"] for r in replicas) / total),
"tx_bytes_per_upload": round(sum(r["delta_tx_bytes"] for r in replicas) / total),
"max_delta_rss_bytes": max(r["delta_rss_bytes"] for r in replicas),
}
def command_run(args):
if args.image:
args.size_bytes = os.path.getsize(args.image)
if args.size_bytes <= 0:
sys.exit("--size-bytes or --image is required")
env = read_env()
ids = containers()
if not ids:
sys.exit("no api containers found")
# One user per worker: the API caps concurrent uploads per user
# (UPLOAD_MAX_IN_FLIGHT_PER_USER), and real traffic comes from many users.
actors = []
for _ in range(args.concurrency):
name = f"bench_{uuid.uuid4().hex[:10]}"
password = "bench:" + uuid.uuid4().hex
status, user = api_json(env, "POST", "/api/v1/users",
body={"username": name, "display_name": "bench", "password": password})
assert status == 201, status
status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}")
assert status == 201, status
token = session["token"]
status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]})
assert status == 201, status
actors.append((token, chat["id"]))
# Warm-up: two uploads per replica, not measured.
for seed in range(len(ids) * 2):
token, chat_id = actors[seed % len(actors)]
one_upload(env, args.mode, token, chat_id, payload(args, 900000 + seed), seed)
limits = []
for c in ids:
limits.append(sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", c))
repeats = [run_once(env, args, actors, ids) for _ in range(args.repeat)]
mem_total = next(int(l.split()[1]) * 1024 for l in open("/proc/meminfo") if l.startswith("MemTotal"))
result = {
"date": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"git": sh("git", "rev-parse", "--short", "HEAD", check=False),
"params": {
"mode": args.mode, "size_bytes": args.size_bytes, "concurrency": args.concurrency,
"uploads_per_worker": args.uploads_per_worker, "repeat": args.repeat,
"users": args.concurrency, "image": os.path.basename(args.image) if args.image else None,
},
"host": {"nproc": os.cpu_count(), "mem_total_bytes": mem_total},
"api_container_limits_nanocpus_memory": limits,
"repeats": repeats,
}
os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
with open(args.output, "w") as handle:
json.dump(result, handle, indent=2)
handle.write("\n")
for index, item in enumerate(repeats):
print(f"repeat {index + 1}: ok={item['succeeded']}/{item['uploads']} "
f"rx/upload={item['rx_bytes_per_upload']} tx/upload={item['tx_bytes_per_upload']} "
f"maxΔrss={item['max_delta_rss_bytes']} p50={item['latency_ms']['p50']}ms "
f"outcomes={item['outcomes']}")
def mib(value):
return f"{value / 1048576:.2f}"
def command_summarize(args):
rows = []
for path in args.paths:
with open(path) as handle:
data = json.load(handle)
p = data["params"]
reps = data["repeats"]
med = lambda key: statistics.median(r[key] for r in reps) # noqa: E731
label = p["mode"] + (f" ({p['image']})" if p.get("image") else "")
rows.append((label, p["size_bytes"], p["concurrency"], reps, med, data))
rows.sort(key=lambda r: (r[0].count("("), r[1], r[0], r[2]))
print("| mode | size, MiB | C | ok | failures | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | peak RSS, MiB | p50, ms | p95, ms |")
print("|---|---|---|---|---|---|---|---|---|---|---|")
for mode, size, conc, reps, med, _data in rows:
peak = max(x["peak_memory_bytes"]["rss"] for r in reps for x in r["replicas"])
failed = {}
for rep in reps:
for name, count in rep.get("outcomes", {}).items():
if name != "ok":
failed[name] = failed.get(name, 0) + count
failures = ", ".join(f"{name}×{count}" for name, count in sorted(failed.items())) or "—"
ok = sum(r["succeeded"] for r in reps)
total = sum(r["uploads"] for r in reps)
p50 = statistics.median(r["latency_ms"]["p50"] for r in reps)
p95 = statistics.median(r["latency_ms"]["p95"] for r in reps)
print(f"| {mode} | {mib(size)} | {conc} | {ok}/{total} | {failures} | {mib(med('rx_bytes_per_upload'))} | "
f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {mib(peak)} | {p50:.0f} | {p95:.0f} |")
def main():
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(dest="command", required=True)
run = sub.add_parser("run")
run.add_argument("--mode", choices=["proxy", "direct"], required=True)
run.add_argument("--size-bytes", type=int, default=0)
run.add_argument("--image", help="upload this file instead of generated noise PNGs")
run.add_argument("--concurrency", type=int, default=10)
run.add_argument("--uploads-per-worker", type=int, default=5)
run.add_argument("--repeat", type=int, default=3)
run.add_argument("--output", required=True)
summarize = sub.add_parser("summarize")
summarize.add_argument("paths", nargs="+")
args = parser.parse_args()
{"run": command_run, "summarize": command_summarize}[args.command](args)
if __name__ == "__main__":
main()
+54
View File
@@ -0,0 +1,54 @@
#!/bin/sh
# Проверка приватности S3 вне API: анонимный доступ, права отдельных credentials
# API, закрытая консоль. Секреты не печатаются.
set -u
cd "$(dirname "$0")/../.."
env_file="${COMPOSE_ENV_FILE:-.env}"
# shellcheck disable=SC1090
set -a; . "./$env_file"; set +a
endpoint="${S3_PUBLIC_ENDPOINT:-http://localhost:9000}"
bucket="${S3_BUCKET:-messenger-attachments}"
compose="docker compose --env-file $env_file"
fail=0
report() { # status name
if [ "$1" = ok ]; then echo "PASS $2"; else echo "FAIL $2"; fail=1; fi
}
key=$($compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -At \
-c "SELECT object_key FROM attachments WHERE status = 'ready' ORDER BY created_at DESC LIMIT 1")
if [ -z "$key" ]; then
echo "FAIL no ready attachment to check; upload one first (make test)"
exit 1
fi
code=$(curl -s -o /dev/null -w '%{http_code}' "$endpoint/$bucket/$key")
[ "$code" = 403 ] && report ok "anonymous GET of a ready object -> 403" || report fail "anonymous GET of a ready object -> $code (want 403)"
code=$(curl -s -o /dev/null -w '%{http_code}' "$endpoint/$bucket/")
[ "$code" = 403 ] && report ok "anonymous list of the bucket -> 403" || report fail "anonymous list of the bucket -> $code (want 403)"
code=$(curl -s -o /dev/null -w '%{http_code}' -X PUT --data-binary x "$endpoint/$bucket/attachments/anonymous-write")
[ "$code" = 403 ] && report ok "anonymous PUT -> 403" || report fail "anonymous PUT -> $code (want 403)"
# The API credentials only, inside a one-off container (secrets come from files).
api_mc() {
$compose run --rm --no-deps -T --entrypoint /bin/sh s3-init -c '
IFS= read -r k < /run/secrets/s3_api_access_key_id
IFS= read -r s < /run/secrets/s3_api_secret_access_key
export MC_CONFIG_DIR=/tmp/.mc MC_HOST_api="http://$k:$s@s3:9000"
exec mc "$@"' sh "$@" >/dev/null 2>&1
}
api_mc mb api/forbidden-bucket && report fail "API credentials can create buckets" || report ok "API credentials cannot create buckets"
api_mc admin info api && report fail "API credentials can use admin API" || report ok "API credentials cannot use admin API"
api_mc stat "api/$bucket/$key" && report ok "API credentials can read attachments/*" || report fail "API credentials cannot read attachments/*"
api_mc stat "api/$bucket/outside-prefix" ; rc=$?
api_mc cp /etc/hostname "api/$bucket/outside-prefix" && report fail "API credentials can write outside attachments/" || report ok "API credentials cannot write outside attachments/"
: "$rc"
port=$($compose port s3 9001 2>/dev/null || true)
[ -z "$port" ] && report ok "console port 9001 is not published" || report fail "console port 9001 is published ($port)"
if curl -s -o /dev/null --max-time 3 http://localhost:9001; then report fail "console reachable on localhost:9001"; else report ok "console not reachable on localhost:9001"; fi
exit "$fail"
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# Сверка bucket и БД (только чтение): orphans — объект без строки, missing —
# ready-строка без объекта. Код выхода 1, если что-то найдено.
set -eu
cd "$(dirname "$0")/../.."
env_file="${COMPOSE_ENV_FILE:-.env}"
# shellcheck disable=SC1090
set -a; . "./$env_file"; set +a
bucket="${S3_BUCKET:-messenger-attachments}"
compose="docker compose --env-file $env_file"
objects=$(mktemp)
rows=$(mktemp)
trap 'rm -f "$objects" "$rows"' EXIT
$compose run --rm --no-deps -T --entrypoint /bin/sh s3-init -c '
IFS= read -r u < /run/secrets/minio_root_user
IFS= read -r p < /run/secrets/minio_root_password
export MC_CONFIG_DIR=/tmp/.mc MC_HOST_stand="http://$u:$p@s3:9000"
exec mc ls --recursive "stand/$S3_BUCKET"' 2>/dev/null \
| sed -n 's/.* \(attachments\/.*\)$/\1/p' | sort > "$objects"
$compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -At \
-c "SELECT object_key, status FROM attachments" | sort > "$rows"
echo "bucket: $bucket"
echo "objects: $(wc -l < "$objects" | tr -d ' ')"
echo "rows by status:"
cut -d'|' -f2 "$rows" | sort | uniq -c | sed 's/^ */ /'
orphans=$(cut -d'|' -f1 "$rows" | sort | comm -13 - "$objects")
missing=$(grep '|ready$' "$rows" | cut -d'|' -f1 | sort | comm -23 - "$objects")
echo "orphans: $(printf '%s' "$orphans" | grep -c . || true)"
[ -z "$orphans" ] || printf '%s\n' "$orphans" | sed 's/^/ orphan /'
echo "missing: $(printf '%s' "$missing" | grep -c . || true)"
[ -z "$missing" ] || printf '%s\n' "$missing" | sed 's/^/ missing /'
[ -z "$orphans$missing" ]
+33
View File
@@ -0,0 +1,33 @@
// Peak RSS of one full decode check with the same sharp pipeline as
// app/src/attachments/image/image-decoder.ts, on worst-case inputs near
// MAX_IMAGE_PIXELS. Runs where the API runs (same libvips, VIPS_DISC_THRESHOLD):
// docker compose exec -T api node - < scripts/lab4/decode_memory.mjs # reads files from IMAGES (paths in the container)
// or on the host from app/: IMAGES="a.png b.jpg" node ../scripts/lab4/decode_memory.mjs
import { readFileSync } from 'node:fs';
import { createRequire } from 'node:module';
const sharp = createRequire(`${process.cwd()}/`)('sharp');
sharp.cache(false);
sharp.concurrency(1);
for (const file of (process.env.IMAGES ?? '').split(/\s+/u).filter(Boolean)) {
const data = readFileSync(file);
const meta = await sharp(data).metadata();
const base = process.memoryUsage().rss;
let peak = base;
const timer = setInterval(() => {
peak = Math.max(peak, process.memoryUsage().rss);
}, 5);
const started = Date.now();
let verdict = 'ok';
try {
await sharp(data, { failOn: 'warning', sequentialRead: true, limitInputPixels: 40_000_000 })
.resize(64, 64, { fit: 'inside' })
.toBuffer();
} catch (error) {
verdict = `rejected: ${String(error.message).split('\n')[0]}`;
}
clearInterval(timer);
console.log(
`${meta.format} ${meta.width}x${meta.height} ${meta.channels}ch ${meta.depth}${meta.isProgressive ? ' progressive' : ''}${meta.isPalette ? ' palette' : ''} bytes=${data.length} decode_ms=${Date.now() - started} peak_delta_rss_mib=${((peak - base) / 1048576).toFixed(1)} ${verdict}`,
);
}
+78
View File
@@ -0,0 +1,78 @@
"""Synthetic PNG helpers (stdlib only); same semantics as app/test/support/png.ts."""
import random
import struct
import zlib
SIGNATURE = b"\x89PNG\r\n\x1a\n"
NOISE_WIDTH = 1024
NOISE_ROW = 1 + NOISE_WIDTH * 3
def chunk(kind: bytes, data: bytes) -> bytes:
crc = zlib.crc32(kind + data) & 0xFFFFFFFF
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", crc)
def build_png(width: int, height: int, idat: bytes) -> bytes:
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return SIGNATURE + chunk(b"IHDR", ihdr) + chunk(b"IDAT", idat) + chunk(b"IEND", b"")
def pad_png(data: bytes, size: int) -> bytes:
"""Insert a private ancillary chunk before IEND to reach exactly `size` bytes."""
extra = size - len(data)
if extra == 0:
return data
if extra < 12:
raise ValueError("padding needs at least 12 bytes")
return data[:-12] + chunk(b"prVt", bytes(extra - 12)) + data[-12:]
def _noise_of_height(height: int, rng: random.Random) -> bytes:
raw = bytearray(rng.randbytes(NOISE_ROW * height))
for row in range(height):
raw[row * NOISE_ROW] = 0 # filter type "none"
compressor = zlib.compressobj(0)
return build_png(NOISE_WIDTH, height, compressor.compress(bytes(raw)) + compressor.flush())
def noise_png(size: int, seed: int = 0) -> bytes:
"""Incompressible, structurally valid PNG of exactly `size` bytes."""
rng = random.Random(seed)
height = max(1, size // NOISE_ROW)
while True:
png = _noise_of_height(height, rng)
if len(png) > size or 0 < size - len(png) < 12:
height -= 1
if height < 1:
raise ValueError("size is too small")
continue
return pad_png(png, size)
ADAM7 = [(0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)]
def worst_decode_png(size: int = 10_485_760, width: int = 8000, height: int = 5000) -> bytes:
"""The heaviest valid upload to decode: 40 MP RGBA 16-bit, Adam7-interlaced
(the decoder must hold the whole image), padded to exactly `size` bytes."""
compressor = zlib.compressobj(1)
parts = []
for x0, y0, dx, dy in ADAM7:
pass_width = (width - x0 + dx - 1) // dx if width > x0 else 0
pass_height = (height - y0 + dy - 1) // dy if height > y0 else 0
if pass_width == 0 or pass_height == 0:
continue
row = bytes(1 + pass_width * 8)
for _ in range(pass_height):
parts.append(compressor.compress(row))
parts.append(compressor.flush())
ihdr = struct.pack(">IIBBBBB", width, height, 16, 6, 0, 0, 1)
png = SIGNATURE + chunk(b"IHDR", ihdr) + chunk(b"IDAT", b"".join(parts)) + chunk(b"IEND", b"")
return pad_png(png, size)
if __name__ == "__main__":
import sys
sys.stdout.buffer.write(worst_decode_png())
+36
View File
@@ -0,0 +1,36 @@
#!/bin/sh
# Experiment matrix: sizes x {proxy,direct} at C=10, plus proxy 9.5 MiB at C=20.
# Needs a running stand (make stand-up) and `make init-env` credentials.
set -eu
cd "$(dirname "$0")/../.."
out=evidence/lab4/bench
mkdir -p "$out"
bench() {
mode=$1 size=$2 conc=$3
python3 scripts/lab4/attachments_bench.py run --mode "$mode" --size-bytes "$size" \
--concurrency "$conc" --uploads-per-worker 5 --repeat 3 \
--output "$out/$mode-$size-c$conc.json"
}
for size in 131072 1048576 9961472; do
for mode in proxy direct; do
echo "== $mode size=$size c=10"
bench "$mode" "$size" 10
done
done
echo "== proxy size=9961472 c=20"
bench proxy 9961472 20
# Worst case for memory: every request carries the heaviest valid image
# (40 MP RGBA16 Adam7 PNG padded to 10 MiB, ~340 MiB per decode) and C=32
# saturates UPLOAD_MAX_IN_FLIGHT (16) on both replicas.
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
worst="$tmp/worst-40mp-rgba16-adam7.png"
python3 scripts/lab4/pngkit.py > "$worst"
for mode in proxy direct; do
echo "== $mode worst-case image c=32"
python3 scripts/lab4/attachments_bench.py run --mode "$mode" --image "$worst" \
--concurrency 32 --uploads-per-worker 2 --repeat 3 \
--output "$out/$mode-worst-c32.json"
done
python3 scripts/lab4/attachments_bench.py summarize "$out"/*.json > evidence/lab4/summary.md
cat evidence/lab4/summary.md
+8
View File
@@ -0,0 +1,8 @@
#!/usr/bin/env bash
# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`.
# Semantic and evidence notes: tests/README.md ("Проверка сохранности") and REPORT.md.
set -euo pipefail
compose=(docker compose --env-file .env)
"${compose[@]}" restart postgres redis s3 api haproxy
"${compose[@]}" up -d --no-recreate --no-deps --wait postgres redis s3 api haproxy
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
# Восстанавливает недостающую API-реплику до объявленного числа реплик.
set -eu
env_file="${COMPOSE_ENV_FILE:-.env}"
replicas="${API_REPLICAS:-}"
if [ -z "$replicas" ] && [ -f "$env_file" ]; then
replicas=$(sed -n 's/^API_REPLICAS=//p' "$env_file" | tail -n 1)
fi
replicas="${replicas:-2}"
exec docker compose --env-file "$env_file" up -d --no-deps --scale "api=${replicas}" api
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# Останавливает ровно одну API-реплику стенда — ту, которую обслужила текущая
# сессия. TARGET_INSTANCE_ID приходит из теста (значение X-Instance-Id ответа),
# а в Docker hostname контейнера равен его короткому ID, поэтому сопоставление
# однозначно. Ничего, кроме этой реплики, скрипт не трогает: ни LB, ни БД, ни
# тома. Возврат управления — быстрый (SIGTERM -> дрейн DRAIN_DELAY_MS=3s ->
# выход), останавливать реплику "навсегда" не требуется.
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
project="${LAB_PROJECT:-mtusi-lab5}"
target=''
for container in $(
docker ps \
--filter "label=com.docker.compose.project=${project}" \
--filter 'label=com.docker.compose.service=api' \
--format '{{.ID}}'
); do
case "$container" in
"$TARGET_INSTANCE_ID"*)
target="$container"
break
;;
esac
done
if [ -z "$target" ]; then
printf 'stop-one: no running api replica of project %s whose id starts with %s\n' \
"$project" "$TARGET_INSTANCE_ID" >&2
exit 1
fi
printf 'stop-one: stopping api replica %s\n' "$target" >&2
docker stop --time 10 "$target" >/dev/null