forked from mtusi2026/mtusi2026lab4
lab4 трек 2: reconciler S3/PostgreSQL, тесты вложений, отказов и согласованности
- python -m messenger.reconcile: merge-join листинга S3 и attachments по object_key; dry-run read-only конструктивно (default_transaction_read_only + S3 без записи), --apply: orphan/stale intent удаляются, ready-строки только в карантин (обратимо), перепроверка под FOR UPDATE, предохранитель от массового удаления; - attachment_ids хранится в строке сообщения (история остаётся одним index range scan), message_attachments — для FK и обратного поиска; - семафор полной проверки изображений (IMAGE_VERIFY_CONCURRENCY); - тесты: вложения (56), отказ/пауза S3 и деградированный режим LB, failover загрузки, приватность bucket, FAULT_POINTS после PUT/после COMMIT, janitor, reconciler; - бенчмарки памяти загрузок и стоимости обхода reconciler. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
335df4b2b2
commit
a2d867bec6
+26
-1
@@ -17,6 +17,29 @@ API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
# Write path of POST message: function = one SQL call (migration 0003), multi = lab3 baseline path.
|
||||
MESSAGE_WRITE_PATH=function
|
||||
# Lab 4: private S3 bucket (SeaweedFS service `s3`). The API gets ITS OWN key pair (Read+Write on
|
||||
# this bucket only) as Docker secrets; admin/reconciler pairs go only to s3-init/reconciler.
|
||||
S3_ENDPOINT=http://s3:8333
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
# Contract limits (may be lowered, never raised above 10485760 / 40000000).
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
# Per replica: bodies buffered at once (RAM bound), wait for a slot, body deadline, image checks at once.
|
||||
UPLOAD_CONCURRENCY=4
|
||||
UPLOAD_QUEUE_TIMEOUT_SECONDS=5
|
||||
UPLOAD_READ_TIMEOUT_SECONDS=30
|
||||
IMAGE_VERIFY_TIMEOUT_SECONDS=10
|
||||
IMAGE_VERIFY_CONCURRENCY=1
|
||||
JPEG_MAX_MEMORY_MB=256
|
||||
S3_TIMEOUT_SECONDS=3
|
||||
# Janitor: 'pending' intents older than this are failed uploads (object + row removed).
|
||||
PENDING_GRACE_SECONDS=900
|
||||
JANITOR_INTERVAL_SECONDS=60
|
||||
# Level 4 failure experiments only (after_s3_put, after_ready_commit); keep empty.
|
||||
FAULT_POINTS=
|
||||
# Additional settings of this implementation (defaults shown; all optional):
|
||||
LOG_LEVEL=info
|
||||
DB_POOL_MIN=1
|
||||
@@ -30,7 +53,9 @@ DRAIN_SECONDS=3
|
||||
SHUTDOWN_TIMEOUT_SECONDS=8
|
||||
# CPU/RAM limits (docker compose deploy.resources.limits); API_* are per replica.
|
||||
API_CPUS=1.0
|
||||
API_MEMORY=256m
|
||||
API_MEMORY=512m
|
||||
S3_CPUS=1.0
|
||||
S3_MEMORY=512m
|
||||
PG_CPUS=2.0
|
||||
PG_MEMORY=1g
|
||||
VALKEY_CPUS=0.5
|
||||
|
||||
@@ -29,9 +29,9 @@ test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Student solution (lab 3, Python). On Windows: PowerShell/Git Bash with GNU make, public
|
||||
# Student solution (lab 4, Python). On Windows: PowerShell/Git Bash with GNU make, public
|
||||
# tests need PYTHONUTF8=1, and the hooks are started through scripts/*.cmd (Git Bash).
|
||||
.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series
|
||||
.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series reconcile reconcile-apply bench-memory bench-reconcile
|
||||
|
||||
secrets:
|
||||
$(PYTHON) scripts/init_secrets.py
|
||||
@@ -80,3 +80,18 @@ unit-test:
|
||||
|
||||
explain:
|
||||
$(PYTHON) scripts/explain_history.py --output evidence/explain-history.txt
|
||||
|
||||
# Track 2: reconciler of S3 objects vs attachment rows (dry-run by default, read-only).
|
||||
reconcile:
|
||||
docker compose run --rm reconciler
|
||||
|
||||
reconcile-apply:
|
||||
docker compose run --rm reconciler --apply
|
||||
|
||||
# Level 4: peak RAM of the replicas under 16 simultaneous uploads (evidence/memory/).
|
||||
bench-memory:
|
||||
$(PYTHON) bench/upload_memory.py --output evidence/memory
|
||||
|
||||
# Track 2: cost of one reconciler pass for 1k/10k/50k attachments (evidence/reconcile/).
|
||||
bench-reconcile:
|
||||
$(PYTHON) bench/reconcile_cost.py --output evidence/reconcile
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 2 experiment: cost of one reconciler pass as a function of the number of attachments.
|
||||
|
||||
Hypothesis: a pass is O(N) with ~N/1000 S3 list requests + N/1000 DB pages and constant memory
|
||||
(merge-join of two key-ordered streams); --verify-checksums adds reading every byte (O(total size)).
|
||||
|
||||
Method: synthetic, CONSISTENT attachments (row 'ready' + 1 KiB object with the matching size and
|
||||
SHA-256, filename 'bench-reconcile') are added in steps up to the largest N; after each step the
|
||||
reconciler runs in dry-run 3 times (median reported), plus page-size and checksum variants.
|
||||
Everything synthetic is deleted at the end (objects first, then rows).
|
||||
|
||||
python bench/reconcile_cost.py --steps 1000 10000 50000 --output evidence/reconcile
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import httpx2 as httpx
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
PASSWORD = "Bench-password-Ω-12345"
|
||||
OBJECT = bytes(range(256)) * 4 # 1 KiB, the same content for every object
|
||||
SHA = hashlib.sha256(OBJECT).hexdigest()
|
||||
|
||||
PUT_SCRIPT = """
|
||||
import asyncio, sys
|
||||
from messenger.config import Settings, s3_settings
|
||||
from messenger.storage import Storage
|
||||
DATA = bytes(range(256)) * 4
|
||||
async def main():
|
||||
s = Storage(Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings()))
|
||||
await s.open()
|
||||
keys = sys.stdin.read().split()
|
||||
sem = asyncio.Semaphore(32)
|
||||
async def one(k):
|
||||
async with sem:
|
||||
await s.{op}(k{args})
|
||||
await asyncio.gather(*(one(k) for k in keys))
|
||||
await s.close()
|
||||
print(len(keys))
|
||||
asyncio.run(main())
|
||||
"""
|
||||
|
||||
|
||||
def compose(*args: str, stdin: str | None = None, timeout: float = 1800, check: bool = True) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(["docker", "compose", *args], cwd=ROOT, input=stdin, capture_output=True, text=True,
|
||||
encoding="utf-8", timeout=timeout, check=check)
|
||||
|
||||
|
||||
def psql(sql: str) -> str:
|
||||
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v",
|
||||
"ON_ERROR_STOP=1", "-c", sql).stdout.strip()
|
||||
|
||||
|
||||
def s3_bulk(op: str, keys: list[str]) -> None:
|
||||
script = PUT_SCRIPT.replace("{op}", op).replace("{args}", ", DATA, 'image/png', {}" if op == "put" else "")
|
||||
compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "python", "reconciler", "-c", script, stdin="\n".join(keys))
|
||||
|
||||
|
||||
def owner_and_chat() -> tuple[str, str]:
|
||||
import base64
|
||||
c = httpx.Client(base_url=BASE, timeout=30)
|
||||
name = "rc_" + uuid.uuid4().hex[:10]
|
||||
c.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
|
||||
token = c.post("/api/v1/auth/sessions", headers={
|
||||
"Authorization": "Basic " + base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()}).json()["token"]
|
||||
me = c.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
|
||||
chat = c.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
|
||||
json={"title": "bench", "member_ids": [me["id"]]}).json()
|
||||
return me["id"], chat["id"]
|
||||
|
||||
|
||||
def add(n: int, owner: str, chat: str) -> None:
|
||||
keys = ["o/" + uuid.uuid4().hex for _ in range(n)]
|
||||
for i in range(0, n, 5000):
|
||||
part = keys[i:i + 5000]
|
||||
values = ",".join(f"('{uuid.uuid4()}','{k}')" for k in part)
|
||||
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
|
||||
"object_key, status, created_at, updated_at) "
|
||||
f"SELECT v.id::uuid, '{chat}', '{owner}', 'bench-reconcile', 'image/png', {len(OBJECT)}, "
|
||||
f"decode('{SHA}', 'hex'), 1, 1, v.k, 'ready', clock_timestamp() - interval '2 hours', "
|
||||
f"clock_timestamp() - interval '2 hours' FROM (VALUES {values}) AS v(id, k)")
|
||||
s3_bulk("put", keys)
|
||||
|
||||
|
||||
def reconcile(*args: str) -> dict:
|
||||
out = compose("run", "--rm", "--no-deps", "-T", "reconciler", *args).stdout
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def measure(label: str, args: list[str], repeats: int) -> dict:
|
||||
runs = [reconcile(*args) for _ in range(repeats)]
|
||||
durations = [r["duration_seconds"] for r in runs]
|
||||
last = runs[-1]
|
||||
return {"label": label, "args": args, "runs": repeats, "duration_s_median": statistics.median(durations),
|
||||
"duration_s_all": durations, "objects": last["scanned"]["objects"], "rows": last["scanned"]["rows"],
|
||||
"findings": {k: v for k, v in last["findings"].items() if v}, "cost": last["cost"]}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--steps", type=int, nargs="+", default=[1000, 10000, 50000])
|
||||
p.add_argument("--repeats", type=int, default=3)
|
||||
p.add_argument("--output", default="evidence/reconcile")
|
||||
args = p.parse_args()
|
||||
out = ROOT / args.output
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
owner, chat = owner_and_chat()
|
||||
present = 0
|
||||
results = []
|
||||
steps = sorted(args.steps)
|
||||
variant_step = steps[min(1, len(steps) - 1)]
|
||||
try:
|
||||
for target in sorted(args.steps):
|
||||
started = time.perf_counter()
|
||||
add(target - present, owner, chat)
|
||||
present = target
|
||||
print(f"populated {present} synthetic attachments in {time.perf_counter() - started:.1f} s", flush=True)
|
||||
results.append({"synthetic": present, **measure("dry-run, page 1000", [], args.repeats)})
|
||||
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
|
||||
if present == variant_step: # page-size and checksum variants at one size
|
||||
for label, extra in (("dry-run, page 100", ["--page-size", "100"]),
|
||||
("dry-run, --verify-checksums", ["--verify-checksums"])):
|
||||
results.append({"synthetic": present, **measure(label, extra, args.repeats)})
|
||||
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
|
||||
finally:
|
||||
keys = psql("SELECT object_key FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = "
|
||||
f"'{chat}'").split()
|
||||
if keys:
|
||||
s3_bulk("delete", keys)
|
||||
psql(f"DELETE FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = '{chat}'")
|
||||
print(f"removed {len(keys)} synthetic attachments", flush=True)
|
||||
meta = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
|
||||
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
|
||||
text=True).stdout.strip(),
|
||||
"object_bytes": len(OBJECT), "reconciler_limits": "cpus 1.0, memory 256m (compose)",
|
||||
"note": "the bucket/table also contain the attachments of earlier test runs (counted in objects/rows)"}
|
||||
(out / "cost.json").write_text(json.dumps({"meta": meta, "results": results}, ensure_ascii=False, indent=1),
|
||||
encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Level 4 experiment: peak memory of the API replicas under >= 10 simultaneous uploads.
|
||||
|
||||
Hypothesis: the memory of one replica is bounded by UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES (bodies
|
||||
waiting for a slot stay in TCP buffers, not in the process) + one image check at a time
|
||||
(IMAGE_VERIFY_CONCURRENCY), independent of how many clients upload at once.
|
||||
|
||||
Method (repeatable, the stand must be up):
|
||||
* for every scenario the api replicas are RECREATED with the scenario's settings, so the cgroup
|
||||
counter memory.peak starts from an idle process (it never decreases during a container's life);
|
||||
* idle memory.current of each replica is read; then N clients upload at the same moment through
|
||||
the LB (a barrier releases them together), each a different file;
|
||||
* during the run memory.current of every replica is sampled every 50 ms inside the container;
|
||||
afterwards memory.peak (cgroup v2: all processes of the container incl. djpeg) and VmHWM of
|
||||
PID 1 (the Python server alone) are read;
|
||||
* the normal configuration is restored at the end.
|
||||
|
||||
python bench/upload_memory.py --output evidence/memory
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import httpx2 as httpx
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
import imagegen # noqa: E402
|
||||
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
PASSWORD = "Bench-password-Ω-12345"
|
||||
MiB = 1024 * 1024
|
||||
|
||||
|
||||
def sh(*args: str, env: dict | None = None, timeout: float = 300) -> str:
|
||||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True, timeout=timeout,
|
||||
env={**os.environ, **(env or {})}).stdout
|
||||
|
||||
|
||||
def replicas() -> list[str]:
|
||||
return sh("docker", "compose", "ps", "-q", "api").split()
|
||||
|
||||
|
||||
def cg(cid: str, name: str) -> int:
|
||||
return int(sh("docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}").split()[0])
|
||||
|
||||
|
||||
def vmhwm(cid: str) -> int:
|
||||
for line in sh("docker", "exec", cid, "cat", "/proc/1/status").splitlines():
|
||||
if line.startswith("VmHWM:"):
|
||||
return int(line.split()[1]) * 1024
|
||||
return 0
|
||||
|
||||
|
||||
def recreate(env: dict) -> None:
|
||||
sh("docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "--wait", "api", env=env, timeout=300)
|
||||
sh(sys.executable, "scripts/lbctl.py", "wait-up", str(len(replicas())), "--timeout", "90") # LB sees them UP
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
class Sampler(threading.Thread):
|
||||
"""memory.current of one container every 50 ms, read by a shell loop inside it."""
|
||||
|
||||
def __init__(self, cid: str):
|
||||
super().__init__(daemon=True)
|
||||
self.cid = cid
|
||||
self.values: list[int] = []
|
||||
self.proc = subprocess.Popen(["docker", "exec", cid, "sh", "-c",
|
||||
"while :; do cat /sys/fs/cgroup/memory.current; sleep 0.05; done"],
|
||||
stdout=subprocess.PIPE, text=True)
|
||||
|
||||
def run(self) -> None:
|
||||
for line in self.proc.stdout:
|
||||
if line.strip().isdigit():
|
||||
self.values.append(int(line))
|
||||
|
||||
def stop(self) -> None:
|
||||
self.proc.kill()
|
||||
|
||||
|
||||
def actor(client: httpx.Client) -> tuple[str, str]:
|
||||
name = "mem_" + uuid.uuid4().hex[:10]
|
||||
client.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
|
||||
import base64
|
||||
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
|
||||
token = client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}).json()["token"]
|
||||
me = client.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
|
||||
chat = client.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
|
||||
json={"title": "mem", "member_ids": [me["id"]]}).json()
|
||||
return token, chat["id"]
|
||||
|
||||
|
||||
def big_jpeg(width: int, height: int, seed: int) -> bytes:
|
||||
"""Large progressive JPEG (smooth content keeps it < 10 MiB): the costliest file to check."""
|
||||
image = Image.linear_gradient("L").resize((width, height)).convert("RGB")
|
||||
image.putpixel((seed % width, 0), (seed % 255, 0, 0))
|
||||
out = io.BytesIO()
|
||||
image.save(out, "JPEG", quality=90, progressive=True, subsampling=0)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
def run_scenario(name: str, env: dict, files: list[tuple[bytes, str]]) -> dict:
|
||||
recreate(env)
|
||||
ids = replicas()
|
||||
idle = {cid: cg(cid, "memory.current") for cid in ids}
|
||||
client = httpx.Client(base_url=BASE, timeout=120)
|
||||
token, chat_id = actor(client)
|
||||
barrier = threading.Barrier(len(files))
|
||||
results: list[tuple[int, float, str]] = [None] * len(files) # type: ignore[list-item]
|
||||
|
||||
def upload(i: int, data: bytes, ctype: str) -> None:
|
||||
c = httpx.Client(base_url=BASE, timeout=120)
|
||||
barrier.wait()
|
||||
started = time.perf_counter()
|
||||
r = c.post(f"/api/v1/chats/{chat_id}/attachments?filename=f{i}", content=data,
|
||||
headers={"Authorization": "Bearer " + token, "Content-Type": ctype})
|
||||
results[i] = (r.status_code, time.perf_counter() - started, r.headers.get("x-instance-id", "-"))
|
||||
c.close()
|
||||
|
||||
samplers = [Sampler(cid) for cid in ids]
|
||||
for s in samplers:
|
||||
s.start()
|
||||
time.sleep(0.5)
|
||||
threads = [threading.Thread(target=upload, args=(i, d, t)) for i, (d, t) in enumerate(files)]
|
||||
wall = time.perf_counter()
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
wall = time.perf_counter() - wall
|
||||
time.sleep(0.5)
|
||||
for s in samplers:
|
||||
s.stop()
|
||||
per_replica = {}
|
||||
for cid, s in zip(ids, samplers):
|
||||
hostname = sh("docker", "inspect", "-f", "{{.Config.Hostname}}", cid).strip()
|
||||
per_replica[hostname] = {
|
||||
"idle_mib": round(idle[cid] / MiB, 1),
|
||||
"peak_mib": round(cg(cid, "memory.peak") / MiB, 1),
|
||||
"sampled_max_mib": round(max(s.values or [0]) / MiB, 1),
|
||||
"python_vmhwm_mib": round(vmhwm(cid) / MiB, 1),
|
||||
"uploads_served": sum(1 for r in results if r[2] == hostname),
|
||||
"limit_mib": round(cg(cid, "memory.max") / MiB, 1),
|
||||
}
|
||||
per_replica[hostname]["peak_minus_idle_mib"] = round(per_replica[hostname]["peak_mib"]
|
||||
- per_replica[hostname]["idle_mib"], 1)
|
||||
lat = sorted(r[1] for r in results if r[0] == 201)
|
||||
codes: dict[str, int] = {}
|
||||
for r in results:
|
||||
codes[str(r[0])] = codes.get(str(r[0]), 0) + 1
|
||||
summary = {
|
||||
"scenario": name, "settings": env, "clients": len(files),
|
||||
"file_mib_each": round(statistics.mean(len(d) for d, _ in files) / MiB, 2),
|
||||
"types": sorted({t for _, t in files}), "status_codes": codes, "wall_seconds": round(wall, 2),
|
||||
"latency_s": {"p50": round(lat[len(lat) // 2], 2), "max": round(lat[-1], 2)} if lat else None,
|
||||
"replicas": per_replica,
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False), flush=True)
|
||||
return summary
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--clients", type=int, default=16)
|
||||
p.add_argument("--output", default="evidence/memory")
|
||||
args = p.parse_args()
|
||||
out = ROOT / args.output
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
print("generating files ...", flush=True)
|
||||
pngs = [(imagegen.png_of_size(int(9.5 * MiB)), "image/png") for _ in range(args.clients)]
|
||||
jpegs = [(big_jpeg(8000, 5000, i), "image/jpeg") for i in range(args.clients)]
|
||||
base = {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}
|
||||
scenarios = [
|
||||
("png-9.5MiB x16, slots=4 (default)", base, pngs),
|
||||
("png-9.5MiB x16, slots=16 (no bound)", {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, pngs),
|
||||
("jpeg-40Mpx-progressive x16, slots=4, verify=1 (default)", base, jpegs),
|
||||
("jpeg-40Mpx-progressive x16, slots=4, verify=4", {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, jpegs),
|
||||
]
|
||||
results = []
|
||||
try:
|
||||
for name, env, files in scenarios:
|
||||
results.append(run_scenario(name, env, files))
|
||||
finally:
|
||||
recreate({}) # back to the compose defaults
|
||||
meta = {
|
||||
"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
|
||||
"commit": sh("git", "rev-parse", "--short", "HEAD").strip(),
|
||||
"api_limits": sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}",
|
||||
replicas()[0]).strip(),
|
||||
"jpeg_file_mib": round(len(jpegs[0][0]) / MiB, 2), "png_file_mib": round(len(pngs[0][0]) / MiB, 2),
|
||||
}
|
||||
(out / "summary.json").write_text(json.dumps({"meta": meta, "scenarios": results}, ensure_ascii=False, indent=1),
|
||||
encoding="utf-8")
|
||||
print(json.dumps(meta, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -42,6 +42,7 @@ x-api-env: &api-env
|
||||
UPLOAD_QUEUE_TIMEOUT_SECONDS: ${UPLOAD_QUEUE_TIMEOUT_SECONDS:-5}
|
||||
UPLOAD_READ_TIMEOUT_SECONDS: ${UPLOAD_READ_TIMEOUT_SECONDS:-30}
|
||||
IMAGE_VERIFY_TIMEOUT_SECONDS: ${IMAGE_VERIFY_TIMEOUT_SECONDS:-10}
|
||||
IMAGE_VERIFY_CONCURRENCY: ${IMAGE_VERIFY_CONCURRENCY:-1}
|
||||
JPEG_MAX_MEMORY_MB: ${JPEG_MAX_MEMORY_MB:-256}
|
||||
PENDING_GRACE_SECONDS: ${PENDING_GRACE_SECONDS:-900}
|
||||
JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-60}
|
||||
|
||||
@@ -50,7 +50,14 @@ CREATE TABLE attachments (
|
||||
-- Janitor: stale intents, oldest first; tiny partial index (only rows of unfinished uploads).
|
||||
CREATE INDEX attachments_pending_created ON attachments (created_at) WHERE status = 'pending';
|
||||
|
||||
-- Which attachments a message references, in the order of attachment_ids.
|
||||
-- attachment_ids of a message, twice on purpose (both written in the message's transaction):
|
||||
-- * messages.attachment_ids (ordered array): the history page reads it from the same row, so the
|
||||
-- keyset range scan of messages stays ONE index scan, without a per-row sub-query;
|
||||
-- * message_attachments: foreign keys (a referenced attachment can never be deleted) and the
|
||||
-- reverse lookup "is this attachment used?" (janitor/reconciler) by an index on attachment_id.
|
||||
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_attachment_ids_len CHECK (cardinality(attachment_ids) <= 10);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages (id),
|
||||
position smallint NOT NULL,
|
||||
@@ -115,8 +122,8 @@ BEGIN
|
||||
IF FOUND THEN
|
||||
UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond')
|
||||
WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created;
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
VALUES (p_message, p_chat, p_user, p_text, v_created);
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
|
||||
VALUES (p_message, p_chat, p_user, p_text, v_created, p_attachments);
|
||||
INSERT INTO message_attachments (message_id, position, attachment_id)
|
||||
SELECT p_message, (a.ord - 1)::smallint, a.id
|
||||
FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord);
|
||||
@@ -127,8 +134,7 @@ BEGIN
|
||||
|
||||
RETURN QUERY
|
||||
SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at,
|
||||
ARRAY(SELECT ma.attachment_id FROM message_attachments ma
|
||||
WHERE ma.message_id = m.id ORDER BY ma.position)
|
||||
m.attachment_ids
|
||||
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
|
||||
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key;
|
||||
IF NOT FOUND THEN
|
||||
|
||||
@@ -389,7 +389,7 @@ async def upload_attachment(request: Request) -> JSONResponse:
|
||||
except asyncio.TimeoutError:
|
||||
raise attachments.Busy("Слишком много одновременных загрузок, повторите позже") from None
|
||||
try:
|
||||
upload = await attachments.read_image(reader, settings, declared)
|
||||
upload = await attachments.read_image(reader, settings, declared, request.app.state.verify_slots)
|
||||
finally:
|
||||
slots.release()
|
||||
except ApiError:
|
||||
@@ -518,7 +518,8 @@ def create_app(settings: Settings | None = None) -> Starlette:
|
||||
"db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds,
|
||||
"instance_id": settings.instance_id, "drain_s": settings.drain_seconds,
|
||||
"message_write_path": settings.message_write_path, "s3_bucket": settings.s3_bucket,
|
||||
"upload_concurrency": settings.upload_concurrency, "pending_grace_s": settings.pending_grace_seconds,
|
||||
"upload_concurrency": settings.upload_concurrency, "verify_concurrency": settings.image_verify_concurrency,
|
||||
"pending_grace_s": settings.pending_grace_seconds,
|
||||
"fault_points": sorted(settings.fault_points)}})
|
||||
tasks = [asyncio.create_task(purge_loop())]
|
||||
if settings.janitor_interval_seconds > 0:
|
||||
@@ -567,5 +568,6 @@ def create_app(settings: Settings | None = None) -> Starlette:
|
||||
app.state.sessions = sessions
|
||||
app.state.storage = storage
|
||||
app.state.upload_slots = asyncio.Semaphore(settings.upload_concurrency)
|
||||
app.state.verify_slots = asyncio.Semaphore(settings.image_verify_concurrency)
|
||||
app.state.draining = False
|
||||
return app
|
||||
|
||||
@@ -227,7 +227,7 @@ def declared_length(request: Request, limit: int) -> int | None:
|
||||
return length
|
||||
|
||||
|
||||
async def read_image(reader: BodyReader, settings: Settings, declared: str) -> Upload:
|
||||
async def read_image(reader: BodyReader, settings: Settings, declared: str, verify_slots: asyncio.Semaphore) -> Upload:
|
||||
"""Body -> verified image. Memory: at most MAX_UPLOAD_BYTES per upload (bytearray); the format
|
||||
and the pixel count are known after the first bytes, so a picture that is too big or not a
|
||||
PNG/JPEG is rejected without reading/decoding the rest."""
|
||||
@@ -260,8 +260,9 @@ async def read_image(reader: BodyReader, settings: Settings, declared: str) -> U
|
||||
data = bytes(buffer)
|
||||
del buffer
|
||||
try:
|
||||
await images.verify(data, info, timeout=settings.image_verify_timeout_seconds,
|
||||
jpeg_max_memory_mb=settings.jpeg_max_memory_mb)
|
||||
async with verify_slots:
|
||||
await images.verify(data, info, timeout=settings.image_verify_timeout_seconds,
|
||||
jpeg_max_memory_mb=settings.jpeg_max_memory_mb)
|
||||
except images.ImageError as exc:
|
||||
raise _image_error(exc) from None
|
||||
except asyncio.TimeoutError:
|
||||
|
||||
@@ -130,6 +130,7 @@ class Settings:
|
||||
upload_queue_timeout_seconds: float = 5.0
|
||||
upload_read_timeout_seconds: float = 30.0
|
||||
image_verify_timeout_seconds: float = 10.0
|
||||
image_verify_concurrency: int = 1
|
||||
jpeg_max_memory_mb: int = 256
|
||||
pending_grace_seconds: int = 900
|
||||
janitor_interval_seconds: float = 60.0
|
||||
@@ -223,6 +224,9 @@ def s3_settings(*, credentials_required: bool = True) -> dict:
|
||||
upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0, 0.0, 60.0),
|
||||
upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 30.0, 1.0, 300.0),
|
||||
image_verify_timeout_seconds=_float("IMAGE_VERIFY_TIMEOUT_SECONDS", 10.0, 1.0, 120.0),
|
||||
# Full checks running at once per replica: a strict JPEG decode of a 40 Mpx progressive
|
||||
# file needs up to ~120-240 MB of coefficient buffers (capped by JPEG_MAX_MEMORY_MB).
|
||||
image_verify_concurrency=_int("IMAGE_VERIFY_CONCURRENCY", 1, 1, 16),
|
||||
jpeg_max_memory_mb=_int("JPEG_MAX_MEMORY_MB", 256, 16, 2048),
|
||||
# An intent row still `pending` after this long belongs to a failed upload: the janitor
|
||||
# deletes its object and the row. Far above the longest possible upload (~45 s).
|
||||
|
||||
@@ -0,0 +1,430 @@
|
||||
"""Reconciler of the two stores (lab 4, track 2): S3 objects vs attachment rows in PostgreSQL.
|
||||
|
||||
docker compose run --rm reconciler # dry-run: report only (default)
|
||||
docker compose run --rm reconciler --apply # fix what is safe to fix
|
||||
docker compose run --rm reconciler --verify-checksums # also read every object (SHA-256)
|
||||
|
||||
How it walks: both sides are read as key-ordered streams and merge-joined - ListObjectsV2
|
||||
pages (prefix o/, byte order) against keyset pages of `attachments ORDER BY object_key`
|
||||
(COLLATE "C" = the same byte order, unique index). Memory does not depend on the bucket size;
|
||||
cost is ~N/1000 S3 list requests + N/page DB pages (measured: docs/CONSISTENCY.md).
|
||||
|
||||
What it finds and what --apply does (nothing else is ever changed):
|
||||
|
||||
kind condition --apply
|
||||
orphan_object object, no row, older than --grace delete the object
|
||||
stale_intent row 'pending' older than --grace (+/- object) delete object, then row
|
||||
missing_object row 'ready', object absent, older than --grace quarantine the row
|
||||
size_mismatch row 'ready', object size != size_bytes quarantine the row
|
||||
checksum_mismatch row 'ready', SHA-256 differs (--verify-checksums) quarantine the row
|
||||
restored_object row quarantined, object present and correct again remove the quarantine
|
||||
fresh_* the same, but younger than --grace nothing (in-flight upload)
|
||||
|
||||
Safety:
|
||||
* dry-run is read-only BY CONSTRUCTION: the PostgreSQL session runs with
|
||||
default_transaction_read_only=on and the S3 wrapper has no write methods, so even a bug in
|
||||
the planner cannot modify a store;
|
||||
* ready rows are never deleted - a used (or unused) ready attachment can only be quarantined,
|
||||
which is reversible; objects that have a row are never deleted unless the row is a stale
|
||||
intent; anything younger than --grace is never touched;
|
||||
* every action re-checks its condition right before acting, under a row lock (FOR UPDATE)
|
||||
where a row exists, so the live API racing with the reconciler cannot be hurt;
|
||||
* object first, row second: an interrupted run leaves a state the next run repairs;
|
||||
re-running is idempotent (the second run finds nothing to do);
|
||||
* safety valve: --apply refuses to delete more than --max-orphan-fraction of the scanned
|
||||
objects (pointing it at the wrong/empty database would otherwise wipe the bucket).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import psycopg
|
||||
|
||||
from .config import ConfigError, Settings, _float, _secret, s3_settings
|
||||
from .errors import Unavailable
|
||||
from .storage import OBJECT_PREFIX, ListedObject, ObjectMissing, Storage
|
||||
|
||||
KINDS = ("orphan_object", "stale_intent", "missing_object", "size_mismatch", "checksum_mismatch",
|
||||
"restored_object", "fresh_orphan_object", "fresh_intent", "fresh_missing_object")
|
||||
MIN_SAFE_GRACE = 60.0 # > the longest possible upload (queue 5 s + body 30 s + verify 10 s + S3)
|
||||
ACTION_OF = {"orphan_object": "delete_object", "stale_intent": "delete_intent", "missing_object": "quarantine",
|
||||
"size_mismatch": "quarantine", "checksum_mismatch": "quarantine", "restored_object": "unquarantine"}
|
||||
|
||||
|
||||
def log(text: str) -> None:
|
||||
print(f"reconcile: {text}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Row:
|
||||
id: str
|
||||
key: str
|
||||
status: str
|
||||
size: int
|
||||
sha256: bytes
|
||||
created_at: dt.datetime
|
||||
updated_at: dt.datetime
|
||||
lost_reason: str | None
|
||||
used: bool
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Finding:
|
||||
kind: str
|
||||
key: str
|
||||
attachment_id: str | None = None
|
||||
used: bool = False
|
||||
detail: str = ""
|
||||
result: str = "planned" # planned | applied | skipped:<why> | failed:<why>
|
||||
|
||||
|
||||
@dataclass
|
||||
class Stats:
|
||||
objects: int = 0
|
||||
rows: int = 0
|
||||
consistent: int = 0
|
||||
s3_list_requests: int = 0
|
||||
s3_head_requests: int = 0
|
||||
s3_get_bytes: int = 0
|
||||
db_pages: int = 0
|
||||
|
||||
|
||||
class ReadOnlyStorage:
|
||||
"""The dry-run view of S3: listing and reading only - there is no delete to call."""
|
||||
|
||||
def __init__(self, storage: Storage):
|
||||
self._s = storage
|
||||
|
||||
async def list_page(self, *args, **kwargs):
|
||||
return await self._s.list_page(*args, **kwargs)
|
||||
|
||||
async def head(self, key: str):
|
||||
return await self._s.head(key)
|
||||
|
||||
async def read_all(self, key: str, limit: int) -> bytes:
|
||||
return await self._s.read_all(key, limit)
|
||||
|
||||
|
||||
class Reconciler:
|
||||
def __init__(self, conn: psycopg.AsyncConnection, storage: Storage | ReadOnlyStorage, *, grace: float,
|
||||
page_size: int, verify: bool, apply: bool):
|
||||
self.conn = conn
|
||||
self.s3 = storage
|
||||
self.grace = grace
|
||||
self.page_size = page_size
|
||||
self.verify = verify
|
||||
self.apply = apply
|
||||
self.stats = Stats()
|
||||
self.findings: list[Finding] = []
|
||||
|
||||
# ---------------------------------------------------------- streams
|
||||
async def objects(self) -> AsyncIterator[ListedObject]:
|
||||
after = None
|
||||
while True:
|
||||
page, more = await self.s3.list_page(OBJECT_PREFIX, after, self.page_size)
|
||||
self.stats.s3_list_requests += 1
|
||||
for item in page:
|
||||
self.stats.objects += 1
|
||||
yield item
|
||||
if not more or not page:
|
||||
return
|
||||
after = page[-1].key
|
||||
|
||||
async def rows(self) -> AsyncIterator[Row]:
|
||||
after = ""
|
||||
while True:
|
||||
cur = await self.conn.execute(
|
||||
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256, a.created_at, a.updated_at, "
|
||||
"a.lost_reason, EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) "
|
||||
"FROM attachments a WHERE a.object_key > %s ORDER BY a.object_key LIMIT %s",
|
||||
(after, self.page_size))
|
||||
page = await cur.fetchall()
|
||||
self.stats.db_pages += 1
|
||||
for r in page:
|
||||
self.stats.rows += 1
|
||||
yield Row(str(r[0]), r[1], r[2], r[3], bytes(r[4]), r[5], r[6], r[7], r[8])
|
||||
if len(page) < self.page_size:
|
||||
return
|
||||
after = page[-1][1]
|
||||
|
||||
# ---------------------------------------------------------- planning (read-only)
|
||||
async def scan(self) -> None:
|
||||
cur = await self.conn.execute("SELECT clock_timestamp()")
|
||||
self.db_now = (await cur.fetchone())[0]
|
||||
self.s3_now = time.time()
|
||||
objects, rows = self.objects(), self.rows()
|
||||
obj = await anext(objects, None)
|
||||
row = await anext(rows, None)
|
||||
while obj is not None or row is not None:
|
||||
if row is None or (obj is not None and obj.key < row.key):
|
||||
self.object_only(obj)
|
||||
obj = await anext(objects, None)
|
||||
elif obj is None or row.key < obj.key:
|
||||
self.row_only(row)
|
||||
row = await anext(rows, None)
|
||||
else:
|
||||
await self.both(obj, row)
|
||||
obj = await anext(objects, None)
|
||||
row = await anext(rows, None)
|
||||
|
||||
def row_old(self, row: Row) -> bool:
|
||||
return (self.db_now - max(row.created_at, row.updated_at)).total_seconds() > self.grace
|
||||
|
||||
def object_only(self, obj: ListedObject) -> None:
|
||||
old = self.s3_now - obj.last_modified_ts > self.grace
|
||||
self.add(Finding("orphan_object" if old else "fresh_orphan_object", obj.key))
|
||||
|
||||
def row_only(self, row: Row) -> None:
|
||||
if row.status == "pending":
|
||||
self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id,
|
||||
detail="object absent"))
|
||||
elif row.lost_reason is not None:
|
||||
self.stats.consistent += 1 # already quarantined, still missing: nothing new
|
||||
else:
|
||||
self.add(Finding("missing_object" if self.row_old(row) else "fresh_missing_object", row.key, row.id,
|
||||
used=row.used))
|
||||
|
||||
async def both(self, obj: ListedObject, row: Row) -> None:
|
||||
if row.status == "pending":
|
||||
self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id,
|
||||
detail="object present"))
|
||||
return
|
||||
problem = await self.object_problem(row, obj.size)
|
||||
if problem:
|
||||
if row.lost_reason is None:
|
||||
self.add(Finding(problem, row.key, row.id, used=row.used))
|
||||
else:
|
||||
self.stats.consistent += 1 # quarantined and still broken
|
||||
elif row.lost_reason is not None:
|
||||
self.add(Finding("restored_object", row.key, row.id, used=row.used))
|
||||
else:
|
||||
self.stats.consistent += 1
|
||||
|
||||
async def object_problem(self, row: Row, size: int) -> str | None:
|
||||
if size != row.size:
|
||||
return "size_mismatch"
|
||||
if self.verify:
|
||||
data = await self.s3.read_all(row.key, row.size)
|
||||
self.stats.s3_get_bytes += len(data)
|
||||
if hashlib.sha256(data).digest() != row.sha256:
|
||||
return "checksum_mismatch"
|
||||
return None
|
||||
|
||||
def add(self, finding: Finding) -> None:
|
||||
if finding.kind not in ACTION_OF:
|
||||
finding.result = "not_actionable"
|
||||
self.findings.append(finding)
|
||||
|
||||
# ---------------------------------------------------------- actions (--apply only)
|
||||
async def run_actions(self, max_orphan_fraction: float) -> None:
|
||||
orphans = sum(1 for f in self.findings if f.kind == "orphan_object")
|
||||
if self.stats.objects and orphans > max_orphan_fraction * self.stats.objects:
|
||||
raise SafetyValve(f"{orphans} of {self.stats.objects} objects look orphaned "
|
||||
f"(> {max_orphan_fraction:.0%}); wrong database? nothing was changed")
|
||||
handlers = {"delete_object": self.delete_object, "delete_intent": self.delete_intent,
|
||||
"quarantine": self.quarantine, "unquarantine": self.unquarantine}
|
||||
for finding in self.findings:
|
||||
action = ACTION_OF.get(finding.kind)
|
||||
if action is None:
|
||||
continue
|
||||
try:
|
||||
finding.result = await handlers[action](finding)
|
||||
except (Unavailable, psycopg.OperationalError) as exc:
|
||||
finding.result = f"failed:{type(exc).__name__}"
|
||||
raise # a store is down: stop; the next run continues from the committed state
|
||||
|
||||
async def delete_object(self, f: Finding) -> str:
|
||||
cur = await self.conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (f.key,))
|
||||
if await cur.fetchone():
|
||||
return "skipped:row_appeared"
|
||||
try:
|
||||
head = await self.s3.head(f.key)
|
||||
self.stats.s3_head_requests += 1
|
||||
except ObjectMissing:
|
||||
return "skipped:already_gone"
|
||||
if time.time() - head.last_modified_ts <= self.grace:
|
||||
return "skipped:object_rewritten_recently"
|
||||
# No row names this key and the object is older than the grace period. Rows are always
|
||||
# written BEFORE their object and keys are random and never reused, so no upload can
|
||||
# still claim it.
|
||||
await self.s3.delete(f.key)
|
||||
return "applied"
|
||||
|
||||
async def delete_intent(self, f: Finding) -> str:
|
||||
async with self.conn.transaction():
|
||||
cur = await self.conn.execute(
|
||||
"SELECT 1 FROM attachments WHERE id = %s AND status = 'pending' "
|
||||
"AND greatest(created_at, updated_at) < clock_timestamp() - make_interval(secs => %s) FOR UPDATE",
|
||||
(f.attachment_id, self.grace))
|
||||
if await cur.fetchone() is None:
|
||||
return "skipped:no_longer_stale_intent"
|
||||
await self.s3.delete(f.key) # object first (idempotent), then the row
|
||||
await self.conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'pending'",
|
||||
(f.attachment_id,))
|
||||
return "applied"
|
||||
|
||||
async def quarantine(self, f: Finding) -> str:
|
||||
async with self.conn.transaction():
|
||||
cur = await self.conn.execute(
|
||||
"SELECT size_bytes, sha256 FROM attachments WHERE id = %s AND status = 'ready' "
|
||||
"AND lost_at IS NULL FOR UPDATE", (f.attachment_id,))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return "skipped:row_changed"
|
||||
reason = await self.current_problem(f.key, row[0], bytes(row[1]))
|
||||
if reason is None:
|
||||
return "skipped:object_ok_now"
|
||||
await self.conn.execute(
|
||||
"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = %s WHERE id = %s",
|
||||
(reason, f.attachment_id))
|
||||
return "applied"
|
||||
|
||||
async def unquarantine(self, f: Finding) -> str:
|
||||
async with self.conn.transaction():
|
||||
cur = await self.conn.execute(
|
||||
"SELECT size_bytes, sha256 FROM attachments WHERE id = %s AND lost_at IS NOT NULL FOR UPDATE",
|
||||
(f.attachment_id,))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return "skipped:row_changed"
|
||||
if await self.current_problem(f.key, row[0], bytes(row[1])) is not None:
|
||||
return "skipped:object_still_bad"
|
||||
await self.conn.execute("UPDATE attachments SET lost_at = NULL, lost_reason = NULL WHERE id = %s",
|
||||
(f.attachment_id,))
|
||||
return "applied"
|
||||
|
||||
async def current_problem(self, key: str, size: int, sha256: bytes) -> str | None:
|
||||
"""Fresh look at the object right before changing the row (the listing may be stale)."""
|
||||
try:
|
||||
head = await self.s3.head(key)
|
||||
self.stats.s3_head_requests += 1
|
||||
except ObjectMissing:
|
||||
return "missing_object"
|
||||
if head.size != size:
|
||||
return "size_mismatch"
|
||||
if self.verify:
|
||||
data = await self.s3.read_all(key, size)
|
||||
self.stats.s3_get_bytes += len(data)
|
||||
if hashlib.sha256(data).digest() != sha256:
|
||||
return "checksum_mismatch"
|
||||
return None
|
||||
|
||||
# ---------------------------------------------------------- report
|
||||
def report(self, mode: str, seconds: float, error: str | None, sample: int) -> dict:
|
||||
kinds = Counter(f.kind for f in self.findings)
|
||||
results = Counter(f.result.split(":")[0] for f in self.findings if f.kind in ACTION_OF)
|
||||
samples = {}
|
||||
for f in self.findings:
|
||||
items = samples.setdefault(f.kind, [])
|
||||
if len(items) < sample:
|
||||
items.append({"key": f.key, "attachment_id": f.attachment_id, "used": f.used,
|
||||
"result": f.result, **({"detail": f.detail} if f.detail else {})})
|
||||
return {
|
||||
"mode": mode, "grace_seconds": self.grace, "verify_checksums": self.verify,
|
||||
"duration_seconds": round(seconds, 3), "error": error,
|
||||
"scanned": {"objects": self.stats.objects, "rows": self.stats.rows, "consistent": self.stats.consistent},
|
||||
"findings": {k: kinds.get(k, 0) for k in KINDS},
|
||||
"used_attachments_in_findings": sum(1 for f in self.findings if f.used),
|
||||
"actions": {"planned": sum(1 for f in self.findings if f.kind in ACTION_OF),
|
||||
"applied": results.get("applied", 0), "skipped": results.get("skipped", 0),
|
||||
"failed": results.get("failed", 0)},
|
||||
"cost": {"s3_list_requests": self.stats.s3_list_requests, "s3_head_requests": self.stats.s3_head_requests,
|
||||
"s3_get_bytes": self.stats.s3_get_bytes, "db_pages": self.stats.db_pages,
|
||||
"page_size": self.page_size},
|
||||
"samples": samples,
|
||||
}
|
||||
|
||||
|
||||
class SafetyValve(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def arguments(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(prog="python -m messenger.reconcile", description=__doc__.split("\n")[0])
|
||||
p.add_argument("--apply", action="store_true", help="fix findings (default: dry-run, read-only)")
|
||||
p.add_argument("--grace", type=float, default=3600.0,
|
||||
help="seconds; anything younger is never touched (default 3600, >= PENDING_GRACE_SECONDS)")
|
||||
p.add_argument("--allow-small-grace", action="store_true",
|
||||
help="permit --grace below 60 s (tests only: a grace shorter than an upload could remove the "
|
||||
"intent of an upload that is still running; that upload then fails with 503)")
|
||||
p.add_argument("--page-size", type=int, default=1000, help="S3 list / DB page size (1..1000)")
|
||||
p.add_argument("--verify-checksums", action="store_true", help="read every ready object and compare SHA-256")
|
||||
p.add_argument("--samples", type=int, default=20, help="keys listed per finding kind in the report")
|
||||
p.add_argument("--max-orphan-fraction", type=float, default=0.5,
|
||||
help="--apply refuses if more than this share of objects look orphaned (default 0.5)")
|
||||
args = p.parse_args(argv)
|
||||
if not 1 <= args.page_size <= 1000:
|
||||
p.error("--page-size must be 1..1000")
|
||||
if args.grace < 1 or (args.grace < MIN_SAFE_GRACE and not args.allow_small_grace):
|
||||
p.error(f"--grace must be >= {MIN_SAFE_GRACE:.0f} s (or >= 1 s with --allow-small-grace)")
|
||||
if not 0 < args.max_orphan_fraction <= 1:
|
||||
p.error("--max-orphan-fraction must be in (0, 1]")
|
||||
return args
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> int:
|
||||
url = _secret("DATABASE_URL", required=True)
|
||||
base = s3_settings()
|
||||
settings = Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024,
|
||||
database_url=url, **base)
|
||||
timeout_ms = int(_float("DB_TIMEOUT_SECONDS", 10.0, 1.0, 120.0) * 1000)
|
||||
options = f"-c statement_timeout={timeout_ms} -c lock_timeout={timeout_ms // 2}"
|
||||
if not args.apply:
|
||||
options += " -c default_transaction_read_only=on" # dry-run: PostgreSQL itself refuses writes
|
||||
mode = "apply" if args.apply else "dry-run"
|
||||
storage = Storage(settings)
|
||||
await storage.open()
|
||||
started = time.perf_counter()
|
||||
error = None
|
||||
code = 0
|
||||
try:
|
||||
conn = await psycopg.AsyncConnection.connect(url, autocommit=True, connect_timeout=5, options=options,
|
||||
application_name=f"messenger-reconcile-{mode}")
|
||||
except psycopg.OperationalError as exc:
|
||||
await storage.close()
|
||||
error = f"store_unavailable: database {type(exc).__name__}"
|
||||
print(json.dumps({"mode": mode, "error": error}, ensure_ascii=False))
|
||||
log(error)
|
||||
return 1
|
||||
rec = Reconciler(conn, storage if args.apply else ReadOnlyStorage(storage), grace=args.grace,
|
||||
page_size=args.page_size, verify=args.verify_checksums, apply=args.apply)
|
||||
try:
|
||||
log(f"{mode}: scanning bucket {settings.s3_bucket} (prefix {OBJECT_PREFIX}) against table attachments")
|
||||
await rec.scan()
|
||||
log(f"scan done: {rec.stats.objects} objects, {rec.stats.rows} rows, {len(rec.findings)} findings")
|
||||
if args.apply:
|
||||
await rec.run_actions(args.max_orphan_fraction)
|
||||
except SafetyValve as exc:
|
||||
error, code = f"safety_valve: {exc}", 3
|
||||
except (Unavailable, psycopg.OperationalError) as exc:
|
||||
error, code = f"store_unavailable: {type(exc).__name__}", 1
|
||||
finally:
|
||||
await conn.close()
|
||||
await storage.close()
|
||||
print(json.dumps(rec.report(mode, time.perf_counter() - started, error, args.samples), ensure_ascii=False,
|
||||
indent=1))
|
||||
if error:
|
||||
log(error)
|
||||
return code
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = arguments(argv)
|
||||
try:
|
||||
return asyncio.run(run(args))
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+11
-13
@@ -219,8 +219,9 @@ async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID,
|
||||
(chat_id,))
|
||||
(created_at,) = await cur.fetchone()
|
||||
await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)",
|
||||
(message_id, chat_id, user_id, _b(text), created_at))
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s::uuid[])",
|
||||
(message_id, chat_id, user_id, _b(text), created_at, attachment_ids))
|
||||
if attachment_ids:
|
||||
await conn.execute(
|
||||
"INSERT INTO message_attachments (message_id, position, attachment_id) "
|
||||
@@ -266,9 +267,7 @@ CLAIM_KEY = (
|
||||
"WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) "
|
||||
"RETURNING 1")
|
||||
STORED_RESULT = (
|
||||
"SELECT k.request_hash, m.id, m.text, m.created_at, "
|
||||
"ARRAY(SELECT ma.attachment_id FROM message_attachments ma WHERE ma.message_id = m.id ORDER BY ma.position) "
|
||||
"FROM idempotency_keys k "
|
||||
"SELECT k.request_hash, m.id, m.text, m.created_at, m.attachment_ids FROM idempotency_keys k "
|
||||
"JOIN messages m ON m.id = k.message_id "
|
||||
"WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s")
|
||||
|
||||
@@ -351,14 +350,13 @@ async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 50
|
||||
|
||||
# Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR
|
||||
# would push the cursor into a Filter and late pages would scan the whole prefix.
|
||||
# attachment_ids: a correlated sub-select per returned row (at most limit + 1 = 101 lookups on the
|
||||
# primary key of message_attachments), so the keyset range scan itself stays unchanged.
|
||||
HISTORY_COLUMNS = ("SELECT m.id, m.sender_id, m.text, m.created_at, "
|
||||
"ARRAY(SELECT ma.attachment_id FROM message_attachments ma "
|
||||
"WHERE ma.message_id = m.id ORDER BY ma.position) FROM messages m ")
|
||||
HISTORY_FIRST = HISTORY_COLUMNS + "WHERE m.chat_id = %s ORDER BY m.created_at, m.id LIMIT %s"
|
||||
HISTORY_AFTER = (HISTORY_COLUMNS +
|
||||
"WHERE m.chat_id = %s AND (m.created_at, m.id) > (%s, %s) ORDER BY m.created_at, m.id LIMIT %s")
|
||||
# attachment_ids is a column of the message row (migration 0004): still one index range scan.
|
||||
HISTORY_FIRST = (
|
||||
"SELECT id, sender_id, text, created_at, attachment_ids FROM messages "
|
||||
"WHERE chat_id = %s ORDER BY created_at, id LIMIT %s")
|
||||
HISTORY_AFTER = (
|
||||
"SELECT id, sender_id, text, created_at, attachment_ids FROM messages "
|
||||
"WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s")
|
||||
|
||||
|
||||
async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit: int) -> tuple[list[Message], bool]:
|
||||
|
||||
@@ -44,6 +44,7 @@ class ObjectHead:
|
||||
size: int
|
||||
content_type: str
|
||||
metadata: dict[str, str]
|
||||
last_modified_ts: float # POSIX seconds
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -116,7 +117,7 @@ class Storage:
|
||||
async def head(self, key: str) -> ObjectHead:
|
||||
response = await self._call("head", self.client.head_object(Bucket=self.bucket, Key=key))
|
||||
return ObjectHead(int(response["ContentLength"]), response.get("ContentType", ""),
|
||||
dict(response.get("Metadata", {})))
|
||||
dict(response.get("Metadata", {})), response["LastModified"].timestamp())
|
||||
|
||||
async def open_stream(self, key: str) -> tuple[int, AsyncIterator[bytes]]:
|
||||
"""Start a GET: (Content-Length, async iterator of chunks). The headers arrive under the
|
||||
|
||||
@@ -21,6 +21,7 @@ import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx2 as httpx
|
||||
import pytest
|
||||
@@ -53,9 +54,9 @@ def basic(username: str, password: str) -> str:
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ docker / psql
|
||||
def compose(*args: str, check: bool = True, timeout: float = 120) -> subprocess.CompletedProcess:
|
||||
def compose(*args: str, check: bool = True, timeout: float = 120, stdin: str | None = None) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True,
|
||||
encoding="utf-8", check=check, timeout=timeout)
|
||||
encoding="utf-8", check=check, timeout=timeout, input=stdin)
|
||||
|
||||
|
||||
def psql(sql: str) -> str:
|
||||
@@ -234,6 +235,24 @@ class Api:
|
||||
def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict:
|
||||
return self.post_message(actor, chat, {"text": text}, key=key).json()
|
||||
|
||||
def upload(self, actor: dict, chat: dict, data, expected=201, *, content_type: str | None = "image/png",
|
||||
filename: str | None = "photo.png", query: str | None = None, headers: dict | None = None):
|
||||
"""POST raw bytes to /chats/{id}/attachments. `query` replaces the whole query string
|
||||
(to send hand-made percent-encoding); `data` may be bytes or an iterator (chunked)."""
|
||||
h = dict(headers or {})
|
||||
if content_type is not None:
|
||||
h["Content-Type"] = content_type
|
||||
if query is None:
|
||||
query = "" if filename is None else "filename=" + quote(filename, safe="")
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments" + (f"?{query}" if query else "")
|
||||
if isinstance(data, (bytes, bytearray)):
|
||||
return self.call("POST", path, expected, token=actor["token"], raw=bytes(data), headers=h,
|
||||
schema="Attachment" if expected == 201 else None)
|
||||
h["Authorization"] = "Bearer " + actor["token"]
|
||||
response = self.client.post(path, content=data, headers=h)
|
||||
assert response.status_code == expected, (response.status_code, response.text[:300])
|
||||
return response
|
||||
|
||||
def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]:
|
||||
"""Walk all pages; returns (items, number_of_pages)."""
|
||||
items, pages, cursor = [], 0, None
|
||||
@@ -281,6 +300,76 @@ def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.Complete
|
||||
timeout=timeout, env={**os.environ, **env})
|
||||
|
||||
|
||||
def s3_exec(body: str, service: str = "api", stdin: str | None = None) -> str:
|
||||
"""Run `body` (async Python, `s` = messenger.storage.Storage) INSIDE a container that holds S3
|
||||
credentials: `api` (Read+Write, the API's own key pair) or `reconciler` (+List). The S3 port is
|
||||
not published on the host; this is how the tests inject faults directly into the bucket."""
|
||||
script = ("import asyncio, base64, json, sys\n"
|
||||
"from messenger.config import Settings, s3_settings\n"
|
||||
"from messenger.storage import Storage, ObjectMissing\n"
|
||||
"async def main():\n"
|
||||
" st = Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings())\n"
|
||||
" s = Storage(st)\n"
|
||||
" await s.open()\n"
|
||||
" try:\n" + "".join(" " + line + "\n" for line in body.splitlines()) +
|
||||
" finally:\n"
|
||||
" await s.close()\n"
|
||||
"asyncio.run(main())\n")
|
||||
if service == "api":
|
||||
cmd = ["exec", "-T", "api", "python", "-c", script]
|
||||
else:
|
||||
cmd = ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service, "-c", script]
|
||||
return compose(*cmd, timeout=120, stdin=stdin).stdout.strip()
|
||||
|
||||
|
||||
def s3_put(key: str, data: bytes) -> None:
|
||||
"""Write raw bytes under `key` (base64 over stdin: Windows limits command lines to 32 KiB)."""
|
||||
s3_exec(f"await s.put({key!r}, base64.b64decode(sys.stdin.read()), 'image/png', {{}})",
|
||||
stdin=base64.b64encode(data).decode())
|
||||
|
||||
|
||||
def s3_delete(key: str) -> None:
|
||||
s3_exec(f"await s.delete({key!r})")
|
||||
|
||||
|
||||
def s3_size(key: str) -> int | None:
|
||||
out = s3_exec(f"try:\n print((await s.head({key!r})).size)\nexcept ObjectMissing:\n print(-1)")
|
||||
size = int(out.splitlines()[-1])
|
||||
return None if size < 0 else size
|
||||
|
||||
|
||||
def s3_keys() -> set[str]:
|
||||
"""Every object key under o/ (needs List: runs with the reconciler's key pair)."""
|
||||
out = s3_exec("keys, after = [], None\n"
|
||||
"while True:\n"
|
||||
" page, more = await s.list_page('o/', after, 1000)\n"
|
||||
" keys += [o.key for o in page]\n"
|
||||
" if not more or not page:\n"
|
||||
" break\n"
|
||||
" after = page[-1].key\n"
|
||||
"print(json.dumps(keys))", service="reconciler")
|
||||
return set(json.loads(out.splitlines()[-1]))
|
||||
|
||||
|
||||
def reconcile(*args: str, expect_code: int = 0) -> dict:
|
||||
"""Run the reconciler CLI (compose service, profile tools) and return its JSON report."""
|
||||
# --no-deps: never let `compose run` start a store that a test has stopped on purpose.
|
||||
result = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "100000", *args, check=False,
|
||||
timeout=300)
|
||||
assert result.returncode == expect_code, (result.returncode, result.stderr[-2000:])
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def findings_by_key(report: dict) -> dict[str, dict]:
|
||||
return {item["key"]: dict(item, kind=kind) for kind, items in report["samples"].items() for item in items}
|
||||
|
||||
|
||||
def attachment_row(attachment_id: str) -> dict | None:
|
||||
out = psql("SELECT json_build_object('status', status, 'object_key', object_key, 'size', size_bytes, "
|
||||
"'lost_reason', lost_reason) FROM attachments WHERE id = '%s'" % attachment_id)
|
||||
return json.loads(out) if out else None
|
||||
|
||||
|
||||
def probe_inside(replica: dict, path: str) -> int:
|
||||
"""HTTP status of `path` asked inside the replica container itself (bypassing the LB)."""
|
||||
code = ("import urllib.request as u, urllib.error as e\n"
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Synthetic images for the lab 4 tests: valid PNG/JPEG of any size, and the broken files the
|
||||
reviewer used against other solutions (truncated, corrupted IDAT with a valid CRC, JPEG without
|
||||
scan data, decompression-bomb headers, MIME spoofing). Nothing big is stored in Git: every file
|
||||
is generated in memory."""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
from PIL import Image
|
||||
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
|
||||
def chunk(ctype: bytes, data: bytes) -> bytes:
|
||||
return struct.pack(">I", len(data)) + ctype + data + struct.pack(">I", zlib.crc32(ctype + data))
|
||||
|
||||
|
||||
def ihdr(width: int, height: int, depth: int = 8, color: int = 2, interlace: int = 0) -> bytes:
|
||||
return chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace))
|
||||
|
||||
|
||||
def png(width: int = 64, height: int = 48, mode: str = "RGB", noise: bool = False, **save) -> bytes:
|
||||
if noise:
|
||||
channels = len(mode)
|
||||
image = Image.frombytes(mode, (width, height), os.urandom(width * height * channels))
|
||||
else:
|
||||
image = Image.new(mode, (width, height), (40, 120, 200) if mode == "RGB" else 128)
|
||||
out = io.BytesIO()
|
||||
image.save(out, "PNG", **save)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
def png_of_size(target: int) -> bytes:
|
||||
"""A valid PNG of (about, at most) `target` bytes: random pixels do not compress."""
|
||||
side = int((target / 3) ** 0.5)
|
||||
data = png(side, side, noise=True, compress_level=1)
|
||||
while len(data) > target:
|
||||
side -= 8
|
||||
data = png(side, side, noise=True, compress_level=1)
|
||||
return data
|
||||
|
||||
|
||||
def png_exactly(size: int) -> bytes:
|
||||
"""A valid PNG of EXACTLY `size` bytes: noise image + an ancillary padding chunk."""
|
||||
base = png_of_size(size - 64)
|
||||
pad = size - len(base) - 12
|
||||
assert pad >= 0
|
||||
return base[:-12] + chunk(b"tEXt", b"pad\x00" + b"x" * (pad - 4)) + base[-12:]
|
||||
|
||||
|
||||
def jpeg(width: int = 64, height: int = 48, progressive: bool = False, noise: bool = False, quality: int = 85) -> bytes:
|
||||
if noise:
|
||||
image = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3))
|
||||
else:
|
||||
image = Image.new("RGB", (width, height), (200, 30, 30))
|
||||
out = io.BytesIO()
|
||||
image.save(out, "JPEG", progressive=progressive, quality=quality)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ broken files
|
||||
def png_corrupt_idat() -> bytes:
|
||||
"""Valid signature, IHDR and CRCs; the IDAT payload is 86 bytes of garbage (reviewer's case)."""
|
||||
return PNG_SIGNATURE + ihdr(20, 20) + chunk(b"IDAT", bytes(range(86))) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def png_short_pixels() -> bytes:
|
||||
"""A complete, CRC-correct PNG whose zlib stream holds fewer rows than the header promises."""
|
||||
raw = b"".join(b"\x00" + bytes(60) for _ in range(19))
|
||||
return PNG_SIGNATURE + ihdr(20, 20) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def png_bomb_header(width: int = 20000, height: int = 10000) -> bytes:
|
||||
"""Header of a huge picture: must be refused (413) from the first 33 bytes."""
|
||||
raw_row = b"\x00" + bytes(width * 3)
|
||||
body = zlib.compress(raw_row * 4, 9) # a few rows only: the decoder must never get that far
|
||||
return PNG_SIGNATURE + ihdr(width, height) + chunk(b"IDAT", body) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def jpeg_without_scan_data() -> bytes:
|
||||
data = jpeg(128, 96)
|
||||
sos = data.index(b"\xff\xda")
|
||||
length = struct.unpack(">H", data[sos + 2:sos + 4])[0]
|
||||
return data[:sos + 2 + length] + b"\xff\xd9"
|
||||
|
||||
|
||||
def jpeg_bomb_header() -> bytes:
|
||||
"""SOF0 announcing 20000 x 20000 = 400 Mpx."""
|
||||
return (b"\xff\xd8\xff\xc0\x00\x11\x08" + struct.pack(">HH", 20000, 20000)
|
||||
+ b"\x03\x01\x22\x00\x02\x11\x01\x03\x11\x01" + b"\xff\xd9")
|
||||
@@ -0,0 +1,268 @@
|
||||
"""Attachments through the load balancer (lab 4, levels 3 and 4): upload/download, ACL, limits,
|
||||
content checks, filename rules and messages with attachment_ids.
|
||||
|
||||
Negative inputs include every case the reviewer used against other lab 4 solutions: corrupted
|
||||
IDAT with valid CRCs, the first 33 bytes of the fixture, JPEG without scan data, a 20000x10000
|
||||
header (DecompressionBomb), exactly 10 MiB through the LB, C1 control U+0085 and a whitespace
|
||||
filename, text=null / missing text / attachment_ids=null in a message."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import imagegen
|
||||
from conftest import attachment_row, compose, s3_size
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
MiB = 1024 * 1024
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def trio(api):
|
||||
"""alice + bob share a chat; eve is an outsider with her own chat."""
|
||||
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
|
||||
return alice, bob, eve, api.chat(alice, bob), api.chat(eve)
|
||||
|
||||
|
||||
def content(api, actor, attachment_id, expected=200, query=""):
|
||||
return api.call("GET", f"/api/v1/attachments/{attachment_id}/content{query}", expected, token=actor["token"])
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ positive path
|
||||
def test_png_and_jpeg_roundtrip_byte_for_byte(api, trio):
|
||||
alice, bob, _, chat, _ = trio
|
||||
jpg = imagegen.jpeg(321, 123, noise=True)
|
||||
for data, ctype in ((SAMPLE, "image/png"), (jpg, "image/jpeg")):
|
||||
att = api.upload(alice, chat, data, content_type=ctype, filename="кот 😺.bin").json()
|
||||
assert att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None
|
||||
assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"]
|
||||
assert att["size_bytes"] == len(data) and att["sha256"] == hashlib.sha256(data).hexdigest()
|
||||
assert att["content_type"] == ctype and att["filename"] == "кот 😺.bin"
|
||||
meta = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"], schema="Attachment").json()
|
||||
assert meta == att
|
||||
got = content(api, bob, att["id"])
|
||||
assert got.content == data and got.headers["content-type"] == ctype
|
||||
assert got.headers["content-length"] == str(len(data))
|
||||
assert got.headers["x-content-type-options"] == "nosniff"
|
||||
assert got.headers["content-disposition"].startswith("attachment;")
|
||||
assert "location" not in got.headers # bytes through the API, never a redirect to S3
|
||||
|
||||
|
||||
def test_metadata_reveals_no_storage_details(api, trio):
|
||||
alice, _, _, chat, _ = trio
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
row = attachment_row(att["id"])
|
||||
assert row["status"] == "ready" and row["object_key"].startswith("o/")
|
||||
text = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=alice["token"]).text
|
||||
assert row["object_key"] not in text and "messenger-attachments" not in text and "s3" not in text.lower()
|
||||
# The key is random: neither the id nor the filename is part of it.
|
||||
assert att["id"].replace("-", "") not in row["object_key"] and "photo" not in row["object_key"]
|
||||
assert s3_size(row["object_key"]) == len(SAMPLE) # the bytes ARE in the bucket
|
||||
|
||||
|
||||
def test_original_is_not_stored_on_api_containers(api, trio):
|
||||
"""The API keeps no file of the original: no recently written file of that size anywhere in
|
||||
the replica's filesystem (the bytes live only in the bucket)."""
|
||||
alice, _, _, chat, _ = trio
|
||||
data = imagegen.png_of_size(300_000)
|
||||
api.upload(alice, chat, data)
|
||||
size = len(data)
|
||||
for cid in compose("ps", "-q", "api").stdout.split():
|
||||
out = subprocess.run(
|
||||
["docker", "exec", cid, "sh", "-c", f"find / -xdev -type f -size {size}c -mmin -10 2>/dev/null | head"],
|
||||
capture_output=True, text=True, timeout=60).stdout.strip()
|
||||
assert out == "", out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ access control
|
||||
def test_outsider_and_unknown_get_404(api, trio):
|
||||
alice, _, eve, chat, _ = trio
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
for suffix in ("", "/content"):
|
||||
api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"])
|
||||
api.call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"])
|
||||
api.call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"])
|
||||
api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401)
|
||||
# Upload into a foreign chat: 404 before anything about the body is looked at.
|
||||
api.upload(eve, chat, SAMPLE, 404)
|
||||
api.upload(eve, chat, b"garbage", 404, content_type="text/plain")
|
||||
|
||||
|
||||
def test_attachment_ids_are_checked_per_chat(api, trio):
|
||||
alice, bob, eve, chat, eve_chat = trio
|
||||
other = api.chat(alice) # alice's second chat
|
||||
att_other = api.upload(alice, other, SAMPLE).json()
|
||||
att_eve = api.upload(eve, eve_chat, SAMPLE).json()
|
||||
# Alice is a member of both chats, but a file is bound to its chat.
|
||||
api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_other["id"]]}, 404)
|
||||
api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404)
|
||||
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404)
|
||||
# Eve cannot post into alice's chat even with her own file: membership first.
|
||||
api.post_message(eve, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404)
|
||||
# Any member may reference an attachment of the chat.
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
msg = api.post_message(bob, chat, {"text": "", "attachment_ids": [att["id"]]}).json()
|
||||
assert msg["attachment_ids"] == [att["id"]] and msg["sender_id"] == bob["id"]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ messages with attachments
|
||||
def test_message_with_attachments_and_history(api, trio):
|
||||
alice, bob, _, chat, _ = trio
|
||||
a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2))
|
||||
m1 = api.post_message(alice, chat, {"text": "", "attachment_ids": [a2, a1]}).json()
|
||||
m2 = api.post_message(alice, chat, {"text": " ", "attachment_ids": [a1]}).json() # blank text + file: ok
|
||||
m3 = api.post_message(alice, chat, {"text": "без файлов"}).json()
|
||||
assert m1["attachment_ids"] == [a2, a1] and m1["text"] == ""
|
||||
assert m2["text"] == " " and m3["attachment_ids"] == []
|
||||
items, _ = api.history(bob, chat, 2)
|
||||
assert [m["attachment_ids"] for m in items] == [[a2, a1], [a1], []]
|
||||
|
||||
|
||||
def test_idempotent_replay_with_attachments(api, trio):
|
||||
alice, _, _, chat, _ = trio
|
||||
a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2))
|
||||
key = uuid.uuid4().hex
|
||||
first = api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a1, a2]}, key=key).json()
|
||||
raw = ('{"attachment_ids":["%s","%s"],"text":"файлы"}' % (a1, a2)).encode()
|
||||
assert api.post_message(alice, chat, None, raw=raw, key=key).json() == first # key order irrelevant
|
||||
api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a2, a1]}, 409, key=key) # other body
|
||||
api.post_message(alice, chat, {"text": "файлы"}, 409, key=key)
|
||||
# A failed attachment check does not burn the key: the same key works with a valid body later.
|
||||
key2 = uuid.uuid4().hex
|
||||
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404, key=key2)
|
||||
api.post_message(alice, chat, {"text": "x", "attachment_ids": [a1]}, key=key2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", [
|
||||
b'{"attachment_ids":["%(a)s"]}', # text missing
|
||||
b'{"text":null,"attachment_ids":["%(a)s"]}', # text null
|
||||
b'{"text":1,"attachment_ids":["%(a)s"]}',
|
||||
b'{"text":"x","attachment_ids":null}', # attachment_ids null
|
||||
b'{"text":"x","attachment_ids":"%(a)s"}',
|
||||
b'{"text":"x","attachment_ids":{}}',
|
||||
b'{"text":"x","attachment_ids":["%(a)s","%(a)s"]}', # duplicates
|
||||
b'{"text":"x","attachment_ids":["%(a)s",1]}',
|
||||
b'{"text":"x","attachment_ids":["not-a-uuid"]}',
|
||||
b'{"text":"","attachment_ids":[]}', # neither text nor files
|
||||
b'{"text":" "}',
|
||||
b'{"text":"x","attachment_ids":[],"sender_id":"%(a)s"}',
|
||||
], ids=lambda raw: raw.decode()[:40])
|
||||
def test_message_body_validation(api, trio, raw):
|
||||
alice, _, _, chat, _ = trio
|
||||
a = api.upload(alice, chat, SAMPLE).json()["id"]
|
||||
api.post_message(alice, chat, None, 400, raw=raw % {b"a": a.encode()})
|
||||
|
||||
|
||||
def test_at_most_ten_attachments(api, trio):
|
||||
alice, _, _, chat, _ = trio
|
||||
ids = [api.upload(alice, chat, imagegen.png(4, 4)).json()["id"] for _ in range(11)]
|
||||
assert len(api.post_message(alice, chat, {"text": "", "attachment_ids": ids[:10]}).json()["attachment_ids"]) == 10
|
||||
api.post_message(alice, chat, {"text": "", "attachment_ids": ids}, 400)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ limits
|
||||
def test_size_limit_through_the_load_balancer(api, trio):
|
||||
alice, bob, _, chat, _ = trio
|
||||
exact = imagegen.png_exactly(10 * MiB)
|
||||
assert len(exact) == 10 * MiB
|
||||
att = api.upload(alice, chat, exact).json() # exactly the limit: accepted via LB
|
||||
assert content(api, bob, att["id"]).content == exact
|
||||
started = time.monotonic()
|
||||
api.upload(alice, chat, exact + b"\x00", 413) # Content-Length over the limit
|
||||
assert time.monotonic() - started < 5
|
||||
over = exact + bytes(1024)
|
||||
|
||||
def chunks(): # no Content-Length: chunked body
|
||||
for i in range(0, len(over), 256 * 1024):
|
||||
yield over[i:i + 256 * 1024]
|
||||
error = api.upload(alice, chat, chunks(), 413).json()
|
||||
assert error["error"]["code"] == "payload_too_large"
|
||||
|
||||
|
||||
def test_pixel_limit_before_decoding(api, trio):
|
||||
alice, _, _, chat, _ = trio
|
||||
for data, ctype in ((imagegen.png_bomb_header(), "image/png"), (imagegen.jpeg_bomb_header(), "image/jpeg"),
|
||||
(imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), "image/png")):
|
||||
started = time.monotonic()
|
||||
assert api.upload(alice, chat, data, 413, content_type=ctype).json()["error"]["code"] == "payload_too_large"
|
||||
assert time.monotonic() - started < 2 # refused from the header, nothing decoded
|
||||
|
||||
|
||||
def test_exactly_40_megapixels_is_accepted(api, trio):
|
||||
alice, _, _, chat, _ = trio
|
||||
data = imagegen.png(8000, 5000, compress_level=9) # 40 000 000 px, ~120 KB file
|
||||
att = api.upload(alice, chat, data).json()
|
||||
assert att["size_bytes"] == len(data)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ content checks
|
||||
BROKEN = {
|
||||
"png: first 33 bytes of the fixture": (SAMPLE[:33], "image/png"),
|
||||
"png: truncated": (SAMPLE[:-20], "image/png"),
|
||||
"png: corrupt IDAT, valid CRC": (imagegen.png_corrupt_idat(), "image/png"),
|
||||
"png: fewer rows than IHDR": (imagegen.png_short_pixels(), "image/png"),
|
||||
"png: flipped byte": (SAMPLE[:700] + bytes([SAMPLE[700] ^ 0x5A]) + SAMPLE[701:], "image/png"),
|
||||
"png: bytes after IEND": (SAMPLE + b"<?php echo 1; ?>", "image/png"),
|
||||
"jpeg: no scan data after SOS": (imagegen.jpeg_without_scan_data(), "image/jpeg"),
|
||||
"jpeg: truncated": (imagegen.jpeg(400, 300, noise=True)[:5000], "image/jpeg"),
|
||||
"mime: png bytes declared jpeg": (SAMPLE, "image/jpeg"),
|
||||
"mime: jpeg bytes declared png": (imagegen.jpeg(), "image/png"),
|
||||
"mime: gif declared png": (b"GIF89a" + bytes(100), "image/png"),
|
||||
"mime: html declared png": (b"<html><script>alert(1)</script></html>", "image/png"),
|
||||
"type: text/plain": (SAMPLE, "text/plain"),
|
||||
"type: octet-stream": (SAMPLE, "application/octet-stream"),
|
||||
"type: multipart": (SAMPLE, "multipart/form-data; boundary=x"),
|
||||
"type: missing": (SAMPLE, None),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", list(BROKEN))
|
||||
def test_bad_content_is_415(api, trio, name):
|
||||
alice, _, _, chat, _ = trio
|
||||
data, ctype = BROKEN[name]
|
||||
error = api.upload(alice, chat, data, 415, content_type=ctype).json()
|
||||
assert error["error"]["code"] == "unsupported_media_type"
|
||||
|
||||
|
||||
def test_empty_body_is_400(api, trio):
|
||||
alice, _, _, chat, _ = trio
|
||||
api.upload(alice, chat, b"", 400)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ filename
|
||||
@pytest.mark.parametrize("query", [
|
||||
"", "filename=", "filename=%20%20%20", "filename=.", "filename=..", "filename=a%2Fb.png", "filename=a%5Cb.png",
|
||||
"filename=a%00b.png", "filename=c1%C2%85.png", "filename=tab%09.png", "filename=photo%E2%80%AEgnp.exe",
|
||||
"filename=%FF%FE.png", "filename=a.png&filename=b.png", "filename=" + "a" * 256,
|
||||
], ids=lambda q: q[:40] or "missing")
|
||||
def test_bad_filename_is_400(api, trio, query):
|
||||
alice, _, _, chat, _ = trio
|
||||
api.upload(alice, chat, SAMPLE, 400, query=query)
|
||||
|
||||
|
||||
def test_unicode_filename_is_kept_and_safely_exposed(api, trio):
|
||||
alice, _, _, chat, _ = trio
|
||||
name = 'отчёт "Q3"; ok.png'
|
||||
att = api.upload(alice, chat, SAMPLE, filename=name).json()
|
||||
assert att["filename"] == name
|
||||
disposition = content(api, alice, att["id"]).headers["content-disposition"]
|
||||
assert '"' not in disposition.split('filename="')[1].split('";')[0] # quotes neutralised in the fallback
|
||||
assert "filename*=UTF-8''" in disposition
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ variants
|
||||
def test_variant_parameter(api, trio):
|
||||
alice, _, eve, chat, _ = trio
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
assert content(api, alice, att["id"], query="?variant=original").content == SAMPLE
|
||||
content(api, alice, att["id"], 404, query="?variant=thumbnail") # schema name, not produced in lab 4
|
||||
content(api, alice, att["id"], 400, query="?variant=huge")
|
||||
content(api, eve, att["id"], 404, query="?variant=thumbnail") # outsider: 404, nothing revealed
|
||||
@@ -278,28 +278,37 @@ def test_scale_up_and_down_is_picked_up_without_errors(api):
|
||||
def test_compose_policy_limits_ports_secrets_images():
|
||||
config = json.loads(compose("config", "--format", "json").stdout)
|
||||
services = config["services"]
|
||||
assert set(services) == {"postgres", "valkey", "migrate", "api", "lb"}
|
||||
# The reconciler (profile "tools") is started on demand and not part of the running stand.
|
||||
assert set(services) == {"postgres", "valkey", "s3", "migrate", "s3-init", "api", "lb"}
|
||||
for name, svc in services.items():
|
||||
limits = svc.get("deploy", {}).get("resources", {}).get("limits", {})
|
||||
assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits"
|
||||
assert "container_name" not in svc, f"{name}: container_name prevents scaling"
|
||||
if name in ("api", "migrate"):
|
||||
assert "build" in svc # built from this repository's source
|
||||
elif name == "s3-init":
|
||||
assert svc["image"] == services["api"]["image"] # the same locally built image
|
||||
else:
|
||||
assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest"
|
||||
assert services["api"]["deploy"]["replicas"] >= 2 and not services["api"].get("ports")
|
||||
published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")}
|
||||
assert set(published) == {"lb"} and all(p["host_ip"] in ("127.0.0.1", "::1") for p in published["lb"])
|
||||
secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()}
|
||||
assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"],
|
||||
"migrate": ["database_url"], "api": ["cache_url", "database_url"], "lb": []}
|
||||
s3_keys = ['s3_admin_access_key_id', 's3_admin_secret_access_key', 's3_api_access_key_id', 's3_api_secret_access_key', 's3_reconciler_access_key_id', 's3_reconciler_secret_access_key']
|
||||
assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"], "s3": s3_keys,
|
||||
"migrate": ["database_url"], "s3-init": s3_keys, "lb": [],
|
||||
# the API: its own S3 key pair only (no admin, no reconciler keys)
|
||||
"api": ["cache_url", "database_url", "s3_api_access_key_id", "s3_api_secret_access_key"]}
|
||||
volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"]
|
||||
for name, svc in services.items()}
|
||||
assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "migrate": [], "api": [], "lb": []}
|
||||
assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "s3": ["s3data"], "migrate": [],
|
||||
"s3-init": [], "api": [], "lb": []}
|
||||
networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()}
|
||||
assert networks == {"postgres": ["data"], "valkey": ["data"], "migrate": ["data"], "api": ["data", "edge"],
|
||||
assert networks == {"postgres": ["data"], "valkey": ["data"], "s3": ["storage"], "migrate": ["data"],
|
||||
"s3-init": ["storage"], "api": ["data", "edge", "storage"],
|
||||
"lb": ["edge"]}, networks # the LB cannot reach the stores at all
|
||||
assert config["networks"]["data"].get("internal") is True
|
||||
assert config["networks"]["storage"].get("internal") is True
|
||||
|
||||
|
||||
def test_running_containers_have_limits_and_no_bypass_ports():
|
||||
@@ -317,7 +326,7 @@ def test_running_containers_have_limits_and_no_bypass_ports():
|
||||
|
||||
def test_migrations_ran_once_by_the_migrate_service():
|
||||
applied = psql("SELECT version FROM schema_migrations ORDER BY version").split()
|
||||
assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function"]
|
||||
assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function", "0004_attachments"]
|
||||
migrate = compose("ps", "-a", "--format", "json", "migrate").stdout
|
||||
rows = [json.loads(line) for line in migrate.splitlines() if line.strip()]
|
||||
assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows)
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Consistency of PostgreSQL and S3 around one upload (lab 4, level 4).
|
||||
|
||||
The two-store write is: intent row 'pending' -> S3 PUT -> row 'ready' -> 201. Failures between
|
||||
the steps are injected with FAULT_POINTS (compose variable, empty in normal runs; the API
|
||||
raises 503 at that exact point, as if the process or the network failed there):
|
||||
|
||||
* after_s3_put object written, row still pending, client got 503 -> the janitor removes
|
||||
the object and the row after PENDING_GRACE_SECONDS (here 3 s);
|
||||
* after_ready_commit row ready and object written, but the answer is lost (client got 503)
|
||||
-> nothing may be deleted: the attachment stays intact and downloadable.
|
||||
|
||||
Plus: an upload interrupted by the client and a stalled body leave nothing behind."""
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import imagegen
|
||||
from conftest import api_with_env, attachment_row, psql, s3_size
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
|
||||
|
||||
def rows_of_chat(chat_id: str) -> list[tuple[str, str, str]]:
|
||||
out = psql(f"SELECT id || ' ' || status || ' ' || object_key FROM attachments WHERE chat_id = '{chat_id}'")
|
||||
return [tuple(line.split()) for line in out.splitlines() if line]
|
||||
|
||||
|
||||
def wait_for(condition, timeout: float, what: str) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while not condition():
|
||||
assert time.monotonic() < deadline, what
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def raw_upload(token: str, chat_id: str, declared: int, body: bytes, *, wait_answer: float | None) -> bytes:
|
||||
"""Hand-made HTTP request: announce `declared` bytes, send only `body`, then either close the
|
||||
socket (client gone) or keep it open and read the answer after `wait_answer` seconds."""
|
||||
sock = socket.create_connection(("127.0.0.1", 8080), timeout=30)
|
||||
head = (f"POST /api/v1/chats/{chat_id}/attachments?filename=x.png HTTP/1.1\r\nHost: localhost\r\n"
|
||||
f"Authorization: Bearer {token}\r\nContent-Type: image/png\r\nContent-Length: {declared}\r\n\r\n")
|
||||
sock.sendall(head.encode() + body)
|
||||
if wait_answer is None:
|
||||
sock.close()
|
||||
return b""
|
||||
time.sleep(wait_answer)
|
||||
answer = sock.recv(65536)
|
||||
sock.close()
|
||||
return answer
|
||||
|
||||
|
||||
def test_interrupted_and_stalled_uploads_leave_nothing(api):
|
||||
alice = api.user("alice")
|
||||
chat = api.chat(alice)
|
||||
data = imagegen.png_of_size(2_000_000)
|
||||
raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=None) # client disconnects
|
||||
with api_with_env(UPLOAD_READ_TIMEOUT_SECONDS="2"):
|
||||
answer = raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=5) # client stalls
|
||||
assert answer.startswith(b"HTTP/1.1 400") and b"invalid_request" in answer, answer[:200]
|
||||
assert b"x-instance-id" in answer.lower()
|
||||
time.sleep(1)
|
||||
assert rows_of_chat(chat["id"]) == [] # nothing was written: the body is checked before any write
|
||||
|
||||
|
||||
def test_crash_between_s3_put_and_commit_is_cleaned_by_janitor(api):
|
||||
alice = api.user("alice")
|
||||
chat = api.chat(alice)
|
||||
with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"):
|
||||
assert api.upload(alice, chat, SAMPLE, 503).json()["error"]["code"] == "unavailable"
|
||||
[(attachment_id, status, key)] = rows_of_chat(chat["id"])
|
||||
assert status == "pending" and s3_size(key) == len(SAMPLE) # the orphan-to-be exists
|
||||
api.call("GET", f"/api/v1/attachments/{attachment_id}", 404, token=alice["token"]) # invisible
|
||||
api.post_message(alice, chat, {"text": "", "attachment_ids": [attachment_id]}, 404) # not attachable
|
||||
wait_for(lambda: rows_of_chat(chat["id"]) == [], 30, "janitor did not remove the stale intent")
|
||||
assert s3_size(key) is None # object removed first
|
||||
# Normal configuration again: an upload in the same chat works and is not touched.
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
assert attachment_row(att["id"])["status"] == "ready"
|
||||
|
||||
|
||||
def test_lost_answer_after_commit_never_deletes_the_object(api):
|
||||
"""The reviewer found this in another PR: COMMIT succeeded, the answer was lost, the API
|
||||
'compensated' by deleting the object -> ready row without bytes. Here nothing compensates
|
||||
in the request path, and the janitor only touches 'pending' rows."""
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
chat = api.chat(alice, bob)
|
||||
with api_with_env(FAULT_POINTS="after_ready_commit", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"):
|
||||
api.upload(alice, chat, SAMPLE, 503)
|
||||
[(attachment_id, status, key)] = rows_of_chat(chat["id"])
|
||||
assert status == "ready"
|
||||
time.sleep(6) # several janitor rounds
|
||||
assert rows_of_chat(chat["id"]) == [(attachment_id, "ready", key)] and s3_size(key) == len(SAMPLE)
|
||||
got = api.call("GET", f"/api/v1/attachments/{attachment_id}/content", 200, token=bob["token"])
|
||||
assert got.content == SAMPLE
|
||||
|
||||
|
||||
def test_fresh_pending_upload_is_not_touched_by_janitor(api):
|
||||
"""An intent younger than PENDING_GRACE_SECONDS is an upload in progress: never removed."""
|
||||
alice = api.user("alice")
|
||||
chat = api.chat(alice)
|
||||
with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="600", JANITOR_INTERVAL_SECONDS="1"):
|
||||
api.upload(alice, chat, SAMPLE, 503)
|
||||
time.sleep(4)
|
||||
[(_, status, key)] = rows_of_chat(chat["id"])
|
||||
assert status == "pending" and s3_size(key) == len(SAMPLE)
|
||||
@@ -78,7 +78,8 @@ def test_migrations_rerun_is_a_noop():
|
||||
assert "0001_init: already applied, skipped" in result.stdout
|
||||
assert "0002_idempotency_keys: already applied, skipped" in result.stdout
|
||||
assert "0003_post_message_function: already applied, skipped" in result.stdout
|
||||
assert psql("SELECT count(*) FROM schema_migrations") == "3"
|
||||
assert "0004_attachments: already applied, skipped" in result.stdout
|
||||
assert psql("SELECT count(*) FROM schema_migrations") == "4"
|
||||
assert psql("SELECT count(*) FROM users") == users_before
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Track 2: the reconciler of S3 and PostgreSQL (python -m messenger.reconcile).
|
||||
|
||||
Faults are injected on BOTH sides, directly into the stores (S3 with the API/reconciler key
|
||||
pairs from inside the storage network, PostgreSQL with psql):
|
||||
|
||||
S3 side: orphan object (no row), object deleted under a ready row, object overwritten with
|
||||
other bytes (size / same-size checksum), object of a quarantined row restored;
|
||||
DB side: stale intent with and without object, quarantine mark on a healthy row;
|
||||
fresh: orphan and intent younger than --grace (an upload in progress).
|
||||
|
||||
The tests prove: dry-run changes nothing in either store; --apply fixes exactly the old findings;
|
||||
fresh and used attachments are never deleted (ready rows only ever get a reversible quarantine);
|
||||
the second run finds nothing to do; the safety valve and store outages stop the run without
|
||||
damage. A small --grace (8-30 s, --allow-small-grace) keeps the tests fast."""
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import imagegen
|
||||
from conftest import (attachment_row, findings_by_key, psql, reconcile, s3_delete, s3_keys, s3_put, s3_size,
|
||||
service_down)
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
GRACE = ["--grace", "8", "--allow-small-grace"]
|
||||
# The main scenario needs "fresh" items to stay younger than --grace for the whole dry-run + snapshot
|
||||
# (each docker exec costs 1-3 s on Windows), hence a longer grace there.
|
||||
LONG_GRACE = ["--grace", "30", "--allow-small-grace"]
|
||||
|
||||
|
||||
def new_key() -> str:
|
||||
return "o/" + uuid.uuid4().hex
|
||||
|
||||
|
||||
def insert_intent(chat: dict, owner: dict, key: str, data: bytes, age_seconds: int) -> str:
|
||||
"""A 'pending' row as an upload leaves it after the intent COMMIT (crash afterwards)."""
|
||||
attachment_id = str(uuid.uuid4())
|
||||
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
|
||||
"object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'x.png', 'image/png', %d, "
|
||||
"decode('%s', 'hex'), 320, 200, '%s', 'pending', clock_timestamp() - make_interval(secs => %d), "
|
||||
"clock_timestamp() - make_interval(secs => %d))"
|
||||
% (attachment_id, chat["id"], owner["id"], len(data), hashlib.sha256(data).hexdigest(), key, age_seconds,
|
||||
age_seconds))
|
||||
return attachment_id
|
||||
|
||||
|
||||
def snapshot() -> tuple[str, frozenset]:
|
||||
"""Fingerprint of everything the reconciler could change: every attachment row + the key list."""
|
||||
rows = psql("SELECT count(*) || ':' || coalesce(md5(string_agg(id::text || status || object_key || "
|
||||
"coalesce(lost_reason, '-') || updated_at::text, ',' ORDER BY id)), '') FROM attachments")
|
||||
return rows, frozenset(s3_keys())
|
||||
|
||||
|
||||
def content_status(api, actor, attachment_id) -> int:
|
||||
return api.client.get(f"/api/v1/attachments/{attachment_id}/content",
|
||||
headers={"Authorization": "Bearer " + actor["token"]}).status_code
|
||||
|
||||
|
||||
def test_detect_and_safely_fix_faults_on_both_sides(api):
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
chat = api.chat(alice, bob)
|
||||
up = {name: api.upload(alice, chat, imagegen.png(30 + i, 20)).json() for i, name in enumerate("ABCDE")}
|
||||
data = {name: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
|
||||
for name, a in up.items()}
|
||||
key = {name: attachment_row(a["id"])["object_key"] for name, a in up.items()}
|
||||
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
|
||||
|
||||
# ---- inject (old)
|
||||
orphan_old = new_key()
|
||||
s3_put(orphan_old, SAMPLE)
|
||||
stale_with, stale_without = new_key(), new_key()
|
||||
s3_put(stale_with, SAMPLE)
|
||||
stale_with_id = insert_intent(chat, alice, stale_with, SAMPLE, 600) # < janitor grace 900 s
|
||||
stale_without_id = insert_intent(chat, alice, stale_without, SAMPLE, 600)
|
||||
s3_delete(key["C"]) # ready + used, bytes lost
|
||||
s3_put(key["D"], data["D"][:len(data["D"]) // 2]) # ready, unused, wrong size
|
||||
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' "
|
||||
f"WHERE id = '{up['E']['id']}'") # quarantined, but bytes are fine
|
||||
time.sleep(32) # everything above is now older than --grace
|
||||
# ---- inject (fresh: an upload in progress)
|
||||
fresh_orphan, fresh_intent = new_key(), new_key()
|
||||
s3_put(fresh_orphan, SAMPLE)
|
||||
s3_put(fresh_intent, SAMPLE)
|
||||
fresh_intent_id = insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
|
||||
|
||||
# ---- dry-run: report everything, change nothing
|
||||
before = snapshot()
|
||||
dry = reconcile(*LONG_GRACE)
|
||||
assert snapshot() == before, "dry-run modified a store"
|
||||
f = findings_by_key(dry)
|
||||
expected = {orphan_old: "orphan_object", fresh_orphan: "fresh_orphan_object", stale_with: "stale_intent",
|
||||
stale_without: "stale_intent", fresh_intent: "fresh_intent", key["C"]: "missing_object",
|
||||
key["D"]: "size_mismatch", key["E"]: "restored_object"}
|
||||
assert {k: f[k]["kind"] for k in expected} == expected
|
||||
assert key["A"] not in f and key["B"] not in f
|
||||
assert f[key["C"]]["used"] is True and dry["mode"] == "dry-run"
|
||||
assert all(f[k]["result"] in ("planned", "not_actionable") for k in expected)
|
||||
|
||||
# ---- apply
|
||||
applied = findings_by_key(reconcile("--apply", *LONG_GRACE))
|
||||
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"]):
|
||||
assert applied[k]["result"] == "applied", (k, applied[k])
|
||||
keys = s3_keys()
|
||||
assert orphan_old not in keys and stale_with not in keys
|
||||
assert attachment_row(stale_with_id) is None and attachment_row(stale_without_id) is None
|
||||
# fresh: untouched
|
||||
assert fresh_orphan in keys and fresh_intent in keys and attachment_row(fresh_intent_id)["status"] == "pending"
|
||||
# used, lost bytes: quarantined, NOT deleted; the message still references it
|
||||
row_c = attachment_row(up["C"]["id"])
|
||||
assert row_c["status"] == "ready" and row_c["lost_reason"] == "missing_object"
|
||||
items, _ = api.history(bob, chat, 100)
|
||||
assert any(up["C"]["id"] in m["attachment_ids"] for m in items)
|
||||
assert content_status(api, bob, up["C"]["id"]) == 404
|
||||
api.post_message(alice, chat, {"text": "", "attachment_ids": [up["C"]["id"]]}, 409) # cannot attach a lost file
|
||||
assert attachment_row(up["D"]["id"])["lost_reason"] == "size_mismatch"
|
||||
assert attachment_row(up["E"]["id"])["lost_reason"] is None # quarantine lifted
|
||||
for name in "ABE":
|
||||
got = api.call("GET", f"/api/v1/attachments/{up[name]['id']}/content", 200, token=bob["token"])
|
||||
assert got.content == data[name]
|
||||
|
||||
# ---- second run: idempotent, nothing left to do for these keys
|
||||
again = findings_by_key(reconcile("--apply", *LONG_GRACE))
|
||||
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"], key["A"], key["B"]):
|
||||
assert k not in again, (k, again.get(k))
|
||||
|
||||
# ---- the lost object comes back from a backup: the quarantine is lifted, the file is served again
|
||||
s3_put(key["C"], data["C"])
|
||||
restored = findings_by_key(reconcile("--apply", *LONG_GRACE))
|
||||
assert restored[key["C"]]["kind"] == "restored_object" and restored[key["C"]]["result"] == "applied"
|
||||
assert api.call("GET", f"/api/v1/attachments/{up['C']['id']}/content", 200, token=bob["token"]).content == data["C"]
|
||||
# tidy up the fresh items of this test once they are old enough
|
||||
time.sleep(31)
|
||||
reconcile("--apply", *LONG_GRACE)
|
||||
assert fresh_orphan not in s3_keys() and attachment_row(fresh_intent_id) is None
|
||||
|
||||
|
||||
def test_checksum_mode_finds_same_size_corruption(api):
|
||||
alice = api.user("alice")
|
||||
chat = api.chat(alice)
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
k = attachment_row(att["id"])["object_key"]
|
||||
s3_put(k, SAMPLE[:1000] + bytes([SAMPLE[1000] ^ 1]) + SAMPLE[1001:]) # same size, other bytes
|
||||
time.sleep(9)
|
||||
assert k not in findings_by_key(reconcile(*GRACE)) # sizes match: listing cannot see it
|
||||
report = reconcile("--verify-checksums", *GRACE)
|
||||
assert findings_by_key(report)[k]["kind"] == "checksum_mismatch"
|
||||
assert report["cost"]["s3_get_bytes"] >= len(SAMPLE)
|
||||
assert findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]["result"] == "applied"
|
||||
assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch"
|
||||
|
||||
|
||||
def test_safety_valve_refuses_mass_deletion():
|
||||
"""If (almost) every object looked orphaned - e.g. the reconciler pointed at an empty database -
|
||||
--apply must stop before deleting anything."""
|
||||
orphan = new_key()
|
||||
s3_put(orphan, SAMPLE)
|
||||
time.sleep(9)
|
||||
report = reconcile("--apply", "--max-orphan-fraction", "0.000001", *GRACE, expect_code=3)
|
||||
assert report["error"].startswith("safety_valve") and report["actions"]["applied"] == 0
|
||||
assert orphan in s3_keys()
|
||||
reconcile("--apply", *GRACE)
|
||||
assert orphan not in s3_keys()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("service", ["s3", "postgres"])
|
||||
def test_store_outage_stops_the_run_without_damage(api, service):
|
||||
with service_down(service, "stop"):
|
||||
report = reconcile(*GRACE, expect_code=1)
|
||||
assert report["error"].startswith("store_unavailable")
|
||||
assert reconcile(*GRACE)["error"] is None
|
||||
|
||||
|
||||
def test_live_uploads_are_never_hurt_by_concurrent_apply(api):
|
||||
"""Uploads and messages run while --apply runs repeatedly: every upload that got 201 stays
|
||||
ready and downloadable byte for byte, every message keeps its attachments."""
|
||||
alice = api.user("alice")
|
||||
chat = api.chat(alice)
|
||||
payloads = [imagegen.png(16 + i, 16) for i in range(40)]
|
||||
done: list[tuple[dict, bytes]] = []
|
||||
|
||||
def worker(chunk: list[bytes]) -> None:
|
||||
for p in chunk:
|
||||
att = api.upload(alice, chat, p).json()
|
||||
api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]})
|
||||
done.append((att, p))
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(4) as pool:
|
||||
futures = [pool.submit(worker, payloads[i::4]) for i in range(4)]
|
||||
runs = 0
|
||||
while not all(x.done() for x in futures):
|
||||
report = reconcile("--apply", *GRACE)
|
||||
assert report["error"] is None
|
||||
runs += 1
|
||||
for x in futures:
|
||||
x.result()
|
||||
assert runs >= 1 and len(done) == len(payloads)
|
||||
for att, p in done:
|
||||
row = attachment_row(att["id"])
|
||||
assert row["status"] == "ready" and row["lost_reason"] is None and s3_size(row["object_key"]) == len(p)
|
||||
for att, p in done[::8]:
|
||||
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]).content == p
|
||||
@@ -0,0 +1,107 @@
|
||||
"""S3 outages, private bucket and failover of attachments (lab 4, levels 3 and 4).
|
||||
|
||||
Every injected failure is undone in `finally` (conftest.service_down / replica_stopped), and the
|
||||
stand is waited for until HAProxy sees all replicas UP again."""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import imagegen
|
||||
from conftest import attachment_row, compose, probe_inside, replica_stopped, running_replicas, service_down
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def pair(api):
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
return alice, bob, api.chat(alice, bob)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("how", ["stop", "pause"])
|
||||
def test_s3_outage_is_503_and_text_keeps_working(api, pair, how):
|
||||
"""S3 down (stopped: connection refused; paused: no answer at all):
|
||||
* readiness is 503 on every replica (and through the LB), quickly;
|
||||
* uploads/downloads are 503 `unavailable` within the S3 deadline, never 500;
|
||||
* the LB's degraded mode keeps chats/text messages working (HAProxy backend api_core);
|
||||
* after recovery the earlier file is intact."""
|
||||
alice, bob, chat = pair
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
with service_down("s3", how):
|
||||
deadline = time.monotonic() + 15
|
||||
while api.client.get("/health/ready").status_code != 503:
|
||||
assert time.monotonic() < deadline, "readiness did not turn 503"
|
||||
time.sleep(0.5)
|
||||
for replica in running_replicas():
|
||||
assert probe_inside(replica, "/health/ready") == 503
|
||||
assert probe_inside(replica, "/health/ready/core") == 200
|
||||
started = time.monotonic()
|
||||
error = api.upload(alice, chat, SAMPLE, 503).json()
|
||||
assert error["error"]["code"] == "unavailable"
|
||||
assert time.monotonic() - started < 8
|
||||
started = time.monotonic()
|
||||
api.call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"])
|
||||
assert time.monotonic() - started < 8
|
||||
api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]) # metadata: PostgreSQL only
|
||||
api.send(alice, chat, "текст работает без S3") # degraded mode
|
||||
got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
|
||||
assert got.content == SAMPLE
|
||||
api.upload(alice, chat, SAMPLE)
|
||||
|
||||
|
||||
def test_upload_survives_failover_of_the_serving_replica(api, pair):
|
||||
"""Upload -> stop the replica that served it -> another replica downloads the same bytes
|
||||
(nothing of the file lives in the replica)."""
|
||||
alice, bob, chat = pair
|
||||
data = imagegen.png_of_size(500_000)
|
||||
response = api.upload(alice, chat, data)
|
||||
att, served_by = response.json(), response.headers["x-instance-id"]
|
||||
replica = next(r for r in running_replicas() if r["hostname"] == served_by)
|
||||
with replica_stopped(replica, "kill"):
|
||||
for _ in range(5):
|
||||
got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
|
||||
assert got.headers["x-instance-id"] != served_by and got.content == data
|
||||
|
||||
|
||||
def test_bucket_is_private_outside_the_api(api, pair):
|
||||
"""Inside the storage network (where the API is): an UNSIGNED GET of a real object and an
|
||||
unsigned LIST are refused; the same GET signed with the API key pair works (control).
|
||||
SeaweedFS admin endpoints (master 9333, filer 8888, volume 8080) are not reachable."""
|
||||
alice, _, chat = pair
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
key = attachment_row(att["id"])["object_key"]
|
||||
probe = (
|
||||
"import urllib.request as u, urllib.error as e, socket\n"
|
||||
"def st(url):\n"
|
||||
" try: return u.urlopen(url, timeout=3).status\n"
|
||||
" except e.HTTPError as x: return x.code\n"
|
||||
f"print(st('http://s3:8333/messenger-attachments/{key}'), st('http://s3:8333/messenger-attachments?list-type=2'))\n"
|
||||
"for port in (9333, 8888, 8080):\n"
|
||||
" s = socket.socket(); s.settimeout(2)\n"
|
||||
" print(port, s.connect_ex(('s3', port)))\n")
|
||||
out = compose("exec", "-T", "api", "python", "-c", probe).stdout.split("\n")
|
||||
get_status, list_status = out[0].split()
|
||||
assert get_status in ("401", "403") and list_status in ("401", "403"), out[0]
|
||||
for line in out[1:4]:
|
||||
port, code = line.split()
|
||||
assert code != "0", f"admin port {port} reachable"
|
||||
signed = compose("exec", "-T", "api", "python", "-c",
|
||||
"import asyncio\nfrom messenger.config import Settings\nfrom messenger.storage import Storage\n"
|
||||
"async def m():\n s=Storage(Settings.from_env()); await s.open()\n"
|
||||
f" print((await s.head('{key}')).size); await s.close()\nasyncio.run(m())").stdout.strip()
|
||||
assert signed == str(len(SAMPLE))
|
||||
|
||||
|
||||
def test_s3_restart_keeps_objects(api, pair):
|
||||
alice, bob, chat = pair
|
||||
data = imagegen.png_of_size(200_000)
|
||||
att = api.upload(alice, chat, data).json()
|
||||
with service_down("s3", "stop"):
|
||||
pass
|
||||
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]).content == data
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Unit tests of the image checks (src/messenger/images.py) and the filename rules. No services.
|
||||
|
||||
The JPEG decoder (djpeg) exists in the API image; locally these tests are skipped without it
|
||||
(the same cases run against the stand in test_attachments.py)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
|
||||
import imagegen # noqa: E402
|
||||
from messenger import images, validation # noqa: E402
|
||||
from messenger.errors import InvalidRequest # noqa: E402
|
||||
|
||||
LIMIT = 40_000_000
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
has_djpeg = pytest.mark.skipif(shutil.which("djpeg") is None, reason="djpeg (libjpeg-turbo-progs) not installed")
|
||||
|
||||
|
||||
def status(data: bytes) -> int:
|
||||
"""201 if accepted, else the HTTP status of the ImageError (413/415)."""
|
||||
try:
|
||||
info = images.sniff(data, LIMIT)
|
||||
if info is None:
|
||||
return 415 # header never completed
|
||||
asyncio.run(images.verify(data, info))
|
||||
return 201
|
||||
except images.ImageError as exc:
|
||||
return exc.status
|
||||
|
||||
|
||||
def test_public_fixture_is_accepted():
|
||||
assert images.sniff(SAMPLE, LIMIT) == images.ImageInfo("image/png", 320, 200)
|
||||
assert status(SAMPLE) == 201
|
||||
|
||||
|
||||
@pytest.mark.parametrize("data", [
|
||||
imagegen.png(37, 23, "RGBA"), imagegen.png(33, 17, "P"), imagegen.png(33, 17, "1"),
|
||||
imagegen.png(33, 17, "L"), imagegen.png(81, 57, "RGB", noise=True), imagegen.png(1, 1),
|
||||
], ids=["rgba", "palette", "1bit", "gray", "noise", "1x1"])
|
||||
def test_valid_png_kinds(data):
|
||||
assert status(data) == 201
|
||||
|
||||
|
||||
def test_interlaced_png_is_accepted():
|
||||
# Pillow cannot write Adam7; build one by hand: 9 x 9 grey, every pass with filter 0.
|
||||
from messenger.images import _png_segments
|
||||
raw = b"".join(b"\x00" + bytes(stride - 1) for size, stride in _png_segments(9, 9, 8, 0, 1)
|
||||
for _ in range(size // stride))
|
||||
data = imagegen.PNG_SIGNATURE + imagegen.ihdr(9, 9, 8, 0, 1) + imagegen.chunk(b"IDAT", __import__("zlib").compress(raw)) \
|
||||
+ imagegen.chunk(b"IEND", b"")
|
||||
assert status(data) == 201
|
||||
|
||||
|
||||
BROKEN_PNG = {
|
||||
"first 33 bytes of the fixture": SAMPLE[:33],
|
||||
"truncated (no IEND)": SAMPLE[:-12],
|
||||
"truncated in IDAT": SAMPLE[:len(SAMPLE) // 2],
|
||||
"flipped byte -> CRC error": SAMPLE[:500] + bytes([SAMPLE[500] ^ 0xFF]) + SAMPLE[501:],
|
||||
"garbage IDAT with valid CRC": imagegen.png_corrupt_idat(),
|
||||
"fewer rows than IHDR": imagegen.png_short_pixels(),
|
||||
"bytes after IEND (polyglot)": SAMPLE + b"PK\x03\x04zip",
|
||||
"text": b"just text, definitely not an image",
|
||||
"gif": b"GIF89a" + bytes(64),
|
||||
"two bytes of a signature": b"\x89P",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", list(BROKEN_PNG))
|
||||
def test_broken_png_is_415(name):
|
||||
assert status(BROKEN_PNG[name]) == 415, name
|
||||
|
||||
|
||||
def test_pixel_limit_is_checked_from_the_header():
|
||||
with pytest.raises(images.ImageError) as exc:
|
||||
images.sniff(imagegen.png_bomb_header()[:33], LIMIT) # 33 bytes are enough
|
||||
assert exc.value.status == 413
|
||||
exact = imagegen.PNG_SIGNATURE + imagegen.ihdr(8000, 5000) # exactly 40 000 000: allowed
|
||||
assert images.sniff(exact, LIMIT).width == 8000
|
||||
with pytest.raises(images.ImageError) as exc:
|
||||
images.sniff(imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), LIMIT)
|
||||
assert exc.value.status == 413
|
||||
with pytest.raises(images.ImageError) as exc:
|
||||
images.sniff(imagegen.jpeg_bomb_header(), LIMIT)
|
||||
assert exc.value.status == 413
|
||||
|
||||
|
||||
def test_png_verification_is_streaming_and_fast():
|
||||
"""40 Mpx RGB = 120 MB of raw pixels: inflated in 1 MiB steps and discarded."""
|
||||
big = imagegen.png(8000, 5000, compress_level=9)
|
||||
started = time.perf_counter()
|
||||
assert status(big) == 201
|
||||
assert time.perf_counter() - started < 5
|
||||
|
||||
|
||||
def test_jpeg_header_is_parsed_incrementally():
|
||||
data = imagegen.jpeg(320, 200)
|
||||
assert images.sniff(data[:4], LIMIT) is None # SOF not arrived yet
|
||||
assert images.sniff(data, LIMIT) == images.ImageInfo("image/jpeg", 320, 200)
|
||||
|
||||
|
||||
@has_djpeg
|
||||
@pytest.mark.parametrize("data", [imagegen.jpeg(), imagegen.jpeg(640, 480, progressive=True)], ids=["base", "prog"])
|
||||
def test_valid_jpeg(data):
|
||||
assert status(data) == 201
|
||||
|
||||
|
||||
@has_djpeg
|
||||
@pytest.mark.parametrize("name,data", [
|
||||
("no scan data after SOS", imagegen.jpeg_without_scan_data()),
|
||||
("truncated", imagegen.jpeg(320, 240, noise=True)[:3000]),
|
||||
], ids=lambda v: v if isinstance(v, str) else "")
|
||||
def test_broken_jpeg_is_415(name, data):
|
||||
assert status(data) == 415, name
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ filename
|
||||
@pytest.mark.parametrize("name", ["photo.png", "фото отпуска.jpg", "a" * 255, "emoji 👨👩👧.png", " x ", "..."[:2] + "x"])
|
||||
def test_good_filenames(name):
|
||||
assert validation.filename_param([("filename", name)]) == name
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["", " ", " ", ".", "..", "a/b.png", "a\\b.png", "a\x00b", "c1\x85.png",
|
||||
"tab\t.png", "del\x7f.png", "x
y", "photognp.exe", "a" * 256])
|
||||
def test_bad_filenames(name):
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.filename_param([("filename", name)])
|
||||
|
||||
|
||||
def test_filename_must_be_single_and_present():
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.filename_param([])
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.filename_param([("filename", "a.png"), ("filename", "b.png")])
|
||||
|
||||
|
||||
def test_query_must_be_valid_utf8():
|
||||
assert validation.query_items(b"filename=%D1%84.png") == [("filename", "ф.png")]
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.query_items(b"filename=%FF%FE.png")
|
||||
@@ -114,6 +114,29 @@ def test_argon2id_hash_and_verify():
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ config
|
||||
@pytest.fixture(autouse=True)
|
||||
def s3_credentials(monkeypatch):
|
||||
"""Lab 4: Settings require the API's S3 key pair; synthetic values for the unit tests."""
|
||||
monkeypatch.setenv("S3_ACCESS_KEY_ID", "AKTEST")
|
||||
monkeypatch.setenv("S3_SECRET_ACCESS_KEY", "secret-for-unit-tests")
|
||||
|
||||
|
||||
def test_s3_settings_are_validated_and_contract_limits_cannot_be_raised(monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
|
||||
settings = Settings.from_env()
|
||||
assert (settings.max_upload_bytes, settings.max_image_pixels) == (10 * 1024 * 1024, 40_000_000)
|
||||
assert settings.fault_points == frozenset() and settings.upload_concurrency == 4
|
||||
for name, value in (("MAX_UPLOAD_BYTES", "10485761"), ("MAX_IMAGE_PIXELS", "40000001"),
|
||||
("FAULT_POINTS", "drop_database"), ("S3_ENDPOINT", "ftp://s3"), ("S3_BUCKET", "Bad_Bucket")):
|
||||
monkeypatch.setenv(name, value)
|
||||
with pytest.raises(ConfigError):
|
||||
Settings.from_env()
|
||||
monkeypatch.delenv(name)
|
||||
monkeypatch.delenv("S3_SECRET_ACCESS_KEY")
|
||||
with pytest.raises(ConfigError):
|
||||
Settings.from_env()
|
||||
|
||||
|
||||
def test_settings_read_secret_files_and_contract_names(tmp_path, monkeypatch):
|
||||
url_file = tmp_path / "database_url"
|
||||
url_file.write_text("postgresql://u:p@db/x\n", encoding="utf-8")
|
||||
|
||||
Reference in New Issue
Block a user