lab4 трек 2: reconciler S3/PostgreSQL, тесты вложений, отказов и согласованности

- python -m messenger.reconcile: merge-join листинга S3 и attachments по object_key;
  dry-run read-only конструктивно (default_transaction_read_only + S3 без записи),
  --apply: orphan/stale intent удаляются, ready-строки только в карантин (обратимо),
  перепроверка под FOR UPDATE, предохранитель от массового удаления;
- attachment_ids хранится в строке сообщения (история остаётся одним index range scan),
  message_attachments — для FK и обратного поиска;
- семафор полной проверки изображений (IMAGE_VERIFY_CONCURRENCY);
- тесты: вложения (56), отказ/пауза S3 и деградированный режим LB, failover загрузки,
  приватность bucket, FAULT_POINTS после PUT/после COMMIT, janitor, reconciler;
- бенчмарки памяти загрузок и стоимости обхода reconciler.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alexey Bodanyuk
2026-10-25 14:05:03 +03:00
co-authored by Claude Opus 5.5
parent 335df4b2b2
commit a2d867bec6
22 changed files with 1943 additions and 36 deletions
+26 -1
View File
@@ -17,6 +17,29 @@ API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
# Write path of POST message: function = one SQL call (migration 0003), multi = lab3 baseline path.
MESSAGE_WRITE_PATH=function
# Lab 4: private S3 bucket (SeaweedFS service `s3`). The API gets ITS OWN key pair (Read+Write on
# this bucket only) as Docker secrets; admin/reconciler pairs go only to s3-init/reconciler.
S3_ENDPOINT=http://s3:8333
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
# Contract limits (may be lowered, never raised above 10485760 / 40000000).
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
# Per replica: bodies buffered at once (RAM bound), wait for a slot, body deadline, image checks at once.
UPLOAD_CONCURRENCY=4
UPLOAD_QUEUE_TIMEOUT_SECONDS=5
UPLOAD_READ_TIMEOUT_SECONDS=30
IMAGE_VERIFY_TIMEOUT_SECONDS=10
IMAGE_VERIFY_CONCURRENCY=1
JPEG_MAX_MEMORY_MB=256
S3_TIMEOUT_SECONDS=3
# Janitor: 'pending' intents older than this are failed uploads (object + row removed).
PENDING_GRACE_SECONDS=900
JANITOR_INTERVAL_SECONDS=60
# Level 4 failure experiments only (after_s3_put, after_ready_commit); keep empty.
FAULT_POINTS=
# Additional settings of this implementation (defaults shown; all optional):
LOG_LEVEL=info
DB_POOL_MIN=1
@@ -30,7 +53,9 @@ DRAIN_SECONDS=3
SHUTDOWN_TIMEOUT_SECONDS=8
# CPU/RAM limits (docker compose deploy.resources.limits); API_* are per replica.
API_CPUS=1.0
API_MEMORY=256m
API_MEMORY=512m
S3_CPUS=1.0
S3_MEMORY=512m
PG_CPUS=2.0
PG_MEMORY=1g
VALKEY_CPUS=0.5
+17 -2
View File
@@ -29,9 +29,9 @@ test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# ---------------------------------------------------------------------------
# Student solution (lab 3, Python). On Windows: PowerShell/Git Bash with GNU make, public
# Student solution (lab 4, Python). On Windows: PowerShell/Git Bash with GNU make, public
# tests need PYTHONUTF8=1, and the hooks are started through scripts/*.cmd (Git Bash).
.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series
.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series reconcile reconcile-apply bench-memory bench-reconcile
secrets:
$(PYTHON) scripts/init_secrets.py
@@ -80,3 +80,18 @@ unit-test:
explain:
$(PYTHON) scripts/explain_history.py --output evidence/explain-history.txt
# Track 2: reconciler of S3 objects vs attachment rows (dry-run by default, read-only).
reconcile:
docker compose run --rm reconciler
reconcile-apply:
docker compose run --rm reconciler --apply
# Level 4: peak RAM of the replicas under 16 simultaneous uploads (evidence/memory/).
bench-memory:
$(PYTHON) bench/upload_memory.py --output evidence/memory
# Track 2: cost of one reconciler pass for 1k/10k/50k attachments (evidence/reconcile/).
bench-reconcile:
$(PYTHON) bench/reconcile_cost.py --output evidence/reconcile
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""Track 2 experiment: cost of one reconciler pass as a function of the number of attachments.
Hypothesis: a pass is O(N) with ~N/1000 S3 list requests + N/1000 DB pages and constant memory
(merge-join of two key-ordered streams); --verify-checksums adds reading every byte (O(total size)).
Method: synthetic, CONSISTENT attachments (row 'ready' + 1 KiB object with the matching size and
SHA-256, filename 'bench-reconcile') are added in steps up to the largest N; after each step the
reconciler runs in dry-run 3 times (median reported), plus page-size and checksum variants.
Everything synthetic is deleted at the end (objects first, then rows).
python bench/reconcile_cost.py --steps 1000 10000 50000 --output evidence/reconcile
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import platform
import statistics
import subprocess
import sys
import time
import uuid
from pathlib import Path
import httpx2 as httpx
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
PASSWORD = "Bench-password-Ω-12345"
OBJECT = bytes(range(256)) * 4 # 1 KiB, the same content for every object
SHA = hashlib.sha256(OBJECT).hexdigest()
PUT_SCRIPT = """
import asyncio, sys
from messenger.config import Settings, s3_settings
from messenger.storage import Storage
DATA = bytes(range(256)) * 4
async def main():
s = Storage(Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings()))
await s.open()
keys = sys.stdin.read().split()
sem = asyncio.Semaphore(32)
async def one(k):
async with sem:
await s.{op}(k{args})
await asyncio.gather(*(one(k) for k in keys))
await s.close()
print(len(keys))
asyncio.run(main())
"""
def compose(*args: str, stdin: str | None = None, timeout: float = 1800, check: bool = True) -> subprocess.CompletedProcess:
return subprocess.run(["docker", "compose", *args], cwd=ROOT, input=stdin, capture_output=True, text=True,
encoding="utf-8", timeout=timeout, check=check)
def psql(sql: str) -> str:
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v",
"ON_ERROR_STOP=1", "-c", sql).stdout.strip()
def s3_bulk(op: str, keys: list[str]) -> None:
script = PUT_SCRIPT.replace("{op}", op).replace("{args}", ", DATA, 'image/png', {}" if op == "put" else "")
compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "python", "reconciler", "-c", script, stdin="\n".join(keys))
def owner_and_chat() -> tuple[str, str]:
import base64
c = httpx.Client(base_url=BASE, timeout=30)
name = "rc_" + uuid.uuid4().hex[:10]
c.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
token = c.post("/api/v1/auth/sessions", headers={
"Authorization": "Basic " + base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()}).json()["token"]
me = c.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
chat = c.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
json={"title": "bench", "member_ids": [me["id"]]}).json()
return me["id"], chat["id"]
def add(n: int, owner: str, chat: str) -> None:
keys = ["o/" + uuid.uuid4().hex for _ in range(n)]
for i in range(0, n, 5000):
part = keys[i:i + 5000]
values = ",".join(f"('{uuid.uuid4()}','{k}')" for k in part)
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
"object_key, status, created_at, updated_at) "
f"SELECT v.id::uuid, '{chat}', '{owner}', 'bench-reconcile', 'image/png', {len(OBJECT)}, "
f"decode('{SHA}', 'hex'), 1, 1, v.k, 'ready', clock_timestamp() - interval '2 hours', "
f"clock_timestamp() - interval '2 hours' FROM (VALUES {values}) AS v(id, k)")
s3_bulk("put", keys)
def reconcile(*args: str) -> dict:
out = compose("run", "--rm", "--no-deps", "-T", "reconciler", *args).stdout
return json.loads(out)
def measure(label: str, args: list[str], repeats: int) -> dict:
runs = [reconcile(*args) for _ in range(repeats)]
durations = [r["duration_seconds"] for r in runs]
last = runs[-1]
return {"label": label, "args": args, "runs": repeats, "duration_s_median": statistics.median(durations),
"duration_s_all": durations, "objects": last["scanned"]["objects"], "rows": last["scanned"]["rows"],
"findings": {k: v for k, v in last["findings"].items() if v}, "cost": last["cost"]}
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--steps", type=int, nargs="+", default=[1000, 10000, 50000])
p.add_argument("--repeats", type=int, default=3)
p.add_argument("--output", default="evidence/reconcile")
args = p.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
owner, chat = owner_and_chat()
present = 0
results = []
steps = sorted(args.steps)
variant_step = steps[min(1, len(steps) - 1)]
try:
for target in sorted(args.steps):
started = time.perf_counter()
add(target - present, owner, chat)
present = target
print(f"populated {present} synthetic attachments in {time.perf_counter() - started:.1f} s", flush=True)
results.append({"synthetic": present, **measure("dry-run, page 1000", [], args.repeats)})
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
if present == variant_step: # page-size and checksum variants at one size
for label, extra in (("dry-run, page 100", ["--page-size", "100"]),
("dry-run, --verify-checksums", ["--verify-checksums"])):
results.append({"synthetic": present, **measure(label, extra, args.repeats)})
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
finally:
keys = psql("SELECT object_key FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = "
f"'{chat}'").split()
if keys:
s3_bulk("delete", keys)
psql(f"DELETE FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = '{chat}'")
print(f"removed {len(keys)} synthetic attachments", flush=True)
meta = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
text=True).stdout.strip(),
"object_bytes": len(OBJECT), "reconciler_limits": "cpus 1.0, memory 256m (compose)",
"note": "the bucket/table also contain the attachments of earlier test runs (counted in objects/rows)"}
(out / "cost.json").write_text(json.dumps({"meta": meta, "results": results}, ensure_ascii=False, indent=1),
encoding="utf-8")
return 0
if __name__ == "__main__":
sys.exit(main())
+211
View File
@@ -0,0 +1,211 @@
#!/usr/bin/env python3
"""Level 4 experiment: peak memory of the API replicas under >= 10 simultaneous uploads.
Hypothesis: the memory of one replica is bounded by UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES (bodies
waiting for a slot stay in TCP buffers, not in the process) + one image check at a time
(IMAGE_VERIFY_CONCURRENCY), independent of how many clients upload at once.
Method (repeatable, the stand must be up):
* for every scenario the api replicas are RECREATED with the scenario's settings, so the cgroup
counter memory.peak starts from an idle process (it never decreases during a container's life);
* idle memory.current of each replica is read; then N clients upload at the same moment through
the LB (a barrier releases them together), each a different file;
* during the run memory.current of every replica is sampled every 50 ms inside the container;
afterwards memory.peak (cgroup v2: all processes of the container incl. djpeg) and VmHWM of
PID 1 (the Python server alone) are read;
* the normal configuration is restored at the end.
python bench/upload_memory.py --output evidence/memory
"""
from __future__ import annotations
import argparse
import io
import json
import os
import platform
import statistics
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
import httpx2 as httpx
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
PASSWORD = "Bench-password-Ω-12345"
MiB = 1024 * 1024
def sh(*args: str, env: dict | None = None, timeout: float = 300) -> str:
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True, timeout=timeout,
env={**os.environ, **(env or {})}).stdout
def replicas() -> list[str]:
return sh("docker", "compose", "ps", "-q", "api").split()
def cg(cid: str, name: str) -> int:
return int(sh("docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}").split()[0])
def vmhwm(cid: str) -> int:
for line in sh("docker", "exec", cid, "cat", "/proc/1/status").splitlines():
if line.startswith("VmHWM:"):
return int(line.split()[1]) * 1024
return 0
def recreate(env: dict) -> None:
sh("docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "--wait", "api", env=env, timeout=300)
sh(sys.executable, "scripts/lbctl.py", "wait-up", str(len(replicas())), "--timeout", "90") # LB sees them UP
time.sleep(3)
class Sampler(threading.Thread):
"""memory.current of one container every 50 ms, read by a shell loop inside it."""
def __init__(self, cid: str):
super().__init__(daemon=True)
self.cid = cid
self.values: list[int] = []
self.proc = subprocess.Popen(["docker", "exec", cid, "sh", "-c",
"while :; do cat /sys/fs/cgroup/memory.current; sleep 0.05; done"],
stdout=subprocess.PIPE, text=True)
def run(self) -> None:
for line in self.proc.stdout:
if line.strip().isdigit():
self.values.append(int(line))
def stop(self) -> None:
self.proc.kill()
def actor(client: httpx.Client) -> tuple[str, str]:
name = "mem_" + uuid.uuid4().hex[:10]
client.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
import base64
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
token = client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}).json()["token"]
me = client.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
chat = client.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
json={"title": "mem", "member_ids": [me["id"]]}).json()
return token, chat["id"]
def big_jpeg(width: int, height: int, seed: int) -> bytes:
"""Large progressive JPEG (smooth content keeps it < 10 MiB): the costliest file to check."""
image = Image.linear_gradient("L").resize((width, height)).convert("RGB")
image.putpixel((seed % width, 0), (seed % 255, 0, 0))
out = io.BytesIO()
image.save(out, "JPEG", quality=90, progressive=True, subsampling=0)
return out.getvalue()
def run_scenario(name: str, env: dict, files: list[tuple[bytes, str]]) -> dict:
recreate(env)
ids = replicas()
idle = {cid: cg(cid, "memory.current") for cid in ids}
client = httpx.Client(base_url=BASE, timeout=120)
token, chat_id = actor(client)
barrier = threading.Barrier(len(files))
results: list[tuple[int, float, str]] = [None] * len(files) # type: ignore[list-item]
def upload(i: int, data: bytes, ctype: str) -> None:
c = httpx.Client(base_url=BASE, timeout=120)
barrier.wait()
started = time.perf_counter()
r = c.post(f"/api/v1/chats/{chat_id}/attachments?filename=f{i}", content=data,
headers={"Authorization": "Bearer " + token, "Content-Type": ctype})
results[i] = (r.status_code, time.perf_counter() - started, r.headers.get("x-instance-id", "-"))
c.close()
samplers = [Sampler(cid) for cid in ids]
for s in samplers:
s.start()
time.sleep(0.5)
threads = [threading.Thread(target=upload, args=(i, d, t)) for i, (d, t) in enumerate(files)]
wall = time.perf_counter()
for t in threads:
t.start()
for t in threads:
t.join()
wall = time.perf_counter() - wall
time.sleep(0.5)
for s in samplers:
s.stop()
per_replica = {}
for cid, s in zip(ids, samplers):
hostname = sh("docker", "inspect", "-f", "{{.Config.Hostname}}", cid).strip()
per_replica[hostname] = {
"idle_mib": round(idle[cid] / MiB, 1),
"peak_mib": round(cg(cid, "memory.peak") / MiB, 1),
"sampled_max_mib": round(max(s.values or [0]) / MiB, 1),
"python_vmhwm_mib": round(vmhwm(cid) / MiB, 1),
"uploads_served": sum(1 for r in results if r[2] == hostname),
"limit_mib": round(cg(cid, "memory.max") / MiB, 1),
}
per_replica[hostname]["peak_minus_idle_mib"] = round(per_replica[hostname]["peak_mib"]
- per_replica[hostname]["idle_mib"], 1)
lat = sorted(r[1] for r in results if r[0] == 201)
codes: dict[str, int] = {}
for r in results:
codes[str(r[0])] = codes.get(str(r[0]), 0) + 1
summary = {
"scenario": name, "settings": env, "clients": len(files),
"file_mib_each": round(statistics.mean(len(d) for d, _ in files) / MiB, 2),
"types": sorted({t for _, t in files}), "status_codes": codes, "wall_seconds": round(wall, 2),
"latency_s": {"p50": round(lat[len(lat) // 2], 2), "max": round(lat[-1], 2)} if lat else None,
"replicas": per_replica,
}
print(json.dumps(summary, ensure_ascii=False), flush=True)
return summary
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--clients", type=int, default=16)
p.add_argument("--output", default="evidence/memory")
args = p.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
print("generating files ...", flush=True)
pngs = [(imagegen.png_of_size(int(9.5 * MiB)), "image/png") for _ in range(args.clients)]
jpegs = [(big_jpeg(8000, 5000, i), "image/jpeg") for i in range(args.clients)]
base = {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}
scenarios = [
("png-9.5MiB x16, slots=4 (default)", base, pngs),
("png-9.5MiB x16, slots=16 (no bound)", {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, pngs),
("jpeg-40Mpx-progressive x16, slots=4, verify=1 (default)", base, jpegs),
("jpeg-40Mpx-progressive x16, slots=4, verify=4", {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, jpegs),
]
results = []
try:
for name, env, files in scenarios:
results.append(run_scenario(name, env, files))
finally:
recreate({}) # back to the compose defaults
meta = {
"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
"commit": sh("git", "rev-parse", "--short", "HEAD").strip(),
"api_limits": sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}",
replicas()[0]).strip(),
"jpeg_file_mib": round(len(jpegs[0][0]) / MiB, 2), "png_file_mib": round(len(pngs[0][0]) / MiB, 2),
}
(out / "summary.json").write_text(json.dumps({"meta": meta, "scenarios": results}, ensure_ascii=False, indent=1),
encoding="utf-8")
print(json.dumps(meta, ensure_ascii=False))
return 0
if __name__ == "__main__":
sys.exit(main())
+1
View File
@@ -42,6 +42,7 @@ x-api-env: &api-env
UPLOAD_QUEUE_TIMEOUT_SECONDS: ${UPLOAD_QUEUE_TIMEOUT_SECONDS:-5}
UPLOAD_READ_TIMEOUT_SECONDS: ${UPLOAD_READ_TIMEOUT_SECONDS:-30}
IMAGE_VERIFY_TIMEOUT_SECONDS: ${IMAGE_VERIFY_TIMEOUT_SECONDS:-10}
IMAGE_VERIFY_CONCURRENCY: ${IMAGE_VERIFY_CONCURRENCY:-1}
JPEG_MAX_MEMORY_MB: ${JPEG_MAX_MEMORY_MB:-256}
PENDING_GRACE_SECONDS: ${PENDING_GRACE_SECONDS:-900}
JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-60}
+11 -5
View File
@@ -50,7 +50,14 @@ CREATE TABLE attachments (
-- Janitor: stale intents, oldest first; tiny partial index (only rows of unfinished uploads).
CREATE INDEX attachments_pending_created ON attachments (created_at) WHERE status = 'pending';
-- Which attachments a message references, in the order of attachment_ids.
-- attachment_ids of a message, twice on purpose (both written in the message's transaction):
-- * messages.attachment_ids (ordered array): the history page reads it from the same row, so the
-- keyset range scan of messages stays ONE index scan, without a per-row sub-query;
-- * message_attachments: foreign keys (a referenced attachment can never be deleted) and the
-- reverse lookup "is this attachment used?" (janitor/reconciler) by an index on attachment_id.
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
ALTER TABLE messages ADD CONSTRAINT messages_attachment_ids_len CHECK (cardinality(attachment_ids) <= 10);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages (id),
position smallint NOT NULL,
@@ -115,8 +122,8 @@ BEGIN
IF FOUND THEN
UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond')
WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
VALUES (p_message, p_chat, p_user, p_text, v_created);
INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
VALUES (p_message, p_chat, p_user, p_text, v_created, p_attachments);
INSERT INTO message_attachments (message_id, position, attachment_id)
SELECT p_message, (a.ord - 1)::smallint, a.id
FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord);
@@ -127,8 +134,7 @@ BEGIN
RETURN QUERY
SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at,
ARRAY(SELECT ma.attachment_id FROM message_attachments ma
WHERE ma.message_id = m.id ORDER BY ma.position)
m.attachment_ids
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key;
IF NOT FOUND THEN
+4 -2
View File
@@ -389,7 +389,7 @@ async def upload_attachment(request: Request) -> JSONResponse:
except asyncio.TimeoutError:
raise attachments.Busy("Слишком много одновременных загрузок, повторите позже") from None
try:
upload = await attachments.read_image(reader, settings, declared)
upload = await attachments.read_image(reader, settings, declared, request.app.state.verify_slots)
finally:
slots.release()
except ApiError:
@@ -518,7 +518,8 @@ def create_app(settings: Settings | None = None) -> Starlette:
"db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds,
"instance_id": settings.instance_id, "drain_s": settings.drain_seconds,
"message_write_path": settings.message_write_path, "s3_bucket": settings.s3_bucket,
"upload_concurrency": settings.upload_concurrency, "pending_grace_s": settings.pending_grace_seconds,
"upload_concurrency": settings.upload_concurrency, "verify_concurrency": settings.image_verify_concurrency,
"pending_grace_s": settings.pending_grace_seconds,
"fault_points": sorted(settings.fault_points)}})
tasks = [asyncio.create_task(purge_loop())]
if settings.janitor_interval_seconds > 0:
@@ -567,5 +568,6 @@ def create_app(settings: Settings | None = None) -> Starlette:
app.state.sessions = sessions
app.state.storage = storage
app.state.upload_slots = asyncio.Semaphore(settings.upload_concurrency)
app.state.verify_slots = asyncio.Semaphore(settings.image_verify_concurrency)
app.state.draining = False
return app
+4 -3
View File
@@ -227,7 +227,7 @@ def declared_length(request: Request, limit: int) -> int | None:
return length
async def read_image(reader: BodyReader, settings: Settings, declared: str) -> Upload:
async def read_image(reader: BodyReader, settings: Settings, declared: str, verify_slots: asyncio.Semaphore) -> Upload:
"""Body -> verified image. Memory: at most MAX_UPLOAD_BYTES per upload (bytearray); the format
and the pixel count are known after the first bytes, so a picture that is too big or not a
PNG/JPEG is rejected without reading/decoding the rest."""
@@ -260,8 +260,9 @@ async def read_image(reader: BodyReader, settings: Settings, declared: str) -> U
data = bytes(buffer)
del buffer
try:
await images.verify(data, info, timeout=settings.image_verify_timeout_seconds,
jpeg_max_memory_mb=settings.jpeg_max_memory_mb)
async with verify_slots:
await images.verify(data, info, timeout=settings.image_verify_timeout_seconds,
jpeg_max_memory_mb=settings.jpeg_max_memory_mb)
except images.ImageError as exc:
raise _image_error(exc) from None
except asyncio.TimeoutError:
+4
View File
@@ -130,6 +130,7 @@ class Settings:
upload_queue_timeout_seconds: float = 5.0
upload_read_timeout_seconds: float = 30.0
image_verify_timeout_seconds: float = 10.0
image_verify_concurrency: int = 1
jpeg_max_memory_mb: int = 256
pending_grace_seconds: int = 900
janitor_interval_seconds: float = 60.0
@@ -223,6 +224,9 @@ def s3_settings(*, credentials_required: bool = True) -> dict:
upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0, 0.0, 60.0),
upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 30.0, 1.0, 300.0),
image_verify_timeout_seconds=_float("IMAGE_VERIFY_TIMEOUT_SECONDS", 10.0, 1.0, 120.0),
# Full checks running at once per replica: a strict JPEG decode of a 40 Mpx progressive
# file needs up to ~120-240 MB of coefficient buffers (capped by JPEG_MAX_MEMORY_MB).
image_verify_concurrency=_int("IMAGE_VERIFY_CONCURRENCY", 1, 1, 16),
jpeg_max_memory_mb=_int("JPEG_MAX_MEMORY_MB", 256, 16, 2048),
# An intent row still `pending` after this long belongs to a failed upload: the janitor
# deletes its object and the row. Far above the longest possible upload (~45 s).
+430
View File
@@ -0,0 +1,430 @@
"""Reconciler of the two stores (lab 4, track 2): S3 objects vs attachment rows in PostgreSQL.
docker compose run --rm reconciler # dry-run: report only (default)
docker compose run --rm reconciler --apply # fix what is safe to fix
docker compose run --rm reconciler --verify-checksums # also read every object (SHA-256)
How it walks: both sides are read as key-ordered streams and merge-joined - ListObjectsV2
pages (prefix o/, byte order) against keyset pages of `attachments ORDER BY object_key`
(COLLATE "C" = the same byte order, unique index). Memory does not depend on the bucket size;
cost is ~N/1000 S3 list requests + N/page DB pages (measured: docs/CONSISTENCY.md).
What it finds and what --apply does (nothing else is ever changed):
kind condition --apply
orphan_object object, no row, older than --grace delete the object
stale_intent row 'pending' older than --grace (+/- object) delete object, then row
missing_object row 'ready', object absent, older than --grace quarantine the row
size_mismatch row 'ready', object size != size_bytes quarantine the row
checksum_mismatch row 'ready', SHA-256 differs (--verify-checksums) quarantine the row
restored_object row quarantined, object present and correct again remove the quarantine
fresh_* the same, but younger than --grace nothing (in-flight upload)
Safety:
* dry-run is read-only BY CONSTRUCTION: the PostgreSQL session runs with
default_transaction_read_only=on and the S3 wrapper has no write methods, so even a bug in
the planner cannot modify a store;
* ready rows are never deleted - a used (or unused) ready attachment can only be quarantined,
which is reversible; objects that have a row are never deleted unless the row is a stale
intent; anything younger than --grace is never touched;
* every action re-checks its condition right before acting, under a row lock (FOR UPDATE)
where a row exists, so the live API racing with the reconciler cannot be hurt;
* object first, row second: an interrupted run leaves a state the next run repairs;
re-running is idempotent (the second run finds nothing to do);
* safety valve: --apply refuses to delete more than --max-orphan-fraction of the scanned
objects (pointing it at the wrong/empty database would otherwise wipe the bucket).
"""
from __future__ import annotations
import argparse
import asyncio
import datetime as dt
import hashlib
import json
import sys
import time
from collections import Counter
from collections.abc import AsyncIterator
from dataclasses import dataclass, field
import psycopg
from .config import ConfigError, Settings, _float, _secret, s3_settings
from .errors import Unavailable
from .storage import OBJECT_PREFIX, ListedObject, ObjectMissing, Storage
KINDS = ("orphan_object", "stale_intent", "missing_object", "size_mismatch", "checksum_mismatch",
"restored_object", "fresh_orphan_object", "fresh_intent", "fresh_missing_object")
MIN_SAFE_GRACE = 60.0 # > the longest possible upload (queue 5 s + body 30 s + verify 10 s + S3)
ACTION_OF = {"orphan_object": "delete_object", "stale_intent": "delete_intent", "missing_object": "quarantine",
"size_mismatch": "quarantine", "checksum_mismatch": "quarantine", "restored_object": "unquarantine"}
def log(text: str) -> None:
print(f"reconcile: {text}", file=sys.stderr, flush=True)
@dataclass(frozen=True, slots=True)
class Row:
id: str
key: str
status: str
size: int
sha256: bytes
created_at: dt.datetime
updated_at: dt.datetime
lost_reason: str | None
used: bool
@dataclass(slots=True)
class Finding:
kind: str
key: str
attachment_id: str | None = None
used: bool = False
detail: str = ""
result: str = "planned" # planned | applied | skipped:<why> | failed:<why>
@dataclass
class Stats:
objects: int = 0
rows: int = 0
consistent: int = 0
s3_list_requests: int = 0
s3_head_requests: int = 0
s3_get_bytes: int = 0
db_pages: int = 0
class ReadOnlyStorage:
"""The dry-run view of S3: listing and reading only - there is no delete to call."""
def __init__(self, storage: Storage):
self._s = storage
async def list_page(self, *args, **kwargs):
return await self._s.list_page(*args, **kwargs)
async def head(self, key: str):
return await self._s.head(key)
async def read_all(self, key: str, limit: int) -> bytes:
return await self._s.read_all(key, limit)
class Reconciler:
def __init__(self, conn: psycopg.AsyncConnection, storage: Storage | ReadOnlyStorage, *, grace: float,
page_size: int, verify: bool, apply: bool):
self.conn = conn
self.s3 = storage
self.grace = grace
self.page_size = page_size
self.verify = verify
self.apply = apply
self.stats = Stats()
self.findings: list[Finding] = []
# ---------------------------------------------------------- streams
async def objects(self) -> AsyncIterator[ListedObject]:
after = None
while True:
page, more = await self.s3.list_page(OBJECT_PREFIX, after, self.page_size)
self.stats.s3_list_requests += 1
for item in page:
self.stats.objects += 1
yield item
if not more or not page:
return
after = page[-1].key
async def rows(self) -> AsyncIterator[Row]:
after = ""
while True:
cur = await self.conn.execute(
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256, a.created_at, a.updated_at, "
"a.lost_reason, EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) "
"FROM attachments a WHERE a.object_key > %s ORDER BY a.object_key LIMIT %s",
(after, self.page_size))
page = await cur.fetchall()
self.stats.db_pages += 1
for r in page:
self.stats.rows += 1
yield Row(str(r[0]), r[1], r[2], r[3], bytes(r[4]), r[5], r[6], r[7], r[8])
if len(page) < self.page_size:
return
after = page[-1][1]
# ---------------------------------------------------------- planning (read-only)
async def scan(self) -> None:
cur = await self.conn.execute("SELECT clock_timestamp()")
self.db_now = (await cur.fetchone())[0]
self.s3_now = time.time()
objects, rows = self.objects(), self.rows()
obj = await anext(objects, None)
row = await anext(rows, None)
while obj is not None or row is not None:
if row is None or (obj is not None and obj.key < row.key):
self.object_only(obj)
obj = await anext(objects, None)
elif obj is None or row.key < obj.key:
self.row_only(row)
row = await anext(rows, None)
else:
await self.both(obj, row)
obj = await anext(objects, None)
row = await anext(rows, None)
def row_old(self, row: Row) -> bool:
return (self.db_now - max(row.created_at, row.updated_at)).total_seconds() > self.grace
def object_only(self, obj: ListedObject) -> None:
old = self.s3_now - obj.last_modified_ts > self.grace
self.add(Finding("orphan_object" if old else "fresh_orphan_object", obj.key))
def row_only(self, row: Row) -> None:
if row.status == "pending":
self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id,
detail="object absent"))
elif row.lost_reason is not None:
self.stats.consistent += 1 # already quarantined, still missing: nothing new
else:
self.add(Finding("missing_object" if self.row_old(row) else "fresh_missing_object", row.key, row.id,
used=row.used))
async def both(self, obj: ListedObject, row: Row) -> None:
if row.status == "pending":
self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id,
detail="object present"))
return
problem = await self.object_problem(row, obj.size)
if problem:
if row.lost_reason is None:
self.add(Finding(problem, row.key, row.id, used=row.used))
else:
self.stats.consistent += 1 # quarantined and still broken
elif row.lost_reason is not None:
self.add(Finding("restored_object", row.key, row.id, used=row.used))
else:
self.stats.consistent += 1
async def object_problem(self, row: Row, size: int) -> str | None:
if size != row.size:
return "size_mismatch"
if self.verify:
data = await self.s3.read_all(row.key, row.size)
self.stats.s3_get_bytes += len(data)
if hashlib.sha256(data).digest() != row.sha256:
return "checksum_mismatch"
return None
def add(self, finding: Finding) -> None:
if finding.kind not in ACTION_OF:
finding.result = "not_actionable"
self.findings.append(finding)
# ---------------------------------------------------------- actions (--apply only)
async def run_actions(self, max_orphan_fraction: float) -> None:
orphans = sum(1 for f in self.findings if f.kind == "orphan_object")
if self.stats.objects and orphans > max_orphan_fraction * self.stats.objects:
raise SafetyValve(f"{orphans} of {self.stats.objects} objects look orphaned "
f"(> {max_orphan_fraction:.0%}); wrong database? nothing was changed")
handlers = {"delete_object": self.delete_object, "delete_intent": self.delete_intent,
"quarantine": self.quarantine, "unquarantine": self.unquarantine}
for finding in self.findings:
action = ACTION_OF.get(finding.kind)
if action is None:
continue
try:
finding.result = await handlers[action](finding)
except (Unavailable, psycopg.OperationalError) as exc:
finding.result = f"failed:{type(exc).__name__}"
raise # a store is down: stop; the next run continues from the committed state
async def delete_object(self, f: Finding) -> str:
cur = await self.conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (f.key,))
if await cur.fetchone():
return "skipped:row_appeared"
try:
head = await self.s3.head(f.key)
self.stats.s3_head_requests += 1
except ObjectMissing:
return "skipped:already_gone"
if time.time() - head.last_modified_ts <= self.grace:
return "skipped:object_rewritten_recently"
# No row names this key and the object is older than the grace period. Rows are always
# written BEFORE their object and keys are random and never reused, so no upload can
# still claim it.
await self.s3.delete(f.key)
return "applied"
async def delete_intent(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT 1 FROM attachments WHERE id = %s AND status = 'pending' "
"AND greatest(created_at, updated_at) < clock_timestamp() - make_interval(secs => %s) FOR UPDATE",
(f.attachment_id, self.grace))
if await cur.fetchone() is None:
return "skipped:no_longer_stale_intent"
await self.s3.delete(f.key) # object first (idempotent), then the row
await self.conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'pending'",
(f.attachment_id,))
return "applied"
async def quarantine(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT size_bytes, sha256 FROM attachments WHERE id = %s AND status = 'ready' "
"AND lost_at IS NULL FOR UPDATE", (f.attachment_id,))
row = await cur.fetchone()
if row is None:
return "skipped:row_changed"
reason = await self.current_problem(f.key, row[0], bytes(row[1]))
if reason is None:
return "skipped:object_ok_now"
await self.conn.execute(
"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = %s WHERE id = %s",
(reason, f.attachment_id))
return "applied"
async def unquarantine(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT size_bytes, sha256 FROM attachments WHERE id = %s AND lost_at IS NOT NULL FOR UPDATE",
(f.attachment_id,))
row = await cur.fetchone()
if row is None:
return "skipped:row_changed"
if await self.current_problem(f.key, row[0], bytes(row[1])) is not None:
return "skipped:object_still_bad"
await self.conn.execute("UPDATE attachments SET lost_at = NULL, lost_reason = NULL WHERE id = %s",
(f.attachment_id,))
return "applied"
async def current_problem(self, key: str, size: int, sha256: bytes) -> str | None:
"""Fresh look at the object right before changing the row (the listing may be stale)."""
try:
head = await self.s3.head(key)
self.stats.s3_head_requests += 1
except ObjectMissing:
return "missing_object"
if head.size != size:
return "size_mismatch"
if self.verify:
data = await self.s3.read_all(key, size)
self.stats.s3_get_bytes += len(data)
if hashlib.sha256(data).digest() != sha256:
return "checksum_mismatch"
return None
# ---------------------------------------------------------- report
def report(self, mode: str, seconds: float, error: str | None, sample: int) -> dict:
kinds = Counter(f.kind for f in self.findings)
results = Counter(f.result.split(":")[0] for f in self.findings if f.kind in ACTION_OF)
samples = {}
for f in self.findings:
items = samples.setdefault(f.kind, [])
if len(items) < sample:
items.append({"key": f.key, "attachment_id": f.attachment_id, "used": f.used,
"result": f.result, **({"detail": f.detail} if f.detail else {})})
return {
"mode": mode, "grace_seconds": self.grace, "verify_checksums": self.verify,
"duration_seconds": round(seconds, 3), "error": error,
"scanned": {"objects": self.stats.objects, "rows": self.stats.rows, "consistent": self.stats.consistent},
"findings": {k: kinds.get(k, 0) for k in KINDS},
"used_attachments_in_findings": sum(1 for f in self.findings if f.used),
"actions": {"planned": sum(1 for f in self.findings if f.kind in ACTION_OF),
"applied": results.get("applied", 0), "skipped": results.get("skipped", 0),
"failed": results.get("failed", 0)},
"cost": {"s3_list_requests": self.stats.s3_list_requests, "s3_head_requests": self.stats.s3_head_requests,
"s3_get_bytes": self.stats.s3_get_bytes, "db_pages": self.stats.db_pages,
"page_size": self.page_size},
"samples": samples,
}
class SafetyValve(Exception):
pass
def arguments(argv: list[str] | None = None) -> argparse.Namespace:
p = argparse.ArgumentParser(prog="python -m messenger.reconcile", description=__doc__.split("\n")[0])
p.add_argument("--apply", action="store_true", help="fix findings (default: dry-run, read-only)")
p.add_argument("--grace", type=float, default=3600.0,
help="seconds; anything younger is never touched (default 3600, >= PENDING_GRACE_SECONDS)")
p.add_argument("--allow-small-grace", action="store_true",
help="permit --grace below 60 s (tests only: a grace shorter than an upload could remove the "
"intent of an upload that is still running; that upload then fails with 503)")
p.add_argument("--page-size", type=int, default=1000, help="S3 list / DB page size (1..1000)")
p.add_argument("--verify-checksums", action="store_true", help="read every ready object and compare SHA-256")
p.add_argument("--samples", type=int, default=20, help="keys listed per finding kind in the report")
p.add_argument("--max-orphan-fraction", type=float, default=0.5,
help="--apply refuses if more than this share of objects look orphaned (default 0.5)")
args = p.parse_args(argv)
if not 1 <= args.page_size <= 1000:
p.error("--page-size must be 1..1000")
if args.grace < 1 or (args.grace < MIN_SAFE_GRACE and not args.allow_small_grace):
p.error(f"--grace must be >= {MIN_SAFE_GRACE:.0f} s (or >= 1 s with --allow-small-grace)")
if not 0 < args.max_orphan_fraction <= 1:
p.error("--max-orphan-fraction must be in (0, 1]")
return args
async def run(args: argparse.Namespace) -> int:
url = _secret("DATABASE_URL", required=True)
base = s3_settings()
settings = Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024,
database_url=url, **base)
timeout_ms = int(_float("DB_TIMEOUT_SECONDS", 10.0, 1.0, 120.0) * 1000)
options = f"-c statement_timeout={timeout_ms} -c lock_timeout={timeout_ms // 2}"
if not args.apply:
options += " -c default_transaction_read_only=on" # dry-run: PostgreSQL itself refuses writes
mode = "apply" if args.apply else "dry-run"
storage = Storage(settings)
await storage.open()
started = time.perf_counter()
error = None
code = 0
try:
conn = await psycopg.AsyncConnection.connect(url, autocommit=True, connect_timeout=5, options=options,
application_name=f"messenger-reconcile-{mode}")
except psycopg.OperationalError as exc:
await storage.close()
error = f"store_unavailable: database {type(exc).__name__}"
print(json.dumps({"mode": mode, "error": error}, ensure_ascii=False))
log(error)
return 1
rec = Reconciler(conn, storage if args.apply else ReadOnlyStorage(storage), grace=args.grace,
page_size=args.page_size, verify=args.verify_checksums, apply=args.apply)
try:
log(f"{mode}: scanning bucket {settings.s3_bucket} (prefix {OBJECT_PREFIX}) against table attachments")
await rec.scan()
log(f"scan done: {rec.stats.objects} objects, {rec.stats.rows} rows, {len(rec.findings)} findings")
if args.apply:
await rec.run_actions(args.max_orphan_fraction)
except SafetyValve as exc:
error, code = f"safety_valve: {exc}", 3
except (Unavailable, psycopg.OperationalError) as exc:
error, code = f"store_unavailable: {type(exc).__name__}", 1
finally:
await conn.close()
await storage.close()
print(json.dumps(rec.report(mode, time.perf_counter() - started, error, args.samples), ensure_ascii=False,
indent=1))
if error:
log(error)
return code
def main(argv: list[str] | None = None) -> int:
args = arguments(argv)
try:
return asyncio.run(run(args))
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())
+11 -13
View File
@@ -219,8 +219,9 @@ async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID,
(chat_id,))
(created_at,) = await cur.fetchone()
await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)",
(message_id, chat_id, user_id, _b(text), created_at))
"INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) "
"VALUES (%s, %s, %s, %s, %s, %s::uuid[])",
(message_id, chat_id, user_id, _b(text), created_at, attachment_ids))
if attachment_ids:
await conn.execute(
"INSERT INTO message_attachments (message_id, position, attachment_id) "
@@ -266,9 +267,7 @@ CLAIM_KEY = (
"WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) "
"RETURNING 1")
STORED_RESULT = (
"SELECT k.request_hash, m.id, m.text, m.created_at, "
"ARRAY(SELECT ma.attachment_id FROM message_attachments ma WHERE ma.message_id = m.id ORDER BY ma.position) "
"FROM idempotency_keys k "
"SELECT k.request_hash, m.id, m.text, m.created_at, m.attachment_ids FROM idempotency_keys k "
"JOIN messages m ON m.id = k.message_id "
"WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s")
@@ -351,14 +350,13 @@ async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 50
# Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR
# would push the cursor into a Filter and late pages would scan the whole prefix.
# attachment_ids: a correlated sub-select per returned row (at most limit + 1 = 101 lookups on the
# primary key of message_attachments), so the keyset range scan itself stays unchanged.
HISTORY_COLUMNS = ("SELECT m.id, m.sender_id, m.text, m.created_at, "
"ARRAY(SELECT ma.attachment_id FROM message_attachments ma "
"WHERE ma.message_id = m.id ORDER BY ma.position) FROM messages m ")
HISTORY_FIRST = HISTORY_COLUMNS + "WHERE m.chat_id = %s ORDER BY m.created_at, m.id LIMIT %s"
HISTORY_AFTER = (HISTORY_COLUMNS +
"WHERE m.chat_id = %s AND (m.created_at, m.id) > (%s, %s) ORDER BY m.created_at, m.id LIMIT %s")
# attachment_ids is a column of the message row (migration 0004): still one index range scan.
HISTORY_FIRST = (
"SELECT id, sender_id, text, created_at, attachment_ids FROM messages "
"WHERE chat_id = %s ORDER BY created_at, id LIMIT %s")
HISTORY_AFTER = (
"SELECT id, sender_id, text, created_at, attachment_ids FROM messages "
"WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s")
async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit: int) -> tuple[list[Message], bool]:
+2 -1
View File
@@ -44,6 +44,7 @@ class ObjectHead:
size: int
content_type: str
metadata: dict[str, str]
last_modified_ts: float # POSIX seconds
@dataclass(frozen=True, slots=True)
@@ -116,7 +117,7 @@ class Storage:
async def head(self, key: str) -> ObjectHead:
response = await self._call("head", self.client.head_object(Bucket=self.bucket, Key=key))
return ObjectHead(int(response["ContentLength"]), response.get("ContentType", ""),
dict(response.get("Metadata", {})))
dict(response.get("Metadata", {})), response["LastModified"].timestamp())
async def open_stream(self, key: str) -> tuple[int, AsyncIterator[bytes]]:
"""Start a GET: (Content-Length, async iterator of chunks). The headers arrive under the
+91 -2
View File
@@ -21,6 +21,7 @@ import sys
import time
import uuid
from pathlib import Path
from urllib.parse import quote
import httpx2 as httpx
import pytest
@@ -53,9 +54,9 @@ def basic(username: str, password: str) -> str:
# ------------------------------------------------------------------ docker / psql
def compose(*args: str, check: bool = True, timeout: float = 120) -> subprocess.CompletedProcess:
def compose(*args: str, check: bool = True, timeout: float = 120, stdin: str | None = None) -> subprocess.CompletedProcess:
return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", check=check, timeout=timeout)
encoding="utf-8", check=check, timeout=timeout, input=stdin)
def psql(sql: str) -> str:
@@ -234,6 +235,24 @@ class Api:
def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict:
return self.post_message(actor, chat, {"text": text}, key=key).json()
def upload(self, actor: dict, chat: dict, data, expected=201, *, content_type: str | None = "image/png",
filename: str | None = "photo.png", query: str | None = None, headers: dict | None = None):
"""POST raw bytes to /chats/{id}/attachments. `query` replaces the whole query string
(to send hand-made percent-encoding); `data` may be bytes or an iterator (chunked)."""
h = dict(headers or {})
if content_type is not None:
h["Content-Type"] = content_type
if query is None:
query = "" if filename is None else "filename=" + quote(filename, safe="")
path = f"/api/v1/chats/{chat['id']}/attachments" + (f"?{query}" if query else "")
if isinstance(data, (bytes, bytearray)):
return self.call("POST", path, expected, token=actor["token"], raw=bytes(data), headers=h,
schema="Attachment" if expected == 201 else None)
h["Authorization"] = "Bearer " + actor["token"]
response = self.client.post(path, content=data, headers=h)
assert response.status_code == expected, (response.status_code, response.text[:300])
return response
def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]:
"""Walk all pages; returns (items, number_of_pages)."""
items, pages, cursor = [], 0, None
@@ -281,6 +300,76 @@ def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.Complete
timeout=timeout, env={**os.environ, **env})
def s3_exec(body: str, service: str = "api", stdin: str | None = None) -> str:
"""Run `body` (async Python, `s` = messenger.storage.Storage) INSIDE a container that holds S3
credentials: `api` (Read+Write, the API's own key pair) or `reconciler` (+List). The S3 port is
not published on the host; this is how the tests inject faults directly into the bucket."""
script = ("import asyncio, base64, json, sys\n"
"from messenger.config import Settings, s3_settings\n"
"from messenger.storage import Storage, ObjectMissing\n"
"async def main():\n"
" st = Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings())\n"
" s = Storage(st)\n"
" await s.open()\n"
" try:\n" + "".join(" " + line + "\n" for line in body.splitlines()) +
" finally:\n"
" await s.close()\n"
"asyncio.run(main())\n")
if service == "api":
cmd = ["exec", "-T", "api", "python", "-c", script]
else:
cmd = ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service, "-c", script]
return compose(*cmd, timeout=120, stdin=stdin).stdout.strip()
def s3_put(key: str, data: bytes) -> None:
"""Write raw bytes under `key` (base64 over stdin: Windows limits command lines to 32 KiB)."""
s3_exec(f"await s.put({key!r}, base64.b64decode(sys.stdin.read()), 'image/png', {{}})",
stdin=base64.b64encode(data).decode())
def s3_delete(key: str) -> None:
s3_exec(f"await s.delete({key!r})")
def s3_size(key: str) -> int | None:
out = s3_exec(f"try:\n print((await s.head({key!r})).size)\nexcept ObjectMissing:\n print(-1)")
size = int(out.splitlines()[-1])
return None if size < 0 else size
def s3_keys() -> set[str]:
"""Every object key under o/ (needs List: runs with the reconciler's key pair)."""
out = s3_exec("keys, after = [], None\n"
"while True:\n"
" page, more = await s.list_page('o/', after, 1000)\n"
" keys += [o.key for o in page]\n"
" if not more or not page:\n"
" break\n"
" after = page[-1].key\n"
"print(json.dumps(keys))", service="reconciler")
return set(json.loads(out.splitlines()[-1]))
def reconcile(*args: str, expect_code: int = 0) -> dict:
"""Run the reconciler CLI (compose service, profile tools) and return its JSON report."""
# --no-deps: never let `compose run` start a store that a test has stopped on purpose.
result = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "100000", *args, check=False,
timeout=300)
assert result.returncode == expect_code, (result.returncode, result.stderr[-2000:])
return json.loads(result.stdout)
def findings_by_key(report: dict) -> dict[str, dict]:
return {item["key"]: dict(item, kind=kind) for kind, items in report["samples"].items() for item in items}
def attachment_row(attachment_id: str) -> dict | None:
out = psql("SELECT json_build_object('status', status, 'object_key', object_key, 'size', size_bytes, "
"'lost_reason', lost_reason) FROM attachments WHERE id = '%s'" % attachment_id)
return json.loads(out) if out else None
def probe_inside(replica: dict, path: str) -> int:
"""HTTP status of `path` asked inside the replica container itself (bypassing the LB)."""
code = ("import urllib.request as u, urllib.error as e\n"
+93
View File
@@ -0,0 +1,93 @@
"""Synthetic images for the lab 4 tests: valid PNG/JPEG of any size, and the broken files the
reviewer used against other solutions (truncated, corrupted IDAT with a valid CRC, JPEG without
scan data, decompression-bomb headers, MIME spoofing). Nothing big is stored in Git: every file
is generated in memory."""
from __future__ import annotations
import io
import os
import struct
import zlib
from PIL import Image
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
def chunk(ctype: bytes, data: bytes) -> bytes:
return struct.pack(">I", len(data)) + ctype + data + struct.pack(">I", zlib.crc32(ctype + data))
def ihdr(width: int, height: int, depth: int = 8, color: int = 2, interlace: int = 0) -> bytes:
return chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace))
def png(width: int = 64, height: int = 48, mode: str = "RGB", noise: bool = False, **save) -> bytes:
if noise:
channels = len(mode)
image = Image.frombytes(mode, (width, height), os.urandom(width * height * channels))
else:
image = Image.new(mode, (width, height), (40, 120, 200) if mode == "RGB" else 128)
out = io.BytesIO()
image.save(out, "PNG", **save)
return out.getvalue()
def png_of_size(target: int) -> bytes:
"""A valid PNG of (about, at most) `target` bytes: random pixels do not compress."""
side = int((target / 3) ** 0.5)
data = png(side, side, noise=True, compress_level=1)
while len(data) > target:
side -= 8
data = png(side, side, noise=True, compress_level=1)
return data
def png_exactly(size: int) -> bytes:
"""A valid PNG of EXACTLY `size` bytes: noise image + an ancillary padding chunk."""
base = png_of_size(size - 64)
pad = size - len(base) - 12
assert pad >= 0
return base[:-12] + chunk(b"tEXt", b"pad\x00" + b"x" * (pad - 4)) + base[-12:]
def jpeg(width: int = 64, height: int = 48, progressive: bool = False, noise: bool = False, quality: int = 85) -> bytes:
if noise:
image = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3))
else:
image = Image.new("RGB", (width, height), (200, 30, 30))
out = io.BytesIO()
image.save(out, "JPEG", progressive=progressive, quality=quality)
return out.getvalue()
# ------------------------------------------------------------------ broken files
def png_corrupt_idat() -> bytes:
"""Valid signature, IHDR and CRCs; the IDAT payload is 86 bytes of garbage (reviewer's case)."""
return PNG_SIGNATURE + ihdr(20, 20) + chunk(b"IDAT", bytes(range(86))) + chunk(b"IEND", b"")
def png_short_pixels() -> bytes:
"""A complete, CRC-correct PNG whose zlib stream holds fewer rows than the header promises."""
raw = b"".join(b"\x00" + bytes(60) for _ in range(19))
return PNG_SIGNATURE + ihdr(20, 20) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")
def png_bomb_header(width: int = 20000, height: int = 10000) -> bytes:
"""Header of a huge picture: must be refused (413) from the first 33 bytes."""
raw_row = b"\x00" + bytes(width * 3)
body = zlib.compress(raw_row * 4, 9) # a few rows only: the decoder must never get that far
return PNG_SIGNATURE + ihdr(width, height) + chunk(b"IDAT", body) + chunk(b"IEND", b"")
def jpeg_without_scan_data() -> bytes:
data = jpeg(128, 96)
sos = data.index(b"\xff\xda")
length = struct.unpack(">H", data[sos + 2:sos + 4])[0]
return data[:sos + 2 + length] + b"\xff\xd9"
def jpeg_bomb_header() -> bytes:
"""SOF0 announcing 20000 x 20000 = 400 Mpx."""
return (b"\xff\xd8\xff\xc0\x00\x11\x08" + struct.pack(">HH", 20000, 20000)
+ b"\x03\x01\x22\x00\x02\x11\x01\x03\x11\x01" + b"\xff\xd9")
+268
View File
@@ -0,0 +1,268 @@
"""Attachments through the load balancer (lab 4, levels 3 and 4): upload/download, ACL, limits,
content checks, filename rules and messages with attachment_ids.
Negative inputs include every case the reviewer used against other lab 4 solutions: corrupted
IDAT with valid CRCs, the first 33 bytes of the fixture, JPEG without scan data, a 20000x10000
header (DecompressionBomb), exactly 10 MiB through the LB, C1 control U+0085 and a whitespace
filename, text=null / missing text / attachment_ids=null in a message."""
from __future__ import annotations
import hashlib
import subprocess
import time
import uuid
from pathlib import Path
import pytest
import imagegen
from conftest import attachment_row, compose, s3_size
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
MiB = 1024 * 1024
@pytest.fixture()
def trio(api):
"""alice + bob share a chat; eve is an outsider with her own chat."""
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
return alice, bob, eve, api.chat(alice, bob), api.chat(eve)
def content(api, actor, attachment_id, expected=200, query=""):
return api.call("GET", f"/api/v1/attachments/{attachment_id}/content{query}", expected, token=actor["token"])
# ------------------------------------------------------------------ positive path
def test_png_and_jpeg_roundtrip_byte_for_byte(api, trio):
alice, bob, _, chat, _ = trio
jpg = imagegen.jpeg(321, 123, noise=True)
for data, ctype in ((SAMPLE, "image/png"), (jpg, "image/jpeg")):
att = api.upload(alice, chat, data, content_type=ctype, filename="кот 😺.bin").json()
assert att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None
assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"]
assert att["size_bytes"] == len(data) and att["sha256"] == hashlib.sha256(data).hexdigest()
assert att["content_type"] == ctype and att["filename"] == "кот 😺.bin"
meta = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"], schema="Attachment").json()
assert meta == att
got = content(api, bob, att["id"])
assert got.content == data and got.headers["content-type"] == ctype
assert got.headers["content-length"] == str(len(data))
assert got.headers["x-content-type-options"] == "nosniff"
assert got.headers["content-disposition"].startswith("attachment;")
assert "location" not in got.headers # bytes through the API, never a redirect to S3
def test_metadata_reveals_no_storage_details(api, trio):
alice, _, _, chat, _ = trio
att = api.upload(alice, chat, SAMPLE).json()
row = attachment_row(att["id"])
assert row["status"] == "ready" and row["object_key"].startswith("o/")
text = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=alice["token"]).text
assert row["object_key"] not in text and "messenger-attachments" not in text and "s3" not in text.lower()
# The key is random: neither the id nor the filename is part of it.
assert att["id"].replace("-", "") not in row["object_key"] and "photo" not in row["object_key"]
assert s3_size(row["object_key"]) == len(SAMPLE) # the bytes ARE in the bucket
def test_original_is_not_stored_on_api_containers(api, trio):
"""The API keeps no file of the original: no recently written file of that size anywhere in
the replica's filesystem (the bytes live only in the bucket)."""
alice, _, _, chat, _ = trio
data = imagegen.png_of_size(300_000)
api.upload(alice, chat, data)
size = len(data)
for cid in compose("ps", "-q", "api").stdout.split():
out = subprocess.run(
["docker", "exec", cid, "sh", "-c", f"find / -xdev -type f -size {size}c -mmin -10 2>/dev/null | head"],
capture_output=True, text=True, timeout=60).stdout.strip()
assert out == "", out
# ------------------------------------------------------------------ access control
def test_outsider_and_unknown_get_404(api, trio):
alice, _, eve, chat, _ = trio
att = api.upload(alice, chat, SAMPLE).json()
for suffix in ("", "/content"):
api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"])
api.call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"])
api.call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"])
api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401)
# Upload into a foreign chat: 404 before anything about the body is looked at.
api.upload(eve, chat, SAMPLE, 404)
api.upload(eve, chat, b"garbage", 404, content_type="text/plain")
def test_attachment_ids_are_checked_per_chat(api, trio):
alice, bob, eve, chat, eve_chat = trio
other = api.chat(alice) # alice's second chat
att_other = api.upload(alice, other, SAMPLE).json()
att_eve = api.upload(eve, eve_chat, SAMPLE).json()
# Alice is a member of both chats, but a file is bound to its chat.
api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_other["id"]]}, 404)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404)
# Eve cannot post into alice's chat even with her own file: membership first.
api.post_message(eve, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404)
# Any member may reference an attachment of the chat.
att = api.upload(alice, chat, SAMPLE).json()
msg = api.post_message(bob, chat, {"text": "", "attachment_ids": [att["id"]]}).json()
assert msg["attachment_ids"] == [att["id"]] and msg["sender_id"] == bob["id"]
# ------------------------------------------------------------------ messages with attachments
def test_message_with_attachments_and_history(api, trio):
alice, bob, _, chat, _ = trio
a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2))
m1 = api.post_message(alice, chat, {"text": "", "attachment_ids": [a2, a1]}).json()
m2 = api.post_message(alice, chat, {"text": " ", "attachment_ids": [a1]}).json() # blank text + file: ok
m3 = api.post_message(alice, chat, {"text": "без файлов"}).json()
assert m1["attachment_ids"] == [a2, a1] and m1["text"] == ""
assert m2["text"] == " " and m3["attachment_ids"] == []
items, _ = api.history(bob, chat, 2)
assert [m["attachment_ids"] for m in items] == [[a2, a1], [a1], []]
def test_idempotent_replay_with_attachments(api, trio):
alice, _, _, chat, _ = trio
a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2))
key = uuid.uuid4().hex
first = api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a1, a2]}, key=key).json()
raw = ('{"attachment_ids":["%s","%s"],"text":"файлы"}' % (a1, a2)).encode()
assert api.post_message(alice, chat, None, raw=raw, key=key).json() == first # key order irrelevant
api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a2, a1]}, 409, key=key) # other body
api.post_message(alice, chat, {"text": "файлы"}, 409, key=key)
# A failed attachment check does not burn the key: the same key works with a valid body later.
key2 = uuid.uuid4().hex
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404, key=key2)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [a1]}, key=key2)
@pytest.mark.parametrize("raw", [
b'{"attachment_ids":["%(a)s"]}', # text missing
b'{"text":null,"attachment_ids":["%(a)s"]}', # text null
b'{"text":1,"attachment_ids":["%(a)s"]}',
b'{"text":"x","attachment_ids":null}', # attachment_ids null
b'{"text":"x","attachment_ids":"%(a)s"}',
b'{"text":"x","attachment_ids":{}}',
b'{"text":"x","attachment_ids":["%(a)s","%(a)s"]}', # duplicates
b'{"text":"x","attachment_ids":["%(a)s",1]}',
b'{"text":"x","attachment_ids":["not-a-uuid"]}',
b'{"text":"","attachment_ids":[]}', # neither text nor files
b'{"text":" "}',
b'{"text":"x","attachment_ids":[],"sender_id":"%(a)s"}',
], ids=lambda raw: raw.decode()[:40])
def test_message_body_validation(api, trio, raw):
alice, _, _, chat, _ = trio
a = api.upload(alice, chat, SAMPLE).json()["id"]
api.post_message(alice, chat, None, 400, raw=raw % {b"a": a.encode()})
def test_at_most_ten_attachments(api, trio):
alice, _, _, chat, _ = trio
ids = [api.upload(alice, chat, imagegen.png(4, 4)).json()["id"] for _ in range(11)]
assert len(api.post_message(alice, chat, {"text": "", "attachment_ids": ids[:10]}).json()["attachment_ids"]) == 10
api.post_message(alice, chat, {"text": "", "attachment_ids": ids}, 400)
# ------------------------------------------------------------------ limits
def test_size_limit_through_the_load_balancer(api, trio):
alice, bob, _, chat, _ = trio
exact = imagegen.png_exactly(10 * MiB)
assert len(exact) == 10 * MiB
att = api.upload(alice, chat, exact).json() # exactly the limit: accepted via LB
assert content(api, bob, att["id"]).content == exact
started = time.monotonic()
api.upload(alice, chat, exact + b"\x00", 413) # Content-Length over the limit
assert time.monotonic() - started < 5
over = exact + bytes(1024)
def chunks(): # no Content-Length: chunked body
for i in range(0, len(over), 256 * 1024):
yield over[i:i + 256 * 1024]
error = api.upload(alice, chat, chunks(), 413).json()
assert error["error"]["code"] == "payload_too_large"
def test_pixel_limit_before_decoding(api, trio):
alice, _, _, chat, _ = trio
for data, ctype in ((imagegen.png_bomb_header(), "image/png"), (imagegen.jpeg_bomb_header(), "image/jpeg"),
(imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), "image/png")):
started = time.monotonic()
assert api.upload(alice, chat, data, 413, content_type=ctype).json()["error"]["code"] == "payload_too_large"
assert time.monotonic() - started < 2 # refused from the header, nothing decoded
def test_exactly_40_megapixels_is_accepted(api, trio):
alice, _, _, chat, _ = trio
data = imagegen.png(8000, 5000, compress_level=9) # 40 000 000 px, ~120 KB file
att = api.upload(alice, chat, data).json()
assert att["size_bytes"] == len(data)
# ------------------------------------------------------------------ content checks
BROKEN = {
"png: first 33 bytes of the fixture": (SAMPLE[:33], "image/png"),
"png: truncated": (SAMPLE[:-20], "image/png"),
"png: corrupt IDAT, valid CRC": (imagegen.png_corrupt_idat(), "image/png"),
"png: fewer rows than IHDR": (imagegen.png_short_pixels(), "image/png"),
"png: flipped byte": (SAMPLE[:700] + bytes([SAMPLE[700] ^ 0x5A]) + SAMPLE[701:], "image/png"),
"png: bytes after IEND": (SAMPLE + b"<?php echo 1; ?>", "image/png"),
"jpeg: no scan data after SOS": (imagegen.jpeg_without_scan_data(), "image/jpeg"),
"jpeg: truncated": (imagegen.jpeg(400, 300, noise=True)[:5000], "image/jpeg"),
"mime: png bytes declared jpeg": (SAMPLE, "image/jpeg"),
"mime: jpeg bytes declared png": (imagegen.jpeg(), "image/png"),
"mime: gif declared png": (b"GIF89a" + bytes(100), "image/png"),
"mime: html declared png": (b"<html><script>alert(1)</script></html>", "image/png"),
"type: text/plain": (SAMPLE, "text/plain"),
"type: octet-stream": (SAMPLE, "application/octet-stream"),
"type: multipart": (SAMPLE, "multipart/form-data; boundary=x"),
"type: missing": (SAMPLE, None),
}
@pytest.mark.parametrize("name", list(BROKEN))
def test_bad_content_is_415(api, trio, name):
alice, _, _, chat, _ = trio
data, ctype = BROKEN[name]
error = api.upload(alice, chat, data, 415, content_type=ctype).json()
assert error["error"]["code"] == "unsupported_media_type"
def test_empty_body_is_400(api, trio):
alice, _, _, chat, _ = trio
api.upload(alice, chat, b"", 400)
# ------------------------------------------------------------------ filename
@pytest.mark.parametrize("query", [
"", "filename=", "filename=%20%20%20", "filename=.", "filename=..", "filename=a%2Fb.png", "filename=a%5Cb.png",
"filename=a%00b.png", "filename=c1%C2%85.png", "filename=tab%09.png", "filename=photo%E2%80%AEgnp.exe",
"filename=%FF%FE.png", "filename=a.png&filename=b.png", "filename=" + "a" * 256,
], ids=lambda q: q[:40] or "missing")
def test_bad_filename_is_400(api, trio, query):
alice, _, _, chat, _ = trio
api.upload(alice, chat, SAMPLE, 400, query=query)
def test_unicode_filename_is_kept_and_safely_exposed(api, trio):
alice, _, _, chat, _ = trio
name = 'отчёт "Q3"; ok.png'
att = api.upload(alice, chat, SAMPLE, filename=name).json()
assert att["filename"] == name
disposition = content(api, alice, att["id"]).headers["content-disposition"]
assert '"' not in disposition.split('filename="')[1].split('";')[0] # quotes neutralised in the fallback
assert "filename*=UTF-8''" in disposition
# ------------------------------------------------------------------ variants
def test_variant_parameter(api, trio):
alice, _, eve, chat, _ = trio
att = api.upload(alice, chat, SAMPLE).json()
assert content(api, alice, att["id"], query="?variant=original").content == SAMPLE
content(api, alice, att["id"], 404, query="?variant=thumbnail") # schema name, not produced in lab 4
content(api, alice, att["id"], 400, query="?variant=huge")
content(api, eve, att["id"], 404, query="?variant=thumbnail") # outsider: 404, nothing revealed
+15 -6
View File
@@ -278,28 +278,37 @@ def test_scale_up_and_down_is_picked_up_without_errors(api):
def test_compose_policy_limits_ports_secrets_images():
config = json.loads(compose("config", "--format", "json").stdout)
services = config["services"]
assert set(services) == {"postgres", "valkey", "migrate", "api", "lb"}
# The reconciler (profile "tools") is started on demand and not part of the running stand.
assert set(services) == {"postgres", "valkey", "s3", "migrate", "s3-init", "api", "lb"}
for name, svc in services.items():
limits = svc.get("deploy", {}).get("resources", {}).get("limits", {})
assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits"
assert "container_name" not in svc, f"{name}: container_name prevents scaling"
if name in ("api", "migrate"):
assert "build" in svc # built from this repository's source
elif name == "s3-init":
assert svc["image"] == services["api"]["image"] # the same locally built image
else:
assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest"
assert services["api"]["deploy"]["replicas"] >= 2 and not services["api"].get("ports")
published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")}
assert set(published) == {"lb"} and all(p["host_ip"] in ("127.0.0.1", "::1") for p in published["lb"])
secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()}
assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"],
"migrate": ["database_url"], "api": ["cache_url", "database_url"], "lb": []}
s3_keys = ['s3_admin_access_key_id', 's3_admin_secret_access_key', 's3_api_access_key_id', 's3_api_secret_access_key', 's3_reconciler_access_key_id', 's3_reconciler_secret_access_key']
assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"], "s3": s3_keys,
"migrate": ["database_url"], "s3-init": s3_keys, "lb": [],
# the API: its own S3 key pair only (no admin, no reconciler keys)
"api": ["cache_url", "database_url", "s3_api_access_key_id", "s3_api_secret_access_key"]}
volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"]
for name, svc in services.items()}
assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "migrate": [], "api": [], "lb": []}
assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "s3": ["s3data"], "migrate": [],
"s3-init": [], "api": [], "lb": []}
networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()}
assert networks == {"postgres": ["data"], "valkey": ["data"], "migrate": ["data"], "api": ["data", "edge"],
assert networks == {"postgres": ["data"], "valkey": ["data"], "s3": ["storage"], "migrate": ["data"],
"s3-init": ["storage"], "api": ["data", "edge", "storage"],
"lb": ["edge"]}, networks # the LB cannot reach the stores at all
assert config["networks"]["data"].get("internal") is True
assert config["networks"]["storage"].get("internal") is True
def test_running_containers_have_limits_and_no_bypass_ports():
@@ -317,7 +326,7 @@ def test_running_containers_have_limits_and_no_bypass_ports():
def test_migrations_ran_once_by_the_migrate_service():
applied = psql("SELECT version FROM schema_migrations ORDER BY version").split()
assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function"]
assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function", "0004_attachments"]
migrate = compose("ps", "-a", "--format", "json", "migrate").stdout
rows = [json.loads(line) for line in migrate.splitlines() if line.strip()]
assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows)
+111
View File
@@ -0,0 +1,111 @@
"""Consistency of PostgreSQL and S3 around one upload (lab 4, level 4).
The two-store write is: intent row 'pending' -> S3 PUT -> row 'ready' -> 201. Failures between
the steps are injected with FAULT_POINTS (compose variable, empty in normal runs; the API
raises 503 at that exact point, as if the process or the network failed there):
* after_s3_put object written, row still pending, client got 503 -> the janitor removes
the object and the row after PENDING_GRACE_SECONDS (here 3 s);
* after_ready_commit row ready and object written, but the answer is lost (client got 503)
-> nothing may be deleted: the attachment stays intact and downloadable.
Plus: an upload interrupted by the client and a stalled body leave nothing behind."""
from __future__ import annotations
import socket
import time
from pathlib import Path
import pytest
import imagegen
from conftest import api_with_env, attachment_row, psql, s3_size
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
def rows_of_chat(chat_id: str) -> list[tuple[str, str, str]]:
out = psql(f"SELECT id || ' ' || status || ' ' || object_key FROM attachments WHERE chat_id = '{chat_id}'")
return [tuple(line.split()) for line in out.splitlines() if line]
def wait_for(condition, timeout: float, what: str) -> None:
deadline = time.monotonic() + timeout
while not condition():
assert time.monotonic() < deadline, what
time.sleep(0.5)
def raw_upload(token: str, chat_id: str, declared: int, body: bytes, *, wait_answer: float | None) -> bytes:
"""Hand-made HTTP request: announce `declared` bytes, send only `body`, then either close the
socket (client gone) or keep it open and read the answer after `wait_answer` seconds."""
sock = socket.create_connection(("127.0.0.1", 8080), timeout=30)
head = (f"POST /api/v1/chats/{chat_id}/attachments?filename=x.png HTTP/1.1\r\nHost: localhost\r\n"
f"Authorization: Bearer {token}\r\nContent-Type: image/png\r\nContent-Length: {declared}\r\n\r\n")
sock.sendall(head.encode() + body)
if wait_answer is None:
sock.close()
return b""
time.sleep(wait_answer)
answer = sock.recv(65536)
sock.close()
return answer
def test_interrupted_and_stalled_uploads_leave_nothing(api):
alice = api.user("alice")
chat = api.chat(alice)
data = imagegen.png_of_size(2_000_000)
raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=None) # client disconnects
with api_with_env(UPLOAD_READ_TIMEOUT_SECONDS="2"):
answer = raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=5) # client stalls
assert answer.startswith(b"HTTP/1.1 400") and b"invalid_request" in answer, answer[:200]
assert b"x-instance-id" in answer.lower()
time.sleep(1)
assert rows_of_chat(chat["id"]) == [] # nothing was written: the body is checked before any write
def test_crash_between_s3_put_and_commit_is_cleaned_by_janitor(api):
alice = api.user("alice")
chat = api.chat(alice)
with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"):
assert api.upload(alice, chat, SAMPLE, 503).json()["error"]["code"] == "unavailable"
[(attachment_id, status, key)] = rows_of_chat(chat["id"])
assert status == "pending" and s3_size(key) == len(SAMPLE) # the orphan-to-be exists
api.call("GET", f"/api/v1/attachments/{attachment_id}", 404, token=alice["token"]) # invisible
api.post_message(alice, chat, {"text": "", "attachment_ids": [attachment_id]}, 404) # not attachable
wait_for(lambda: rows_of_chat(chat["id"]) == [], 30, "janitor did not remove the stale intent")
assert s3_size(key) is None # object removed first
# Normal configuration again: an upload in the same chat works and is not touched.
att = api.upload(alice, chat, SAMPLE).json()
assert attachment_row(att["id"])["status"] == "ready"
def test_lost_answer_after_commit_never_deletes_the_object(api):
"""The reviewer found this in another PR: COMMIT succeeded, the answer was lost, the API
'compensated' by deleting the object -> ready row without bytes. Here nothing compensates
in the request path, and the janitor only touches 'pending' rows."""
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
with api_with_env(FAULT_POINTS="after_ready_commit", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"):
api.upload(alice, chat, SAMPLE, 503)
[(attachment_id, status, key)] = rows_of_chat(chat["id"])
assert status == "ready"
time.sleep(6) # several janitor rounds
assert rows_of_chat(chat["id"]) == [(attachment_id, "ready", key)] and s3_size(key) == len(SAMPLE)
got = api.call("GET", f"/api/v1/attachments/{attachment_id}/content", 200, token=bob["token"])
assert got.content == SAMPLE
def test_fresh_pending_upload_is_not_touched_by_janitor(api):
"""An intent younger than PENDING_GRACE_SECONDS is an upload in progress: never removed."""
alice = api.user("alice")
chat = api.chat(alice)
with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="600", JANITOR_INTERVAL_SECONDS="1"):
api.upload(alice, chat, SAMPLE, 503)
time.sleep(4)
[(_, status, key)] = rows_of_chat(chat["id"])
assert status == "pending" and s3_size(key) == len(SAMPLE)
+2 -1
View File
@@ -78,7 +78,8 @@ def test_migrations_rerun_is_a_noop():
assert "0001_init: already applied, skipped" in result.stdout
assert "0002_idempotency_keys: already applied, skipped" in result.stdout
assert "0003_post_message_function: already applied, skipped" in result.stdout
assert psql("SELECT count(*) FROM schema_migrations") == "3"
assert "0004_attachments: already applied, skipped" in result.stdout
assert psql("SELECT count(*) FROM schema_migrations") == "4"
assert psql("SELECT count(*) FROM users") == users_before
+209
View File
@@ -0,0 +1,209 @@
"""Track 2: the reconciler of S3 and PostgreSQL (python -m messenger.reconcile).
Faults are injected on BOTH sides, directly into the stores (S3 with the API/reconciler key
pairs from inside the storage network, PostgreSQL with psql):
S3 side: orphan object (no row), object deleted under a ready row, object overwritten with
other bytes (size / same-size checksum), object of a quarantined row restored;
DB side: stale intent with and without object, quarantine mark on a healthy row;
fresh: orphan and intent younger than --grace (an upload in progress).
The tests prove: dry-run changes nothing in either store; --apply fixes exactly the old findings;
fresh and used attachments are never deleted (ready rows only ever get a reversible quarantine);
the second run finds nothing to do; the safety valve and store outages stop the run without
damage. A small --grace (8-30 s, --allow-small-grace) keeps the tests fast."""
from __future__ import annotations
import concurrent.futures
import hashlib
import time
import uuid
from pathlib import Path
import pytest
import imagegen
from conftest import (attachment_row, findings_by_key, psql, reconcile, s3_delete, s3_keys, s3_put, s3_size,
service_down)
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
GRACE = ["--grace", "8", "--allow-small-grace"]
# The main scenario needs "fresh" items to stay younger than --grace for the whole dry-run + snapshot
# (each docker exec costs 1-3 s on Windows), hence a longer grace there.
LONG_GRACE = ["--grace", "30", "--allow-small-grace"]
def new_key() -> str:
return "o/" + uuid.uuid4().hex
def insert_intent(chat: dict, owner: dict, key: str, data: bytes, age_seconds: int) -> str:
"""A 'pending' row as an upload leaves it after the intent COMMIT (crash afterwards)."""
attachment_id = str(uuid.uuid4())
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
"object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'x.png', 'image/png', %d, "
"decode('%s', 'hex'), 320, 200, '%s', 'pending', clock_timestamp() - make_interval(secs => %d), "
"clock_timestamp() - make_interval(secs => %d))"
% (attachment_id, chat["id"], owner["id"], len(data), hashlib.sha256(data).hexdigest(), key, age_seconds,
age_seconds))
return attachment_id
def snapshot() -> tuple[str, frozenset]:
"""Fingerprint of everything the reconciler could change: every attachment row + the key list."""
rows = psql("SELECT count(*) || ':' || coalesce(md5(string_agg(id::text || status || object_key || "
"coalesce(lost_reason, '-') || updated_at::text, ',' ORDER BY id)), '') FROM attachments")
return rows, frozenset(s3_keys())
def content_status(api, actor, attachment_id) -> int:
return api.client.get(f"/api/v1/attachments/{attachment_id}/content",
headers={"Authorization": "Bearer " + actor["token"]}).status_code
def test_detect_and_safely_fix_faults_on_both_sides(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
up = {name: api.upload(alice, chat, imagegen.png(30 + i, 20)).json() for i, name in enumerate("ABCDE")}
data = {name: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
for name, a in up.items()}
key = {name: attachment_row(a["id"])["object_key"] for name, a in up.items()}
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
# ---- inject (old)
orphan_old = new_key()
s3_put(orphan_old, SAMPLE)
stale_with, stale_without = new_key(), new_key()
s3_put(stale_with, SAMPLE)
stale_with_id = insert_intent(chat, alice, stale_with, SAMPLE, 600) # < janitor grace 900 s
stale_without_id = insert_intent(chat, alice, stale_without, SAMPLE, 600)
s3_delete(key["C"]) # ready + used, bytes lost
s3_put(key["D"], data["D"][:len(data["D"]) // 2]) # ready, unused, wrong size
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' "
f"WHERE id = '{up['E']['id']}'") # quarantined, but bytes are fine
time.sleep(32) # everything above is now older than --grace
# ---- inject (fresh: an upload in progress)
fresh_orphan, fresh_intent = new_key(), new_key()
s3_put(fresh_orphan, SAMPLE)
s3_put(fresh_intent, SAMPLE)
fresh_intent_id = insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
# ---- dry-run: report everything, change nothing
before = snapshot()
dry = reconcile(*LONG_GRACE)
assert snapshot() == before, "dry-run modified a store"
f = findings_by_key(dry)
expected = {orphan_old: "orphan_object", fresh_orphan: "fresh_orphan_object", stale_with: "stale_intent",
stale_without: "stale_intent", fresh_intent: "fresh_intent", key["C"]: "missing_object",
key["D"]: "size_mismatch", key["E"]: "restored_object"}
assert {k: f[k]["kind"] for k in expected} == expected
assert key["A"] not in f and key["B"] not in f
assert f[key["C"]]["used"] is True and dry["mode"] == "dry-run"
assert all(f[k]["result"] in ("planned", "not_actionable") for k in expected)
# ---- apply
applied = findings_by_key(reconcile("--apply", *LONG_GRACE))
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"]):
assert applied[k]["result"] == "applied", (k, applied[k])
keys = s3_keys()
assert orphan_old not in keys and stale_with not in keys
assert attachment_row(stale_with_id) is None and attachment_row(stale_without_id) is None
# fresh: untouched
assert fresh_orphan in keys and fresh_intent in keys and attachment_row(fresh_intent_id)["status"] == "pending"
# used, lost bytes: quarantined, NOT deleted; the message still references it
row_c = attachment_row(up["C"]["id"])
assert row_c["status"] == "ready" and row_c["lost_reason"] == "missing_object"
items, _ = api.history(bob, chat, 100)
assert any(up["C"]["id"] in m["attachment_ids"] for m in items)
assert content_status(api, bob, up["C"]["id"]) == 404
api.post_message(alice, chat, {"text": "", "attachment_ids": [up["C"]["id"]]}, 409) # cannot attach a lost file
assert attachment_row(up["D"]["id"])["lost_reason"] == "size_mismatch"
assert attachment_row(up["E"]["id"])["lost_reason"] is None # quarantine lifted
for name in "ABE":
got = api.call("GET", f"/api/v1/attachments/{up[name]['id']}/content", 200, token=bob["token"])
assert got.content == data[name]
# ---- second run: idempotent, nothing left to do for these keys
again = findings_by_key(reconcile("--apply", *LONG_GRACE))
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"], key["A"], key["B"]):
assert k not in again, (k, again.get(k))
# ---- the lost object comes back from a backup: the quarantine is lifted, the file is served again
s3_put(key["C"], data["C"])
restored = findings_by_key(reconcile("--apply", *LONG_GRACE))
assert restored[key["C"]]["kind"] == "restored_object" and restored[key["C"]]["result"] == "applied"
assert api.call("GET", f"/api/v1/attachments/{up['C']['id']}/content", 200, token=bob["token"]).content == data["C"]
# tidy up the fresh items of this test once they are old enough
time.sleep(31)
reconcile("--apply", *LONG_GRACE)
assert fresh_orphan not in s3_keys() and attachment_row(fresh_intent_id) is None
def test_checksum_mode_finds_same_size_corruption(api):
alice = api.user("alice")
chat = api.chat(alice)
att = api.upload(alice, chat, SAMPLE).json()
k = attachment_row(att["id"])["object_key"]
s3_put(k, SAMPLE[:1000] + bytes([SAMPLE[1000] ^ 1]) + SAMPLE[1001:]) # same size, other bytes
time.sleep(9)
assert k not in findings_by_key(reconcile(*GRACE)) # sizes match: listing cannot see it
report = reconcile("--verify-checksums", *GRACE)
assert findings_by_key(report)[k]["kind"] == "checksum_mismatch"
assert report["cost"]["s3_get_bytes"] >= len(SAMPLE)
assert findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]["result"] == "applied"
assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch"
def test_safety_valve_refuses_mass_deletion():
"""If (almost) every object looked orphaned - e.g. the reconciler pointed at an empty database -
--apply must stop before deleting anything."""
orphan = new_key()
s3_put(orphan, SAMPLE)
time.sleep(9)
report = reconcile("--apply", "--max-orphan-fraction", "0.000001", *GRACE, expect_code=3)
assert report["error"].startswith("safety_valve") and report["actions"]["applied"] == 0
assert orphan in s3_keys()
reconcile("--apply", *GRACE)
assert orphan not in s3_keys()
@pytest.mark.parametrize("service", ["s3", "postgres"])
def test_store_outage_stops_the_run_without_damage(api, service):
with service_down(service, "stop"):
report = reconcile(*GRACE, expect_code=1)
assert report["error"].startswith("store_unavailable")
assert reconcile(*GRACE)["error"] is None
def test_live_uploads_are_never_hurt_by_concurrent_apply(api):
"""Uploads and messages run while --apply runs repeatedly: every upload that got 201 stays
ready and downloadable byte for byte, every message keeps its attachments."""
alice = api.user("alice")
chat = api.chat(alice)
payloads = [imagegen.png(16 + i, 16) for i in range(40)]
done: list[tuple[dict, bytes]] = []
def worker(chunk: list[bytes]) -> None:
for p in chunk:
att = api.upload(alice, chat, p).json()
api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]})
done.append((att, p))
with concurrent.futures.ThreadPoolExecutor(4) as pool:
futures = [pool.submit(worker, payloads[i::4]) for i in range(4)]
runs = 0
while not all(x.done() for x in futures):
report = reconcile("--apply", *GRACE)
assert report["error"] is None
runs += 1
for x in futures:
x.result()
assert runs >= 1 and len(done) == len(payloads)
for att, p in done:
row = attachment_row(att["id"])
assert row["status"] == "ready" and row["lost_reason"] is None and s3_size(row["object_key"]) == len(p)
for att, p in done[::8]:
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]).content == p
+107
View File
@@ -0,0 +1,107 @@
"""S3 outages, private bucket and failover of attachments (lab 4, levels 3 and 4).
Every injected failure is undone in `finally` (conftest.service_down / replica_stopped), and the
stand is waited for until HAProxy sees all replicas UP again."""
from __future__ import annotations
import time
from pathlib import Path
import pytest
import imagegen
from conftest import attachment_row, compose, probe_inside, replica_stopped, running_replicas, service_down
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
@pytest.fixture()
def pair(api):
alice, bob = api.user("alice"), api.user("bob")
return alice, bob, api.chat(alice, bob)
@pytest.mark.parametrize("how", ["stop", "pause"])
def test_s3_outage_is_503_and_text_keeps_working(api, pair, how):
"""S3 down (stopped: connection refused; paused: no answer at all):
* readiness is 503 on every replica (and through the LB), quickly;
* uploads/downloads are 503 `unavailable` within the S3 deadline, never 500;
* the LB's degraded mode keeps chats/text messages working (HAProxy backend api_core);
* after recovery the earlier file is intact."""
alice, bob, chat = pair
att = api.upload(alice, chat, SAMPLE).json()
with service_down("s3", how):
deadline = time.monotonic() + 15
while api.client.get("/health/ready").status_code != 503:
assert time.monotonic() < deadline, "readiness did not turn 503"
time.sleep(0.5)
for replica in running_replicas():
assert probe_inside(replica, "/health/ready") == 503
assert probe_inside(replica, "/health/ready/core") == 200
started = time.monotonic()
error = api.upload(alice, chat, SAMPLE, 503).json()
assert error["error"]["code"] == "unavailable"
assert time.monotonic() - started < 8
started = time.monotonic()
api.call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"])
assert time.monotonic() - started < 8
api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]) # metadata: PostgreSQL only
api.send(alice, chat, "текст работает без S3") # degraded mode
got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
assert got.content == SAMPLE
api.upload(alice, chat, SAMPLE)
def test_upload_survives_failover_of_the_serving_replica(api, pair):
"""Upload -> stop the replica that served it -> another replica downloads the same bytes
(nothing of the file lives in the replica)."""
alice, bob, chat = pair
data = imagegen.png_of_size(500_000)
response = api.upload(alice, chat, data)
att, served_by = response.json(), response.headers["x-instance-id"]
replica = next(r for r in running_replicas() if r["hostname"] == served_by)
with replica_stopped(replica, "kill"):
for _ in range(5):
got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
assert got.headers["x-instance-id"] != served_by and got.content == data
def test_bucket_is_private_outside_the_api(api, pair):
"""Inside the storage network (where the API is): an UNSIGNED GET of a real object and an
unsigned LIST are refused; the same GET signed with the API key pair works (control).
SeaweedFS admin endpoints (master 9333, filer 8888, volume 8080) are not reachable."""
alice, _, chat = pair
att = api.upload(alice, chat, SAMPLE).json()
key = attachment_row(att["id"])["object_key"]
probe = (
"import urllib.request as u, urllib.error as e, socket\n"
"def st(url):\n"
" try: return u.urlopen(url, timeout=3).status\n"
" except e.HTTPError as x: return x.code\n"
f"print(st('http://s3:8333/messenger-attachments/{key}'), st('http://s3:8333/messenger-attachments?list-type=2'))\n"
"for port in (9333, 8888, 8080):\n"
" s = socket.socket(); s.settimeout(2)\n"
" print(port, s.connect_ex(('s3', port)))\n")
out = compose("exec", "-T", "api", "python", "-c", probe).stdout.split("\n")
get_status, list_status = out[0].split()
assert get_status in ("401", "403") and list_status in ("401", "403"), out[0]
for line in out[1:4]:
port, code = line.split()
assert code != "0", f"admin port {port} reachable"
signed = compose("exec", "-T", "api", "python", "-c",
"import asyncio\nfrom messenger.config import Settings\nfrom messenger.storage import Storage\n"
"async def m():\n s=Storage(Settings.from_env()); await s.open()\n"
f" print((await s.head('{key}')).size); await s.close()\nasyncio.run(m())").stdout.strip()
assert signed == str(len(SAMPLE))
def test_s3_restart_keeps_objects(api, pair):
alice, bob, chat = pair
data = imagegen.png_of_size(200_000)
att = api.upload(alice, chat, data).json()
with service_down("s3", "stop"):
pass
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]).content == data
+148
View File
@@ -0,0 +1,148 @@
"""Unit tests of the image checks (src/messenger/images.py) and the filename rules. No services.
The JPEG decoder (djpeg) exists in the API image; locally these tests are skipped without it
(the same cases run against the stand in test_attachments.py)."""
from __future__ import annotations
import asyncio
import shutil
import sys
import time
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "src"))
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
from messenger import images, validation # noqa: E402
from messenger.errors import InvalidRequest # noqa: E402
LIMIT = 40_000_000
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
has_djpeg = pytest.mark.skipif(shutil.which("djpeg") is None, reason="djpeg (libjpeg-turbo-progs) not installed")
def status(data: bytes) -> int:
"""201 if accepted, else the HTTP status of the ImageError (413/415)."""
try:
info = images.sniff(data, LIMIT)
if info is None:
return 415 # header never completed
asyncio.run(images.verify(data, info))
return 201
except images.ImageError as exc:
return exc.status
def test_public_fixture_is_accepted():
assert images.sniff(SAMPLE, LIMIT) == images.ImageInfo("image/png", 320, 200)
assert status(SAMPLE) == 201
@pytest.mark.parametrize("data", [
imagegen.png(37, 23, "RGBA"), imagegen.png(33, 17, "P"), imagegen.png(33, 17, "1"),
imagegen.png(33, 17, "L"), imagegen.png(81, 57, "RGB", noise=True), imagegen.png(1, 1),
], ids=["rgba", "palette", "1bit", "gray", "noise", "1x1"])
def test_valid_png_kinds(data):
assert status(data) == 201
def test_interlaced_png_is_accepted():
# Pillow cannot write Adam7; build one by hand: 9 x 9 grey, every pass with filter 0.
from messenger.images import _png_segments
raw = b"".join(b"\x00" + bytes(stride - 1) for size, stride in _png_segments(9, 9, 8, 0, 1)
for _ in range(size // stride))
data = imagegen.PNG_SIGNATURE + imagegen.ihdr(9, 9, 8, 0, 1) + imagegen.chunk(b"IDAT", __import__("zlib").compress(raw)) \
+ imagegen.chunk(b"IEND", b"")
assert status(data) == 201
BROKEN_PNG = {
"first 33 bytes of the fixture": SAMPLE[:33],
"truncated (no IEND)": SAMPLE[:-12],
"truncated in IDAT": SAMPLE[:len(SAMPLE) // 2],
"flipped byte -> CRC error": SAMPLE[:500] + bytes([SAMPLE[500] ^ 0xFF]) + SAMPLE[501:],
"garbage IDAT with valid CRC": imagegen.png_corrupt_idat(),
"fewer rows than IHDR": imagegen.png_short_pixels(),
"bytes after IEND (polyglot)": SAMPLE + b"PK\x03\x04zip",
"text": b"just text, definitely not an image",
"gif": b"GIF89a" + bytes(64),
"two bytes of a signature": b"\x89P",
}
@pytest.mark.parametrize("name", list(BROKEN_PNG))
def test_broken_png_is_415(name):
assert status(BROKEN_PNG[name]) == 415, name
def test_pixel_limit_is_checked_from_the_header():
with pytest.raises(images.ImageError) as exc:
images.sniff(imagegen.png_bomb_header()[:33], LIMIT) # 33 bytes are enough
assert exc.value.status == 413
exact = imagegen.PNG_SIGNATURE + imagegen.ihdr(8000, 5000) # exactly 40 000 000: allowed
assert images.sniff(exact, LIMIT).width == 8000
with pytest.raises(images.ImageError) as exc:
images.sniff(imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), LIMIT)
assert exc.value.status == 413
with pytest.raises(images.ImageError) as exc:
images.sniff(imagegen.jpeg_bomb_header(), LIMIT)
assert exc.value.status == 413
def test_png_verification_is_streaming_and_fast():
"""40 Mpx RGB = 120 MB of raw pixels: inflated in 1 MiB steps and discarded."""
big = imagegen.png(8000, 5000, compress_level=9)
started = time.perf_counter()
assert status(big) == 201
assert time.perf_counter() - started < 5
def test_jpeg_header_is_parsed_incrementally():
data = imagegen.jpeg(320, 200)
assert images.sniff(data[:4], LIMIT) is None # SOF not arrived yet
assert images.sniff(data, LIMIT) == images.ImageInfo("image/jpeg", 320, 200)
@has_djpeg
@pytest.mark.parametrize("data", [imagegen.jpeg(), imagegen.jpeg(640, 480, progressive=True)], ids=["base", "prog"])
def test_valid_jpeg(data):
assert status(data) == 201
@has_djpeg
@pytest.mark.parametrize("name,data", [
("no scan data after SOS", imagegen.jpeg_without_scan_data()),
("truncated", imagegen.jpeg(320, 240, noise=True)[:3000]),
], ids=lambda v: v if isinstance(v, str) else "")
def test_broken_jpeg_is_415(name, data):
assert status(data) == 415, name
# ------------------------------------------------------------------ filename
@pytest.mark.parametrize("name", ["photo.png", "фото отпуска.jpg", "a" * 255, "emoji 👨‍👩‍👧.png", " x ", "..."[:2] + "x"])
def test_good_filenames(name):
assert validation.filename_param([("filename", name)]) == name
@pytest.mark.parametrize("name", ["", " ", " ", ".", "..", "a/b.png", "a\\b.png", "a\x00b", "c1\x85.png",
"tab\t.png", "del\x7f.png", "x
y", "photo‮gnp.exe", "a" * 256])
def test_bad_filenames(name):
with pytest.raises(InvalidRequest):
validation.filename_param([("filename", name)])
def test_filename_must_be_single_and_present():
with pytest.raises(InvalidRequest):
validation.filename_param([])
with pytest.raises(InvalidRequest):
validation.filename_param([("filename", "a.png"), ("filename", "b.png")])
def test_query_must_be_valid_utf8():
assert validation.query_items(b"filename=%D1%84.png") == [("filename", "ф.png")]
with pytest.raises(InvalidRequest):
validation.query_items(b"filename=%FF%FE.png")
+23
View File
@@ -114,6 +114,29 @@ def test_argon2id_hash_and_verify():
# ------------------------------------------------------------------ config
@pytest.fixture(autouse=True)
def s3_credentials(monkeypatch):
"""Lab 4: Settings require the API's S3 key pair; synthetic values for the unit tests."""
monkeypatch.setenv("S3_ACCESS_KEY_ID", "AKTEST")
monkeypatch.setenv("S3_SECRET_ACCESS_KEY", "secret-for-unit-tests")
def test_s3_settings_are_validated_and_contract_limits_cannot_be_raised(monkeypatch):
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
settings = Settings.from_env()
assert (settings.max_upload_bytes, settings.max_image_pixels) == (10 * 1024 * 1024, 40_000_000)
assert settings.fault_points == frozenset() and settings.upload_concurrency == 4
for name, value in (("MAX_UPLOAD_BYTES", "10485761"), ("MAX_IMAGE_PIXELS", "40000001"),
("FAULT_POINTS", "drop_database"), ("S3_ENDPOINT", "ftp://s3"), ("S3_BUCKET", "Bad_Bucket")):
monkeypatch.setenv(name, value)
with pytest.raises(ConfigError):
Settings.from_env()
monkeypatch.delenv(name)
monkeypatch.delenv("S3_SECRET_ACCESS_KEY")
with pytest.raises(ConfigError):
Settings.from_env()
def test_settings_read_secret_files_and_contract_names(tmp_path, monkeypatch):
url_file = tmp_path / "database_url"
url_file.write_text("postgresql://u:p@db/x\n", encoding="utf-8")