перенос: реализация lab4 без изменений (API, миграции 0001-0004, HAProxy, Valkey, SeaweedFS, тесты)

Скопировано из mtusi2026lab4 (ветка solution/bvt2302-bodanyuk, commit 17c33cd) через
git archive: исходники, миграции, собственные тесты, Dockerfile, compose, конфигурации
infra/, скрипты и bench/. README, COURSE, contracts/, tests/ и отчёт lab4 не копировались.
Makefile и .env.example — версии lab4 (skeleton-часть совпадает с lab5, строки broker
добавляются в следующих коммитах).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alexey Bodanyuk
2026-10-31 15:19:00 +03:00
co-authored by Claude Opus 5.5
parent 276dbdad8a
commit 5112be83cc
70 changed files with 10544 additions and 7 deletions
+48 -7
View File
@@ -5,21 +5,62 @@ APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
# The files are created by `python scripts/init_secrets.py` and mounted by compose as secrets.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
# valkey = sessions live only in Valkey (AOF, appendfsync always); postgres = table `sessions`.
SESSION_STORE=valkey
CACHE_URL_FILE=/run/secrets/cache_url
# Number of interchangeable API replicas behind HAProxy (compose deploy.replicas).
API_REPLICAS=2
# Idempotency-Key results are kept at least this long (contract: >= 24 h).
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
# Write path of POST message: function = one SQL call (migration 0003), multi = lab3 baseline path.
MESSAGE_WRITE_PATH=function
# Lab 4: private S3 bucket (SeaweedFS service `s3`). The API gets ITS OWN key pair (Read+Write on
# this bucket only) as Docker secrets; admin/reconciler pairs go only to s3-init/reconciler.
S3_ENDPOINT=http://s3:8333
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
# Contract limits (may be lowered, never raised above 10485760 / 40000000).
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
# Per replica: bodies buffered at once (RAM bound), wait for a slot, body deadline, image checks at once.
UPLOAD_CONCURRENCY=4
UPLOAD_QUEUE_TIMEOUT_SECONDS=5
UPLOAD_READ_TIMEOUT_SECONDS=30
IMAGE_VERIFY_TIMEOUT_SECONDS=10
IMAGE_VERIFY_CONCURRENCY=1
JPEG_MAX_MEMORY_MB=256
S3_TIMEOUT_SECONDS=3
# Janitor: 'pending' intents older than this are failed uploads (object + row removed).
PENDING_GRACE_SECONDS=900
JANITOR_INTERVAL_SECONDS=60
# Level 4 failure experiments only (after_s3_put, after_ready_commit); keep empty.
FAULT_POINTS=
# Additional settings of this implementation (defaults shown; all optional):
LOG_LEVEL=info
DB_POOL_MIN=1
DB_POOL_MAX=10
DB_TIMEOUT_SECONDS=3
CACHE_TIMEOUT_SECONDS=0.5
PASSWORD_HASH_CONCURRENCY=2
DOCS_ENABLED=true
# Graceful stop of a replica: readiness 503 + drain, then up to SHUTDOWN for in-flight requests.
DRAIN_SECONDS=3
SHUTDOWN_TIMEOUT_SECONDS=8
# CPU/RAM limits (docker compose deploy.resources.limits); API_* are per replica.
API_CPUS=1.0
API_MEMORY=512m
S3_CPUS=1.0
S3_MEMORY=1g
PG_CPUS=2.0
PG_MEMORY=1g
VALKEY_CPUS=0.5
LB_CPUS=1.0
PG_MAX_CONNECTIONS=100
# Host ports of the load balancer (published on 127.0.0.1 only; API replicas publish nothing)
APP_PUBLISH_PORT=8080
LB_STATS_PORT=8404
+5
View File
@@ -0,0 +1,5 @@
# LF everywhere: identical files on Windows/Linux and inside `git archive | docker build -`.
* text=auto eol=lf
*.png binary
# cmd.exe expects CRLF in batch files.
*.cmd text eol=crlf
+38
View File
@@ -0,0 +1,38 @@
name: Application (lab4)
on: [push, pull_request]
permissions:
contents: read
jobs:
app:
runs-on: ubuntu-latest
timeout-minutes: 40
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.13'
- name: Synthetic secrets and env (CI only, never committed)
run: python scripts/init_secrets.py && cp .env.example .env
- name: Build, migrate once, bootstrap the S3 bucket, start 2 API replicas behind HAProxy
run: docker compose --env-file .env up -d --build --wait
- name: Contract and public smoke
run: make check test PYTHON=python
- name: Persistence across restart of LB, API replicas, PostgreSQL, Valkey and S3
run: make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python
- name: Failover of the replica that served the test, then restore it
run: |
make test-failover ACTION_SCRIPT=scripts/stop-one.sh PYTHON=python
sh scripts/restore-api.sh && python scripts/lbctl.py wait-up 2
- name: Reconciler (track 2) dry-run on the stand - read-only report
run: docker compose run --rm reconciler
- name: Own tests (unit + stand; every failure is restored in finally)
run: |
python -m pip install --require-hashes --no-deps -r requirements.txt
python -m pip install -r requirements-test.txt
make student-test PYTHON=python
- name: Logs
if: always()
run: docker compose logs --tail 100 api lb s3 s3-init
- name: Remove CI stand and its synthetic data
if: always()
run: docker compose down -v
+66
View File
@@ -0,0 +1,66 @@
# syntax=docker/dockerfile:1
# Messenger API, lab 4 (replicas behind HAProxy, PostgreSQL + Valkey + S3). Built by compose.yaml
# (services api, migrate, s3-init, reconciler: one image, different commands).
#
# Python is interpreted, so there is no separate compile stage; the image
# contains only the interpreter, locked dependencies and the source code.
# Base image is pinned by version AND digest (python 3.13.16, Debian 12 bookworm).
FROM python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1
WORKDIR /app
# 1. Dependencies first (layer cache survives source edits). Every package,
# including transitive ones, is pinned with sha256 hashes in requirements.txt;
# --no-deps guarantees nothing outside the lock file is installed.
COPY requirements.txt ./
RUN pip install --root-user-action=ignore --require-hashes --no-deps -r requirements.txt
# 2. JPEG integrity check: libjpeg-turbo's djpeg (strict mode, see src/messenger/images.py).
# Version pinned to the Debian 12 package; apt lists are removed in the same layer.
RUN apt-get update \
&& apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \
&& rm -rf /var/lib/apt/lists/*
# 3. Unprivileged runtime user. Code stays owned by root, so the process
# cannot modify its own code.
RUN groupadd --system --gid 10001 app \
&& useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app
COPY src/ ./src/
# Versioned SQL migrations, applied by `python -m messenger.migrate` (compose service migrate).
COPY migrations/ ./migrations/
# Course contract, served read-only at /openapi.json for Swagger UI (/docs).
COPY contracts/openapi.json ./contracts/openapi.json
# Non-secret defaults; compose.yaml passes the rest. Passwords come only as files
# (DATABASE_URL_FILE / CACHE_URL_FILE -> Docker secrets), never as ENV or COPY.
ENV PYTHONPATH=/app/src \
APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab \
LOG_LEVEL=info \
SHUTDOWN_TIMEOUT_SECONDS=8 \
DOCS_ENABLED=true \
SESSION_TTL_SECONDS=3600 \
DATABASE_URL_FILE=/run/secrets/database_url
# Build revision (scripts/rolling-update.sh passes git commit + time): makes every rolled-out
# image distinguishable in `docker inspect` and in the "starting" log line of each replica.
ARG BUILD_REVISION=dev
LABEL org.opencontainers.image.revision=$BUILD_REVISION
ENV APP_REVISION=$BUILD_REVISION
USER app:app
EXPOSE 8080
STOPSIGNAL SIGTERM
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD ["python", "-m", "messenger.healthcheck"]
# Exec form: the Python server is PID 1 and receives SIGTERM directly (no shell in between).
CMD ["python", "-m", "messenger"]
+67
View File
@@ -28,3 +28,70 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# ---------------------------------------------------------------------------
# Student solution (lab 4, Python). On Windows: PowerShell/Git Bash with GNU make, public
# tests need PYTHONUTF8=1, and the hooks are started through scripts/*.cmd (Git Bash).
.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series reconcile reconcile-apply bench-memory bench-reconcile
secrets:
$(PYTHON) scripts/init_secrets.py
# Build, apply migrations ONCE (service migrate), start API replicas and the LB, wait for healthy.
up: secrets
docker compose up -d --build --wait
# Safe stop: containers stop, named volumes (data) stay.
down:
docker compose down
logs:
docker compose logs --tail 50 api lb
ps:
docker compose ps -a
migrate:
docker compose run --rm migrate
# HAProxy view of the replicas (UP/DOWN) mapped to containers and X-Instance-Id.
lb-status:
$(PYTHON) scripts/lbctl.py status
$(PYTHON) scripts/lbctl.py replicas
# Start replicas stopped by scripts/stop-one.sh again and wait until the LB has them all UP.
restore:
sh scripts/restore-api.sh
$(PYTHON) scripts/lbctl.py wait-up $${API_REPLICAS:-2}
# Replace every API replica one by one with a freshly built image (no downtime).
rolling-update:
sh scripts/rolling-update.sh
# Level 4: stop each replica in turn under load, measure RPS/errors/recovery (evidence/failover/).
failover-series:
$(PYTHON) bench/failover_series.py --output evidence/failover
# Own tests: unit + black-box against the running stand (needs requirements*.txt in a venv).
student-test:
$(PYTHON) -m pytest student-tests -v
unit-test:
$(PYTHON) -m pytest student-tests -m "not stack" -q
explain:
$(PYTHON) scripts/explain_history.py --output evidence/explain-history.txt
# Track 2: reconciler of S3 objects vs attachment rows (dry-run by default, read-only).
reconcile:
docker compose run --rm reconciler
reconcile-apply:
docker compose run --rm reconciler --apply
# Level 4: peak RAM of the replicas under 16 simultaneous uploads (evidence/memory/).
bench-memory:
$(PYTHON) bench/upload_memory.py --output evidence/memory
# Track 2: cost of one reconciler pass for 1k/10k/50k attachments (evidence/reconcile/).
bench-reconcile:
$(PYTHON) bench/reconcile_cost.py --output evidence/reconcile
+141
View File
@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""Level 4: stop EACH API replica in turn under load (restoring the previous one first) and
measure RPS, errors and recovery. Modes: graceful `stop` (SIGTERM + drain), `kill` (SIGKILL)
and `pause` (frozen process: the kernel still accepts TCP, nothing answers - the worst case
for detection, only the active health check can find it).
python bench/failover_series.py --output evidence/failover # ~5 min for 2 replicas x 3 modes
Timeline of one round (all times from the moment the stop command is issued):
warm-up 6 s -> stop replica -> 12 s with it down -> start it again -> wait until HAProxy
has every replica UP (rejoin time) -> 6 s steady state.
Measured per round: requests/ok/failed and ok-RPS before / during / after, the last failed
request (+s), the first moment after which the stopped replica never answered again
(= removal from rotation), time until the restarted replica is UP in HAProxy and serves
traffic again, plus the history check (no confirmed message lost, no duplicate).
Output: <output>/series.json (summary), <output>/timeline-<mode>-<replica>.csv (per second),
<output>/series.md (table for REPORT.md). No tokens are written.
"""
from __future__ import annotations
import argparse
import csv
import datetime as dt
import json
import platform
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
sys.path.insert(0, str(ROOT / "bench"))
import lbctl # noqa: E402
from loadgen import Load # noqa: E402
def docker(*args: str) -> str:
return subprocess.run(["docker", *args], capture_output=True, text=True, check=True, timeout=120).stdout
def one_round(replica: dict, mode: str, replicas: int, workers: int, out: Path) -> dict:
load = Load(workers).start()
time.sleep(6)
t_stop = load.now()
if mode == "pause":
docker("pause", replica["id"])
else:
docker("stop", "--time", "15" if mode == "stop" else "0", replica["id"])
t_stopped = load.now()
time.sleep(12)
t_start = load.now()
docker("unpause" if mode == "pause" else "start", replica["id"])
lbctl.wait_up(replicas, timeout=90)
t_up = load.now()
# The restarted container keeps its hostname: wait until it serves requests again.
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
if any(e[3] == replica["hostname"] and e[0] > t_up for e in load.events):
break
time.sleep(0.1)
t_serves = load.now()
time.sleep(6)
load.stop()
t_end = load.now()
failed_after_stop = [e for e in load.events if e[0] >= t_stop and e[2] not in (200, 201)]
served_by_target = [e[0] for e in load.events if e[3] == replica["hostname"] and t_stop <= e[0] < t_start]
out_of_rotation = (max(served_by_target) - t_stop) if served_by_target else 0.0
rows = load.per_second()
with (out / f"timeline-{mode}-{replica['name']}.csv").open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"])
writer.writeheader()
for row in rows:
row = dict(row)
row["instances"] = row["instances"].replace(replica["hostname"], "TARGET")
writer.writerow(row)
state = json.loads(docker("inspect", replica["id"]))[0]["State"]
return {
"mode": mode, "replica": replica["name"], "instance_id": replica["hostname"],
"stop_command_seconds": round(t_stopped - t_stop, 2),
"before": load.window(0.5, t_stop), "during": load.window(t_stop, t_start), "after": load.window(t_serves, t_end),
"failed_after_stop": len(failed_after_stop),
"failed_statuses": sorted({e[2] for e in failed_after_stop}),
"last_failure_after_stop_s": round(max((e[0] for e in failed_after_stop), default=t_stop) - t_stop, 3),
"target_last_served_after_stop_s": round(out_of_rotation, 3),
"rejoin_up_in_lb_s": round(t_up - t_start, 2), "rejoin_serving_s": round(t_serves - t_start, 2),
"client_retries_same_key": load.retries,
"history": load.verify_history(),
"restarted_state": f"{state['Status']} health={(state.get('Health') or {}).get('Status')}",
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--output", default="evidence/failover")
parser.add_argument("--modes", default="stop,kill,pause")
parser.add_argument("--workers", type=int, default=4)
args = parser.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
replicas = sorted((r for r in lbctl.replicas() if r["state"] == "running"), key=lambda r: r["name"])
assert len(replicas) >= 2, "need at least two running replicas"
lbctl.wait_up(len(replicas))
results = []
for mode in args.modes.split(","):
for replica in replicas: # each replica in turn; the previous one is already restored
print(f"== {mode} {replica['name']} ...", flush=True)
result = one_round(replica, mode, len(replicas), args.workers, out)
results.append(result)
print(json.dumps({k: result[k] for k in ("failed_after_stop", "last_failure_after_stop_s",
"target_last_served_after_stop_s", "rejoin_serving_s")}), flush=True)
meta = {"date": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
text=True).stdout.strip(),
"host": f"{platform.system()} {platform.release()}, docker {docker('version', '--format', '{{.Server.Version}}').strip()}",
"workload": f"{args.workers} closed-loop workers from the host via 127.0.0.1:8080: POST message "
"(Idempotency-Key, uncertain -> same key) + GET last page; client timeout 3 s",
"lb": "HAProxy: check GET /health/ready inter 1s fastinter 500ms fall 2 rise 2, timeout check 1s; "
"retry-on conn-failure + redispatch",
"api": "DRAIN_SECONDS=3, SHUTDOWN_TIMEOUT_SECONDS=8, stop_grace_period 15s"}
(out / "series.json").write_text(json.dumps({"meta": meta, "rounds": results}, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8")
lines = ["| Режим | Реплика | Запросов во время отказа | Ошибок после stop | ok-RPS до / во время / после | "
"Последняя ошибка, с | Цель ушла из ротации, с | Возврат в LB / обслуживает, с | Потери / дубли |",
"| --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
for r in results:
h = r["history"]
lines.append(f"| {r['mode']} | {r['replica']} | {r['during']['requests']} | {r['failed_after_stop']} "
f"{r['failed_statuses'] or ''} | {r['before']['ok_rps']} / {r['during']['ok_rps']} / {r['after']['ok_rps']} | "
f"{r['last_failure_after_stop_s']} | {r['target_last_served_after_stop_s']} | "
f"{r['rejoin_up_in_lb_s']} / {r['rejoin_serving_s']} | "
f"{h['lost_confirmed']} / {h['duplicate_ids'] + h['duplicate_texts']} |")
(out / "series.md").write_text("\n".join([f"Дата {meta['date']}, commit {meta['commit']}. {meta['workload']}.", "",
*lines, ""]), encoding="utf-8")
print("\n".join(lines))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+212
View File
@@ -0,0 +1,212 @@
#!/usr/bin/env python3
"""Background HTTP traffic through the LB for failover / rolling-update experiments (level 4).
Closed loop, N worker threads, each: POST a message (Idempotency-Key) + GET the last page.
An uncertain POST (timeout, connection error, 5xx) is retried with the SAME key and body,
exactly as a correct client must do; the history is checked afterwards for losses and
duplicates. Every request is recorded as (t, kind, status, instance, latency).
This is a correctness/availability probe, not a capacity benchmark (see bench/track1 for that).
python bench/loadgen.py --duration 30 --workers 4 # prints a summary JSON
"""
from __future__ import annotations
import argparse
import base64
import json
import math
import os
import threading
import time
import uuid
from collections import Counter, defaultdict
import urllib.error
import urllib.request
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
class Http:
"""Minimal stdlib client with keep-alive per thread (http.client under urllib is not)."""
def __init__(self, base_url: str, timeout: float):
import http.client
from urllib.parse import urlsplit
parts = urlsplit(base_url)
self.host, self.port = parts.hostname, parts.port or 80
self.timeout = timeout
self.factory = lambda: http.client.HTTPConnection(self.host, self.port, timeout=self.timeout)
self.conn = self.factory()
def request(self, method: str, path: str, body: bytes | None = None, headers: dict | None = None):
try:
self.conn.request(method, path, body=body, headers=headers or {})
response = self.conn.getresponse()
data = response.read()
if response.getheader("connection", "").lower() == "close":
self.conn.close()
self.conn = self.factory()
return response.status, dict(response.getheaders()), data
except Exception:
self.conn.close()
self.conn = self.factory()
raise
def call(method: str, path: str, body=None, headers=None, timeout=10.0):
data = json.dumps(body).encode() if body is not None else None
h = {"Content-Type": "application/json", **(headers or {})}
req = urllib.request.Request(BASE_URL + path, data=data, headers=h, method=method)
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.status, json.loads(r.read() or b"null")
except urllib.error.HTTPError as e:
return e.code, json.loads(e.read() or b"null")
def make_user(label: str) -> dict:
username = f"{label}_{uuid.uuid4().hex[:12]}"
password = "Load-password-" + uuid.uuid4().hex
status, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
assert status == 201, (status, user)
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
status, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert status == 201, (status, session)
return {"id": user["id"], "token": session["token"]}
class Load:
def __init__(self, workers: int = 4, timeout: float = 3.0, pause: float = 0.0):
self.workers = workers
self.timeout = timeout
self.pause = pause
self.alice, self.bob = make_user("load"), make_user("load")
status, chat = call("POST", "/api/v1/chats", {"title": "load", "member_ids": [self.alice["id"], self.bob["id"]]},
headers={"Authorization": "Bearer " + self.alice["token"]})
assert status == 201, chat
self.chat = chat
self.stop_event = threading.Event()
self.lock = threading.Lock()
self.events: list[tuple[float, str, int, str, float]] = []
self.confirmed: dict[str, dict] = {}
self.retries = 0
self.t0 = time.monotonic()
self.threads = [threading.Thread(target=self._worker, args=(i,), daemon=True) for i in range(workers)]
def now(self) -> float:
return time.monotonic() - self.t0
def _record(self, kind: str, status: int, instance: str, latency: float) -> None:
with self.lock:
self.events.append((round(self.now(), 4), kind, status, instance, round(latency * 1000, 2)))
def _worker(self, index: int) -> None:
http = Http(BASE_URL, self.timeout)
auth = {"Authorization": "Bearer " + self.alice["token"]}
path = f"/api/v1/chats/{self.chat['id']}/messages"
n, key = 0, uuid.uuid4().hex
text = f"w{index}-0-{key[:6]}"
while not self.stop_event.is_set():
started = time.monotonic()
try:
status, headers, data = http.request("POST", path, json.dumps({"text": text}).encode(),
{**auth, "Idempotency-Key": key, "Content-Type": "application/json"})
instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-"
except Exception:
status, instance, data = 0, "-", b""
self._record("POST", status, instance, time.monotonic() - started)
if status == 201:
msg = json.loads(data)
with self.lock:
self.confirmed[msg["id"]] = msg
n += 1
key = uuid.uuid4().hex
text = f"w{index}-{n}-{key[:6]}"
elif status in (0, 502, 503, 504, 500):
with self.lock:
self.retries += 1 # same key + same text on the next iteration
started = time.monotonic()
try:
status, headers, _ = http.request("GET", path + "?limit=5", None, auth)
instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-"
except Exception:
status, instance = 0, "-"
self._record("GET", status, instance, time.monotonic() - started)
if self.pause:
time.sleep(self.pause)
def start(self) -> "Load":
self.t0 = time.monotonic()
for t in self.threads:
t.start()
return self
def stop(self) -> None:
self.stop_event.set()
for t in self.threads:
t.join(timeout=15)
def verify_history(self) -> dict:
"""Walk the whole history with the second user: no confirmed message lost, no duplicate."""
items, cursor = [], None
while True:
q = "?limit=100" + (f"&cursor={cursor}" if cursor else "")
status, page = call("GET", f"/api/v1/chats/{self.chat['id']}/messages{q}",
headers={"Authorization": "Bearer " + self.bob["token"]}, timeout=30)
assert status == 200, page
items += page["items"]
cursor = page["next_cursor"]
if not cursor:
break
ids = [m["id"] for m in items]
texts = [m["text"] for m in items]
lost = [i for i in self.confirmed if i not in set(ids)]
changed = [m for m in items if m["id"] in self.confirmed and self.confirmed[m["id"]] != m]
return {"history_messages": len(items), "confirmed_messages": len(self.confirmed),
"lost_confirmed": len(lost), "altered_confirmed": len(changed),
"duplicate_ids": len(ids) - len(set(ids)), "duplicate_texts": len(texts) - len(set(texts)),
"unconfirmed_but_stored": len(set(ids) - set(self.confirmed))}
# ------------------------------------------------------------------ summaries
def window(self, start: float, end: float) -> dict:
ev = [e for e in self.events if start <= e[0] < end]
ok = [e for e in ev if e[2] in (200, 201)]
lat = sorted(e[4] for e in ok)
def pct(p: float):
return lat[max(0, math.ceil(p * len(lat)) - 1)] if lat else None
span = max(end - start, 1e-9)
return {"seconds": round(span, 2), "requests": len(ev), "ok": len(ok), "failed": len(ev) - len(ok),
"ok_rps": round(len(ok) / span, 1), "error_ratio": round((len(ev) - len(ok)) / len(ev), 5) if ev else None,
"statuses": dict(Counter(e[2] for e in ev)), "instances": dict(Counter(e[3] for e in ok)),
"p50_ms": pct(.5), "p95_ms": pct(.95), "p99_ms": pct(.99)}
def per_second(self) -> list[dict]:
buckets: dict[int, list] = defaultdict(list)
for e in self.events:
buckets[int(e[0])].append(e)
rows = []
for second in sorted(buckets):
ev = buckets[second]
rows.append({"second": second, "ok": sum(1 for e in ev if e[2] in (200, 201)),
"failed": sum(1 for e in ev if e[2] not in (200, 201)),
"instances": "|".join(f"{k}:{v}" for k, v in sorted(Counter(e[3] for e in ev).items()))})
return rows
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--duration", type=float, default=20)
parser.add_argument("--workers", type=int, default=4)
args = parser.parse_args()
load = Load(args.workers).start()
time.sleep(args.duration)
load.stop()
print(json.dumps({"total": load.window(0, load.now()), "history": load.verify_history(),
"client_retries_same_key": load.retries}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+156
View File
@@ -0,0 +1,156 @@
#!/usr/bin/env python3
"""Track 2 experiment: cost of one reconciler pass as a function of the number of attachments.
Hypothesis: a pass is O(N) with ~N/1000 S3 list requests + N/1000 DB pages and constant memory
(merge-join of two key-ordered streams); --verify-checksums adds reading every byte (O(total size)).
Method: synthetic, CONSISTENT attachments (row 'ready' + 1 KiB object with the matching size and
SHA-256, filename 'bench-reconcile') are added in steps up to the largest N; after each step the
reconciler runs in dry-run 3 times (median reported), plus page-size and checksum variants.
Everything synthetic is deleted at the end (objects first, then rows).
python bench/reconcile_cost.py --steps 1000 10000 50000 --output evidence/reconcile
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import platform
import statistics
import subprocess
import sys
import time
import uuid
from pathlib import Path
import httpx2 as httpx
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
PASSWORD = "Bench-password-Ω-12345"
OBJECT = bytes(range(256)) * 4 # 1 KiB, the same content for every object
SHA = hashlib.sha256(OBJECT).hexdigest()
PUT_SCRIPT = """
import asyncio, sys
from messenger.config import Settings, s3_settings
from messenger.storage import Storage
DATA = bytes(range(256)) * 4
async def main():
s = Storage(Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings()))
await s.open()
keys = sys.stdin.read().split()
sem = asyncio.Semaphore(32)
async def one(k):
async with sem:
await s.{op}(k{args})
await asyncio.gather(*(one(k) for k in keys))
await s.close()
print(len(keys))
asyncio.run(main())
"""
def compose(*args: str, stdin: str | None = None, timeout: float = 1800, check: bool = True) -> subprocess.CompletedProcess:
return subprocess.run(["docker", "compose", *args], cwd=ROOT, input=stdin, capture_output=True, text=True,
encoding="utf-8", timeout=timeout, check=check)
def psql(sql: str) -> str:
"""SQL through stdin: a 5000-row INSERT does not fit into a Windows command line (32 KiB)."""
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v",
"ON_ERROR_STOP=1", "-f", "-", stdin=sql).stdout.strip()
def s3_bulk(op: str, keys: list[str]) -> None:
script = PUT_SCRIPT.replace("{op}", op).replace("{args}", ", DATA, 'image/png', {}" if op == "put" else "")
compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "python", "reconciler", "-c", script, stdin="\n".join(keys))
def owner_and_chat() -> tuple[str, str]:
import base64
c = httpx.Client(base_url=BASE, timeout=30)
name = "rc_" + uuid.uuid4().hex[:10]
c.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
token = c.post("/api/v1/auth/sessions", headers={
"Authorization": "Basic " + base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()}).json()["token"]
me = c.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
chat = c.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
json={"title": "bench", "member_ids": [me["id"]]}).json()
return me["id"], chat["id"]
def add(n: int, owner: str, chat: str) -> None:
keys = ["o/" + uuid.uuid4().hex for _ in range(n)]
for i in range(0, n, 5000):
part = keys[i:i + 5000]
values = ",".join(f"('{uuid.uuid4()}','{k}')" for k in part)
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
"object_key, status, created_at, updated_at) "
f"SELECT v.id::uuid, '{chat}', '{owner}', 'bench-reconcile', 'image/png', {len(OBJECT)}, "
f"decode('{SHA}', 'hex'), 1, 1, v.k, 'ready', clock_timestamp() - interval '2 hours', "
f"clock_timestamp() - interval '2 hours' FROM (VALUES {values}) AS v(id, k)")
s3_bulk("put", keys)
def reconcile(*args: str) -> dict:
out = compose("run", "--rm", "--no-deps", "-T", "reconciler", *args).stdout
return json.loads(out)
def measure(label: str, args: list[str], repeats: int) -> dict:
runs = [reconcile(*args) for _ in range(repeats)]
durations = [r["duration_seconds"] for r in runs]
last = runs[-1]
return {"label": label, "args": args, "runs": repeats, "duration_s_median": statistics.median(durations),
"duration_s_all": durations, "objects": last["scanned"]["objects"], "rows": last["scanned"]["rows"],
"findings": {k: v for k, v in last["findings"].items() if v}, "cost": last["cost"]}
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--steps", type=int, nargs="+", default=[1000, 10000, 50000])
p.add_argument("--repeats", type=int, default=3)
p.add_argument("--output", default="evidence/reconcile")
args = p.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
owner, chat = owner_and_chat()
present = 0
results = []
steps = sorted(args.steps)
variant_step = steps[min(1, len(steps) - 1)]
try:
for target in sorted(args.steps):
started = time.perf_counter()
add(target - present, owner, chat)
present = target
print(f"populated {present} synthetic attachments in {time.perf_counter() - started:.1f} s", flush=True)
results.append({"synthetic": present, **measure("dry-run, page 1000", [], args.repeats)})
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
if present == variant_step: # page-size and checksum variants at one size
for label, extra in (("dry-run, page 100", ["--page-size", "100"]),
("dry-run, --verify-checksums", ["--verify-checksums"])):
results.append({"synthetic": present, **measure(label, extra, args.repeats)})
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
finally:
keys = psql("SELECT object_key FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = "
f"'{chat}'").split()
if keys:
s3_bulk("delete", keys)
psql(f"DELETE FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = '{chat}'")
print(f"removed {len(keys)} synthetic attachments", flush=True)
meta = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
text=True).stdout.strip(),
"object_bytes": len(OBJECT), "reconciler_limits": "cpus 1.0, memory 256m (compose)",
"note": "the bucket/table also contain the attachments of earlier test runs (counted in objects/rows)"}
(out / "cost.json").write_text(json.dumps({"meta": meta, "results": results}, ensure_ascii=False, indent=1),
encoding="utf-8")
return 0
if __name__ == "__main__":
sys.exit(main())
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env python3
"""Track 2 evidence: inject faults on both sides, run the reconciler, keep every JSON report.
Scenario A (all fault kinds; the same as student-tests/test_reconcile.py, but the reports are saved):
S3 side: old orphan objects, object deleted under a USED ready row, object truncated under an
unused ready row, same-size corruption (found only with --verify-checksums);
DB side: stale intents (with / without object), quarantine mark on a healthy row;
fresh: orphan + intent younger than --grace.
Runs: dry-run (+ snapshot of both stores before/after) -> apply -> apply again -> restore of the
lost object from a "backup" -> apply (quarantine lifted).
Scenario B (interrupted apply): 2000 old orphans, S3 is paused while --apply deletes them ->
the run stops with an error, nothing else is damaged; the next run finishes the rest; the
third run finds nothing.
python bench/reconcile_faults.py --output evidence/reconcile
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
from conftest import (Api, attachment_row, compose, findings_by_key, psql, s3_delete, s3_exec, s3_keys, # noqa: E402
s3_put)
from test_reconcile import insert_intent, snapshot # noqa: E402
GRACE = ["--grace", "30", "--allow-small-grace"]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
def reconcile(*args: str) -> tuple[int, dict]:
r = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "1000", *args, check=False, timeout=600)
return r.returncode, json.loads(r.stdout)
def save(out: Path, name: str, data) -> None:
(out / name).write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
def ours(report: dict, keys: dict[str, str]) -> dict:
"""The findings of this scenario's keys only (the stand also holds other test data)."""
f = findings_by_key(report)
return {label: ({"kind": f[k]["kind"], "result": f[k]["result"], "used": f[k]["used"]} if k in f else None)
for label, k in keys.items()}
def scenario_a(out: Path) -> dict:
api = Api()
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
up = {n: api.upload(alice, chat, imagegen.png(40 + i, 30, noise=True)).json() for i, n in enumerate("ABCDEF")}
data = {n: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
for n, a in up.items()}
key = {n: attachment_row(a["id"])["object_key"] for n, a in up.items()}
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
keys = {"healthy used A": key["A"], "healthy unused B": key["B"], "used, object deleted C": key["C"],
"unused, object truncated D": key["D"], "quarantined but healthy E": key["E"],
"same-size corruption F": key["F"]}
orphan = "o/" + uuid.uuid4().hex
stale_with, stale_without = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex
s3_put(orphan, SAMPLE)
s3_put(stale_with, SAMPLE)
insert_intent(chat, alice, stale_with, SAMPLE, 600)
insert_intent(chat, alice, stale_without, SAMPLE, 600)
s3_delete(key["C"])
s3_put(key["D"], data["D"][: len(data["D"]) // 2])
s3_put(key["F"], data["F"][:200] + bytes([data["F"][200] ^ 1]) + data["F"][201:])
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' WHERE id = '{up['E']['id']}'")
keys.update({"old orphan object": orphan, "stale intent + object": stale_with, "stale intent, no object": stale_without})
time.sleep(32)
fresh_orphan, fresh_intent = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex
s3_put(fresh_orphan, SAMPLE)
s3_put(fresh_intent, SAMPLE)
insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
keys.update({"fresh orphan (in-flight)": fresh_orphan, "fresh intent (in-flight)": fresh_intent})
before = snapshot()
_, dry = reconcile("--verify-checksums", *GRACE)
after = snapshot()
_, apply1 = reconcile("--apply", "--verify-checksums", *GRACE)
_, apply2 = reconcile("--apply", "--verify-checksums", *GRACE)
s3_put(key["C"], data["C"]) # the lost object restored from a backup
_, apply3 = reconcile("--apply", "--verify-checksums", *GRACE)
for name, report in (("faults-1-dry-run.json", dry), ("faults-2-apply.json", apply1),
("faults-3-apply-again.json", apply2), ("faults-4-after-restore.json", apply3)):
save(out, name, report)
rows = {label: attachment_row(up[n]["id"]) for label, n in (("C", "C"), ("D", "D"), ("E", "E"), ("F", "F"), ("A", "A"), ("B", "B"))}
content = {n: api.client.get(f"/api/v1/attachments/{up[n]['id']}/content",
headers={"Authorization": "Bearer " + bob["token"]}).status_code for n in "ABCDEF"}
present = s3_keys()
summary = {
"dry_run_changed_nothing": before == after,
"per_key": {"dry-run": ours(dry, keys), "apply": ours(apply1, keys), "apply again": ours(apply2, keys),
"after restore of C": ours(apply3, keys)},
"objects_present_at_end (fresh items were older than --grace by the later runs)": {label: k in present for label, k in keys.items()},
"rows_after": {k: (v and {"status": v["status"], "lost_reason": v["lost_reason"]}) for k, v in rows.items()},
"content_status_after": content,
"durations_s": [dry["duration_seconds"], apply1["duration_seconds"], apply2["duration_seconds"],
apply3["duration_seconds"]],
}
print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True)
return summary
def scenario_b(out: Path, count: int = 2000) -> dict:
keys = ["o/" + uuid.uuid4().hex for _ in range(count)]
s3_exec("ks = sys.stdin.read().split()\n"
"sem = asyncio.Semaphore(32)\n"
"async def one(k):\n"
" async with sem:\n"
" await s.put(k, b'orphan', 'image/png', {})\n"
"await asyncio.gather(*(one(k) for k in ks))", stdin="\n".join(keys))
time.sleep(32)
result: dict = {}
def run_apply() -> None:
result["code"], result["report"] = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE)
t = threading.Thread(target=run_apply)
t.start()
# pause S3 as soon as deletions are under way (the reconciler is planning/listing first)
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
left = len(s3_keys() & set(keys))
if left < count:
break
time.sleep(0.2)
subprocess.run(["docker", "compose", "pause", "s3"], cwd=ROOT, check=True, capture_output=True)
t.join()
subprocess.run(["docker", "compose", "unpause", "s3"], cwd=ROOT, check=True, capture_output=True)
time.sleep(5) # let S3 settle after unpause before counting
left_after_interrupt = len(s3_keys() & set(keys))
code2, second = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE)
code3, third = reconcile("--apply", *GRACE)
save(out, "interrupted-1-apply.json", result["report"])
save(out, "interrupted-2-rerun.json", second)
save(out, "interrupted-3-third.json", third)
summary = {
"orphans_injected": count,
"interrupted_run": {"exit_code": result["code"], "error": result["report"].get("error"),
"applied": result["report"]["actions"]["applied"]},
"orphans_left_after_interrupt": left_after_interrupt,
"rerun": {"exit_code": code2, "applied": second["actions"]["applied"], "error": second["error"]},
"third": {"exit_code": code3, "planned": third["actions"]["planned"]},
"orphans_left_at_end": len(s3_keys() & set(keys)),
}
print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True)
return summary
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--output", default="evidence/reconcile")
args = p.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
a = scenario_a(out)
b = scenario_b(out)
commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip()
save(out, "faults-summary.json", {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "commit": commit,
"grace_seconds": 30, "scenario_a": a, "scenario_b_interrupted": b})
return 0
if __name__ == "__main__":
sys.exit(main())
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env python3
"""Level 4: scripts/rolling-update.sh while bench/loadgen.py traffic runs through the LB.
python bench/rolling_under_load.py --output evidence/rolling-update
Writes <output>/summary.json (errors, RPS, history check, revisions before/after),
<output>/timeline.csv (per second ok/failed/instances) and <output>/script-output.txt.
"""
from __future__ import annotations
import argparse
import csv
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
sys.path.insert(0, str(ROOT / "bench"))
import lbctl # noqa: E402
from loadgen import Load # noqa: E402
def revisions() -> dict[str, str]:
result = {}
for r in lbctl.replicas():
if r["state"] == "running":
label = subprocess.run(["docker", "inspect", "-f", '{{index .Config.Labels "org.opencontainers.image.revision"}}',
r["id"]], capture_output=True, text=True).stdout.strip()
result[r["name"]] = f"{r['hostname']} revision={label}"
return result
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--output", default="evidence/rolling-update")
parser.add_argument("--workers", type=int, default=4)
args = parser.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
before = revisions()
load = Load(args.workers).start()
time.sleep(5)
t_begin = load.now()
bash = (os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe")
if os.name == "nt" else "sh")
env = {**os.environ, "PYTHON": sys.executable}
script = subprocess.run([bash, str(ROOT / "scripts" / "rolling-update.sh")], cwd=ROOT, env=env,
capture_output=True, text=True, encoding="utf-8", timeout=900)
t_end = load.now()
time.sleep(5)
load.stop()
(out / "script-output.txt").write_text(f"$ sh scripts/rolling-update.sh (exit {script.returncode})\n"
+ script.stdout + script.stderr, encoding="utf-8")
with (out / "timeline.csv").open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"])
writer.writeheader()
writer.writerows(load.per_second())
summary = {"script_exit_code": script.returncode, "update_seconds": round(t_end - t_begin, 1),
"before": before, "after": revisions(),
"during_update": load.window(t_begin, t_end), "whole_run": load.window(0, load.now()),
"client_retries_same_key": load.retries, "history": load.verify_history(),
"workload": f"{args.workers} closed-loop workers: POST (Idempotency-Key) + GET page via 127.0.0.1:8080"}
(out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({k: summary[k] for k in ("script_exit_code", "update_seconds", "before", "after", "history")},
ensure_ascii=False, indent=2))
print(json.dumps(summary["during_update"], ensure_ascii=False))
return 0 if script.returncode == 0 else 1
if __name__ == "__main__":
raise SystemExit(main())
+244
View File
@@ -0,0 +1,244 @@
#!/usr/bin/env python3
"""Level 4 experiment: peak memory of the API replicas under >= 10 simultaneous uploads.
Hypothesis: the memory of one replica is bounded by UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES (bodies
waiting for a slot stay in TCP buffers, not in the process) + one image check at a time
(IMAGE_VERIFY_CONCURRENCY), independent of how many clients upload at once.
Method (repeatable, the stand must be up):
* for every scenario the api replicas are RECREATED with the scenario's settings, so the cgroup
counter memory.peak starts from an idle process (it never decreases during a container's life);
* idle memory.current of each replica is read; then N clients upload at the same moment through
the LB (a barrier releases them together), each a different file;
* during the run memory.current of every replica is sampled every 50 ms inside the container;
afterwards memory.peak (cgroup v2: all processes of the container incl. djpeg) and VmHWM of
PID 1 (the Python server alone) are read;
* the normal configuration is restored at the end.
python bench/upload_memory.py --output evidence/memory
"""
from __future__ import annotations
import argparse
import io
import json
import os
import platform
import statistics
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
import httpx2 as httpx
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
PASSWORD = "Bench-password-Ω-12345"
MiB = 1024 * 1024
def sh(*args: str, env: dict | None = None, timeout: float = 300) -> str:
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True, timeout=timeout,
env={**os.environ, **(env or {})}).stdout
def replicas() -> list[str]:
return sh("docker", "compose", "ps", "-q", "api").split()
def cg(cid: str, name: str) -> int:
return int(sh("docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}").split()[0])
def memory_stat(cid: str, key: str) -> int:
for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat").splitlines():
name, value = line.split()
if name == key:
return int(value)
return 0
def vmhwm(cid: str) -> int:
for line in sh("docker", "exec", cid, "cat", "/proc/1/status").splitlines():
if line.startswith("VmHWM:"):
return int(line.split()[1]) * 1024
return 0
def container_events() -> dict[str, str]:
"""restart count + OOM-kill events per container (cgroup memory.events oom_kill counter)."""
out = {}
for cid in sh("docker", "compose", "ps", "-q").split():
name, restarts = sh("docker", "inspect", "-f", "{{.Name}} {{.RestartCount}}", cid).split()
events = dict(line.split() for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.events").splitlines())
out[name.lstrip("/")] = f"restarts={restarts} oom_kill={events.get('oom_kill', '?')}"
return out
def recreate(env: dict) -> None:
sh("docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "--wait", "api", env=env, timeout=300)
sh(sys.executable, "scripts/lbctl.py", "wait-up", str(len(replicas())), "--timeout", "90") # LB sees them UP
time.sleep(3)
class Sampler(threading.Thread):
"""memory.current of one container every 50 ms, read by a shell loop inside it."""
def __init__(self, cid: str):
super().__init__(daemon=True)
self.cid = cid
self.values: list[int] = []
self.proc = subprocess.Popen(["docker", "exec", cid, "sh", "-c",
"while :; do cat /sys/fs/cgroup/memory.current; sleep 0.05; done"],
stdout=subprocess.PIPE, text=True)
def run(self) -> None:
for line in self.proc.stdout:
if line.strip().isdigit():
self.values.append(int(line))
def stop(self) -> None:
self.proc.kill()
def actor(client: httpx.Client) -> tuple[str, str]:
name = "mem_" + uuid.uuid4().hex[:10]
client.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
import base64
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
token = client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}).json()["token"]
me = client.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
chat = client.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
json={"title": "mem", "member_ids": [me["id"]]}).json()
return token, chat["id"]
def big_jpeg(width: int, height: int, seed: int) -> bytes:
"""Large progressive JPEG (smooth content keeps it < 10 MiB): the costliest file to check."""
image = Image.linear_gradient("L").resize((width, height)).convert("RGB")
image.putpixel((seed % width, 0), (seed % 255, 0, 0))
out = io.BytesIO()
image.save(out, "JPEG", quality=90, progressive=True, subsampling=0)
return out.getvalue()
def run_scenario(name: str, env: dict, files: list[tuple[bytes, str]]) -> dict:
recreate(env)
ids = replicas()
idle = {cid: cg(cid, "memory.current") for cid in ids}
client = httpx.Client(base_url=BASE, timeout=120)
token, chat_id = actor(client)
barrier = threading.Barrier(len(files))
results: list[tuple[int, float, str]] = [None] * len(files) # type: ignore[list-item]
def upload(i: int, data: bytes, ctype: str) -> None:
c = httpx.Client(base_url=BASE, timeout=120)
barrier.wait()
started = time.perf_counter()
r = c.post(f"/api/v1/chats/{chat_id}/attachments?filename=f{i}", content=data,
headers={"Authorization": "Bearer " + token, "Content-Type": ctype})
reason = r.json()["error"]["message"] if r.status_code != 201 else ""
results[i] = (r.status_code, time.perf_counter() - started, r.headers.get("x-instance-id", "-"), reason)
c.close()
s3_id = sh("docker", "compose", "ps", "-q", "s3").strip()
restarts_before = container_events()
samplers = [Sampler(cid) for cid in ids] + [Sampler(s3_id)]
for s in samplers:
s.start()
time.sleep(0.5)
threads = [threading.Thread(target=upload, args=(i, d, t)) for i, (d, t) in enumerate(files)]
wall = time.perf_counter()
for t in threads:
t.start()
for t in threads:
t.join()
wall = time.perf_counter() - wall
time.sleep(0.5)
for s in samplers:
s.stop()
per_replica = {}
for cid, s in zip(ids, samplers[:-1]):
hostname = sh("docker", "inspect", "-f", "{{.Config.Hostname}}", cid).strip()
per_replica[hostname] = {
"idle_mib": round(idle[cid] / MiB, 1),
"peak_mib": round(cg(cid, "memory.peak") / MiB, 1),
"sampled_max_mib": round(max(s.values or [0]) / MiB, 1),
"python_vmhwm_mib": round(vmhwm(cid) / MiB, 1),
"uploads_served": sum(1 for r in results if r[2] == hostname),
"limit_mib": round(cg(cid, "memory.max") / MiB, 1),
}
per_replica[hostname]["peak_minus_idle_mib"] = round(per_replica[hostname]["peak_mib"]
- per_replica[hostname]["idle_mib"], 1)
lat = sorted(r[1] for r in results if r[0] == 201)
codes: dict[str, int] = {}
reasons: dict[str, int] = {}
for r in results:
codes[str(r[0])] = codes.get(str(r[0]), 0) + 1
if r[3]:
reasons[r[3]] = reasons.get(r[3], 0) + 1
summary = {
"scenario": name, "settings": env, "clients": len(files),
"s3_sampled_max_mib": round(max(samplers[-1].values or [0]) / MiB, 1),
"s3_anon_after_mib": round(memory_stat(s3_id, "anon") / MiB, 1),
"s3_limit_mib": round(cg(s3_id, "memory.max") / MiB, 1),
# OOM kills / restarts DURING the scenario (any container of the stand): must be empty
"incidents": {k: v for k, v in container_events().items() if v != restarts_before.get(k)},
"file_mib_each": round(statistics.mean(len(d) for d, _ in files) / MiB, 2),
"types": sorted({t for _, t in files}), "status_codes": codes, "error_reasons": reasons, "wall_seconds": round(wall, 2),
"latency_s": {"p50": round(lat[len(lat) // 2], 2), "max": round(lat[-1], 2)} if lat else None,
"replicas": per_replica,
}
print(json.dumps(summary, ensure_ascii=False), flush=True)
return summary
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--clients", type=int, default=16)
p.add_argument("--output", default="evidence/memory")
args = p.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
print("generating files ...", flush=True)
pngs = [(imagegen.png_of_size(int(9.5 * MiB)), "image/png") for _ in range(2 * args.clients)]
jpegs = [(big_jpeg(8000, 5000, i), "image/jpeg") for i in range(args.clients)]
base = {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}
n, n2 = args.clients, 2 * args.clients
scenarios = [
(f"png-9.4MiB x{n}, slots=4 (default)", base, pngs[:n]),
(f"png-9.4MiB x{n2}, slots=4 (default)", base, pngs),
(f"png-9.4MiB x{n2}, slots=16 (bound raised)", {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, pngs),
(f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=1 (default)", base, jpegs),
(f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=4", {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, jpegs),
]
results = []
sh("docker", "compose", "restart", "s3") # S3 memory counters start from an idle process
sh("docker", "compose", "up", "-d", "--wait", "s3")
try:
for name, env, files in scenarios:
results.append(run_scenario(name, env, files))
finally:
recreate({}) # back to the compose defaults
meta = {
"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
"commit": sh("git", "rev-parse", "--short", "HEAD").strip(),
"api_limits": sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}",
replicas()[0]).strip(),
"jpeg_file_mib": round(len(jpegs[0][0]) / MiB, 2), "png_file_mib": round(len(pngs[0][0]) / MiB, 2),
}
(out / "summary.json").write_text(json.dumps({"meta": meta, "scenarios": results}, ensure_ascii=False, indent=1),
encoding="utf-8")
print(json.dumps(meta, ensure_ascii=False))
return 0
if __name__ == "__main__":
sys.exit(main())
+338
View File
@@ -0,0 +1,338 @@
# Lab 4 stand: client -> HAProxy (lb) -> API x N (one scalable service `api`)
# -> PostgreSQL (metadata, source of truth) + Valkey (sessions) + SeaweedFS S3 (bytes).
# From a clean clone (details and Windows notes: REPORT.md, docs/WINDOWS.md):
# python scripts/init_secrets.py # once: random local credentials into ./secrets (git-ignored)
# cp .env.example .env # optional overrides (API_REPLICAS, limits, ...)
# docker compose --env-file .env up -d --build --wait
# Order: postgres/valkey/s3 healthy -> migrate + s3-init (ONE run each, exit 0) -> api healthy -> lb.
# Images are pinned by version AND digest. Passwords/keys are passed only as secret files, and
# every service gets only the secrets it needs. Only the LB publishes ports (loopback only).
# Three networks: `edge` (lb <-> api), `data` (api/migrate/reconciler <-> postgres/valkey) and
# `storage` (api/s3-init/reconciler <-> s3); data and storage are internal (no route outside).
name: mtusi-lab4
x-api-env: &api-env
APP_ENV: ${APP_ENV:-lab}
LOG_LEVEL: ${LOG_LEVEL:-info}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: ${SESSION_STORE:-valkey}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
DB_POOL_MIN: ${DB_POOL_MIN:-1}
DB_POOL_MAX: ${DB_POOL_MAX:-10}
DB_TIMEOUT_SECONDS: ${DB_TIMEOUT_SECONDS:-3}
CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.5}
PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-2}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
# Graceful stop: 3 s drain (readiness 503, still serving) + up to 8 s for in-flight requests.
DRAIN_SECONDS: ${DRAIN_SECONDS:-3}
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-8}
DOCS_ENABLED: ${DOCS_ENABLED:-true}
MESSAGE_WRITE_PATH: ${MESSAGE_WRITE_PATH:-function}
# Lab 4: S3 (the API's own Read+Write key pair, see infra/seaweedfs/start.sh) and upload limits.
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-4}
UPLOAD_QUEUE_TIMEOUT_SECONDS: ${UPLOAD_QUEUE_TIMEOUT_SECONDS:-5}
UPLOAD_READ_TIMEOUT_SECONDS: ${UPLOAD_READ_TIMEOUT_SECONDS:-30}
IMAGE_VERIFY_TIMEOUT_SECONDS: ${IMAGE_VERIFY_TIMEOUT_SECONDS:-10}
IMAGE_VERIFY_CONCURRENCY: ${IMAGE_VERIFY_CONCURRENCY:-1}
JPEG_MAX_MEMORY_MB: ${JPEG_MAX_MEMORY_MB:-256}
PENDING_GRACE_SECONDS: ${PENDING_GRACE_SECONDS:-900}
JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-60}
# Level 4 failure experiments only (compose.faults.yaml); empty = no injected failures.
FAULT_POINTS: ${FAULT_POINTS:-}
services:
postgres:
image: postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
# pg_stat_statements: per-statement counters for the experiments.
command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"]
secrets: [postgres_password]
volumes:
- pgdata:/var/lib/postgresql # PostgreSQL 18 image keeps PGDATA under this path
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -q"]
interval: 2s
timeout: 3s
retries: 30
start_period: 10s
stop_grace_period: 30s # clean shutdown checkpoint instead of crash recovery
restart: unless-stopped
networks: [data]
deploy:
resources:
limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1g}"}
valkey:
image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
# The password goes into a config include readable only by the valkey user,
# so it never appears in the process arguments. docker-entrypoint.sh drops root.
command:
- sh
- -c
- >-
umask 077 &&
printf 'requirepass %s\n' "$$(cat /run/secrets/valkey_password)" > /tmp/requirepass.conf &&
chown valkey /tmp/requirepass.conf &&
exec docker-entrypoint.sh valkey-server /etc/valkey/valkey.conf
secrets: [valkey_password]
volumes:
- ./infra/valkey/valkey.conf:/etc/valkey/valkey.conf:ro
- valkeydata:/data
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/valkey_password)\" valkey-cli ping | grep -q PONG"]
interval: 2s
timeout: 3s
retries: 30
start_period: 5s
restart: unless-stopped
networks: [data]
deploy:
resources:
limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m}
# Lab 4: S3-compatible object storage (SeaweedFS: master + volume + filer + S3 gateway in one
# process). Bytes of the attachments live ONLY here, on the named volume s3data. Identities
# and exposure: infra/seaweedfs/start.sh. Not published on the host; its web UIs listen on
# 127.0.0.1 inside the container only.
s3:
image: chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62
entrypoint: ["/bin/sh", "/etc/seaweedfs/start.sh"]
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
# Go heap target below the container limit: the GC returns memory before the cgroup OOM.
GOMEMLIMIT: ${S3_GOMEMLIMIT:-600MiB}
secrets:
- s3_admin_access_key_id
- s3_admin_secret_access_key
- s3_api_access_key_id
- s3_api_secret_access_key
- s3_reconciler_access_key_id
- s3_reconciler_secret_access_key
volumes:
- ./infra/seaweedfs/start.sh:/etc/seaweedfs/start.sh:ro
- s3data:/data
healthcheck:
# Unauthenticated health endpoint of the S3 gateway (200 once it serves requests).
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"]
interval: 2s
timeout: 3s
retries: 30
start_period: 10s
stop_grace_period: 30s
restart: unless-stopped
networks: [storage]
deploy:
resources:
limits: {cpus: "${S3_CPUS:-1.0}", memory: "${S3_MEMORY:-1g}"}
# One-shot schema migration: runs ONCE per `up`, before any API replica starts
# (api depends_on service_completed_successfully). Replicas never migrate themselves.
migrate:
build:
context: .
args:
BUILD_REVISION: ${BUILD_REVISION:-dev}
image: ${API_IMAGE:-mtusi-messenger:lab4}
command: ["python", "-m", "messenger.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
depends_on:
postgres:
condition: service_healthy
healthcheck:
disable: true
restart: "no"
networks: [data]
deploy:
resources:
limits: {cpus: "0.5", memory: 256m}
# One-shot, idempotent S3 bootstrap: bucket, sentinel object, verification of the access model
# (private bucket, least-privilege keys). The ONLY holder of the admin key pair besides s3.
s3-init:
image: ${API_IMAGE:-mtusi-messenger:lab4}
command: ["python", "-m", "messenger.s3_bootstrap"]
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key
S3_RECONCILER_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id
S3_RECONCILER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key
secrets:
- s3_admin_access_key_id
- s3_admin_secret_access_key
- s3_api_access_key_id
- s3_api_secret_access_key
- s3_reconciler_access_key_id
- s3_reconciler_secret_access_key
depends_on:
s3:
condition: service_healthy
migrate:
condition: service_completed_successfully # same image: built once by `migrate`
healthcheck:
disable: true
restart: "no"
networks: [storage]
deploy:
resources:
limits: {cpus: "0.5", memory: 256m}
api:
build:
context: .
args:
BUILD_REVISION: ${BUILD_REVISION:-dev}
image: ${API_IMAGE:-mtusi-messenger:lab4}
environment: *api-env
secrets:
- database_url
- cache_url
- source: s3_api_access_key_id
target: s3_access_key_id
- source: s3_api_secret_access_key
target: s3_secret_access_key
# No `ports` and no container_name: replicas are reachable only through `lb`,
# and `--scale api=N` / deploy.replicas works. X-Instance-Id = container hostname.
expose: ["8080"]
healthcheck:
test: ["CMD", "python", "-m", "messenger.healthcheck"]
interval: 2s
timeout: 2s
retries: 3
start_period: 5s
depends_on:
postgres:
condition: service_healthy
valkey:
condition: service_healthy
s3:
condition: service_healthy
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
stop_grace_period: 15s # > DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (11 s)
restart: unless-stopped
networks: [edge, data, storage]
deploy:
replicas: ${API_REPLICAS:-2}
resources:
# 512m: UPLOAD_CONCURRENCY (4) x 10 MiB bodies + one djpeg (<= JPEG_MAX_MEMORY_MB) + runtime;
# measured in evidence/memory/ (bench/upload_memory.py).
limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512m}"}
# No swap on top of the limit (Docker's default would allow as much swap again): the RAM bound
# of the experiments is a real bound, and running out of it is visible, not hidden in swap.
memswap_limit: ${API_MEMORY:-512m}
lb:
image: haproxy:3.2.26-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
# The ONLY public entry point, on the loopback interfaces only. Both IPv4 and IPv6:
# on Windows `localhost` resolves to ::1 first, and a refused ::1 connection costs ~2 s
# per request before the fallback to 127.0.0.1 (measured in lab 3: 2.0 s vs 0.007 s).
- "127.0.0.1:${APP_PUBLISH_PORT:-8080}:8080"
- "[::1]:${APP_PUBLISH_PORT:-8080}:8080"
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404" # local status page (UP/DOWN of replicas)
healthcheck:
# Config is valid and the process answers; whether any API is ready is the LB's own job.
test: ["CMD-SHELL", "haproxy -c -q -f /usr/local/etc/haproxy/haproxy.cfg && wget -q -O /dev/null http://127.0.0.1:8404/"]
interval: 5s
timeout: 3s
retries: 3
depends_on:
api:
condition: service_healthy
restart: unless-stopped
networks: [edge]
deploy:
resources:
limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m}
# Track 2 (lab 4): reconciler of S3 objects vs PostgreSQL metadata. Not a daemon: started on
# demand, dry-run by default (read-only on both stores):
# docker compose run --rm reconciler # report only
# docker compose run --rm reconciler --apply # fix: orphans, stale intents, quarantine
reconciler:
profiles: ["tools"]
image: ${API_IMAGE:-mtusi-messenger:lab4}
entrypoint: ["python", "-m", "messenger.reconcile"]
command: []
environment:
DATABASE_URL_FILE: /run/secrets/database_url
DB_TIMEOUT_SECONDS: ${RECONCILE_DB_TIMEOUT_SECONDS:-10}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3}
secrets:
- database_url
- source: s3_reconciler_access_key_id
target: s3_access_key_id
- source: s3_reconciler_secret_access_key
target: s3_secret_access_key
depends_on:
postgres:
condition: service_healthy
s3:
condition: service_healthy
restart: "no"
networks: [data, storage]
deploy:
resources:
limits: {cpus: "${RECONCILER_CPUS:-1.0}", memory: 256m}
networks:
edge:
data:
internal: true # no route outside: stores talk only to api/migrate
storage:
internal: true # S3 API reachable only by api, s3-init, reconciler
volumes:
pgdata:
valkeydata:
s3data:
secrets:
postgres_password:
file: ./secrets/postgres_password
valkey_password:
file: ./secrets/valkey_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_admin_access_key_id:
file: ./secrets/s3_admin_access_key_id
s3_admin_secret_access_key:
file: ./secrets/s3_admin_secret_access_key
s3_api_access_key_id:
file: ./secrets/s3_api_access_key_id
s3_api_secret_access_key:
file: ./secrets/s3_api_secret_access_key
s3_reconciler_access_key_id:
file: ./secrets/s3_reconciler_access_key_id
s3_reconciler_secret_access_key:
file: ./secrets/s3_reconciler_secret_access_key
+116
View File
@@ -0,0 +1,116 @@
# HAProxy 3.2 LTS — the single public entry point of the lab 4 stand (compose service `lb`).
#
# Backend discovery: Docker DNS name `api` returns one A record per RUNNING replica of the
# compose service `api` (deploy.replicas / --scale). server-template keeps 8 slots and
# re-resolves every second, so scale up/down, a restarted container with a new IP and a
# full stack restart are picked up without restarting the LB. A stopped container drops
# out of DNS and its slot goes to MAINT.
#
# Readiness: an ACTIVE check `GET /health/ready` (PostgreSQL + Valkey + S3 reachable, not
# draining) every 1 s; 2 failures -> DOWN, 2 successes -> UP. Passive errors do not decide
# anything on their own, and there are NO fallback servers without checks: a replica that
# answers /health/live but not /health/ready never gets traffic while a fully ready one exists.
#
# Degraded mode (lab 4): S3 is a SHARED dependency. If it is down, /health/ready is 503 on
# every replica and backend `api` has no server. Instead of answering 503 to everything, the
# frontend then switches to backend `api_core`: the same replicas, checked by
# /health/ready/core (PostgreSQL + Valkey, not draining). Chats and text messages keep working,
# every attachment request is answered 503 by the API itself (fast, S3_TIMEOUT_SECONDS).
# A replica that alone lost S3 (e.g. its storage network) is DOWN in `api` while the others are
# UP, so it gets no traffic at all.
# Worst case detection of a failed replica (a check answered at once counts at once, a check
# that hangs counts only after `timeout check`):
# fast explicit 503 (drain after SIGTERM): inter 1 s + fastinter 0.5 s ~= 1.5 s;
# slow ready (PostgreSQL/Valkey unreachable: the answer takes ~1 s = timeout check)
# or frozen process: inter 1 s + 1 s + fastinter 0.5 s + 1 s ~= 3.5 s.
# Requests that could not even be SENT (connection refused/reset before the request)
# are retried on another replica (retry-on conn-failure + redispatch) - safe for POST,
# because the API has not seen them. Nothing that may have reached the API is retried by
# the LB: an uncertain POST is retried by the CLIENT with the same Idempotency-Key.
# No sticky sessions: sessions live in Valkey, every replica serves every user.
global
log stdout format raw local0 info
maxconn 4096
# Runtime API for `show servers state` / `show stat` (scripts and tests, no network port).
stats socket /var/lib/haproxy/admin.sock mode 600 level admin
hard-stop-after 15s
resolvers docker
nameserver dns1 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold other 1s
hold refused 1s
hold nx 1s
hold timeout 1s
hold obsolete 1s
defaults
mode http
log global
option httplog
option dontlognull
option http-keep-alive
timeout connect 1s
timeout client 30s
timeout server 10s # > DB_TIMEOUT_SECONDS (3 s): the API answers 503 itself first
timeout http-request 10s
timeout http-keep-alive 10s
timeout queue 3s
timeout check 1s
retries 2
retry-on conn-failure
option redispatch 1
# Errors generated by the LB itself, in the contract error format with the same request id.
# They carry no X-Instance-Id: no API replica produced them. 502/504 = the outcome of the
# request is unknown (a replica died mid-request / did not answer in time): retry with the
# same Idempotency-Key. 503 = no ready replica at all.
http-error status 502 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API-реплика оборвала ответ; повторите с тем же Idempotency-Key\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]"
http-error status 503 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет готовых API-реплик\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]"
http-error status 504 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API-реплика не ответила вовремя; повторите с тем же Idempotency-Key\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]"
frontend public
bind :8080
# One request id end to end: keep a valid client X-Request-Id, otherwise create one; the
# API echoes it, and errors generated by the LB itself carry the same value.
http-request set-header X-Request-Id %[uuid()] unless { req.hdr(x-request-id) -m reg ^[A-Za-z0-9._:-]{1,128}$ }
http-request set-var(txn.request_id) req.hdr(x-request-id)
log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.request_id)] \"%HM %HPO\""
use_backend api if { nbsrv(api) gt 0 }
default_backend api_core
backend api
balance roundrobin
# Uploads (<= 10 MiB body + verification + S3 PUT): the API's own budget is
# UPLOAD_READ_TIMEOUT 30 s + IMAGE_VERIFY 10 s + S3; the LB must not cut it at 10 s (a 504
# after a successful upload would make the client upload a duplicate).
acl is_upload path_reg ^/api/v1/chats/[^/]+/attachments$
http-request set-timeout server 60s if is_upload METH_POST
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 maxconn 200
server-template api 8 api:8080
# Degraded mode only (see the header): used when NO replica is fully ready.
backend api_core
balance roundrobin
acl is_upload path_reg ^/api/v1/chats/[^/]+/attachments$
http-request set-timeout server 60s if is_upload METH_POST
option httpchk
http-check send meth GET uri /health/ready/core ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 maxconn 200
server-template core 8 api:8080
# Local-only status page (127.0.0.1:8404 on the host): which replica is UP/DOWN right now.
frontend stats
bind :8404
stats enable
stats uri /
stats refresh 2s
stats show-legends
+45
View File
@@ -0,0 +1,45 @@
#!/bin/sh
# Entry point of the compose service `s3` (SeaweedFS 4.46, one container: master + volume +
# filer + S3 gateway). Renders the S3 identities from Docker secrets, then execs `weed`.
#
# Identities (SeaweedFS s3.json, actions scoped to the bucket):
# bootstrap-admin Admin (create bucket) -> only s3-init has these keys
# messenger-api Read + Write on S3_BUCKET -> API replicas (no List, no Admin)
# messenger-reconciler Read + Write + List on S3_BUCKET -> reconciler CLI (needs the listing)
# There is NO anonymous identity, so every unsigned request is refused: the bucket is private.
#
# Exposure: master (9333), volume (8080), filer (8888) with their web UIs and gRPC listen on
# 127.0.0.1 INSIDE the container only (-ip.bind); only the S3 API port 8333 listens on the
# container network `storage` (internal, not published on the host). Telemetry is off.
set -eu
umask 077
secret() { tr -d '\r\n' < "/run/secrets/$1"; }
bucket="${S3_BUCKET:?S3_BUCKET is required}"
cat > /tmp/s3.json <<EOF
{
"identities": [
{"name": "bootstrap-admin",
"credentials": [{"accessKey": "$(secret s3_admin_access_key_id)", "secretKey": "$(secret s3_admin_secret_access_key)"}],
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
{"name": "messenger-api",
"credentials": [{"accessKey": "$(secret s3_api_access_key_id)", "secretKey": "$(secret s3_api_secret_access_key)"}],
"actions": ["Read:${bucket}", "Write:${bucket}"]},
{"name": "messenger-reconciler",
"credentials": [{"accessKey": "$(secret s3_reconciler_access_key_id)", "secretKey": "$(secret s3_reconciler_secret_access_key)"}],
"actions": ["Read:${bucket}", "Write:${bucket}", "List:${bucket}"]}
]
}
EOF
# Memory: by default SeaweedFS buffers every in-flight upload without a limit and the Go runtime
# keeps freed heap; measured ~1 GB under 16 parallel 9.4 MiB PUTs (evidence/memory). The upload
# limits make excess PUTs wait inside SeaweedFS instead of growing the heap, and GOMEMLIMIT (set
# in compose.yaml below the container limit) makes the Go GC return memory before the cgroup OOM.
exec weed server \
-dir=/data \
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
-master.telemetry=false \
-master.volumeSizeLimitMB=1024 -volume.max=0 \
-filer -filer.disableDirListing -filer.concurrentUploadLimitMB=128 \
-volume.concurrentUploadLimitMB=128 \
-s3 -s3.ip.bind=0.0.0.0 -s3.port=8333 -s3.config=/tmp/s3.json -s3.concurrentUploadLimitMB=128 \
-s3.iam=false -s3.autoCreateBucket=false -s3.port.iceberg=0 -s3.port.lance=0
+17
View File
@@ -0,0 +1,17 @@
# Valkey is the single store of sessions (SESSION_STORE=valkey), so it must be durable.
# AOF with fsync on every write: a confirmed login or logout survives a crash/restart,
# a logout can never be lost and "resurrect" a token. Sessions are small and rare
# writes, so the fsync cost is acceptable here (see REPORT.md).
appendonly yes
appendfsync always
appenddirname "appendonlydir"
# RDB snapshots are not needed on top of AOF.
save ""
dir /data
port 6379
protected-mode yes
# Sessions are never evicted silently: if memory is full, writes fail and login returns 503.
maxmemory 128mb
maxmemory-policy noeviction
# Password, generated by scripts/init_secrets.py and written here at container start.
include /tmp/requirepass.conf
+65
View File
@@ -0,0 +1,65 @@
-- 0001: initial schema of the messenger (lab 2). Design rationale: docs/DATA_MODEL.md
-- Applied once by `python -m messenger.migrate`; the version is recorded in schema_migrations.
-- User-supplied free text (display_name, title, message text) is stored as UTF-8 in BYTEA:
-- the contract allows U+0000, and PostgreSQL TEXT cannot contain NUL bytes.
-- Lengths are limited in code points by the API; the CHECKs below are byte-level safety nets
-- (one code point is at most 4 bytes in UTF-8).
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL,
display_name bytea NOT NULL,
password_hash text NOT NULL, -- Argon2id PHC string, never the password
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT users_username_key UNIQUE (username),
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_len CHECK (octet_length(display_name) BETWEEN 1 AND 400)
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title bytea NOT NULL,
created_by uuid NOT NULL REFERENCES users (id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
-- Per-chat clock: created_at of the newest message. Updating this row serialises writers
-- of one chat, so message timestamps strictly increase in commit order (see DATA_MODEL.md).
last_message_at timestamptz,
CONSTRAINT chats_title_len CHECK (octet_length(title) BETWEEN 1 AND 400)
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id),
user_id uuid NOT NULL REFERENCES users (id),
position smallint NOT NULL, -- order of member_ids sent by the creator
chat_created_at timestamptz NOT NULL, -- copy of chats.created_at for keyset listing
PRIMARY KEY (chat_id, user_id),
CONSTRAINT chat_members_position_key UNIQUE (chat_id, position)
);
-- GET /chats of one user: (user_id, created_at, id) range scan, no sort.
CREATE INDEX chat_members_user_keyset ON chat_members (user_id, chat_created_at, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL,
sender_id uuid NOT NULL,
text bytea NOT NULL,
created_at timestamptz NOT NULL,
-- The sender must be a member of this chat: enforced by the database, not only by the API.
CONSTRAINT messages_sender_is_member FOREIGN KEY (chat_id, sender_id)
REFERENCES chat_members (chat_id, user_id),
CONSTRAINT messages_text_len CHECK (octet_length(text) BETWEEN 1 AND 16000)
);
-- History page: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n
-- becomes an index range scan that starts exactly at the cursor.
CREATE INDEX messages_chat_keyset ON messages (chat_id, created_at, id);
-- Sessions for SESSION_STORE=postgres (the default store is Valkey; see docs/DATA_MODEL.md).
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY, -- SHA-256 of the token; the token is not stored
user_id uuid NOT NULL REFERENCES users (id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32)
);
CREATE INDEX sessions_expires_at ON sessions (expires_at);
+25
View File
@@ -0,0 +1,25 @@
-- 0002 (lab 3): shared idempotency of POST /chats/{chat_id}/messages.
-- Additive only (new table), so the lab 2 API keeps working while this runs: it is safe
-- to apply before a rolling update of the API replicas (see REPORT.md, "Миграции").
--
-- Scope of a key (contracts/README.md): current user + HTTP method + chat path + key.
-- The method (POST) and the path template are fixed for this table, so the scope
-- is (user_id, chat_id, key). The key row, the message and the result are written in
-- ONE transaction (repo.post_message): there is no "pending" state that a crash could
-- leave behind, and the stored result is the message row itself.
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users (id),
chat_id uuid NOT NULL REFERENCES chats (id),
key text NOT NULL,
request_hash bytea NOT NULL, -- SHA-256 of the canonical JSON body
-- Inserted before the message in the same transaction, hence checked at COMMIT.
message_id uuid NOT NULL REFERENCES messages (id) DEFERRABLE INITIALLY DEFERRED,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (user_id, chat_id, key),
CONSTRAINT idempotency_keys_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
CONSTRAINT idempotency_keys_hash_len CHECK (octet_length(request_hash) = 32),
CONSTRAINT idempotency_keys_message_key UNIQUE (message_id)
);
-- Housekeeping of keys older than IDEMPOTENCY_TTL_SECONDS (>= 24 h).
CREATE INDEX idempotency_keys_created_at ON idempotency_keys (created_at);
+50
View File
@@ -0,0 +1,50 @@
-- 0003 (lab 3, track 1 improvement): the whole idempotent "send message" as ONE statement.
-- Additive (a new function), so replicas of the previous version keep working while it is
-- applied, and the previous write path (MESSAGE_WRITE_PATH=multi) stays usable for A/B runs.
--
-- Same semantics and the same SQL as repo.post_message (multi-statement path), but executed
-- inside the server: the API sends one call instead of ~11 round trips in two transactions
-- (membership check, BEGIN, key claim, chat clock, message, COMMIT, pool checks). Called in
-- autocommit mode, so the function call IS the transaction: key row, chat clock and message
-- commit together or not at all. VOLATILE + READ COMMITTED: every statement inside takes a
-- fresh snapshot, so after waiting on a concurrent claim of the same key the stored result
-- of the winner is visible, exactly as in the multi-statement path.
--
-- o_status: created | replayed | conflict | not_found | retry (key vanished in between: cleanup)
CREATE FUNCTION post_message(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid,
p_text bytea, p_ttl_seconds integer)
RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz)
LANGUAGE plpgsql VOLATILE AS $$
DECLARE
v_created timestamptz;
BEGIN
PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user;
IF NOT FOUND THEN
RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz;
RETURN;
END IF;
INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id)
VALUES (p_user, p_chat, p_key, p_hash, p_message)
ON CONFLICT (user_id, chat_id, key) DO UPDATE
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp()
WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds);
IF FOUND THEN
-- Per-chat clock under the chat row lock (see migrations/0001 and docs/DATA_MODEL.md).
UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond')
WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
VALUES (p_message, p_chat, p_user, p_text, v_created);
RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created;
RETURN;
END IF;
RETURN QUERY
SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key;
IF NOT FOUND THEN
RETURN QUERY SELECT 'retry'::text, NULL::uuid, NULL::bytea, NULL::timestamptz;
END IF;
END
$$;
+145
View File
@@ -0,0 +1,145 @@
-- 0004 (lab 4): attachments. Metadata live here, the bytes in the private S3 bucket.
-- Additive (new tables, a relaxed CHECK, a NEW function post_message_v2), so replicas of the
-- lab 3 version keep working while it is applied (they never send attachment_ids).
--
-- Consistency between the two stores (docs/CONSISTENCY.md):
-- 1. INSERT the row as status='pending' (intent: the server-generated object_key is known
-- BEFORE anything is written to S3) and COMMIT;
-- 2. PUT the object into S3;
-- 3. UPDATE status='ready' WHERE status='pending'; only then 201.
-- Invariant: an object never exists without a row that names its key (row first, object
-- second; on cleanup object first, row second). A crash/outage between the steps leaves a
-- 'pending' row; the janitor deletes its object (idempotent) and then the row once it is older
-- than PENDING_GRACE_SECONDS. The reconciler (python -m messenger.reconcile) checks the
-- invariant over the whole bucket.
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL,
owner_id uuid NOT NULL,
filename text NOT NULL, -- display name only, never a path or a key
content_type text NOT NULL, -- detected from the bytes, not from the header
size_bytes integer NOT NULL,
sha256 bytea NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
-- Random server-generated key ("o/" + 128 random bits in hex), unrelated to id/filename.
-- COLLATE "C": byte order = S3 ListObjectsV2 order, the reconciler merge-joins both lists.
object_key text COLLATE "C" NOT NULL,
status text NOT NULL DEFAULT 'pending',
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
-- Set by the reconciler when the object of a READY row is missing or has the wrong size:
-- the row is quarantined (content -> 404, cannot be attached), never deleted, and the mark
-- is removed again if the object comes back (restore from backup).
lost_at timestamptz,
lost_reason text,
-- The uploader must be a member of the chat (enforced by the database too).
CONSTRAINT attachments_owner_is_member FOREIGN KEY (chat_id, owner_id)
REFERENCES chat_members (chat_id, user_id),
CONSTRAINT attachments_object_key_key UNIQUE (object_key),
CONSTRAINT attachments_object_key_format CHECK (object_key ~ '^o/[0-9a-f]{32}$'),
CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready')),
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256_len CHECK (octet_length(sha256) = 32),
CONSTRAINT attachments_pixels CHECK (width >= 1 AND height >= 1 AND width::bigint * height <= 40000000),
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
CONSTRAINT attachments_lost CHECK ((lost_at IS NULL) = (lost_reason IS NULL) AND (lost_at IS NULL OR status = 'ready'))
);
-- Janitor: stale intents, oldest first; tiny partial index (only rows of unfinished uploads).
CREATE INDEX attachments_pending_created ON attachments (created_at) WHERE status = 'pending';
-- attachment_ids of a message, twice on purpose (both written in the message's transaction):
-- * messages.attachment_ids (ordered array): the history page reads it from the same row, so the
-- keyset range scan of messages stays ONE index scan, without a per-row sub-query;
-- * message_attachments: foreign keys (a referenced attachment can never be deleted) and the
-- reverse lookup "is this attachment used?" (janitor/reconciler) by an index on attachment_id.
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
ALTER TABLE messages ADD CONSTRAINT messages_attachment_ids_len CHECK (cardinality(attachment_ids) <= 10);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages (id),
position smallint NOT NULL,
attachment_id uuid NOT NULL REFERENCES attachments (id),
PRIMARY KEY (message_id, position),
CONSTRAINT message_attachments_unique UNIQUE (message_id, attachment_id),
CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9)
);
-- "Is this attachment used by any message?" (janitor/reconciler never touch used ones).
CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id);
-- Lab 4: text may be "" when the message has attachments (the API enforces "text or files").
-- DROP CONSTRAINT takes a short exclusive lock on messages and the new CHECK scans the table
-- once, inside this migration's transaction (seconds for a lab-sized table; for a big table
-- this would be ADD ... NOT VALID + VALIDATE in a separate migration).
ALTER TABLE messages DROP CONSTRAINT messages_text_len;
ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (octet_length(text) <= 16000);
-- post_message_v2: post_message (0003) + attachment_ids, still ONE statement and ONE transaction.
-- The attachments are checked BEFORE the key is claimed and the key is claimed only if they are
-- fine, so a failed check never leaves a key behind. A replay (same key, same body) returns the
-- stored message even if an attachment was quarantined since: the result of a key is final.
--
-- o_status: created | replayed | conflict | not_found | retry | attachment_not_found | attachment_conflict
CREATE FUNCTION post_message_v2(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid,
p_text bytea, p_ttl_seconds integer, p_attachments uuid[])
RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz, o_attachments uuid[])
LANGUAGE plpgsql VOLATILE AS $$
DECLARE
v_created timestamptz;
v_found integer;
v_usable integer;
v_check text := 'ok';
BEGIN
PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user;
IF NOT FOUND THEN
RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
RETURN;
END IF;
IF cardinality(p_attachments) > 0 THEN
-- Visible = ready attachment of THIS chat (pending, foreign or unknown -> 404).
-- Usable = visible and not quarantined (lost -> 409, like a failed attachment).
-- FOR SHARE: a concurrent quarantine waits until this message is committed.
SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL) INTO v_found, v_usable
FROM (SELECT a.lost_at FROM attachments a
WHERE a.id = ANY (p_attachments) AND a.chat_id = p_chat AND a.status = 'ready'
FOR SHARE) a;
IF v_found <> cardinality(p_attachments) THEN
v_check := 'attachment_not_found';
ELSIF v_usable <> v_found THEN
v_check := 'attachment_conflict';
END IF;
END IF;
IF v_check = 'ok' THEN
INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id)
VALUES (p_user, p_chat, p_key, p_hash, p_message)
ON CONFLICT (user_id, chat_id, key) DO UPDATE
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp()
WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds);
IF FOUND THEN
UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond')
WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created;
INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
VALUES (p_message, p_chat, p_user, p_text, v_created, p_attachments);
INSERT INTO message_attachments (message_id, position, attachment_id)
SELECT p_message, (a.ord - 1)::smallint, a.id
FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord);
RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created, p_attachments;
RETURN;
END IF;
END IF;
RETURN QUERY
SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at,
m.attachment_ids
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key;
IF NOT FOUND THEN
RETURN QUERY SELECT CASE WHEN v_check = 'ok' THEN 'retry' ELSE v_check END,
NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
END IF;
END
$$;
+6
View File
@@ -0,0 +1,6 @@
# Student tests (in-process, via Starlette TestClient). Runtime deps come from requirements.txt.
pytest==9.1.1
httpx2==2.13.1
jsonschema==4.26.0
# Lab 4 tests: generate PNG/JPEG fixtures (valid, corrupt, oversized) on the fly.
pillow==12.3.0
+20
View File
@@ -0,0 +1,20 @@
# Direct runtime dependencies. The pinned lock with hashes is requirements.txt:
# docker run --rm -v "$PWD:/w" -w /w python:3.13.16-slim-bookworm \
# sh -c "pip install -q pip-tools==7.6.1 && pip-compile --generate-hashes --strip-extras --no-emit-index-url -o requirements.txt requirements.in"
starlette==1.7.0
uvicorn==0.54.0
# Fast C parser/event loop used by uvicorn in the container (Linux only).
httptools==0.9.0
uvloop==0.23.0 ; sys_platform != "win32"
# PostgreSQL driver (libpq bundled in the binary wheel) and its async connection pool.
psycopg==3.3.6
psycopg-binary==3.3.6
psycopg-pool==3.3.3
# Valkey client (Valkey speaks the Redis protocol).
redis==8.1.0
# Argon2id password KDF.
argon2-cffi==25.1.0
# Lab 4: S3 client (async botocore over aiohttp; requests are cancellable, unlike libpq cancel).
aiobotocore==3.9.2
# Images are checked without Pillow: PNG by src/messenger/images.py, JPEG by libjpeg-turbo
# `djpeg -strict` (Debian package libjpeg-turbo-progs, see Dockerfile).
+1356
View File
File diff suppressed because it is too large Load Diff
+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env python3
"""Run every check of REPORT.md on the current checkout and save the real output in evidence/.
python scripts/collect_evidence.py # ~60 min; the stand must NOT be running yet
Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the
stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the
lab 4 experiments (memory under concurrent uploads, reconciler faults and cost), and records
container states before/after every hook. Prints no secrets
(hook output is shown by the public tests as suppressed; our scripts print names and states only).
On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin.
"""
from __future__ import annotations
import datetime as dt
import json
import os
import shutil
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / "evidence"
WINDOWS = os.name == "nt"
HOOK = ".cmd" if WINDOWS else ".sh"
PYTHON = sys.executable
VENV_PY = str(ROOT / ".venv" / ("Scripts/python.exe" if WINDOWS else "bin/python"))
# Makefile recipes use $(PYTHON) unquoted: pass interpreters WITHOUT spaces in the path.
MAKE_PY = "python" if WINDOWS else "python3"
MAKE_VENV = ".venv/Scripts/python.exe" if WINDOWS else ".venv/bin/python"
ENV = {**os.environ, "PYTHONUTF8": "1"}
if WINDOWS:
ENV["PATH"] = ENV["PATH"] + os.pathsep + r"C:\Program Files\Git\usr\bin"
def now() -> str:
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def sh(cmd: list[str], timeout: float = 1800, env: dict | None = None) -> tuple[int, str]:
started = time.monotonic()
result = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace",
timeout=timeout, env={**ENV, **(env or {})})
took = time.monotonic() - started
return result.returncode, (result.stdout + result.stderr).rstrip() + f"\n[exit {result.returncode}, {took:.1f} s]\n"
class Log:
def __init__(self, name: str, title: str):
self.path = EVIDENCE / name
self.path.parent.mkdir(parents=True, exist_ok=True)
commit = subprocess.run(["git", "log", "-1", "--format=%h %s"], cwd=ROOT, capture_output=True, text=True,
encoding="utf-8").stdout.strip()
self.text = f"# {title}\n# {now()}, commit {commit}\n"
def run(self, cmd: list[str], shown: str | None = None, **kwargs) -> int:
code, out = sh(cmd, **kwargs)
self.text += f"\n$ {shown or ' '.join(cmd)}\n{out}"
self.save()
print(f" {self.path.name}: {shown or ' '.join(cmd[:4])} -> exit {code}", flush=True)
return code
def note(self, text: str) -> None:
self.text += f"\n{text}\n"
self.save()
def save(self) -> None:
self.path.write_text(self.text, encoding="utf-8")
def states() -> list[str]:
ids = subprocess.run(["docker", "compose", "ps", "-a", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
rows = []
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True).stdout or "[]"):
st = item["State"]
rows.append(f"{item['Name'].lstrip('/'):24} host={item['Config']['Hostname']:13} {st['Status']:8} "
f"exit={st['ExitCode']:<3} pid={st['Pid']:<6} started={st['StartedAt'][:23]}")
return sorted(rows)
def main() -> int:
EVIDENCE.mkdir(exist_ok=True)
make = shutil.which("make", path=ENV["PATH"]) or "make"
# 1. Clean clone -> secrets -> build -> start
log = Log("clean-clone.txt", "Clean clone: secrets, config, build, migrations from zero, start, versions")
log.run(["git", "status", "--short", "--branch"])
log.run([PYTHON, "--version"]); log.run(["docker", "version", "--format", "client {{.Client.Version}} / server {{.Server.Version}}"])
log.run(["docker", "compose", "version"]); log.run([make, "--version"], shown="make --version")
log.run([PYTHON, "scripts/init_secrets.py"], shown="python scripts/init_secrets.py")
if not (ROOT / ".env").exists():
shutil.copy(ROOT / ".env.example", ROOT / ".env")
log.note("$ cp .env.example .env")
log.run(["docker", "compose", "--env-file", ".env", "config", "--quiet"])
log.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait"], timeout=1800)
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Image}}\t{{.Status}}"])
log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"])
log.run(["docker", "compose", "logs", "--no-log-prefix", "s3-init"]) # bucket + access-model verdicts
log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"])
# venv for validate / own tests
if not Path(VENV_PY).exists():
log.run([PYTHON, "-m", "venv", ".venv"])
log.run([VENV_PY, "-m", "pip", "install", "-q", "--require-hashes", "--no-deps", "-r", "requirements.txt"], timeout=1800)
log.run([VENV_PY, "-m", "pip", "install", "-q", "-r", "requirements-test.txt", "-r", "requirements-dev.txt"], timeout=1800)
# 2. make check / validate / test
log = Log("make-check-validate-test.txt", "make check, make validate (venv), make test")
log.run([make, "check", f"PYTHON={MAKE_PY}"], shown="make check")
log.run([make, "validate", f"PYTHON={MAKE_VENV}"], shown=f"make validate PYTHON={MAKE_VENV}")
log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test")
# 3. persistence with states before/after
log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey, S3")
log.note("Containers BEFORE:\n" + "\n".join(states()))
log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"],
shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}")
log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey, s3; volumes kept):\n" + "\n".join(states()))
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
# 4. failover: until BOTH replicas have been the stopped one (each run is a real public test)
log = Log("test-failover.txt", "make test-failover with scripts/stop-one: which container is really stopped, restore, repeat")
stopped: set[str] = set()
import urllib.request
for attempt in range(1, 7):
# The public test stops the replica that served its first POST; its request sequence is
# fixed, so with round robin it hits the same replica every time. Shift the rotation by
# (attempt - 1) extra requests so that the other replica becomes the target as well.
for _ in range(attempt - 1):
urllib.request.urlopen("http://127.0.0.1:8080/health/live", timeout=5).read()
before = {r for r in states() if "-api-" in r and " running " in r}
log.note(f"--- run {attempt}; running API replicas BEFORE:\n" + "\n".join(sorted(before)))
log.run([make, "test-failover", f"ACTION_SCRIPT=scripts/stop-one{HOOK}", f"PYTHON={MAKE_PY}"],
shown=f"make test-failover ACTION_SCRIPT=scripts/stop-one{HOOK}")
after = states()
gone = [r for r in after if "-api-" in r and " exited " in r]
log.note("AFTER (the stopped replica, left stopped by the hook):\n" + "\n".join(gone))
log.run([PYTHON, "scripts/lbctl.py", "status"])
stopped.update(r.split()[0] for r in gone)
log.run(["sh" if not WINDOWS else r"C:\Program Files\Git\bin\bash.exe", "scripts/restore-api.sh"],
shown="sh scripts/restore-api.sh")
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
if len(stopped) >= 2:
break
log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}")
# 5. own tests
log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally")
log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v",
timeout=3600)
# 6. lab 4 experiments (level 4 memory, track 2 reconciler) and the public load script
log = Log("lab4-runs.txt", "Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py")
log.run([VENV_PY, "bench/upload_memory.py", "--output", "evidence/memory"], timeout=3600)
log.run([VENV_PY, "bench/reconcile_faults.py", "--output", "evidence/reconcile"], timeout=1800)
log.run([VENV_PY, "bench/reconcile_cost.py", "--output", "evidence/reconcile"], timeout=3600)
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"])
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"])
print("done:", EVIDENCE)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+127
View File
@@ -0,0 +1,127 @@
#!/usr/bin/env python3
"""Defense scenario, steps 1-2 (README): upload the fixture, send a message with attachment_ids,
download as the second member and compare SHA-256; the outsider gets 404; an unauthenticated
request straight to S3 (from inside the storage network) is refused, the signed one works.
python scripts/demo_attachments.py [--base-url http://127.0.0.1:8080]
Prints requests and answers; never prints tokens or keys. Creates synthetic users.
"""
from __future__ import annotations
import argparse
import base64
import hashlib
import json
import subprocess
import sys
import urllib.error
import urllib.request
import uuid
from email.message import Message
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
PASSWORD = "Demo-password-Ω-12345"
class Client:
def __init__(self, base: str):
self.base = base.rstrip("/")
def call(self, method: str, path: str, *, token: str | None = None, body=None, raw: bytes | None = None,
headers: dict | None = None) -> tuple[int, Message, bytes]:
h = dict(headers or {})
if token:
h["Authorization"] = "Bearer " + token
data = raw
if body is not None:
data = json.dumps(body, ensure_ascii=False).encode()
h["Content-Type"] = "application/json"
req = urllib.request.Request(self.base + path, data=data, method=method, headers=h)
try:
with urllib.request.urlopen(req, timeout=30) as r:
return r.status, r.headers, r.read() # case-insensitive HTTPMessage
except urllib.error.HTTPError as e:
return e.code, e.headers, e.read()
def show(label: str, status: int, headers: Message, payload: bytes, *, binary: bool = False) -> None:
inst = headers.get("X-Instance-Id", "-")
if binary:
text = f"{len(payload)} bytes, Content-Type {headers.get('Content-Type')}"
else:
text = payload.decode("utf-8", "replace")[:300]
print(f" {label:58} -> {status} [X-Instance-Id {inst}] {text}")
def user(c: Client, label: str) -> dict:
name = f"{label}_{uuid.uuid4().hex[:8]}"
status, _, body = c.call("POST", "/api/v1/users", body={"username": name, "display_name": label.title(),
"password": PASSWORD})
assert status == 201, body
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
status, _, body = c.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
session = json.loads(body)
return {"id": session["user"]["id"], "token": session["token"], "name": name}
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--base-url", default="http://127.0.0.1:8080")
args = p.parse_args()
c = Client(args.base_url)
content = FIXTURE.read_bytes()
print(f"fixture {FIXTURE.name}: {len(content)} bytes, sha256 {hashlib.sha256(content).hexdigest()}")
alice, bob, eve = user(c, "alice"), user(c, "bob"), user(c, "eve")
print(f"users: alice, bob (members), eve (outsider)")
status, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
body={"title": "Демо lab4", "member_ids": [alice["id"], bob["id"]]})
chat = json.loads(body)
print("\n1. upload -> message -> download by the second member")
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],
raw=content, headers={"Content-Type": "image/png"})
show("POST /chats/{id}/attachments (alice, raw PNG)", st, h, b)
att = json.loads(b)
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
show("POST /chats/{id}/messages text='' attachment_ids=[id]", st, h, b)
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"])
show("GET /attachments/{id} (bob)", st, h, b)
st, h, got = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
show("GET /attachments/{id}/content (bob)", st, h, got, binary=True)
same = hashlib.sha256(got).hexdigest() == hashlib.sha256(content).hexdigest() == att["sha256"]
print(f" SHA-256 of the download == fixture == metadata: {same}")
print("\n2. outsider and direct S3 access")
for suffix in ("", "/content"):
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", token=eve["token"])
show(f"GET /attachments/{{id}}{suffix} (eve, not a member)", st, h, b)
key = subprocess.run(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
"-At", "-c", f"SELECT object_key FROM attachments WHERE id = '{att['id']}'"],
cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
print(f" object key in PostgreSQL: {key} (random; not in any API response: "
f"{key not in json.dumps(att)})")
probe = (
"import urllib.request as u, urllib.error as e, asyncio\n"
"def st(url):\n"
" try: return u.urlopen(url, timeout=3).status\n"
" except e.HTTPError as x: return x.code\n"
f"print('anonymous GET http://s3:8333/messenger-attachments/{key} ->', st('http://s3:8333/messenger-attachments/{key}'))\n"
"print('anonymous LIST http://s3:8333/messenger-attachments?list-type=2 ->', st('http://s3:8333/messenger-attachments?list-type=2'))\n"
"from messenger.config import Settings\nfrom messenger.storage import Storage\n"
"async def m():\n s = Storage(Settings.from_env()); await s.open()\n"
f" print('signed HEAD with the API key pair ->', (await s.head('{key}')).size, 'bytes'); await s.close()\n"
"asyncio.run(m())\n")
out = subprocess.run(["docker", "compose", "exec", "-T", "api", "python", "-c", probe], cwd=ROOT,
capture_output=True, text=True).stdout
for line in out.splitlines():
print(" (inside network storage) " + line)
print(" S3 is not published on the host at all: only the LB has ports (127.0.0.1:8080, 8404).")
return 0 if same else 1
if __name__ == "__main__":
sys.exit(main())
+114
View File
@@ -0,0 +1,114 @@
#!/usr/bin/env python3
"""EXPLAIN (ANALYZE, BUFFERS) of the history query on >= 10 000 messages (level 4).
python scripts/explain_history.py --output evidence/explain-history.txt
Creates (once, idempotent) a synthetic chat with 20 000 messages plus 30 000 messages in
30 other chats, then explains the EXACT statements of src/messenger/repo.py:
1. first page (HISTORY_FIRST)
2. late page via cursor (HISTORY_AFTER) — generic plan forced, as a prepared statement
reused by the driver would get it
3. the rejected variant `($2 IS NULL OR (created_at, id) > ...)` — generic plan
4. the rejected OFFSET pagination for the same late page
Runs psql inside the postgres container; no database port is published.
"""
from __future__ import annotations
import argparse
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
from messenger.repo import HISTORY_AFTER, HISTORY_FIRST # noqa: E402 (the app's real SQL)
CHAT = "00000000-0000-4000-8000-00000000e002"
USER = "00000000-0000-4000-8000-00000000e001"
TARGET = 20_000
LATE = 19_950 # cursor position: the 50 newest messages remain
SEED = f"""
\\set ON_ERROR_STOP on
BEGIN;
INSERT INTO users (id, username, display_name, password_hash)
VALUES ('{USER}', 'explain_seed', convert_to('Explain seed', 'UTF8'), 'not-a-login') ON CONFLICT DO NOTHING;
INSERT INTO chats (id, title, created_by, created_at)
SELECT ('00000000-0000-4000-8000-0000000' || lpad(to_hex(n), 5, '0'))::uuid, convert_to('explain ' || n, 'UTF8'),
'{USER}', timestamptz '2026-01-01 00:00:00+00'
FROM generate_series(57346::int, 57376::int) AS n -- e002 .. e020: target + 30 noise chats
ON CONFLICT DO NOTHING;
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
SELECT id, '{USER}', 0, created_at FROM chats WHERE created_by = '{USER}' ON CONFLICT DO NOTHING;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), '{CHAT}', '{USER}', convert_to('synthetic message ' || g, 'UTF8'),
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
FROM generate_series(1::int, {TARGET}::int) AS g
WHERE NOT EXISTS (SELECT 1 FROM messages WHERE chat_id = '{CHAT}');
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), c.id, '{USER}', convert_to('noise ' || g, 'UTF8'),
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
FROM chats c CROSS JOIN generate_series(1::int, 1000::int) AS g
WHERE c.created_by = '{USER}' AND c.id <> '{CHAT}'
AND NOT EXISTS (SELECT 1 FROM messages m WHERE m.chat_id = c.id);
COMMIT;
ANALYZE messages;
"""
def pg(sql_param: str) -> str:
"""Turn psycopg %s placeholders into $1..$n for a PREPARE statement."""
parts = sql_param.split("%s")
return "".join(p + (f"${i + 1}" if i < len(parts) - 1 else "") for i, p in enumerate(parts))
EXPLAIN = f"""
\\set ON_ERROR_STOP on
\\pset pager off
SELECT count(*) AS messages_total, count(*) FILTER (WHERE chat_id = '{CHAT}') AS messages_in_target_chat FROM messages;
SELECT created_at AS cur_ts, id AS cur_id FROM messages WHERE chat_id = '{CHAT}'
ORDER BY created_at, id OFFSET {LATE - 1} LIMIT 1 \\gset
SET plan_cache_mode = force_generic_plan;
PREPARE first_page(uuid, int) AS {pg(HISTORY_FIRST)};
PREPARE after_page(uuid, timestamptz, uuid, int) AS {pg(HISTORY_AFTER)};
PREPARE or_page(uuid, timestamptz, uuid, int) AS
SELECT id, sender_id, text, created_at FROM messages
WHERE chat_id = $1 AND ($2 IS NULL OR (created_at, id) > ($2, $3)) ORDER BY created_at, id LIMIT $4;
PREPARE offset_page(uuid, int, int) AS
SELECT id, sender_id, text, created_at FROM messages WHERE chat_id = $1 ORDER BY created_at, id OFFSET $2 LIMIT $3;
\\echo '=== 1. first page, app query HISTORY_FIRST, generic plan'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE first_page('{CHAT}', 51);
\\echo '=== 2. late page (cursor at {LATE}), app query HISTORY_AFTER, generic plan'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE after_page('{CHAT}', :'cur_ts', :'cur_id', 51);
\\echo '=== 3. REJECTED: one query with ($2 IS NULL OR ...), same late page, generic plan'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE or_page('{CHAT}', :'cur_ts', :'cur_id', 51);
\\echo '=== 4. REJECTED: OFFSET pagination, same late page'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE offset_page('{CHAT}', {LATE}, 51);
"""
def psql(script: str) -> str:
result = subprocess.run(
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-f", "-"],
cwd=ROOT, input=script, capture_output=True, text=True, encoding="utf-8", check=True)
return result.stdout
def run() -> str:
psql(SEED)
return psql(EXPLAIN)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--output", default=None)
args = parser.parse_args()
text = run()
print(text)
if args.output:
Path(args.output).write_text(text, encoding="utf-8", newline="\n")
return 0
if __name__ == "__main__":
sys.exit(main())
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""Create local synthetic credentials for the stand in ./secrets (ignored by Git).
python scripts/init_secrets.py
Idempotent: existing files are kept, so re-running never changes passwords of a
database that already has data. Prints only file names, never values.
Delete ./secrets together with the volumes (`docker compose down -v`) to start over.
"""
from __future__ import annotations
import os
import secrets
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRETS = ROOT / "secrets"
def write(name: str, value: str) -> None:
path = SECRETS / name
if path.exists():
print(f"keep secrets/{name}")
return
path.write_text(value + "\n", encoding="utf-8", newline="\n")
# Readable by the container users (postgres, valkey, seaweedfs, app uid 10001) through the
# compose bind mount. Local synthetic lab credentials only.
os.chmod(path, 0o644)
print(f"created secrets/{name}")
def read(name: str) -> str:
return (SECRETS / name).read_text(encoding="utf-8").strip()
def main() -> int:
SECRETS.mkdir(exist_ok=True)
write("postgres_password", secrets.token_urlsafe(24)) # [A-Za-z0-9_-]: safe inside URLs
write("valkey_password", secrets.token_urlsafe(24))
write("database_url", f"postgresql://messenger:{read('postgres_password')}@postgres:5432/messenger")
write("cache_url", f"redis://:{read('valkey_password')}@valkey:6379/0")
# Lab 4: three separate S3 key pairs (SeaweedFS identities, infra/seaweedfs/start.sh):
# admin = bucket bootstrap only; api = Read+Write on the bucket; reconciler = + List.
for who in ("admin", "api", "reconciler"):
write(f"s3_{who}_access_key_id", "AK" + secrets.token_hex(9).upper())
write(f"s3_{who}_secret_access_key", secrets.token_hex(20))
return 0
if __name__ == "__main__":
sys.exit(main())
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env python3
"""What the load balancer thinks right now, mapped to real containers.
python scripts/lbctl.py status # table: HAProxy slot, IP, UP/DOWN/MAINT, container, X-Instance-Id
python scripts/lbctl.py wait-up 2 # wait until exactly 2 API replicas are UP in HAProxy
python scripts/lbctl.py replicas # api containers of this compose project: name, hostname, state
Reads HAProxy's runtime API (`show servers state api`) through `docker compose exec lb socat`,
so no admin port is published. Container hostname = X-Instance-Id of that replica.
Prints no secrets. Used by student-tests and bench scripts as a library as well.
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
# srv_op_state of HAProxy (src/server.h): 0 stopped, 1 starting, 2 running, 3 stopping.
OP_STATE = {0: "DOWN", 1: "STARTING", 2: "UP", 3: "STOPPING"}
ADMIN_RMAINT = 0x20 # slot without a resolved address (no running container behind it)
def run(*args: str, timeout: float = 60) -> str:
result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding="utf-8",
timeout=timeout)
if result.returncode != 0:
raise RuntimeError(f"{' '.join(args[:3])}... failed: {result.stderr.strip()[:300]}")
return result.stdout
def replicas(all_states: bool = True) -> list[dict]:
"""API containers of this compose project (running and stopped)."""
ids = run("docker", "compose", "ps", "-q", *(["-a"] if all_states else []), "api").split()
if not ids:
return []
data = json.loads(run("docker", "inspect", *ids))
result = []
for item in data:
networks = item["NetworkSettings"]["Networks"] or {}
result.append({
"id": item["Id"][:12],
"name": item["Name"].lstrip("/"),
"hostname": item["Config"]["Hostname"],
"state": item["State"]["Status"],
"health": (item["State"].get("Health") or {}).get("Status", "-"),
"exit_code": item["State"]["ExitCode"],
"started_at": item["State"]["StartedAt"],
"pid": item["State"]["Pid"],
"ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")),
# compose network name without the project prefix -> IP, e.g. {"edge": ..., "data": ...}
"networks": {name.split("_", 1)[-1]: n["IPAddress"] for name, n in networks.items() if n.get("IPAddress")},
})
return sorted(result, key=lambda r: r["name"])
def lb_servers() -> list[dict]:
raw = run("docker", "compose", "exec", "-T", "lb", "sh", "-c",
'echo "show servers state api" | socat stdio /var/lib/haproxy/admin.sock')
by_ip = {ip: r for r in replicas() for ip in r["ips"]}
servers = []
for line in raw.splitlines():
parts = line.split()
if len(parts) < 7 or not parts[0].isdigit():
continue
name, addr, op, admin = parts[3], parts[4], int(parts[5]), int(parts[6])
if admin & ADMIN_RMAINT or addr == "-":
state = "MAINT"
else:
state = OP_STATE.get(op, str(op))
container = by_ip.get(addr)
servers.append({"slot": name, "addr": addr, "state": state,
"container": container["name"] if container else None,
"instance_id": container["hostname"] if container else None})
return servers
def up_instances() -> set[str]:
return {s["instance_id"] or s["addr"] for s in lb_servers() if s["state"] == "UP"}
def wait_up(count: int, timeout: float = 60) -> float:
"""Seconds until exactly `count` servers are UP (raises TimeoutError otherwise)."""
started = time.monotonic()
while True:
try:
if len(up_instances()) == count:
return time.monotonic() - started
except RuntimeError:
pass # lb restarting
if time.monotonic() - started > timeout:
raise TimeoutError(f"HAProxy does not report {count} UP API replicas after {timeout:.0f} s")
time.sleep(0.25)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("status")
sub.add_parser("replicas")
wait = sub.add_parser("wait-up")
wait.add_argument("count", type=int)
wait.add_argument("--timeout", type=float, default=60)
args = parser.parse_args()
if args.command == "status":
for s in lb_servers():
if s["state"] != "MAINT" or s["container"]:
print(f"{s['slot']:6} {s['addr']:15} {s['state']:8} {s['container'] or '-':24} X-Instance-Id={s['instance_id'] or '-'}")
print(f"(other slots: MAINT, no container)")
elif args.command == "replicas":
for r in replicas():
print(f"{r['name']:24} X-Instance-Id={r['hostname']} {r['state']:8} health={r['health']:9} "
f"pid={r['pid']} started={r['started_at'][:23]} ip={','.join(r['ips']) or '-'}")
else:
took = wait_up(args.count, args.timeout)
print(f"{args.count} API replica(s) UP in HAProxy after {took:.1f} s: {sorted(up_instances())}")
return 0
if __name__ == "__main__":
sys.exit(main())
+6
View File
@@ -0,0 +1,6 @@
@echo off
rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows
rem cannot execute a .sh file directly. Runs the same scripts\restart.sh with Git Bash.
rem python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
"%ProgramFiles%\Git\bin\bash.exe" "%~dp0restart.sh"
exit /b %ERRORLEVEL%
+18
View File
@@ -0,0 +1,18 @@
#!/bin/sh
# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`.
# Restarts EVERY process of the stand: LB, all API replicas, PostgreSQL, Valkey and S3 (SeaweedFS).
# Volumes, schema and data are preserved: no `down -v`, no FLUSH, no re-seed.
# Output is suppressed by the test; run it by hand to see the steps.
set -eu
cd "$(dirname "$0")/.."
# 1. Entry point and API first: no request is served while the stores restart underneath.
docker compose stop lb api
# 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync, SeaweedFS volume/filer
# flush), then start again. The S3 bytes live on the named volume s3data.
docker compose restart postgres valkey s3
# 3. API replicas (wait for healthy stores via depends_on), then the LB. HAProxy resolves
# the new replica addresses through Docker DNS by itself - no stale backend list.
docker compose start api
docker compose start lb
echo "restart.sh: lb, api (all replicas), postgres, valkey and s3 restarted with volumes preserved"
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
# Start every stopped API replica again (after scripts/stop-one.sh) and wait until Docker
# reports them healthy. The LB picks them up by itself (DNS re-resolution + checks, ~2-3 s);
# `python scripts/lbctl.py wait-up 2` waits for that explicitly.
set -eu
cd "$(dirname "$0")/.."
docker compose up -d --no-recreate --no-deps --wait api
docker compose ps api --format '{{.Name}} {{.Status}}'
+37
View File
@@ -0,0 +1,37 @@
#!/bin/sh
# Rolling update of the API without downtime (Compose has no built-in one for a scaled service).
# 1. build the new image from the working tree;
# 2. run migrations ONCE with the new image (they must be additive / compatible with the
# running old version: expand first, contract in a later release);
# 3. for every OLD replica: start one NEW replica (scale N+1, old ones untouched), wait until
# Docker says healthy AND HAProxy has N+1 replicas UP, then stop the old one gracefully
# (SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit) and remove it.
# At every moment at least N replicas serve traffic. Stops on the first error (set -e);
# the old replicas that were not replaced yet keep running.
set -eu
cd "$(dirname "$0")/.."
PY="${PYTHON:-$(command -v python3 || command -v python)}"
# New image = new revision label (git commit + UTC time), visible in docker inspect and logs.
BUILD_REVISION="${BUILD_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date -u +%Y%m%dT%H%M%SZ)}"
export BUILD_REVISION
replicas="$(docker compose ps -q api | wc -l | tr -d ' ')"
[ "$replicas" -ge 1 ] || { echo "rolling-update: no running api replica" >&2; exit 2; }
old="$(docker compose ps -q api)"
echo "rolling-update: $replicas replica(s) to replace, new revision $BUILD_REVISION"
docker compose build api
docker compose run --rm --no-deps migrate
for id in $old; do
name="$(docker inspect -f '{{.Name}}' "$id" | tr -d '/')"
docker compose up -d --no-deps --no-recreate --wait --scale "api=$((replicas + 1))" api
"$PY" scripts/lbctl.py wait-up "$((replicas + 1))" --timeout 60
docker stop --time 15 "$id" >/dev/null
docker rm "$id" >/dev/null
"$PY" scripts/lbctl.py wait-up "$replicas" --timeout 60
echo "rolling-update: replaced $name"
done
for id in $(docker compose ps -q api); do
docker inspect -f '{{.Name}} revision={{index .Config.Labels "org.opencontainers.image.revision"}} {{.State.Status}}' "$id"
done
+7
View File
@@ -0,0 +1,7 @@
@echo off
rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows
rem cannot execute a .sh file directly. Runs the same scripts\stop-one.sh with Git Bash;
rem TARGET_INSTANCE_ID and STOP_MODE are inherited from the environment.
rem python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd
"%ProgramFiles%\Git\bin\bash.exe" "%~dp0stop-one.sh"
exit /b %ERRORLEVEL%
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# Failover hook for `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`.
# Stops exactly ONE API replica: the container whose X-Instance-Id (= container hostname,
# see src/messenger/config.py) is in TARGET_INSTANCE_ID. DB, Valkey, LB and volumes are
# not touched. The replica is LEFT STOPPED (`docker stop` = no restart by the policy);
# restore it explicitly afterwards: scripts/restore-api.sh.
# STOP_MODE=stop (default) SIGTERM: readiness 503, 3 s drain, graceful exit (~4 s)
# STOP_MODE=kill SIGKILL at once (`docker stop -t 0`): crash of the process
# Returns within 10 s. The test hides the output; run by hand to see it (no secrets printed).
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
*[!A-Za-z0-9._-]*) echo "stop-one: invalid TARGET_INSTANCE_ID" >&2; exit 2 ;;
esac
mode="${STOP_MODE:-stop}"
target=""
# Only RUNNING containers of THIS compose project's service `api` are candidates.
for id in $(docker compose ps -q api); do
if [ "$(docker inspect -f '{{.Config.Hostname}}' "$id")" = "$TARGET_INSTANCE_ID" ]; then
target="$id"
fi
done
if [ -z "$target" ]; then
echo "stop-one: no running api replica with X-Instance-Id=$TARGET_INSTANCE_ID" >&2
exit 3
fi
name="$(docker inspect -f '{{.Name}}' "$target")"
case "$mode" in
stop) docker stop --time 15 "$target" >/dev/null ;;
kill) docker stop --time 0 "$target" >/dev/null ;;
*) echo "stop-one: STOP_MODE must be stop or kill" >&2; exit 2 ;;
esac
echo "stop-one: $mode $name (X-Instance-Id=$TARGET_INSTANCE_ID) -> $(docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}}' "$target")"
+3
View File
@@ -0,0 +1,3 @@
"""In-memory messenger for lab 1 (MTUSI, «Высоконагруженные приложения»)."""
__version__ = "1.0.0"
+88
View File
@@ -0,0 +1,88 @@
"""Entry point: `python -m messenger`. The HTTP server is the main process (PID 1 in Docker).
Shutdown sequence on SIGTERM/SIGINT (lab 3: the replica stands behind HAProxy):
1. readiness flips to 503 (`app.state.draining = True`), requests are still served;
2. after DRAIN_SECONDS (compose: 3 s > HAProxy detection 2 x 1 s) the load balancer has
marked this replica DOWN and sends nothing new here;
3. uvicorn closes the listening socket; in-flight requests get up to
SHUTDOWN_TIMEOUT_SECONDS (compose: 8 s) to finish;
4. the lifespan closes the DB pool and the Valkey client; the process exits with code 0.
DRAIN + SHUTDOWN (11 s) < compose stop_grace_period (15 s), so Docker never needs SIGKILL.
A second signal during the drain skips the rest of it. Data stays in PostgreSQL and Valkey.
"""
from __future__ import annotations
import logging
import os
import sys
import threading
from types import FrameType
import uvicorn
from . import __version__, logging_setup
from .app import create_app
from .config import ConfigError, Settings
log = logging.getLogger("messenger")
class GracefulServer(uvicorn.Server):
"""uvicorn.Server that drains (readiness 503, still serving) before the normal graceful stop."""
def __init__(self, config: uvicorn.Config, drain_seconds: float):
super().__init__(config)
self.drain_seconds = drain_seconds
self.drain_timer: threading.Timer | None = None
def handle_exit(self, sig: int, frame: FrameType | None) -> None:
self.config.app.state.draining = True # config.app is our Starlette instance
if self.drain_timer is None and self.drain_seconds > 0 and not self.should_exit:
log.info("drain_started", extra={"fields": {"signal": sig, "drain_s": self.drain_seconds,
"timeout_s": self.config.timeout_graceful_shutdown}})
# uvicorn polls should_exit in its main loop, so setting it from a timer thread is safe.
self.drain_timer = threading.Timer(self.drain_seconds, self._stop, args=(sig, frame))
self.drain_timer.daemon = True
self.drain_timer.start()
return
if self.drain_timer is not None:
self.drain_timer.cancel() # second signal: stop draining now
self._stop(sig, frame)
def _stop(self, sig: int, frame: FrameType | None) -> None:
if not self.should_exit:
log.info("shutdown_requested", extra={"fields": {"signal": sig}})
super().handle_exit(sig, frame)
def main() -> int:
try:
settings = Settings.from_env()
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
logging_setup.configure(settings.log_level)
app = create_app(settings)
config = uvicorn.Config(
app,
host=settings.host,
port=settings.port,
log_config=None,
access_log=False,
server_header=False,
proxy_headers=False,
lifespan="on",
timeout_graceful_shutdown=settings.shutdown_timeout,
)
server = GracefulServer(config, settings.drain_seconds)
log.info("starting", extra={"fields": {
"version": __version__, "revision": os.environ.get("APP_REVISION", "dev"), "env": settings.env, "host": settings.host, "port": settings.port,
"python": sys.version.split()[0], "storage": "postgresql", "instance_id": settings.instance_id,
}})
server.run()
log.info("stopped")
return 0
if __name__ == "__main__":
sys.exit(main())
+584
View File
@@ -0,0 +1,584 @@
"""HTTP layer: Starlette routes, request-id/logging middleware and error mapping."""
from __future__ import annotations
import asyncio
import contextlib
import hashlib
import json
import logging
import random
import re
import time
import uuid
from typing import Any
from starlette.applications import Starlette
from starlette.exceptions import HTTPException
from starlette.requests import Request
from starlette.responses import JSONResponse, Response, StreamingResponse
from starlette.routing import Route
from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Send
from . import attachments, authhdr, cursor, docs, passwords, repo, validation
from .config import Settings
from .db import Database
from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unauthorized, Unavailable
from .sessions import SessionStore, build_store
from .storage import ObjectMissing, Storage
from .timefmt import rfc3339
log = logging.getLogger("messenger")
access_log = logging.getLogger("messenger.access")
REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
# --------------------------------------------------------------------------
# Responses
# --------------------------------------------------------------------------
def error_response(request_id: str, status: int, code: str, message: str, headers: dict[str, str] | None = None) -> JSONResponse:
body = {"error": {"code": code, "message": message, "request_id": request_id}}
return JSONResponse(body, status_code=status, headers={"X-Request-Id": request_id, **(headers or {})})
async def api_error_handler(request: Request, exc: Exception) -> JSONResponse:
assert isinstance(exc, ApiError)
return error_response(request.state.request_id, exc.status, exc.code, exc.message, exc.headers)
async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse:
"""Unknown routes and wrong methods also answer in the contract error format."""
assert isinstance(exc, HTTPException)
if exc.status_code == 404:
return error_response(request.state.request_id, 404, "not_found", "Объект не найден")
if exc.status_code == 405:
return error_response(request.state.request_id, 405, "invalid_request", "Метод не поддерживается", exc.headers)
return error_response(request.state.request_id, exc.status_code, "invalid_request", "Неверный запрос")
# --------------------------------------------------------------------------
# Middleware (pure ASGI, so it also covers 404/405 and unexpected exceptions)
# --------------------------------------------------------------------------
class RequestContextMiddleware:
"""Adds X-Request-Id and X-Instance-Id to EVERY response (errors, health and crash 500/503
included), writes one access-log line, turns crashes into 500."""
def __init__(self, app: ASGIApp, instance_id: str):
self.app = app
self.instance_id = instance_id.encode("ascii")
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] != "http":
await self.app(scope, receive, send)
return
incoming = dict(scope["headers"]).get(b"x-request-id", b"").decode("latin-1")
request_id = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex
scope.setdefault("state", {})["request_id"] = request_id
status = 500
started = False
begin = time.perf_counter()
async def send_with_id(message: AsgiMessage) -> None:
nonlocal status, started
if message["type"] == "http.response.start":
started = True
status = message["status"]
headers = [(k, v) for k, v in message.get("headers", [])
if k.lower() not in (b"x-request-id", b"x-instance-id")]
headers.append((b"x-request-id", request_id.encode("latin-1")))
headers.append((b"x-instance-id", self.instance_id))
message = {**message, "headers": headers}
await send(message)
try:
await self.app(scope, receive, send_with_id)
except asyncio.CancelledError:
# uvicorn cancels requests still running when SHUTDOWN_TIMEOUT_SECONDS expires:
# answer 503 (retryable) instead of a generic 500, then let the cancellation proceed.
if not started:
status = 503
response = error_response(request_id, 503, "unavailable", "Сервер завершает работу")
await response(scope, receive, send_with_id)
raise
except Exception:
log.exception("unhandled_error", extra={"fields": {"request_id": request_id}})
if not started:
status = 500
response = error_response(request_id, 500, "internal_error", "Внутренняя ошибка сервера")
await response(scope, receive, send_with_id)
finally:
access_log.info("request", extra={"fields": {
"request_id": request_id,
"method": scope["method"],
"path": scope["path"], # ids only; query (cursor) and bodies are never logged
"status": status,
"instance_id": self.instance_id.decode("ascii"),
"duration_ms": round((time.perf_counter() - begin) * 1000, 3),
}})
# --------------------------------------------------------------------------
# Request helpers
# --------------------------------------------------------------------------
def db_of(request: Request) -> Database:
return request.app.state.db
def sessions_of(request: Request) -> SessionStore:
return request.app.state.sessions
def storage_of(request: Request) -> Storage:
return request.app.state.storage
async def authenticate(request: Request) -> str:
"""User id of a valid Bearer session, otherwise 401. X-User-Id is ignored since lab 2.
Store failures are 503 (Unavailable), never 401 and never anonymous access.
"""
token = authhdr.parse_bearer(request.headers.get("authorization"))
user_id = await sessions_of(request).user_id_for(token)
if user_id is None:
raise Unauthorized("Сессия недействительна, истекла или отозвана")
return user_id
def path_uuid(request: Request, name: str) -> str:
return validation.parse_uuid(request.path_params[name], name)
async def read_json(request: Request) -> dict[str, Any]:
content_type = request.headers.get("content-type", "")
media_type, _, params = content_type.partition(";")
if media_type.strip().lower() != "application/json":
raise InvalidRequest("Content-Type должен быть application/json")
charset = params.strip().lower()
if charset and charset.replace(" ", "") not in {"charset=utf-8", "charset=\"utf-8\""}:
raise InvalidRequest("Поддерживается только кодировка UTF-8")
limit: int = request.app.state.settings.max_body_bytes
declared = request.headers.get("content-length")
if declared is not None and declared.isdigit() and int(declared) > limit:
raise PayloadTooLarge()
chunks: list[bytes] = []
size = 0
async for chunk in request.stream():
size += len(chunk)
if size > limit:
raise PayloadTooLarge()
chunks.append(chunk)
return validation.parse_json_object(b"".join(chunks))
def page_params(request: Request) -> tuple[int, str | None]:
query = request.query_params.multi_items()
limit = validation.parse_limit(validation.single_query_param(query, "limit"))
raw_cursor = validation.single_query_param(query, "cursor")
if raw_cursor is not None and raw_cursor == "":
raise InvalidRequest("cursor не может быть пустым")
return limit, raw_cursor
async def reject_body(request: Request) -> None:
"""POST /auth/sessions takes credentials only from the Basic header; any body is 400."""
if request.headers.get("transfer-encoding") or request.headers.get("content-length", "0") != "0":
raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела")
async for chunk in request.stream():
if chunk:
raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела")
def idempotency_key(request: Request) -> str:
values = request.headers.getlist("idempotency-key")
return validation.idempotency_key(values[0] if len(values) == 1 else None)
def request_fingerprint(text: str, attachment_ids: list[str]) -> bytes:
"""SHA-256 of the canonical body: key order and an absent attachment_ids vs [] do not matter."""
canonical = json.dumps({"attachment_ids": attachment_ids, "text": text},
ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(canonical.encode("utf-8")).digest()
def ok(body: Any, status: int = 200) -> JSONResponse:
return JSONResponse(body, status_code=status)
# --------------------------------------------------------------------------
# Handlers
# --------------------------------------------------------------------------
async def live(request: Request) -> JSONResponse:
return ok({"status": "ok"})
async def ready(request: Request) -> JSONResponse:
"""200 only if the WHOLE mandatory synchronous path can be served: PostgreSQL, Valkey
(SESSION_STORE=valkey) and, since lab 4, the S3 bucket (upload/download of attachments)
answer. 503 while draining after SIGTERM. HAProxy routes to such replicas first."""
if request.app.state.draining:
raise Unavailable("Сервер завершает работу")
db_ok, store_ok, s3_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping(),
storage_of(request).ping())
if not db_ok:
raise Unavailable("PostgreSQL недоступен")
if not store_ok:
raise Unavailable("Хранилище сессий недоступно")
if not s3_ok:
raise Unavailable("Хранилище файлов (S3) недоступно")
return ok({"status": "ok"})
async def ready_core(request: Request) -> JSONResponse:
"""Not in the contract: readiness WITHOUT S3, for the load balancer's degraded mode only.
If S3 is down for every replica, /health/ready is 503 everywhere; HAProxy then routes to
replicas that are at least core-ready, so chats and messages keep working while every
attachment request answers 503 itself (see infra/haproxy/haproxy.cfg)."""
if request.app.state.draining:
raise Unavailable("Сервер завершает работу")
db_ok, store_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping())
if not (db_ok and store_ok):
raise Unavailable("PostgreSQL или хранилище сессий недоступно")
return ok({"status": "ok"})
async def register_user(request: Request) -> JSONResponse:
data = await read_json(request)
validation.check_fields(data, {"username", "display_name", "password"})
username = validation.username_field(data)
display_name = validation.text_field(data, "display_name", 100)
password = validation.password_field(data)
password_hash = await passwords.hash_password(password)
user = await repo.create_user(db_of(request), username, display_name, password_hash)
log.info("user_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}})
return ok(user.to_json(), 201)
async def create_session(request: Request) -> JSONResponse:
"""Login: only `Authorization: Basic`; wrong user and wrong password give the same 401."""
username, password = authhdr.parse_basic(request.headers.get("authorization"))
await reject_body(request)
found = await repo.get_credentials(db_of(request), username)
user, stored_hash = found if found else (None, None)
if not await passwords.verify_password(stored_hash, password) or user is None:
log.info("login_failed", extra={"fields": {"request_id": request.state.request_id}})
raise Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE)
ttl = request.app.state.settings.session_ttl_seconds
token, expires_at = await sessions_of(request).create(user.id, ttl)
log.info("session_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}})
return ok({"token": token, "token_type": "Bearer", "expires_at": rfc3339(expires_at), "user": user.to_json()}, 201)
async def revoke_session(request: Request) -> Response:
token = authhdr.parse_bearer(request.headers.get("authorization"))
if not await sessions_of(request).revoke(token):
raise Unauthorized("Сессия недействительна, истекла или отозвана")
log.info("session_revoked", extra={"fields": {"request_id": request.state.request_id}})
return Response(status_code=204)
async def me(request: Request) -> JSONResponse:
user_id = await authenticate(request)
user = await repo.get_user(db_of(request), user_id)
if user is None:
raise Unauthorized("Сессия недействительна, истекла или отозвана")
return ok(user.to_json())
async def create_chat(request: Request) -> JSONResponse:
user_id = await authenticate(request)
data = await read_json(request)
validation.check_fields(data, {"title", "member_ids"})
title = validation.text_field(data, "title", 100)
member_ids = validation.member_ids_field(data)
chat = await repo.create_chat(db_of(request), user_id, title, member_ids)
log.info("chat_created", extra={"fields": {"request_id": request.state.request_id, "chat_id": chat.id, "members": len(member_ids)}})
return ok(chat.to_json(), 201)
async def list_chats(request: Request) -> JSONResponse:
user_id = await authenticate(request)
limit, raw_cursor = page_params(request)
after = cursor.decode(raw_cursor, cursor.CHATS, user_id) if raw_cursor else None
items, has_more = await repo.list_chats(db_of(request), user_id, after, limit)
next_cursor = cursor.encode(cursor.CHATS, user_id, items[-1].key) if has_more else None
return ok({"items": [c.to_json() for c in items], "next_cursor": next_cursor})
async def get_chat(request: Request) -> JSONResponse:
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
return ok((await repo.get_chat(db_of(request), user_id, chat_id)).to_json())
def parse_message(data: dict[str, Any]) -> tuple[str, list[str]]:
"""`text` is required and a string (also with attachments); `attachment_ids`, if present, an
array of 0-10 unique UUIDs. "" or whitespace-only text is allowed only with attachments."""
validation.check_fields(data, {"text"}, frozenset({"attachment_ids"}))
attachment_ids = validation.attachment_ids_field(data)
text = validation.text_field(data, "text", 4000, min_len=0 if attachment_ids else 1,
allow_blank=bool(attachment_ids))
return text, attachment_ids
async def create_message(request: Request) -> JSONResponse:
"""Order of answers for both write paths: 401 -> 400 bad chat id -> 404 outsider -> 400 key/body
-> 409 other body with the same key -> 201 (new or replayed). sender_id comes from the session."""
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
db = db_of(request)
settings = request.app.state.settings
if settings.message_write_path == "multi":
await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before key/body are looked at
key = idempotency_key(request)
text, attachment_ids = parse_message(await read_json(request))
result = await repo.post_message(db, user_id, chat_id, key, request_fingerprint(text, attachment_ids), text,
settings.idempotency_ttl_seconds, attachment_ids)
else:
try:
key = idempotency_key(request)
text, attachment_ids = parse_message(await read_json(request))
except ApiError:
# Rare error path only: an outsider must still get 404, not a hint about the body.
await repo.ensure_member(db, user_id, chat_id)
raise
# Membership check, key claim, chat clock and message: one SQL call, one transaction.
result = await repo.post_message_single_call(db, user_id, chat_id, key,
request_fingerprint(text, attachment_ids), text,
settings.idempotency_ttl_seconds, attachment_ids)
if result.replayed:
log.info("idempotent_replay", extra={"fields": {"request_id": request.state.request_id,
"message_id": result.message.id}})
return ok(result.message.to_json(), 201)
async def list_messages(request: Request) -> JSONResponse:
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
db = db_of(request)
await repo.ensure_member(db, user_id, chat_id) # 404 for outsiders before cursor details leak
limit, raw_cursor = page_params(request)
after = cursor.decode(raw_cursor, cursor.MESSAGES, chat_id) if raw_cursor else None
items, has_more = await repo.list_messages(db, chat_id, after, limit)
next_cursor = cursor.encode(cursor.MESSAGES, chat_id, items[-1].key) if has_more else None
return ok({"items": [m.to_json() for m in items], "next_cursor": next_cursor})
# ------------------------------------------------------------------ attachments (lab 4)
def fault(request: Request, point: str) -> None:
"""Failure injection for the level 4 experiments (FAULT_POINTS, compose.faults.yaml only)."""
if point in request.app.state.settings.fault_points:
log.warning("fault_injected", extra={"fields": {"request_id": request.state.request_id, "point": point}})
raise Unavailable("Внедрённый отказ: " + point)
async def upload_attachment(request: Request) -> JSONResponse:
"""POST /chats/{chat_id}/attachments?filename=...: raw PNG/JPEG -> 201 ready (attachments.py)."""
settings: Settings = request.app.state.settings
reader = attachments.BodyReader(request)
slots: asyncio.Semaphore = request.app.state.upload_slots
acquired = False
try:
try:
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
db = db_of(request)
await repo.ensure_member(db, user_id, chat_id) # outsiders: 404 before anything about the body
filename = validation.filename_param(validation.query_items(request.scope["query_string"]))
declared = attachments.declared_type(request)
length = attachments.declared_length(request, settings.max_upload_bytes)
try:
await asyncio.wait_for(slots.acquire(), settings.upload_queue_timeout_seconds)
except asyncio.TimeoutError:
raise attachments.Busy("Слишком много одновременных загрузок, повторите позже") from None
acquired = True
upload = await attachments.read_image(reader, settings, declared, request.app.state.verify_slots, length)
except ApiError:
# Early rejection: drop (not store) a bounded rest of the body so the client reads our error.
await reader.discard(settings.max_upload_bytes + (1 << 20), timeout=5.0)
raise
# The slot is held until the buffer is no longer needed (after the S3 PUT), so at most
# UPLOAD_CONCURRENCY bodies of this replica exist in memory at any moment.
return await store_upload(request, db, chat_id, user_id, filename, upload)
finally:
if acquired:
slots.release()
async def store_upload(request: Request, db: Database, chat_id: str, user_id: str, filename: str,
upload: attachments.Upload) -> JSONResponse:
"""intent (pending) -> S3 PUT -> ready -> 201 (order and failure analysis: docs/CONSISTENCY.md)."""
attachment_id = uuid.uuid4()
object_key = attachments.new_object_key()
storage = storage_of(request)
await attachments.create_intent(db, attachment_id, chat_id, user_id, filename, upload.info, len(upload.data),
upload.sha256, object_key)
await storage.put(object_key, upload.data, upload.info.content_type,
{"attachment-id": str(attachment_id), "sha256": upload.sha256.hex()})
fault(request, "after_s3_put")
attachment = await attachments.mark_ready(db, attachment_id)
if attachment is None:
raise Unavailable("Загрузка не завершена, повторите запрос")
fault(request, "after_ready_commit")
log.info("attachment_created", extra={"fields": {"request_id": request.state.request_id,
"attachment_id": attachment.id, "chat_id": chat_id,
"size": attachment.size_bytes, "type": attachment.content_type}})
return ok(attachment.to_json(), 201)
async def get_attachment(request: Request) -> JSONResponse:
user_id = await authenticate(request)
attachment_id = path_uuid(request, "attachment_id")
return ok((await attachments.get_visible(db_of(request), user_id, attachment_id)).to_json())
def content_disposition(filename: str) -> str:
"""RFC 6266/5987: ASCII fallback + UTF-8 name; the stored name itself is never interpreted."""
from urllib.parse import quote
fallback = "".join(c if 32 <= ord(c) < 127 and c not in '"\\%;' else "_" for c in filename)
return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{quote(filename, safe='')}"
async def download_attachment(request: Request) -> Response:
"""Original bytes through the authorized API: no redirect, no public S3 URL. Streamed from
S3 in 64 KiB chunks (memory per download does not depend on the file size)."""
user_id = await authenticate(request)
attachment_id = path_uuid(request, "attachment_id")
query = validation.query_items(request.scope["query_string"])
try:
validation.variant_param(query)
except validation.NotFoundVariant:
await attachments.get_visible(db_of(request), user_id, attachment_id) # 404 for outsiders first
raise NotFound("Такого варианта у вложения нет") from None
attachment = await attachments.get_visible(db_of(request), user_id, attachment_id)
if attachment.lost:
raise NotFound("Оригинал вложения утрачен")
try:
size, chunks = await storage_of(request).open_stream(attachment.object_key)
except ObjectMissing:
# A ready row without its object: data loss, reported by the reconciler (quarantine).
log.error("attachment_object_missing", extra={"fields": {"request_id": request.state.request_id,
"attachment_id": attachment.id}})
raise NotFound("Оригинал вложения утрачен") from None
if size != attachment.size_bytes:
log.error("attachment_size_mismatch", extra={"fields": {"request_id": request.state.request_id,
"attachment_id": attachment.id}})
await chunks.aclose()
raise Unavailable("Оригинал вложения временно недоступен")
headers = {
"Content-Length": str(size),
"Content-Disposition": content_disposition(attachment.filename),
"X-Content-Type-Options": "nosniff",
"Cache-Control": "private, no-store",
"ETag": '"' + attachment.sha256 + '"',
}
return StreamingResponse(chunks, media_type=attachment.content_type, headers=headers)
def create_app(settings: Settings | None = None) -> Starlette:
settings = settings or Settings.from_env()
db = Database(settings)
sessions = build_store(settings, db)
storage = Storage(settings)
async def purge_loop() -> None:
"""Housekeeping: expired Idempotency-Key rows (older than IDEMPOTENCY_TTL_SECONDS)."""
while True:
await asyncio.sleep(600)
try:
deleted = await repo.purge_idempotency_keys(db, settings.idempotency_ttl_seconds)
if deleted:
log.info("idempotency_keys_purged", extra={"fields": {"deleted": deleted}})
except ApiError:
pass # database unavailable right now; next round
async def janitor_loop(every: float) -> None:
"""Level 4 cleanup: intents of failed uploads older than PENDING_GRACE_SECONDS (attachments.py).
Every replica runs it; the advisory lock lets one round run at a time."""
while True:
await asyncio.sleep(every * (0.5 + random.random()))
try:
removed = await attachments.clean_stale_pending(db, storage, settings.pending_grace_seconds)
if removed:
log.info("stale_uploads_removed", extra={"fields": {"count": removed}})
except ApiError:
pass # PostgreSQL or S3 unavailable right now; the rows stay pending for the next round
stats_log = logging.getLogger("messenger.stats")
async def stats_loop(every: float) -> None:
"""Experiments only: DB pool saturation (waiting requests, wait time) and process CPU."""
cpu = time.process_time()
db.pool.pop_stats()
while True:
await asyncio.sleep(every)
stats = db.pool.pop_stats()
now_cpu = time.process_time()
stats_log.info("pool_stats", extra={"fields": {
"interval_s": every, "cpu_s": round(now_cpu - cpu, 4),
**{k: stats.get(k, 0) for k in ("pool_size", "pool_available", "requests_waiting", "requests_num",
"requests_queued", "requests_wait_ms", "usage_ms", "requests_errors")}}})
cpu = now_cpu
@contextlib.asynccontextmanager
async def lifespan(app: Starlette):
passwords.configure(settings.hash_concurrency)
await db.open()
await storage.open()
log.info("dependencies_configured", extra={"fields": {
"session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds,
"db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds,
"instance_id": settings.instance_id, "drain_s": settings.drain_seconds,
"message_write_path": settings.message_write_path, "s3_bucket": settings.s3_bucket,
"upload_concurrency": settings.upload_concurrency, "verify_concurrency": settings.image_verify_concurrency,
"pending_grace_s": settings.pending_grace_seconds,
"fault_points": sorted(settings.fault_points)}})
tasks = [asyncio.create_task(purge_loop())]
if settings.janitor_interval_seconds > 0:
tasks.append(asyncio.create_task(janitor_loop(settings.janitor_interval_seconds)))
if settings.stats_log_seconds > 0:
tasks.append(asyncio.create_task(stats_loop(settings.stats_log_seconds)))
try:
yield
finally:
for task in tasks:
task.cancel()
await sessions.close()
await storage.close()
await db.close()
routes = [
Route("/health/live", live, methods=["GET"]),
Route("/health/ready", ready, methods=["GET"]),
Route("/health/ready/core", ready_core, methods=["GET"]),
Route("/api/v1/users", register_user, methods=["POST"]),
Route("/api/v1/users/me", me, methods=["GET"]),
Route("/api/v1/auth/sessions", create_session, methods=["POST"]),
Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]),
Route("/api/v1/chats", create_chat, methods=["POST"]),
Route("/api/v1/chats", list_chats, methods=["GET"]),
Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]),
Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]),
Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]),
Route("/api/v1/chats/{chat_id}/attachments", upload_attachment, methods=["POST"]),
Route("/api/v1/attachments/{attachment_id}", get_attachment, methods=["GET"]),
Route("/api/v1/attachments/{attachment_id}/content", download_attachment, methods=["GET"]),
]
if settings.docs_enabled: # Swagger UI for manual experiments, not part of the contract
routes += [
Route("/docs", docs.swagger_page, methods=["GET"]),
Route("/openapi.json", docs.openapi_json, methods=["GET"]),
]
app = Starlette(
routes=routes,
exception_handlers={ApiError: api_error_handler, HTTPException: http_exception_handler},
lifespan=lifespan,
)
app.add_middleware(RequestContextMiddleware, instance_id=settings.instance_id)
app.state.settings = settings
app.state.db = db
app.state.sessions = sessions
app.state.storage = storage
app.state.upload_slots = asyncio.Semaphore(settings.upload_concurrency)
app.state.verify_slots = asyncio.Semaphore(settings.image_verify_concurrency)
app.state.draining = False
return app
+300
View File
@@ -0,0 +1,300 @@
"""Attachments (lab 4): metadata in PostgreSQL, original bytes in the private S3 bucket.
Upload (POST /chats/{id}/attachments, raw PNG/JPEG body), in this order:
auth -> membership (404) -> filename / Content-Type / Content-Length (400/415/413)
-> upload slot (at most UPLOAD_CONCURRENCY bodies per replica, else 503 + Retry-After)
-> body in chunks: size limit, read deadline, format + pixel count from the first bytes
-> full integrity check (images.verify) -> SHA-256 / size known
-> INSERT row status='pending' with a fresh random object_key, COMMIT (intent)
-> PUT object to S3 (Content-MD5)
-> UPDATE status='ready' WHERE status='pending', COMMIT -> 201 ready.
Why 201 is honest: it is sent only after BOTH the object and the ready row are durable.
What a failure between the steps leaves behind, and who removes it (docs/CONSISTENCY.md):
* before the intent COMMIT: nothing (no object was written);
* after the intent, before/while PUT, before the ready COMMIT: a 'pending' row (+ maybe the
object). The client got 503 and may retry (a new attachment). The janitor (below) deletes the
object - idempotent, a missing key is fine - and then the row, once it is older than
PENDING_GRACE_SECONDS. Nothing is compensated in the request path: after a lost answer of
COMMIT we do not know whether the row is ready, and deleting the object could destroy a
ready attachment. Only the janitor decides, under a row lock, from the committed state;
* after the ready COMMIT (answer lost): a correct ready attachment the client does not know.
"""
from __future__ import annotations
import asyncio
import datetime as dt
import hashlib
import logging
import secrets
import uuid
from dataclasses import dataclass
import psycopg
from psycopg import errors as pg_errors
from starlette.requests import Request
from . import images
from .config import Settings
from .db import Database
from .errors import ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType
from .storage import OBJECT_PREFIX, Storage
from .timefmt import rfc3339
log = logging.getLogger("messenger.attachments")
JANITOR_LOCK_ID = 2026_04 # advisory lock: one janitor round at a time across all replicas
@dataclass(frozen=True, slots=True)
class Attachment:
id: str
chat_id: str
owner_id: str
filename: str
content_type: str
size_bytes: int
sha256: str
object_key: str
status: str
created_at: dt.datetime
updated_at: dt.datetime
lost: bool = False
def to_json(self) -> dict:
"""Public representation: no object key, no bucket, no internal state."""
return {"id": self.id, "chat_id": self.chat_id, "owner_id": self.owner_id, "filename": self.filename,
"content_type": self.content_type, "size_bytes": self.size_bytes, "sha256": self.sha256,
"status": "ready", "variants": [], "error_code": None,
"created_at": rfc3339(self.created_at), "updated_at": rfc3339(self.updated_at)}
COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.object_key, "
"a.status, a.created_at, a.updated_at, a.lost_at IS NOT NULL")
def _attachment(row) -> Attachment:
return Attachment(str(row[0]), str(row[1]), str(row[2]), row[3], row[4], row[5], bytes(row[6]).hex(), row[7],
row[8], row[9], row[10], row[11])
def new_object_key() -> str:
"""Server-generated, 128 random bits, unrelated to the attachment id and the filename."""
return OBJECT_PREFIX + secrets.token_hex(16)
# ------------------------------------------------------------------ SQL
async def create_intent(db: Database, attachment_id: uuid.UUID, chat_id: str, owner_id: str, filename: str,
info: images.ImageInfo, size: int, sha256: bytes, object_key: str) -> None:
async def work(conn: psycopg.AsyncConnection) -> None:
await conn.execute(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"width, height, object_key, status) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending')",
(attachment_id, chat_id, owner_id, filename, info.content_type, size, sha256, info.width, info.height,
object_key))
try:
await db.run(work, transaction=False)
except pg_errors.ForeignKeyViolation:
raise NotFound("Чат не найден") from None
async def mark_ready(db: Database, attachment_id: uuid.UUID) -> Attachment | None:
"""pending -> ready. None if the row is no longer pending (the janitor removed a stale intent)."""
async def work(conn: psycopg.AsyncConnection) -> Attachment | None:
cur = await conn.execute(
"UPDATE attachments a SET status = 'ready', updated_at = clock_timestamp() "
f"WHERE a.id = %s AND a.status = 'pending' RETURNING {COLUMNS}", (attachment_id,))
row = await cur.fetchone()
return _attachment(row) if row else None
return await db.run(work, transaction=False)
async def get_visible(db: Database, user_id: str, attachment_id: str) -> Attachment:
"""A READY attachment of a chat the user is a member of; anything else is 404 (unknown,
foreign chat, unfinished upload) - existence of foreign objects is never revealed."""
async def work(conn: psycopg.AsyncConnection) -> Attachment:
cur = await conn.execute(
f"SELECT {COLUMNS} FROM attachments a "
"JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s "
"WHERE a.id = %s AND a.status = 'ready'", (user_id, attachment_id))
row = await cur.fetchone()
if row is None:
raise NotFound("Вложение не найдено")
return _attachment(row)
return await db.run(work, transaction=False)
async def clean_stale_pending(db: Database, storage: Storage, grace_seconds: int, batch: int = 20) -> int:
"""Janitor round: remove intents of uploads that never became ready.
Under FOR UPDATE: an upload that is (impossibly) still running would block on the row in
mark_ready and then find it gone (-> 503), so a ready attachment can never lose its object.
Order: object first (idempotent), row second; a crash in between leaves the pending row and
the next round repeats. Only status='pending' rows older than the grace period are touched:
fresh uploads and ready/used attachments are never selected.
"""
async def work(conn: psycopg.AsyncConnection) -> int:
cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (JANITOR_LOCK_ID,))
if not (await cur.fetchone())[0]:
return 0
cur = await conn.execute(
"SELECT id, object_key FROM attachments WHERE status = 'pending' "
"AND created_at < clock_timestamp() - make_interval(secs => %s) "
"ORDER BY created_at LIMIT %s FOR UPDATE SKIP LOCKED", (grace_seconds, batch))
rows = await cur.fetchall()
if not rows:
return 0
await asyncio.gather(*(storage.delete(key) for _, key in rows))
await conn.execute("DELETE FROM attachments WHERE id = ANY(%s) AND status = 'pending'",
([r[0] for r in rows],))
return len(rows)
return await db.run(work, deadline=storage.timeout + 5)
# ------------------------------------------------------------------ request body
class Busy(Unavailable):
"""All upload slots of this replica are taken: retry shortly (no body was read)."""
@property
def headers(self) -> dict[str, str]:
return {"Retry-After": "1"}
@dataclass
class Upload:
data: bytearray
info: images.ImageInfo
sha256: bytes
class BodyReader:
"""Reads the raw ASGI body ourselves (not request.stream()), so we always know how much of it
is consumed and can discard the rest after an early rejection."""
def __init__(self, request: Request):
self.receive = request.receive
self.done = False
async def chunk(self, timeout: float) -> bytes | None:
if self.done:
return None
message = await asyncio.wait_for(self.receive(), timeout)
if message["type"] == "http.disconnect":
self.done = True
raise InvalidRequest("Клиент прервал загрузку")
if not message.get("more_body", False):
self.done = True
return message.get("body", b"")
async def discard(self, limit: int, timeout: float) -> None:
"""After an early 4xx: read and DROP up to `limit` more bytes, so an ordinary client that
writes the whole body before reading the answer still gets our JSON error instead of a
connection reset. Nothing is kept in memory; beyond the limit the connection is dropped."""
loop = asyncio.get_running_loop()
deadline = loop.time() + timeout
seen = 0
try:
while not self.done and seen <= limit:
remaining = deadline - loop.time()
if remaining <= 0:
return
seen += len(await self.chunk(remaining) or b"")
except (ApiError, asyncio.TimeoutError):
return
def declared_type(request: Request) -> str:
"""Content-Type of the upload must be image/png or image/jpeg (parameters ignored)."""
media_type = request.headers.get("content-type", "").partition(";")[0].strip().lower()
if media_type not in (images.PNG, images.JPEG):
raise UnsupportedMediaType("Content-Type должен быть image/png или image/jpeg")
return media_type
def declared_length(request: Request, limit: int) -> int | None:
raw = request.headers.get("content-length")
if raw is None:
return None # chunked transfer: the limit is enforced while reading
if not raw.isdigit():
raise InvalidRequest("Неверный Content-Length")
length = int(raw)
if length > limit:
raise PayloadTooLarge(f"Файл больше {limit} байт")
if length == 0:
raise InvalidRequest("Тело запроса пустое: нужны байты изображения")
return length
async def read_image(reader: BodyReader, settings: Settings, declared: str, verify_slots: asyncio.Semaphore,
length: int | None) -> Upload:
"""Body -> verified image. Memory: ONE buffer of at most MAX_UPLOAD_BYTES per upload - exactly
Content-Length bytes allocated once when the length is declared (no growth, no final copy);
it is passed as is to the checks and to the S3 PUT. The format and the pixel count are known
after the first bytes, so a picture that is too big or not a PNG/JPEG is rejected without
reading/decoding the rest."""
loop = asyncio.get_running_loop()
deadline = loop.time() + settings.upload_read_timeout_seconds
limit = settings.max_upload_bytes
buffer = bytearray(length) if length else bytearray()
view = memoryview(buffer) if length else None
filled = 0
digest = hashlib.sha256()
info: images.ImageInfo | None = None
while True:
remaining = deadline - loop.time()
if remaining <= 0:
raise InvalidRequest("Тело запроса не получено вовремя")
try:
chunk = await reader.chunk(remaining)
except asyncio.TimeoutError:
raise InvalidRequest("Тело запроса не получено вовремя") from None
if chunk is None:
break
if filled + len(chunk) > (length or limit):
raise PayloadTooLarge(f"Файл больше {limit} байт")
if view is not None:
view[filled:filled + len(chunk)] = chunk
else:
buffer += chunk
filled += len(chunk)
digest.update(chunk)
if info is None and filled:
info = _sniff(view[:filled] if view is not None else buffer, settings, declared)
if view is not None:
view.release()
if not filled:
raise InvalidRequest("Тело запроса пустое: нужны байты изображения")
if length and filled != length:
raise InvalidRequest("Тело запроса короче Content-Length")
if info is None:
raise UnsupportedMediaType("Файл не является корректным изображением PNG или JPEG")
try:
async with verify_slots:
await images.verify(buffer, info, timeout=settings.image_verify_timeout_seconds,
jpeg_max_memory_mb=settings.jpeg_max_memory_mb)
except images.ImageError as exc:
raise _image_error(exc) from None
except asyncio.TimeoutError:
raise UnsupportedMediaType("Изображение не удалось проверить за отведённое время") from None
except (images.VerifierUnavailable, MemoryError):
log.warning("image_verifier_unavailable", extra={"fields": {"type": info.content_type, "size": filled}})
raise Busy("Не хватило ресурсов для проверки изображения, повторите позже") from None
return Upload(buffer, info, digest.digest())
def _sniff(buffer: bytearray, settings: Settings, declared: str) -> images.ImageInfo | None:
try:
info = images.sniff(buffer, settings.max_image_pixels)
except images.ImageError as exc:
raise _image_error(exc) from None
if info is not None and info.content_type != declared:
# "MIME spoofing": the header says PNG, the bytes are JPEG (or vice versa).
raise UnsupportedMediaType("Содержимое не совпадает с Content-Type")
return info
def _image_error(exc: images.ImageError) -> ApiError:
return PayloadTooLarge(exc.message) if exc.status == 413 else UnsupportedMediaType(exc.message)
+48
View File
@@ -0,0 +1,48 @@
"""Parsing of the Authorization header. Any malformed value is 401, never 500.
Basic (RFC 7617) is accepted only by POST /auth/sessions; everything else needs
Bearer. Basic is an encoding, not encryption: plain HTTP is acceptable only for the
local synthetic lab stand (lab 7 adds TLS).
"""
from __future__ import annotations
import base64
import binascii
import re
from .errors import BASIC_CHALLENGE, BEARER_CHALLENGE, Unauthorized
from .validation import USERNAME_RE
B64_RE = re.compile(r"[A-Za-z0-9+/]+={0,2}")
TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}") # token_urlsafe(32) gives 43 characters
def _split(header: str | None) -> tuple[str, str]:
if not header:
return "", ""
scheme, _, value = header.strip().partition(" ")
return scheme.lower(), value.strip()
def parse_basic(header: str | None) -> tuple[str, str]:
"""(username, password) from `Basic base64(username:password)`; the password may contain ':'."""
invalid = Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE)
scheme, value = _split(header)
if scheme != "basic" or B64_RE.fullmatch(value) is None or len(value) > 1024:
raise invalid
try:
decoded = base64.b64decode(value, validate=True).decode("utf-8")
except (binascii.Error, ValueError, UnicodeDecodeError):
raise invalid from None
username, sep, password = decoded.partition(":") # split at the FIRST colon
if not sep or USERNAME_RE.fullmatch(username) is None or not password:
raise invalid
return username, password
def parse_bearer(header: str | None) -> str:
scheme, value = _split(header)
if scheme != "bearer" or TOKEN_RE.fullmatch(value) is None:
# Missing header, Basic on a business endpoint, garbage token: one answer.
raise Unauthorized("Требуется действительная Bearer-сессия", challenge=BEARER_CHALLENGE)
return value
+236
View File
@@ -0,0 +1,236 @@
"""Runtime configuration read from environment variables (12-factor style).
Non-secret settings have safe defaults. Connection strings with passwords are
read from files (`DATABASE_URL_FILE`, `CACHE_URL_FILE` → Docker secrets in
/run/secrets), so they never appear in `docker inspect`, compose files or logs.
Invalid values stop the process at startup with a clear message.
"""
from __future__ import annotations
import os
import re
import socket
from dataclasses import dataclass
from pathlib import Path
class ConfigError(ValueError):
pass
INSTANCE_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
def _instance_id() -> str:
"""X-Instance-Id of this process: INSTANCE_ID or the container hostname.
Docker sets the hostname of a container to its short container ID, so the value
is unique per replica, stable for the life of the process and maps 1:1 to a real
container (`docker inspect`), which scripts/stop-one.sh relies on.
"""
value = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname()
if INSTANCE_ID_RE.fullmatch(value) is None:
raise ConfigError("INSTANCE_ID must match [A-Za-z0-9._-]{1,64}")
return value
def _int(name: str, default: int, low: int, high: int) -> int:
raw = os.environ.get(name)
if raw is None or raw.strip() == "":
return default
try:
value = int(raw.strip())
except ValueError:
raise ConfigError(f"{name} must be an integer, got {raw!r}") from None
if not low <= value <= high:
raise ConfigError(f"{name} must be in [{low}, {high}], got {value}")
return value
def _float(name: str, default: float, low: float, high: float) -> float:
raw = os.environ.get(name)
if raw is None or raw.strip() == "":
return default
try:
value = float(raw.strip())
except ValueError:
raise ConfigError(f"{name} must be a number, got {raw!r}") from None
if not low <= value <= high:
raise ConfigError(f"{name} must be in [{low}, {high}], got {value}")
return value
def _bool(name: str, default: bool) -> bool:
raw = os.environ.get(name, "").strip().lower()
if raw == "":
return default
if raw not in {"true", "false", "1", "0"}:
raise ConfigError(f"{name} must be true/false, got {raw!r}")
return raw in {"true", "1"}
def _choice(name: str, default: str, allowed: set[str]) -> str:
value = os.environ.get(name, "").strip().lower() or default
if value not in allowed:
raise ConfigError(f"{name} must be one of {sorted(allowed)}, got {value!r}")
return value
def _secret(name: str, required: bool) -> str:
"""Value of <NAME>_FILE (preferred, Docker secret) or <NAME> (local runs/tests)."""
path = os.environ.get(f"{name}_FILE", "").strip()
if path:
try:
value = Path(path).read_text(encoding="utf-8").strip()
except OSError as exc:
# The message names the file, never its content.
raise ConfigError(f"{name}_FILE={path} cannot be read: {exc.strerror}") from None
else:
value = os.environ.get(name, "").strip()
if required and not value:
raise ConfigError(f"{name}_FILE (or {name}) must be set")
return value
@dataclass(frozen=True)
class Settings:
host: str
port: int
env: str
log_level: str
shutdown_timeout: float
max_body_bytes: int
docs_enabled: bool = True
database_url: str = ""
session_store: str = "postgres"
cache_url: str = ""
session_ttl_seconds: int = 3600
db_pool_min: int = 1
db_pool_max: int = 10
db_timeout_seconds: float = 3.0
cache_timeout_seconds: float = 0.5
hash_concurrency: int = 2
instance_id: str = "local"
drain_seconds: float = 0.0
idempotency_ttl_seconds: int = 86400
stats_log_seconds: float = 0.0
message_write_path: str = "function"
# Lab 4: private S3 bucket and attachment limits (see storage.py, attachments.py).
s3_endpoint: str = "http://s3:8333"
s3_region: str = "us-east-1"
s3_bucket: str = "messenger-attachments"
s3_access_key_id: str = ""
s3_secret_access_key: str = ""
s3_timeout_seconds: float = 3.0
s3_connect_timeout_seconds: float = 1.0
s3_max_connections: int = 20
max_upload_bytes: int = 10 * 1024 * 1024
max_image_pixels: int = 40_000_000
upload_concurrency: int = 4
upload_queue_timeout_seconds: float = 5.0
upload_read_timeout_seconds: float = 30.0
image_verify_timeout_seconds: float = 10.0
image_verify_concurrency: int = 1
jpeg_max_memory_mb: int = 256
pending_grace_seconds: int = 900
janitor_interval_seconds: float = 60.0
fault_points: frozenset[str] = frozenset()
@property
def db_statement_timeout_ms(self) -> int:
# The server stops a statement a bit before the client-side deadline fires.
return max(100, int(self.db_timeout_seconds * 1000) - 500)
@classmethod
def from_env(cls) -> "Settings":
log_level = os.environ.get("LOG_LEVEL", "info").strip().lower() or "info"
if log_level not in {"debug", "info", "warning", "error"}:
raise ConfigError(f"LOG_LEVEL must be debug/info/warning/error, got {log_level!r}")
session_store = os.environ.get("SESSION_STORE", "postgres").strip().lower() or "postgres"
if session_store == "redis":
session_store = "valkey" # same protocol; the course allows Redis OR Valkey
if session_store not in {"postgres", "valkey"}:
raise ConfigError(f"SESSION_STORE must be postgres or valkey, got {session_store!r}")
db_pool_min = _int("DB_POOL_MIN", 1, 0, 50)
db_pool_max = _int("DB_POOL_MAX", 10, 1, 100)
if db_pool_min > db_pool_max:
raise ConfigError("DB_POOL_MIN must not exceed DB_POOL_MAX")
return cls(
host=os.environ.get("APP_HOST", "0.0.0.0").strip() or "0.0.0.0",
port=_int("APP_PORT", 8080, 1, 65535),
env=os.environ.get("APP_ENV", "lab").strip() or "lab",
log_level=log_level,
# DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS must stay below compose stop_grace_period.
shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 30.0),
max_body_bytes=_int("MAX_BODY_BYTES", 64 * 1024, 1024, 1024 * 1024),
docs_enabled=_bool("DOCS_ENABLED", True),
database_url=_secret("DATABASE_URL", required=True),
session_store=session_store,
cache_url=_secret("CACHE_URL", required=session_store == "valkey"),
# Contract name. Minimum 1 s so the expiry test can use 2 s.
session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600),
db_pool_min=db_pool_min,
db_pool_max=db_pool_max,
db_timeout_seconds=_float("DB_TIMEOUT_SECONDS", 3.0, 0.5, 30.0),
cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.5, 0.05, 10.0),
hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 2, 1, 16),
instance_id=_instance_id(),
# After SIGTERM: readiness is 503 but requests are still served for this long,
# so the load balancer stops routing here before the listener closes.
drain_seconds=_float("DRAIN_SECONDS", 0.0, 0.0, 10.0),
# Contract: a replay within 24 h returns the original result.
idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 86400, 30 * 24 * 3600),
# Experiments (bench/track1): log DB pool counters and process CPU every N s; 0 = off.
stats_log_seconds=_float("STATS_LOG_SECONDS", 0.0, 0.0, 60.0),
# Track 1: `function` = one SQL call per send (migration 0003), `multi` = the previous
# multi-statement transaction; both have identical semantics (A/B on one image).
message_write_path=_choice("MESSAGE_WRITE_PATH", "function", {"function", "multi"}),
**s3_settings(),
)
# Contract limits (contracts/README.md): configurable DOWN for experiments, never above them.
CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024
CONTRACT_MAX_IMAGE_PIXELS = 40_000_000
# Failure points for the level 4 experiments only (compose.faults.yaml); empty in normal runs.
FAULT_POINTS = {"after_s3_put", "after_ready_commit"}
def s3_settings(*, credentials_required: bool = True) -> dict:
"""S3 part of Settings; also used by the reconciler/bootstrap CLIs with their own keys."""
faults = frozenset(f.strip() for f in os.environ.get("FAULT_POINTS", "").split(",") if f.strip())
if faults - FAULT_POINTS:
raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}")
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").strip() or "http://s3:8333"
if not re.fullmatch(r"https?://[A-Za-z0-9.\-]+(:[0-9]{1,5})?/?", endpoint):
raise ConfigError(f"S3_ENDPOINT must be http(s)://host[:port], got {endpoint!r}")
bucket = os.environ.get("S3_BUCKET", "messenger-attachments").strip() or "messenger-attachments"
if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket):
raise ConfigError(f"S3_BUCKET is not a valid bucket name: {bucket!r}")
return dict(
s3_endpoint=endpoint,
s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1",
s3_bucket=bucket,
s3_access_key_id=_secret("S3_ACCESS_KEY_ID", required=credentials_required),
s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY", required=credentials_required),
s3_timeout_seconds=_float("S3_TIMEOUT_SECONDS", 3.0, 0.2, 30.0),
s3_connect_timeout_seconds=_float("S3_CONNECT_TIMEOUT_SECONDS", 1.0, 0.1, 10.0),
s3_max_connections=_int("S3_MAX_CONNECTIONS", 20, 1, 200),
max_upload_bytes=_int("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES, 1024, CONTRACT_MAX_UPLOAD_BYTES),
max_image_pixels=_int("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS, 1, CONTRACT_MAX_IMAGE_PIXELS),
# At most this many request bodies of one replica are buffered/checked at once (RAM bound:
# UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES); further uploads wait, then get 503.
upload_concurrency=_int("UPLOAD_CONCURRENCY", 4, 1, 64),
upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0, 0.0, 60.0),
upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 30.0, 1.0, 300.0),
image_verify_timeout_seconds=_float("IMAGE_VERIFY_TIMEOUT_SECONDS", 10.0, 1.0, 120.0),
# Full checks running at once per replica: a strict JPEG decode of a 40 Mpx progressive
# file needs up to ~120-240 MB of coefficient buffers (capped by JPEG_MAX_MEMORY_MB).
image_verify_concurrency=_int("IMAGE_VERIFY_CONCURRENCY", 1, 1, 16),
jpeg_max_memory_mb=_int("JPEG_MAX_MEMORY_MB", 256, 16, 2048),
# An intent row still `pending` after this long belongs to a failed upload: the janitor
# deletes its object and the row. Far above the longest possible upload (~45 s).
pending_grace_seconds=_int("PENDING_GRACE_SECONDS", 900, 1, 7 * 24 * 3600),
janitor_interval_seconds=_float("JANITOR_INTERVAL_SECONDS", 60.0, 0.0, 3600.0),
fault_points=faults,
)
+56
View File
@@ -0,0 +1,56 @@
"""Opaque keyset cursors.
A cursor encodes the sort key `(created_at, id)` of the last item on the page
plus the resource it belongs to, e.g. `v1|m|<chat_id>|<ts_us>|<message_id>`,
packed in URL-safe base64. The client treats it as an opaque string; the server
rejects cursors of another chat/user (400) and any non-canonical encoding.
"""
from __future__ import annotations
import base64
import re
from .errors import InvalidRequest
from .validation import UUID_RE
B64_RE = re.compile(r"[A-Za-z0-9_-]{1,512}")
TS_RE = re.compile(r"[0-9]{1,19}")
# Latest instant both Python datetime and PostgreSQL accept: 9999-12-31T23:59:59.999999Z.
MAX_TS_US = 253402300799999999
MESSAGES = "m"
CHATS = "c"
SortKey = tuple[int, str]
def encode(scope: str, owner_id: str, key: SortKey) -> str:
ts, item_id = key
raw = f"v1|{scope}|{owner_id}|{ts}|{item_id}".encode("ascii")
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
def decode(cursor: str, scope: str, owner_id: str) -> SortKey:
"""Return the sort key stored in the cursor or raise InvalidRequest."""
bad = InvalidRequest("Неверный cursor")
if B64_RE.fullmatch(cursor) is None:
raise bad
try:
raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)).decode("ascii")
except (ValueError, UnicodeDecodeError):
raise bad from None
parts = raw.split("|")
if len(parts) != 5 or parts[0] != "v1":
raise bad
_, got_scope, got_owner, ts, item_id = parts
if TS_RE.fullmatch(ts) is None or UUID_RE.fullmatch(item_id) is None or item_id != item_id.lower():
raise bad
key = (int(ts), item_id)
if key[0] > MAX_TS_US:
raise bad # out-of-range time must be 400, not an overflow (500) in datetime/SQL
if encode(got_scope, got_owner, key) != cursor:
raise bad # only the canonical form produced by the server is accepted
if got_scope != scope or got_owner != owner_id:
raise InvalidRequest("cursor относится к другому ресурсу")
return key
+131
View File
@@ -0,0 +1,131 @@
"""PostgreSQL access: a bounded connection pool and a hard deadline for every unit of work.
Three independent limits keep a request from hanging when the database misbehaves:
* pool `timeout` — waiting for a free connection (pool exhausted);
* server `statement_timeout` / `lock_timeout` — PostgreSQL itself cancels a slow
statement or a statement stuck behind a lock;
* client deadline — covers what the server cannot report, e.g. a
paused/frozen PostgreSQL container or a dead network.
The client deadline needs care. Every unit of work runs as its own task; at the deadline
the REQUEST stops waiting and gets 503 at once, but the task is NOT cancelled: cancelling a
psycopg query makes psycopg send a cancel request, and libpq starts that connection
(PQcancelStart) synchronously in the event-loop thread, resolving the host name first.
When the API has lost its network to PostgreSQL (lab 3 review scenario: `docker network
disconnect`), that blocked the WHOLE replica, /health/live included, for ~1 minute.
The abandoned task instead ends by itself, bounded by:
* TCP keepalives + `tcp_user_timeout` (= the deadline): a dead peer or a vanished network
breaks the socket within seconds, the connection is discarded by the pool;
* `statement_timeout` / `lock_timeout` on a live but slow server;
* a frozen (paused) server keeps the connection until it resumes; meanwhile new work waits
at most the pool timeout and gets 503.
Any failure becomes 503 `unavailable` — never 401, never a hang.
"""
from __future__ import annotations
import asyncio
import logging
from collections.abc import Awaitable, Callable
from typing import TypeVar
import psycopg
from psycopg_pool import AsyncConnectionPool, PoolTimeout
from .config import Settings
from .errors import Unavailable
log = logging.getLogger("messenger.db")
T = TypeVar("T")
Work = Callable[[psycopg.AsyncConnection], Awaitable[T]]
# Errors that mean "the database is unavailable right now", not "the request is wrong".
# QueryCanceled (statement_timeout), LockNotAvailable (lock_timeout) and AdminShutdown
# (server restart) are subclasses of OperationalError.
UNAVAILABLE_ERRORS = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TimeoutError)
def _forget(task: asyncio.Task) -> None:
"""Done-callback for abandoned tasks: retrieve the result so nothing is logged as 'never retrieved'."""
if not task.cancelled():
task.exception()
class Database:
def __init__(self, settings: Settings):
self.deadline = settings.db_timeout_seconds
options = (
f"-c statement_timeout={settings.db_statement_timeout_ms} "
f"-c lock_timeout={max(100, settings.db_statement_timeout_ms // 2)} "
"-c idle_in_transaction_session_timeout=10000"
)
self.pool = AsyncConnectionPool(
conninfo=settings.database_url,
min_size=settings.db_pool_min,
max_size=settings.db_pool_max,
timeout=settings.db_timeout_seconds, # max wait for a free connection
open=False,
name="messenger",
check=AsyncConnectionPool.check_connection, # drop connections broken by a DB restart
kwargs={
"options": options,
"connect_timeout": max(1, int(settings.db_timeout_seconds)),
"application_name": "messenger-api",
# Autocommit connections: a single statement (post_message function, ping) is its
# own transaction with ONE round trip; multi-statement work uses an explicit
# conn.transaction() (BEGIN ... COMMIT), see run().
"autocommit": True,
# Detect a dead peer / lost network without any cancel request (see module doc).
"keepalives": 1,
"keepalives_idle": 5,
"keepalives_interval": 1,
"keepalives_count": 3,
"tcp_user_timeout": int(settings.db_timeout_seconds * 1000),
},
)
async def open(self) -> None:
# wait=False: the API starts (liveness 200) even if PostgreSQL is not up yet;
# readiness stays 503 until the database answers.
await self.pool.open(wait=False)
async def close(self) -> None:
await self.pool.close(timeout=2)
async def run(self, work: Work[T], deadline: float | None = None, *, transaction: bool = True) -> T:
"""Run `work(conn)` in one transaction (COMMIT on success, ROLLBACK on any exception).
transaction=False: `work` issues exactly ONE statement, which autocommit makes atomic.
Domain errors raised by `work` (NotFound, Conflict, ...) propagate unchanged;
database unavailability and the deadline become Unavailable (503).
"""
async def unit() -> T:
async with self.pool.connection() as conn:
if not transaction:
return await work(conn)
async with conn.transaction():
return await work(conn)
task = asyncio.ensure_future(unit())
try:
return await asyncio.wait_for(asyncio.shield(task), deadline or self.deadline)
except UNAVAILABLE_ERRORS as exc:
log.warning("database_unavailable", extra={"fields": {"error": type(exc).__name__}})
raise Unavailable("База данных временно недоступна") from None
finally:
if not task.done():
# Deadline or cancelled request: stop waiting now. The task is left to finish
# on its own (no task.cancel(): that would trigger psycopg's blocking cancel).
task.add_done_callback(_forget)
async def ping(self, timeout: float = 1.0) -> bool:
async def select_one(conn: psycopg.AsyncConnection) -> None:
await conn.execute("SELECT 1")
try:
await self.run(select_one, deadline=timeout, transaction=False)
return True
except Unavailable:
return False
+57
View File
@@ -0,0 +1,57 @@
"""Swagger UI for manual experiments: http://127.0.0.1:8080/docs
The page shows the course contract contracts/openapi.json as is (it is not
generated from our code). Only `servers` is replaced with "/" so that the
"Try it out" requests go to the same host the page was opened from.
Swagger UI itself (JS/CSS) is loaded by the browser from the jsDelivr CDN.
Disable with DOCS_ENABLED=false.
"""
from __future__ import annotations
import json
from functools import lru_cache
from pathlib import Path
from starlette.requests import Request
from starlette.responses import HTMLResponse, JSONResponse
SPEC_PATH = Path(__file__).resolve().parents[2] / "contracts" / "openapi.json"
SWAGGER_UI = "https://cdn.jsdelivr.net/npm/swagger-ui-dist@5.33.0"
PAGE = f"""<!doctype html>
<html lang="ru">
<head>
<meta charset="utf-8">
<title>Messenger lab1 — Swagger UI</title>
<link rel="stylesheet" href="{SWAGGER_UI}/swagger-ui.css">
</head>
<body>
<div id="swagger-ui"></div>
<script src="{SWAGGER_UI}/swagger-ui-bundle.js"></script>
<script>
window.ui = SwaggerUIBundle({{
url: "/openapi.json",
dom_id: "#swagger-ui",
tryItOutEnabled: true, // fields are editable right away
persistAuthorization: true, // X-User-Id from "Authorize" survives page reload
displayRequestDuration: true,
}});
</script>
</body>
</html>
"""
@lru_cache(maxsize=1)
def _spec() -> dict:
spec = json.loads(SPEC_PATH.read_text(encoding="utf-8"))
spec["servers"] = [{"url": "/", "description": "этот сервер"}]
return spec
async def swagger_page(request: Request) -> HTMLResponse:
return HTMLResponse(PAGE)
async def openapi_json(request: Request) -> JSONResponse:
return JSONResponse(_spec())
+71
View File
@@ -0,0 +1,71 @@
"""Domain errors mapped 1:1 to the contract error codes (contracts/README.md)."""
from __future__ import annotations
BEARER_CHALLENGE = 'Bearer realm="messenger"'
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
class ApiError(Exception):
status = 500
code = "internal_error"
default_message = "Внутренняя ошибка сервера"
def __init__(self, message: str | None = None):
self.message = message or self.default_message
super().__init__(self.message)
@property
def headers(self) -> dict[str, str]:
return {}
class InvalidRequest(ApiError):
status = 400
code = "invalid_request"
default_message = "Неверный запрос"
class Unauthorized(ApiError):
"""401 with the challenge of the scheme the endpoint expects (RFC 7235)."""
status = 401
code = "unauthorized"
default_message = "Требуется действительная сессия"
def __init__(self, message: str | None = None, challenge: str = BEARER_CHALLENGE):
super().__init__(message)
self.challenge = challenge
@property
def headers(self) -> dict[str, str]:
return {"WWW-Authenticate": self.challenge}
class NotFound(ApiError):
status = 404
code = "not_found"
default_message = "Объект не найден"
class Conflict(ApiError):
status = 409
code = "conflict"
default_message = "Конфликт"
class PayloadTooLarge(ApiError):
status = 413
code = "payload_too_large"
default_message = "Тело запроса слишком большое"
class UnsupportedMediaType(ApiError):
status = 415
code = "unsupported_media_type"
default_message = "Неподдерживаемый тип содержимого"
class Unavailable(ApiError):
status = 503
code = "unavailable"
default_message = "Сервис временно недоступен"
+19
View File
@@ -0,0 +1,19 @@
"""Docker HEALTHCHECK probe without curl/wget in the image: `python -m messenger.healthcheck`."""
from __future__ import annotations
import os
import sys
import urllib.request
def main() -> int:
port = os.environ.get("APP_PORT", "8080").strip() or "8080"
try:
with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/ready", timeout=2) as response:
return 0 if response.status == 200 else 1
except Exception:
return 1
if __name__ == "__main__":
sys.exit(main())
+328
View File
@@ -0,0 +1,328 @@
"""Content checks of uploaded images (lab 4): what the bytes ARE, not what the client says.
Two stages, both with bounded memory and both BEFORE anything is stored:
1. `sniff(head)` — runs on the first bytes while the body is still arriving. Recognises
PNG/JPEG by the signature and reads the dimensions from the header (PNG IHDR, JPEG SOFn).
A foreign format is rejected (415) and a picture with more than MAX_IMAGE_PIXELS pixels
(413) before the rest of the body is read and long before any pixel is decoded.
2. `verify(data, info)` — full integrity check of the complete file, run in a worker thread:
* PNG: our own streaming validator. Every chunk CRC is checked, the IDAT zlib stream is
inflated in 1 MiB steps and DISCARDED (no raster in memory), its size must be exactly
height x (1 + row bytes) (Adam7 per pass), every row filter byte must be 0..4, IEND
must close the file. A truncated file, a broken IDAT or extra/missing pixel data fail.
Pillow is not used for PNG: it would allocate the whole raster (40 Mpx RGBA = 160 MB).
* JPEG: libjpeg-turbo `djpeg -strict` decodes the whole entropy-coded data at 1/8 scale
(DCT scaling: the output raster is 64 times smaller) in a child process; any decoder
warning is fatal, so a truncated file or a file without scan data after SOS fails.
Every problem of the CONTENT is an ImageError carrying the HTTP status (415 or 413);
nothing here may turn user input into a 500.
"""
from __future__ import annotations
import asyncio
import os
import shutil
import struct
import zlib
from dataclasses import dataclass
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
JPEG_SOI = b"\xff\xd8\xff"
PNG = "image/png"
JPEG = "image/jpeg"
# Bits per pixel channel count by PNG colour type, and the bit depths allowed for it.
PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}}
# Adam7 passes: (x start, y start, x step, y step).
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
# JPEG start-of-frame markers that carry the frame size (not DHT C4, JPG C8, DAC CC).
JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}
# Standalone markers without a length field.
JPEG_STANDALONE = {0x01, *range(0xD0, 0xD8)}
DJPEG = shutil.which("djpeg") or "djpeg" # libjpeg-turbo-progs, installed in the Dockerfile
INPUT_STEP = 1 << 16 # compressed bytes handed to zlib at once
INFLATE_STEP = 1 << 20 # bytes of raw PNG data produced per zlib call
MAX_DIMENSION = 2**31 - 1
class ImageError(Exception):
"""The uploaded bytes are not an acceptable image. `status`: 415 (format/corrupt) or 413."""
def __init__(self, status: int, message: str):
super().__init__(message)
self.status = status
self.message = message
class VerifierUnavailable(Exception):
"""The checker itself could not run to the end (resources): the API answers 503, retryable."""
def unsupported(message: str ="Файл не является корректным изображением PNG или JPEG") -> ImageError:
return ImageError(415, message)
@dataclass(frozen=True, slots=True)
class ImageInfo:
content_type: str
width: int
height: int
def detect(head: bytes) -> str | None:
"""Format by signature; None while fewer bytes than a signature have arrived."""
if head[:8] == PNG_SIGNATURE:
return PNG
if head[:3] == JPEG_SOI:
return JPEG
probe = min(len(head), 8)
if PNG_SIGNATURE.startswith(head[:probe]) or JPEG_SOI.startswith(head[:min(probe, 3)]):
return None
raise unsupported()
def sniff(head: bytes, max_pixels: int) -> ImageInfo | None:
"""Format and size from the beginning of the file, or None if more bytes are needed.
Raises ImageError: 415 for a foreign/broken header, 413 for too many pixels."""
kind = detect(head)
if kind is None:
return None
info = _png_header(head) if kind == PNG else _jpeg_header(head)
if info is None:
return None
if info.width * info.height > max_pixels:
raise ImageError(413, f"Изображение больше {max_pixels} пикселей")
return info
def _png_header(head: bytes) -> ImageInfo | None:
if len(head) < 33:
return None
length, ctype = struct.unpack(">I4s", head[8:16])
if ctype != b"IHDR" or length != 13:
raise unsupported()
if zlib.crc32(head[12:29]) != struct.unpack(">I", head[29:33])[0]:
raise unsupported()
width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", head[16:29])
if not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION):
raise unsupported()
if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1:
raise unsupported()
return ImageInfo(PNG, width, height)
def _jpeg_header(head: bytes) -> ImageInfo | None:
"""Walk the marker segments up to the first SOFn. None = the SOF has not arrived yet."""
i = 2
n = len(head)
while True:
if i >= n:
return None
if head[i] != 0xFF:
raise unsupported()
while i < n and head[i] == 0xFF: # fill bytes
i += 1
if i >= n:
return None
marker = head[i]
i += 1
if marker in JPEG_STANDALONE:
continue
if marker in (0xD8, 0xD9, 0xDA, 0x00):
# Second SOI, EOI or scan data before any frame header: not a usable JPEG.
raise unsupported()
if i + 2 > n:
return None
length = struct.unpack(">H", head[i:i + 2])[0]
if length < 2:
raise unsupported()
if marker in JPEG_SOF:
if i + 7 > n:
return None
if length < 8:
raise unsupported()
height, width = struct.unpack(">HH", head[i + 3:i + 7])
if width == 0 or height == 0: # height 0 = DNL-defined height: not supported
raise unsupported()
return ImageInfo(JPEG, width, height)
i += length
# ------------------------------------------------------------------ full verification
async def verify(data: bytes | bytearray | memoryview, info: ImageInfo, *, timeout: float = 10.0,
jpeg_max_memory_mb: int = 256) -> None:
"""Complete integrity check of the whole file. PNG: our validator in a worker thread (zlib
releases the GIL); JPEG: a strict libjpeg decoder in a child process."""
if info.content_type == PNG:
await asyncio.wait_for(asyncio.to_thread(verify_png, data), timeout)
else:
await verify_jpeg(data, timeout, jpeg_max_memory_mb)
def _png_segments(width: int, height: int, depth: int, color: int, interlace: int) -> list[tuple[int, int]]:
"""(byte count, row stride) of every pass of the raw (filtered) image data."""
bpp = PNG_CHANNELS[color] * depth
passes = ADAM7 if interlace else ((0, 0, 1, 1),)
segments = []
for xs, ys, dx, dy in passes:
pw = (width - xs + dx - 1) // dx if width > xs else 0
ph = (height - ys + dy - 1) // dy if height > ys else 0
if pw and ph:
stride = 1 + (pw * bpp + 7) // 8
segments.append((ph * stride, stride))
return segments
class _RawChecker:
"""Counts inflated bytes against the expected size and checks the filter byte of every row
using C-speed slicing (`piece[first::stride]`), so even 40 000 000 one-pixel rows are fast."""
VALID_FILTERS = bytes(range(5))
def __init__(self, segments: list[tuple[int, int]]):
self.segments = segments
self.index = 0 # current pass
self.offset = 0 # bytes already seen inside the current pass
self.expected = sum(size for size, _ in segments)
self.total = 0
def feed(self, piece: bytes) -> None:
self.total += len(piece)
if self.total > self.expected:
raise unsupported()
pos = 0
while pos < len(piece):
size, stride = self.segments[self.index]
take = min(size - self.offset, len(piece) - pos)
part = piece[pos:pos + take]
first = (-self.offset) % stride
if part[first::stride].translate(None, self.VALID_FILTERS):
raise unsupported()
self.offset += take
pos += take
if self.offset == size:
self.index += 1
self.offset = 0
def verify_png(data: bytes | bytearray | memoryview) -> None:
view = memoryview(data)
if bytes(view[:8]) != PNG_SIGNATURE:
raise unsupported()
pos = 8
n = len(view)
checker: _RawChecker | None = None
inflater = zlib.decompressobj()
color = None
seen_plte = seen_idat = idat_done = False
first = True
while True:
if pos + 12 > n:
raise unsupported() # truncated: no room for another chunk header + CRC
length, ctype = struct.unpack(">I4s", view[pos:pos + 8])
if length > n - pos - 12:
raise unsupported()
body = view[pos + 8:pos + 8 + length]
crc = struct.unpack(">I", view[pos + 8 + length:pos + 12 + length])[0]
if zlib.crc32(body, zlib.crc32(ctype)) != crc:
raise unsupported()
pos += 12 + length
if not all(65 <= c <= 90 or 97 <= c <= 122 for c in ctype):
raise unsupported()
if first:
if ctype != b"IHDR" or length != 13:
raise unsupported()
width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", body)
if (not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION) or color not in PNG_DEPTHS
or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1):
raise unsupported()
checker = _RawChecker(_png_segments(width, height, depth, color, interlace))
first = False
continue
assert checker is not None
if ctype == b"IHDR":
raise unsupported()
if ctype == b"PLTE":
if seen_plte or seen_idat or color in (0, 4) or length % 3 or not 3 <= length <= 768:
raise unsupported()
seen_plte = True
elif ctype == b"IDAT":
if idat_done:
raise unsupported() # IDAT chunks must be consecutive
if color == 3 and not seen_plte:
raise unsupported()
seen_idat = True
_inflate(inflater, body, checker)
elif ctype == b"IEND":
if length != 0 or not seen_idat:
raise unsupported()
if pos != n:
raise unsupported() # bytes after IEND (polyglot files) are not accepted
break
elif not 97 <= ctype[0] <= 122:
raise unsupported() # unknown CRITICAL chunk: a decoder could not show the image
if ctype != b"IDAT" and seen_idat:
idat_done = True
# The zlib stream must end exactly with the last pixel row, without trailing data.
if not inflater.eof or inflater.unused_data or checker.total != checker.expected:
raise unsupported()
def _inflate(inflater, body: memoryview, checker: _RawChecker) -> None:
if inflater.eof:
if len(body):
raise unsupported() # data after the end of the zlib stream
return
# Input is fed in small slices: zlib's `unconsumed_tail` is a COPY of the unread input, so
# feeding the whole 10 MiB IDAT at once would copy the rest of it after every output step.
try:
for start in range(0, len(body), INPUT_STEP):
pending: bytes | memoryview = body[start:start + INPUT_STEP]
while pending:
if inflater.eof:
raise unsupported() # data after the end of the zlib stream
checker.feed(inflater.decompress(pending, INFLATE_STEP))
pending = inflater.unconsumed_tail
if inflater.eof and inflater.unused_data:
raise unsupported()
except zlib.error:
raise unsupported() from None
async def verify_jpeg(data: bytes | bytearray | memoryview, timeout: float, max_memory_mb: int) -> None:
"""Decode the WHOLE file with the reference decoder (libjpeg-turbo `djpeg`) in strict mode.
`-strict` turns every libjpeg warning into a failure: "premature end of data segment"
(scan data missing or cut before EOI), "premature end of JPEG file", corrupt Huffman data.
Pillow cannot be used for this: it ignores libjpeg warnings and returns a grey picture for
a JPEG whose scan data is missing. `-scale 1/8` decodes every block but writes a 64x
smaller raster to /dev/null; `-maxmemory` caps libjpeg's working memory (coefficient
buffers of progressive files); the separate process is killed at the deadline.
"""
process = await asyncio.create_subprocess_exec(
DJPEG, "-strict", "-scale", "1/8", "-maxmemory", str(max_memory_mb * 1024), "-outfile", os.devnull,
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE)
stderr = b""
try:
_, stderr = await asyncio.wait_for(process.communicate(data), timeout)
except (asyncio.TimeoutError, asyncio.CancelledError):
process.kill()
await process.wait()
raise
except (BrokenPipeError, ConnectionResetError):
await process.wait() # djpeg stopped reading: the file is already known to be broken
if process.returncode is not None and process.returncode < 0:
# Killed by a signal (the cgroup OOM killer under memory pressure): a problem of the
# SERVER, not of the file - never report it as 415.
raise VerifierUnavailable()
if process.returncode != 0:
if b"Insufficient memory" in (stderr or b""):
# A valid but extremely memory-hungry file (progressive, 4:4:4, ~40 Mpx): a limit, not corruption.
raise ImageError(413, "Изображение слишком велико для обработки")
raise unsupported()
+36
View File
@@ -0,0 +1,36 @@
"""JSON-lines logging to stdout. Logs never contain message texts or request bodies."""
from __future__ import annotations
import datetime as dt
import json
import logging
import sys
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
entry = {
"ts": dt.datetime.fromtimestamp(record.created, dt.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z"),
"level": record.levelname.lower(),
"logger": record.name,
"event": record.getMessage(),
}
fields = getattr(record, "fields", None)
if fields:
entry.update(fields)
if record.exc_info:
# Stack traces go only to the server log, never to the HTTP client.
entry["exception"] = self.formatException(record.exc_info)
return json.dumps(entry, ensure_ascii=False)
def configure(level: str) -> None:
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
root = logging.getLogger()
root.handlers[:] = [handler]
root.setLevel(level.upper())
# uvicorn's own access log is disabled; our middleware writes one line per request.
for name in ("uvicorn", "uvicorn.error"):
logging.getLogger(name).handlers[:] = []
logging.getLogger(name).propagate = True
+76
View File
@@ -0,0 +1,76 @@
"""Versioned SQL migrations: `python -m messenger.migrate` (compose service `migrate`).
* Files `migrations/NNNN_name.sql` are applied in name order, each in its own
transaction together with its row in `schema_migrations` (DDL is transactional
in PostgreSQL, so a failed migration leaves nothing behind).
* Re-running is safe: applied versions are skipped, so data is never duplicated.
* The SHA-256 of every applied file is stored; if an applied file was edited later,
the run stops instead of silently diverging from the real schema.
* `pg_advisory_lock` serialises concurrent runs (two API replicas starting at once).
"""
from __future__ import annotations
import hashlib
import sys
import time
from pathlib import Path
import psycopg
from .config import ConfigError, _secret
MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / "migrations"
LOCK_ID = 2026_02 # arbitrary application-wide advisory lock id
def say(text: str) -> None:
print(f"migrate: {text}", flush=True)
def connect(url: str, wait_seconds: float = 60) -> psycopg.Connection:
deadline = time.monotonic() + wait_seconds
while True:
try:
return psycopg.connect(url, autocommit=True, connect_timeout=3, application_name="messenger-migrate")
except psycopg.OperationalError as exc:
if time.monotonic() > deadline:
raise SystemExit(f"migrate: database is not reachable: {type(exc).__name__}") from None
time.sleep(1)
def main() -> int:
try:
url = _secret("DATABASE_URL", required=True)
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
files = sorted(MIGRATIONS_DIR.glob("*.sql"))
with connect(url) as conn:
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
" version text PRIMARY KEY, checksum text NOT NULL,"
" applied_at timestamptz NOT NULL DEFAULT now())")
applied = dict(conn.execute("SELECT version, checksum FROM schema_migrations").fetchall())
for path in files:
version, sql = path.stem, path.read_bytes()
checksum = hashlib.sha256(sql).hexdigest()
if version in applied:
if applied[version] != checksum:
say(f"{version}: file changed after it was applied; create a new migration instead")
return 1
say(f"{version}: already applied, skipped")
continue
with conn.transaction():
conn.execute(sql.decode("utf-8"))
conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)", (version, checksum))
say(f"{version}: applied")
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
say(f"done, {len(files)} migration file(s) checked")
return 0
if __name__ == "__main__":
sys.exit(main())
+53
View File
@@ -0,0 +1,53 @@
"""Password hashing with Argon2id (memory-hard password KDF; also mandatory in lab 7).
* Parameters follow the OWASP minimum for Argon2id: m=19 MiB, t=2, p=1. Unlike bcrypt
there is no 72-byte limit, so any 12-128 code point UTF-8 password is hashed whole.
* Hashing takes tens of milliseconds of CPU and 19 MiB of RAM. It runs in a worker
thread (argon2-cffi releases the GIL), so the event loop keeps serving other
requests, and a semaphore caps parallel hashes (memory/CPU DoS protection).
* An unknown username is checked against a dummy hash, so "no such user" and
"wrong password" take the same time and return the same 401.
"""
from __future__ import annotations
import asyncio
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
_hasher = PasswordHasher(time_cost=2, memory_cost=19 * 1024, parallelism=1, hash_len=32, salt_len=16)
_semaphore: asyncio.Semaphore | None = None
_dummy_hash: str | None = None
def configure(concurrency: int) -> None:
global _semaphore
_semaphore = asyncio.Semaphore(concurrency)
async def _run(func, *args):
if _semaphore is None:
configure(2)
async with _semaphore:
return await asyncio.to_thread(func, *args)
async def hash_password(password: str) -> str:
return await _run(_hasher.hash, password)
def _verify(stored_hash: str, password: str) -> bool:
try:
return _hasher.verify(stored_hash, password)
except (VerifyMismatchError, VerificationError, InvalidHashError):
return False
async def verify_password(stored_hash: str | None, password: str) -> bool:
global _dummy_hash
if stored_hash is None:
if _dummy_hash is None:
_dummy_hash = await hash_password("dummy-password-for-timing")
await _run(_verify, _dummy_hash, password)
return False
return await _run(_verify, stored_hash, password)
+437
View File
@@ -0,0 +1,437 @@
"""Reconciler of the two stores (lab 4, track 2): S3 objects vs attachment rows in PostgreSQL.
docker compose run --rm reconciler # dry-run: report only (default)
docker compose run --rm reconciler --apply # fix what is safe to fix
docker compose run --rm reconciler --verify-checksums # also read every object (SHA-256)
How it walks: both sides are read as key-ordered streams and merge-joined - ListObjectsV2
pages (prefix o/, byte order) against keyset pages of `attachments ORDER BY object_key`
(COLLATE "C" = the same byte order, unique index). Memory does not depend on the bucket size;
cost is ~N/1000 S3 list requests + N/page DB pages (measured: docs/CONSISTENCY.md).
What it finds and what --apply does (nothing else is ever changed):
kind condition --apply
orphan_object object, no row, older than --grace delete the object
stale_intent row 'pending' older than --grace (+/- object) delete object, then row
missing_object row 'ready', object absent, older than --grace quarantine the row
size_mismatch row 'ready', object size != size_bytes quarantine the row
checksum_mismatch row 'ready', SHA-256 differs (--verify-checksums) quarantine the row
restored_object row quarantined, object present and correct again remove the quarantine
(a checksum quarantine only with --verify-checksums)
fresh_* the same, but younger than --grace nothing (in-flight upload)
Safety:
* dry-run is read-only BY CONSTRUCTION: the PostgreSQL session runs with
default_transaction_read_only=on and the S3 wrapper has no write methods, so even a bug in
the planner cannot modify a store;
* ready rows are never deleted - a used (or unused) ready attachment can only be quarantined,
which is reversible; objects that have a row are never deleted unless the row is a stale
intent; anything younger than --grace is never touched;
* every action re-checks its condition right before acting, under a row lock (FOR UPDATE)
where a row exists, so the live API racing with the reconciler cannot be hurt;
* object first, row second: an interrupted run leaves a state the next run repairs;
re-running is idempotent (the second run finds nothing to do);
* safety valve: --apply refuses to delete more than --max-orphan-fraction of the scanned
objects (pointing it at the wrong/empty database would otherwise wipe the bucket).
"""
from __future__ import annotations
import argparse
import asyncio
import datetime as dt
import hashlib
import json
import sys
import time
from collections import Counter
from collections.abc import AsyncIterator
from dataclasses import dataclass, field
import psycopg
from .config import ConfigError, Settings, _float, _secret, s3_settings
from .errors import Unavailable
from .storage import OBJECT_PREFIX, ListedObject, ObjectMissing, Storage
KINDS = ("orphan_object", "stale_intent", "missing_object", "size_mismatch", "checksum_mismatch",
"restored_object", "fresh_orphan_object", "fresh_intent", "fresh_missing_object")
MIN_SAFE_GRACE = 60.0 # > the longest possible upload (queue 5 s + body 30 s + verify 10 s + S3)
ACTION_OF = {"orphan_object": "delete_object", "stale_intent": "delete_intent", "missing_object": "quarantine",
"size_mismatch": "quarantine", "checksum_mismatch": "quarantine", "restored_object": "unquarantine"}
def log(text: str) -> None:
print(f"reconcile: {text}", file=sys.stderr, flush=True)
@dataclass(frozen=True, slots=True)
class Row:
id: str
key: str
status: str
size: int
sha256: bytes
created_at: dt.datetime
updated_at: dt.datetime
lost_reason: str | None
used: bool
@dataclass(slots=True)
class Finding:
kind: str
key: str
attachment_id: str | None = None
used: bool = False
detail: str = ""
result: str = "planned" # planned | applied | skipped:<why> | failed:<why>
@dataclass
class Stats:
objects: int = 0
rows: int = 0
consistent: int = 0
s3_list_requests: int = 0
s3_head_requests: int = 0
s3_get_bytes: int = 0
db_pages: int = 0
class ReadOnlyStorage:
"""The dry-run view of S3: listing and reading only - there is no delete to call."""
def __init__(self, storage: Storage):
self._s = storage
async def list_page(self, *args, **kwargs):
return await self._s.list_page(*args, **kwargs)
async def head(self, key: str):
return await self._s.head(key)
async def read_all(self, key: str, limit: int) -> bytes:
return await self._s.read_all(key, limit)
class Reconciler:
def __init__(self, conn: psycopg.AsyncConnection, storage: Storage | ReadOnlyStorage, *, grace: float,
page_size: int, verify: bool, apply: bool):
self.conn = conn
self.s3 = storage
self.grace = grace
self.page_size = page_size
self.verify = verify
self.apply = apply
self.stats = Stats()
self.findings: list[Finding] = []
# ---------------------------------------------------------- streams
async def objects(self) -> AsyncIterator[ListedObject]:
after = None
while True:
page, more = await self.s3.list_page(OBJECT_PREFIX, after, self.page_size)
self.stats.s3_list_requests += 1
for item in page:
self.stats.objects += 1
yield item
if not more or not page:
return
after = page[-1].key
async def rows(self) -> AsyncIterator[Row]:
after = ""
while True:
cur = await self.conn.execute(
"SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256, a.created_at, a.updated_at, "
"a.lost_reason, EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) "
"FROM attachments a WHERE a.object_key > %s ORDER BY a.object_key LIMIT %s",
(after, self.page_size))
page = await cur.fetchall()
self.stats.db_pages += 1
for r in page:
self.stats.rows += 1
yield Row(str(r[0]), r[1], r[2], r[3], bytes(r[4]), r[5], r[6], r[7], r[8])
if len(page) < self.page_size:
return
after = page[-1][1]
# ---------------------------------------------------------- planning (read-only)
async def scan(self) -> None:
cur = await self.conn.execute("SELECT clock_timestamp()")
self.db_now = (await cur.fetchone())[0]
self.s3_now = time.time()
objects, rows = self.objects(), self.rows()
obj = await anext(objects, None)
row = await anext(rows, None)
while obj is not None or row is not None:
if row is None or (obj is not None and obj.key < row.key):
self.object_only(obj)
obj = await anext(objects, None)
elif obj is None or row.key < obj.key:
self.row_only(row)
row = await anext(rows, None)
else:
await self.both(obj, row)
obj = await anext(objects, None)
row = await anext(rows, None)
def row_old(self, row: Row) -> bool:
return (self.db_now - max(row.created_at, row.updated_at)).total_seconds() > self.grace
def object_only(self, obj: ListedObject) -> None:
old = self.s3_now - obj.last_modified_ts > self.grace
self.add(Finding("orphan_object" if old else "fresh_orphan_object", obj.key))
def row_only(self, row: Row) -> None:
if row.status == "pending":
self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id,
detail="object absent"))
elif row.lost_reason is not None:
self.stats.consistent += 1 # already quarantined, still missing: nothing new
else:
self.add(Finding("missing_object" if self.row_old(row) else "fresh_missing_object", row.key, row.id,
used=row.used))
async def both(self, obj: ListedObject, row: Row) -> None:
if row.status == "pending":
self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id,
detail="object present"))
return
problem = await self.object_problem(row, obj.size)
if problem:
if row.lost_reason is None:
self.add(Finding(problem, row.key, row.id, used=row.used))
else:
self.stats.consistent += 1 # quarantined and still broken
elif row.lost_reason == "checksum_mismatch" and not self.verify:
# Same size, but this pass does not read the bytes: it cannot prove the object is the
# original again, so a checksum quarantine is lifted only by --verify-checksums.
self.stats.consistent += 1
elif row.lost_reason is not None:
self.add(Finding("restored_object", row.key, row.id, used=row.used))
else:
self.stats.consistent += 1
async def object_problem(self, row: Row, size: int) -> str | None:
if size != row.size:
return "size_mismatch"
if self.verify:
data = await self.s3.read_all(row.key, row.size)
self.stats.s3_get_bytes += len(data)
if hashlib.sha256(data).digest() != row.sha256:
return "checksum_mismatch"
return None
def add(self, finding: Finding) -> None:
if finding.kind not in ACTION_OF:
finding.result = "not_actionable"
self.findings.append(finding)
# ---------------------------------------------------------- actions (--apply only)
async def run_actions(self, max_orphan_fraction: float) -> None:
orphans = sum(1 for f in self.findings if f.kind == "orphan_object")
if self.stats.objects and orphans > max_orphan_fraction * self.stats.objects:
raise SafetyValve(f"{orphans} of {self.stats.objects} objects look orphaned "
f"(> {max_orphan_fraction:.0%}); wrong database? nothing was changed")
handlers = {"delete_object": self.delete_object, "delete_intent": self.delete_intent,
"quarantine": self.quarantine, "unquarantine": self.unquarantine}
for finding in self.findings:
action = ACTION_OF.get(finding.kind)
if action is None:
continue
try:
finding.result = await handlers[action](finding)
except (Unavailable, psycopg.OperationalError) as exc:
finding.result = f"failed:{type(exc).__name__}"
raise # a store is down: stop; the next run continues from the committed state
async def delete_object(self, f: Finding) -> str:
cur = await self.conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (f.key,))
if await cur.fetchone():
return "skipped:row_appeared"
try:
head = await self.s3.head(f.key)
self.stats.s3_head_requests += 1
except ObjectMissing:
return "skipped:already_gone"
if time.time() - head.last_modified_ts <= self.grace:
return "skipped:object_rewritten_recently"
# No row names this key and the object is older than the grace period. Rows are always
# written BEFORE their object and keys are random and never reused, so no upload can
# still claim it.
await self.s3.delete(f.key)
return "applied"
async def delete_intent(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT 1 FROM attachments WHERE id = %s AND status = 'pending' "
"AND greatest(created_at, updated_at) < clock_timestamp() - make_interval(secs => %s) FOR UPDATE",
(f.attachment_id, self.grace))
if await cur.fetchone() is None:
return "skipped:no_longer_stale_intent"
await self.s3.delete(f.key) # object first (idempotent), then the row
await self.conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'pending'",
(f.attachment_id,))
return "applied"
async def quarantine(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT size_bytes, sha256 FROM attachments WHERE id = %s AND status = 'ready' "
"AND lost_at IS NULL FOR UPDATE", (f.attachment_id,))
row = await cur.fetchone()
if row is None:
return "skipped:row_changed"
reason = await self.current_problem(f.key, row[0], bytes(row[1]))
if reason is None:
return "skipped:object_ok_now"
await self.conn.execute(
"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = %s WHERE id = %s",
(reason, f.attachment_id))
return "applied"
async def unquarantine(self, f: Finding) -> str:
async with self.conn.transaction():
cur = await self.conn.execute(
"SELECT size_bytes, sha256, lost_reason FROM attachments WHERE id = %s AND lost_at IS NOT NULL FOR UPDATE",
(f.attachment_id,))
row = await cur.fetchone()
if row is None:
return "skipped:row_changed"
if row[2] == "checksum_mismatch" and not self.verify:
return "skipped:needs_verify_checksums"
if await self.current_problem(f.key, row[0], bytes(row[1])) is not None:
return "skipped:object_still_bad"
await self.conn.execute("UPDATE attachments SET lost_at = NULL, lost_reason = NULL WHERE id = %s",
(f.attachment_id,))
return "applied"
async def current_problem(self, key: str, size: int, sha256: bytes) -> str | None:
"""Fresh look at the object right before changing the row (the listing may be stale)."""
try:
head = await self.s3.head(key)
self.stats.s3_head_requests += 1
except ObjectMissing:
return "missing_object"
if head.size != size:
return "size_mismatch"
if self.verify:
data = await self.s3.read_all(key, size)
self.stats.s3_get_bytes += len(data)
if hashlib.sha256(data).digest() != sha256:
return "checksum_mismatch"
return None
# ---------------------------------------------------------- report
def report(self, mode: str, seconds: float, error: str | None, sample: int) -> dict:
kinds = Counter(f.kind for f in self.findings)
results = Counter(f.result.split(":")[0] for f in self.findings if f.kind in ACTION_OF)
samples = {}
for f in self.findings:
items = samples.setdefault(f.kind, [])
if len(items) < sample:
items.append({"key": f.key, "attachment_id": f.attachment_id, "used": f.used,
"result": f.result, **({"detail": f.detail} if f.detail else {})})
return {
"mode": mode, "grace_seconds": self.grace, "verify_checksums": self.verify,
"duration_seconds": round(seconds, 3), "error": error,
"scanned": {"objects": self.stats.objects, "rows": self.stats.rows, "consistent": self.stats.consistent},
"findings": {k: kinds.get(k, 0) for k in KINDS},
"used_attachments_in_findings": sum(1 for f in self.findings if f.used),
"actions": {"planned": sum(1 for f in self.findings if f.kind in ACTION_OF),
"applied": results.get("applied", 0), "skipped": results.get("skipped", 0),
"failed": results.get("failed", 0)},
"cost": {"s3_list_requests": self.stats.s3_list_requests, "s3_head_requests": self.stats.s3_head_requests,
"s3_get_bytes": self.stats.s3_get_bytes, "db_pages": self.stats.db_pages,
"page_size": self.page_size},
"samples": samples,
}
class SafetyValve(Exception):
pass
def arguments(argv: list[str] | None = None) -> argparse.Namespace:
p = argparse.ArgumentParser(prog="python -m messenger.reconcile", description=__doc__.split("\n")[0])
p.add_argument("--apply", action="store_true", help="fix findings (default: dry-run, read-only)")
p.add_argument("--grace", type=float, default=3600.0,
help="seconds; anything younger is never touched (default 3600, >= PENDING_GRACE_SECONDS)")
p.add_argument("--allow-small-grace", action="store_true",
help="permit --grace below 60 s (tests only: a grace shorter than an upload could remove the "
"intent of an upload that is still running; that upload then fails with 503)")
p.add_argument("--page-size", type=int, default=1000, help="S3 list / DB page size (1..1000)")
p.add_argument("--verify-checksums", action="store_true", help="read every ready object and compare SHA-256")
p.add_argument("--samples", type=int, default=20, help="keys listed per finding kind in the report")
p.add_argument("--max-orphan-fraction", type=float, default=0.5,
help="--apply refuses if more than this share of objects look orphaned (default 0.5)")
args = p.parse_args(argv)
if not 1 <= args.page_size <= 1000:
p.error("--page-size must be 1..1000")
if args.grace < 1 or (args.grace < MIN_SAFE_GRACE and not args.allow_small_grace):
p.error(f"--grace must be >= {MIN_SAFE_GRACE:.0f} s (or >= 1 s with --allow-small-grace)")
if not 0 < args.max_orphan_fraction <= 1:
p.error("--max-orphan-fraction must be in (0, 1]")
return args
async def run(args: argparse.Namespace) -> int:
url = _secret("DATABASE_URL", required=True)
base = s3_settings()
settings = Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024,
database_url=url, **base)
timeout_ms = int(_float("DB_TIMEOUT_SECONDS", 10.0, 1.0, 120.0) * 1000)
options = f"-c statement_timeout={timeout_ms} -c lock_timeout={timeout_ms // 2}"
if not args.apply:
options += " -c default_transaction_read_only=on" # dry-run: PostgreSQL itself refuses writes
mode = "apply" if args.apply else "dry-run"
storage = Storage(settings)
await storage.open()
started = time.perf_counter()
error = None
code = 0
try:
conn = await psycopg.AsyncConnection.connect(url, autocommit=True, connect_timeout=5, options=options,
application_name=f"messenger-reconcile-{mode}")
except psycopg.OperationalError as exc:
await storage.close()
error = f"store_unavailable: database {type(exc).__name__}"
print(json.dumps({"mode": mode, "error": error}, ensure_ascii=False))
log(error)
return 1
rec = Reconciler(conn, storage if args.apply else ReadOnlyStorage(storage), grace=args.grace,
page_size=args.page_size, verify=args.verify_checksums, apply=args.apply)
try:
log(f"{mode}: scanning bucket {settings.s3_bucket} (prefix {OBJECT_PREFIX}) against table attachments")
await rec.scan()
log(f"scan done: {rec.stats.objects} objects, {rec.stats.rows} rows, {len(rec.findings)} findings")
if args.apply:
await rec.run_actions(args.max_orphan_fraction)
except SafetyValve as exc:
error, code = f"safety_valve: {exc}", 3
except (Unavailable, psycopg.OperationalError) as exc:
error, code = f"store_unavailable: {type(exc).__name__}", 1
finally:
await conn.close()
await storage.close()
print(json.dumps(rec.report(mode, time.perf_counter() - started, error, args.samples), ensure_ascii=False,
indent=1))
if error:
log(error)
return code
def main(argv: list[str] | None = None) -> int:
args = arguments(argv)
try:
return asyncio.run(run(args))
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())
+372
View File
@@ -0,0 +1,372 @@
"""SQL for users, chats and messages. All statements are parameterised (%s placeholders):
values travel separately from the SQL text, so input can never change the query.
Free text is stored as UTF-8 BYTEA (PostgreSQL TEXT cannot hold U+0000), so it is
encoded on the way in and decoded on the way out. See docs/DATA_MODEL.md.
"""
from __future__ import annotations
import datetime as dt
import uuid
from dataclasses import dataclass
import psycopg
from psycopg import errors as pg_errors
from .cursor import SortKey
from .db import Database
from .errors import Conflict, InvalidRequest, NotFound, Unavailable
from .timefmt import from_us, rfc3339, to_us
def _b(text: str) -> bytes:
return text.encode("utf-8")
def _s(raw: bytes | memoryview) -> str:
return bytes(raw).decode("utf-8")
@dataclass(slots=True, frozen=True)
class User:
id: str
username: str
display_name: str
created_at: dt.datetime
def to_json(self) -> dict:
return {"id": self.id, "username": self.username, "display_name": self.display_name,
"created_at": rfc3339(self.created_at)}
@dataclass(slots=True, frozen=True)
class Chat:
id: str
title: str
member_ids: tuple[str, ...]
created_at: dt.datetime
@property
def key(self) -> SortKey:
return (to_us(self.created_at), self.id)
def to_json(self) -> dict:
return {"id": self.id, "title": self.title, "member_ids": list(self.member_ids),
"created_at": rfc3339(self.created_at)}
@dataclass(slots=True, frozen=True)
class Message:
id: str
chat_id: str
sender_id: str
text: str
created_at: dt.datetime
attachment_ids: tuple[str, ...] = ()
@property
def key(self) -> SortKey:
return (to_us(self.created_at), self.id)
def to_json(self) -> dict:
return {"id": self.id, "chat_id": self.chat_id, "sender_id": self.sender_id, "text": self.text,
"created_at": rfc3339(self.created_at), "attachment_ids": list(self.attachment_ids)}
def _user(row) -> User:
return User(str(row[0]), row[1], _s(row[2]), row[3])
# ------------------------------------------------------------------ users
async def create_user(db: Database, username: str, display_name: str, password_hash: str) -> User:
async def work(conn: psycopg.AsyncConnection) -> User:
cur = await conn.execute(
"INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) "
"RETURNING id, username, display_name, created_at",
(uuid.uuid4(), username, _b(display_name), password_hash))
return _user(await cur.fetchone())
try:
return await db.run(work)
except pg_errors.UniqueViolation:
# Two concurrent registrations: the unique index lets exactly one INSERT commit.
raise Conflict("Имя пользователя уже занято") from None
async def get_user(db: Database, user_id: str) -> User | None:
async def work(conn: psycopg.AsyncConnection) -> User | None:
cur = await conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,))
row = await cur.fetchone()
return _user(row) if row else None
return await db.run(work)
async def get_credentials(db: Database, username: str) -> tuple[User, str] | None:
async def work(conn: psycopg.AsyncConnection) -> tuple[User, str] | None:
cur = await conn.execute(
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
(username,))
row = await cur.fetchone()
return (_user(row), row[4]) if row else None
return await db.run(work)
# ------------------------------------------------------------------ chats
async def create_chat(db: Database, creator_id: str, title: str, member_ids: list[str]) -> Chat:
"""Atomic: the chat row and all memberships commit together or not at all.
The chat is inserted FIRST; an unknown member then fails the membership INSERT
with a foreign-key violation and the whole transaction (chat row included) is
rolled back. student-tests check that nothing is left behind.
"""
if creator_id not in member_ids:
raise InvalidRequest("member_ids должен включать текущего пользователя")
chat_id = uuid.uuid4()
async def work(conn: psycopg.AsyncConnection) -> dt.datetime:
cur = await conn.execute(
"INSERT INTO chats (id, title, created_by) VALUES (%s, %s, %s) RETURNING created_at",
(chat_id, _b(title), creator_id))
(created_at,) = await cur.fetchone()
await conn.execute(
"INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) "
"SELECT %s, m.user_id, m.ord - 1, %s "
"FROM unnest(%s::uuid[]) WITH ORDINALITY AS m(user_id, ord)",
(chat_id, created_at, member_ids))
return created_at
try:
created_at = await db.run(work)
except pg_errors.ForeignKeyViolation:
raise InvalidRequest("member_ids содержит несуществующих пользователей") from None
return Chat(str(chat_id), title, tuple(member_ids), created_at)
async def _load_chats(conn: psycopg.AsyncConnection, rows) -> list[Chat]:
if not rows:
return []
ids = [r[0] for r in rows]
cur = await conn.execute(
"SELECT chat_id, user_id FROM chat_members WHERE chat_id = ANY(%s) ORDER BY chat_id, position", (ids,))
members: dict[uuid.UUID, list[str]] = {i: [] for i in ids}
for chat_id, user_id in await cur.fetchall():
members[chat_id].append(str(user_id))
return [Chat(str(r[0]), _s(r[1]), tuple(members[r[0]]), r[2]) for r in rows]
async def get_chat(db: Database, user_id: str, chat_id: str) -> Chat:
"""The chat if the user is a member; a missing chat and a foreign chat are both 404."""
async def work(conn: psycopg.AsyncConnection) -> Chat:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
(user_id, chat_id))
row = await cur.fetchone()
if row is None:
raise NotFound("Чат не найден")
return (await _load_chats(conn, [row]))[0]
return await db.run(work)
async def _check_member(conn: psycopg.AsyncConnection, user_id: str, chat_id: str) -> None:
cur = await conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id))
if await cur.fetchone() is None:
raise NotFound("Чат не найден")
async def ensure_member(db: Database, user_id: str, chat_id: str) -> None:
async def work(conn: psycopg.AsyncConnection) -> None:
await _check_member(conn, user_id, chat_id)
await db.run(work)
async def list_chats(db: Database, user_id: str, after: SortKey | None, limit: int) -> tuple[list[Chat], bool]:
async def work(conn: psycopg.AsyncConnection) -> tuple[list[Chat], bool]:
if after is None:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id "
"WHERE m.user_id = %s ORDER BY m.chat_created_at, m.chat_id LIMIT %s",
(user_id, limit + 1))
else:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id "
"WHERE m.user_id = %s AND (m.chat_created_at, m.chat_id) > (%s, %s) "
"ORDER BY m.chat_created_at, m.chat_id LIMIT %s",
(user_id, from_us(after[0]), after[1], limit + 1))
rows = await cur.fetchall()
return await _load_chats(conn, rows[:limit]), len(rows) > limit
return await db.run(work)
# ------------------------------------------------------------------ messages
@dataclass(slots=True, frozen=True)
class PostResult:
message: Message
replayed: bool # True: the key was already used with this body, nothing new was written
async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, user_id: str, chat_id: str,
text: str, attachment_ids: list[str]) -> dt.datetime:
"""Insert with a per-chat strictly increasing timestamp.
`UPDATE chats ... RETURNING` locks the chat row until COMMIT, so writers of one
chat are serialised: a message with a smaller created_at can never commit after
a message with a bigger one, and a reader paging by cursor cannot skip it.
"""
cur = await conn.execute(
"UPDATE chats SET last_message_at = greatest(clock_timestamp(), "
"last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at",
(chat_id,))
(created_at,) = await cur.fetchone()
await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) "
"VALUES (%s, %s, %s, %s, %s, %s::uuid[])",
(message_id, chat_id, user_id, _b(text), created_at, attachment_ids))
if attachment_ids:
await conn.execute(
"INSERT INTO message_attachments (message_id, position, attachment_id) "
"SELECT %s, a.ord - 1, a.id FROM unnest(%s::uuid[]) WITH ORDINALITY AS a(id, ord)",
(message_id, attachment_ids))
return created_at
# Attachments a message may reference: ready, of THIS chat (else 404), not quarantined (else 409).
# FOR SHARE keeps a concurrent quarantine (reconciler) from slipping in before COMMIT.
CHECK_ATTACHMENTS = (
"SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL) FROM ("
"SELECT lost_at FROM attachments WHERE id = ANY(%s::uuid[]) AND chat_id = %s AND status = 'ready' "
"FOR SHARE) a")
async def _check_attachments(conn: psycopg.AsyncConnection, chat_id: str, attachment_ids: list[str]) -> str:
if not attachment_ids:
return "ok"
cur = await conn.execute(CHECK_ATTACHMENTS, (attachment_ids, chat_id))
found, usable = await cur.fetchone()
if found != len(attachment_ids):
return "attachment_not_found"
return "ok" if usable == found else "attachment_conflict"
def _attachment_error(status: str) -> None:
if status == "attachment_not_found":
raise NotFound("Вложение не найдено")
if status == "attachment_conflict":
raise Conflict("Вложение недоступно и не может быть прикреплено")
# Claim the key. A row of a concurrent transaction that is not committed yet makes this
# INSERT wait on the unique index until that transaction ends: after COMMIT it is a conflict
# (the key is taken), after ROLLBACK the INSERT goes through. A key older than the TTL is
# re-used as if it were new (ON CONFLICT ... DO UPDATE ... WHERE expired).
CLAIM_KEY = (
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) "
"VALUES (%s, %s, %s, %s, %s) "
"ON CONFLICT (user_id, chat_id, key) DO UPDATE SET "
"request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() "
"WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) "
"RETURNING 1")
STORED_RESULT = (
"SELECT k.request_hash, m.id, m.text, m.created_at, m.attachment_ids FROM idempotency_keys k "
"JOIN messages m ON m.id = k.message_id "
"WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s")
async def post_message(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, text: str,
ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult:
"""Idempotent send: the key, the message and the stored result commit in ONE transaction.
* first request -> key row + message + chat clock, all or nothing (no "pending" key is
ever left behind: a crash or 5xx rolls everything back and the client may retry);
* same key, same body (any replica, also concurrently) -> the stored Message, 201, replayed;
* same key, other body -> 409; the membership check comes first, so outsiders always get 404;
* attachments (lab 4) are checked before the key is claimed (same order as post_message_v2).
"""
message_id = uuid.uuid4()
attachment_ids = list(attachment_ids)
async def work(conn: psycopg.AsyncConnection) -> PostResult:
await _check_member(conn, user_id, chat_id)
check = await _check_attachments(conn, chat_id, attachment_ids)
if check == "ok":
cur = await conn.execute(CLAIM_KEY, (user_id, chat_id, key, request_hash, message_id, ttl_seconds))
if await cur.fetchone() is not None:
created_at = await _insert_message(conn, message_id, user_id, chat_id, text, attachment_ids)
return PostResult(Message(str(message_id), chat_id, user_id, text, created_at,
tuple(attachment_ids)), False)
cur = await conn.execute(STORED_RESULT, (user_id, chat_id, key))
row = await cur.fetchone()
if row is None:
_attachment_error(check)
# The key row vanished between the two statements (expired-key cleanup): retryable.
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
if bytes(row[0]) != request_hash:
raise Conflict("Idempotency-Key уже использован с другим сообщением")
return PostResult(Message(str(row[1]), chat_id, user_id, _s(row[2]), row[3],
tuple(str(a) for a in row[4])), True)
return await db.run(work)
POST_MESSAGE_CALL = ("SELECT o_status, o_id, o_text, o_created_at, o_attachments "
"FROM post_message_v2(%s, %s, %s, %s, %s, %s, %s, %s::uuid[])")
async def post_message_single_call(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes,
text: str, ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult:
"""Same contract as post_message, ONE round trip: SQL function post_message_v2 (migration 0004 =
0003 + attachment_ids) in autocommit mode, i.e. the call is the whole transaction."""
message_id = uuid.uuid4()
async def work(conn: psycopg.AsyncConnection) -> tuple:
cur = await conn.execute(POST_MESSAGE_CALL, (user_id, chat_id, key, request_hash, message_id, _b(text),
ttl_seconds, list(attachment_ids)))
return await cur.fetchone()
status, stored_id, stored_text, created_at, stored_attachments = await db.run(work, transaction=False)
if status == "not_found":
raise NotFound("Чат не найден")
_attachment_error(status)
if status == "conflict":
raise Conflict("Idempotency-Key уже использован с другим сообщением")
if status == "retry":
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
return PostResult(Message(str(stored_id), chat_id, user_id, _s(stored_text), created_at,
tuple(str(a) for a in stored_attachments)), status == "replayed")
async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 5000) -> int:
"""Delete expired keys in bounded batches; one replica at a time (advisory xact lock)."""
async def work(conn: psycopg.AsyncConnection) -> int:
cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (2026_03,))
if not (await cur.fetchone())[0]:
return 0
cur = await conn.execute(
"DELETE FROM idempotency_keys WHERE ctid = ANY(ARRAY("
"SELECT ctid FROM idempotency_keys "
"WHERE created_at < clock_timestamp() - make_interval(secs => %s) LIMIT %s))",
(ttl_seconds, batch))
return cur.rowcount
return await db.run(work)
# Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR
# would push the cursor into a Filter and late pages would scan the whole prefix.
# attachment_ids is a column of the message row (migration 0004): still one index range scan.
HISTORY_FIRST = (
"SELECT id, sender_id, text, created_at, attachment_ids FROM messages "
"WHERE chat_id = %s ORDER BY created_at, id LIMIT %s")
HISTORY_AFTER = (
"SELECT id, sender_id, text, created_at, attachment_ids FROM messages "
"WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s")
async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit: int) -> tuple[list[Message], bool]:
async def work(conn: psycopg.AsyncConnection) -> list:
if after is None:
cur = await conn.execute(HISTORY_FIRST, (chat_id, limit + 1))
else:
cur = await conn.execute(HISTORY_AFTER, (chat_id, from_us(after[0]), after[1], limit + 1))
return await cur.fetchall()
rows = await db.run(work)
items = [Message(str(r[0]), chat_id, str(r[1]), _s(r[2]), r[3], tuple(str(a) for a in r[4]))
for r in rows[:limit]]
return items, len(rows) > limit
+151
View File
@@ -0,0 +1,151 @@
"""One-shot S3 bootstrap: `python -m messenger.s3_bootstrap` (compose service `s3-init`).
Idempotent, safe to run on every `docker compose up`:
1. waits until the S3 endpoint answers;
2. creates the bucket with the ADMIN key pair if it does not exist yet (an existing bucket and
its objects are left untouched);
3. writes the sentinel object `health/sentinel` that the API's readiness probe HEADs;
4. proves the access model and FAILS the stand (exit 1) if it is wrong:
* anonymous (unsigned) GET of an object and anonymous LIST of the bucket are refused;
* the API key pair can read/write objects of ITS bucket, but can neither list it, nor create
a bucket, nor see or read another bucket (a canary bucket created here for this check);
* the reconciler key pair can list the bucket but cannot create one.
Only file names and verdicts are printed, never key values.
"""
from __future__ import annotations
import asyncio
import os
import sys
import time
import urllib.error
import urllib.request
from botocore.exceptions import ClientError
from .config import ConfigError, Settings, _secret, s3_settings
from .errors import Unavailable
from .storage import SENTINEL_KEY, Storage
def say(text: str) -> None:
print(f"s3-bootstrap: {text}", flush=True)
def anonymous_status(url: str) -> int:
"""HTTP status of an UNSIGNED request (what anyone on the network could do)."""
try:
with urllib.request.urlopen(url, timeout=3) as response:
return response.status
except urllib.error.HTTPError as exc:
return exc.code
async def error_code(coro) -> str:
"""S3 error code of a call that is expected to be refused ('' if it succeeded)."""
try:
await coro
return ""
except ClientError as exc:
return str(exc.response.get("Error", {}).get("Code", "error"))
def settings_with(keys: tuple[str, str]) -> Settings:
base = s3_settings(credentials_required=False)
base.update(s3_access_key_id=keys[0], s3_secret_access_key=keys[1])
return Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024, **base)
async def run() -> int:
admin = (_secret("S3_ADMIN_ACCESS_KEY_ID", True), _secret("S3_ADMIN_SECRET_ACCESS_KEY", True))
api = (_secret("S3_ACCESS_KEY_ID", True), _secret("S3_SECRET_ACCESS_KEY", True))
reconciler = (_secret("S3_RECONCILER_ACCESS_KEY_ID", True), _secret("S3_RECONCILER_SECRET_ACCESS_KEY", True))
settings = settings_with(admin)
bucket = settings.s3_bucket
admin_s3, api_s3, rec_s3 = Storage(settings), Storage(settings_with(api)), Storage(settings_with(reconciler))
for s in (admin_s3, api_s3, rec_s3):
await s.open()
try:
deadline = time.monotonic() + 60
while True:
try:
await admin_s3._call("list_buckets", admin_s3.client.list_buckets())
break
except Unavailable:
if time.monotonic() > deadline:
say("S3 endpoint does not answer within 60 s")
return 1
await asyncio.sleep(1)
code = await error_code(admin_s3.client.head_bucket(Bucket=bucket))
if code == "":
say(f"bucket {bucket}: exists, kept as is")
elif code in ("404", "NoSuchBucket", "NotFound"):
code = await error_code(admin_s3.client.create_bucket(Bucket=bucket))
if code not in ("", "BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
say(f"bucket {bucket}: create failed ({code})")
return 1
say(f"bucket {bucket}: created")
else:
say(f"bucket {bucket}: head failed ({code})")
return 1
await admin_s3.put(SENTINEL_KEY, b"ok\n", "text/plain", {})
say(f"sentinel {SENTINEL_KEY}: written")
canary = bucket + "-canary"
if await error_code(admin_s3.client.head_bucket(Bucket=canary)):
await admin_s3.client.create_bucket(Bucket=canary)
await admin_s3.client.put_object(Bucket=canary, Key="canary", Body=b"secret of another bucket")
failures = []
base = settings.s3_endpoint.rstrip("/")
for what, url in (("anonymous GET object", f"{base}/{bucket}/{SENTINEL_KEY}"),
("anonymous LIST bucket", f"{base}/{bucket}?list-type=2")):
status = anonymous_status(url)
verdict = "refused" if status in (401, 403) else "ALLOWED"
say(f"{what}: HTTP {status} -> {verdict}")
if verdict != "refused":
failures.append(what)
probe = "health/bootstrap-probe"
checks = [
("api: read object", await error_code(api_s3.client.head_object(Bucket=bucket, Key=SENTINEL_KEY)), True),
("api: write object", await error_code(api_s3.client.put_object(Bucket=bucket, Key=probe, Body=b"p")), True),
("api: delete object", await error_code(api_s3.client.delete_object(Bucket=bucket, Key=probe)), True),
("api: list bucket", await error_code(api_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), False),
("api: create bucket", await error_code(api_s3.client.create_bucket(Bucket=bucket + "-x")), False),
("api: read object of another bucket",
await error_code(api_s3.client.head_object(Bucket=canary, Key="canary")), False),
("reconciler: list bucket", await error_code(rec_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), True),
("reconciler: create bucket", await error_code(rec_s3.client.create_bucket(Bucket=bucket + "-x")), False),
]
for what, code, allowed in checks:
ok = (code == "") == allowed
say(f"{what}: {'allowed' if code == '' else 'refused (' + code + ')'} -> {'OK' if ok else 'WRONG'}")
if not ok:
failures.append(what)
visible = [b["Name"] for b in (await api_s3.client.list_buckets()).get("Buckets", [])]
say(f"api: buckets visible in ListBuckets: {visible} -> {'OK' if canary not in visible else 'WRONG'}")
if canary in visible:
failures.append("api sees another bucket")
if failures:
say("access model is WRONG: " + ", ".join(failures))
return 1
say("done: private bucket, least-privilege keys verified")
return 0
finally:
for s in (admin_s3, api_s3, rec_s3):
await s.close()
def main() -> int:
try:
return asyncio.run(run())
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
if __name__ == "__main__":
os.environ.setdefault("PYTHONUNBUFFERED", "1")
sys.exit(main())
+160
View File
@@ -0,0 +1,160 @@
"""Server-side sessions behind an opaque Bearer token.
* Token: `secrets.token_urlsafe(32)` = 256 random bits. Only SHA-256(token) is
stored, so a leaked database/AOF does not contain usable tokens. A fast hash is
enough for a 256-bit random value; passwords need a slow KDF (passwords.py).
* Expiry is absolute: `expires_at = login time + SESSION_TTL_SECONDS`. Every lookup
compares it with the current time, and a restart never extends it.
Two interchangeable stores (SESSION_STORE):
* `valkey` (compose default): the ONLY copy of a session is a Valkey key with
`PXAT expires_at`, persisted by AOF with `appendfsync always`. There is no second
copy (no cache over PostgreSQL), so a logout cannot be "forgotten" and a revoked
token cannot come back after Valkey fails and recovers. If Valkey is unreachable
a session cannot be checked: protected requests and logout get 503, never access.
* `postgres`: table `sessions` (migrations/0001_init.sql).
"""
from __future__ import annotations
import asyncio
import datetime as dt
import hashlib
import json
import logging
import secrets
from typing import Protocol
from redis import exceptions as redis_errors
from redis.asyncio import Redis
from redis.backoff import NoBackoff
from redis.retry import Retry
from .config import Settings
from .db import Database
from .errors import Unavailable
from .timefmt import UTC, from_us, to_us
log = logging.getLogger("messenger.sessions")
def new_token() -> str:
return secrets.token_urlsafe(32)
def token_hash(token: str) -> bytes:
return hashlib.sha256(token.encode("ascii")).digest()
def now() -> dt.datetime:
return dt.datetime.now(UTC)
class SessionStore(Protocol):
name: str
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]: ...
async def user_id_for(self, token: str) -> str | None: ...
async def revoke(self, token: str) -> bool: ...
async def ping(self) -> bool: ...
async def close(self) -> None: ...
class PostgresSessionStore:
name = "postgres"
def __init__(self, db: Database):
self.db = db
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]:
token = new_token()
expires_at = now() + dt.timedelta(seconds=ttl_seconds)
async def work(conn) -> None:
await conn.execute(
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
(token_hash(token), user_id, expires_at))
# Housekeeping: drop this user's expired sessions (bounded by the user's own rows).
await conn.execute("DELETE FROM sessions WHERE user_id = %s AND expires_at <= %s", (user_id, now()))
await self.db.run(work)
return token, expires_at
async def user_id_for(self, token: str) -> str | None:
async def work(conn):
cur = await conn.execute(
"SELECT user_id FROM sessions WHERE token_hash = %s AND expires_at > %s", (token_hash(token), now()))
return await cur.fetchone()
row = await self.db.run(work)
return str(row[0]) if row else None
async def revoke(self, token: str) -> bool:
async def work(conn) -> bool:
cur = await conn.execute(
"DELETE FROM sessions WHERE token_hash = %s AND expires_at > %s", (token_hash(token), now()))
return cur.rowcount == 1
return await self.db.run(work)
async def ping(self) -> bool:
return True # covered by the database readiness check
async def close(self) -> None:
return None
class ValkeySessionStore:
name = "valkey"
PREFIX = "session:"
def __init__(self, settings: Settings):
self.timeout = settings.cache_timeout_seconds
self.client = Redis.from_url(
settings.cache_url,
socket_timeout=self.timeout,
socket_connect_timeout=self.timeout,
retry=Retry(NoBackoff(), 0), # no hidden retries: a failure is reported at once
decode_responses=False,
)
def _key(self, token: str) -> str:
return self.PREFIX + token_hash(token).hex()
async def _call(self, coro):
try:
async with asyncio.timeout(self.timeout * 2):
return await coro
except (redis_errors.ConnectionError, redis_errors.TimeoutError, TimeoutError, OSError) as exc:
log.warning("session_store_unavailable", extra={"fields": {"error": type(exc).__name__}})
raise Unavailable("Хранилище сессий временно недоступно") from None
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]:
token = new_token()
expires_at = now() + dt.timedelta(seconds=ttl_seconds)
value = json.dumps({"user_id": user_id, "expires_at_us": to_us(expires_at)}).encode()
# PXAT = absolute expiry time: survives restarts (AOF) without being extended.
await self._call(self.client.set(self._key(token), value, pxat=to_us(expires_at) // 1000))
return token, expires_at
async def user_id_for(self, token: str) -> str | None:
raw = await self._call(self.client.get(self._key(token)))
if raw is None:
return None
data = json.loads(raw)
if from_us(data["expires_at_us"]) <= now():
return None # do not rely only on Valkey's lazy expiry
return data["user_id"]
async def revoke(self, token: str) -> bool:
return await self._call(self.client.delete(self._key(token))) == 1
async def ping(self) -> bool:
try:
return bool(await self._call(self.client.ping()))
except Unavailable:
return False
async def close(self) -> None:
await self.client.aclose()
def build_store(settings: Settings, db: Database) -> SessionStore:
return ValkeySessionStore(settings) if settings.session_store == "valkey" else PostgresSessionStore(db)
+177
View File
@@ -0,0 +1,177 @@
"""Private S3 bucket (SeaweedFS in compose) through aiobotocore, with a hard deadline per call.
* Credentials: the API's OWN key pair (S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE ->
Docker secrets). SeaweedFS grants it only Read + Write on this one bucket: no List, no
Admin (cannot create/delete buckets or read other buckets). The admin pair is given only
to the S3 server and the one-shot bootstrap (infra/seaweedfs, messenger.s3_bootstrap).
* Deadlines: botocore connect/read timeouts AND an asyncio deadline around every call, so a
paused or unreachable S3 costs at most S3_TIMEOUT_SECONDS, then 503 `unavailable`. Unlike a
libpq query (db.py), an aiohttp request can be cancelled safely, so the call is cancelled.
* ONE retry inside botocore (total_max_attempts=2), within the same asyncio deadline: after S3
restarts, pooled keep-alive connections to the old process are dead and the first request on
such a connection fails although S3 is up again (found by test_s3_outage_..._stop on a clean
clone). Every call here is idempotent PER KEY (PUT of the same bytes to the same server-chosen
key, GET, HEAD, DELETE), so repeating it cannot create a second attachment.
* Checksums: Content-MD5 is sent with every PUT, so S3 rejects bytes damaged on the way.
botocore's newer default (aws-chunked CRC32 trailers) is switched off ("when_required"):
plain, widely supported S3 requests.
"""
from __future__ import annotations
import asyncio
import base64
import contextlib
import hashlib
import logging
from collections.abc import AsyncIterator
from dataclasses import dataclass
from aiobotocore.config import AioConfig
from aiobotocore.session import get_session
from botocore.exceptions import BotoCoreError, ClientError
from .config import Settings
from .errors import Unavailable
log = logging.getLogger("messenger.storage")
MISSING_CODES = {"NoSuchKey", "404", "NotFound"}
class ObjectMissing(Exception):
"""The key does not exist in the bucket (definitive answer of S3, not an outage)."""
@dataclass(frozen=True, slots=True)
class ObjectHead:
size: int
content_type: str
metadata: dict[str, str]
last_modified_ts: float # POSIX seconds
@dataclass(frozen=True, slots=True)
class ListedObject:
key: str
size: int
last_modified_ts: float # POSIX seconds
class Storage:
def __init__(self, settings: Settings, *, access_key: str | None = None, secret_key: str | None = None):
self.bucket = settings.s3_bucket
self.timeout = settings.s3_timeout_seconds
self._endpoint = settings.s3_endpoint
self._region = settings.s3_region
self._access_key = access_key or settings.s3_access_key_id
self._secret_key = secret_key or settings.s3_secret_access_key
self._config = AioConfig(
connect_timeout=settings.s3_connect_timeout_seconds,
read_timeout=settings.s3_timeout_seconds,
retries={"total_max_attempts": 2, "mode": "standard"},
s3={"addressing_style": "path"},
signature_version="s3v4",
request_checksum_calculation="when_required",
response_checksum_validation="when_required",
max_pool_connections=settings.s3_max_connections,
)
self._stack: contextlib.AsyncExitStack | None = None
self._client = None
async def open(self) -> None:
self._stack = contextlib.AsyncExitStack()
session = get_session()
self._client = await self._stack.enter_async_context(session.create_client(
"s3", endpoint_url=self._endpoint, region_name=self._region,
aws_access_key_id=self._access_key, aws_secret_access_key=self._secret_key, config=self._config))
async def close(self) -> None:
if self._stack is not None:
await self._stack.aclose()
self._stack = None
@property
def client(self):
assert self._client is not None, "Storage.open() was not called"
return self._client
async def _call(self, what: str, coro, deadline: float | None = None):
"""Await one S3 call under the deadline; outages become Unavailable (503)."""
try:
return await asyncio.wait_for(coro, deadline or self.timeout)
except ClientError as exc:
code = str(exc.response.get("Error", {}).get("Code", ""))
if code in MISSING_CODES:
raise ObjectMissing() from None
log.warning("s3_error", extra={"fields": {"op": what, "code": code}})
raise Unavailable("Хранилище файлов временно недоступно") from None
except (asyncio.TimeoutError, BotoCoreError, OSError) as exc:
log.warning("s3_unavailable", extra={"fields": {"op": what, "error": type(exc).__name__}})
raise Unavailable("Хранилище файлов временно недоступно") from None
async def put(self, key: str, data: bytes | bytearray, content_type: str, metadata: dict[str, str]) -> None:
md5 = base64.b64encode(hashlib.md5(data, usedforsecurity=False).digest()).decode("ascii")
# Time budget grows with the size: 10 MiB over a local network takes well under a second.
deadline = self.timeout + len(data) / (8 * 1024 * 1024)
await self._call("put", self.client.put_object(
Bucket=self.bucket, Key=key, Body=data, ContentType=content_type, ContentLength=len(data),
ContentMD5=md5, Metadata=metadata), deadline)
async def head(self, key: str) -> ObjectHead:
response = await self._call("head", self.client.head_object(Bucket=self.bucket, Key=key))
return ObjectHead(int(response["ContentLength"]), response.get("ContentType", ""),
dict(response.get("Metadata", {})), response["LastModified"].timestamp())
async def open_stream(self, key: str) -> tuple[int, AsyncIterator[bytes]]:
"""Start a GET: (Content-Length, async iterator of chunks). The headers arrive under the
deadline; every chunk read afterwards has its own deadline (a stalled S3 aborts the
download instead of hanging the connection)."""
response = await self._call("get", self.client.get_object(Bucket=self.bucket, Key=key))
body = response["Body"]
size = int(response["ContentLength"])
async def chunks() -> AsyncIterator[bytes]:
try:
while True:
chunk = await self._call("get_body", body.read(64 * 1024))
if not chunk:
return
yield chunk
finally:
body.close()
return size, chunks()
async def read_all(self, key: str, limit: int) -> bytes:
"""Whole object (reconciler checksum verification); refuses objects above `limit`."""
size, chunks = await self.open_stream(key)
if size > limit:
raise Unavailable("Объект больше допустимого размера")
parts = [chunk async for chunk in chunks]
return b"".join(parts)
async def delete(self, key: str) -> None:
"""Idempotent: deleting a missing key is a success in S3."""
await self._call("delete", self.client.delete_object(Bucket=self.bucket, Key=key))
async def list_page(self, prefix: str, start_after: str | None, page_size: int) -> tuple[list[ListedObject], bool]:
"""One ListObjectsV2 page in key order (needs the List permission: reconciler only)."""
params = {"Bucket": self.bucket, "Prefix": prefix, "MaxKeys": page_size}
if start_after:
params["StartAfter"] = start_after
response = await self._call("list", self.client.list_objects_v2(**params))
items = [ListedObject(o["Key"], int(o["Size"]), o["LastModified"].timestamp())
for o in response.get("Contents", [])]
return items, bool(response.get("IsTruncated"))
async def ping(self, timeout: float = 1.0) -> bool:
"""Readiness: HEAD of the sentinel object written by the bootstrap. Proves that S3
answers, the bucket exists and the API credentials are accepted (Read permission)."""
try:
await self._call("ping", self.client.head_object(Bucket=self.bucket, Key=SENTINEL_KEY), timeout)
return True
except (Unavailable, ObjectMissing):
return False
SENTINEL_KEY = "health/sentinel"
OBJECT_PREFIX = "o/" # every attachment original lives under o/<random 128-bit hex>
+22
View File
@@ -0,0 +1,22 @@
"""Conversions between PostgreSQL timestamptz values, cursor integers and RFC 3339 strings."""
from __future__ import annotations
import datetime as dt
UTC = dt.timezone.utc
_EPOCH = dt.datetime(1970, 1, 1, tzinfo=UTC)
def to_us(value: dt.datetime) -> int:
"""Microseconds since the Unix epoch (exact: timestamptz has microsecond precision)."""
delta = value - _EPOCH
return (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds
def from_us(us: int) -> dt.datetime:
return _EPOCH + dt.timedelta(microseconds=us)
def rfc3339(value: dt.datetime) -> str:
"""RFC 3339 UTC with fixed microsecond precision, e.g. 2026-10-12T12:00:00.000001Z."""
return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+199
View File
@@ -0,0 +1,199 @@
"""Strict request validation according to contracts/openapi.json + contracts/README.md.
Written by hand instead of a schema library so that every rule is visible:
extra fields, wrong JSON types, whitespace-only strings and lengths counted in
Unicode code points all produce 400 invalid_request.
"""
from __future__ import annotations
import json
import re
import unicodedata
from typing import Any
from urllib.parse import parse_qsl
from .errors import InvalidRequest
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
# Canonical textual UUID (8-4-4-4-12 hex). Case-insensitive, normalised to lower case.
UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
LIMIT_RE = re.compile(r"[0-9]{1,3}")
# Idempotency-Key (contract, lab 3): ASCII [A-Za-z0-9._:-]{1,128}.
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
SURROGATES_RE = re.compile("[\ud800-\udfff]")
DEFAULT_LIMIT = 50
MAX_LIMIT = 100
def _reject_constant(name: str) -> Any:
# Python's json accepts NaN/Infinity, which are not valid JSON.
raise ValueError(f"invalid JSON constant {name}")
def _unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
# {"text":"a","text":"b"} is ambiguous (which one is meant?) -> 400, not "last one wins".
result = dict(pairs)
if len(result) != len(pairs):
raise ValueError("duplicate key in JSON object")
return result
def parse_json_object(body: bytes) -> dict[str, Any]:
"""Decode a UTF-8 JSON object or raise InvalidRequest (malformed JSON -> 400)."""
try:
text = body.decode("utf-8")
value = json.loads(text, parse_constant=_reject_constant, object_pairs_hook=_unique_keys)
except (UnicodeDecodeError, ValueError):
raise InvalidRequest("Тело запроса должно быть корректным JSON в UTF-8") from None
if not isinstance(value, dict):
raise InvalidRequest("Тело запроса должно быть JSON-объектом")
return value
def check_fields(data: dict[str, Any], required: set[str], optional: frozenset[str] = frozenset()) -> None:
extra = set(data) - required - optional
if extra:
raise InvalidRequest("Неизвестные поля: " + ", ".join(sorted(extra)))
missing = required - set(data)
if missing:
raise InvalidRequest("Отсутствуют обязательные поля: " + ", ".join(sorted(missing)))
def text_field(data: dict[str, Any], name: str, max_len: int, *, min_len: int = 1, allow_blank: bool = False) -> str:
value = data[name]
if not isinstance(value, str):
raise InvalidRequest(f"Поле {name} должно быть строкой")
if SURROGATES_RE.search(value):
# "\ud800" escapes decode to lone surrogates that cannot be encoded as UTF-8.
raise InvalidRequest(f"Поле {name} содержит недопустимые символы")
# len() of a Python str is the number of Unicode code points, as the contract requires.
if not min_len <= len(value) <= max_len:
raise InvalidRequest(f"Длина поля {name} должна быть от {min_len} до {max_len} символов")
if value.strip() == "" and not allow_blank:
raise InvalidRequest(f"Поле {name} не может состоять только из пробелов")
return value
def username_field(data: dict[str, Any]) -> str:
value = data["username"]
if not isinstance(value, str) or USERNAME_RE.fullmatch(value) is None:
raise InvalidRequest("username должен соответствовать [a-z0-9_]{3,32}")
return value
def password_field(data: dict[str, Any]) -> str:
"""12-128 Unicode code points, not whitespace-only; any other character (':' too) is allowed."""
value = data["password"]
if not isinstance(value, str):
raise InvalidRequest("Поле password должно быть строкой")
if SURROGATES_RE.search(value):
raise InvalidRequest("Поле password содержит недопустимые символы")
if not 12 <= len(value) <= 128:
raise InvalidRequest("Длина пароля должна быть от 12 до 128 символов")
if value.strip() == "":
raise InvalidRequest("Пароль не может состоять только из пробелов")
return value
def parse_uuid(value: Any, what: str = "UUID") -> str:
if not isinstance(value, str) or UUID_RE.fullmatch(value) is None:
raise InvalidRequest(f"Неверный {what}")
return value.lower()
def member_ids_field(data: dict[str, Any]) -> list[str]:
value = data["member_ids"]
if not isinstance(value, list):
raise InvalidRequest("member_ids должен быть массивом")
if not 1 <= len(value) <= 100:
raise InvalidRequest("member_ids должен содержать от 1 до 100 элементов")
ids = [parse_uuid(item, "UUID участника") for item in value]
if len(set(ids)) != len(ids):
raise InvalidRequest("member_ids содержит дубликаты")
return ids
def attachment_ids_field(data: dict[str, Any]) -> list[str]:
"""Optional `attachment_ids` (lab 4+): absent and [] are the same message (contracts/README.md);
null or any other non-array type is 400."""
value = data.get("attachment_ids", [])
if not isinstance(value, list):
raise InvalidRequest("attachment_ids должен быть массивом")
if len(value) > 10:
raise InvalidRequest("attachment_ids содержит больше 10 элементов")
ids = [parse_uuid(item, "UUID вложения") for item in value]
if len(set(ids)) != len(ids):
raise InvalidRequest("attachment_ids содержит дубликаты")
return ids
def idempotency_key(raw: str | None) -> str:
if raw is None or IDEMPOTENCY_KEY_RE.fullmatch(raw) is None:
raise InvalidRequest("Заголовок Idempotency-Key обязателен: [A-Za-z0-9._:-]{1,128}")
return raw
def single_query_param(query: list[tuple[str, str]], name: str) -> str | None:
values = [v for k, v in query if k == name]
if len(values) > 1:
raise InvalidRequest(f"Параметр {name} указан несколько раз")
return values[0] if values else None
def parse_limit(raw: str | None) -> int:
if raw is None:
return DEFAULT_LIMIT
if LIMIT_RE.fullmatch(raw) is None or not 1 <= int(raw) <= MAX_LIMIT:
raise InvalidRequest(f"limit должен быть целым числом от 1 до {MAX_LIMIT}")
return int(raw)
# Lab 4: the display name of an attachment. Never used as a path or an S3 key, but it is shown
# to other users, so characters that change how a name LOOKS are refused: control characters
# (C0, DEL, C1 such as U+0085), line/paragraph separators, unpaired surrogates and bidi controls
# ("photo<U+202E>gnp.exe" would display as "photoexe.png").
BIDI_CONTROLS = frozenset("\u061c\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069")
FORBIDDEN_CATEGORIES = frozenset({"Cc", "Cs", "Zl", "Zp"})
def query_items(raw: bytes) -> list[tuple[str, str]]:
"""Strict query parsing: percent-encoded UTF-8 only (Starlette would replace bad bytes by U+FFFD)."""
try:
return parse_qsl(raw.decode("ascii"), keep_blank_values=True, encoding="utf-8", errors="strict")
except (UnicodeDecodeError, ValueError):
raise InvalidRequest("Параметры запроса должны быть в UTF-8 с percent-encoding") from None
def filename_param(query: list[tuple[str, str]]) -> str:
value = single_query_param(query, "filename")
if value is None:
raise InvalidRequest("Параметр filename обязателен")
if not 1 <= len(value) <= 255:
raise InvalidRequest("Длина filename должна быть от 1 до 255 символов")
if value.strip() == "" or value in {".", ".."}:
raise InvalidRequest("filename не может состоять только из пробелов или точек")
if "/" in value or "\\" in value:
raise InvalidRequest("filename не может содержать / или \\")
for char in value:
if unicodedata.category(char) in FORBIDDEN_CATEGORIES or char in BIDI_CONTROLS:
raise InvalidRequest("filename содержит управляющие символы")
return value
ATTACHMENT_VARIANTS = frozenset({"thumbnail", "crop", "optimized"}) # exist in the schema, not in lab 4
def variant_param(query: list[tuple[str, str]]) -> str:
"""`variant` of GET /attachments/{id}/content: only `original` exists in lab 4. A known variant
name is a missing variant of a ready object (404); anything else is an invalid value (400)."""
value = single_query_param(query, "variant")
if value is None or value == "original":
return "original"
if value in ATTACHMENT_VARIANTS:
raise NotFoundVariant()
raise InvalidRequest("variant должен быть original")
class NotFoundVariant(Exception):
"""Raised for a variant that lab 4 does not produce; mapped to 404 by the handler."""
+380
View File
@@ -0,0 +1,380 @@
"""Shared helpers for the student tests.
* `unit/` — pure functions (validation, cursor, auth headers, Argon2id); no services.
* `test_*` in this folder — marker `stack`: black-box HTTP against the running compose
stand THROUGH THE LOAD BALANCER (BASE_URL, default http://127.0.0.1:8080) plus
`docker compose` for database checks (psql inside the postgres container), the HAProxy
runtime state (scripts/lbctl.py) and failure injection. Every failure test restores the
stopped/paused service or replica in `finally` and waits until HAProxy sees all replicas UP.
python -m pytest student-tests # everything (stand must be up)
python -m pytest student-tests -m "not stack" # unit only
"""
from __future__ import annotations
import base64
import contextlib
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
from urllib.parse import quote
import httpx2 as httpx
import pytest
from jsonschema import Draft202012Validator
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
sys.path.insert(0, str(ROOT / "scripts"))
import lbctl # noqa: E402
SPEC = json.loads((ROOT / "contracts" / "openapi.json").read_text(encoding="utf-8"))
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
PASSWORD = "Test-password-Ω-12345" # synthetic; 12+ code points, non-ASCII on purpose
REPLICAS = int(os.environ.get("API_REPLICAS", "2")) # replicas the stand normally runs
def pytest_configure(config):
config.addinivalue_line("markers", "stack: needs the running compose stand (docker compose up -d --wait)")
def schema_errors(value, name: str) -> list[str]:
"""Validate against components/schemas/<name> of the course contract (JSON Schema 2020-12)."""
schema = {"$ref": f"#/components/schemas/{name}", "components": SPEC["components"]}
validator = Draft202012Validator(schema, format_checker=Draft202012Validator.FORMAT_CHECKER)
return [e.message for e in validator.iter_errors(value)]
def basic(username: str, password: str) -> str:
return "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode()
# ------------------------------------------------------------------ docker / psql
def compose(*args: str, check: bool = True, timeout: float = 120, stdin: str | None = None) -> subprocess.CompletedProcess:
return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", check=check, timeout=timeout, input=stdin)
def psql(sql: str) -> str:
"""Run SQL inside the postgres container (no DB port is published) and return unaligned output."""
result = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
"-At", "-v", "ON_ERROR_STOP=1", "-c", sql)
return result.stdout.strip()
def wait_ready(timeout: float = 90) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
if httpx.get(BASE_URL + "/health/ready", timeout=2).status_code == 200:
return
except httpx.HTTPError:
pass
time.sleep(0.3)
raise AssertionError("stand did not become ready")
def wait_healthy(*services: str, timeout: float = 120) -> None:
"""Poll Docker health until EVERY container of every service is healthy again. `compose up
--wait` is not used: it gives up at once on a container marked unhealthy while paused."""
deadline = time.monotonic() + timeout
pending = set(services)
while pending and time.monotonic() < deadline:
for service in list(pending):
ids = compose("ps", "-q", service, check=False).stdout.split()
if ids:
states = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", *ids],
capture_output=True, text=True).stdout.split()
if states and all(state == "healthy" for state in states):
pending.discard(service)
time.sleep(0.5)
assert not pending, f"not healthy: {pending}"
def wait_stand(replicas: int = REPLICAS) -> None:
"""Readiness through the LB AND every replica back in HAProxy rotation."""
wait_ready()
lbctl.wait_up(replicas, timeout=90)
def restart_services(*services: str) -> None:
compose("restart", *services)
wait_healthy(*services)
wait_stand()
@contextlib.contextmanager
def service_down(service: str, how: str = "stop"):
"""Stop or pause a compose service; ALWAYS bring it back and wait for readiness."""
assert how in {"stop", "pause"}
compose(how, service)
try:
yield
finally:
compose("unpause" if how == "pause" else "start", service)
wait_healthy(service)
wait_stand()
@contextlib.contextmanager
def api_with_env(**overrides: str):
"""Recreate the api container with environment overrides (e.g. SESSION_TTL_SECONDS=2),
then ALWAYS recreate it with the normal configuration."""
env = {**os.environ, **overrides}
subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=env,
capture_output=True, text=True, check=True, timeout=180)
wait_stand()
try:
yield
finally:
clean = {k: v for k, v in os.environ.items() if k not in overrides}
subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=clean,
capture_output=True, text=True, check=True, timeout=180)
wait_stand()
def running_replicas() -> list[dict]:
return [r for r in lbctl.replicas() if r["state"] == "running"]
@contextlib.contextmanager
def replica_stopped(replica: dict, mode: str = "stop"):
"""Stop ONE api container (stop = SIGTERM + drain, kill = SIGKILL); ALWAYS start it again
and wait until HAProxy has every replica UP."""
assert mode in {"stop", "kill"}
subprocess.run(["docker", "stop", "--time", "15" if mode == "stop" else "0", replica["id"]],
capture_output=True, text=True, check=True, timeout=60)
try:
yield
finally:
subprocess.run(["docker", "start", replica["id"]], capture_output=True, text=True, check=True, timeout=60)
wait_healthy("api")
wait_stand()
def api_env(name: str) -> str:
"""Current value of an environment variable inside the running api container."""
return compose("exec", "-T", "api", "printenv", name).stdout.strip()
# ------------------------------------------------------------------ HTTP
class Api:
"""HTTP helper that checks the invariants every response must satisfy."""
def __init__(self, base_url: str = BASE_URL):
self.client = httpx.Client(base_url=base_url, timeout=15)
def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None,
json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None,
lb_error_ok: bool = False):
"""lb_error_ok: a 502/503/504 generated by HAProxy itself (no API replica answered) is
accepted without X-Instance-Id; every response of an API replica must carry it."""
h = dict(headers or {})
if token is not None:
h["Authorization"] = "Bearer " + token
if json_body is not None:
raw = json.dumps(json_body, ensure_ascii=False).encode("utf-8")
h.setdefault("Content-Type", "application/json")
response = self.client.request(method, path, content=raw, headers=h)
allowed = (expected,) if isinstance(expected, int) else expected
assert response.status_code in allowed, (method, path, response.status_code, response.text[:300])
request_id = response.headers.get("x-request-id")
assert request_id, "every response must carry X-Request-Id"
if not response.headers.get("x-instance-id"):
assert lb_error_ok and response.status_code in (502, 503, 504), \
("missing X-Instance-Id", method, path, response.status_code)
if response.status_code >= 400:
body = response.json()
assert schema_errors(body, "Error") == [], response.text
assert body["error"]["request_id"] == request_id
assert "Traceback" not in response.text and "psycopg" not in response.text
if response.status_code == 401:
challenge = "basic" if path == "/api/v1/auth/sessions" else "bearer"
assert response.headers.get("www-authenticate", "").lower().startswith(challenge)
elif schema:
assert schema_errors(response.json(), schema) == [], response.text
return response
def register(self, label: str = "u", *, password: str = PASSWORD, display_name: str | None = None) -> dict:
username = f"{label}_{uuid.uuid4().hex[:12]}"[:32]
body = {"username": username, "display_name": display_name or f"Студент {label}", "password": password}
user = self.call("POST", "/api/v1/users", 201, json_body=body, schema="User").json()
return {"id": user["id"], "user": user, "username": username, "password": password}
def login(self, actor: dict) -> str:
session = self.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": basic(actor["username"], actor["password"])},
schema="Session").json()
assert session["user"] == actor["user"] and session["token_type"] == "Bearer"
return session["token"]
def user(self, label: str = "u", **kwargs) -> dict:
actor = self.register(label, **kwargs)
actor["token"] = self.login(actor)
return actor
def chat(self, owner: dict, *members: dict, title: str = "Тестовый чат") -> dict:
ids = [owner["id"], *(m["id"] for m in members)]
return self.call("POST", "/api/v1/chats", 201, token=owner["token"],
json_body={"title": title, "member_ids": ids}, schema="Chat").json()
def post_message(self, actor: dict, chat: dict, body, expected=201, *, key: str | None = None,
raw: bytes | None = None, headers: dict | None = None, **kwargs):
"""POST a message with an Idempotency-Key (a fresh one unless `key` is given)."""
h = {"Idempotency-Key": key or uuid.uuid4().hex, **(headers or {})}
if raw is not None:
h.setdefault("Content-Type", "application/json")
return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", expected, token=actor["token"],
json_body=body if raw is None else None, raw=raw, headers=h,
schema="Message" if expected == 201 else None, **kwargs)
def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict:
return self.post_message(actor, chat, {"text": text}, key=key).json()
def upload(self, actor: dict, chat: dict, data, expected=201, *, content_type: str | None = "image/png",
filename: str | None = "photo.png", query: str | None = None, headers: dict | None = None):
"""POST raw bytes to /chats/{id}/attachments. `query` replaces the whole query string
(to send hand-made percent-encoding); `data` may be bytes or an iterator (chunked)."""
h = dict(headers or {})
if content_type is not None:
h["Content-Type"] = content_type
if query is None:
query = "" if filename is None else "filename=" + quote(filename, safe="")
path = f"/api/v1/chats/{chat['id']}/attachments" + (f"?{query}" if query else "")
if isinstance(data, (bytes, bytearray)):
return self.call("POST", path, expected, token=actor["token"], raw=bytes(data), headers=h,
schema="Attachment" if expected == 201 else None)
h["Authorization"] = "Bearer " + actor["token"]
response = self.client.post(path, content=data, headers=h)
assert response.status_code == expected, (response.status_code, response.text[:300])
return response
def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]:
"""Walk all pages; returns (items, number_of_pages)."""
items, pages, cursor = [], 0, None
while True:
query = f"?limit={limit}" + (f"&cursor={cursor}" if cursor else "")
page = self.call("GET", f"/api/v1/chats/{chat['id']}/messages{query}", 200,
token=actor["token"], schema="MessagePage").json()
pages += 1
items.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return items, pages
assert len(page["items"]) == limit, "a non-final page must be full"
assert len(items) < 100_000, "cursor loop"
def _stand_is_up() -> bool:
try:
return httpx.get(BASE_URL + "/health/ready", timeout=3).status_code == 200
except httpx.HTTPError:
return False
def pytest_collection_modifyitems(config, items):
if any("stack" in item.keywords for item in items) and not _stand_is_up():
skip = pytest.mark.skip(reason=f"stand is not running at {BASE_URL}: docker compose up -d --wait")
for item in items:
if "stack" in item.keywords:
item.add_marker(skip)
@pytest.fixture(scope="session")
def api() -> Api:
return Api()
def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.CompletedProcess:
"""Run scripts/<name> (a POSIX sh hook); on Windows through Git Bash like the .cmd wrappers."""
script = str(ROOT / "scripts" / name)
if os.name == "nt":
command = [os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe"), script]
else:
command = [script]
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True,
timeout=timeout, env={**os.environ, **env})
def s3_exec(body: str, service: str = "api", stdin: str | None = None) -> str:
"""Run `body` (async Python, `s` = messenger.storage.Storage) INSIDE a container that holds S3
credentials: `api` (Read+Write, the API's own key pair) or `reconciler` (+List). The S3 port is
not published on the host; this is how the tests inject faults directly into the bucket."""
script = ("import asyncio, base64, json, sys\n"
"from messenger.config import Settings, s3_settings\n"
"from messenger.storage import Storage, ObjectMissing\n"
"async def main():\n"
" st = Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings())\n"
" s = Storage(st)\n"
" await s.open()\n"
" try:\n" + "".join(" " + line + "\n" for line in body.splitlines()) +
" finally:\n"
" await s.close()\n"
"asyncio.run(main())\n")
if service == "api":
cmd = ["exec", "-T", "api", "python", "-c", script]
else:
cmd = ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service, "-c", script]
return compose(*cmd, timeout=120, stdin=stdin).stdout.strip()
def s3_put(key: str, data: bytes) -> None:
"""Write raw bytes under `key` (base64 over stdin: Windows limits command lines to 32 KiB)."""
s3_exec(f"await s.put({key!r}, base64.b64decode(sys.stdin.read()), 'image/png', {{}})",
stdin=base64.b64encode(data).decode())
def s3_delete(key: str) -> None:
s3_exec(f"await s.delete({key!r})")
def s3_size(key: str) -> int | None:
out = s3_exec(f"try:\n print((await s.head({key!r})).size)\nexcept ObjectMissing:\n print(-1)")
size = int(out.splitlines()[-1])
return None if size < 0 else size
def s3_keys() -> set[str]:
"""Every object key under o/ (needs List: runs with the reconciler's key pair)."""
out = s3_exec("keys, after = [], None\n"
"while True:\n"
" page, more = await s.list_page('o/', after, 1000)\n"
" keys += [o.key for o in page]\n"
" if not more or not page:\n"
" break\n"
" after = page[-1].key\n"
"print(json.dumps(keys))", service="reconciler")
return set(json.loads(out.splitlines()[-1]))
def reconcile(*args: str, expect_code: int = 0) -> dict:
"""Run the reconciler CLI (compose service, profile tools) and return its JSON report."""
# --no-deps: never let `compose run` start a store that a test has stopped on purpose.
result = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "100000", *args, check=False,
timeout=300)
assert result.returncode == expect_code, (result.returncode, result.stderr[-2000:])
return json.loads(result.stdout)
def findings_by_key(report: dict) -> dict[str, dict]:
return {item["key"]: dict(item, kind=kind) for kind, items in report["samples"].items() for item in items}
def attachment_row(attachment_id: str) -> dict | None:
out = psql("SELECT json_build_object('status', status, 'object_key', object_key, 'size', size_bytes, "
"'lost_reason', lost_reason) FROM attachments WHERE id = '%s'" % attachment_id)
return json.loads(out) if out else None
def probe_inside(replica: dict, path: str) -> int:
"""HTTP status of `path` asked inside the replica container itself (bypassing the LB)."""
code = ("import urllib.request as u, urllib.error as e\n"
"try: print(u.urlopen('http://127.0.0.1:8080%s', timeout=3).status)\n"
"except e.HTTPError as x: print(x.code)") % path
out = subprocess.run(["docker", "exec", replica["id"], "python", "-c", code], capture_output=True, text=True,
timeout=30).stdout.strip()
return int(out or 0)
+93
View File
@@ -0,0 +1,93 @@
"""Synthetic images for the lab 4 tests: valid PNG/JPEG of any size, and the broken files the
reviewer used against other solutions (truncated, corrupted IDAT with a valid CRC, JPEG without
scan data, decompression-bomb headers, MIME spoofing). Nothing big is stored in Git: every file
is generated in memory."""
from __future__ import annotations
import io
import os
import struct
import zlib
from PIL import Image
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
def chunk(ctype: bytes, data: bytes) -> bytes:
return struct.pack(">I", len(data)) + ctype + data + struct.pack(">I", zlib.crc32(ctype + data))
def ihdr(width: int, height: int, depth: int = 8, color: int = 2, interlace: int = 0) -> bytes:
return chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace))
def png(width: int = 64, height: int = 48, mode: str = "RGB", noise: bool = False, **save) -> bytes:
if noise:
channels = len(mode)
image = Image.frombytes(mode, (width, height), os.urandom(width * height * channels))
else:
image = Image.new(mode, (width, height), (40, 120, 200) if mode == "RGB" else 128)
out = io.BytesIO()
image.save(out, "PNG", **save)
return out.getvalue()
def png_of_size(target: int) -> bytes:
"""A valid PNG of (about, at most) `target` bytes: random pixels do not compress."""
side = int((target / 3) ** 0.5)
data = png(side, side, noise=True, compress_level=1)
while len(data) > target:
side -= 8
data = png(side, side, noise=True, compress_level=1)
return data
def png_exactly(size: int) -> bytes:
"""A valid PNG of EXACTLY `size` bytes: noise image + an ancillary padding chunk."""
base = png_of_size(size - 64)
pad = size - len(base) - 12
assert pad >= 0
return base[:-12] + chunk(b"tEXt", b"pad\x00" + b"x" * (pad - 4)) + base[-12:]
def jpeg(width: int = 64, height: int = 48, progressive: bool = False, noise: bool = False, quality: int = 85) -> bytes:
if noise:
image = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3))
else:
image = Image.new("RGB", (width, height), (200, 30, 30))
out = io.BytesIO()
image.save(out, "JPEG", progressive=progressive, quality=quality)
return out.getvalue()
# ------------------------------------------------------------------ broken files
def png_corrupt_idat() -> bytes:
"""Valid signature, IHDR and CRCs; the IDAT payload is 86 bytes of garbage (reviewer's case)."""
return PNG_SIGNATURE + ihdr(20, 20) + chunk(b"IDAT", bytes(range(86))) + chunk(b"IEND", b"")
def png_short_pixels() -> bytes:
"""A complete, CRC-correct PNG whose zlib stream holds fewer rows than the header promises."""
raw = b"".join(b"\x00" + bytes(60) for _ in range(19))
return PNG_SIGNATURE + ihdr(20, 20) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")
def png_bomb_header(width: int = 20000, height: int = 10000) -> bytes:
"""Header of a huge picture: must be refused (413) from the first 33 bytes."""
raw_row = b"\x00" + bytes(width * 3)
body = zlib.compress(raw_row * 4, 9) # a few rows only: the decoder must never get that far
return PNG_SIGNATURE + ihdr(width, height) + chunk(b"IDAT", body) + chunk(b"IEND", b"")
def jpeg_without_scan_data() -> bytes:
data = jpeg(128, 96)
sos = data.index(b"\xff\xda")
length = struct.unpack(">H", data[sos + 2:sos + 4])[0]
return data[:sos + 2 + length] + b"\xff\xd9"
def jpeg_bomb_header() -> bytes:
"""SOF0 announcing 20000 x 20000 = 400 Mpx."""
return (b"\xff\xd8\xff\xc0\x00\x11\x08" + struct.pack(">HH", 20000, 20000)
+ b"\x03\x01\x22\x00\x02\x11\x01\x03\x11\x01" + b"\xff\xd9")
+344
View File
@@ -0,0 +1,344 @@
"""Contract behaviour over real HTTP (stand must be running). Lab 1 checks moved to Bearer + new lab 2 auth."""
from __future__ import annotations
import base64
import uuid
import pytest
from conftest import PASSWORD, basic
pytestmark = pytest.mark.stack
def key(item: dict) -> tuple[str, int]:
return item["created_at"], uuid.UUID(item["id"]).int
# ================================================================== registration & login
def test_register_never_returns_password_and_login_returns_bearer(api):
actor = api.register("alice", display_name="Алиса 👩‍💻")
assert set(actor["user"]) == {"id", "username", "display_name", "created_at"}
session = api.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": basic(actor["username"], PASSWORD)}, schema="Session").json()
assert session["token_type"] == "Bearer" and len(session["token"]) >= 43
assert actor["username"] not in session["token"] and actor["id"] not in session["token"]
assert api.call("GET", "/api/v1/users/me", 200, token=session["token"], schema="User").json() == actor["user"]
def test_duplicate_username_is_409(api):
actor = api.register("dup")
body = {"username": actor["username"], "display_name": "Другой", "password": PASSWORD}
assert api.call("POST", "/api/v1/users", 409, json_body=body).json()["error"]["code"] == "conflict"
@pytest.mark.parametrize("password", ["short-11ch", " " * 12, " \t \n " * 3, "я" * 129])
def test_bad_password_on_registration_is_400(api, password):
body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "x", "password": password}
api.call("POST", "/api/v1/users", 400, json_body=body)
@pytest.mark.parametrize("password", ["я" * 128, "with:colons:inside", "Twelve chars"])
def test_edge_passwords_register_and_login(api, password):
actor = api.user("edge", password=password)
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
def test_wrong_password_and_unknown_user_give_the_same_401(api):
actor = api.register("victim")
wrong = api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic(actor["username"], "Wrong-password-1")})
unknown = api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic("nobody_" + uuid.uuid4().hex[:8], PASSWORD)})
assert wrong.json()["error"]["message"] == unknown.json()["error"]["message"]
assert wrong.headers["www-authenticate"] == 'Basic realm="messenger", charset="UTF-8"'
@pytest.mark.parametrize("header", ["Basic é", "Basic !!!", "Basic " + base64.b64encode(b"no_colon").decode(),
"Basic " + base64.b64encode(b"\xff\xfe:x").decode(), "Bearer " + "a" * 43, ""])
def test_malformed_basic_is_401_not_500(api, header):
# Raw UTF-8 bytes, as curl would send them (httpx refuses non-ASCII str headers).
api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": header.encode("utf-8")} if header else {})
def test_logout_revokes_only_this_token(api):
actor = api.user("logout")
second = api.login(actor)
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"])
api.call("GET", "/api/v1/users/me", 401, token=actor["token"])
api.call("DELETE", "/api/v1/auth/sessions/current", 401, token=actor["token"])
api.call("GET", "/api/v1/users/me", 200, token=second) # other session of the same user survives
def test_x_user_id_basic_and_garbage_do_not_authenticate(api):
actor = api.user("hdr")
for headers in ({"X-User-Id": actor["id"]},
{"Authorization": basic(actor["username"], PASSWORD)},
{"Authorization": "Bearer " + "A" * 43},
{"Authorization": "Bearer"}):
api.call("GET", "/api/v1/users/me", 401, headers=headers)
me = api.call("GET", "/api/v1/users/me", 200, token=actor["token"], headers={"X-User-Id": str(uuid.uuid4())})
assert me.json()["id"] == actor["id"]
# ================================================================== validation / errors (from lab 1)
@pytest.mark.parametrize("raw", [b"{", b"", b"[]", b"null", b'{"username": "abc_def", "display_name": NaN, "password": "Twelve-chars"}',
b'{"username": "abc_def", "display_name": "x", "password": "Twelve-chars"}{"a": 1}',
b'{"username": "abc_def", "display_name": "\\ud800", "password": "Twelve-chars"}'])
def test_malformed_json_is_400(api, raw):
api.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"})
@pytest.mark.parametrize("body", [
{"username": "abc_def", "display_name": "x"}, # no password
{"username": "abc_def", "display_name": "x", "password": "Twelve-chars", "extra": 1},
{"username": "Abc_def", "display_name": "x", "password": "Twelve-chars"},
{"username": "abc_def", "display_name": " ", "password": "Twelve-chars"},
{"username": "abc_def", "display_name": "x" * 101, "password": "Twelve-chars"},
{"username": 5, "display_name": "x", "password": "Twelve-chars"},
])
def test_invalid_registration_fields_are_400(api, body):
api.call("POST", "/api/v1/users", 400, json_body=body)
def test_unicode_limits_are_code_points_and_nul_is_stored(api):
actor = api.user("uni", display_name="😀" * 100)
assert actor["user"]["display_name"] == "😀" * 100
nul = api.user("nul", display_name="A\u0000B")
assert api.call("GET", "/api/v1/users/me", 200, token=nul["token"]).json()["display_name"] == "A\u0000B"
chat = api.chat(actor, title="T\u0000itle")
assert chat["title"] == "T\u0000itle"
assert api.send(actor, chat, "hello\u0000world")["text"] == "hello\u0000world"
assert api.send(actor, chat, "🙂" * 4000)["text"] == "🙂" * 4000
api.post_message(actor, chat, {"text": "я" * 4001}, 400)
texts = [m["text"] for m in api.history(actor, chat, 100)[0]]
assert texts == ["hello\u0000world", "🙂" * 4000]
def test_oversized_body_and_wrong_content_type(api):
api.call("POST", "/api/v1/users", 413, json_body={"username": "big_body", "display_name": "x" * 70_000, "password": PASSWORD})
api.call("POST", "/api/v1/users", 400, raw=b'{"username":"abc_def"}', headers={"Content-Type": "text/plain"})
# ================================================================== chats & membership
def test_chat_validation(api):
alice, bob = api.user("alice"), api.user("bob")
for body in ({"title": "x", "member_ids": [alice["id"], bob["id"], bob["id"]]},
{"title": "x", "member_ids": [alice["id"], str(uuid.uuid4())]},
{"title": "x", "member_ids": [bob["id"]]},
{"title": " ", "member_ids": [alice["id"]]},
{"title": "x", "member_ids": [alice["id"], "bob"]}):
api.call("POST", "/api/v1/chats", 400, token=alice["token"], json_body=body)
def test_foreign_chat_is_indistinguishable_from_missing(api):
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
chat = api.chat(alice, bob)
for chat_id in (chat["id"], str(uuid.uuid4())):
base = f"/api/v1/chats/{chat_id}"
errors = [api.call("GET", base, 404, token=eve["token"]).json(),
api.call("GET", base + "/messages", 404, token=eve["token"]).json(),
api.call("POST", base + "/messages", 404, token=eve["token"], json_body={"text": "чужое"}, headers={"Idempotency-Key": "k1"}).json(),
api.call("GET", base + "/messages?cursor=garbage", 404, token=eve["token"]).json()]
assert len({e["error"]["message"] for e in errors}) == 1
api.call("GET", "/api/v1/chats/not-a-uuid", 401)
api.call("GET", "/api/v1/chats/not-a-uuid", 400, token=alice["token"])
api.call("GET", "/api/v1/chats/NOT-A-UUID/messages", 400, token=alice["token"])
def test_chat_list_order_pagination_and_isolation(api):
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
created = [api.chat(alice, bob, title=f"Чат {i}") for i in range(5)]
api.chat(eve, title="Чат Евы")
walked, cursor = [], None
while True:
query = "?limit=2" + (f"&cursor={cursor}" if cursor else "")
page = api.call("GET", "/api/v1/chats" + query, 200, token=bob["token"], schema="ChatPage").json()
walked += page["items"]
cursor = page["next_cursor"]
if not cursor:
break
assert walked == created
first = api.call("GET", "/api/v1/chats?limit=1", 200, token=alice["token"]).json()["next_cursor"]
api.call("GET", f"/api/v1/chats?limit=1&cursor={first}", 400, token=bob["token"]) # cursor of another user
api.call("GET", "/api/v1/chats?limit=0", 400, token=bob["token"])
# ================================================================== messages & keyset pagination
def test_history_pages_without_loss_or_duplicates(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
sent = [api.send(alice if i % 2 else bob, chat, f"сообщение {i}") for i in range(25)]
items, pages = api.history(bob, chat, 7)
assert pages == 4 and items == sent and items == sorted(items, key=key)
assert sent[0]["sender_id"] == bob["id"] and sent[1]["sender_id"] == alice["id"]
@pytest.mark.parametrize("query", ["limit=0", "limit=101", "limit=abc", "limit=1&limit=2", "cursor=", "cursor=garbage",
"cursor=djF8bXx4fDF8Mg", "cursor=" + base64.urlsafe_b64encode(b"v1|m|x|99999999999999999999|y").decode()])
def test_bad_limit_or_cursor_is_400(api, query):
alice = api.user("alice")
chat = api.chat(alice)
api.call("GET", f"/api/v1/chats/{chat['id']}/messages?{query}", 400, token=alice["token"])
def test_cursor_bound_to_chat(api):
alice = api.user("alice")
a, b = api.chat(alice, title="A"), api.chat(alice, title="B")
for chat in (a, b):
api.send(alice, chat, "1")
api.send(alice, chat, "2")
cursor = api.call("GET", f"/api/v1/chats/{a['id']}/messages?limit=1", 200, token=alice["token"]).json()["next_cursor"]
api.call("GET", f"/api/v1/chats/{b['id']}/messages?cursor={cursor}", 400, token=alice["token"])
empty = api.chat(alice, title="Empty")
api.call("GET", f"/api/v1/chats/{empty['id']}/messages?cursor=garbage", 400, token=alice["token"])
def test_read_after_write(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
for i in range(5):
message = api.send(alice, chat, f"m{i}")
last = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"]).json()["items"][-1]
assert last == message
# ================================================================== lab 3: inputs found by the reviewer in other PRs
def b64(raw: bytes) -> str:
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
def bad_cursors(owner_id: str) -> list[str]:
"""Wrong types/values inside the cursor, empty and whitespace cursors, out-of-range time."""
good_uuid = str(uuid.uuid4())
return ["", " ", "%20", "garbage", b64(b"[]"), b64(b"{}"), b64(b"123"), b64(b"false"), b64(b"null"),
b64(b'{"created_at":1,"id":1}'), b64(b'{"created_at":null,"id":"x"}'),
b64(f"v1|m|{owner_id}|1|1".encode()), b64(f"v1|c|{owner_id}|1|1".encode()),
b64(f"v1|m|{owner_id}|abc|{good_uuid}".encode()), b64(f"v1|m|{owner_id}|-1|{good_uuid}".encode()),
b64(f"v1|m|{owner_id}|9223372036854775807|{good_uuid}".encode()),
b64(f"v1|c|{owner_id}|9223372036854775807|{good_uuid}".encode()),
b64(f"v1|m|{owner_id}|1|{good_uuid.upper()}".encode()), b64(f"v2|m|{owner_id}|1|{good_uuid}".encode())]
def test_malformed_cursor_values_are_400_on_both_lists(api):
alice = api.user("cur")
chat = api.chat(alice)
for cursor in bad_cursors(chat["id"]) + bad_cursors(alice["id"]):
quoted = cursor if cursor in ("", "%20") else cursor.replace(" ", "%20")
for path in (f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"):
error = api.call("GET", f"{path}?cursor={quoted}", 400, token=alice["token"]).json()
assert error["error"]["code"] == "invalid_request", (path, cursor)
@pytest.mark.parametrize("raw", [b"\xff", b'{"text":"first","TEXT":"second"}', b'{"text":"a","text":"b"}',
b'{"text":123}', b'{"text":"valid"}}', b'{"text":"valid"}]', b'{"text":"x"} {}',
b'{"text":"x","sender_id":"11111111-1111-4111-8111-111111111111"}',
b'{"text":"x","attachment_ids":null}', b'{"text":"x","attachment_ids":["nope"]}',
b'{"text":" \n\t "}'])
def test_bad_message_body_is_400_and_nothing_is_stored(api, raw):
alice = api.user("body")
chat = api.chat(alice)
error = api.post_message(alice, chat, None, 400, raw=raw).json()
assert error["error"]["code"] == "invalid_request"
assert api.history(alice, chat, 100)[0] == []
def test_ff_byte_body_on_registration_is_contract_error(api):
response = api.call("POST", "/api/v1/users", 400, raw=b"\xff", headers={"Content-Type": "application/json"})
assert response.json()["error"]["request_id"] == response.headers["x-request-id"]
def test_text_is_stored_verbatim_and_length_counts_raw_text(api):
alice = api.user("verbatim")
chat = api.chat(alice)
padded = " строка\nс пробелами "
assert api.send(alice, chat, padded)["text"] == padded
api.post_message(alice, chat, {"text": " " + "a" * 4000}, 400) # 4001 code points before any trimming
assert [m["text"] for m in api.history(alice, chat, 100)[0]] == [padded]
def test_unknown_attachment_ids_are_404(api):
alice = api.user("att")
chat = api.chat(alice)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())] * 2}, 400) # duplicates
assert api.history(alice, chat, 100)[0] == []
def test_member_ids_are_compared_as_uuids_not_strings(api):
alice, bob = api.user("case"), api.user("case")
api.call("POST", "/api/v1/chats", 400, token=alice["token"],
json_body={"title": "x", "member_ids": [alice["id"], alice["id"].upper()]})
api.call("POST", "/api/v1/chats", 400, token=alice["token"],
json_body={"title": "x", "member_ids": [alice["id"], bob["id"]], "id": str(uuid.uuid4())})
chat = api.call("POST", "/api/v1/chats", 201, token=alice["token"],
json_body={"title": "x", "member_ids": [alice["id"].upper(), bob["id"]]}, schema="Chat").json()
assert chat["member_ids"] == [alice["id"], bob["id"]]
@pytest.mark.parametrize("header", ["Bearer é", "Bearer .", "Bearer a b", "Bearer " + "é" * 43, "bearer\t" + "a" * 43])
def test_non_ascii_or_garbage_bearer_is_401_not_500(api, header):
api.call("GET", "/api/v1/users/me", 401, headers={"Authorization": header.encode("utf-8")})
def test_login_with_a_body_is_rejected(api):
actor = api.register("login_body")
auth = {"Authorization": basic(actor["username"], actor["password"])}
api.call("POST", "/api/v1/auth/sessions", 400, raw=b'{"extra":1}', headers={**auth, "Content-Type": "application/json"})
api.call("POST", "/api/v1/auth/sessions", 400, raw=b"x", headers=auth)
api.call("POST", "/api/v1/auth/sessions", 201, headers=auth, schema="Session")
def test_password_range_is_12_to_128_code_points(api):
api.call("POST", "/api/v1/users", 400, json_body={"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "x", "password": "a"})
actor = api.user("pw128", password="я:" * 64)
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
def test_every_api_response_carries_instance_and_request_id(api):
"""X-Instance-Id also on 404 of unknown routes, 405, 401, 400 and health (Api.call asserts it)."""
api.call("GET", "/health/live", 200)
api.call("GET", "/health/ready", 200)
api.call("GET", "/no/such/route", 404)
api.call("PUT", "/api/v1/users", 405)
api.call("GET", "/api/v1/users/me", 401)
response = api.call("GET", "/api/v1/users/me", 401, headers={"X-Request-Id": "client-supplied.id-1"})
assert response.headers["x-request-id"] == "client-supplied.id-1" # one id from LB log to API log
@pytest.mark.parametrize("blank", [" ", "  ", "  \t "])
def test_unicode_whitespace_only_strings_are_400(api, blank):
alice = api.user("blank")
chat = api.chat(alice)
api.call("POST", "/api/v1/users", 400, json_body={"username": "bl_" + uuid.uuid4().hex[:10], "display_name": blank,
"password": PASSWORD})
api.call("POST", "/api/v1/chats", 400, token=alice["token"], json_body={"title": blank, "member_ids": [alice["id"]]})
api.post_message(alice, chat, {"text": blank}, 400)
assert api.history(alice, chat, 100)[0] == []
@pytest.mark.parametrize("raw", [b'{"username":"num_probe","display_name":"x","password":1e1000}',
'{"username":"ff_probe","display_name":"x","password":"12345678901'.encode() + b'\xff"}',
b'{"username":"num_probe","display_name":1,"password":"Twelve-chars-ok"}'])
def test_odd_registration_bodies_are_400_and_the_api_survives(api, raw):
api.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"})
api.call("GET", "/health/ready", 200)
def test_more_than_100_members_is_400(api):
owner = api.user("big")
others = [api.register(f"m{i}")["id"] for i in range(100)]
api.call("POST", "/api/v1/chats", 400, token=owner["token"], json_body={"title": "x", "member_ids": [owner["id"], *others]})
chat = api.call("POST", "/api/v1/chats", 201, token=owner["token"],
json_body={"title": "x", "member_ids": [owner["id"], *others[:99]]}, schema="Chat").json()
assert len(chat["member_ids"]) == 100
def test_cursor_of_one_list_is_rejected_by_the_other(api):
alice = api.user("cross")
chats = [api.chat(alice, title=f"c{i}") for i in range(2)]
for _ in range(2):
api.send(alice, chats[0], "m")
message_cursor = api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?limit=1", 200,
token=alice["token"]).json()["next_cursor"]
chat_cursor = api.call("GET", "/api/v1/chats?limit=1", 200, token=alice["token"]).json()["next_cursor"]
api.call("GET", f"/api/v1/chats?cursor={message_cursor}", 400, token=alice["token"])
api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?cursor={chat_cursor}", 400, token=alice["token"])
+268
View File
@@ -0,0 +1,268 @@
"""Attachments through the load balancer (lab 4, levels 3 and 4): upload/download, ACL, limits,
content checks, filename rules and messages with attachment_ids.
Negative inputs include every case the reviewer used against other lab 4 solutions: corrupted
IDAT with valid CRCs, the first 33 bytes of the fixture, JPEG without scan data, a 20000x10000
header (DecompressionBomb), exactly 10 MiB through the LB, C1 control U+0085 and a whitespace
filename, text=null / missing text / attachment_ids=null in a message."""
from __future__ import annotations
import hashlib
import subprocess
import time
import uuid
from pathlib import Path
import pytest
import imagegen
from conftest import attachment_row, compose, s3_size
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
MiB = 1024 * 1024
@pytest.fixture()
def trio(api):
"""alice + bob share a chat; eve is an outsider with her own chat."""
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
return alice, bob, eve, api.chat(alice, bob), api.chat(eve)
def content(api, actor, attachment_id, expected=200, query=""):
return api.call("GET", f"/api/v1/attachments/{attachment_id}/content{query}", expected, token=actor["token"])
# ------------------------------------------------------------------ positive path
def test_png_and_jpeg_roundtrip_byte_for_byte(api, trio):
alice, bob, _, chat, _ = trio
jpg = imagegen.jpeg(321, 123, noise=True)
for data, ctype in ((SAMPLE, "image/png"), (jpg, "image/jpeg")):
att = api.upload(alice, chat, data, content_type=ctype, filename="кот 😺.bin").json()
assert att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None
assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"]
assert att["size_bytes"] == len(data) and att["sha256"] == hashlib.sha256(data).hexdigest()
assert att["content_type"] == ctype and att["filename"] == "кот 😺.bin"
meta = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"], schema="Attachment").json()
assert meta == att
got = content(api, bob, att["id"])
assert got.content == data and got.headers["content-type"] == ctype
assert got.headers["content-length"] == str(len(data))
assert got.headers["x-content-type-options"] == "nosniff"
assert got.headers["content-disposition"].startswith("attachment;")
assert "location" not in got.headers # bytes through the API, never a redirect to S3
def test_metadata_reveals_no_storage_details(api, trio):
alice, _, _, chat, _ = trio
att = api.upload(alice, chat, SAMPLE).json()
row = attachment_row(att["id"])
assert row["status"] == "ready" and row["object_key"].startswith("o/")
text = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=alice["token"]).text
assert row["object_key"] not in text and "messenger-attachments" not in text and "s3" not in text.lower()
# The key is random: neither the id nor the filename is part of it.
assert att["id"].replace("-", "") not in row["object_key"] and "photo" not in row["object_key"]
assert s3_size(row["object_key"]) == len(SAMPLE) # the bytes ARE in the bucket
def test_original_is_not_stored_on_api_containers(api, trio):
"""The API keeps no file of the original: no recently written file of that size anywhere in
the replica's filesystem (the bytes live only in the bucket)."""
alice, _, _, chat, _ = trio
data = imagegen.png_of_size(300_000)
api.upload(alice, chat, data)
size = len(data)
for cid in compose("ps", "-q", "api").stdout.split():
out = subprocess.run(
["docker", "exec", cid, "sh", "-c", f"find / -xdev -type f -size {size}c -mmin -10 2>/dev/null | head"],
capture_output=True, text=True, timeout=60).stdout.strip()
assert out == "", out
# ------------------------------------------------------------------ access control
def test_outsider_and_unknown_get_404(api, trio):
alice, _, eve, chat, _ = trio
att = api.upload(alice, chat, SAMPLE).json()
for suffix in ("", "/content"):
api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"])
api.call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"])
api.call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"])
api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401)
# Upload into a foreign chat: 404 before anything about the body is looked at.
api.upload(eve, chat, SAMPLE, 404)
api.upload(eve, chat, b"garbage", 404, content_type="text/plain")
def test_attachment_ids_are_checked_per_chat(api, trio):
alice, bob, eve, chat, eve_chat = trio
other = api.chat(alice) # alice's second chat
att_other = api.upload(alice, other, SAMPLE).json()
att_eve = api.upload(eve, eve_chat, SAMPLE).json()
# Alice is a member of both chats, but a file is bound to its chat.
api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_other["id"]]}, 404)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404)
# Eve cannot post into alice's chat even with her own file: membership first.
api.post_message(eve, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404)
# Any member may reference an attachment of the chat.
att = api.upload(alice, chat, SAMPLE).json()
msg = api.post_message(bob, chat, {"text": "", "attachment_ids": [att["id"]]}).json()
assert msg["attachment_ids"] == [att["id"]] and msg["sender_id"] == bob["id"]
# ------------------------------------------------------------------ messages with attachments
def test_message_with_attachments_and_history(api, trio):
alice, bob, _, chat, _ = trio
a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2))
m1 = api.post_message(alice, chat, {"text": "", "attachment_ids": [a2, a1]}).json()
m2 = api.post_message(alice, chat, {"text": " ", "attachment_ids": [a1]}).json() # blank text + file: ok
m3 = api.post_message(alice, chat, {"text": "без файлов"}).json()
assert m1["attachment_ids"] == [a2, a1] and m1["text"] == ""
assert m2["text"] == " " and m3["attachment_ids"] == []
items, _ = api.history(bob, chat, 2)
assert [m["attachment_ids"] for m in items] == [[a2, a1], [a1], []]
def test_idempotent_replay_with_attachments(api, trio):
alice, _, _, chat, _ = trio
a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2))
key = uuid.uuid4().hex
first = api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a1, a2]}, key=key).json()
raw = ('{"attachment_ids":["%s","%s"],"text":"файлы"}' % (a1, a2)).encode()
assert api.post_message(alice, chat, None, raw=raw, key=key).json() == first # key order irrelevant
api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a2, a1]}, 409, key=key) # other body
api.post_message(alice, chat, {"text": "файлы"}, 409, key=key)
# A failed attachment check does not burn the key: the same key works with a valid body later.
key2 = uuid.uuid4().hex
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404, key=key2)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [a1]}, key=key2)
@pytest.mark.parametrize("raw", [
b'{"attachment_ids":["%(a)s"]}', # text missing
b'{"text":null,"attachment_ids":["%(a)s"]}', # text null
b'{"text":1,"attachment_ids":["%(a)s"]}',
b'{"text":"x","attachment_ids":null}', # attachment_ids null
b'{"text":"x","attachment_ids":"%(a)s"}',
b'{"text":"x","attachment_ids":{}}',
b'{"text":"x","attachment_ids":["%(a)s","%(a)s"]}', # duplicates
b'{"text":"x","attachment_ids":["%(a)s",1]}',
b'{"text":"x","attachment_ids":["not-a-uuid"]}',
b'{"text":"","attachment_ids":[]}', # neither text nor files
b'{"text":" "}',
b'{"text":"x","attachment_ids":[],"sender_id":"%(a)s"}',
], ids=lambda raw: raw.decode()[:40])
def test_message_body_validation(api, trio, raw):
alice, _, _, chat, _ = trio
a = api.upload(alice, chat, SAMPLE).json()["id"]
api.post_message(alice, chat, None, 400, raw=raw % {b"a": a.encode()})
def test_at_most_ten_attachments(api, trio):
alice, _, _, chat, _ = trio
ids = [api.upload(alice, chat, imagegen.png(4, 4)).json()["id"] for _ in range(11)]
assert len(api.post_message(alice, chat, {"text": "", "attachment_ids": ids[:10]}).json()["attachment_ids"]) == 10
api.post_message(alice, chat, {"text": "", "attachment_ids": ids}, 400)
# ------------------------------------------------------------------ limits
def test_size_limit_through_the_load_balancer(api, trio):
alice, bob, _, chat, _ = trio
exact = imagegen.png_exactly(10 * MiB)
assert len(exact) == 10 * MiB
att = api.upload(alice, chat, exact).json() # exactly the limit: accepted via LB
assert content(api, bob, att["id"]).content == exact
started = time.monotonic()
api.upload(alice, chat, exact + b"\x00", 413) # Content-Length over the limit
assert time.monotonic() - started < 5
over = exact + bytes(1024)
def chunks(): # no Content-Length: chunked body
for i in range(0, len(over), 256 * 1024):
yield over[i:i + 256 * 1024]
error = api.upload(alice, chat, chunks(), 413).json()
assert error["error"]["code"] == "payload_too_large"
def test_pixel_limit_before_decoding(api, trio):
alice, _, _, chat, _ = trio
for data, ctype in ((imagegen.png_bomb_header(), "image/png"), (imagegen.jpeg_bomb_header(), "image/jpeg"),
(imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), "image/png")):
started = time.monotonic()
assert api.upload(alice, chat, data, 413, content_type=ctype).json()["error"]["code"] == "payload_too_large"
assert time.monotonic() - started < 2 # refused from the header, nothing decoded
def test_exactly_40_megapixels_is_accepted(api, trio):
alice, _, _, chat, _ = trio
data = imagegen.png(8000, 5000, compress_level=9) # 40 000 000 px, ~120 KB file
att = api.upload(alice, chat, data).json()
assert att["size_bytes"] == len(data)
# ------------------------------------------------------------------ content checks
BROKEN = {
"png: first 33 bytes of the fixture": (SAMPLE[:33], "image/png"),
"png: truncated": (SAMPLE[:-20], "image/png"),
"png: corrupt IDAT, valid CRC": (imagegen.png_corrupt_idat(), "image/png"),
"png: fewer rows than IHDR": (imagegen.png_short_pixels(), "image/png"),
"png: flipped byte": (SAMPLE[:700] + bytes([SAMPLE[700] ^ 0x5A]) + SAMPLE[701:], "image/png"),
"png: bytes after IEND": (SAMPLE + b"<?php echo 1; ?>", "image/png"),
"jpeg: no scan data after SOS": (imagegen.jpeg_without_scan_data(), "image/jpeg"),
"jpeg: truncated": (imagegen.jpeg(400, 300, noise=True)[:5000], "image/jpeg"),
"mime: png bytes declared jpeg": (SAMPLE, "image/jpeg"),
"mime: jpeg bytes declared png": (imagegen.jpeg(), "image/png"),
"mime: gif declared png": (b"GIF89a" + bytes(100), "image/png"),
"mime: html declared png": (b"<html><script>alert(1)</script></html>", "image/png"),
"type: text/plain": (SAMPLE, "text/plain"),
"type: octet-stream": (SAMPLE, "application/octet-stream"),
"type: multipart": (SAMPLE, "multipart/form-data; boundary=x"),
"type: missing": (SAMPLE, None),
}
@pytest.mark.parametrize("name", list(BROKEN))
def test_bad_content_is_415(api, trio, name):
alice, _, _, chat, _ = trio
data, ctype = BROKEN[name]
error = api.upload(alice, chat, data, 415, content_type=ctype).json()
assert error["error"]["code"] == "unsupported_media_type"
def test_empty_body_is_400(api, trio):
alice, _, _, chat, _ = trio
api.upload(alice, chat, b"", 400)
# ------------------------------------------------------------------ filename
@pytest.mark.parametrize("query", [
"", "filename=", "filename=%20%20%20", "filename=.", "filename=..", "filename=a%2Fb.png", "filename=a%5Cb.png",
"filename=a%00b.png", "filename=c1%C2%85.png", "filename=tab%09.png", "filename=photo%E2%80%AEgnp.exe",
"filename=%FF%FE.png", "filename=a.png&filename=b.png", "filename=" + "a" * 256,
], ids=lambda q: q[:40] or "missing")
def test_bad_filename_is_400(api, trio, query):
alice, _, _, chat, _ = trio
api.upload(alice, chat, SAMPLE, 400, query=query)
def test_unicode_filename_is_kept_and_safely_exposed(api, trio):
alice, _, _, chat, _ = trio
name = 'отчёт "Q3"; ok.png'
att = api.upload(alice, chat, SAMPLE, filename=name).json()
assert att["filename"] == name
disposition = content(api, alice, att["id"]).headers["content-disposition"]
assert '"' not in disposition.split('filename="')[1].split('";')[0] # quotes neutralised in the fallback
assert "filename*=UTF-8''" in disposition
# ------------------------------------------------------------------ variants
def test_variant_parameter(api, trio):
alice, _, eve, chat, _ = trio
att = api.upload(alice, chat, SAMPLE).json()
assert content(api, alice, att["id"], query="?variant=original").content == SAMPLE
content(api, alice, att["id"], 404, query="?variant=thumbnail") # schema name, not produced in lab 4
content(api, alice, att["id"], 400, query="?variant=huge")
content(api, eve, att["id"], 404, query="?variant=thumbnail") # outsider: 404, nothing revealed
+334
View File
@@ -0,0 +1,334 @@
"""Load balancer, replicas and graceful shutdown (lab 3, levels 3 and 4).
Each test that stops, kills, rescales or isolates a replica restores the stand in `finally`
and waits until HAProxy reports every replica UP again (conftest.wait_stand).
"""
from __future__ import annotations
import contextlib
import json
import subprocess
import threading
import time
import uuid
import httpx2 as httpx
import pytest
from conftest import BASE_URL, REPLICAS, Api, compose, lbctl, probe_inside, psql, replica_stopped, run_hook, \
running_replicas, wait_healthy, wait_stand
pytestmark = pytest.mark.stack
def instances_seen(api: Api, token: str, requests: int = 40) -> list[str]:
return [api.call("GET", "/api/v1/users/me", 200, token=token).headers["x-instance-id"] for _ in range(requests)]
# ------------------------------------------------------------------ level 3: replicas, no affinity
def test_one_session_is_served_by_every_real_replica(api):
actor = api.user("rr")
seen = instances_seen(api, actor["token"])
running = {r["hostname"] for r in running_replicas()}
assert len(running) == REPLICAS >= 2
assert set(seen) == running, "X-Instance-Id values must be exactly the hostnames of the running containers"
# Round robin on ONE keep-alive client connection: consecutive requests switch replicas,
# so there is no sticky session hiding split state.
switches = sum(1 for a, b in zip(seen, seen[1:]) if a != b)
assert switches >= len(seen) // 2, seen
assert {s["instance_id"] for s in lbctl.lb_servers() if s["state"] == "UP"} == running
def test_write_on_one_replica_is_read_on_the_other(api):
alice, bob = api.user("rw"), api.user("rw")
chat = api.chat(alice, bob)
for i in range(10):
sent = api.post_message(alice, chat, {"text": f"m{i}"})
read = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"])
assert read.json()["items"][-1] == sent.json()
logout = api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
for _ in range(6): # the revocation is visible on every replica at once (shared Valkey)
r = api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
assert r.headers["x-instance-id"]
assert logout.headers["x-instance-id"]
# ------------------------------------------------------------------ readiness drives the LB
def hba_file() -> str:
return psql("SHOW hba_file")
@contextlib.contextmanager
def isolated_from_postgres(replica: dict):
"""Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its
connections), while it keeps running and answering /health/live. Always undone."""
ip = replica["networks"]["data"] # PostgreSQL sees the replica's address on the data network
path = hba_file()
marker = "lab3-test-isolate"
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}")
try:
psql("SELECT pg_reload_conf()")
psql(f"SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE client_addr = '{ip}'")
yield replica
finally:
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}")
psql("SELECT pg_reload_conf()")
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: no restart of the replica needed
wait_stand()
@contextlib.contextmanager
def disconnected_from_data_network(replica: dict):
"""`docker network disconnect <project>_data <api>`: the replica keeps its edge link to the
LB but loses PostgreSQL AND Valkey. Always reconnected."""
network = next(n for n in json.loads(subprocess.run(
["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", replica["id"]],
capture_output=True, text=True, check=True).stdout) if n.endswith("_data"))
subprocess.run(["docker", "network", "disconnect", network, replica["id"]], check=True, capture_output=True)
try:
yield
finally:
subprocess.run(["docker", "network", "connect", network, replica["id"]], check=True, capture_output=True)
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: dead sockets are dropped (tcp_user_timeout)
wait_stand()
@pytest.mark.parametrize("how", ["pg_hba", "network"])
@pytest.mark.parametrize("position", range(REPLICAS))
def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position, how):
"""The reviewers' scenarios: one API loses PostgreSQL (pg_hba reject) or its whole data
network (docker network disconnect) but keeps running. /health/live 200, /health/ready
503 -> HAProxy must take it out; no business request may reach it."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
actor = api.user("ready")
isolate = isolated_from_postgres if how == "pg_hba" else disconnected_from_data_network
with isolate(replica):
started = time.monotonic()
while replica["hostname"] in lbctl.up_instances():
assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s"
time.sleep(0.1)
detected = time.monotonic() - started
for _ in range(3): # the process stays responsive (event loop never blocked) ...
started_probe = time.monotonic()
assert probe_inside(replica, "/health/live") == 200
assert probe_inside(replica, "/health/ready") == 503 # ... but not ready
assert time.monotonic() - started_probe < 4.5, "replica froze while its dependencies were unreachable"
seen = instances_seen(api, actor["token"], 40)
assert replica["hostname"] not in seen and len(seen) == 40
for i in range(20): # writes keep working, only through the ready replicas
assert api.register(f"r{i}")["id"]
print(f"{replica['name']} ({how}): unready replica out of rotation after {detected:.2f} s")
# ------------------------------------------------------------------ stopping replicas under traffic
class Traffic:
"""Background writers/readers through the LB; uncertain POSTs are retried with the SAME key."""
def __init__(self, api: Api, workers: int = 4):
self.alice, self.bob = api.user("load"), api.user("load")
self.chat = api.chat(self.alice, self.bob)
self.workers = workers
self.stop = threading.Event()
self.lock = threading.Lock()
self.events: list[tuple[float, str, int, str | None]] = [] # (time, kind, status, instance)
self.confirmed: list[dict] = []
self.threads = [threading.Thread(target=self.run, args=(i,), daemon=True) for i in range(workers)]
def record(self, kind: str, status: int, instance: str | None) -> None:
with self.lock:
self.events.append((time.monotonic(), kind, status, instance))
def run(self, index: int) -> None:
client = httpx.Client(base_url=BASE_URL, timeout=3)
auth = {"Authorization": "Bearer " + self.alice["token"]}
n = 0
key, text = uuid.uuid4().hex, f"w{index}-0"
while not self.stop.is_set():
try:
r = client.post(f"/api/v1/chats/{self.chat['id']}/messages", json={"text": text},
headers={**auth, "Idempotency-Key": key})
self.record("POST", r.status_code, r.headers.get("x-instance-id"))
if r.status_code == 201:
with self.lock:
self.confirmed.append(r.json())
n += 1
key, text = uuid.uuid4().hex, f"w{index}-{n}"
# any other outcome: keep key and text, retry the same request
except httpx.HTTPError:
self.record("POST", 0, None)
try:
r = client.get(f"/api/v1/chats/{self.chat['id']}/messages?limit=5", headers=auth)
self.record("GET", r.status_code, r.headers.get("x-instance-id"))
except httpx.HTTPError:
self.record("GET", 0, None)
time.sleep(0.02)
client.close()
def __enter__(self) -> "Traffic":
for t in self.threads:
t.start()
time.sleep(1.0)
return self
def __exit__(self, *exc) -> None:
self.stop.set()
for t in self.threads:
t.join(timeout=10)
def failures(self, since: float = 0.0) -> list[tuple]:
return [e for e in self.events if e[0] >= since and e[2] not in (200, 201)]
def check_history(self, api: Api) -> None:
history = api.history(self.bob, self.chat, 100)[0]
ids, texts = [m["id"] for m in history], [m["text"] for m in history]
assert len(ids) == len(set(ids)) and len(texts) == len(set(texts)), "duplicate after retry"
assert {m["id"] for m in self.confirmed} <= set(ids), "a confirmed message was lost"
@pytest.mark.parametrize("position", range(REPLICAS))
def test_graceful_stop_of_each_replica_loses_no_request(api, position):
"""SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit 0: not one failed request."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
with Traffic(api) as traffic:
started = time.monotonic()
with replica_stopped(replica, "stop"):
stopped_after = time.monotonic() - started
time.sleep(3)
after = [e for e in traffic.events if e[0] > time.monotonic() - 2]
assert after and replica["hostname"] not in {e[3] for e in after}
time.sleep(1)
state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True,
check=True).stdout)[0]["State"]
assert traffic.failures() == [], traffic.failures()[:10]
assert stopped_after < 15, "graceful stop must finish before stop_grace_period (no SIGKILL)"
assert state["Status"] == "running" # restored by replica_stopped
traffic.check_history(api)
print(f"{replica['name']}: stop took {stopped_after:.1f} s, {len(traffic.events)} requests, 0 failed")
@pytest.mark.parametrize("position", range(REPLICAS))
def test_killed_replica_bounded_errors_recovery_and_no_duplicates(api, position):
"""SIGKILL: requests in flight on that replica may fail (counted), new ones succeed within
10 s, the killed replica gets nothing after detection, retries do not duplicate."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
with Traffic(api) as traffic:
killed_at = time.monotonic()
with replica_stopped(replica, "kill"):
time.sleep(4)
failures = traffic.failures(killed_at)
last_failure = max((e[0] for e in failures), default=killed_at) - killed_at
post_ok = [e for e in traffic.events if e[0] > killed_at and e[1] == "POST" and e[2] == 201]
assert post_ok and post_ok[0][0] - killed_at <= 10, "writes did not recover within 10 s"
late = [e for e in traffic.events if e[0] > killed_at + 3.5]
assert late and replica["hostname"] not in {e[3] for e in late}
assert last_failure <= 10
traffic.check_history(api)
print(f"{replica['name']}: killed; {len(failures)} failed of {len(traffic.events)}, last failure +{last_failure:.2f} s")
def test_graceful_shutdown_order_in_logs(api):
"""Logs prove the order: drain_started -> (HAProxy marks DOWN) -> shutdown_requested -> exit 0."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[-1]
since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 1))
with replica_stopped(replica, "stop"):
logs = subprocess.run(["docker", "logs", "--since", since, replica["id"]], capture_output=True,
text=True, encoding="utf-8").stdout
lb_logs = compose("logs", "--no-log-prefix", "--since", since, "lb").stdout
state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True,
check=True).stdout)[0]["State"]
events = [json.loads(line) for line in logs.splitlines() if line.startswith("{")]
names = [e["event"] for e in events]
assert names.index("drain_started") < names.index("shutdown_requested") < names.index("stopped")
t = {e["event"]: e["ts"] for e in events}
assert t["drain_started"] < t["shutdown_requested"]
assert state["ExitCode"] == 0 and not state["OOMKilled"]
assert "is DOWN" in lb_logs and ("503" in lb_logs or "Connection refused" in lb_logs)
# ------------------------------------------------------------------ discovery: restart and scaling
def test_full_restart_lb_rediscovers_all_replicas(api):
actor = api.user("restart")
before = {r["hostname"]: r["started_at"] for r in running_replicas()}
run_hook("restart.sh")
wait_healthy("postgres", "valkey", "api")
wait_stand()
after = {r["hostname"]: r["started_at"] for r in running_replicas()}
assert set(after) == set(before) and all(after[h] != before[h] for h in after), "every replica restarted"
assert set(instances_seen(api, actor["token"])) == set(after) # same token, both replicas, no LB restart needed
def test_scale_up_and_down_is_picked_up_without_errors(api):
with Traffic(api, workers=2) as traffic:
try:
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS + 1}", "api",
timeout=180)
lbctl.wait_up(REPLICAS + 1, timeout=30)
actor = api.user("scale")
assert len(set(instances_seen(api, actor["token"], 30))) == REPLICAS + 1
finally:
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS}", "api",
timeout=180)
wait_stand()
time.sleep(1)
assert traffic.failures() == [], traffic.failures()[:10]
traffic.check_history(api)
# ------------------------------------------------------------------ level 4: deployment policy
def test_compose_policy_limits_ports_secrets_images():
config = json.loads(compose("config", "--format", "json").stdout)
services = config["services"]
# The reconciler (profile "tools") is started on demand and not part of the running stand.
assert set(services) == {"postgres", "valkey", "s3", "migrate", "s3-init", "api", "lb"}
for name, svc in services.items():
limits = svc.get("deploy", {}).get("resources", {}).get("limits", {})
assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits"
assert "container_name" not in svc, f"{name}: container_name prevents scaling"
if name in ("api", "migrate"):
assert "build" in svc # built from this repository's source
elif name == "s3-init":
assert svc["image"] == services["api"]["image"] # the same locally built image
else:
assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest"
assert services["api"]["deploy"]["replicas"] >= 2 and not services["api"].get("ports")
published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")}
assert set(published) == {"lb"} and all(p["host_ip"] in ("127.0.0.1", "::1") for p in published["lb"])
secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()}
s3_keys = ['s3_admin_access_key_id', 's3_admin_secret_access_key', 's3_api_access_key_id', 's3_api_secret_access_key', 's3_reconciler_access_key_id', 's3_reconciler_secret_access_key']
assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"], "s3": s3_keys,
"migrate": ["database_url"], "s3-init": s3_keys, "lb": [],
# the API: its own S3 key pair only (no admin, no reconciler keys)
"api": ["cache_url", "database_url", "s3_api_access_key_id", "s3_api_secret_access_key"]}
volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"]
for name, svc in services.items()}
assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "s3": ["s3data"], "migrate": [],
"s3-init": [], "api": [], "lb": []}
networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()}
assert networks == {"postgres": ["data"], "valkey": ["data"], "s3": ["storage"], "migrate": ["data"],
"s3-init": ["storage"], "api": ["data", "edge", "storage"],
"lb": ["edge"]}, networks # the LB cannot reach the stores at all
assert config["networks"]["data"].get("internal") is True
assert config["networks"]["storage"].get("internal") is True
def test_running_containers_have_limits_and_no_bypass_ports():
ids = compose("ps", "-q").stdout.split()
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True,
check=True).stdout):
name = item["Name"]
assert item["HostConfig"]["NanoCpus"] > 0 and item["HostConfig"]["Memory"] > 0, name
bindings = item["HostConfig"]["PortBindings"] or {}
if "-lb-" in name:
assert all(b["HostIp"] in ("127.0.0.1", "::1") for v in bindings.values() for b in v) # loopback only
else:
assert not bindings, f"{name} publishes ports in bypass of the LB"
def test_migrations_ran_once_by_the_migrate_service():
applied = psql("SELECT version FROM schema_migrations ORDER BY version").split()
assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function", "0004_attachments"]
migrate = compose("ps", "-a", "--format", "json", "migrate").stdout
rows = [json.loads(line) for line in migrate.splitlines() if line.strip()]
assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows)
api_logs = compose("logs", "--no-log-prefix", "api").stdout
assert "migrate:" not in api_logs and "schema_migrations" not in api_logs # replicas never migrate
+111
View File
@@ -0,0 +1,111 @@
"""Consistency of PostgreSQL and S3 around one upload (lab 4, level 4).
The two-store write is: intent row 'pending' -> S3 PUT -> row 'ready' -> 201. Failures between
the steps are injected with FAULT_POINTS (compose variable, empty in normal runs; the API
raises 503 at that exact point, as if the process or the network failed there):
* after_s3_put object written, row still pending, client got 503 -> the janitor removes
the object and the row after PENDING_GRACE_SECONDS (here 3 s);
* after_ready_commit row ready and object written, but the answer is lost (client got 503)
-> nothing may be deleted: the attachment stays intact and downloadable.
Plus: an upload interrupted by the client and a stalled body leave nothing behind."""
from __future__ import annotations
import socket
import time
from pathlib import Path
import pytest
import imagegen
from conftest import api_with_env, attachment_row, psql, s3_size
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
def rows_of_chat(chat_id: str) -> list[tuple[str, str, str]]:
out = psql(f"SELECT id || ' ' || status || ' ' || object_key FROM attachments WHERE chat_id = '{chat_id}'")
return [tuple(line.split()) for line in out.splitlines() if line]
def wait_for(condition, timeout: float, what: str) -> None:
deadline = time.monotonic() + timeout
while not condition():
assert time.monotonic() < deadline, what
time.sleep(0.5)
def raw_upload(token: str, chat_id: str, declared: int, body: bytes, *, wait_answer: float | None) -> bytes:
"""Hand-made HTTP request: announce `declared` bytes, send only `body`, then either close the
socket (client gone) or keep it open and read the answer after `wait_answer` seconds."""
sock = socket.create_connection(("127.0.0.1", 8080), timeout=30)
head = (f"POST /api/v1/chats/{chat_id}/attachments?filename=x.png HTTP/1.1\r\nHost: localhost\r\n"
f"Authorization: Bearer {token}\r\nContent-Type: image/png\r\nContent-Length: {declared}\r\n\r\n")
sock.sendall(head.encode() + body)
if wait_answer is None:
sock.close()
return b""
time.sleep(wait_answer)
answer = sock.recv(65536)
sock.close()
return answer
def test_interrupted_and_stalled_uploads_leave_nothing(api):
alice = api.user("alice")
chat = api.chat(alice)
data = imagegen.png_of_size(2_000_000)
raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=None) # client disconnects
with api_with_env(UPLOAD_READ_TIMEOUT_SECONDS="2"):
answer = raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=5) # client stalls
assert answer.startswith(b"HTTP/1.1 400") and b"invalid_request" in answer, answer[:200]
assert b"x-instance-id" in answer.lower()
time.sleep(1)
assert rows_of_chat(chat["id"]) == [] # nothing was written: the body is checked before any write
def test_crash_between_s3_put_and_commit_is_cleaned_by_janitor(api):
alice = api.user("alice")
chat = api.chat(alice)
with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"):
assert api.upload(alice, chat, SAMPLE, 503).json()["error"]["code"] == "unavailable"
[(attachment_id, status, key)] = rows_of_chat(chat["id"])
assert status == "pending" and s3_size(key) == len(SAMPLE) # the orphan-to-be exists
api.call("GET", f"/api/v1/attachments/{attachment_id}", 404, token=alice["token"]) # invisible
api.post_message(alice, chat, {"text": "", "attachment_ids": [attachment_id]}, 404) # not attachable
wait_for(lambda: rows_of_chat(chat["id"]) == [], 30, "janitor did not remove the stale intent")
assert s3_size(key) is None # object removed first
# Normal configuration again: an upload in the same chat works and is not touched.
att = api.upload(alice, chat, SAMPLE).json()
assert attachment_row(att["id"])["status"] == "ready"
def test_lost_answer_after_commit_never_deletes_the_object(api):
"""The reviewer found this in another PR: COMMIT succeeded, the answer was lost, the API
'compensated' by deleting the object -> ready row without bytes. Here nothing compensates
in the request path, and the janitor only touches 'pending' rows."""
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
with api_with_env(FAULT_POINTS="after_ready_commit", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"):
api.upload(alice, chat, SAMPLE, 503)
[(attachment_id, status, key)] = rows_of_chat(chat["id"])
assert status == "ready"
time.sleep(6) # several janitor rounds
assert rows_of_chat(chat["id"]) == [(attachment_id, "ready", key)] and s3_size(key) == len(SAMPLE)
got = api.call("GET", f"/api/v1/attachments/{attachment_id}/content", 200, token=bob["token"])
assert got.content == SAMPLE
def test_fresh_pending_upload_is_not_touched_by_janitor(api):
"""An intent younger than PENDING_GRACE_SECONDS is an upload in progress: never removed."""
alice = api.user("alice")
chat = api.chat(alice)
with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="600", JANITOR_INTERVAL_SECONDS="1"):
api.upload(alice, chat, SAMPLE, 503)
time.sleep(4)
[(_, status, key)] = rows_of_chat(chat["id"])
assert status == "pending" and s3_size(key) == len(SAMPLE)
+117
View File
@@ -0,0 +1,117 @@
"""Database-level guarantees, checked over HTTP and verified directly in PostgreSQL / Valkey."""
from __future__ import annotations
import threading
import time
import uuid
import pytest
from conftest import PASSWORD, Api, compose, psql
pytestmark = pytest.mark.stack
def run_concurrently(workers: int, target) -> list:
barrier = threading.Barrier(workers)
results: list = [None] * workers
def body(index: int) -> None:
local = Api() # one HTTP client (connection) per worker
barrier.wait() # all workers fire at the same moment
try:
results[index] = target(local, index)
except BaseException as exc: # noqa: BLE001 - reported by the test
results[index] = exc
threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)]
for t in threads:
t.start()
for t in threads:
t.join()
return results
def test_registration_race_exactly_one_201_and_seven_409():
username = "race_" + uuid.uuid4().hex[:10]
def register(client: Api, _index: int) -> int:
body = {"username": username, "display_name": "Гонка", "password": PASSWORD}
response = client.client.post("/api/v1/users", json=body)
return response.status_code
statuses = run_concurrently(8, register)
# Real HTTP codes are compared: a timeout, 500 or exception fails the test instead of counting as 409.
assert sorted(statuses, key=str) == [201] + [409] * 7, statuses
assert psql(f"SELECT count(*) FROM users WHERE username = '{username}'") == "1"
def _ins_counter() -> int:
return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'"))
def test_create_chat_rolls_back_after_the_chat_row_was_inserted(api):
"""The chat INSERT runs first; the unknown member then breaks the membership INSERT (FK).
Proof that the first INSERT really happened and was rolled back: PostgreSQL counts
attempted inserts in pg_stat_user_tables.n_tup_ins even for aborted transactions,
while the table contains no such chat and no new membership rows.
"""
alice = api.user("rollback")
title = "rollback-" + uuid.uuid4().hex
members_before = int(psql("SELECT count(*) FROM chat_members"))
attempts_before = _ins_counter()
error = api.call("POST", "/api/v1/chats", 400, token=alice["token"],
json_body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}).json()
assert "несуществующих" in error["error"]["message"]
deadline = time.monotonic() + 20 # statistics are flushed by the API backend asynchronously
while _ins_counter() == attempts_before and time.monotonic() < deadline:
time.sleep(0.5)
assert _ins_counter() > attempts_before, "the chat INSERT was never attempted"
assert psql(f"SELECT count(*) FROM chats WHERE title = convert_to('{title}', 'UTF8')") == "0"
assert int(psql("SELECT count(*) FROM chat_members")) == members_before
def test_migrations_rerun_is_a_noop():
users_before = psql("SELECT count(*) FROM users")
result = compose("run", "--rm", "--no-deps", "migrate")
assert "0001_init: already applied, skipped" in result.stdout
assert "0002_idempotency_keys: already applied, skipped" in result.stdout
assert "0003_post_message_function: already applied, skipped" in result.stdout
assert "0004_attachments: already applied, skipped" in result.stdout
assert psql("SELECT count(*) FROM schema_migrations") == "4"
assert psql("SELECT count(*) FROM users") == users_before
def test_100_messages_from_10_clients_keep_order_and_cursor_walk(api):
clients = [api.user(f"w{i}") for i in range(10)]
chat = api.chat(clients[0], *clients[1:])
def writer(client: Api, index: int) -> list[dict]:
return [client.send(clients[index], chat, f"c{index}-m{j}") for j in range(10)]
results = run_concurrently(10, writer)
assert all(isinstance(r, list) for r in results), results
confirmed = [m for batch in results for m in batch]
history, pages = api.history(clients[5], chat, 7)
ids = [m["id"] for m in history]
assert len(confirmed) == 100 and len(history) == 100 and len(set(ids)) == 100 and pages == 15
assert set(ids) == {m["id"] for m in confirmed}
keys = [(m["created_at"], uuid.UUID(m["id"]).int) for m in history]
assert all(a < b for a, b in zip(keys, keys[1:])), "history is not strictly ordered"
stamps = [m["created_at"] for m in history]
assert len(set(stamps)) == 100, "per-chat clock must give unique timestamps"
for i, client in enumerate(clients):
own = [m["text"] for m in history if m["sender_id"] == client["id"]]
assert own == [f"c{i}-m{j}" for j in range(10)]
def test_secrets_are_not_stored_in_plain_text(api):
actor = api.user("store")
stored = psql(f"SELECT password_hash FROM users WHERE id = '{actor['id']}'")
assert stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$") and PASSWORD not in stored
token_hex = actor["token"].encode().hex()
keys = compose("exec", "-T", "valkey", "sh", "-c",
'REDISCLI_AUTH="$(cat /run/secrets/valkey_password)" valkey-cli --scan --pattern "session:*"').stdout.split()
assert keys and all(len(k) == len("session:") + 64 for k in keys)
assert not any(actor["token"] in k or token_hex in k for k in keys)
+35
View File
@@ -0,0 +1,35 @@
"""The history index really bounds the work of a late page (>= 10 000 messages, generic plan)."""
from __future__ import annotations
import re
import sys
import pytest
from conftest import ROOT
pytestmark = pytest.mark.stack
sys.path.insert(0, str(ROOT / "scripts"))
import explain_history # noqa: E402
def section(text: str, number: int) -> str:
parts = re.split(r"^=== ", text, flags=re.M)
return next(p for p in parts if p.startswith(f"{number}."))
def buffers(plan: str) -> int:
return int(re.search(r"Buffers: shared hit=(\d+)", plan).group(1))
def test_late_page_uses_index_range_not_filter():
text = explain_history.run()
assert "20000" in text # the target chat holds 20 000 messages
app_late = section(text, 2)
assert "Index Cond: ((chat_id = $1) AND (ROW(created_at, id) > ROW($2, $3)))" in app_late
assert "Rows Removed by Filter" not in app_late and "Sort" not in app_late
rejected = section(text, 3)
assert "Rows Removed by Filter: 19950" in rejected
# The app query touches a handful of pages; the rejected variants touch the whole prefix.
assert buffers(app_late) <= 10 < buffers(rejected) and buffers(app_late) <= 10 < buffers(section(text, 4))
+111
View File
@@ -0,0 +1,111 @@
"""Dependency failures with a VALID session: bounded 503, never anonymous access, recovery afterwards.
Every test stops/pauses a real container and restores it in `finally` (service_down).
Timing bounds: DB_TIMEOUT_SECONDS=3 (client deadline), CACHE_TIMEOUT_SECONDS=0.5.
Lab 3: requests go through HAProxy. While the outage is being detected the replicas answer
503 themselves; once every replica fails /health/ready, HAProxy answers 503 in the same
contract format (without X-Instance-Id). Both are "unavailable"; never 200 and never 401
for a valid token. Liveness/readiness of the processes are checked inside each container.
"""
from __future__ import annotations
import subprocess
import time
import pytest
from conftest import ROOT, probe_inside, restart_services, running_replicas, service_down
pytestmark = pytest.mark.stack
DB_BOUND = 3.6 # DB_TIMEOUT_SECONDS + network/HTTP overhead
CACHE_BOUND = 1.6 # 2 x CACHE_TIMEOUT_SECONDS + overhead
def timed(api, *args, **kwargs):
started = time.monotonic()
response = api.call(*args, lb_error_ok=True, **kwargs)
return response, time.monotonic() - started
def processes_alive_but_not_ready() -> None:
for replica in running_replicas():
assert probe_inside(replica, "/health/live") == 200
assert probe_inside(replica, "/health/ready") == 503
@pytest.fixture
def prepared(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
message = api.send(alice, chat, "до отказа")
return alice, bob, chat, message
def check_intact(api, alice, bob, chat, message):
"""After recovery the SAME tokens work and confirmed data is unchanged."""
assert api.call("GET", "/api/v1/users/me", 200, token=alice["token"]).json() == alice["user"]
items = api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json()["items"]
assert items == [message]
@pytest.mark.parametrize("how", ["stop", "pause"])
def test_valkey_failure(api, prepared, how):
alice, bob, chat, message = prepared
with service_down("valkey", how):
response, took = timed(api, "GET", "/api/v1/users/me", 503, token=alice["token"])
assert response.json()["error"]["code"] == "unavailable" and took < CACHE_BOUND, took
api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access
api.call("GET", "/health/ready", 503, lb_error_ok=True)
processes_alive_but_not_ready()
check_intact(api, alice, bob, chat, message)
@pytest.mark.parametrize("how", ["stop", "pause"])
def test_postgres_failure(api, prepared, how):
alice, bob, chat, message = prepared
with service_down("postgres", how):
for path in ("/api/v1/users/me", f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"):
response, took = timed(api, "GET", path, 503, token=alice["token"])
assert response.json()["error"]["code"] == "unavailable", path
assert took < DB_BOUND, (path, took) # bounded even when PostgreSQL is frozen (pause)
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access
api.call("GET", "/health/ready", 503, lb_error_ok=True)
processes_alive_but_not_ready()
check_intact(api, alice, bob, chat, message)
def test_locked_table_gives_bounded_503(api, prepared):
"""A statement stuck behind a lock is stopped by lock_timeout, not after minutes."""
alice, bob, chat, message = prepared
locker = subprocess.Popen(
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c",
"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(6); COMMIT;"],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
time.sleep(1.5) # let the lock be taken
response, took = timed(api, "GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
assert response.json()["error"]["code"] == "unavailable" and took < DB_BOUND, took
assert response.headers.get("x-instance-id"), "a lock is the API's own 503, the replicas stay ready"
finally:
locker.wait(timeout=30)
check_intact(api, alice, bob, chat, message)
def test_logout_during_valkey_outage_is_never_lost(api, prepared):
"""Valkey is the single source of truth for sessions, so a logout cannot be 'forgotten':
during the outage logout answers 503 (nothing revoked, the client knows it failed);
after recovery logout works, and the revocation survives a Valkey and API restart (AOF)."""
alice, bob, chat, message = prepared
with service_down("valkey", "stop"):
api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"], lb_error_ok=True)
api.call("GET", "/api/v1/users/me", 200, token=alice["token"]) # logout did not happen
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
restart_services("valkey", "api")
for _ in range(10): # repeated checks: the revoked token must never come back
api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], json_body={"text": "x"})
api.call("GET", "/api/v1/users/me", 200, token=bob["token"]) # other sessions survived the restart
+281
View File
@@ -0,0 +1,281 @@
"""Shared idempotency of POST /chats/{id}/messages (lab 3), through the load balancer.
Strict assertions: exact HTTP codes for every request, identical bodies for replays,
the number of rows in PostgreSQL (messages AND idempotency_keys), and which replica
answered (X-Instance-Id). Fault injection (trigger, row lock, stopped replica) is always
undone in `finally`.
"""
from __future__ import annotations
import subprocess
import threading
import time
import uuid
import httpx2 as httpx
import pytest
from conftest import BASE_URL, ROOT, Api, psql, replica_stopped, running_replicas
pytestmark = pytest.mark.stack
def messages_in_db(chat: dict) -> int:
return int(psql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}'"))
def key_rows(actor: dict, key: str) -> int:
return int(psql(f"SELECT count(*) FROM idempotency_keys WHERE user_id = '{actor['id']}' AND key = '{key}'"))
@pytest.fixture
def pair(api):
alice, bob = api.user("alice"), api.user("bob")
return alice, bob, api.chat(alice, bob)
def test_replay_returns_the_same_201_and_message_from_every_replica(api, pair):
alice, bob, chat = pair
key = "replay-" + uuid.uuid4().hex
first = api.post_message(alice, chat, {"text": "один раз"}, key=key)
served = {first.headers["x-instance-id"]}
for _ in range(20):
again = api.post_message(alice, chat, {"text": "один раз"}, key=key)
assert again.status_code == 201 and again.json() == first.json()
served.add(again.headers["x-instance-id"])
assert len(served) >= 2, "replays must be answered by different replicas, not a local cache"
assert api.history(bob, chat, 100)[0] == [first.json()]
assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1
def test_other_body_with_the_same_key_is_409_and_keeps_the_original(api, pair):
alice, bob, chat = pair
key = uuid.uuid4().hex
original = api.send(alice, chat, "оригинал", key=key)
for body in ({"text": "другой текст"}, {"text": "оригинал "}, {"text": "Оригинал"}):
assert api.post_message(alice, chat, body, 409, key=key).json()["error"]["code"] == "conflict"
assert api.send(alice, chat, "оригинал", key=key) == original
assert api.history(bob, chat, 100)[0] == [original] and key_rows(alice, key) == 1
def test_key_order_and_missing_attachment_ids_are_the_same_body(api, pair):
alice, _, chat = pair
key = uuid.uuid4().hex
first = api.post_message(alice, chat, None, raw='{"text":"порядок"}'.encode(), key=key).json()
for raw in ('{"attachment_ids":[],"text":"порядок"}', '{"text":"порядок","attachment_ids":[]}',
'{ "text" : "\\u043f\\u043e\\u0440\\u044f\\u0434\\u043e\\u043a" }'):
assert api.post_message(alice, chat, None, raw=raw.encode(), key=key).json() == first
assert messages_in_db(chat) == 1
@pytest.mark.parametrize("value", ["", "a" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "*"])
def test_missing_or_invalid_key_is_400(api, pair, value):
alice, _, chat = pair
raw_value = value.encode("utf-8")
error = api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"],
json_body={"text": "x"}, headers={"Idempotency-Key": raw_value} if value else {}).json()
assert error["error"]["code"] == "invalid_request"
assert messages_in_db(chat) == 0
def test_repeated_key_header_is_400(pair):
alice, _, chat = pair
response = httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "x"}, timeout=10,
headers=[("Authorization", "Bearer " + alice["token"]),
("Idempotency-Key", "one"), ("Idempotency-Key", "two")])
assert response.status_code == 400 and response.headers["x-instance-id"]
assert messages_in_db(chat) == 0
def test_longest_key_and_every_allowed_character(api, pair):
alice, _, chat = pair
for key in ("k" * 128, "A-Za-z0-9._:-" + uuid.uuid4().hex):
first = api.send(alice, chat, key[:20], key=key)
assert api.send(alice, chat, key[:20], key=key) == first
assert messages_in_db(chat) == 2
def test_scope_is_user_plus_chat_plus_key(api, pair):
alice, bob, chat = pair
other = api.chat(alice, bob, title="второй")
key = "shared-key"
a1 = api.send(alice, chat, "текст", key=key)
b1 = api.send(bob, chat, "текст", key=key) # another user: independent
a2 = api.send(alice, other, "текст", key=key) # another chat: independent
assert len({a1["id"], b1["id"], a2["id"]}) == 3
assert messages_in_db(chat) == 2 and messages_in_db(other) == 1
def test_outsider_and_revoked_session_never_see_the_stored_result(api, pair):
alice, bob, chat = pair
eve = api.user("eve")
key = uuid.uuid4().hex
api.send(alice, chat, "секрет", key=key)
for _ in range(3): # a replay checks access again: always 404, never 409 or the stored message
assert api.post_message(eve, chat, {"text": "секрет"}, 404, key=key).json()["error"]["code"] == "not_found"
api.post_message(eve, chat, {"text": "другое"}, 404, key=key)
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
api.post_message(alice, chat, {"text": "секрет"}, 401, key=key) # session checked again
assert messages_in_db(chat) == 1
def fire_together(workers: int, request) -> list:
barrier = threading.Barrier(workers)
results: list = [None] * workers
def body(index: int) -> None:
client = httpx.Client(base_url=BASE_URL, timeout=15) # own connection -> LB spreads them
barrier.wait()
try:
results[index] = request(client, index)
except BaseException as exc: # noqa: BLE001 - asserted by the test
results[index] = exc
finally:
client.close()
threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)]
for t in threads:
t.start()
for t in threads:
t.join()
return results
def test_concurrent_same_key_through_different_replicas_creates_exactly_one(pair):
alice, bob, chat = pair
key = "race-" + uuid.uuid4().hex
def post(client: httpx.Client, _index: int):
r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": "гонка"},
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
return r.status_code, r.headers.get("x-instance-id"), r.json()
for _ in range(3): # three rounds: the first creates, later rounds replay concurrently
results = fire_together(24, post)
assert all(isinstance(r, tuple) for r in results), results
assert [r[0] for r in results] == [201] * 24, [r[0] for r in results]
assert len({str(r[2]) for r in results}) == 1, "every concurrent replay must return the same Message"
assert len({r[1] for r in results}) >= 2, "requests did not reach two replicas"
assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1
def test_concurrent_same_key_different_bodies_one_wins(pair):
alice, _, chat = pair
key = "conflict-" + uuid.uuid4().hex
def post(client: httpx.Client, index: int):
r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": f"вариант {index % 2}"},
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
return r.status_code, r.json()
results = fire_together(20, post)
winners = [body for status, body in results if status == 201]
losers = [body for status, body in results if status == 409]
assert len(winners) + len(losers) == 20, results
assert winners and len({str(b) for b in winners}) == 1
assert all(b["error"]["code"] == "conflict" for b in losers)
assert {b["text"] for b in winners} <= {"вариант 0", "вариант 1"}
assert len(losers) == sum(1 for i in range(20) if f"вариант {i % 2}" != winners[0]["text"])
assert messages_in_db(chat) == 1
def inserts_into_keys() -> int:
return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'idempotency_keys'"))
def test_failure_after_the_key_insert_rolls_everything_back(api, pair):
"""A trigger makes the MESSAGE insert fail AFTER the key row was inserted in the same
transaction. Expected: 500 for the client, no key row and no message left behind
(n_tup_ins proves the key INSERT really ran), and a retry with the same key succeeds."""
alice, bob, chat = pair
marker = "boom-" + uuid.uuid4().hex
key = uuid.uuid4().hex
psql("CREATE OR REPLACE FUNCTION lab3_test_fail() RETURNS trigger LANGUAGE plpgsql AS $$ "
f"BEGIN IF NEW.text = convert_to('{marker}', 'UTF8') THEN RAISE EXCEPTION 'injected failure'; END IF; "
"RETURN NEW; END $$; "
"CREATE TRIGGER lab3_test_fail BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION lab3_test_fail()")
try:
before = inserts_into_keys()
error = api.post_message(alice, chat, {"text": marker}, 500, key=key)
assert error.json()["error"]["code"] == "internal_error" and "injected" not in error.text
assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0
deadline = time.monotonic() + 20 # table statistics are flushed asynchronously
while inserts_into_keys() == before and time.monotonic() < deadline:
time.sleep(0.5)
assert inserts_into_keys() > before, "the key INSERT never ran before the failure"
finally:
psql("DROP TRIGGER IF EXISTS lab3_test_fail ON messages; DROP FUNCTION IF EXISTS lab3_test_fail()")
retried = api.send(alice, chat, marker, key=key) # transient failure was not recorded
assert api.send(alice, chat, marker, key=key) == retried
assert api.history(bob, chat, 100)[0] == [retried] and key_rows(alice, key) == 1
def lock_chat_row(chat: dict, seconds: float) -> subprocess.Popen:
sql = (f"BEGIN; SELECT 1 FROM chats WHERE id = '{chat['id']}' FOR UPDATE; "
f"SELECT pg_sleep({seconds}); COMMIT;")
return subprocess.Popen(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d",
"messenger", "-c", sql], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def wait_lock_taken(chat: dict) -> None:
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
if psql("SELECT count(*) FROM pg_stat_activity WHERE query LIKE '%pg_sleep%' AND state = 'active' "
"AND pid <> pg_backend_pid()") != "0":
return
time.sleep(0.2)
raise AssertionError("the row lock was not taken")
def test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message(api, pair):
"""Uncertain outcome: the client gives up after 0.4 s, the server commits later (the row
lock is held 1 s < lock_timeout). The retry must return THAT message, not a second one."""
alice, bob, chat = pair
key = uuid.uuid4().hex
locker = lock_chat_row(chat, 1.0)
try:
wait_lock_taken(chat)
with pytest.raises(httpx.TimeoutException):
httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "неопределённо"}, timeout=0.4,
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
finally:
locker.wait(timeout=30)
deadline = time.monotonic() + 10
while messages_in_db(chat) == 0 and time.monotonic() < deadline:
time.sleep(0.2)
stored = psql(f"SELECT id FROM messages WHERE chat_id = '{chat['id']}'")
assert stored, "the server-side transaction was expected to commit after the lock was released"
retried = api.send(alice, chat, "неопределённо", key=key)
assert retried["id"] == stored and messages_in_db(chat) == 1 and key_rows(alice, key) == 1
def test_lock_timeout_is_a_transient_503_that_is_not_recorded(api, pair):
"""The row lock outlives lock_timeout: the API answers 503 within its deadline and stores
nothing; after the lock is gone the same key creates the message exactly once."""
alice, bob, chat = pair
key = uuid.uuid4().hex
locker = lock_chat_row(chat, 4.0)
try:
wait_lock_taken(chat)
started = time.monotonic()
api.post_message(alice, chat, {"text": "после блокировки"}, 503, key=key)
assert time.monotonic() - started < 3.6
assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0
finally:
locker.wait(timeout=30)
created = api.send(alice, chat, "после блокировки", key=key)
assert api.send(alice, chat, "после блокировки", key=key) == created
assert api.history(bob, chat, 100)[0] == [created] and key_rows(alice, key) == 1
def test_replay_after_the_serving_replica_is_stopped(api, pair):
alice, bob, chat = pair
key = uuid.uuid4().hex
first = api.post_message(alice, chat, {"text": "до остановки"}, key=key)
served_by = first.headers["x-instance-id"]
replica = next(r for r in running_replicas() if r["hostname"] == served_by)
with replica_stopped(replica):
for _ in range(5):
again = Api().post_message(alice, chat, {"text": "до остановки"}, key=key)
assert again.json() == first.json() and again.headers["x-instance-id"] != served_by
assert messages_in_db(chat) == 1
+217
View File
@@ -0,0 +1,217 @@
"""Track 2: the reconciler of S3 and PostgreSQL (python -m messenger.reconcile).
Faults are injected on BOTH sides, directly into the stores (S3 with the API/reconciler key
pairs from inside the storage network, PostgreSQL with psql):
S3 side: orphan object (no row), object deleted under a ready row, object overwritten with
other bytes (size / same-size checksum), object of a quarantined row restored;
DB side: stale intent with and without object, quarantine mark on a healthy row;
fresh: orphan and intent younger than --grace (an upload in progress).
The tests prove: dry-run changes nothing in either store; --apply fixes exactly the old findings;
fresh and used attachments are never deleted (ready rows only ever get a reversible quarantine);
the second run finds nothing to do; the safety valve and store outages stop the run without
damage. A small --grace (8-30 s, --allow-small-grace) keeps the tests fast."""
from __future__ import annotations
import concurrent.futures
import hashlib
import time
import uuid
from pathlib import Path
import pytest
import imagegen
from conftest import (attachment_row, findings_by_key, psql, reconcile, s3_delete, s3_keys, s3_put, s3_size,
service_down)
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
GRACE = ["--grace", "8", "--allow-small-grace"]
# The main scenario needs "fresh" items to stay younger than --grace for the whole dry-run + snapshot
# (each docker exec costs 1-3 s on Windows), hence a longer grace there.
LONG_GRACE = ["--grace", "30", "--allow-small-grace"]
def new_key() -> str:
return "o/" + uuid.uuid4().hex
def insert_intent(chat: dict, owner: dict, key: str, data: bytes, age_seconds: int) -> str:
"""A 'pending' row as an upload leaves it after the intent COMMIT (crash afterwards)."""
attachment_id = str(uuid.uuid4())
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
"object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'x.png', 'image/png', %d, "
"decode('%s', 'hex'), 320, 200, '%s', 'pending', clock_timestamp() - make_interval(secs => %d), "
"clock_timestamp() - make_interval(secs => %d))"
% (attachment_id, chat["id"], owner["id"], len(data), hashlib.sha256(data).hexdigest(), key, age_seconds,
age_seconds))
return attachment_id
def snapshot() -> tuple[str, frozenset]:
"""Fingerprint of everything the reconciler could change: every attachment row + the key list."""
rows = psql("SELECT count(*) || ':' || coalesce(md5(string_agg(id::text || status || object_key || "
"coalesce(lost_reason, '-') || updated_at::text, ',' ORDER BY id)), '') FROM attachments")
return rows, frozenset(s3_keys())
def content_status(api, actor, attachment_id) -> int:
return api.client.get(f"/api/v1/attachments/{attachment_id}/content",
headers={"Authorization": "Bearer " + actor["token"]}).status_code
def test_detect_and_safely_fix_faults_on_both_sides(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
up = {name: api.upload(alice, chat, imagegen.png(30 + i, 20)).json() for i, name in enumerate("ABCDE")}
data = {name: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
for name, a in up.items()}
key = {name: attachment_row(a["id"])["object_key"] for name, a in up.items()}
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
# ---- inject (old)
orphan_old = new_key()
s3_put(orphan_old, SAMPLE)
stale_with, stale_without = new_key(), new_key()
s3_put(stale_with, SAMPLE)
stale_with_id = insert_intent(chat, alice, stale_with, SAMPLE, 600) # < janitor grace 900 s
stale_without_id = insert_intent(chat, alice, stale_without, SAMPLE, 600)
s3_delete(key["C"]) # ready + used, bytes lost
s3_put(key["D"], data["D"][:len(data["D"]) // 2]) # ready, unused, wrong size
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' "
f"WHERE id = '{up['E']['id']}'") # quarantined, but bytes are fine
time.sleep(32) # everything above is now older than --grace
# ---- inject (fresh: an upload in progress)
fresh_orphan, fresh_intent = new_key(), new_key()
s3_put(fresh_orphan, SAMPLE)
s3_put(fresh_intent, SAMPLE)
fresh_intent_id = insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
# ---- dry-run: report everything, change nothing
before = snapshot()
dry = reconcile(*LONG_GRACE)
assert snapshot() == before, "dry-run modified a store"
f = findings_by_key(dry)
expected = {orphan_old: "orphan_object", fresh_orphan: "fresh_orphan_object", stale_with: "stale_intent",
stale_without: "stale_intent", fresh_intent: "fresh_intent", key["C"]: "missing_object",
key["D"]: "size_mismatch", key["E"]: "restored_object"}
assert {k: f[k]["kind"] for k in expected} == expected
assert key["A"] not in f and key["B"] not in f
assert f[key["C"]]["used"] is True and dry["mode"] == "dry-run"
assert all(f[k]["result"] in ("planned", "not_actionable") for k in expected)
# ---- apply
applied = findings_by_key(reconcile("--apply", *LONG_GRACE))
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"]):
assert applied[k]["result"] == "applied", (k, applied[k])
keys = s3_keys()
assert orphan_old not in keys and stale_with not in keys
assert attachment_row(stale_with_id) is None and attachment_row(stale_without_id) is None
# fresh: untouched
assert fresh_orphan in keys and fresh_intent in keys and attachment_row(fresh_intent_id)["status"] == "pending"
# used, lost bytes: quarantined, NOT deleted; the message still references it
row_c = attachment_row(up["C"]["id"])
assert row_c["status"] == "ready" and row_c["lost_reason"] == "missing_object"
items, _ = api.history(bob, chat, 100)
assert any(up["C"]["id"] in m["attachment_ids"] for m in items)
assert content_status(api, bob, up["C"]["id"]) == 404
api.post_message(alice, chat, {"text": "", "attachment_ids": [up["C"]["id"]]}, 409) # cannot attach a lost file
assert attachment_row(up["D"]["id"])["lost_reason"] == "size_mismatch"
assert attachment_row(up["E"]["id"])["lost_reason"] is None # quarantine lifted
for name in "ABE":
got = api.call("GET", f"/api/v1/attachments/{up[name]['id']}/content", 200, token=bob["token"])
assert got.content == data[name]
# ---- second run: idempotent, nothing left to do for these keys
again = findings_by_key(reconcile("--apply", *LONG_GRACE))
for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"], key["A"], key["B"]):
assert k not in again, (k, again.get(k))
# ---- the lost object comes back from a backup: the quarantine is lifted, the file is served again
s3_put(key["C"], data["C"])
restored = findings_by_key(reconcile("--apply", *LONG_GRACE))
assert restored[key["C"]]["kind"] == "restored_object" and restored[key["C"]]["result"] == "applied"
assert api.call("GET", f"/api/v1/attachments/{up['C']['id']}/content", 200, token=bob["token"]).content == data["C"]
# tidy up the fresh items of this test once they are old enough
time.sleep(31)
reconcile("--apply", *LONG_GRACE)
assert fresh_orphan not in s3_keys() and attachment_row(fresh_intent_id) is None
def test_checksum_mode_finds_same_size_corruption(api):
alice = api.user("alice")
chat = api.chat(alice)
att = api.upload(alice, chat, SAMPLE).json()
k = attachment_row(att["id"])["object_key"]
s3_put(k, SAMPLE[:1000] + bytes([SAMPLE[1000] ^ 1]) + SAMPLE[1001:]) # same size, other bytes
time.sleep(9)
assert k not in findings_by_key(reconcile(*GRACE)) # sizes match: listing cannot see it
report = reconcile("--verify-checksums", *GRACE)
assert findings_by_key(report)[k]["kind"] == "checksum_mismatch"
assert report["cost"]["s3_get_bytes"] >= len(SAMPLE)
assert findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]["result"] == "applied"
assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch"
# A later pass WITHOUT checksums sees "same size" but cannot prove the bytes: the quarantine stays.
assert k not in findings_by_key(reconcile("--apply", *GRACE))
assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch"
# Restoring the original and verifying lifts it.
s3_put(k, SAMPLE)
lifted = findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]
assert (lifted["kind"], lifted["result"]) == ("restored_object", "applied")
assert attachment_row(att["id"])["lost_reason"] is None
def test_safety_valve_refuses_mass_deletion():
"""If (almost) every object looked orphaned - e.g. the reconciler pointed at an empty database -
--apply must stop before deleting anything."""
orphan = new_key()
s3_put(orphan, SAMPLE)
time.sleep(9)
report = reconcile("--apply", "--max-orphan-fraction", "0.000001", *GRACE, expect_code=3)
assert report["error"].startswith("safety_valve") and report["actions"]["applied"] == 0
assert orphan in s3_keys()
reconcile("--apply", *GRACE)
assert orphan not in s3_keys()
@pytest.mark.parametrize("service", ["s3", "postgres"])
def test_store_outage_stops_the_run_without_damage(api, service):
with service_down(service, "stop"):
report = reconcile(*GRACE, expect_code=1)
assert report["error"].startswith("store_unavailable")
assert reconcile(*GRACE)["error"] is None
def test_live_uploads_are_never_hurt_by_concurrent_apply(api):
"""Uploads and messages run while --apply runs repeatedly: every upload that got 201 stays
ready and downloadable byte for byte, every message keeps its attachments."""
alice = api.user("alice")
chat = api.chat(alice)
payloads = [imagegen.png(16 + i, 16) for i in range(40)]
done: list[tuple[dict, bytes]] = []
def worker(chunk: list[bytes]) -> None:
for p in chunk:
att = api.upload(alice, chat, p).json()
api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]})
done.append((att, p))
with concurrent.futures.ThreadPoolExecutor(4) as pool:
futures = [pool.submit(worker, payloads[i::4]) for i in range(4)]
runs = 0
while not all(x.done() for x in futures):
report = reconcile("--apply", *GRACE)
assert report["error"] is None
runs += 1
for x in futures:
x.result()
assert runs >= 1 and len(done) == len(payloads)
for att, p in done:
row = attachment_row(att["id"])
assert row["status"] == "ready" and row["lost_reason"] is None and s3_size(row["object_key"]) == len(p)
for att, p in done[::8]:
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]).content == p
+66
View File
@@ -0,0 +1,66 @@
"""Session lifetime: expiry with a short SESSION_TTL_SECONDS, no extension by restarts,
and the alternative SESSION_STORE=postgres. The api container is recreated with the
override and ALWAYS recreated back with the normal configuration (api_with_env)."""
from __future__ import annotations
import datetime as dt
import time
import pytest
from conftest import api_env, api_with_env, restart_services, service_down
pytestmark = pytest.mark.stack
def seconds_left(expires_at: str) -> float:
moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00"))
return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds()
def login_session(api, actor) -> dict:
from conftest import basic
return api.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": basic(actor["username"], actor["password"])}, schema="Session").json()
@pytest.mark.parametrize("store", ["valkey", "postgres"])
def test_short_ttl_expires(api, store):
with api_with_env(SESSION_TTL_SECONDS="2", SESSION_STORE=store):
assert api_env("SESSION_TTL_SECONDS") == "2" # the contract variable reaches the container
actor = api.register("ttl")
session = login_session(api, actor)
assert 1.0 < seconds_left(session["expires_at"]) <= 2.1
api.call("GET", "/api/v1/users/me", 200, token=session["token"])
time.sleep(max(0.0, seconds_left(session["expires_at"])) + 0.3)
api.call("GET", "/api/v1/users/me", 401, token=session["token"])
api.call("DELETE", "/api/v1/auth/sessions/current", 401, token=session["token"])
def test_restart_does_not_extend_ttl(api):
# 30 s: a restart of Valkey + both API replicas (readiness now also waits for S3) takes ~12-15 s.
with api_with_env(SESSION_TTL_SECONDS="30"):
actor = api.register("ttlrestart")
session = login_session(api, actor)
restart_services("valkey", "api")
assert seconds_left(session["expires_at"]) > 0, "restart took longer than the TTL; increase it"
api.call("GET", "/api/v1/users/me", 200, token=session["token"])
time.sleep(max(0.0, seconds_left(session["expires_at"])) + 0.3)
api.call("GET", "/api/v1/users/me", 401, token=session["token"])
def test_postgres_session_store(api):
"""SESSION_STORE=postgres: sessions survive an API restart, logout revokes, and a DB outage
is 503 for a valid token (not 401 and not anonymous access)."""
with api_with_env(SESSION_STORE="postgres"):
assert api_env("SESSION_STORE") == "postgres"
actor = api.user("pgstore")
restart_services("api")
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
with service_down("postgres", "stop"):
assert api.call("GET", "/api/v1/users/me", 503, token=actor["token"],
lb_error_ok=True).json()["error"]["code"] == "unavailable"
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True)
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"])
api.call("GET", "/api/v1/users/me", 401, token=actor["token"])
+107
View File
@@ -0,0 +1,107 @@
"""S3 outages, private bucket and failover of attachments (lab 4, levels 3 and 4).
Every injected failure is undone in `finally` (conftest.service_down / replica_stopped), and the
stand is waited for until HAProxy sees all replicas UP again."""
from __future__ import annotations
import time
from pathlib import Path
import pytest
import imagegen
from conftest import attachment_row, compose, probe_inside, replica_stopped, running_replicas, service_down
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
@pytest.fixture()
def pair(api):
alice, bob = api.user("alice"), api.user("bob")
return alice, bob, api.chat(alice, bob)
@pytest.mark.parametrize("how", ["stop", "pause"])
def test_s3_outage_is_503_and_text_keeps_working(api, pair, how):
"""S3 down (stopped: connection refused; paused: no answer at all):
* readiness is 503 on every replica (and through the LB), quickly;
* uploads/downloads are 503 `unavailable` within the S3 deadline, never 500;
* the LB's degraded mode keeps chats/text messages working (HAProxy backend api_core);
* after recovery the earlier file is intact."""
alice, bob, chat = pair
att = api.upload(alice, chat, SAMPLE).json()
with service_down("s3", how):
deadline = time.monotonic() + 15
while api.client.get("/health/ready").status_code != 503:
assert time.monotonic() < deadline, "readiness did not turn 503"
time.sleep(0.5)
for replica in running_replicas():
assert probe_inside(replica, "/health/ready") == 503
assert probe_inside(replica, "/health/ready/core") == 200
started = time.monotonic()
error = api.upload(alice, chat, SAMPLE, 503).json()
assert error["error"]["code"] == "unavailable"
assert time.monotonic() - started < 8
started = time.monotonic()
api.call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"])
assert time.monotonic() - started < 8
api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]) # metadata: PostgreSQL only
api.send(alice, chat, "текст работает без S3") # degraded mode
got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
assert got.content == SAMPLE
api.upload(alice, chat, SAMPLE)
def test_upload_survives_failover_of_the_serving_replica(api, pair):
"""Upload -> stop the replica that served it -> another replica downloads the same bytes
(nothing of the file lives in the replica)."""
alice, bob, chat = pair
data = imagegen.png_of_size(500_000)
response = api.upload(alice, chat, data)
att, served_by = response.json(), response.headers["x-instance-id"]
replica = next(r for r in running_replicas() if r["hostname"] == served_by)
with replica_stopped(replica, "kill"):
for _ in range(5):
got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
assert got.headers["x-instance-id"] != served_by and got.content == data
def test_bucket_is_private_outside_the_api(api, pair):
"""Inside the storage network (where the API is): an UNSIGNED GET of a real object and an
unsigned LIST are refused; the same GET signed with the API key pair works (control).
SeaweedFS admin endpoints (master 9333, filer 8888, volume 8080) are not reachable."""
alice, _, chat = pair
att = api.upload(alice, chat, SAMPLE).json()
key = attachment_row(att["id"])["object_key"]
probe = (
"import urllib.request as u, urllib.error as e, socket\n"
"def st(url):\n"
" try: return u.urlopen(url, timeout=3).status\n"
" except e.HTTPError as x: return x.code\n"
f"print(st('http://s3:8333/messenger-attachments/{key}'), st('http://s3:8333/messenger-attachments?list-type=2'))\n"
"for port in (9333, 8888, 8080):\n"
" s = socket.socket(); s.settimeout(2)\n"
" print(port, s.connect_ex(('s3', port)))\n")
out = compose("exec", "-T", "api", "python", "-c", probe).stdout.split("\n")
get_status, list_status = out[0].split()
assert get_status in ("401", "403") and list_status in ("401", "403"), out[0]
for line in out[1:4]:
port, code = line.split()
assert code != "0", f"admin port {port} reachable"
signed = compose("exec", "-T", "api", "python", "-c",
"import asyncio\nfrom messenger.config import Settings\nfrom messenger.storage import Storage\n"
"async def m():\n s=Storage(Settings.from_env()); await s.open()\n"
f" print((await s.head('{key}')).size); await s.close()\nasyncio.run(m())").stdout.strip()
assert signed == str(len(SAMPLE))
def test_s3_restart_keeps_objects(api, pair):
alice, bob, chat = pair
data = imagegen.png_of_size(200_000)
att = api.upload(alice, chat, data).json()
with service_down("s3", "stop"):
pass
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]).content == data
+148
View File
@@ -0,0 +1,148 @@
"""Unit tests of the image checks (src/messenger/images.py) and the filename rules. No services.
The JPEG decoder (djpeg) exists in the API image; locally these tests are skipped without it
(the same cases run against the stand in test_attachments.py)."""
from __future__ import annotations
import asyncio
import shutil
import sys
import time
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "src"))
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
from messenger import images, validation # noqa: E402
from messenger.errors import InvalidRequest # noqa: E402
LIMIT = 40_000_000
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
has_djpeg = pytest.mark.skipif(shutil.which("djpeg") is None, reason="djpeg (libjpeg-turbo-progs) not installed")
def status(data: bytes) -> int:
"""201 if accepted, else the HTTP status of the ImageError (413/415)."""
try:
info = images.sniff(data, LIMIT)
if info is None:
return 415 # header never completed
asyncio.run(images.verify(data, info))
return 201
except images.ImageError as exc:
return exc.status
def test_public_fixture_is_accepted():
assert images.sniff(SAMPLE, LIMIT) == images.ImageInfo("image/png", 320, 200)
assert status(SAMPLE) == 201
@pytest.mark.parametrize("data", [
imagegen.png(37, 23, "RGBA"), imagegen.png(33, 17, "P"), imagegen.png(33, 17, "1"),
imagegen.png(33, 17, "L"), imagegen.png(81, 57, "RGB", noise=True), imagegen.png(1, 1),
], ids=["rgba", "palette", "1bit", "gray", "noise", "1x1"])
def test_valid_png_kinds(data):
assert status(data) == 201
def test_interlaced_png_is_accepted():
# Pillow cannot write Adam7; build one by hand: 9 x 9 grey, every pass with filter 0.
from messenger.images import _png_segments
raw = b"".join(b"\x00" + bytes(stride - 1) for size, stride in _png_segments(9, 9, 8, 0, 1)
for _ in range(size // stride))
data = imagegen.PNG_SIGNATURE + imagegen.ihdr(9, 9, 8, 0, 1) + imagegen.chunk(b"IDAT", __import__("zlib").compress(raw)) \
+ imagegen.chunk(b"IEND", b"")
assert status(data) == 201
BROKEN_PNG = {
"first 33 bytes of the fixture": SAMPLE[:33],
"truncated (no IEND)": SAMPLE[:-12],
"truncated in IDAT": SAMPLE[:len(SAMPLE) // 2],
"flipped byte -> CRC error": SAMPLE[:500] + bytes([SAMPLE[500] ^ 0xFF]) + SAMPLE[501:],
"garbage IDAT with valid CRC": imagegen.png_corrupt_idat(),
"fewer rows than IHDR": imagegen.png_short_pixels(),
"bytes after IEND (polyglot)": SAMPLE + b"PK\x03\x04zip",
"text": b"just text, definitely not an image",
"gif": b"GIF89a" + bytes(64),
"two bytes of a signature": b"\x89P",
}
@pytest.mark.parametrize("name", list(BROKEN_PNG))
def test_broken_png_is_415(name):
assert status(BROKEN_PNG[name]) == 415, name
def test_pixel_limit_is_checked_from_the_header():
with pytest.raises(images.ImageError) as exc:
images.sniff(imagegen.png_bomb_header()[:33], LIMIT) # 33 bytes are enough
assert exc.value.status == 413
exact = imagegen.PNG_SIGNATURE + imagegen.ihdr(8000, 5000) # exactly 40 000 000: allowed
assert images.sniff(exact, LIMIT).width == 8000
with pytest.raises(images.ImageError) as exc:
images.sniff(imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), LIMIT)
assert exc.value.status == 413
with pytest.raises(images.ImageError) as exc:
images.sniff(imagegen.jpeg_bomb_header(), LIMIT)
assert exc.value.status == 413
def test_png_verification_is_streaming_and_fast():
"""40 Mpx RGB = 120 MB of raw pixels: inflated in 1 MiB steps and discarded."""
big = imagegen.png(8000, 5000, compress_level=9)
started = time.perf_counter()
assert status(big) == 201
assert time.perf_counter() - started < 5
def test_jpeg_header_is_parsed_incrementally():
data = imagegen.jpeg(320, 200)
assert images.sniff(data[:4], LIMIT) is None # SOF not arrived yet
assert images.sniff(data, LIMIT) == images.ImageInfo("image/jpeg", 320, 200)
@has_djpeg
@pytest.mark.parametrize("data", [imagegen.jpeg(), imagegen.jpeg(640, 480, progressive=True)], ids=["base", "prog"])
def test_valid_jpeg(data):
assert status(data) == 201
@has_djpeg
@pytest.mark.parametrize("name,data", [
("no scan data after SOS", imagegen.jpeg_without_scan_data()),
("truncated", imagegen.jpeg(320, 240, noise=True)[:3000]),
], ids=lambda v: v if isinstance(v, str) else "")
def test_broken_jpeg_is_415(name, data):
assert status(data) == 415, name
# ------------------------------------------------------------------ filename
@pytest.mark.parametrize("name", ["photo.png", "фото отпуска.jpg", "a" * 255, "emoji 👨‍👩‍👧.png", " x ", "..."[:2] + "x"])
def test_good_filenames(name):
assert validation.filename_param([("filename", name)]) == name
@pytest.mark.parametrize("name", ["", " ", " ", ".", "..", "a/b.png", "a\\b.png", "a\x00b", "c1\x85.png",
"tab\t.png", "del\x7f.png", "x
y", "photo‮gnp.exe", "a" * 256])
def test_bad_filenames(name):
with pytest.raises(InvalidRequest):
validation.filename_param([("filename", name)])
def test_filename_must_be_single_and_present():
with pytest.raises(InvalidRequest):
validation.filename_param([])
with pytest.raises(InvalidRequest):
validation.filename_param([("filename", "a.png"), ("filename", "b.png")])
def test_query_must_be_valid_utf8():
assert validation.query_items(b"filename=%D1%84.png") == [("filename", "ф.png")]
with pytest.raises(InvalidRequest):
validation.query_items(b"filename=%FF%FE.png")
+225
View File
@@ -0,0 +1,225 @@
"""Pure functions, no services: validation, cursor, Authorization parsing, Argon2id, config."""
from __future__ import annotations
import asyncio
import base64
import datetime as dt
import pytest
from messenger import authhdr, cursor, passwords, validation
from messenger.config import ConfigError, Settings
from messenger.errors import InvalidRequest, Unauthorized
from messenger.timefmt import UTC, from_us, rfc3339, to_us
def b64(raw: bytes) -> str:
return base64.b64encode(raw).decode()
# ------------------------------------------------------------------ password rules
@pytest.mark.parametrize("password", ["Twelve-chars", "я" * 128, "пароль:с:двоеточиями", "😀" * 12, "a b c d e f g"])
def test_valid_passwords(password):
assert validation.password_field({"password": password}) == password
@pytest.mark.parametrize("password", ["short-11ch", "я" * 129, " " * 12, "\t\n " * 5, "\ud800" * 12, None, 123456789012])
def test_invalid_passwords(password):
with pytest.raises(InvalidRequest):
validation.password_field({"password": password})
def test_text_may_contain_nul_but_not_lone_surrogates():
assert validation.text_field({"t": "a\u0000b"}, "t", 10) == "a\u0000b"
with pytest.raises(InvalidRequest):
validation.text_field({"t": "a\ud800"}, "t", 10)
# ------------------------------------------------------------------ cursor
CHAT = "3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11"
def test_cursor_roundtrip_and_binding():
key = (to_us(dt.datetime(2026, 10, 12, 12, 0, 0, 123456, tzinfo=UTC)), "11111111-1111-4111-8111-111111111111")
token = cursor.encode(cursor.MESSAGES, CHAT, key)
assert cursor.decode(token, cursor.MESSAGES, CHAT) == key
with pytest.raises(InvalidRequest):
cursor.decode(token, cursor.MESSAGES, "00000000-0000-4000-8000-000000000000") # other chat
with pytest.raises(InvalidRequest):
cursor.decode(token, cursor.CHATS, CHAT) # other resource
@pytest.mark.parametrize("payload", [
b"v1|m|" + CHAT.encode() + b"|253402300800000000|11111111-1111-4111-8111-111111111111", # 1 us after 9999-12-31
b"v1|m|" + CHAT.encode() + b"|9999999999999999999|11111111-1111-4111-8111-111111111111", # int64 edge
b"v1|m|" + CHAT.encode() + b"|-1|11111111-1111-4111-8111-111111111111",
b"v1|m|" + CHAT.encode() + b"|1|not-a-uuid",
b"v1|m|" + CHAT.encode() + b"|1|11111111-1111-4111-8111-11111111111G",
b"v2|m|" + CHAT.encode() + b"|1|11111111-1111-4111-8111-111111111111",
])
def test_broken_cursors_are_400_not_500(payload):
token = base64.urlsafe_b64encode(payload).rstrip(b"=").decode()
with pytest.raises(InvalidRequest):
cursor.decode(token, cursor.MESSAGES, CHAT)
def test_time_helpers_are_exact():
moment = dt.datetime(2026, 2, 28, 23, 59, 59, 999999, tzinfo=UTC)
assert from_us(to_us(moment)) == moment
assert rfc3339(moment) == "2026-02-28T23:59:59.999999Z"
assert from_us(cursor.MAX_TS_US).year == 9999
# ------------------------------------------------------------------ Authorization header
def test_basic_is_parsed_and_password_may_contain_colons():
assert authhdr.parse_basic("Basic " + b64("alice:pa:ss:wörd".encode())) == ("alice", "pa:ss:wörd")
assert authhdr.parse_basic("basic " + b64(b"bob_1:x" * 1)) == ("bob_1", "x")
@pytest.mark.parametrize("header", [
None, "", "Basic", "Basic é", "Basic !!!!", "Basic " + b64(b"no-colon"), "Basic " + b64(b"\xff\xfe:pw"),
"Basic " + b64(b":password"), "Basic " + b64(b"alice:"), "Basic " + b64("Алиса:pw".encode()),
"Bearer " + "a" * 43, "Basic " + "A" * 2000,
])
def test_bad_basic_is_401_with_basic_challenge(header):
with pytest.raises(Unauthorized) as info:
authhdr.parse_basic(header)
assert info.value.headers["WWW-Authenticate"].startswith("Basic realm=\"messenger\"")
@pytest.mark.parametrize("header", [None, "", "Bearer", "Bearer short", "Bearer " + "a" * 43 + "!",
"Basic " + b64(b"alice:password-123"), "Token " + "a" * 43, "Bearer " + "é" * 43])
def test_bad_bearer_is_401_with_bearer_challenge(header):
with pytest.raises(Unauthorized) as info:
authhdr.parse_bearer(header)
assert info.value.headers["WWW-Authenticate"] == 'Bearer realm="messenger"'
def test_bearer_ok():
token = "A_b-" + "x" * 39
assert authhdr.parse_bearer("Bearer " + token) == token
# ------------------------------------------------------------------ Argon2id
def test_argon2id_hash_and_verify():
async def scenario():
stored = await passwords.hash_password("я" * 128)
assert stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$")
assert await passwords.verify_password(stored, "я" * 128)
assert not await passwords.verify_password(stored, "я" * 127)
assert not await passwords.verify_password(None, "anything") # unknown user, dummy hash
assert not await passwords.verify_password("not-a-hash", "anything")
assert await passwords.hash_password("same password!") != await passwords.hash_password("same password!") # salt
asyncio.run(scenario())
# ------------------------------------------------------------------ config
@pytest.fixture(autouse=True)
def s3_credentials(monkeypatch):
"""Lab 4: Settings require the API's S3 key pair; synthetic values for the unit tests."""
monkeypatch.setenv("S3_ACCESS_KEY_ID", "AKTEST")
monkeypatch.setenv("S3_SECRET_ACCESS_KEY", "secret-for-unit-tests")
def test_s3_settings_are_validated_and_contract_limits_cannot_be_raised(monkeypatch):
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
settings = Settings.from_env()
assert (settings.max_upload_bytes, settings.max_image_pixels) == (10 * 1024 * 1024, 40_000_000)
assert settings.fault_points == frozenset() and settings.upload_concurrency == 4
for name, value in (("MAX_UPLOAD_BYTES", "10485761"), ("MAX_IMAGE_PIXELS", "40000001"),
("FAULT_POINTS", "drop_database"), ("S3_ENDPOINT", "ftp://s3"), ("S3_BUCKET", "Bad_Bucket")):
monkeypatch.setenv(name, value)
with pytest.raises(ConfigError):
Settings.from_env()
monkeypatch.delenv(name)
monkeypatch.delenv("S3_SECRET_ACCESS_KEY")
with pytest.raises(ConfigError):
Settings.from_env()
def test_settings_read_secret_files_and_contract_names(tmp_path, monkeypatch):
url_file = tmp_path / "database_url"
url_file.write_text("postgresql://u:p@db/x\n", encoding="utf-8")
monkeypatch.setenv("DATABASE_URL_FILE", str(url_file))
monkeypatch.setenv("SESSION_STORE", "redis")
monkeypatch.setenv("CACHE_URL", "redis://:p@valkey:6379/0")
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
settings = Settings.from_env()
assert settings.database_url == "postgresql://u:p@db/x"
assert settings.session_store == "valkey" and settings.session_ttl_seconds == 2
monkeypatch.setenv("SESSION_TTL_SECONDS", "0")
with pytest.raises(ConfigError):
Settings.from_env()
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
monkeypatch.setenv("DATABASE_URL_FILE", str(tmp_path / "missing"))
with pytest.raises(ConfigError) as info:
Settings.from_env()
assert "u:p" not in str(info.value)
# ------------------------------------------------------------------ lab 3: idempotency, JSON, instance id
@pytest.mark.parametrize("key", ["a", "k" * 128, "A-Za-z0-9._:-", "2026-10-16T10:00:00.1"])
def test_valid_idempotency_keys(key):
assert validation.idempotency_key(key) == key
@pytest.mark.parametrize("key", [None, "", "k" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "a\x00b", "*", "é"])
def test_invalid_idempotency_keys(key):
with pytest.raises(InvalidRequest):
validation.idempotency_key(key)
@pytest.mark.parametrize("raw", [b'{"text":"a","text":"b"}', b'{"a":{"x":1,"x":2}}', b'\xff', b'{"text":"x"} {}'])
def test_duplicate_keys_and_garbage_json_are_rejected(raw):
with pytest.raises(InvalidRequest):
validation.parse_json_object(raw)
def test_attachment_ids_absent_equals_empty_and_is_validated():
assert validation.attachment_ids_field({"text": "x"}) == []
assert validation.attachment_ids_field({"attachment_ids": []}) == []
one = "11111111-1111-4111-8111-111111111111"
assert validation.attachment_ids_field({"attachment_ids": [one.upper()]}) == [one]
for bad in (None, "x", [1], ["nope"], [one, one.upper()], [one] * 11):
with pytest.raises(InvalidRequest):
validation.attachment_ids_field({"attachment_ids": bad})
def test_request_fingerprint_ignores_key_order_and_default_attachments():
from messenger.app import request_fingerprint
assert request_fingerprint("привет", []) == request_fingerprint("привет", [])
assert request_fingerprint("привет", []) != request_fingerprint("привет ", [])
assert request_fingerprint("a\u0000b", []) != request_fingerprint("ab", [])
one = "11111111-1111-4111-8111-111111111111"
assert request_fingerprint("x", [one]) != request_fingerprint("x", [])
def test_instance_id_defaults_to_hostname_and_is_validated(monkeypatch):
import socket
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
monkeypatch.delenv("INSTANCE_ID", raising=False)
assert Settings.from_env().instance_id == socket.gethostname()
monkeypatch.setenv("INSTANCE_ID", "replica-a")
assert Settings.from_env().instance_id == "replica-a"
for bad in ("with space", "x" * 65, "ip/1"):
monkeypatch.setenv("INSTANCE_ID", bad)
with pytest.raises(ConfigError):
Settings.from_env()
def test_idempotency_ttl_cannot_be_below_24_hours(monkeypatch):
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
monkeypatch.setenv("IDEMPOTENCY_TTL_SECONDS", "3600")
with pytest.raises(ConfigError):
Settings.from_env()
def test_message_write_path_setting(monkeypatch):
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
monkeypatch.delenv("MESSAGE_WRITE_PATH", raising=False)
assert Settings.from_env().message_write_path == "function"
monkeypatch.setenv("MESSAGE_WRITE_PATH", "multi")
assert Settings.from_env().message_write_path == "multi"
monkeypatch.setenv("MESSAGE_WRITE_PATH", "fast")
with pytest.raises(ConfigError):
Settings.from_env()