test(pipeline): add repeatable crash duplicate and stale version matrix

This commit is contained in:
2026-11-15 22:25:22 +03:00
parent 5be41d8646
commit d3e156e8a1
12 changed files with 2077 additions and 4 deletions
+5 -4
View File
@@ -32,13 +32,14 @@ select = ["E", "F", "I", "B", "UP"]
"research/*.py" = ["E501"]
"scripts/*.py" = ["E501"]
[tool.pytest.ini_options]
testpaths = ["student-tests"]
addopts = "-ra"
# Keep SQL statements together; the formatter controls executable statements.
"src/attachments.py" = ["E501"]
"src/pipeline.py" = ["E501"]
"src/worker.py" = ["E501"]
"src/operator.py" = ["E501"]
"src/reconcile.py" = ["E501"]
[tool.pytest.ini_options]
testpaths = ["student-tests"]
addopts = "-ra"
+215
View File
@@ -0,0 +1,215 @@
import argparse
import atexit
import json
import os
import socket
import subprocess
import tempfile
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
PROGRESS = {}
def save_incomplete():
receipt = PROGRESS.get("receipt")
if receipt and receipt.get("result") != "PASS":
target = PROGRESS["output"].with_name(
PROGRESS["output"].stem + "-failed.json"
)
if target.exists():
target = target.with_name(
target.stem + "-" + uuid4().hex[:8] + target.suffix
)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(
json.dumps(
{**receipt, "result": "INCOMPLETE"},
ensure_ascii=False,
indent=2,
)
+ "\n"
)
atexit.register(save_incomplete)
def run(args, *, cwd, env, expected=0):
result = subprocess.run(
args, cwd=cwd, env=env, capture_output=True, text=True
)
def redact(text):
return (
text.replace(str(cwd), "<clean-clone>")
.replace(str(ROOT), "<repository>")
.replace(str(Path.home()), "<home>")
)
if result.returncode != expected:
if "receipt" in PROGRESS:
PROGRESS["receipt"]["failed_action"] = redact(" ".join(args))
raise RuntimeError(redact(result.stdout + result.stderr))
return redact(result.stdout.strip())
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/clean-clone.json")
args = parser.parse_args()
output = ROOT / args.output
if output.exists():
raise SystemExit("Choose new output path")
assert not subprocess.check_output(
["git", "status", "--porcelain"], cwd=ROOT
)
sha = subprocess.check_output(
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
).strip()
identity = uuid4().hex[:10]
project = "mtusi-lab5-clean-" + identity
with socket.socket() as reservation:
reservation.bind(("127.0.0.1", 0))
public_port = str(reservation.getsockname()[1])
env = {
**os.environ,
"COMPOSE_PROJECT_NAME": project,
"API_IMAGE": "mtusi-messenger:lab5-clean-" + identity,
"PUBLIC_PORT": public_port,
"VCS_REF": sha,
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
receipt = {
"date": datetime.now(UTC).isoformat(),
"source_commit": sha,
"fresh_secrets_environment_and_volumes": True,
"command": "python scripts/verify_clean.py",
"fixed_host_port_for_restart": int(public_port),
}
PROGRESS.update(receipt=receipt, output=output)
with tempfile.TemporaryDirectory(prefix="mtusi-lab5-clean-") as temporary:
clone = Path(temporary) / "lab5"
run(
[
"git",
"clone",
"--no-hardlinks",
"--branch",
"solution/bvt2301-maslov",
str(ROOT),
str(clone),
],
cwd=ROOT,
env=env,
)
for name in [".env", ".venv", "secrets"]:
assert not (clone / name).exists()
run(["uv", "sync", "--locked"], cwd=clone, env=env)
receipt["static_checks"] = run(
["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"],
cwd=clone,
env=env,
)
run(["make", "bootstrap"], cwd=clone, env=env)
try:
run(
["docker", "compose", "up", "-d", "--build", "--wait"],
cwd=clone,
env=env,
)
env["BASE_URL"] = "http://127.0.0.1:" + public_port
receipt["smoke"] = run(
["make", "test", "BASE_URL=" + env["BASE_URL"]],
cwd=clone,
env=env,
)
receipt["http_tests"] = run(
[".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env
)
run(
[
".venv/bin/python",
"-m",
"scripts.verify_cluster",
"--output",
"evidence/clean-cluster.json",
],
cwd=clone,
env=env,
)
run(
[
".venv/bin/python",
"-m",
"scripts.verify_storage",
"--output",
"evidence/clean-storage.json",
],
cwd=clone,
env=env,
)
receipt["storage"] = json.loads(
(clone / "evidence/clean-storage.json").read_text()
)
receipt["cluster"] = json.loads(
(clone / "evidence/clean-cluster.json").read_text()
)
import hashlib
expected = {
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in (clone / "src").glob("*.py")
}
ids = run(
["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env
).split()
assert len(ids) == 2
code = (
"import hashlib,json,pathlib; print(json.dumps({p.name:"
"hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))"
)
for cid in ids:
actual = json.loads(
run(
["docker", "exec", cid, "python", "-c", code],
cwd=clone,
env=env,
)
)
assert actual == expected
info = json.loads(
run(["docker", "inspect", cid], cwd=clone, env=env)
)[0]
assert (
info["Config"]["Labels"][
"org.opencontainers.image.revision"
]
== sha
)
assert not info["HostConfig"]["PortBindings"]
receipt["both_replicas_runtime_hashes_and_revision_match"] = True
receipt["fsck"] = run(
["git", "fsck", "--full"], cwd=clone, env=env
)
receipt["checks_completed"] = True
finally:
assert project.startswith("mtusi-lab5-clean-")
run(
["docker", "compose", "down", "--volumes", "--remove-orphans"],
cwd=clone,
env=env,
)
receipt["result"] = "PASS"
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
print(
"PASS: fresh clone, build, HTTP tests, failover, rolling, persistence and runtime hashes"
)
if __name__ == "__main__":
main()
+342
View File
@@ -0,0 +1,342 @@
"""Проверки реальных процессов, failover, rolling и общей идемпотентности."""
import argparse
import json
import os
import subprocess
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
import httpx
from scripts import cluster
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
def snapshot():
ids = compose("ps", "-q").split()
result = []
for c in json.loads(command(["docker", "inspect", *ids])):
role = c["Config"]["Labels"]["com.docker.compose.service"]
entry = {
"role": role,
"container": c["Id"][:12],
"pid": c["State"]["Pid"],
"started_at": c["State"]["StartedAt"],
"ports": c["HostConfig"]["PortBindings"],
"cpu_nano": c["HostConfig"]["NanoCpus"],
"memory_limit": c["HostConfig"]["Memory"],
}
if role == "api":
entry["instance"] = cluster.alias(c["Id"])
assert not entry["ports"]
result.append(entry)
return result
def exercise(client, session, chat, action, *, seconds=15):
path = f"/api/v1/chats/{chat['id']}/messages"
observations, confirmed, errors, reads = [], {}, 0, 0
started = time.monotonic()
finished = threading.Event()
action_result = {}
def operate():
time.sleep(0.5)
try:
action()
except BaseException as exc:
action_result["error"] = type(exc).__name__
finally:
action_result["finished"] = round(time.monotonic() - started, 3)
finished.set()
thread = threading.Thread(target=operate)
thread.start()
key = uuid4().hex
try:
while time.monotonic() - started < seconds or not finished.is_set():
h = {**auth(session), "Idempotency-Key": key}
try:
r = client.post(path, headers=h, json={"text": key})
if r.status_code != 201:
errors += 1
continue
m = r.json()
confirmed[m["id"]] = m
observations.append(
{
"at": round(time.monotonic() - started, 3),
"instance": r.headers["x-instance-id"],
}
)
key = uuid4().hex
read = client.get(
path, headers=auth(session), params={"limit": 100}
)
reads += read.status_code == 200
errors += read.status_code != 200
except httpx.HTTPError:
errors += 1
time.sleep(0.04)
finally:
thread.join(timeout=90)
assert not action_result.get("error"), action_result
found, cursor = [], None
while True:
r = client.get(
path,
headers=auth(session),
params={"limit": 100, **({"cursor": cursor} if cursor else {})},
)
assert r.status_code == 200
page = r.json()
found.extend(page["items"])
cursor = page["next_cursor"]
if not cursor:
break
by_id = {m["id"]: m for m in found}
assert all(by_id.get(k) == v for k, v in confirmed.items())
texts = [m["text"] for m in found]
assert len(texts) == len(set(texts))
recovery = next(
(
x["at"]
for x in observations
if x["at"] >= action_result["finished"]
),
None,
)
assert recovery is not None and (seconds > 20 or recovery <= 10)
elapsed = time.monotonic() - started
return {
"confirmed_writes": len(confirmed),
"successful_reads": reads,
"errors": errors,
"successful_rps": round((len(confirmed) + reads) / elapsed, 2),
"first_success_after_action_seconds": recovery,
"action": action_result,
"observations": observations,
"confirmed_preserved": True,
"duplicate_texts": False,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/cluster-repeat.json")
args = parser.parse_args()
report = {
"date": datetime.now(timezone.utc).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"complete": False,
}
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
try:
ready()
initial = snapshot()
apis = [c for c in initial if c["role"] == "api"]
assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2
report["before"] = initial
with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client:
user, _, session = actor(client)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={
"title": "Cluster verification",
"member_ids": [user["id"]],
},
)
assert r.status_code == 201
chat = r.json()
path = f"/api/v1/chats/{chat['id']}/messages"
key = uuid4().hex
def repeat(_):
with httpx.Client(
base_url=BASE, trust_env=False, timeout=8
) as c:
r = c.post(
path,
headers={**auth(session), "Idempotency-Key": key},
json={"text": "Same concurrent request"},
)
assert r.status_code == 201
return r.json(), r.headers["x-instance-id"]
with ThreadPoolExecutor(max_workers=10) as pool:
repeated = list(pool.map(repeat, range(30)))
assert len({m[0]["id"] for m in repeated}) == 1
assert len({m[1] for m in repeated}) == 2
report["parallel_replay"] = {
"requests": 30,
"messages": 1,
"instances": sorted({m[1] for m in repeated}),
}
# Сбой второй записи транзакции не должен оставить сообщение.
fault_key = uuid4().hex
fault_text = "Atomic rollback " + fault_key
sql(
"CREATE FUNCTION student_idempo_fault() RETURNS trigger "
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='"
+ fault_key
+ "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; "
"CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency "
"FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()"
)
try:
failed = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": fault_text},
)
assert failed.status_code == 500
assert (
sql(
"SELECT count(*) FROM messages WHERE text='"
+ fault_text
+ "'"
)
== "0"
)
finally:
sql(
"DROP TRIGGER student_idempo_fault ON idempotency; "
"DROP FUNCTION student_idempo_fault()"
)
retried = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": fault_text},
)
assert retried.status_code == 201
report["atomic_rollback_and_5xx_retry"] = True
sql(
"UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' "
"WHERE key='" + fault_key + "'"
)
expired = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": "New retention window"},
)
assert expired.status_code == 201
assert expired.json()["id"] != retried.json()["id"]
report["expired_key_can_be_reused"] = True
# Отдельный чат каждой серии: проверки дубликатов не смешивают тексты.
report["failovers"] = []
for api in apis:
def stop(target=api["instance"]):
env = {**os.environ, "TARGET_INSTANCE_ID": target}
subprocess.run(
["scripts/stop-one.sh"],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={"title": "Failover", "member_ids": [user["id"]]},
)
result = exercise(client, session, r.json(), stop)
assert api["instance"] not in [
o["instance"] for o in result["observations"][-5:]
]
assert (
command(
[
"docker",
"inspect",
"-f",
"{{.State.Running}}",
api["container"],
]
)
== "false"
)
result["stopped"] = api
report["failovers"].append(result)
compose("up", "-d", "--no-build", "--wait")
ready()
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={"title": "Rolling", "member_ids": [user["id"]]},
)
before_roll = snapshot()
report["rolling"] = exercise(
client,
session,
r.json(),
lambda: command(["scripts/rolling-update.sh"]),
seconds=45,
)
after_roll = snapshot()
assert not (
{c["container"] for c in before_roll if c["role"] == "api"}
& {c["container"] for c in after_roll if c["role"] == "api"}
)
report["rolling"]["before"] = before_roll
report["rolling"]["after"] = after_roll
replay = repeat(0)[0]
assert replay == repeated[0][0]
report["replay_after_replacement"] = True
before = snapshot()
env = {
**os.environ,
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
for mode, hook in [
("failover", "scripts/stop-one.sh"),
("persistence", "scripts/restart.sh"),
]:
if mode == "persistence":
compose("up", "-d", "--no-build", "--wait")
ready()
before = snapshot()
run = subprocess.run(
[
".venv/bin/python",
"tests/resilience.py",
"--mode",
mode,
"--action-script",
hook,
"--base-url",
BASE,
],
cwd=ROOT,
env=env,
capture_output=True,
text=True,
)
assert run.returncode == 0, run.stdout + run.stderr
report["public_" + mode] = run.stdout.strip()
after = snapshot()
for role in ["postgres", "cache", "s3", "broker", "lb"]:
old = next(c for c in before if c["role"] == role)
new = next(c for c in after if c["role"] == role)
assert old["started_at"] != new["started_at"]
report["after_persistence"] = after
report["complete"] = True
finally:
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS: process failover, rolling, replay and full persistence")
if __name__ == "__main__":
main()
+526
View File
@@ -0,0 +1,526 @@
"""Repeatable fault injection for grade 5, track 2; synthetic data only."""
import argparse
import hashlib
import io
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from uuid import uuid4
import httpx
from PIL import Image
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1")
os.environ.setdefault("no_proxy", "localhost,127.0.0.1")
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
PREFIX = """import io,json,hashlib,time,urllib.request
from uuid import UUID,uuid4
from psycopg.types.json import Jsonb
from src.pipeline import database,broker,envelope,STREAM,GROUP
from src.object_store import ObjectStore,BUCKET
from src.operator import change
from src.reconcile import reconcile
db,store,queue=database(),ObjectStore(),broker()
"""
def inside(code):
cid = compose("ps", "-q", "api").split()[0]
return json.loads(
command(
["docker", "exec", "-i", cid, "python", "-"], input=PREFIX + code
)
)
def wait(fn, timeout=90):
end = time.monotonic() + timeout
while time.monotonic() < end:
value = fn()
if value:
return value
time.sleep(0.15)
raise TimeoutError("Fault scenario condition not reached")
def stop(role):
compose("stop", "--timeout", "1", role)
def start(role):
compose("up", "-d", "--no-deps", "--no-recreate", role)
def row(job):
return inside(
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\nprint(json.dumps(r,default=str))"
)
def inject(job, phase, action, remaining=1):
return inside(
f"with db.connection() as c: r=c.execute('INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES(%s,%s,%s,%s) RETURNING id',({job!r},{phase!r},{action!r},{remaining})).fetchone()\nprint(json.dumps(r['id']))"
)
def hit(identity):
return inside(
f"with db.connection() as c: r=c.execute('SELECT hit_at IS NOT NULL hit FROM pipeline_faults WHERE id=%s',({identity},)).fetchone()\nprint(json.dumps(r['hit']))"
)
def operator(job, version=None):
return inside(
f"print(json.dumps(change(db,store,job_id=UUID({job!r}),operator='synthetic-verifier',reason='Explicit fault matrix replay',version={version!r})))"
)
def duplicate(job, count=5):
return inside(
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\ne=envelope(r)\nids=[queue.xadd(STREAM,{{'job':json.dumps(e)}}) for _ in range({count})]\nprint(json.dumps(ids))"
)
def inspect_effect(job):
return inside(f"""with db.connection() as c:
j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()
v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall()
p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n']
e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall()
keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")]
print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e}},default=str))""")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/pipeline-repeat.json")
parser.add_argument("--repeats", type=int, default=2)
args = parser.parse_args()
output = ROOT / args.output
if output.exists():
raise ValueError("Choose a new output path")
report = {
"source_commit": command(["git", "rev-parse", "HEAD"]),
"date": datetime.now(timezone.utc).isoformat(),
"track": 2,
"complete": False,
"cases": [],
}
output.parent.mkdir(parents=True, exist_ok=True)
def save(name, details):
report["cases"].append({"case": name, **details})
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS:", name, flush=True)
try:
ready()
# Check instrumentation is really enabled and production default is off.
assert inside(
"import os\nprint(json.dumps(os.getenv('PIPELINE_FAULTS')=='1'))"
)
stop("reconciler")
with httpx.Client(
base_url=BASE, trust_env=False, timeout=25
) as client:
user, _, session = actor(client)
h = auth(session)
c = client.post(
"/api/v1/chats",
headers=h,
json={
"title": "Pipeline synthetic faults",
"member_ids": [user["id"]],
},
)
assert c.status_code == 201
chat = c.json()["id"]
raw_io = io.BytesIO()
Image.new("RGBA", (320, 200), (20, 50, 170, 120)).save(
raw_io, format="PNG"
)
raw = raw_io.getvalue()
def upload():
r = client.post(
f"/api/v1/chats/{chat}/attachments",
headers={**h, "Content-Type": "image/png"},
params={"filename": "synthetic.png"},
content=raw,
)
assert r.status_code == 202, r.text
a = r.json()
assert a["status"] == "queued" and not a["variants"]
j = inside(
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
)
return a, j
def metadata(a):
r = client.get("/api/v1/attachments/" + a["id"], headers=h)
assert r.status_code == 200, r.text
return r.json()
def until(a, state="ready", timeout=90):
return wait(
lambda: (
r if (r := metadata(a))["status"] == state else None
),
timeout,
)
def confirm(a, j):
until(a)
downloaded = client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
)
assert (
downloaded.status_code == 200 and downloaded.content == raw
)
m = metadata(a)
assert (
m["sha256"] == hashlib.sha256(raw).hexdigest()
and m["error_code"] is None
)
assert {v["name"] for v in m["variants"]} == {
"thumbnail",
"crop",
"optimized",
}
for v in m["variants"]:
r = client.get(
f"/api/v1/attachments/{a['id']}/content",
headers=h,
params={"variant": v["name"]},
)
assert (
r.status_code == 200
and hashlib.sha256(r.content).hexdigest()
== v["sha256"]
)
with Image.open(io.BytesIO(r.content)) as image:
assert (
image.size
== {
"thumbnail": (256, 160),
"crop": (128, 128),
"optimized": (320, 200),
}[v["name"]]
)
assert not image.getexif()
effect = inspect_effect(j)
assert (
effect["publications"] == 1
and len(effect["variants"]) == 4
and len(effect["physical_keys"]) == 4
)
return effect
# Source-only crash before the DB handoff must never be reported as accepted.
stop("worker")
stop("relay")
marker = inject(chat, "api_after_staging", "wait")
before = inside(
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
)
pool = ThreadPoolExecutor(max_workers=1)
future = pool.submit(
lambda: client.post(
f"/api/v1/chats/{chat}/attachments",
headers={**h, "Content-Type": "image/png"},
params={"filename": "unconfirmed.png"},
content=raw,
)
)
wait(lambda marker=marker: hit(marker))
compose("kill", "-s", "SIGKILL", "api")
try:
response = future.result(15)
assert response.status_code != 202
except httpx.HTTPError:
pass
pool.shutdown()
start("api")
ready()
after = inside(
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
)
assert before == after
save(
"staging_before_handoff_crash",
{
"no_202": True,
"no_attachment_or_job_created": True,
"orphan_retained_for_cleanup": True,
},
)
# API durable commit/outbox survives restart without a broker or worker.
a, j = upload()
linked = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**h, "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
)
assert linked.status_code == 201
assert (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 409
)
compose("restart", "api")
ready()
compose("kill", "-s", "SIGKILL", "broker")
start("broker")
start("relay")
start("worker")
effect = confirm(a, j)
save(
"commit_before_publish_and_worker_stopped",
{
"attachment": a["id"],
"effect": effect,
"queued_link_allowed": True,
"api_and_broker_restarted": True,
},
)
# Broker-volume survival with a confirmed, still-unconsumed stream entry.
stop("worker")
a, j = upload()
wait(
lambda: inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))"
)
)
ids = inside("print(json.dumps(queue.xrange(STREAM)))")
compose("kill", "-s", "SIGKILL", "broker")
start("broker")
wait(lambda: inside("print(json.dumps(queue.ping()))"))
restored = inside("print(json.dumps(queue.xrange(STREAM)))")
assert restored == ids
start("worker")
save(
"confirmed_broker_entry_survives_sigkill",
{
"effect": confirm(a, j),
"same_stream_entries": True,
"entry_count": len(ids),
},
)
# Confirm succeeded, PG marker did not: relay recovery may publish a duplicate.
stop("worker")
stop("relay")
a, j = upload()
marker = inject(j, "relay_after_publish", "crash")
start("relay")
wait(lambda marker=marker: hit(marker))
time.sleep(0.5)
unpublished = inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']))"
)
assert unpublished >= 1
start("relay")
start("worker")
save(
"relay_crash_after_xadd_before_pg_marker",
{
"unpublished_after_crash": unpublished,
"effect": confirm(a, j),
},
)
# Repeated before/after effect and commit/ack windows.
for repeat in range(args.repeats):
for phase in ["before_effect", "after_effect", "after_commit"]:
stop("worker")
a, j = upload()
marker = inject(j, phase, "wait")
start("worker")
wait(lambda marker=marker: hit(marker))
at_crash = inspect_effect(j)
assert at_crash["publications"] == (
1 if phase == "after_commit" else 0
)
assert len(at_crash["physical_keys"]) == (
0 if phase == "before_effect" else 4
)
compose("kill", "-s", "SIGKILL", "worker")
start("worker")
effect = confirm(a, j)
save(
f"{phase}_sigkill_repeat_{repeat + 1}",
{"at_crash": at_crash, "effect": effect},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "transient")
start("worker")
effect = confirm(a, j)
assert effect["job"]["attempt"] == 2
save("transient_retry_with_jitter", {"effect": effect})
stop("worker")
a, j = upload()
inject(j, "before_effect", "hang")
start("worker")
effect = confirm(a, j)
assert any(
e["event"] == "retry"
and e["detail"]["code"] == "worker_timeout"
for e in effect["events"]
)
save("hung_child_killed_and_retried", {"effect": effect})
stop("worker")
a, j = upload()
inject(j, "before_effect", "permanent")
start("worker")
failed = until(a, "failed")
assert failed["error_code"] == "invalid_image"
assert (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 409
)
r = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**h, "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
)
assert r.status_code == 409
assert operator(j) == j
effect = confirm(a, j)
assert effect["job"]["cycle"] == 2
try:
operator(j)
except RuntimeError:
pass
else:
raise AssertionError("Replay of ready must be rejected")
save(
"permanent_failed_registry_and_audited_replay",
{
"failed": failed,
"effect": effect,
"ready_replay_rejected": True,
},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "transient", remaining=5)
start("worker")
failed = until(a, "failed", timeout=150)
assert (
failed["error_code"] == "retry_exhausted"
and row(j)["attempt"] == 5
)
operator(j)
save(
"finite_retry_exhaustion_then_explicit_replay",
{"failed": failed, "effect": confirm(a, j)},
)
# Reverse notification order and add concurrent copies: one publication/key.
stop("worker")
stop("relay")
a, j = upload()
b, k = upload()
duplicate(k, 8)
duplicate(j, 8)
start("worker")
start("relay")
ea = confirm(a, j)
confirm(b, k)
duplicate(j, 10)
duplicate(k, 10)
wait(
lambda: any(
e["event"] == "duplicate_decision"
for e in inspect_effect(j)["events"]
)
)
assert inspect_effect(j)["job"]["attempt"] == ea["job"]["attempt"]
save(
"reordered_and_duplicate_notifications",
{
"effects": [inspect_effect(j), inspect_effect(k)],
"added_duplicates": 36,
"no_extra_attempt_after_ready": True,
},
)
# Hold v1 after its PUTs, complete v2 on the other bounded slot, release old CAS.
for repeat in range(args.repeats):
stop("worker")
a, j = upload()
marker = inject(j, "after_effect", "wait")
start("worker")
wait(lambda marker=marker: hit(marker))
k = operator(j, version=2)
v2 = confirm(a, k)
hashes = {v["name"]: v["sha256"] for v in v2["variants"]}
inside(
f"with db.connection() as c: c.execute('UPDATE pipeline_faults SET released_at=clock_timestamp() WHERE id=%s',({marker},))\nprint('true')"
)
wait(
lambda j=j: any(
e["event"] == "stale_completion_rejected"
for e in inspect_effect(j)["events"]
)
)
duplicate(j, 12)
wait(
lambda j=j: any(
e["event"] == "duplicate_decision"
for e in inspect_effect(j)["events"]
)
)
assert hashes == {
v["name"]: v["sha256"] for v in confirm(a, k)["variants"]
}
assert inspect_effect(j)["publications"] == 0
save(
f"stale_v1_after_v2_repeat_{repeat + 1}",
{
"v1": inspect_effect(j),
"v2": inspect_effect(k),
"new_hashes_unchanged": True,
},
)
# No arbitrary URL/bytes fields in queue envelopes.
invalid = inside(
"bad={'schema_version':1,'url':'http://example.invalid/raw'}\nqueue.xadd(STREAM,{'job':json.dumps(bad)})\nprint('true')"
)
assert invalid
wait(
lambda: inside(
"with db.connection() as c: r=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE event='invalid_envelope'\").fetchone()\nprint(json.dumps(r['n']>0))"
)
)
save(
"malformed_notification_rejected", {"durable_diagnostic": True}
)
report["worker_metrics"] = inside(
"with db.connection() as c: r=c.execute(\"SELECT job_id,detail FROM pipeline_events WHERE event='child_finished' ORDER BY id\").fetchall()\nprint(json.dumps(r,default=str))"
)
report["audit"] = inside(
"with db.connection() as c: r=c.execute('SELECT job_id,action,operator,reason,created_at FROM pipeline_audit ORDER BY id').fetchall()\nprint(json.dumps(r,default=str))"
)
report["complete"] = True
finally:
for role in ["api", "broker", "relay", "worker", "reconciler"]:
try:
start(role)
except Exception:
pass
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS: complete versioned pipeline fault matrix", flush=True)
if __name__ == "__main__":
main()
+8
View File
@@ -343,6 +343,14 @@ def consume(slot):
with db.connection() as conn:
event(conn, row["id"], "ready_effect_missing")
continue
if state in {"ready", "obsolete", "failed"}:
with db.connection() as conn:
event(
conn,
data["job_id"],
"duplicate_decision",
{"state": state, "entry": entry},
)
if state == "claimed":
receiver, sender = context.Pipe(duplex=False)
child = context.Process(
+119
View File
@@ -0,0 +1,119 @@
import json
import os
from pathlib import Path
from uuid import uuid4
import httpx
import pytest
from jsonschema import Draft202012Validator, FormatChecker
CONTRACT = json.loads(
(Path(__file__).parents[1] / "contracts/openapi.json").read_text()
)
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8084")
PASSWORD = "Synthetic-Password-🌍-123"
TOKENS = {}
def check(result: httpx.Response, status: int, schema: str) -> dict:
assert result.status_code == status, result.text
assert result.headers["x-request-id"]
assert result.headers["x-instance-id"]
assert result.headers["content-type"].startswith("application/json")
payload = result.json()
spec = {"$ref": f"#/components/schemas/{schema}", **CONTRACT}
Draft202012Validator(spec, format_checker=FormatChecker()).validate(
payload
)
if schema == "Error":
assert payload["error"]["request_id"] == result.headers["x-request-id"]
assert (
payload["error"]["code"]
== {
400: "invalid_request",
401: "unauthorized",
404: "not_found",
409: "conflict",
413: "payload_too_large",
415: "unsupported_media_type",
429: "rate_limited",
500: "internal_error",
503: "unavailable",
}[status]
)
return payload
def register(client: httpx.Client, **overrides) -> dict:
data = {
"username": "u_" + uuid4().hex[:24],
"display_name": "Алиса 🌍",
"password": PASSWORD,
}
data.update(overrides)
user = check(client.post("/api/v1/users", json=data), 201, "User")
session = check(
client.post(
"/api/v1/auth/sessions",
auth=httpx.BasicAuth(data["username"], data["password"]),
),
201,
"Session",
)
TOKENS[user["id"]] = session["token"]
return user
@pytest.fixture
def client():
with httpx.Client(
base_url=BASE_URL, timeout=15, trust_env=False
) as client:
yield client
@pytest.fixture
def actors(client):
return register(client), register(client), register(client)
@pytest.fixture
def chat(client, actors):
alice, bob, _ = actors
return check(
client.post(
"/api/v1/chats",
headers=headers(alice),
json={"title": "Учёба 🌍", "member_ids": [alice["id"], bob["id"]]},
),
201,
"Chat",
)
def headers(actor: dict) -> dict:
return {
"Authorization": "Bearer " + TOKENS[actor["id"]],
"Idempotency-Key": uuid4().hex,
}
def history(client, actor, chat_id, limit=7):
found, cursor = [], None
while True:
params = {"limit": limit}
if cursor:
params["cursor"] = cursor
page = check(
client.get(
f"/api/v1/chats/{chat_id}/messages",
headers=headers(actor),
params=params,
),
200,
"MessagePage",
)
found.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return found
+412
View File
@@ -0,0 +1,412 @@
import base64
import json
import threading
from concurrent.futures import ThreadPoolExecutor
from uuid import UUID, uuid4
import httpx
import pytest
from conftest import BASE_URL, PASSWORD, check, headers, history, register
@pytest.mark.parametrize("path", ["/health/live", "/health/ready"])
def test_health(client, path):
assert check(client.get(path), 200, "Health") == {"status": "ok"}
@pytest.mark.parametrize(
"value", [None, "not-a-uuid", str(uuid4()), uuid4().hex]
)
def test_identification(client, value):
check(
client.get(
"/api/v1/users/me", headers={"X-User-Id": value} if value else {}
),
401,
"Error",
)
def test_identity(client, actors):
assert (
check(
client.get("/api/v1/users/me", headers=headers(actors[0])),
200,
"User",
)
== actors[0]
)
@pytest.mark.parametrize(
"data",
[
{},
[],
None,
{"username": "ABC", "display_name": "A"},
{"username": "abc\n", "display_name": "A"},
{"username": 123, "display_name": "A"},
{"username": "abc", "display_name": " "},
{"username": "abc", "display_name": "x" * 101},
{"username": "abc", "display_name": "A", "extra": True},
{"username": "abc", "display_name": "\ud800"},
],
)
def test_invalid_user(client, data):
if isinstance(data, dict):
data = {"password": PASSWORD, **data}
check(client.post("/api/v1/users", content=json.dumps(data)), 400, "Error")
@pytest.mark.parametrize(
"raw", [b"{", b"NaN", b'{"username":"abc", "display_name":NaN}', b"\xff"]
)
def test_malformed_json(client, raw):
check(client.post("/api/v1/users", content=raw), 400, "Error")
def test_auth_before_json(client):
check(client.post("/api/v1/chats", content=b"{"), 401, "Error")
def test_duplicate_username(client, actors):
check(
client.post(
"/api/v1/users",
json={
"username": actors[0]["username"],
"display_name": "B",
"password": PASSWORD,
},
),
409,
"Error",
)
def test_unicode_codepoints(client, actors, chat):
user = register(client, display_name="🌍" * 100)
assert user["display_name"] == "🌍" * 100
text = "🌍" * 4000
result = check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": text},
),
201,
"Message",
)
assert result["text"] == text
assert result["sender_id"] == actors[0]["id"]
@pytest.mark.parametrize(
"method,suffix,body",
[
("GET", "", None),
("GET", "/messages", None),
("POST", "/messages", {"text": "hello"}),
],
)
def test_acl(client, actors, chat, method, suffix, body):
outsider = headers(actors[2])
actual = check(
client.request(
method,
f"/api/v1/chats/{chat['id']}{suffix}",
headers=outsider,
json=body,
),
404,
"Error",
)
missing = check(
client.request(
method,
f"/api/v1/chats/{uuid4()}{suffix}",
headers=outsider,
json=body,
),
404,
"Error",
)
assert actual["error"]["message"] == missing["error"]["message"]
@pytest.mark.parametrize("value", ["", " \t\n", 42, True, None, "a" * 4001])
def test_bad_message(client, actors, chat, value):
check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": value},
),
400,
"Error",
)
def test_sender_not_client_controlled(client, actors, chat):
check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": "Hi", "sender_id": actors[1]["id"]},
),
400,
"Error",
)
@pytest.mark.parametrize(
"case",
[
"duplicate",
"unknown",
"no_caller",
"empty",
"wrong_type",
"bad_uuid",
"too_many",
"blank_title",
],
)
def test_invalid_chat(client, actors, case):
a, b, _ = actors
members = {
"duplicate": [a["id"], a["id"]],
"unknown": [a["id"], str(uuid4())],
"no_caller": [b["id"]],
"empty": [],
"wrong_type": a["id"],
"bad_uuid": ["broken"],
"too_many": [str(uuid4()) for _ in range(101)],
"blank_title": [a["id"]],
}[case]
check(
client.post(
"/api/v1/chats",
headers=headers(a),
json={
"title": " " if case == "blank_title" else "Title",
"member_ids": members,
},
),
400,
"Error",
)
@pytest.mark.parametrize(
"limit", ["0", "101", "-1", "1.5", "true", "1e1", "", "12"]
)
def test_bad_limit(client, actors, chat, limit):
check(
client.get(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
params={"limit": limit},
),
400,
"Error",
)
@pytest.mark.parametrize(
"cursor", ["", "broken", "W10", "e30", "*", "x" * 2049]
)
def test_bad_cursor(client, actors, chat, cursor):
check(
client.get(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
params={"cursor": cursor},
),
400,
"Error",
)
def test_pagination(client, actors, chat):
a, b, outsider = actors
sent = []
path = f"/api/v1/chats/{chat['id']}/messages"
for i in range(23):
sent.append(
check(
client.post(
path,
headers=headers(a if i % 2 else b),
json={"text": f"Msg {i}"},
),
201,
"Message",
)
)
got = history(client, a, chat["id"])
assert got == sorted(
sent, key=lambda m: (m["created_at"], UUID(m["id"]).int)
)
first = check(
client.get(path, headers=headers(a), params={"limit": 1}),
200,
"MessagePage",
)
second = check(
client.get(
path,
headers=headers(b),
params={"limit": 22, "cursor": first["next_cursor"]},
),
200,
"MessagePage",
)
assert second["items"] == got[1:]
assert second["next_cursor"] is None
chats = check(
client.get("/api/v1/chats", headers=headers(outsider)), 200, "ChatPage"
)
assert chats["items"] == []
other = check(
client.post(
"/api/v1/chats",
headers=headers(a),
json={"title": "Other", "member_ids": [a["id"]]},
),
201,
"Chat",
)
check(
client.get(
f"/api/v1/chats/{other['id']}/messages",
headers=headers(a),
params={"cursor": first["next_cursor"]},
),
400,
"Error",
)
# Валидный base64/JSON с неверным типом ключа тоже даёт 400.
data = json.loads(
base64.urlsafe_b64decode(
first["next_cursor"] + "=" * (-len(first["next_cursor"]) % 4)
)
)
data["at"] = 17
forged = (
base64.urlsafe_b64encode(json.dumps(data).encode())
.decode()
.rstrip("=")
)
check(
client.get(path, headers=headers(a), params={"cursor": forged}),
400,
"Error",
)
def test_chat_pagination_scope(client, actors):
a, b, _ = actors
created = []
for i in range(4):
created.append(
check(
client.post(
"/api/v1/chats",
headers=headers(a),
json={"title": str(i), "member_ids": [a["id"], b["id"]]},
),
201,
"Chat",
)
)
page = check(
client.get("/api/v1/chats", headers=headers(a), params={"limit": 2}),
200,
"ChatPage",
)
assert page["items"] == created[:2]
last = check(
client.get(
"/api/v1/chats",
headers=headers(a),
params={"limit": 2, "cursor": page["next_cursor"]},
),
200,
"ChatPage",
)
assert last["items"] == created[2:]
assert last["next_cursor"] is None
check(
client.get(
"/api/v1/chats",
headers=headers(b),
params={"cursor": page["next_cursor"]},
),
400,
"Error",
)
def test_concurrent_messages(client, actors, chat):
barrier = threading.Barrier(10)
a = actors[0]
path = f"/api/v1/chats/{chat['id']}/messages"
def worker(i):
with httpx.Client(
base_url=BASE_URL, timeout=15, trust_env=False
) as conn:
barrier.wait(timeout=15)
return [
check(
conn.post(
path, headers=headers(a), json={"text": f"{i}:{j}"}
),
201,
"Message",
)["id"]
for j in range(10)
]
with ThreadPoolExecutor(max_workers=10) as pool:
ids = [item for batch in pool.map(worker, range(10)) for item in batch]
rows = history(client, a, chat["id"])
assert len(rows) == len(ids) == len(set(ids)) == 100
assert set(ids) == {row["id"] for row in rows}
assert len({row["text"] for row in rows}) == 100
def test_registration_race(client):
username = "race_" + uuid4().hex[:20]
barrier = threading.Barrier(10)
def worker(_):
with httpx.Client(
base_url=BASE_URL, timeout=15, trust_env=False
) as conn:
barrier.wait(timeout=15)
result = conn.post(
"/api/v1/users",
json={
"username": username,
"display_name": "Test",
"password": PASSWORD,
},
)
check(
result,
result.status_code,
"User" if result.status_code == 201 else "Error",
)
return result.status_code
with ThreadPoolExecutor(max_workers=10) as pool:
statuses = list(pool.map(worker, range(10)))
assert statuses.count(201) == 1
assert statuses.count(409) == 9
def test_unknown_path(client):
check(client.get("/does-not-exist"), 404, "Error")
+212
View File
@@ -0,0 +1,212 @@
import hashlib
import io
import time
from pathlib import Path
from uuid import uuid4
import pytest
from conftest import check, headers
from PIL import Image
PNG = (Path(__file__).parents[1] / "tests/fixtures/sample.png").read_bytes()
def upload(
client, actor, chat, *, data=PNG, filename="Фото 🌍.png", mime="image/png"
):
return client.post(
f"/api/v1/chats/{chat['id']}/attachments",
headers={**headers(actor), "Content-Type": mime},
params={"filename": filename},
content=data,
)
def wait_ready(client, actor, attachment):
end = time.monotonic() + 60
while time.monotonic() < end:
row = check(
client.get(
f"/api/v1/attachments/{attachment['id']}",
headers=headers(actor),
),
200,
"Attachment",
)
if row["status"] == "ready":
return row
assert row["status"] != "failed", row
time.sleep(0.2)
raise TimeoutError("Pipeline readiness")
def test_attachment_roundtrip_acl_and_message(client, actors, chat):
attachment = check(upload(client, actors[0], chat), 202, "Attachment")
assert attachment["status"] == "queued" and attachment["variants"] == []
attachment = wait_ready(client, actors[0], attachment)
assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest()
assert attachment["size_bytes"] == len(PNG)
path = f"/api/v1/attachments/{attachment['id']}"
assert (
check(client.get(path, headers=headers(actors[1])), 200, "Attachment")
== attachment
)
downloaded = client.get(path + "/content", headers=headers(actors[1]))
assert downloaded.status_code == 200 and downloaded.content == PNG
assert downloaded.headers["Content-Type"] == "image/png"
assert not downloaded.history
for suffix in ["", "/content"]:
check(
client.get(path + suffix, headers=headers(actors[2])), 404, "Error"
)
check(client.get(path + suffix), 401, "Error")
thumb = client.get(
path + "/content",
headers=headers(actors[1]),
params={"variant": "thumbnail"},
)
assert thumb.status_code == 200
with Image.open(io.BytesIO(thumb.content)) as image:
assert image.format == "JPEG" and image.size == (256, 160)
sent = check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[1]),
json={"text": "", "attachment_ids": [attachment["id"]]},
),
201,
"Message",
)
assert sent["attachment_ids"] == [attachment["id"]]
assert sent["sender_id"] == actors[1]["id"]
@pytest.mark.parametrize(
"name",
[
"../x.png",
"x/y.png",
"x\\y.png",
"x\x00.png",
"x\x7f.png",
"x\n.png",
"a" * 256,
"",
],
)
def test_filename_guard(client, actors, chat, name):
check(upload(client, actors[0], chat, filename=name), 400, "Error")
@pytest.mark.parametrize(
"data,mime,status",
[
(b"not an image", "image/png", 415),
(PNG, "image/jpeg", 415),
(PNG, "application/octet-stream", 415),
(PNG[:25], "image/png", 415),
(b"", "image/png", 415),
(b"x" * (10 * 1024 * 1024 + 1), "image/png", 413),
],
)
def test_image_guards(client, actors, chat, data, mime, status):
r = upload(client, actors[0], chat, data=data, mime=mime)
check(r, status, "Error")
def test_jpeg_original_unchanged(client, actors, chat):
stream = io.BytesIO()
Image.new("RGB", (320, 200), (30, 60, 90)).save(stream, format="JPEG")
jpeg = stream.getvalue()
attachment = check(
upload(
client,
actors[0],
chat,
data=jpeg,
filename="camera.jpg",
mime="image/jpeg",
),
202,
"Attachment",
)
attachment = wait_ready(client, actors[0], attachment)
r = client.get(
f"/api/v1/attachments/{attachment['id']}/content",
headers=headers(actors[1]),
)
assert r.content == jpeg
def test_cross_chat_link_and_unknown(client, actors, chat):
a = check(upload(client, actors[0], chat), 202, "Attachment")
other = check(
client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "Other", "member_ids": [actors[0]["id"]]},
),
201,
"Chat",
)
for identity in [a["id"], str(uuid4())]:
check(
client.post(
f"/api/v1/chats/{other['id']}/messages",
headers=headers(actors[0]),
json={"text": "", "attachment_ids": [identity]},
),
404,
"Error",
)
check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=headers(actors[0]),
json={"text": "", "attachment_ids": [a["id"], a["id"]]},
),
400,
"Error",
)
def test_default_attachments_are_semantically_equal(client, actors, chat):
h = headers(actors[0])
path = f"/api/v1/chats/{chat['id']}/messages"
first = check(
client.post(path, headers=h, json={"text": "Semantic default"}),
201,
"Message",
)
second = check(
client.post(
path,
headers=h,
json={"attachment_ids": [], "text": "Semantic default"},
),
201,
"Message",
)
assert first == second and first["attachment_ids"] == []
def test_pixel_limit_before_decode(client, actors, chat):
# Compressible synthetic image stays below10MiB but exceeds40M pixels.
stream = io.BytesIO()
Image.new("1", (8000, 5001)).save(stream, format="PNG")
assert len(stream.getvalue()) < 10 * 1024 * 1024
r = upload(client, actors[0], chat, data=stream.getvalue())
assert r.status_code == 413
def test_chunked_body_limit_without_content_length(client, actors, chat):
r = client.post(
f"/api/v1/chats/{chat['id']}/attachments",
headers={**headers(actors[0]), "Content-Type": "image/png"},
params={"filename": "chunked.png"},
content=iter([b"x" * (10 * 1024 * 1024 + 1)]),
)
assert r.request.headers.get("content-length") is None
assert r.status_code == 413
assert r.json()["error"]["code"] == "payload_too_large"
assert r.json()["error"]["request_id"] == r.headers["x-request-id"]
+26
View File
@@ -0,0 +1,26 @@
import json
from uuid import uuid4
import pytest
from conftest import check, headers
@pytest.mark.parametrize("encoding", ["utf-16", "utf-32"])
def test_only_utf8_json(client, encoding):
payload = {"username": "u_" + uuid4().hex[:20], "display_name": "UTF"}
payload["password"] = "Synthetic-encoding-test-123"
result = client.post(
"/api/v1/users",
content=json.dumps(payload).encode(encoding),
headers={"Content-Type": "application/json"},
)
check(result, 400, "Error")
def test_leading_zero_limit(client, actors, chat):
result = client.get(
"/api/v1/chats/" + chat["id"] + "/messages",
headers=headers(actors[0]),
params={"limit": "0001"},
)
check(result, 200, "MessagePage")
+23
View File
@@ -0,0 +1,23 @@
from conftest import check, headers
def test_history_acl_precedes_query_validation(client, actors, chat):
path = f"/api/v1/chats/{chat['id']}/messages"
check(
client.get(
path,
headers=headers(actors[2]),
params={"limit": 0, "cursor": "broken"},
),
404,
"Error",
)
check(
client.get(
path,
headers=headers(actors[0]),
params={"limit": 0, "cursor": "broken"},
),
400,
"Error",
)
+75
View File
@@ -0,0 +1,75 @@
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
import httpx
import pytest
from conftest import BASE_URL, check, headers, history
@pytest.mark.parametrize("key", [None, "", "a b", "x" * 129, "a/b"])
def test_key_required_and_valid(client, actors, chat, key):
h = headers(actors[0])
h.pop("Idempotency-Key")
if key is not None:
h["Idempotency-Key"] = key
check(
client.post(
f"/api/v1/chats/{chat['id']}/messages",
headers=h,
json={"text": "Key"},
),
400,
"Error",
)
def test_parallel_replay_across_instances(client, actors, chat):
h = headers(actors[0])
path = f"/api/v1/chats/{chat['id']}/messages"
def send(_):
with httpx.Client(base_url=BASE_URL, trust_env=False, timeout=15) as c:
r = c.post(path, headers=h, json={"text": "Один результат 🌍"})
return check(r, 201, "Message"), r.headers["x-instance-id"]
with ThreadPoolExecutor(max_workers=10) as pool:
results = list(pool.map(send, range(30)))
assert len({r[0]["id"] for r in results}) == 1
assert len({r[1] for r in results}) >= 2
assert history(client, actors[0], chat["id"]) == [results[0][0]]
check(
client.post(path, headers=h, json={"text": "Другой текст"}),
409,
"Error",
)
# Другой пользователь может независимо использовать тот же ключ.
other = headers(actors[1])
other["Idempotency-Key"] = h["Idempotency-Key"]
second = check(
client.post(path, headers=other, json={"text": "Боб"}), 201, "Message"
)
assert second["id"] != results[0][0]["id"]
def test_replay_rechecks_acl(client, actors, chat):
h = headers(actors[0])
path = f"/api/v1/chats/{chat['id']}/messages"
check(
client.post(path, headers=h, json={"text": "Existing"}), 201, "Message"
)
outsider = headers(actors[2])
outsider["Idempotency-Key"] = h["Idempotency-Key"]
check(
client.post(path, headers=outsider, json={"text": "Existing"}),
404,
"Error",
)
check(
client.post(
path,
headers={"Idempotency-Key": uuid4().hex},
json={"text": "Existing"},
),
401,
"Error",
)
+114
View File
@@ -0,0 +1,114 @@
from uuid import uuid4
import httpx
import pytest
from conftest import PASSWORD, TOKENS, check, headers, register
@pytest.mark.parametrize(
"password", ["", "x" * 11, "x" * 129, " " * 12, 42, True, None]
)
def test_password_validation(client, password):
check(
client.post(
"/api/v1/users",
json={
"username": "pwd_" + uuid4().hex[:20],
"display_name": "Test",
"password": password,
},
),
400,
"Error",
)
def test_full_utf8_password(client):
password = "🌍" * 127 + ":"
user = register(client, password=password)
assert (
check(
client.get("/api/v1/users/me", headers=headers(user)), 200, "User"
)
== user
)
@pytest.mark.parametrize(
"authorization", [None, "Basic broken", "Bearer unknown", "Basic /w=="]
)
def test_bad_basic(client, authorization):
result = client.post(
"/api/v1/auth/sessions",
headers={"Authorization": authorization} if authorization else {},
)
check(result, 401, "Error")
assert (
result.headers["www-authenticate"]
== 'Basic realm="messenger", charset="UTF-8"'
)
def test_wrong_password_and_unknown_user(client):
user = register(client)
for username in [user["username"], "missing_" + uuid4().hex[:20]]:
result = client.post(
"/api/v1/auth/sessions",
auth=httpx.BasicAuth(username, PASSWORD + "wrong"),
)
data = check(result, 401, "Error")
assert data["error"]["message"] == "Authentication required"
def test_logout_is_not_revived_by_cache(client):
user = register(client)
auth = headers(user)
check(client.get("/api/v1/users/me", headers=auth), 200, "User")
result = client.delete("/api/v1/auth/sessions/current", headers=auth)
assert result.status_code == 204 and result.content == b""
assert result.headers["x-request-id"]
check(client.get("/api/v1/users/me", headers=auth), 401, "Error")
check(
client.delete("/api/v1/auth/sessions/current", headers=auth),
401,
"Error",
)
def test_only_current_session_revoked(client):
user = register(client)
another = check(
client.post(
"/api/v1/auth/sessions",
auth=httpx.BasicAuth(user["username"], PASSWORD),
),
201,
"Session",
)
assert another["token"] != TOKENS[user["id"]]
assert len(another["token"]) >= 32
assert (
client.delete(
"/api/v1/auth/sessions/current", headers=headers(user)
).status_code
== 204
)
result = client.get(
"/api/v1/users/me",
headers={"Authorization": "Bearer " + another["token"]},
)
assert check(result, 200, "User") == user
def test_bearer_source_and_ignored_user_header(client, actors):
alice, bob, _ = actors
auth = {**headers(alice), "X-User-Id": bob["id"]}
assert (
check(client.get("/api/v1/users/me", headers=auth), 200, "User")["id"]
== alice["id"]
)
result = client.get(
"/api/v1/users/me", auth=httpx.BasicAuth(alice["username"], PASSWORD)
)
check(result, 401, "Error")
assert result.headers["www-authenticate"] == 'Bearer realm="messenger"'