Files
mtusi2026lab5/student-tests/test_pipeline.py
T

358 lines
13 KiB
Python

import hashlib
import io
import json
import struct
import zlib
from dataclasses import replace
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
from urllib.parse import quote
from uuid import uuid4
import jsonschema
import pytest
from conftest import ROOT
from PIL import Image
from test_api import error, headers
from messenger.broker import GROUP, STREAM, Broker, Job
from messenger.cleanup import collect
from messenger.domain import DomainError
from messenger.images import variants
from messenger.pipeline import Pipeline
from messenger.relay import publish_one
from messenger.worker import handle
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
def upload(client, actor, room, data=PNG, media="image/png", name="Фото.png"):
return client.post(
f"/api/v1/chats/{room['id']}/attachments?filename={quote(name, safe='')}",
headers={**headers(actor), "Content-Type": media},
content=data,
)
@pytest.fixture
def broker(settings):
b = Broker(settings)
assert settings.broker_url.endswith("/1")
b.client.flushdb()
b.group()
yield b
b.client.flushdb()
b.close()
def process(client, settings, broker):
db = client.app.state.repository
assert publish_one(db, broker, settings)
id, fields = broker.client.xreadgroup(GROUP, "unit", {STREAM: ">"}, count=1)[0][1][0]
jsonschema.validate(
json.loads(fields["job"]),
json.loads((ROOT / "contracts/jobs.schema.json").read_text()),
format_checker=jsonschema.FormatChecker(),
)
handle(db, broker, client.app.state.attachments.store, settings, id, fields["job"])
def test_queued_message_ready_and_all_variants(client, actors, room, settings, broker):
r = upload(client, actors[0], room)
assert r.status_code == 202 and r.json()["status"] == "queued"
a = r.json()
assert not a["variants"] and a["sha256"] == hashlib.sha256(PNG).hexdigest()
path = "/api/v1/attachments/" + a["id"]
error(client.get(path + "/content", headers=headers(actors[0])), 409, "conflict")
key = uuid4().hex
msg = client.post(
f"/api/v1/chats/{room['id']}/messages",
json={"text": "", "attachment_ids": [a["id"]]},
headers={**headers(actors[0]), "Idempotency-Key": key},
)
assert msg.status_code == 201
for suffix in ("", "/content"):
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
error(client.get(path + suffix), 401, "unauthorized")
process(client, settings, broker)
ready = client.get(path, headers=headers(actors[1])).json()
assert ready["status"] == "ready" and ready["error_code"] is None
assert {v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"}
assert client.get(path + "/content", headers=headers(actors[1])).content == PNG
for v in ready["variants"]:
content = client.get(path + "/content?variant=" + v["name"], headers=headers(actors[1]))
assert content.status_code == 200 and content.headers["content-type"] == v["content_type"]
assert len(content.content) == v["size_bytes"]
assert hashlib.sha256(content.content).hexdigest() == v["sha256"]
with Image.open(io.BytesIO(content.content)) as image:
assert image.size == (v["width"], v["height"])
assert not image.getexif()
with client.app.state.repository.connection() as conn:
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (a["id"],)
).fetchone()
job = Job(
schema_version=1,
type="attachment.process.v1",
job_id=str(j["job_id"]),
attachment_id=a["id"],
pipeline_version=1,
attempt=1,
created_at=a["created_at"],
)
sid = broker.publish(job)
handle(
client.app.state.repository,
broker,
client.app.state.attachments.store,
settings,
sid,
job.model_dump_json(exclude_none=True),
)
assert client.get(path, headers=headers(actors[0])).json() == ready
assert broker.client.xlen(STREAM) == 0
@pytest.mark.parametrize("name", ["", "../a.png", "a\\b.png", "a\x00.png", "\n.png", "x" * 256])
def test_bad_filename(client, actors, room, name):
error(upload(client, actors[0], room, name=name), 400, "invalid_request")
@pytest.mark.parametrize(
"data,media",
[
(b"", "image/png"),
(b"bad", "image/png"),
(PNG[:30], "image/png"),
(PNG, "image/jpeg"),
(PNG, "text/plain"),
],
)
def test_bad_image(client, actors, room, data, media):
error(upload(client, actors[0], room, data, media), 415, "unsupported_media_type")
def test_upload_limits_and_pixel_bomb(client, actors, room):
error(upload(client, actors[0], room, b"x" * (10485760 + 1)), 413, "payload_too_large")
error(
upload(client, actors[0], room, (b"x" * 1048576 for _ in range(11))),
413,
"payload_too_large",
)
data = bytearray(PNG)
data[16:24] = struct.pack(">II", 10000, 10000)
data[29:33] = struct.pack(">I", zlib.crc32(data[12:29]) & 0xFFFFFFFF)
error(upload(client, actors[0], room, bytes(data)), 413, "payload_too_large")
@pytest.mark.parametrize("size", [(320, 200), (17, 11), (1, 400), (400, 1), (1281, 713)])
def test_transform_dimensions_alpha_and_exif(size):
raw = io.BytesIO()
Image.new("RGBA", size, (1, 2, 3, 0)).save(raw, "PNG")
results = {name: (data, dims, media) for name, data, dims, media in variants(raw, 1)}
scale = max(size)
assert results["thumbnail"][1] == tuple(max(1, n * min(256, scale) // scale) for n in size)
assert results["crop"][1] == (128, 128)
assert max(results["optimized"][1]) <= 1280
with Image.open(io.BytesIO(results["thumbnail"][0])) as image:
assert image.getpixel((0, 0)) == (255, 255, 255)
def test_exif_orientation_and_original_unchanged():
raw = io.BytesIO()
exif = Image.Exif()
exif[274] = 6
Image.new("RGB", (320, 200), "red").save(raw, "JPEG", exif=exif)
before = raw.getvalue()
result = list(variants(raw, 1))
assert result[0][2] == (160, 256) and raw.getvalue() == before
for _, data, _, _ in result:
with Image.open(io.BytesIO(data)) as image:
assert not image.getexif()
def test_failed_retry_replay_and_ready_replay_rejected(
client, actors, room, settings, broker, monkeypatch
):
a = upload(client, actors[0], room).json()
store, db = client.app.state.attachments.store, client.app.state.repository
original = store.put
monkeypatch.setattr(
store, "put", lambda *_: (_ for _ in ()).throw(DomainError(503, "unavailable", "test"))
)
process(client, replace(settings, job_max_attempts=1), broker)
path = "/api/v1/attachments/" + a["id"]
assert client.get(path, headers=headers(actors[0])).json()["error_code"] == "retry_exhausted"
error(
client.post(
f"/api/v1/chats/{room['id']}/messages",
json={"text": "", "attachment_ids": [a["id"]]},
headers={**headers(actors[0]), "Idempotency-Key": uuid4().hex},
),
409,
"conflict",
)
Pipeline(db, settings).replay(a["id"])
monkeypatch.setattr(store, "put", original)
process(client, settings, broker)
assert client.get(path, headers=headers(actors[0])).json()["status"] == "ready"
with pytest.raises(ValueError):
Pipeline(db, settings).replay(a["id"])
def test_invalid_envelope_quarantined(client, settings, broker):
sid = broker.client.xadd(STREAM, {"job": '{"url":"http://untrusted.invalid/"}'})
handle(
client.app.state.repository,
broker,
client.app.state.attachments.store,
settings,
sid,
'{"url":"http://untrusted.invalid/"}',
)
assert broker.client.xlen("invalid-jobs") == 1
assert "url" not in json.dumps(broker.client.xrange("invalid-jobs"))
def test_staging_failure_has_no_accepted_job(client, actors, room, monkeypatch):
store = client.app.state.attachments.store
monkeypatch.setattr(
store, "put", lambda *_: (_ for _ in ()).throw(DomainError(503, "unavailable", "test"))
)
error(upload(client, actors[0], room), 503, "unavailable")
with client.app.state.repository.connection() as conn:
assert conn.execute("SELECT count(*) AS n FROM attachment_jobs").fetchone()["n"] == 0
def test_gc_minimum_retention(client):
with pytest.raises(ValueError):
collect(client.app.state.repository, client.app.state.attachments.store, 0, True)
def test_gc_fences_paused_upload_before_job_commit(client, actors, room, settings, monkeypatch):
db, store = client.app.state.repository, client.app.state.attachments.store
p = Pipeline(db, settings)
id = str(uuid4())
key = "staging/" + id
p.reserve(id, key)
store.put(key, io.BytesIO(PNG), len(PNG), "image/png", hashlib.sha256(PNG).hexdigest())
# Simulated S3 object age; actual private test-bucket deletion and DB locks are real.
old = datetime.now(UTC) - timedelta(seconds=301)
head = store.head(key)
head["LastModified"] = old
objects = [{"Key": key, "ETag": head["ETag"], "Size": len(PNG), "LastModified": old}]
monkeypatch.setattr(store, "head", lambda _: head)
monkeypatch.setattr(
store.client,
"get_paginator",
lambda _: SimpleNamespace(paginate=lambda **_: [{"Contents": objects}]),
)
assert collect(db, store, 300, True)["protected"] == 1
with db.connection(write=True) as conn:
conn.execute(
"UPDATE upload_reservations SET expires_at=clock_timestamp()-interval '1 second' "
"WHERE id=%s",
(id,),
)
assert collect(db, store, 300, False)["eligible"] == 1
assert collect(db, store, 300, True)["deleted"] == 1
with pytest.raises(DomainError) as expired:
p.accept(
id,
room["id"],
actors[0]["id"],
"paused.png",
"image/png",
len(PNG),
hashlib.sha256(PNG).hexdigest(),
key,
)
assert expired.value.status == 503 and db.attachment(id) is None
def test_version_fence_rejects_old_writer(client, actors, room, settings, broker):
a = upload(client, actors[0], room).json()
db, store = client.app.state.repository, client.app.state.attachments.store
with db.connection() as conn:
j = conn.execute(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (a["id"],)
).fetchone()
envelope = Job(
schema_version=1,
type="attachment.process.v1",
job_id=str(j["job_id"]),
attachment_id=a["id"],
pipeline_version=1,
attempt=1,
created_at=a["created_at"],
)
pipeline = Pipeline(db, settings)
outcome, old = pipeline.claim(envelope)
assert outcome == "claimed"
pipeline.replay(a["id"], new_version=True)
process(client, settings, broker)
current = db.attachment(a["id"])
assert current["pipeline_version"] == 2 and current["storage_state"] == "ready"
assert not pipeline.publish(old, "objects/old-result", [])
handle(
db,
broker,
store,
settings,
broker.publish(envelope),
envelope.model_dump_json(exclude_none=True),
)
assert db.attachment(a["id"])["object_key"] == current["object_key"]
def test_foreign_chat_attachment_is_not_linkable(client, actors, room):
a = upload(client, actors[0], room).json()
other = client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "other", "member_ids": [actors[0]["id"]]},
).json()
error(
client.post(
f"/api/v1/chats/{other['id']}/messages",
headers={**headers(actors[0]), "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
),
404,
"not_found",
)
@pytest.mark.parametrize(
"patch",
[
{"schema_version": True},
{"traceparent": None},
{"tracestate": None},
{"url": "https://untrusted.invalid/"},
{"job_id": "not-a-uuid"},
{"pipeline_version": 0},
{"attempt": 0},
{"created_at": "2026-10-17T12:00:00"},
{"traceparent": "invalid"},
{"type": "download.anything"},
],
)
def test_job_schema_rejects_invalid_carrier(client, settings, broker, patch):
payload = {
"schema_version": 1,
"type": "attachment.process.v1",
"job_id": str(uuid4()),
"attachment_id": str(uuid4()),
"pipeline_version": 1,
"attempt": 1,
"created_at": "2026-10-17T12:00:00Z",
**patch,
}
data = json.dumps(payload)
sid = broker.client.xadd(STREAM, {"job": data})
handle(
client.app.state.repository, broker, client.app.state.attachments.store, settings, sid, data
)
assert broker.client.xrange("invalid-jobs")[0][1]["reason"] == "invalid_envelope"