forked from mtusi2026/mtusi2026lab5
358 lines
13 KiB
Python
358 lines
13 KiB
Python
import hashlib
|
|
import io
|
|
import json
|
|
import struct
|
|
import zlib
|
|
from dataclasses import replace
|
|
from datetime import UTC, datetime, timedelta
|
|
from types import SimpleNamespace
|
|
from urllib.parse import quote
|
|
from uuid import uuid4
|
|
|
|
import jsonschema
|
|
import pytest
|
|
from conftest import ROOT
|
|
from PIL import Image
|
|
from test_api import error, headers
|
|
|
|
from messenger.broker import GROUP, STREAM, Broker, Job
|
|
from messenger.cleanup import collect
|
|
from messenger.domain import DomainError
|
|
from messenger.images import variants
|
|
from messenger.pipeline import Pipeline
|
|
from messenger.relay import publish_one
|
|
from messenger.worker import handle
|
|
|
|
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
|
|
|
|
def upload(client, actor, room, data=PNG, media="image/png", name="Фото.png"):
|
|
return client.post(
|
|
f"/api/v1/chats/{room['id']}/attachments?filename={quote(name, safe='')}",
|
|
headers={**headers(actor), "Content-Type": media},
|
|
content=data,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def broker(settings):
|
|
b = Broker(settings)
|
|
assert settings.broker_url.endswith("/1")
|
|
b.client.flushdb()
|
|
b.group()
|
|
yield b
|
|
b.client.flushdb()
|
|
b.close()
|
|
|
|
|
|
def process(client, settings, broker):
|
|
db = client.app.state.repository
|
|
assert publish_one(db, broker, settings)
|
|
id, fields = broker.client.xreadgroup(GROUP, "unit", {STREAM: ">"}, count=1)[0][1][0]
|
|
jsonschema.validate(
|
|
json.loads(fields["job"]),
|
|
json.loads((ROOT / "contracts/jobs.schema.json").read_text()),
|
|
format_checker=jsonschema.FormatChecker(),
|
|
)
|
|
handle(db, broker, client.app.state.attachments.store, settings, id, fields["job"])
|
|
|
|
|
|
def test_queued_message_ready_and_all_variants(client, actors, room, settings, broker):
|
|
r = upload(client, actors[0], room)
|
|
assert r.status_code == 202 and r.json()["status"] == "queued"
|
|
a = r.json()
|
|
assert not a["variants"] and a["sha256"] == hashlib.sha256(PNG).hexdigest()
|
|
path = "/api/v1/attachments/" + a["id"]
|
|
error(client.get(path + "/content", headers=headers(actors[0])), 409, "conflict")
|
|
key = uuid4().hex
|
|
msg = client.post(
|
|
f"/api/v1/chats/{room['id']}/messages",
|
|
json={"text": "", "attachment_ids": [a["id"]]},
|
|
headers={**headers(actors[0]), "Idempotency-Key": key},
|
|
)
|
|
assert msg.status_code == 201
|
|
for suffix in ("", "/content"):
|
|
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
|
|
error(client.get(path + suffix), 401, "unauthorized")
|
|
process(client, settings, broker)
|
|
ready = client.get(path, headers=headers(actors[1])).json()
|
|
assert ready["status"] == "ready" and ready["error_code"] is None
|
|
assert {v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"}
|
|
assert client.get(path + "/content", headers=headers(actors[1])).content == PNG
|
|
for v in ready["variants"]:
|
|
content = client.get(path + "/content?variant=" + v["name"], headers=headers(actors[1]))
|
|
assert content.status_code == 200 and content.headers["content-type"] == v["content_type"]
|
|
assert len(content.content) == v["size_bytes"]
|
|
assert hashlib.sha256(content.content).hexdigest() == v["sha256"]
|
|
with Image.open(io.BytesIO(content.content)) as image:
|
|
assert image.size == (v["width"], v["height"])
|
|
assert not image.getexif()
|
|
with client.app.state.repository.connection() as conn:
|
|
j = conn.execute(
|
|
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (a["id"],)
|
|
).fetchone()
|
|
job = Job(
|
|
schema_version=1,
|
|
type="attachment.process.v1",
|
|
job_id=str(j["job_id"]),
|
|
attachment_id=a["id"],
|
|
pipeline_version=1,
|
|
attempt=1,
|
|
created_at=a["created_at"],
|
|
)
|
|
sid = broker.publish(job)
|
|
handle(
|
|
client.app.state.repository,
|
|
broker,
|
|
client.app.state.attachments.store,
|
|
settings,
|
|
sid,
|
|
job.model_dump_json(exclude_none=True),
|
|
)
|
|
assert client.get(path, headers=headers(actors[0])).json() == ready
|
|
assert broker.client.xlen(STREAM) == 0
|
|
|
|
|
|
@pytest.mark.parametrize("name", ["", "../a.png", "a\\b.png", "a\x00.png", "\n.png", "x" * 256])
|
|
def test_bad_filename(client, actors, room, name):
|
|
error(upload(client, actors[0], room, name=name), 400, "invalid_request")
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"data,media",
|
|
[
|
|
(b"", "image/png"),
|
|
(b"bad", "image/png"),
|
|
(PNG[:30], "image/png"),
|
|
(PNG, "image/jpeg"),
|
|
(PNG, "text/plain"),
|
|
],
|
|
)
|
|
def test_bad_image(client, actors, room, data, media):
|
|
error(upload(client, actors[0], room, data, media), 415, "unsupported_media_type")
|
|
|
|
|
|
def test_upload_limits_and_pixel_bomb(client, actors, room):
|
|
error(upload(client, actors[0], room, b"x" * (10485760 + 1)), 413, "payload_too_large")
|
|
error(
|
|
upload(client, actors[0], room, (b"x" * 1048576 for _ in range(11))),
|
|
413,
|
|
"payload_too_large",
|
|
)
|
|
data = bytearray(PNG)
|
|
data[16:24] = struct.pack(">II", 10000, 10000)
|
|
data[29:33] = struct.pack(">I", zlib.crc32(data[12:29]) & 0xFFFFFFFF)
|
|
error(upload(client, actors[0], room, bytes(data)), 413, "payload_too_large")
|
|
|
|
|
|
@pytest.mark.parametrize("size", [(320, 200), (17, 11), (1, 400), (400, 1), (1281, 713)])
|
|
def test_transform_dimensions_alpha_and_exif(size):
|
|
raw = io.BytesIO()
|
|
Image.new("RGBA", size, (1, 2, 3, 0)).save(raw, "PNG")
|
|
results = {name: (data, dims, media) for name, data, dims, media in variants(raw, 1)}
|
|
scale = max(size)
|
|
assert results["thumbnail"][1] == tuple(max(1, n * min(256, scale) // scale) for n in size)
|
|
assert results["crop"][1] == (128, 128)
|
|
assert max(results["optimized"][1]) <= 1280
|
|
with Image.open(io.BytesIO(results["thumbnail"][0])) as image:
|
|
assert image.getpixel((0, 0)) == (255, 255, 255)
|
|
|
|
|
|
def test_exif_orientation_and_original_unchanged():
|
|
raw = io.BytesIO()
|
|
exif = Image.Exif()
|
|
exif[274] = 6
|
|
Image.new("RGB", (320, 200), "red").save(raw, "JPEG", exif=exif)
|
|
before = raw.getvalue()
|
|
result = list(variants(raw, 1))
|
|
assert result[0][2] == (160, 256) and raw.getvalue() == before
|
|
for _, data, _, _ in result:
|
|
with Image.open(io.BytesIO(data)) as image:
|
|
assert not image.getexif()
|
|
|
|
|
|
def test_failed_retry_replay_and_ready_replay_rejected(
|
|
client, actors, room, settings, broker, monkeypatch
|
|
):
|
|
a = upload(client, actors[0], room).json()
|
|
store, db = client.app.state.attachments.store, client.app.state.repository
|
|
original = store.put
|
|
monkeypatch.setattr(
|
|
store, "put", lambda *_: (_ for _ in ()).throw(DomainError(503, "unavailable", "test"))
|
|
)
|
|
process(client, replace(settings, job_max_attempts=1), broker)
|
|
path = "/api/v1/attachments/" + a["id"]
|
|
assert client.get(path, headers=headers(actors[0])).json()["error_code"] == "retry_exhausted"
|
|
error(
|
|
client.post(
|
|
f"/api/v1/chats/{room['id']}/messages",
|
|
json={"text": "", "attachment_ids": [a["id"]]},
|
|
headers={**headers(actors[0]), "Idempotency-Key": uuid4().hex},
|
|
),
|
|
409,
|
|
"conflict",
|
|
)
|
|
Pipeline(db, settings).replay(a["id"])
|
|
monkeypatch.setattr(store, "put", original)
|
|
process(client, settings, broker)
|
|
assert client.get(path, headers=headers(actors[0])).json()["status"] == "ready"
|
|
with pytest.raises(ValueError):
|
|
Pipeline(db, settings).replay(a["id"])
|
|
|
|
|
|
def test_invalid_envelope_quarantined(client, settings, broker):
|
|
sid = broker.client.xadd(STREAM, {"job": '{"url":"http://untrusted.invalid/"}'})
|
|
handle(
|
|
client.app.state.repository,
|
|
broker,
|
|
client.app.state.attachments.store,
|
|
settings,
|
|
sid,
|
|
'{"url":"http://untrusted.invalid/"}',
|
|
)
|
|
assert broker.client.xlen("invalid-jobs") == 1
|
|
assert "url" not in json.dumps(broker.client.xrange("invalid-jobs"))
|
|
|
|
|
|
def test_staging_failure_has_no_accepted_job(client, actors, room, monkeypatch):
|
|
store = client.app.state.attachments.store
|
|
monkeypatch.setattr(
|
|
store, "put", lambda *_: (_ for _ in ()).throw(DomainError(503, "unavailable", "test"))
|
|
)
|
|
error(upload(client, actors[0], room), 503, "unavailable")
|
|
with client.app.state.repository.connection() as conn:
|
|
assert conn.execute("SELECT count(*) AS n FROM attachment_jobs").fetchone()["n"] == 0
|
|
|
|
|
|
def test_gc_minimum_retention(client):
|
|
with pytest.raises(ValueError):
|
|
collect(client.app.state.repository, client.app.state.attachments.store, 0, True)
|
|
|
|
|
|
def test_gc_fences_paused_upload_before_job_commit(client, actors, room, settings, monkeypatch):
|
|
db, store = client.app.state.repository, client.app.state.attachments.store
|
|
p = Pipeline(db, settings)
|
|
id = str(uuid4())
|
|
key = "staging/" + id
|
|
p.reserve(id, key)
|
|
store.put(key, io.BytesIO(PNG), len(PNG), "image/png", hashlib.sha256(PNG).hexdigest())
|
|
# Simulated S3 object age; actual private test-bucket deletion and DB locks are real.
|
|
old = datetime.now(UTC) - timedelta(seconds=301)
|
|
head = store.head(key)
|
|
head["LastModified"] = old
|
|
objects = [{"Key": key, "ETag": head["ETag"], "Size": len(PNG), "LastModified": old}]
|
|
monkeypatch.setattr(store, "head", lambda _: head)
|
|
monkeypatch.setattr(
|
|
store.client,
|
|
"get_paginator",
|
|
lambda _: SimpleNamespace(paginate=lambda **_: [{"Contents": objects}]),
|
|
)
|
|
assert collect(db, store, 300, True)["protected"] == 1
|
|
with db.connection(write=True) as conn:
|
|
conn.execute(
|
|
"UPDATE upload_reservations SET expires_at=clock_timestamp()-interval '1 second' "
|
|
"WHERE id=%s",
|
|
(id,),
|
|
)
|
|
assert collect(db, store, 300, False)["eligible"] == 1
|
|
assert collect(db, store, 300, True)["deleted"] == 1
|
|
with pytest.raises(DomainError) as expired:
|
|
p.accept(
|
|
id,
|
|
room["id"],
|
|
actors[0]["id"],
|
|
"paused.png",
|
|
"image/png",
|
|
len(PNG),
|
|
hashlib.sha256(PNG).hexdigest(),
|
|
key,
|
|
)
|
|
assert expired.value.status == 503 and db.attachment(id) is None
|
|
|
|
|
|
def test_version_fence_rejects_old_writer(client, actors, room, settings, broker):
|
|
a = upload(client, actors[0], room).json()
|
|
db, store = client.app.state.repository, client.app.state.attachments.store
|
|
with db.connection() as conn:
|
|
j = conn.execute(
|
|
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (a["id"],)
|
|
).fetchone()
|
|
envelope = Job(
|
|
schema_version=1,
|
|
type="attachment.process.v1",
|
|
job_id=str(j["job_id"]),
|
|
attachment_id=a["id"],
|
|
pipeline_version=1,
|
|
attempt=1,
|
|
created_at=a["created_at"],
|
|
)
|
|
pipeline = Pipeline(db, settings)
|
|
outcome, old = pipeline.claim(envelope)
|
|
assert outcome == "claimed"
|
|
pipeline.replay(a["id"], new_version=True)
|
|
process(client, settings, broker)
|
|
current = db.attachment(a["id"])
|
|
assert current["pipeline_version"] == 2 and current["storage_state"] == "ready"
|
|
assert not pipeline.publish(old, "objects/old-result", [])
|
|
handle(
|
|
db,
|
|
broker,
|
|
store,
|
|
settings,
|
|
broker.publish(envelope),
|
|
envelope.model_dump_json(exclude_none=True),
|
|
)
|
|
assert db.attachment(a["id"])["object_key"] == current["object_key"]
|
|
|
|
|
|
def test_foreign_chat_attachment_is_not_linkable(client, actors, room):
|
|
a = upload(client, actors[0], room).json()
|
|
other = client.post(
|
|
"/api/v1/chats",
|
|
headers=headers(actors[0]),
|
|
json={"title": "other", "member_ids": [actors[0]["id"]]},
|
|
).json()
|
|
error(
|
|
client.post(
|
|
f"/api/v1/chats/{other['id']}/messages",
|
|
headers={**headers(actors[0]), "Idempotency-Key": uuid4().hex},
|
|
json={"text": "", "attachment_ids": [a["id"]]},
|
|
),
|
|
404,
|
|
"not_found",
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"patch",
|
|
[
|
|
{"schema_version": True},
|
|
{"traceparent": None},
|
|
{"tracestate": None},
|
|
{"url": "https://untrusted.invalid/"},
|
|
{"job_id": "not-a-uuid"},
|
|
{"pipeline_version": 0},
|
|
{"attempt": 0},
|
|
{"created_at": "2026-10-17T12:00:00"},
|
|
{"traceparent": "invalid"},
|
|
{"type": "download.anything"},
|
|
],
|
|
)
|
|
def test_job_schema_rejects_invalid_carrier(client, settings, broker, patch):
|
|
payload = {
|
|
"schema_version": 1,
|
|
"type": "attachment.process.v1",
|
|
"job_id": str(uuid4()),
|
|
"attachment_id": str(uuid4()),
|
|
"pipeline_version": 1,
|
|
"attempt": 1,
|
|
"created_at": "2026-10-17T12:00:00Z",
|
|
**patch,
|
|
}
|
|
data = json.dumps(payload)
|
|
sid = broker.client.xadd(STREAM, {"job": data})
|
|
handle(
|
|
client.app.state.repository, broker, client.app.state.attachments.store, settings, sid, data
|
|
)
|
|
assert broker.client.xrange("invalid-jobs")[0][1]["reason"] == "invalid_envelope"
|