"""Attachments through the load balancer (lab 4, levels 3 and 4; lab 5: upload is 202 queued and the worker publishes the original + variants): upload/download, ACL, limits, content checks, filename rules and messages with attachment_ids. Negative inputs include every case the reviewer used against other lab 4 solutions: corrupted IDAT with valid CRCs, the first 33 bytes of the fixture, JPEG without scan data, a 20000x10000 header (DecompressionBomb), exactly 10 MiB through the LB, C1 control U+0085 and a whitespace filename, text=null / missing text / attachment_ids=null in a message; and the lab 4/5 review cases: PNG row filter 5 and 0xff, IDAT that is not a zlib stream, the 27-byte JPEG without tables and scan data, a JPEG whose SOS names a component missing from SOF - all 415 AT UPLOAD, never 202 followed by failed/invalid_image.""" from __future__ import annotations import hashlib import subprocess import time import uuid from pathlib import Path import pytest import imagegen from conftest import attachment_row, compose, s3_size pytestmark = pytest.mark.stack ROOT = Path(__file__).resolve().parents[1] SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() MiB = 1024 * 1024 @pytest.fixture() def trio(api): """alice + bob share a chat; eve is an outsider with her own chat.""" alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve") return alice, bob, eve, api.chat(alice, bob), api.chat(eve) def content(api, actor, attachment_id, expected=200, query=""): return api.call("GET", f"/api/v1/attachments/{attachment_id}/content{query}", expected, token=actor["token"]) # ------------------------------------------------------------------ positive path def test_png_and_jpeg_roundtrip_byte_for_byte(api, trio): alice, bob, _, chat, _ = trio jpg = imagegen.jpeg(321, 123, noise=True) for data, ctype in ((SAMPLE, "image/png"), (jpg, "image/jpeg")): att = api.upload(alice, chat, data, content_type=ctype, filename="кот 😺.bin").json() assert att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"] assert att["size_bytes"] == len(data) and att["sha256"] == hashlib.sha256(data).hexdigest() assert att["content_type"] == ctype and att["filename"] == "кот 😺.bin" meta = api.wait_status(bob, att["id"], "ready") assert {k: v for k, v in meta.items() if k not in ("status", "variants", "updated_at")} == \ {k: v for k, v in att.items() if k not in ("status", "variants", "updated_at")} assert [v["name"] for v in meta["variants"]] == ["thumbnail", "crop", "optimized"] got = content(api, bob, att["id"]) assert got.content == data and got.headers["content-type"] == ctype assert got.headers["content-length"] == str(len(data)) assert got.headers["x-content-type-options"] == "nosniff" assert got.headers["content-disposition"].startswith("attachment;") assert "location" not in got.headers # bytes through the API, never a redirect to S3 def test_metadata_reveals_no_storage_details(api, trio): alice, _, _, chat, _ = trio att = api.upload(alice, chat, SAMPLE).json() api.wait_status(alice, att["id"], "ready") row = attachment_row(att["id"]) assert row["status"] == "ready" and row["object_key"].startswith(f"a/{att['id']}/") assert row["staging_key"].startswith("s/") text = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=alice["token"]).text for secret in (row["object_key"], row["staging_key"], "messenger-attachments", "messenger-staging", "a/", "s3"): assert secret not in text.lower(), secret # The staging key is random: neither the id nor the filename is part of it. assert att["id"].replace("-", "") not in row["staging_key"] and "photo" not in row["staging_key"] assert s3_size(row["object_key"]) == len(SAMPLE) # the published original IS in the final bucket def test_original_is_not_stored_on_api_containers(api, trio): """The API keeps no file of the original: no recently written file of that size anywhere in the replica's filesystem (the bytes live only in the bucket).""" alice, _, _, chat, _ = trio data = imagegen.png_of_size(300_000) api.upload(alice, chat, data) size = len(data) for cid in compose("ps", "-q", "api").stdout.split(): out = subprocess.run( ["docker", "exec", cid, "sh", "-c", f"find / -xdev -type f -size {size}c -mmin -10 2>/dev/null | head"], capture_output=True, text=True, timeout=60).stdout.strip() assert out == "", out # ------------------------------------------------------------------ access control def test_outsider_and_unknown_get_404(api, trio): alice, _, eve, chat, _ = trio att = api.upload(alice, chat, SAMPLE).json() for suffix in ("", "/content"): api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"]) api.call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"]) api.call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"]) api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401) # Upload into a foreign chat: 404 before anything about the body is looked at. api.upload(eve, chat, SAMPLE, 404) api.upload(eve, chat, b"garbage", 404, content_type="text/plain") def test_attachment_ids_are_checked_per_chat(api, trio): alice, bob, eve, chat, eve_chat = trio other = api.chat(alice) # alice's second chat att_other = api.upload(alice, other, SAMPLE).json() att_eve = api.upload(eve, eve_chat, SAMPLE).json() # Alice is a member of both chats, but a file is bound to its chat. api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_other["id"]]}, 404) api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404) api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404) # Eve cannot post into alice's chat even with her own file: membership first. api.post_message(eve, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404) # Any member may reference an attachment of the chat. att = api.upload(alice, chat, SAMPLE).json() msg = api.post_message(bob, chat, {"text": "", "attachment_ids": [att["id"]]}).json() assert msg["attachment_ids"] == [att["id"]] and msg["sender_id"] == bob["id"] # ------------------------------------------------------------------ messages with attachments def test_message_with_attachments_and_history(api, trio): alice, bob, _, chat, _ = trio a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2)) m1 = api.post_message(alice, chat, {"text": "", "attachment_ids": [a2, a1]}).json() m2 = api.post_message(alice, chat, {"text": " ", "attachment_ids": [a1]}).json() # blank text + file: ok m3 = api.post_message(alice, chat, {"text": "без файлов"}).json() assert m1["attachment_ids"] == [a2, a1] and m1["text"] == "" assert m2["text"] == " " and m3["attachment_ids"] == [] items, _ = api.history(bob, chat, 2) assert [m["attachment_ids"] for m in items] == [[a2, a1], [a1], []] def test_idempotent_replay_with_attachments(api, trio): alice, _, _, chat, _ = trio a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2)) key = uuid.uuid4().hex first = api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a1, a2]}, key=key).json() raw = ('{"attachment_ids":["%s","%s"],"text":"файлы"}' % (a1, a2)).encode() assert api.post_message(alice, chat, None, raw=raw, key=key).json() == first # key order irrelevant api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a2, a1]}, 409, key=key) # other body api.post_message(alice, chat, {"text": "файлы"}, 409, key=key) # A failed attachment check does not burn the key: the same key works with a valid body later. key2 = uuid.uuid4().hex api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404, key=key2) api.post_message(alice, chat, {"text": "x", "attachment_ids": [a1]}, key=key2) @pytest.mark.parametrize("raw", [ b'{"attachment_ids":["%(a)s"]}', # text missing b'{"text":null,"attachment_ids":["%(a)s"]}', # text null b'{"text":1,"attachment_ids":["%(a)s"]}', b'{"text":"x","attachment_ids":null}', # attachment_ids null b'{"text":"x","attachment_ids":"%(a)s"}', b'{"text":"x","attachment_ids":{}}', b'{"text":"x","attachment_ids":["%(a)s","%(a)s"]}', # duplicates b'{"text":"x","attachment_ids":["%(a)s",1]}', b'{"text":"x","attachment_ids":["not-a-uuid"]}', b'{"text":"","attachment_ids":[]}', # neither text nor files b'{"text":" "}', b'{"text":"x","attachment_ids":[],"sender_id":"%(a)s"}', ], ids=lambda raw: raw.decode()[:40]) def test_message_body_validation(api, trio, raw): alice, _, _, chat, _ = trio a = api.upload(alice, chat, SAMPLE).json()["id"] api.post_message(alice, chat, None, 400, raw=raw % {b"a": a.encode()}) def test_at_most_ten_attachments(api, trio): alice, _, _, chat, _ = trio ids = [api.upload(alice, chat, imagegen.png(4, 4)).json()["id"] for _ in range(11)] assert len(api.post_message(alice, chat, {"text": "", "attachment_ids": ids[:10]}).json()["attachment_ids"]) == 10 api.post_message(alice, chat, {"text": "", "attachment_ids": ids}, 400) # ------------------------------------------------------------------ limits def test_size_limit_through_the_load_balancer(api, trio): alice, bob, _, chat, _ = trio exact = imagegen.png_exactly(10 * MiB) assert len(exact) == 10 * MiB att = api.upload(alice, chat, exact).json() # exactly the limit: accepted via LB api.wait_status(bob, att["id"], "ready") assert content(api, bob, att["id"]).content == exact started = time.monotonic() api.upload(alice, chat, exact + b"\x00", 413) # Content-Length over the limit assert time.monotonic() - started < 5 over = exact + bytes(1024) def chunks(): # no Content-Length: chunked body for i in range(0, len(over), 256 * 1024): yield over[i:i + 256 * 1024] error = api.upload(alice, chat, chunks(), 413).json() assert error["error"]["code"] == "payload_too_large" def test_pixel_limit_before_decoding(api, trio): alice, _, _, chat, _ = trio for data, ctype in ((imagegen.png_bomb_header(), "image/png"), (imagegen.jpeg_bomb_header(), "image/jpeg"), (imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), "image/png")): started = time.monotonic() assert api.upload(alice, chat, data, 413, content_type=ctype).json()["error"]["code"] == "payload_too_large" assert time.monotonic() - started < 2 # refused from the header, nothing decoded def test_exactly_40_megapixels_is_accepted(api, trio): alice, _, _, chat, _ = trio data = imagegen.png(8000, 5000, compress_level=9) # 40 000 000 px, ~120 KB file att = api.upload(alice, chat, data).json() assert att["size_bytes"] == len(data) # Lab 5: the worker processes the biggest allowed picture within its memory limit. ready = api.wait_status(alice, att["id"], "ready", timeout=120) assert {v["name"]: (v["width"], v["height"]) for v in ready["variants"]} == \ {"thumbnail": (256, 160), "crop": (128, 128), "optimized": (1280, 800)} # ------------------------------------------------------------------ content checks BROKEN = { "png: first 33 bytes of the fixture": (SAMPLE[:33], "image/png"), "png: truncated": (SAMPLE[:-20], "image/png"), "png: corrupt IDAT, valid CRC": (imagegen.png_corrupt_idat(), "image/png"), "png: fewer rows than IHDR": (imagegen.png_short_pixels(), "image/png"), "png: flipped byte": (SAMPLE[:700] + bytes([SAMPLE[700] ^ 0x5A]) + SAMPLE[701:], "image/png"), "png: bytes after IEND": (SAMPLE + b"", "image/png"), "jpeg: no scan data after SOS": (imagegen.jpeg_without_scan_data(), "image/jpeg"), "jpeg: truncated": (imagegen.jpeg(400, 300, noise=True)[:5000], "image/jpeg"), "review: png row filter 5": (imagegen.png_bad_filter(5), "image/png"), "review: png row filter 0xff": (imagegen.png_bad_filter(0xFF), "image/png"), "review: png IDAT not-a-zlib-stream": (imagegen.png_idat_not_zlib(), "image/png"), "review: jpeg 27 bytes, no tables/scan": (imagegen.JPEG_27_BYTES, "image/jpeg"), "review: jpeg SOS component not in SOF": (imagegen.jpeg_sos_unknown_component(), "image/jpeg"), "mime: png bytes declared jpeg": (SAMPLE, "image/jpeg"), "mime: jpeg bytes declared png": (imagegen.jpeg(), "image/png"), "mime: gif declared png": (b"GIF89a" + bytes(100), "image/png"), "mime: html declared png": (b"", "image/png"), "type: text/plain": (SAMPLE, "text/plain"), "type: octet-stream": (SAMPLE, "application/octet-stream"), "type: multipart": (SAMPLE, "multipart/form-data; boundary=x"), "type: missing": (SAMPLE, None), } @pytest.mark.parametrize("name", list(BROKEN)) def test_bad_content_is_415(api, trio, name): alice, _, _, chat, _ = trio data, ctype = BROKEN[name] error = api.upload(alice, chat, data, 415, content_type=ctype).json() assert error["error"]["code"] == "unsupported_media_type" def test_empty_body_is_400(api, trio): alice, _, _, chat, _ = trio api.upload(alice, chat, b"", 400) # ------------------------------------------------------------------ filename @pytest.mark.parametrize("query", [ "", "filename=", "filename=%20%20%20", "filename=.", "filename=..", "filename=a%2Fb.png", "filename=a%5Cb.png", "filename=a%00b.png", "filename=c1%C2%85.png", "filename=tab%09.png", "filename=photo%E2%80%AEgnp.exe", "filename=%FF%FE.png", "filename=a.png&filename=b.png", "filename=" + "a" * 256, ], ids=lambda q: q[:40] or "missing") def test_bad_filename_is_400(api, trio, query): alice, _, _, chat, _ = trio api.upload(alice, chat, SAMPLE, 400, query=query) def test_unicode_filename_is_kept_and_safely_exposed(api, trio): alice, _, _, chat, _ = trio name = 'отчёт "Q3"; ok.png' att = api.upload(alice, chat, SAMPLE, filename=name).json() assert att["filename"] == name api.wait_status(alice, att["id"], "ready") disposition = content(api, alice, att["id"]).headers["content-disposition"] assert '"' not in disposition.split('filename="')[1].split('";')[0] # quotes neutralised in the fallback assert "filename*=UTF-8''" in disposition # ------------------------------------------------------------------ variants def test_variant_parameter(api, trio): alice, _, eve, chat, _ = trio att = api.upload(alice, chat, SAMPLE).json() api.wait_status(alice, att["id"], "ready") assert content(api, alice, att["id"], query="?variant=original").content == SAMPLE assert content(api, alice, att["id"], query="?variant=thumbnail").headers["content-type"] == "image/jpeg" content(api, alice, att["id"], 400, query="?variant=huge") content(api, alice, att["id"], 400, query="?variant=thumbnail&variant=crop") content(api, eve, att["id"], 404, query="?variant=thumbnail") # outsider: 404, nothing revealed content(api, eve, att["id"], 404, query="?variant=huge") # outsider first, then the value