import argparse import base64 import concurrent.futures import hashlib import http.client import json import os import struct import subprocess import sys import time import urllib.parse import uuid import zlib from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, check, run COMPOSE = [ "docker", "compose", "--project-name", "highload-lab4", "--env-file", ".env", "-f", "compose.yml", ] def command(args, **kwargs): return subprocess.check_output(args, cwd=ROOT, text=True, **kwargs).strip() def ready(client): deadline = time.monotonic() + 90 while time.monotonic() < deadline: try: client.call("GET", "/health/ready", schema="Health") return except Exception: time.sleep(1) raise AssertionError("readiness did not recover") def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default="http://localhost:8080") parser.add_argument("--failures", action="store_true") args = parser.parse_args() c = Client(args.base_url) alice, bob, eve, chat = c.bootstrap() png = (ROOT / "tests/fixtures/sample.png").read_bytes() path = "/api/v1/chats/" + chat["id"] + "/attachments" def upload( data=png, mime="image/png", filename="photo.png", status=201, actor=alice, target=path, ): return c.call( "POST", target + "?" + urllib.parse.urlencode({"filename": filename}), status, actor=actor, raw=data, headers={"Content-Type": mime}, schema="Attachment" if status == 201 else None, ) jpeg = base64.b64decode( "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAACAAMDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDj6KKKyPJP/9k=" ) jpeg_result = upload(data=jpeg, mime="image/jpeg", filename="photo.jpg")["json"] check( c.call( "GET", "/api/v1/attachments/" + jpeg_result["id"] + "/content", actor=bob )["bytes"] == jpeg, "JPEG changed", ) first = upload()["json"] second = upload(filename="Второе фото.png")["json"] for a in (first, second): content = c.call( "GET", "/api/v1/attachments/" + a["id"] + "/content", actor=bob ) check(content["bytes"] == png, "download bytes changed") check(a["sha256"] == hashlib.sha256(png).hexdigest(), "checksum differs") for suffix in ("", "/content"): c.call("GET", "/api/v1/attachments/" + first["id"] + suffix, 404, actor=eve) c.call("GET", "/api/v1/attachments/" + first["id"] + suffix, 401) c.call( "GET", "/api/v1/attachments/" + first["id"] + "/content?variant=thumbnail", 404, actor=alice, ) upload(actor=eve, status=404) upload(data=b"x" * (10485760 + 1), status=413) upload(data=png[:40], status=415) upload(mime="image/jpeg", status=415) upload(mime="text/plain", status=415) for filename in ( "", " ", "../file.png", "a\\b.png", "a\x00.png", "a\n.png", "x" * 256, ): upload(filename=filename, status=400) huge = bytearray(png) huge[16:24] = struct.pack(">II", 10001, 4000) huge[29:33] = struct.pack(">I", zlib.crc32(huge[12:29]) & 0xFFFFFFFF) upload(data=bytes(huge), status=413) # Unknown-length uploads must obey the same byte limit. endpoint = urllib.parse.urlsplit(args.base_url) conn = http.client.HTTPConnection( endpoint.hostname, endpoint.port or 80, timeout=35 ) conn.request( "POST", path + "?filename=chunked.png", body=iter([png, b"x" * 10485760]), headers={ "Authorization": "Bearer " + alice["token"], "Content-Type": "image/png", }, encode_chunked=True, ) response = conn.getresponse() check(response.status == 413, "chunked body limit not enforced") response.read() conn.close() ids = [first["id"], second["id"]] key = uuid.uuid4().hex def send(_): return c.message(alice, chat, "", key=key, attachments=ids) with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool: responses = list(pool.map(send, range(24))) check( len({r["headers"]["X-Instance-Id"] for r in responses}) >= 2, "did not use two API replicas", ) check( all(r["json"] == responses[0]["json"] for r in responses), "replay changed message", ) messages_path = "/api/v1/chats/" + chat["id"] + "/messages" history = c.call("GET", messages_path, actor=bob, schema="MessagePage")["json"][ "items" ] check( len(history) == 1 and history[0]["attachment_ids"] == ids, "duplicate or lost attachments", ) for body, expected in ( ({"text": "", "attachment_ids": ids[::-1]}, 409), ({"text": "changed", "attachment_ids": ids}, 409), ): c.call( "POST", messages_path, expected, actor=alice, body=body, headers={"Idempotency-Key": key}, ) for bad_ids, expected in ( ([first["id"], first["id"]], 400), ([str(uuid.uuid4())], 404), (["invalid"], 400), (None, 400), (ids * 6, 400), ): c.call( "POST", messages_path, expected, actor=alice, body={"text": "x", "attachment_ids": bad_ids}, headers={"Idempotency-Key": uuid.uuid4().hex}, ) other_chat = c.call( "POST", "/api/v1/chats", 201, actor=alice, body={"title": "Other", "member_ids": [alice["id"]]}, schema="Chat", )["json"] c.call( "POST", "/api/v1/chats/" + other_chat["id"] + "/messages", 404, actor=alice, body={"text": "", "attachment_ids": [first["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}, ) text_key = uuid.uuid4().hex text = c.message(alice, chat, "text", key=text_key)["json"] check( c.message(alice, chat, "text", key=text_key, attachments=[])["json"] == text, "omitted/empty array differ", ) print( "PASS: image limits, chunked upload, ACL, message attachments, concurrent replay across replicas" ) if args.failures: records = [] for _ in range(2): result = upload() target = result["headers"]["X-Instance-Id"] a = result["json"] check( not records or target != records[0]["upload_instance"], "repeat did not target the other replica", ) try: command( [str(ROOT / "scripts/stop-one.sh")], env={**os.environ, "TARGET_INSTANCE_ID": target}, ) time.sleep(3) downloaded = c.call( "GET", "/api/v1/attachments/" + a["id"] + "/content", actor=bob ) check(downloaded["bytes"] == png, "original lost across failover") served = downloaded["headers"]["X-Instance-Id"] check(served != target, "download came from stopped instance") records.append( { "upload_instance": target, "download_instance": served, "sha256": a["sha256"], } ) if len(records) == 1: # Upload the next fixture while only the second replica runs. # Restore below, then select that same replica via LB uploads. next_target = served finally: command( COMPOSE + ["up", "-d", "--no-build", "--wait"], stderr=subprocess.STDOUT, ) ready(c) if len(records) == 1: # Round-robin state may change after recovery; keep requesting until it # chooses the other replica, then use that upload in the next pass. original_upload = upload def upload(*a, **kw): for attempt in range(30): candidate = original_upload(*a, **kw) if candidate["headers"]["X-Instance-Id"] == next_target: return candidate raise AssertionError("could not reach second replica") print( "PASS: stop the uploading replica, download from the other (both replicas)" ) try: command(COMPOSE + ["stop", "s3"], stderr=subprocess.STDOUT) # Probe through an existing image on the data network; no API port exposure. image = command( [ "docker", "inspect", "--format", "{{.Config.Image}}", command(COMPOSE + ["ps", "--all", "-q", "s3"]), ] ) probe = [ "docker", "run", "--rm", "--network", "highload-lab4_data", "--entrypoint", "curl", image, "-s", "-o", "/dev/null", "-w", "%{http_code}", ] for container in command(COMPOSE + ["ps", "-q", "api"]).splitlines(): # Container hostname aliases are not DNS names; use the data-network IP. info = json.loads(command(["docker", "inspect", container]))[0] ip = info["NetworkSettings"]["Networks"]["highload-lab4_data"][ "IPAddress" ] check( command(probe + [f"http://{ip}:8080/health/live"]) == "200", "liveness depends on S3", ) check( command(probe + [f"http://{ip}:8080/health/ready"]) == "503", "readiness ignores S3 failure", ) records.append({"s3_failure": "live=200 ready=503 on both API replicas"}) finally: command( COMPOSE + ["up", "-d", "--no-build", "--wait"], stderr=subprocess.STDOUT ) ready(c) check( c.call("GET", "/api/v1/attachments/" + first["id"] + "/content", actor=bob)[ "bytes" ] == png, "S3 recovery lost bytes", ) image = command( [ "docker", "inspect", "--format", "{{.Config.Image}}", command(COMPOSE + ["ps", "--all", "-q", "s3"]), ] ) code = command( [ "docker", "run", "--rm", "--network", "highload-lab4_data", "--entrypoint", "curl", image, "-s", "-o", "/dev/null", "-w", "%{http_code}", "http://s3:9000/messenger-attachments/originals/" + first["id"], ] ) check(code == "403", "anonymous object access was permitted") records.append({"anonymous_object_get": code}) directory = ROOT / ".memory/grade3" directory.mkdir(parents=True, exist_ok=True) (directory / "failures.json").write_text(json.dumps(records, indent=2) + "\n") print("PASS: S3 unavailable/recovery, private bucket") if __name__ == "__main__": run(main)