166 lines
10 KiB
Python
166 lines
10 KiB
Python
import argparse
|
|
import base64
|
|
import concurrent.futures
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import uuid
|
|
import zlib
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, check, run
|
|
|
|
COMPOSE = ["docker", "compose", "--project-name", "highload-lab4", "--env-file", ".env", "-f", "compose.yml"]
|
|
|
|
def command(args, **kwargs):
|
|
return subprocess.check_output(args, cwd=ROOT, text=True, **kwargs).strip()
|
|
|
|
def ready(client):
|
|
deadline = time.monotonic() + 90
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
client.call("GET", "/health/ready", schema="Health")
|
|
return
|
|
except Exception:
|
|
time.sleep(1)
|
|
raise AssertionError("readiness did not recover")
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--base-url", default="http://localhost:8080")
|
|
parser.add_argument("--failures", action="store_true")
|
|
args = parser.parse_args()
|
|
c = Client(args.base_url)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
png = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
path = "/api/v1/chats/" + chat["id"] + "/attachments"
|
|
def upload(data=png, mime="image/png", filename="photo.png", status=201, actor=alice, target=path):
|
|
return c.call("POST", target + "?" + urllib.parse.urlencode({"filename": filename}), status,
|
|
actor=actor, raw=data, headers={"Content-Type": mime}, schema="Attachment" if status == 201 else None)
|
|
jpeg = base64.b64decode('/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAACAAMDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDj6KKKyPJP/9k=')
|
|
jpeg_result = upload(data=jpeg,mime="image/jpeg",filename="photo.jpg")["json"]
|
|
check(c.call("GET", "/api/v1/attachments/"+jpeg_result["id"]+"/content",actor=bob)["bytes"] == jpeg,"JPEG changed")
|
|
first = upload()["json"]
|
|
second = upload(filename="Второе фото.png")["json"]
|
|
for a in (first, second):
|
|
content = c.call("GET", "/api/v1/attachments/" + a["id"] + "/content", actor=bob)
|
|
check(content["bytes"] == png, "download bytes changed")
|
|
check(a["sha256"] == hashlib.sha256(png).hexdigest(), "checksum differs")
|
|
for suffix in ("", "/content"):
|
|
c.call("GET", "/api/v1/attachments/" + first["id"] + suffix, 404, actor=eve)
|
|
c.call("GET", "/api/v1/attachments/" + first["id"] + suffix, 401)
|
|
c.call("GET", "/api/v1/attachments/" + first["id"] + "/content?variant=thumbnail", 404, actor=alice)
|
|
upload(actor=eve, status=404)
|
|
upload(data=b"x" * (10485760 + 1), status=413)
|
|
upload(data=png[:40], status=415)
|
|
upload(mime="image/jpeg", status=415)
|
|
upload(mime="text/plain", status=415)
|
|
for filename in ("", " ", "../file.png", "a\\b.png", "a\x00.png", "a\n.png", "x" * 256):
|
|
upload(filename=filename, status=400)
|
|
huge = bytearray(png)
|
|
huge[16:24] = struct.pack(">II", 10001, 4000)
|
|
huge[29:33] = struct.pack(">I", zlib.crc32(huge[12:29]) & 0xffffffff)
|
|
upload(data=bytes(huge), status=413)
|
|
|
|
# Unknown-length uploads must obey the same byte limit.
|
|
endpoint = urllib.parse.urlsplit(args.base_url)
|
|
conn = http.client.HTTPConnection(endpoint.hostname, endpoint.port or 80, timeout=35)
|
|
conn.request("POST", path + "?filename=chunked.png", body=iter([png, b"x" * 10485760]),
|
|
headers={"Authorization": "Bearer " + alice["token"], "Content-Type": "image/png"}, encode_chunked=True)
|
|
response = conn.getresponse()
|
|
check(response.status == 413, "chunked body limit not enforced")
|
|
response.read(); conn.close()
|
|
|
|
ids = [first["id"], second["id"]]
|
|
key = uuid.uuid4().hex
|
|
def send(_):
|
|
return c.message(alice, chat, "", key=key, attachments=ids)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
|
|
responses = list(pool.map(send, range(24)))
|
|
check(len({r["headers"]["X-Instance-Id"] for r in responses}) >= 2, "did not use two API replicas")
|
|
check(all(r["json"] == responses[0]["json"] for r in responses), "replay changed message")
|
|
messages_path = "/api/v1/chats/" + chat["id"] + "/messages"
|
|
history = c.call("GET", messages_path, actor=bob, schema="MessagePage")["json"]["items"]
|
|
check(len(history) == 1 and history[0]["attachment_ids"] == ids, "duplicate or lost attachments")
|
|
for body, expected in (({"text": "", "attachment_ids": ids[::-1]}, 409),
|
|
({"text": "changed", "attachment_ids": ids}, 409)):
|
|
c.call("POST", messages_path, expected, actor=alice, body=body, headers={"Idempotency-Key": key})
|
|
for bad_ids, expected in (([first["id"], first["id"]], 400), ([str(uuid.uuid4())], 404), (["invalid"], 400), (None, 400), (ids * 6, 400)):
|
|
c.call("POST", messages_path, expected, actor=alice, body={"text": "x", "attachment_ids": bad_ids}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
|
other_chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "Other", "member_ids": [alice["id"]]}, schema="Chat")["json"]
|
|
c.call("POST", "/api/v1/chats/" + other_chat["id"] + "/messages", 404, actor=alice,
|
|
body={"text": "", "attachment_ids": [first["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
|
text_key = uuid.uuid4().hex
|
|
text = c.message(alice, chat, "text", key=text_key)["json"]
|
|
check(c.message(alice, chat, "text", key=text_key, attachments=[])["json"] == text, "omitted/empty array differ")
|
|
print("PASS: image limits, chunked upload, ACL, message attachments, concurrent replay across replicas")
|
|
|
|
if args.failures:
|
|
records = []
|
|
for _ in range(2):
|
|
result = upload()
|
|
target = result["headers"]["X-Instance-Id"]
|
|
a = result["json"]
|
|
check(not records or target != records[0]["upload_instance"], "repeat did not target the other replica")
|
|
try:
|
|
command([str(ROOT / "scripts/stop-one.sh")], env={**os.environ, "TARGET_INSTANCE_ID": target})
|
|
time.sleep(3)
|
|
downloaded = c.call("GET", "/api/v1/attachments/" + a["id"] + "/content", actor=bob)
|
|
check(downloaded["bytes"] == png, "original lost across failover")
|
|
served = downloaded["headers"]["X-Instance-Id"]
|
|
check(served != target, "download came from stopped instance")
|
|
records.append({"upload_instance": target, "download_instance": served, "sha256": a["sha256"]})
|
|
if len(records) == 1:
|
|
# Upload the next fixture while only the second replica runs.
|
|
# Restore below, then select that same replica via LB uploads.
|
|
next_target = served
|
|
finally:
|
|
command(COMPOSE + ["up", "-d", "--no-build", "--wait"], stderr=subprocess.STDOUT)
|
|
ready(c)
|
|
if len(records) == 1:
|
|
# Round-robin state may change after recovery; keep requesting until it
|
|
# chooses the other replica, then use that upload in the next pass.
|
|
original_upload = upload
|
|
def upload(*a, **kw):
|
|
for attempt in range(30):
|
|
candidate = original_upload(*a, **kw)
|
|
if candidate["headers"]["X-Instance-Id"] == next_target:
|
|
return candidate
|
|
raise AssertionError("could not reach second replica")
|
|
print("PASS: stop the uploading replica, download from the other (both replicas)")
|
|
try:
|
|
command(COMPOSE + ["stop", "s3"], stderr=subprocess.STDOUT)
|
|
# Probe through an existing image on the data network; no API port exposure.
|
|
image = command(["docker", "inspect", "--format", "{{.Config.Image}}", command(COMPOSE + ["ps", "--all", "-q", "s3"])])
|
|
probe = ["docker", "run", "--rm", "--network", "highload-lab4_data", "--entrypoint", "curl", image, "-s", "-o", "/dev/null", "-w", "%{http_code}"]
|
|
for container in command(COMPOSE + ["ps", "-q", "api"]).splitlines():
|
|
# Container hostname aliases are not DNS names; use the data-network IP.
|
|
info = json.loads(command(["docker", "inspect", container]))[0]
|
|
ip = info["NetworkSettings"]["Networks"]["highload-lab4_data"]["IPAddress"]
|
|
check(command(probe + [f"http://{ip}:8080/health/live"]) == "200", "liveness depends on S3")
|
|
check(command(probe + [f"http://{ip}:8080/health/ready"]) == "503", "readiness ignores S3 failure")
|
|
records.append({"s3_failure": "live=200 ready=503 on both API replicas"})
|
|
finally:
|
|
command(COMPOSE + ["up", "-d", "--no-build", "--wait"], stderr=subprocess.STDOUT)
|
|
ready(c)
|
|
check(c.call("GET", "/api/v1/attachments/" + first["id"] + "/content", actor=bob)["bytes"] == png, "S3 recovery lost bytes")
|
|
image = command(["docker", "inspect", "--format", "{{.Config.Image}}", command(COMPOSE + ["ps", "--all", "-q", "s3"])])
|
|
code = command(["docker", "run", "--rm", "--network", "highload-lab4_data", "--entrypoint", "curl", image, "-s", "-o", "/dev/null", "-w", "%{http_code}", "http://s3:9000/messenger-attachments/originals/" + first["id"]])
|
|
check(code == "403", "anonymous object access was permitted")
|
|
records.append({"anonymous_object_get": code})
|
|
directory = ROOT / ".memory/grade3"
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
(directory / "failures.json").write_text(json.dumps(records, indent=2) + "\n")
|
|
print("PASS: S3 unavailable/recovery, private bucket")
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|