366 lines
12 KiB
Python
366 lines
12 KiB
Python
import argparse
|
|
import base64
|
|
import concurrent.futures
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import uuid
|
|
import zlib
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, check, run
|
|
|
|
COMPOSE = [
|
|
"docker",
|
|
"compose",
|
|
"--project-name",
|
|
"highload-lab4",
|
|
"--env-file",
|
|
".env",
|
|
"-f",
|
|
"compose.yml",
|
|
]
|
|
|
|
|
|
def command(args, **kwargs):
|
|
return subprocess.check_output(args, cwd=ROOT, text=True, **kwargs).strip()
|
|
|
|
|
|
def ready(client):
|
|
deadline = time.monotonic() + 90
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
client.call("GET", "/health/ready", schema="Health")
|
|
return
|
|
except Exception:
|
|
time.sleep(1)
|
|
raise AssertionError("readiness did not recover")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--base-url", default="http://localhost:8080")
|
|
parser.add_argument("--failures", action="store_true")
|
|
args = parser.parse_args()
|
|
c = Client(args.base_url)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
png = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
path = "/api/v1/chats/" + chat["id"] + "/attachments"
|
|
|
|
def upload(
|
|
data=png,
|
|
mime="image/png",
|
|
filename="photo.png",
|
|
status=201,
|
|
actor=alice,
|
|
target=path,
|
|
):
|
|
return c.call(
|
|
"POST",
|
|
target + "?" + urllib.parse.urlencode({"filename": filename}),
|
|
status,
|
|
actor=actor,
|
|
raw=data,
|
|
headers={"Content-Type": mime},
|
|
schema="Attachment" if status == 201 else None,
|
|
)
|
|
|
|
jpeg = base64.b64decode(
|
|
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAACAAMDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDj6KKKyPJP/9k="
|
|
)
|
|
jpeg_result = upload(data=jpeg, mime="image/jpeg", filename="photo.jpg")["json"]
|
|
check(
|
|
c.call(
|
|
"GET", "/api/v1/attachments/" + jpeg_result["id"] + "/content", actor=bob
|
|
)["bytes"]
|
|
== jpeg,
|
|
"JPEG changed",
|
|
)
|
|
first = upload()["json"]
|
|
second = upload(filename="Второе фото.png")["json"]
|
|
for a in (first, second):
|
|
content = c.call(
|
|
"GET", "/api/v1/attachments/" + a["id"] + "/content", actor=bob
|
|
)
|
|
check(content["bytes"] == png, "download bytes changed")
|
|
check(a["sha256"] == hashlib.sha256(png).hexdigest(), "checksum differs")
|
|
for suffix in ("", "/content"):
|
|
c.call("GET", "/api/v1/attachments/" + first["id"] + suffix, 404, actor=eve)
|
|
c.call("GET", "/api/v1/attachments/" + first["id"] + suffix, 401)
|
|
c.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + first["id"] + "/content?variant=thumbnail",
|
|
404,
|
|
actor=alice,
|
|
)
|
|
upload(actor=eve, status=404)
|
|
upload(data=b"x" * (10485760 + 1), status=413)
|
|
upload(data=png[:40], status=415)
|
|
upload(mime="image/jpeg", status=415)
|
|
upload(mime="text/plain", status=415)
|
|
for filename in (
|
|
"",
|
|
" ",
|
|
"../file.png",
|
|
"a\\b.png",
|
|
"a\x00.png",
|
|
"a\n.png",
|
|
"x" * 256,
|
|
):
|
|
upload(filename=filename, status=400)
|
|
huge = bytearray(png)
|
|
huge[16:24] = struct.pack(">II", 10001, 4000)
|
|
huge[29:33] = struct.pack(">I", zlib.crc32(huge[12:29]) & 0xFFFFFFFF)
|
|
upload(data=bytes(huge), status=413)
|
|
|
|
# Unknown-length uploads must obey the same byte limit.
|
|
endpoint = urllib.parse.urlsplit(args.base_url)
|
|
conn = http.client.HTTPConnection(
|
|
endpoint.hostname, endpoint.port or 80, timeout=35
|
|
)
|
|
conn.request(
|
|
"POST",
|
|
path + "?filename=chunked.png",
|
|
body=iter([png, b"x" * 10485760]),
|
|
headers={
|
|
"Authorization": "Bearer " + alice["token"],
|
|
"Content-Type": "image/png",
|
|
},
|
|
encode_chunked=True,
|
|
)
|
|
response = conn.getresponse()
|
|
check(response.status == 413, "chunked body limit not enforced")
|
|
response.read()
|
|
conn.close()
|
|
|
|
ids = [first["id"], second["id"]]
|
|
key = uuid.uuid4().hex
|
|
|
|
def send(_):
|
|
return c.message(alice, chat, "", key=key, attachments=ids)
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
|
|
responses = list(pool.map(send, range(24)))
|
|
check(
|
|
len({r["headers"]["X-Instance-Id"] for r in responses}) >= 2,
|
|
"did not use two API replicas",
|
|
)
|
|
check(
|
|
all(r["json"] == responses[0]["json"] for r in responses),
|
|
"replay changed message",
|
|
)
|
|
messages_path = "/api/v1/chats/" + chat["id"] + "/messages"
|
|
history = c.call("GET", messages_path, actor=bob, schema="MessagePage")["json"][
|
|
"items"
|
|
]
|
|
check(
|
|
len(history) == 1 and history[0]["attachment_ids"] == ids,
|
|
"duplicate or lost attachments",
|
|
)
|
|
for body, expected in (
|
|
({"text": "", "attachment_ids": ids[::-1]}, 409),
|
|
({"text": "changed", "attachment_ids": ids}, 409),
|
|
):
|
|
c.call(
|
|
"POST",
|
|
messages_path,
|
|
expected,
|
|
actor=alice,
|
|
body=body,
|
|
headers={"Idempotency-Key": key},
|
|
)
|
|
for bad_ids, expected in (
|
|
([first["id"], first["id"]], 400),
|
|
([str(uuid.uuid4())], 404),
|
|
(["invalid"], 400),
|
|
(None, 400),
|
|
(ids * 6, 400),
|
|
):
|
|
c.call(
|
|
"POST",
|
|
messages_path,
|
|
expected,
|
|
actor=alice,
|
|
body={"text": "x", "attachment_ids": bad_ids},
|
|
headers={"Idempotency-Key": uuid.uuid4().hex},
|
|
)
|
|
other_chat = c.call(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
201,
|
|
actor=alice,
|
|
body={"title": "Other", "member_ids": [alice["id"]]},
|
|
schema="Chat",
|
|
)["json"]
|
|
c.call(
|
|
"POST",
|
|
"/api/v1/chats/" + other_chat["id"] + "/messages",
|
|
404,
|
|
actor=alice,
|
|
body={"text": "", "attachment_ids": [first["id"]]},
|
|
headers={"Idempotency-Key": uuid.uuid4().hex},
|
|
)
|
|
text_key = uuid.uuid4().hex
|
|
text = c.message(alice, chat, "text", key=text_key)["json"]
|
|
check(
|
|
c.message(alice, chat, "text", key=text_key, attachments=[])["json"] == text,
|
|
"omitted/empty array differ",
|
|
)
|
|
print(
|
|
"PASS: image limits, chunked upload, ACL, message attachments, concurrent replay across replicas"
|
|
)
|
|
|
|
if args.failures:
|
|
records = []
|
|
for _ in range(2):
|
|
result = upload()
|
|
target = result["headers"]["X-Instance-Id"]
|
|
a = result["json"]
|
|
check(
|
|
not records or target != records[0]["upload_instance"],
|
|
"repeat did not target the other replica",
|
|
)
|
|
try:
|
|
command(
|
|
[str(ROOT / "scripts/stop-one.sh")],
|
|
env={**os.environ, "TARGET_INSTANCE_ID": target},
|
|
)
|
|
time.sleep(3)
|
|
downloaded = c.call(
|
|
"GET", "/api/v1/attachments/" + a["id"] + "/content", actor=bob
|
|
)
|
|
check(downloaded["bytes"] == png, "original lost across failover")
|
|
served = downloaded["headers"]["X-Instance-Id"]
|
|
check(served != target, "download came from stopped instance")
|
|
records.append(
|
|
{
|
|
"upload_instance": target,
|
|
"download_instance": served,
|
|
"sha256": a["sha256"],
|
|
}
|
|
)
|
|
if len(records) == 1:
|
|
# Upload the next fixture while only the second replica runs.
|
|
# Restore below, then select that same replica via LB uploads.
|
|
next_target = served
|
|
finally:
|
|
command(
|
|
COMPOSE + ["up", "-d", "--no-build", "--wait"],
|
|
stderr=subprocess.STDOUT,
|
|
)
|
|
ready(c)
|
|
if len(records) == 1:
|
|
# Round-robin state may change after recovery; keep requesting until it
|
|
# chooses the other replica, then use that upload in the next pass.
|
|
original_upload = upload
|
|
|
|
def upload(*a, **kw):
|
|
for attempt in range(30):
|
|
candidate = original_upload(*a, **kw)
|
|
if candidate["headers"]["X-Instance-Id"] == next_target:
|
|
return candidate
|
|
raise AssertionError("could not reach second replica")
|
|
|
|
print(
|
|
"PASS: stop the uploading replica, download from the other (both replicas)"
|
|
)
|
|
try:
|
|
command(COMPOSE + ["stop", "s3"], stderr=subprocess.STDOUT)
|
|
# Probe through an existing image on the data network; no API port exposure.
|
|
image = command(
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
"{{.Config.Image}}",
|
|
command(COMPOSE + ["ps", "--all", "-q", "s3"]),
|
|
]
|
|
)
|
|
probe = [
|
|
"docker",
|
|
"run",
|
|
"--rm",
|
|
"--network",
|
|
"highload-lab4_data",
|
|
"--entrypoint",
|
|
"curl",
|
|
image,
|
|
"-s",
|
|
"-o",
|
|
"/dev/null",
|
|
"-w",
|
|
"%{http_code}",
|
|
]
|
|
for container in command(COMPOSE + ["ps", "-q", "api"]).splitlines():
|
|
# Container hostname aliases are not DNS names; use the data-network IP.
|
|
info = json.loads(command(["docker", "inspect", container]))[0]
|
|
ip = info["NetworkSettings"]["Networks"]["highload-lab4_data"][
|
|
"IPAddress"
|
|
]
|
|
check(
|
|
command(probe + [f"http://{ip}:8080/health/live"]) == "200",
|
|
"liveness depends on S3",
|
|
)
|
|
check(
|
|
command(probe + [f"http://{ip}:8080/health/ready"]) == "503",
|
|
"readiness ignores S3 failure",
|
|
)
|
|
records.append({"s3_failure": "live=200 ready=503 on both API replicas"})
|
|
finally:
|
|
command(
|
|
COMPOSE + ["up", "-d", "--no-build", "--wait"], stderr=subprocess.STDOUT
|
|
)
|
|
ready(c)
|
|
check(
|
|
c.call("GET", "/api/v1/attachments/" + first["id"] + "/content", actor=bob)[
|
|
"bytes"
|
|
]
|
|
== png,
|
|
"S3 recovery lost bytes",
|
|
)
|
|
image = command(
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
"{{.Config.Image}}",
|
|
command(COMPOSE + ["ps", "--all", "-q", "s3"]),
|
|
]
|
|
)
|
|
code = command(
|
|
[
|
|
"docker",
|
|
"run",
|
|
"--rm",
|
|
"--network",
|
|
"highload-lab4_data",
|
|
"--entrypoint",
|
|
"curl",
|
|
image,
|
|
"-s",
|
|
"-o",
|
|
"/dev/null",
|
|
"-w",
|
|
"%{http_code}",
|
|
"http://s3:9000/messenger-attachments/originals/" + first["id"],
|
|
]
|
|
)
|
|
check(code == "403", "anonymous object access was permitted")
|
|
records.append({"anonymous_object_get": code})
|
|
directory = ROOT / ".memory/grade3"
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
(directory / "failures.json").write_text(json.dumps(records, indent=2) + "\n")
|
|
print("PASS: S3 unavailable/recovery, private bucket")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|