feat: add direct upload to 5 track
This commit is contained in:
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real presigned POST flow; synthetic data only, never prints signed fields/tokens."""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check, run
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.check_output(
|
||||
["docker", "compose", *args], cwd=ROOT, text=True, stderr=subprocess.STDOUT
|
||||
)
|
||||
|
||||
|
||||
def upload(c, actor, chat, data, concurrent=True):
|
||||
path = f"/api/v1/chats/{chat['id']}/attachment-uploads"
|
||||
result = c.call("POST", path, 201, actor=actor, body={
|
||||
"filename": "direct.png", "content_type": "image/png",
|
||||
"size_bytes": len(data), "sha256": hashlib.sha256(data).hexdigest(),
|
||||
})
|
||||
instruction = json.loads(result["bytes"])
|
||||
boundary = "direct-" + uuid.uuid4().hex
|
||||
body = bytearray()
|
||||
for key, value in instruction["upload"]["fields"].items():
|
||||
body.extend(f'--{boundary}\r\nContent-Disposition: form-data; name="{key}"\r\n\r\n{value}\r\n'.encode())
|
||||
body.extend(f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="direct.png"\r\nContent-Type: image/png\r\n\r\n'.encode())
|
||||
body.extend(data)
|
||||
body.extend(f"\r\n--{boundary}--\r\n".encode())
|
||||
req = urllib.request.Request(instruction["upload"]["url"], data=bytes(body), method="POST", headers={"Content-Type": "multipart/form-data; boundary=" + boundary})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=25) as response:
|
||||
check(response.status == 204, "S3 POST did not return 204")
|
||||
except Exception:
|
||||
raise Failure("S3 POST failed (request details suppressed)") from None
|
||||
endpoint = path + "/" + instruction["upload_id"] + "/complete"
|
||||
if not concurrent:
|
||||
return c.call("POST", endpoint, 201, actor=actor, schema="Attachment"), endpoint
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
futures = [pool.submit(c.call, "POST", endpoint, 201, actor=actor, schema="Attachment") for _ in range(2)]
|
||||
results = [future.result() for future in futures]
|
||||
check(results[0]["json"] == results[1]["json"], "concurrent complete returned different metadata")
|
||||
instances = {r["headers"]["X-Instance-Id"] for r in results}
|
||||
check(len(instances) == 2, "complete did not exercise both API replicas")
|
||||
return results[0], endpoint
|
||||
|
||||
|
||||
def wait_content(c, actor, attachment, data, excluded=None):
|
||||
deadline = time.monotonic() + 30
|
||||
while True:
|
||||
try:
|
||||
response = c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=actor)
|
||||
check(response["bytes"] == data, "download differs from original")
|
||||
if excluded:
|
||||
check(response["headers"]["X-Instance-Id"] != excluded, "stopped replica served download")
|
||||
return
|
||||
except Failure:
|
||||
if time.monotonic() >= deadline:
|
||||
raise
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://localhost:8080")
|
||||
parser.add_argument("--failures", action="store_true", help="stop each confirming replica in turn, restoring it afterwards")
|
||||
parser.add_argument("--restart", action="store_true", help="restart this lab using scripts/restart.sh")
|
||||
args = parser.parse_args()
|
||||
c = Client(args.base_url, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
data = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
confirmed, endpoint = upload(c, alice, chat, data)
|
||||
attachment = confirmed["json"]
|
||||
c.call("GET", f"/api/v1/attachments/{attachment['id']}", 404, actor=eve)
|
||||
wait_content(c, bob, attachment, data)
|
||||
c.message(alice, chat, "", attachments=[attachment["id"]])
|
||||
if args.failures:
|
||||
ids = compose("ps", "-q", "api").splitlines()
|
||||
check(len(ids) == 2, "expected two API replicas")
|
||||
for cid in ids:
|
||||
hostname = json.loads(subprocess.check_output(["docker", "inspect", cid], text=True))[0]["Config"]["Hostname"]
|
||||
# Force complete onto the replica about to stop by stopping its peer.
|
||||
peer = next(other for other in ids if other != cid)
|
||||
try:
|
||||
subprocess.run(["docker", "stop", peer], check=True, stdout=subprocess.DEVNULL)
|
||||
# Wait for HAProxy to mark the stopped peer down.
|
||||
deadline = time.monotonic() + 20
|
||||
while True:
|
||||
try:
|
||||
fresh, _ = upload(c, alice, chat, data, concurrent=False)
|
||||
check(fresh["headers"]["X-Instance-Id"] == hostname, "wrong confirming replica")
|
||||
break
|
||||
except Failure:
|
||||
if time.monotonic() >= deadline: raise
|
||||
time.sleep(0.2)
|
||||
compose("up", "-d", "--no-deps", "--no-build", "--wait", "api")
|
||||
subprocess.run(["docker", "stop", cid], check=True, stdout=subprocess.DEVNULL)
|
||||
wait_content(c, bob, fresh["json"], data, excluded=hostname)
|
||||
print("PASS: original survives stopping confirming replica " + hostname)
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "--no-build", "--wait", "api")
|
||||
if args.restart:
|
||||
subprocess.run([str(ROOT / "scripts/restart.sh")], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
|
||||
wait_content(c, bob, attachment, data)
|
||||
again = c.call("POST", endpoint, 201, actor=alice, schema="Attachment")
|
||||
check(again["json"] == attachment, "restart changed direct complete result")
|
||||
print("PASS: session, direct upload receipt, metadata and bytes survive restart")
|
||||
print("PASS: real S3 form, concurrent complete through two replicas, authorized bytes and attachment-only message")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -22,12 +22,14 @@ def main():
|
||||
if mount["Type"] == "bind"
|
||||
]
|
||||
if service == "s3":
|
||||
assert not info["HostConfig"]["PortBindings"], "S3 ports are published"
|
||||
assert info["HostConfig"]["PortBindings"] == {
|
||||
"9000/tcp": [{"HostIp": "127.0.0.1", "HostPort": "9000"}]
|
||||
}, "only loopback S3 port may be published"
|
||||
assert "MINIO_BROWSER=off" in info["Config"]["Env"], (
|
||||
"MinIO console is enabled"
|
||||
)
|
||||
assert set(info["NetworkSettings"]["Networks"]) == {
|
||||
"highload-lab4_data"
|
||||
"highload-lab4_data", "highload-lab4_edge"
|
||||
}
|
||||
else:
|
||||
assert (
|
||||
@@ -56,7 +58,7 @@ def main():
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
(directory / "isolation.json").write_text(json.dumps(evidence, indent=2) + "\n")
|
||||
print(
|
||||
"PASS: private S3 network, disabled console, restricted API mounts, bounded tmpfs"
|
||||
"PASS: loopback-only private S3, disabled console, restricted API mounts, bounded tmpfs"
|
||||
)
|
||||
|
||||
|
||||
|
||||
+3
-1
@@ -26,7 +26,9 @@ def main():
|
||||
POSTGRES_USER="messenger",
|
||||
POSTGRES_PASSWORD=secrets.token_hex(24),
|
||||
)
|
||||
defaults.update(UPLOAD_CLEANUP_GRACE_SECONDS="600", UPLOAD_TMP_DIR="/tmp/uploads")
|
||||
defaults.update(UPLOAD_CLEANUP_GRACE_SECONDS="600", UPLOAD_TMP_DIR="/tmp/uploads",
|
||||
S3_PUBLIC_ENDPOINT="http://127.0.0.1:9000",
|
||||
S3_CORS_ORIGINS="http://127.0.0.1:5173,http://localhost:5173")
|
||||
values = defaults | existing
|
||||
for name, lower, upper in (
|
||||
("APP_PORT", 1, 65535),
|
||||
|
||||
Reference in New Issue
Block a user