forked from mtusi2026/mtusi2026lab4
Негативные загрузки, приватный bucket, очистка orphan и пик памяти при десяти одновременных upload лежат рядом с командами, которыми это воспроизводится.
232 lines
7.9 KiB
Python
232 lines
7.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Private bucket, orphan grace, and a bounded concurrent-upload memory sample."""
|
|
|
|
import json
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.request
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
OUT = ROOT / "evidence" / "grade4-infra.json"
|
|
|
|
|
|
def compose(*args, input_text=None):
|
|
result = subprocess.run(
|
|
["docker", "compose", "--env-file", ".env", *args],
|
|
cwd=ROOT,
|
|
input=input_text,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
if result.returncode != 0:
|
|
raise SystemExit(result.stderr[-500:] or result.stdout[-500:] or "compose failed")
|
|
return result.stdout
|
|
|
|
|
|
def anonymous_status():
|
|
script = r"""
|
|
code=$(wget -S -O /dev/null http://127.0.0.1:9000/messenger-attachments/ 2>&1 | awk 'BEGIN{c=""} /^[[:space:]]*HTTP\//{c=$2} END{print c}')
|
|
printf '%s' "$code"
|
|
"""
|
|
output = compose("exec", "-T", "s3", "sh", "-c", script).strip()
|
|
if output != "403":
|
|
raise SystemExit("anonymous bucket status %s" % output)
|
|
return output
|
|
|
|
|
|
def other_bucket_denied():
|
|
code = r"""
|
|
import os
|
|
import boto3
|
|
from botocore.client import Config
|
|
from botocore.exceptions import ClientError
|
|
client = boto3.client(
|
|
"s3",
|
|
endpoint_url=os.environ["S3_ENDPOINT"],
|
|
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],
|
|
aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
|
|
region_name=os.environ.get("S3_REGION", "us-east-1"),
|
|
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
|
)
|
|
try:
|
|
client.create_bucket(Bucket="not-the-messenger-bucket")
|
|
print("allowed")
|
|
except ClientError as exc:
|
|
print(exc.response.get("Error", {}).get("Code", "denied"))
|
|
"""
|
|
output = compose("exec", "-T", "api", "python", "-c", code).strip()
|
|
if output == "allowed":
|
|
raise SystemExit("app credentials created a second bucket")
|
|
return output
|
|
|
|
|
|
def plant_and_sweep():
|
|
code = r"""
|
|
import asyncio
|
|
from datetime import datetime, timedelta, timezone
|
|
from app import config, db, s3
|
|
|
|
async def main():
|
|
await db.connect()
|
|
old_key = "originals/orphan-evidence"
|
|
fresh_key = "originals/fresh-evidence"
|
|
payload = b"not-a-real-image"
|
|
await s3.put_bytes(old_key, payload, "application/octet-stream")
|
|
await s3.put_bytes(fresh_key, payload, "application/octet-stream")
|
|
old = datetime.now(timezone.utc) - timedelta(seconds=config.ORPHAN_GRACE_SECONDS + 60)
|
|
fresh = datetime.now(timezone.utc)
|
|
async with db.require().acquire() as conn:
|
|
await conn.execute("DELETE FROM blob_writes WHERE object_key = ANY($1::text[])", [old_key, fresh_key])
|
|
await conn.execute(
|
|
"INSERT INTO blob_writes (object_key, committed, created_at) VALUES ($1, FALSE, $2), ($3, FALSE, $4)",
|
|
old_key, old, fresh_key, fresh,
|
|
)
|
|
from app.attachments import sweep_orphans
|
|
deleted = await sweep_orphans()
|
|
async def exists(key):
|
|
try:
|
|
await s3.get_bytes(key)
|
|
return True
|
|
except Exception:
|
|
return False
|
|
print("%s %s %s" % (deleted, int(await exists(old_key)), int(await exists(fresh_key))))
|
|
await s3.delete_key(fresh_key)
|
|
async with db.require().acquire() as conn:
|
|
await conn.execute("DELETE FROM blob_writes WHERE object_key = $1 AND NOT committed", fresh_key)
|
|
await db.close()
|
|
|
|
asyncio.run(main())
|
|
"""
|
|
output = compose("exec", "-T", "api", "python", "-c", code).strip().split()
|
|
deleted, old_exists, fresh_exists = int(output[0]), int(output[1]), int(output[2])
|
|
if deleted < 1 or old_exists != 0 or fresh_exists != 1:
|
|
raise SystemExit("sweep result deleted=%s old=%s fresh=%s" % (deleted, old_exists, fresh_exists))
|
|
return {"deleted": deleted, "old_removed": True, "fresh_kept": True}
|
|
|
|
|
|
def memory_sample():
|
|
import base64
|
|
|
|
username = "mem_" + uuid.uuid4().hex[:10]
|
|
password = "T3st-" + uuid.uuid4().hex
|
|
body = json.dumps({"username": username, "display_name": "Mem", "password": password}).encode()
|
|
|
|
def call(method, path, data=None, headers=None):
|
|
request = urllib.request.Request("http://127.0.0.1:8080" + path, data=data, headers=headers or {}, method=method)
|
|
try:
|
|
response = urllib.request.urlopen(request, timeout=60)
|
|
except urllib.error.HTTPError as error:
|
|
response = error
|
|
with response:
|
|
return response.status, response.read()
|
|
|
|
status, raw = call("POST", "/api/v1/users", body, {"Content-Type": "application/json"})
|
|
if status != 201:
|
|
raise SystemExit("memory user %s" % status)
|
|
user = json.loads(raw)
|
|
basic = base64.b64encode((username + ":" + password).encode()).decode()
|
|
status, raw = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
|
if status != 201:
|
|
raise SystemExit("memory login %s" % status)
|
|
token = json.loads(raw)["token"]
|
|
status, raw = call(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
json.dumps({"title": "mem", "member_ids": [user["id"]]}).encode(),
|
|
{"Content-Type": "application/json", "Authorization": "Bearer " + token},
|
|
)
|
|
if status != 201:
|
|
raise SystemExit("memory chat %s" % status)
|
|
chat_id = json.loads(raw)["id"]
|
|
import os
|
|
import struct
|
|
import zlib
|
|
|
|
def png():
|
|
signature = b"\x89PNG\r\n\x1a\n"
|
|
|
|
def chunk(kind, data):
|
|
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
|
|
|
width, height = 320, 240
|
|
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
|
raw_rows = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
|
|
return signature + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw_rows, 1)) + chunk(b"IEND", b"")
|
|
|
|
blob = png()
|
|
codes = []
|
|
|
|
def upload():
|
|
status, _ = call(
|
|
"POST",
|
|
"/api/v1/chats/%s/attachments?filename=m.png" % chat_id,
|
|
blob,
|
|
{"Content-Type": "image/png", "Authorization": "Bearer " + token},
|
|
)
|
|
codes.append(status)
|
|
|
|
samples = []
|
|
|
|
def watch():
|
|
end = time.monotonic() + 8
|
|
while time.monotonic() < end:
|
|
stats = subprocess.run(
|
|
["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
for line in stats.stdout.splitlines():
|
|
if "mtusi2026lab4-api-" in line:
|
|
samples.append(line.strip())
|
|
time.sleep(0.4)
|
|
|
|
watcher = threading.Thread(target=watch)
|
|
watcher.start()
|
|
threads = [threading.Thread(target=upload) for _ in range(10)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
watcher.join()
|
|
if codes.count(201) != 10:
|
|
raise SystemExit("concurrent uploads %s" % codes)
|
|
|
|
def mib(line):
|
|
value = line.split()[1]
|
|
number = float(value[:-3]) if value.lower().endswith("mib") else float(value[:-3]) / 1024
|
|
return number
|
|
|
|
peak = max(mib(line) for line in samples) if samples else 0
|
|
return {"uploads": 10, "bytes_each": len(blob), "success": 10, "peak_api_mib": round(peak, 1), "samples": len(samples)}
|
|
|
|
|
|
def main():
|
|
published = subprocess.run(
|
|
["docker", "compose", "--env-file", ".env", "port", "s3", "9000"],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if published.returncode == 0 and published.stdout.strip():
|
|
raise SystemExit("s3 port is published: " + published.stdout.strip())
|
|
summary = {
|
|
"anonymous_bucket_status": anonymous_status(),
|
|
"second_bucket": other_bucket_denied(),
|
|
"sweep": plant_and_sweep(),
|
|
"s3_port_published": False,
|
|
"memory": memory_sample(),
|
|
}
|
|
OUT.parent.mkdir(exist_ok=True)
|
|
OUT.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
|
print(json.dumps(summary, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import urllib.error
|
|
|
|
main()
|