Зафиксировать проверки уровня 4 и отчёт.

Негативные загрузки, приватный bucket, очистка orphan и пик памяти при десяти одновременных upload лежат рядом с командами, которыми это воспроизводится.
This commit is contained in:
2026-11-09 00:25:42 +03:00
parent 00445bc41b
commit 8faf08d8eb
7 changed files with 643 additions and 28 deletions
+304
View File
@@ -0,0 +1,304 @@
#!/usr/bin/env python3
"""Negative attachment cases. Prints statuses only, never tokens."""
import base64
import hashlib
import json
import socket
import struct
import uuid
import urllib.error
import urllib.request
import zlib
from pathlib import Path
BASE = "http://127.0.0.1:8080"
ROOT = Path(__file__).resolve().parents[1]
JPEG = base64.b64decode(
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAARCAAIAAgDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD5soooroMD/9k="
)
def header_only_png(width: int, height: int) -> bytes:
signature = b"\x89PNG\r\n\x1a\n"
def chunk(kind: bytes, data: bytes) -> bytes:
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return signature + chunk(b"IHDR", ihdr) + chunk(b"IEND", b"")
def png(width: int, height: int) -> bytes:
signature = b"\x89PNG\r\n\x1a\n"
def chunk(kind: bytes, data: bytes) -> bytes:
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
raw = b"".join(b"\x00" + (b"\x10\x20\x30" * width) for _ in range(height))
return signature + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")
class Api:
def __init__(self) -> None:
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
def call(self, method, path, body=None, token=None, headers=None, raw=None, content_type=None):
hdrs = {}
data = raw
if body is not None:
data = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if content_type:
hdrs["Content-Type"] = content_type
if token:
hdrs["Authorization"] = "Bearer " + token
if headers:
hdrs.update(headers)
request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method)
try:
response = self.opener.open(request, timeout=30)
except urllib.error.HTTPError as error:
response = error
with response:
payload = response.read()
parsed = None
if payload and response.headers.get_content_type() == "application/json":
parsed = json.loads(payload)
return response.status, response.headers, parsed, payload
def expect(status, got, label):
if got == 500:
raise SystemExit("FAIL %s returned 500" % label)
if got != status:
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
print("PASS", label, got)
def user(api: Api, label: str):
username = "g4_" + label + "_" + uuid.uuid4().hex[:10]
password = "T3st-" + uuid.uuid4().hex
status, _, body, _ = api.call(
"POST",
"/api/v1/users",
{"username": username, "display_name": "Grade " + label, "password": password},
)
expect(201, status, "register " + label)
basic = base64.b64encode((username + ":" + password).encode()).decode()
status, _, session, _ = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
expect(201, status, "login " + label)
return body["id"], session["token"]
def declared_length(token, chat_id):
request = (
"POST /api/v1/chats/%s/attachments?filename=big.png HTTP/1.1\r\n"
"Host: 127.0.0.1:8080\r\n"
"Authorization: Bearer %s\r\n"
"Content-Type: image/png\r\n"
"Content-Length: 10485761\r\n"
"Connection: close\r\n\r\n" % (chat_id, token)
).encode() + b"\x89PNG"
sock = socket.create_connection(("127.0.0.1", 8080), timeout=5)
try:
sock.sendall(request)
data = b""
while b"\r\n" not in data:
chunk = sock.recv(256)
if not chunk:
break
data += chunk
finally:
sock.close()
line = data.split(b"\r\n", 1)[0].decode("latin1", "replace")
code = int(line.split()[1])
expect(413, code, "content-length above 10MiB before body is read")
def main():
api = Api()
alice, alice_token = user(api, "alice")
bob, bob_token = user(api, "bob")
eve, eve_token = user(api, "eve")
status, _, chat, _ = api.call(
"POST",
"/api/v1/chats",
{"title": "grade4", "member_ids": [alice, bob]},
token=alice_token,
)
expect(201, status, "create chat")
chat_id = chat["id"]
status, _, other, _ = api.call(
"POST",
"/api/v1/chats",
{"title": "other", "member_ids": [alice, eve]},
token=alice_token,
)
expect(201, status, "create other chat")
status, _, message, _ = api.call(
"POST",
"/api/v1/chats/%s/messages" % chat_id,
{"text": "только текст"},
token=alice_token,
headers={"Idempotency-Key": uuid.uuid4().hex},
)
expect(201, status, "text message")
if message["attachment_ids"] != []:
raise SystemExit("FAIL text message lost empty attachment_ids")
good = png(32, 32)
status, _, attachment, _ = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=ok.png" % chat_id,
token=alice_token,
raw=good,
content_type="image/png",
)
expect(201, status, "png upload")
if attachment["sha256"] != hashlib.sha256(good).hexdigest() or attachment["status"] != "ready":
raise SystemExit("FAIL png metadata")
status, headers, _, downloaded = api.call(
"GET",
"/api/v1/attachments/%s/content" % attachment["id"],
token=bob_token,
)
expect(200, status, "member download")
if downloaded != good or headers.get_content_type() != "image/png":
raise SystemExit("FAIL download bytes")
if headers.get("Location"):
raise SystemExit("FAIL download redirected")
status, _, jpeg_meta, _ = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=ok.jpg" % chat_id,
token=alice_token,
raw=JPEG,
content_type="image/jpeg",
)
expect(201, status, "jpeg upload")
if jpeg_meta["content_type"] != "image/jpeg" or jpeg_meta["sha256"] != hashlib.sha256(JPEG).hexdigest():
raise SystemExit("FAIL jpeg metadata")
status, _, _, _ = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=a/b.png" % chat_id,
token=alice_token,
raw=good,
content_type="image/png",
)
expect(400, status, "filename with slash")
status, _, _, _ = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=%s" % (chat_id, "bad%0Aname.png"),
token=alice_token,
raw=good,
content_type="image/png",
)
expect(400, status, "filename with control character")
status, _, _, _ = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=lie.png" % chat_id,
token=alice_token,
raw=good,
content_type="image/jpeg",
)
expect(415, status, "png bytes declared as jpeg")
status, _, _, _ = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=cut.png" % chat_id,
token=alice_token,
raw=b"\x89PNG\r\n\x1a\ntruncated",
content_type="image/png",
)
expect(415, status, "truncated png")
status, _, _, _ = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=cut.jpg" % chat_id,
token=alice_token,
raw=JPEG[:-40],
content_type="image/jpeg",
)
expect(415, status, "truncated jpeg")
status, _, _, _ = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=huge.png" % chat_id,
token=alice_token,
raw=header_only_png(10000, 5000),
content_type="image/png",
)
expect(413, status, "pixel limit")
declared_length(alice_token, chat_id)
status, _, _, _ = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=big.png" % chat_id,
token=alice_token,
raw=b"\x89PNG" + b"\x00" * (10_485_760),
content_type="image/png",
)
expect(413, status, "body above 10MiB")
status, _, _, _ = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=nope.png" % chat_id,
token=eve_token,
raw=good,
content_type="image/png",
)
expect(404, status, "outsider upload")
for suffix in ("", "/content"):
status, _, _, _ = api.call("GET", "/api/v1/attachments/%s%s" % (attachment["id"], suffix), token=eve_token)
expect(404, status, "outsider get " + (suffix or "metadata"))
missing = str(uuid.uuid4())
status, _, _, _ = api.call(
"POST",
"/api/v1/chats/%s/messages" % chat_id,
{"text": "", "attachment_ids": [missing]},
token=alice_token,
headers={"Idempotency-Key": uuid.uuid4().hex},
)
expect(404, status, "unknown attachment id")
status, _, foreign, _ = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=other.png" % other["id"],
token=alice_token,
raw=good,
content_type="image/png",
)
expect(201, status, "upload into other chat")
status, _, _, _ = api.call(
"POST",
"/api/v1/chats/%s/messages" % chat_id,
{"text": "", "attachment_ids": [foreign["id"]]},
token=alice_token,
headers={"Idempotency-Key": uuid.uuid4().hex},
)
expect(404, status, "attachment from another chat")
ids = [str(uuid.uuid4()) for _ in range(11)]
status, _, _, _ = api.call(
"POST",
"/api/v1/chats/%s/messages" % chat_id,
{"text": "x", "attachment_ids": ids},
token=alice_token,
headers={"Idempotency-Key": uuid.uuid4().hex},
)
expect(400, status, "more than 10 attachment ids")
status, _, linked, _ = api.call(
"POST",
"/api/v1/chats/%s/messages" % chat_id,
{"text": "", "attachment_ids": [attachment["id"]]},
token=bob_token,
headers={"Idempotency-Key": uuid.uuid4().hex},
)
expect(201, status, "message with attachment")
if linked["attachment_ids"] != [attachment["id"]] or linked["text"] != "":
raise SystemExit("FAIL linked message")
print("PASS grade4 api checks")
if __name__ == "__main__":
main()
+231
View File
@@ -0,0 +1,231 @@
#!/usr/bin/env python3
"""Private bucket, orphan grace, and a bounded concurrent-upload memory sample."""
import json
import subprocess
import threading
import time
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "evidence" / "grade4-infra.json"
def compose(*args, input_text=None):
result = subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=ROOT,
input=input_text,
text=True,
capture_output=True,
)
if result.returncode != 0:
raise SystemExit(result.stderr[-500:] or result.stdout[-500:] or "compose failed")
return result.stdout
def anonymous_status():
script = r"""
code=$(wget -S -O /dev/null http://127.0.0.1:9000/messenger-attachments/ 2>&1 | awk 'BEGIN{c=""} /^[[:space:]]*HTTP\//{c=$2} END{print c}')
printf '%s' "$code"
"""
output = compose("exec", "-T", "s3", "sh", "-c", script).strip()
if output != "403":
raise SystemExit("anonymous bucket status %s" % output)
return output
def other_bucket_denied():
code = r"""
import os
import boto3
from botocore.client import Config
from botocore.exceptions import ClientError
client = boto3.client(
"s3",
endpoint_url=os.environ["S3_ENDPOINT"],
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],
aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
region_name=os.environ.get("S3_REGION", "us-east-1"),
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
)
try:
client.create_bucket(Bucket="not-the-messenger-bucket")
print("allowed")
except ClientError as exc:
print(exc.response.get("Error", {}).get("Code", "denied"))
"""
output = compose("exec", "-T", "api", "python", "-c", code).strip()
if output == "allowed":
raise SystemExit("app credentials created a second bucket")
return output
def plant_and_sweep():
code = r"""
import asyncio
from datetime import datetime, timedelta, timezone
from app import config, db, s3
async def main():
await db.connect()
old_key = "originals/orphan-evidence"
fresh_key = "originals/fresh-evidence"
payload = b"not-a-real-image"
await s3.put_bytes(old_key, payload, "application/octet-stream")
await s3.put_bytes(fresh_key, payload, "application/octet-stream")
old = datetime.now(timezone.utc) - timedelta(seconds=config.ORPHAN_GRACE_SECONDS + 60)
fresh = datetime.now(timezone.utc)
async with db.require().acquire() as conn:
await conn.execute("DELETE FROM blob_writes WHERE object_key = ANY($1::text[])", [old_key, fresh_key])
await conn.execute(
"INSERT INTO blob_writes (object_key, committed, created_at) VALUES ($1, FALSE, $2), ($3, FALSE, $4)",
old_key, old, fresh_key, fresh,
)
from app.attachments import sweep_orphans
deleted = await sweep_orphans()
async def exists(key):
try:
await s3.get_bytes(key)
return True
except Exception:
return False
print("%s %s %s" % (deleted, int(await exists(old_key)), int(await exists(fresh_key))))
await s3.delete_key(fresh_key)
async with db.require().acquire() as conn:
await conn.execute("DELETE FROM blob_writes WHERE object_key = $1 AND NOT committed", fresh_key)
await db.close()
asyncio.run(main())
"""
output = compose("exec", "-T", "api", "python", "-c", code).strip().split()
deleted, old_exists, fresh_exists = int(output[0]), int(output[1]), int(output[2])
if deleted < 1 or old_exists != 0 or fresh_exists != 1:
raise SystemExit("sweep result deleted=%s old=%s fresh=%s" % (deleted, old_exists, fresh_exists))
return {"deleted": deleted, "old_removed": True, "fresh_kept": True}
def memory_sample():
import base64
username = "mem_" + uuid.uuid4().hex[:10]
password = "T3st-" + uuid.uuid4().hex
body = json.dumps({"username": username, "display_name": "Mem", "password": password}).encode()
def call(method, path, data=None, headers=None):
request = urllib.request.Request("http://127.0.0.1:8080" + path, data=data, headers=headers or {}, method=method)
try:
response = urllib.request.urlopen(request, timeout=60)
except urllib.error.HTTPError as error:
response = error
with response:
return response.status, response.read()
status, raw = call("POST", "/api/v1/users", body, {"Content-Type": "application/json"})
if status != 201:
raise SystemExit("memory user %s" % status)
user = json.loads(raw)
basic = base64.b64encode((username + ":" + password).encode()).decode()
status, raw = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
if status != 201:
raise SystemExit("memory login %s" % status)
token = json.loads(raw)["token"]
status, raw = call(
"POST",
"/api/v1/chats",
json.dumps({"title": "mem", "member_ids": [user["id"]]}).encode(),
{"Content-Type": "application/json", "Authorization": "Bearer " + token},
)
if status != 201:
raise SystemExit("memory chat %s" % status)
chat_id = json.loads(raw)["id"]
import os
import struct
import zlib
def png():
signature = b"\x89PNG\r\n\x1a\n"
def chunk(kind, data):
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
width, height = 320, 240
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
raw_rows = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
return signature + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw_rows, 1)) + chunk(b"IEND", b"")
blob = png()
codes = []
def upload():
status, _ = call(
"POST",
"/api/v1/chats/%s/attachments?filename=m.png" % chat_id,
blob,
{"Content-Type": "image/png", "Authorization": "Bearer " + token},
)
codes.append(status)
samples = []
def watch():
end = time.monotonic() + 8
while time.monotonic() < end:
stats = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"],
capture_output=True,
text=True,
check=False,
)
for line in stats.stdout.splitlines():
if "mtusi2026lab4-api-" in line:
samples.append(line.strip())
time.sleep(0.4)
watcher = threading.Thread(target=watch)
watcher.start()
threads = [threading.Thread(target=upload) for _ in range(10)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
watcher.join()
if codes.count(201) != 10:
raise SystemExit("concurrent uploads %s" % codes)
def mib(line):
value = line.split()[1]
number = float(value[:-3]) if value.lower().endswith("mib") else float(value[:-3]) / 1024
return number
peak = max(mib(line) for line in samples) if samples else 0
return {"uploads": 10, "bytes_each": len(blob), "success": 10, "peak_api_mib": round(peak, 1), "samples": len(samples)}
def main():
published = subprocess.run(
["docker", "compose", "--env-file", ".env", "port", "s3", "9000"],
cwd=ROOT,
capture_output=True,
text=True,
)
if published.returncode == 0 and published.stdout.strip():
raise SystemExit("s3 port is published: " + published.stdout.strip())
summary = {
"anonymous_bucket_status": anonymous_status(),
"second_bucket": other_bucket_denied(),
"sweep": plant_and_sweep(),
"s3_port_published": False,
"memory": memory_sample(),
}
OUT.parent.mkdir(exist_ok=True)
OUT.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
print(json.dumps(summary, indent=2))
if __name__ == "__main__":
import urllib.error
main()