forked from mtusi2026/mtusi2026lab4
Хранить вложения в приватном S3 и сверять их с PostgreSQL.
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
"""Create a local .env with synthetic secrets. Refuses to overwrite an existing file."""
|
||||
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
TARGET = ROOT / ".env"
|
||||
|
||||
|
||||
def token(length: int = 32) -> str:
|
||||
alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
return "".join(secrets.choice(alphabet) for _ in range(length))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if TARGET.exists():
|
||||
print(".env already exists; left unchanged")
|
||||
return
|
||||
content = "\n".join(
|
||||
[
|
||||
"POSTGRES_USER=messenger",
|
||||
f"POSTGRES_PASSWORD={token()}",
|
||||
"POSTGRES_DB=messenger",
|
||||
"MINIO_ROOT_USER=labminio",
|
||||
f"MINIO_ROOT_PASSWORD={token(40)}",
|
||||
"S3_BUCKET=messenger-attachments",
|
||||
"S3_REGION=us-east-1",
|
||||
"S3_ACCESS_KEY_ID=messengerapi",
|
||||
f"S3_SECRET_ACCESS_KEY={token(40)}",
|
||||
"SESSION_TTL_SECONDS=3600",
|
||||
"IDEMPOTENCY_TTL_SECONDS=86400",
|
||||
"MAX_UPLOAD_BYTES=10485760",
|
||||
"MAX_IMAGE_PIXELS=40000000",
|
||||
"ORPHAN_GRACE_SECONDS=120",
|
||||
"APP_ENV=lab",
|
||||
"",
|
||||
]
|
||||
)
|
||||
TARGET.write_text(content, encoding="utf-8")
|
||||
print("wrote .env with local synthetic secrets; do not commit it")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,33 @@
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
|
||||
class Program
|
||||
{
|
||||
static int Run(string args)
|
||||
{
|
||||
string dir = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, ".."));
|
||||
var start = new ProcessStartInfo("docker", args)
|
||||
{
|
||||
WorkingDirectory = dir,
|
||||
UseShellExecute = false
|
||||
};
|
||||
var process = Process.Start(start);
|
||||
process.WaitForExit();
|
||||
return process.ExitCode;
|
||||
}
|
||||
|
||||
static int Main()
|
||||
{
|
||||
string name = Path.GetFileName(Environment.GetCommandLineArgs()[0]).ToLowerInvariant();
|
||||
if (name.Contains("stop"))
|
||||
{
|
||||
string id = Environment.GetEnvironmentVariable("TARGET_INSTANCE_ID");
|
||||
if (id != "api-1" && id != "api-2") return 2;
|
||||
return Run("compose --env-file .env stop -t 2 " + id);
|
||||
}
|
||||
int up = Run("compose --env-file .env up -d --no-recreate --no-build postgres minio api-1 api-2 lb");
|
||||
if (up != 0) return up;
|
||||
return Run("compose --env-file .env restart postgres minio api-1 api-2 lb");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
"""Ten concurrent uploads at two sizes. Records container RSS, not a universal capacity claim."""
|
||||
|
||||
import json
|
||||
import random
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
|
||||
def chunk(tag: bytes, data: bytes) -> bytes:
|
||||
crc = zlib.crc32(tag + data) & 0xFFFFFFFF
|
||||
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", crc)
|
||||
|
||||
|
||||
def make_png(width: int, height: int, seed: int) -> bytes:
|
||||
rng = random.Random(seed)
|
||||
raw = bytearray()
|
||||
for _y in range(height):
|
||||
raw.append(0)
|
||||
raw.extend(rng.randbytes(width * 3))
|
||||
compressor = zlib.compressobj(level=0)
|
||||
payload = compressor.compress(bytes(raw)) + compressor.flush()
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", payload) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def parse_bytes(text: str) -> int:
|
||||
"""docker stats MemUsage looks like '12.3MiB / 768MiB'."""
|
||||
value = text.split("/")[0].strip()
|
||||
number = ""
|
||||
unit = ""
|
||||
for char in value:
|
||||
if char.isdigit() or char == ".":
|
||||
number += char
|
||||
elif char != " ":
|
||||
unit += char
|
||||
scale = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3, "kB": 1000, "MB": 1000**2, "GB": 1000**3}
|
||||
return int(float(number) * scale[unit])
|
||||
|
||||
|
||||
def sample(stop: threading.Event, peaks: dict) -> None:
|
||||
while not stop.is_set():
|
||||
try:
|
||||
raw = subprocess.check_output(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}"],
|
||||
text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError:
|
||||
time.sleep(0.3)
|
||||
continue
|
||||
for line in raw.splitlines():
|
||||
name, usage = line.split("\t", 1)
|
||||
if name not in {"mtusi-api-1", "mtusi-api-2"}:
|
||||
continue
|
||||
peaks[name] = max(peaks.get(name, 0), parse_bytes(usage))
|
||||
time.sleep(0.25)
|
||||
|
||||
|
||||
def vmhwm() -> dict:
|
||||
result = {}
|
||||
for name in ("mtusi-api-1", "mtusi-api-2"):
|
||||
text = subprocess.check_output(["docker", "exec", name, "sh", "-c", "grep VmHWM /proc/1/status"], text=True)
|
||||
result[name] = int(text.split()[1])
|
||||
return result
|
||||
|
||||
|
||||
def wave(client: Client, actor, chat, payload: bytes, label: str) -> dict:
|
||||
stop = threading.Event()
|
||||
peaks = {}
|
||||
thread = threading.Thread(target=sample, args=(stop, peaks))
|
||||
before = vmhwm()
|
||||
thread.start()
|
||||
errors = []
|
||||
|
||||
def once(index):
|
||||
local = Client("http://127.0.0.1:8080", timeout=60)
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename={label}-{index}.png"
|
||||
try:
|
||||
response = local.call(
|
||||
"POST",
|
||||
path,
|
||||
201,
|
||||
actor=actor,
|
||||
raw=payload,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
return response["json"]["id"]
|
||||
except Exception as exc:
|
||||
errors.append(str(exc))
|
||||
return None
|
||||
|
||||
started = time.perf_counter()
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
ids = list(pool.map(once, range(10)))
|
||||
elapsed = time.perf_counter() - started
|
||||
time.sleep(0.6)
|
||||
stop.set()
|
||||
thread.join()
|
||||
after = vmhwm()
|
||||
return {
|
||||
"label": label,
|
||||
"file_bytes": len(payload),
|
||||
"concurrency": 10,
|
||||
"successes": sum(1 for item in ids if item),
|
||||
"errors": errors[:5],
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"docker_stats_peak_bytes": peaks,
|
||||
"vmhwm_kib_before": before,
|
||||
"vmhwm_kib_after": after,
|
||||
}
|
||||
|
||||
|
||||
def restart_apis() -> None:
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "restart", "api-1", "api-2"], cwd=ROOT)
|
||||
deadline = time.time() + 90
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2).read()
|
||||
return
|
||||
except Exception:
|
||||
time.sleep(1)
|
||||
raise SystemExit("API did not become ready after restart")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
client = Client("http://127.0.0.1:8080", timeout=30)
|
||||
alice = client.user("mem_alice")
|
||||
bob = client.user("mem_bob")
|
||||
chat = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "Память", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
small = make_png(64, 64, 1)
|
||||
large = make_png(1200, 900, 2)
|
||||
restart_apis()
|
||||
first = wave(client, alice, chat, small, "small")
|
||||
restart_apis()
|
||||
second = wave(client, alice, chat, large, "large")
|
||||
report = {
|
||||
"hypothesis": "A single upload is capped at 10 MiB and does not decode bitmaps above 1_000_000 pixels; ten concurrent uploads stay inside the 768 MiB container limit.",
|
||||
"limits": {"container_memory": "768MiB", "max_upload_bytes": 10485760, "full_decode_pixels": 1000000},
|
||||
"waves": [first, second],
|
||||
"note": "docker stats includes page cache. VmHWM is the process high-water RSS in KiB after a fresh API restart, so the two waves are comparable.",
|
||||
}
|
||||
dest = ROOT / "evidence" / "memory.json"
|
||||
dest.parent.mkdir(exist_ok=True)
|
||||
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if first["successes"] != 10 or second["successes"] != 10:
|
||||
raise SystemExit("not all concurrent uploads succeeded")
|
||||
print("PASS: ten concurrent uploads at two sizes")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Inject faults on both sides, dry-run, apply, and repeat. Fresh and in-use rows survive."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
FIXTURE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
|
||||
|
||||
def admin(*args) -> dict:
|
||||
raw = subprocess.check_output(
|
||||
["docker", "exec", "mtusi-api-1", "python", "-m", "app.admin", *args],
|
||||
text=True,
|
||||
)
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def redact(report: dict) -> dict:
|
||||
cloned = json.loads(json.dumps(report))
|
||||
for action in cloned.get("actions", []):
|
||||
action.pop("object_key", None)
|
||||
return cloned
|
||||
|
||||
|
||||
def main() -> None:
|
||||
client = Client("http://127.0.0.1:8080", timeout=30)
|
||||
alice = client.user("rec_alice")
|
||||
bob = client.user("rec_bob")
|
||||
chat = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "Сверка", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
fresh = client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=kept.png",
|
||||
201,
|
||||
actor=alice,
|
||||
raw=FIXTURE,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)["json"]
|
||||
client.message(alice, chat, "", attachments=[fresh["id"]])
|
||||
loose = client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=loose.png",
|
||||
201,
|
||||
actor=alice,
|
||||
raw=FIXTURE,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)["json"]
|
||||
faults = admin("inject", "--chat-id", chat["id"], "--owner-id", alice["id"])
|
||||
dry = admin("reconcile", "--dry-run")
|
||||
notes = {item.get("note") for item in dry["actions"]}
|
||||
check("old-orphan" in notes, "dry-run missed the old orphan")
|
||||
check("fault-stale-missing" in notes, "dry-run missed stale metadata")
|
||||
check("fresh-orphan" not in notes, "dry-run wanted to delete a fresh orphan")
|
||||
check("fault-in-use-missing" not in notes, "dry-run wanted to delete an in-use attachment")
|
||||
check(dry["skipped"].get("fresh_orphan", 0) >= 1, "fresh orphan was not skipped")
|
||||
check(dry["skipped"].get("in_use_missing", 0) >= 1, "in-use missing attachment was not skipped")
|
||||
applied = admin("reconcile", "--apply")
|
||||
again = admin("reconcile", "--dry-run")
|
||||
check(again["action_count"] == 0, "second dry-run was not a no-op")
|
||||
old_object = admin("probe", "--object-key", faults["old_orphan"]["object_key"])
|
||||
fresh_object = admin("probe", "--object-key", faults["fresh_orphan"]["object_key"])
|
||||
stale_meta = admin("probe", "--attachment-id", faults["stale_missing"]["attachment_id"])
|
||||
used_meta = admin("probe", "--attachment-id", faults["in_use_missing"]["attachment_id"])
|
||||
check(old_object["object"] is False, "old orphan object survived apply")
|
||||
check(fresh_object["object"] is True, "fresh orphan object was deleted")
|
||||
check(stale_meta["metadata"] is False, "stale unreferenced metadata survived apply")
|
||||
check(used_meta["metadata"] is True, "in-use metadata was deleted")
|
||||
downloaded = client.call("GET", "/api/v1/attachments/" + fresh["id"] + "/content", actor=bob)
|
||||
check(downloaded["bytes"] == FIXTURE, "fresh in-use original changed")
|
||||
check(hashlib.sha256(downloaded["bytes"]).hexdigest() == fresh["sha256"], "checksum changed")
|
||||
client.call("GET", "/api/v1/attachments/" + loose["id"], actor=bob, schema="Attachment")
|
||||
client.call("GET", "/api/v1/attachments/" + faults["in_use_missing"]["attachment_id"], actor=bob, schema="Attachment")
|
||||
missing_bytes = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + faults["in_use_missing"]["attachment_id"] + "/content",
|
||||
404,
|
||||
actor=bob,
|
||||
)
|
||||
check(missing_bytes["json"]["error"]["code"] == "not_found", "missing in-use bytes were not 404")
|
||||
client.call("GET", "/api/v1/attachments/" + faults["stale_missing"]["attachment_id"], 404, actor=alice)
|
||||
report = {
|
||||
"hypothesis": "A full-scan reconciler with a 120s grace period removes only expired orphans and unreferenced metadata whose object is already gone.",
|
||||
"method": "Inject a fresh orphan, a 2h orphan, a 2h missing unreferenced row, and a 2h missing row referenced by a message. Dry-run, apply, dry-run again.",
|
||||
"dry_run": redact(dry),
|
||||
"apply": redact(applied),
|
||||
"second_dry_run": redact(again),
|
||||
"survived": {
|
||||
"fresh_orphan_object": True,
|
||||
"in_use_metadata": True,
|
||||
"fresh_message_attachment_sha256": fresh["sha256"],
|
||||
"loose_unreferenced_attachment": loose["id"],
|
||||
},
|
||||
"limitations": [
|
||||
"Walk cost is one full ListObjects plus one pass over attachment and ledger rows; this stand has one page.",
|
||||
"Grace uses the ledger or metadata timestamp, not a backdated S3 LastModified, because MinIO does not let the lab rewrite LastModified.",
|
||||
"In-use metadata whose bytes are gone is reported and kept; the reconciler does not invent replacement bytes.",
|
||||
"A second dry-run being empty shows the repair is repeatable, not that a continuous watcher is running.",
|
||||
],
|
||||
}
|
||||
dest = ROOT / "evidence" / "reconcile.json"
|
||||
dest.parent.mkdir(exist_ok=True)
|
||||
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"action_count": dry["action_count"], "second": again["action_count"], "duration_ms": dry["duration_ms"], "objects_seen": dry["objects_seen"], "list_pages": dry["list_pages"]}, ensure_ascii=False))
|
||||
print("PASS: reconciler dry-run/apply/repeat kept fresh and in-use attachments")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
print("FAIL:", exc)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
# Restart API and every store used by the lab. Volumes are preserved.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env up -d --no-recreate --no-build postgres minio api-1 api-2 lb
|
||||
docker compose --env-file .env restart postgres minio api-1 api-2 lb
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Stop MinIO, observe ready=503, start it again, and read the same bytes."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
FIXTURE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
|
||||
|
||||
def status(path: str) -> int:
|
||||
try:
|
||||
with urllib.request.urlopen("http://127.0.0.1:8080" + path, timeout=15) as response:
|
||||
return response.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code
|
||||
|
||||
|
||||
def wait_ready(expected: int) -> None:
|
||||
deadline = time.time() + 90
|
||||
last = None
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
last = status("/health/ready")
|
||||
except Exception:
|
||||
last = None
|
||||
if last == expected:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise SystemExit(f"ready did not become {expected}; last={last}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
client = Client("http://127.0.0.1:8080", timeout=20)
|
||||
alice = client.user("out_alice")
|
||||
bob = client.user("out_bob")
|
||||
chat = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "Отказ S3", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
uploaded = client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=outage.png",
|
||||
201,
|
||||
actor=alice,
|
||||
raw=FIXTURE,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)["json"]
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "minio"], cwd=ROOT)
|
||||
time.sleep(2)
|
||||
live = status("/health/live")
|
||||
ready = status("/health/ready")
|
||||
check(live == 200, "liveness failed while S3 was stopped")
|
||||
check(ready == 503, "readiness stayed 200 while S3 was stopped")
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "minio"], cwd=ROOT)
|
||||
wait_ready(200)
|
||||
downloaded = client.call("GET", "/api/v1/attachments/" + uploaded["id"] + "/content", actor=bob)
|
||||
check(downloaded["bytes"] == FIXTURE, "original changed after the S3 restart")
|
||||
check(hashlib.sha256(downloaded["bytes"]).hexdigest() == uploaded["sha256"], "checksum changed after S3 restart")
|
||||
report = {"live_while_s3_down": live, "ready_while_s3_down": ready, "download_after_start": "match"}
|
||||
dest = ROOT / "evidence" / "s3_outage.json"
|
||||
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False))
|
||||
print("PASS: S3 outage fails readiness and preserves the original")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception as exc:
|
||||
print("FAIL:", exc)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
# Stop exactly one API replica and leave it stopped. Do not touch volumes or stores.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
api-1|api-2) ;;
|
||||
*)
|
||||
echo "unexpected instance" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
docker compose --env-file .env stop -t 2 "$TARGET_INSTANCE_ID"
|
||||
Reference in New Issue
Block a user