Хранить вложения в приватном S3 и сверять их с PostgreSQL.

This commit is contained in:
2026-11-06 20:13:44 +03:00
parent ec540a1a72
commit a6bb7cbb8b
34 changed files with 3285 additions and 31 deletions
+44
View File
@@ -0,0 +1,44 @@
"""Create a local .env with synthetic secrets. Refuses to overwrite an existing file."""
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
TARGET = ROOT / ".env"
def token(length: int = 32) -> str:
alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
return "".join(secrets.choice(alphabet) for _ in range(length))
def main() -> None:
if TARGET.exists():
print(".env already exists; left unchanged")
return
content = "\n".join(
[
"POSTGRES_USER=messenger",
f"POSTGRES_PASSWORD={token()}",
"POSTGRES_DB=messenger",
"MINIO_ROOT_USER=labminio",
f"MINIO_ROOT_PASSWORD={token(40)}",
"S3_BUCKET=messenger-attachments",
"S3_REGION=us-east-1",
"S3_ACCESS_KEY_ID=messengerapi",
f"S3_SECRET_ACCESS_KEY={token(40)}",
"SESSION_TTL_SECONDS=3600",
"IDEMPOTENCY_TTL_SECONDS=86400",
"MAX_UPLOAD_BYTES=10485760",
"MAX_IMAGE_PIXELS=40000000",
"ORPHAN_GRACE_SECONDS=120",
"APP_ENV=lab",
"",
]
)
TARGET.write_text(content, encoding="utf-8")
print("wrote .env with local synthetic secrets; do not commit it")
if __name__ == "__main__":
main()
+33
View File
@@ -0,0 +1,33 @@
using System;
using System.Diagnostics;
using System.IO;
class Program
{
static int Run(string args)
{
string dir = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, ".."));
var start = new ProcessStartInfo("docker", args)
{
WorkingDirectory = dir,
UseShellExecute = false
};
var process = Process.Start(start);
process.WaitForExit();
return process.ExitCode;
}
static int Main()
{
string name = Path.GetFileName(Environment.GetCommandLineArgs()[0]).ToLowerInvariant();
if (name.Contains("stop"))
{
string id = Environment.GetEnvironmentVariable("TARGET_INSTANCE_ID");
if (id != "api-1" && id != "api-2") return 2;
return Run("compose --env-file .env stop -t 2 " + id);
}
int up = Run("compose --env-file .env up -d --no-recreate --no-build postgres minio api-1 api-2 lb");
if (up != 0) return up;
return Run("compose --env-file .env restart postgres minio api-1 api-2 lb");
}
}
+171
View File
@@ -0,0 +1,171 @@
"""Ten concurrent uploads at two sizes. Records container RSS, not a universal capacity claim."""
import json
import random
import struct
import subprocess
import sys
import threading
import time
import urllib.request
import zlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
def chunk(tag: bytes, data: bytes) -> bytes:
crc = zlib.crc32(tag + data) & 0xFFFFFFFF
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", crc)
def make_png(width: int, height: int, seed: int) -> bytes:
rng = random.Random(seed)
raw = bytearray()
for _y in range(height):
raw.append(0)
raw.extend(rng.randbytes(width * 3))
compressor = zlib.compressobj(level=0)
payload = compressor.compress(bytes(raw)) + compressor.flush()
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", payload) + chunk(b"IEND", b"")
def parse_bytes(text: str) -> int:
"""docker stats MemUsage looks like '12.3MiB / 768MiB'."""
value = text.split("/")[0].strip()
number = ""
unit = ""
for char in value:
if char.isdigit() or char == ".":
number += char
elif char != " ":
unit += char
scale = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3, "kB": 1000, "MB": 1000**2, "GB": 1000**3}
return int(float(number) * scale[unit])
def sample(stop: threading.Event, peaks: dict) -> None:
while not stop.is_set():
try:
raw = subprocess.check_output(
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}"],
text=True,
)
except subprocess.CalledProcessError:
time.sleep(0.3)
continue
for line in raw.splitlines():
name, usage = line.split("\t", 1)
if name not in {"mtusi-api-1", "mtusi-api-2"}:
continue
peaks[name] = max(peaks.get(name, 0), parse_bytes(usage))
time.sleep(0.25)
def vmhwm() -> dict:
result = {}
for name in ("mtusi-api-1", "mtusi-api-2"):
text = subprocess.check_output(["docker", "exec", name, "sh", "-c", "grep VmHWM /proc/1/status"], text=True)
result[name] = int(text.split()[1])
return result
def wave(client: Client, actor, chat, payload: bytes, label: str) -> dict:
stop = threading.Event()
peaks = {}
thread = threading.Thread(target=sample, args=(stop, peaks))
before = vmhwm()
thread.start()
errors = []
def once(index):
local = Client("http://127.0.0.1:8080", timeout=60)
path = f"/api/v1/chats/{chat['id']}/attachments?filename={label}-{index}.png"
try:
response = local.call(
"POST",
path,
201,
actor=actor,
raw=payload,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
return response["json"]["id"]
except Exception as exc:
errors.append(str(exc))
return None
started = time.perf_counter()
from concurrent.futures import ThreadPoolExecutor
with ThreadPoolExecutor(max_workers=10) as pool:
ids = list(pool.map(once, range(10)))
elapsed = time.perf_counter() - started
time.sleep(0.6)
stop.set()
thread.join()
after = vmhwm()
return {
"label": label,
"file_bytes": len(payload),
"concurrency": 10,
"successes": sum(1 for item in ids if item),
"errors": errors[:5],
"elapsed_seconds": round(elapsed, 3),
"docker_stats_peak_bytes": peaks,
"vmhwm_kib_before": before,
"vmhwm_kib_after": after,
}
def restart_apis() -> None:
subprocess.check_call(["docker", "compose", "--env-file", ".env", "restart", "api-1", "api-2"], cwd=ROOT)
deadline = time.time() + 90
while time.time() < deadline:
try:
urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2).read()
return
except Exception:
time.sleep(1)
raise SystemExit("API did not become ready after restart")
def main() -> None:
client = Client("http://127.0.0.1:8080", timeout=30)
alice = client.user("mem_alice")
bob = client.user("mem_bob")
chat = client.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "Память", "member_ids": [alice["id"], bob["id"]]},
schema="Chat",
)["json"]
small = make_png(64, 64, 1)
large = make_png(1200, 900, 2)
restart_apis()
first = wave(client, alice, chat, small, "small")
restart_apis()
second = wave(client, alice, chat, large, "large")
report = {
"hypothesis": "A single upload is capped at 10 MiB and does not decode bitmaps above 1_000_000 pixels; ten concurrent uploads stay inside the 768 MiB container limit.",
"limits": {"container_memory": "768MiB", "max_upload_bytes": 10485760, "full_decode_pixels": 1000000},
"waves": [first, second],
"note": "docker stats includes page cache. VmHWM is the process high-water RSS in KiB after a fresh API restart, so the two waves are comparable.",
}
dest = ROOT / "evidence" / "memory.json"
dest.parent.mkdir(exist_ok=True)
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
if first["successes"] != 10 or second["successes"] != 10:
raise SystemExit("not all concurrent uploads succeeded")
print("PASS: ten concurrent uploads at two sizes")
if __name__ == "__main__":
main()
+126
View File
@@ -0,0 +1,126 @@
"""Inject faults on both sides, dry-run, apply, and repeat. Fresh and in-use rows survive."""
import hashlib
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
FIXTURE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
def admin(*args) -> dict:
raw = subprocess.check_output(
["docker", "exec", "mtusi-api-1", "python", "-m", "app.admin", *args],
text=True,
)
return json.loads(raw)
def redact(report: dict) -> dict:
cloned = json.loads(json.dumps(report))
for action in cloned.get("actions", []):
action.pop("object_key", None)
return cloned
def main() -> None:
client = Client("http://127.0.0.1:8080", timeout=30)
alice = client.user("rec_alice")
bob = client.user("rec_bob")
chat = client.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "Сверка", "member_ids": [alice["id"], bob["id"]]},
schema="Chat",
)["json"]
fresh = client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=kept.png",
201,
actor=alice,
raw=FIXTURE,
headers={"Content-Type": "image/png"},
schema="Attachment",
)["json"]
client.message(alice, chat, "", attachments=[fresh["id"]])
loose = client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=loose.png",
201,
actor=alice,
raw=FIXTURE,
headers={"Content-Type": "image/png"},
schema="Attachment",
)["json"]
faults = admin("inject", "--chat-id", chat["id"], "--owner-id", alice["id"])
dry = admin("reconcile", "--dry-run")
notes = {item.get("note") for item in dry["actions"]}
check("old-orphan" in notes, "dry-run missed the old orphan")
check("fault-stale-missing" in notes, "dry-run missed stale metadata")
check("fresh-orphan" not in notes, "dry-run wanted to delete a fresh orphan")
check("fault-in-use-missing" not in notes, "dry-run wanted to delete an in-use attachment")
check(dry["skipped"].get("fresh_orphan", 0) >= 1, "fresh orphan was not skipped")
check(dry["skipped"].get("in_use_missing", 0) >= 1, "in-use missing attachment was not skipped")
applied = admin("reconcile", "--apply")
again = admin("reconcile", "--dry-run")
check(again["action_count"] == 0, "second dry-run was not a no-op")
old_object = admin("probe", "--object-key", faults["old_orphan"]["object_key"])
fresh_object = admin("probe", "--object-key", faults["fresh_orphan"]["object_key"])
stale_meta = admin("probe", "--attachment-id", faults["stale_missing"]["attachment_id"])
used_meta = admin("probe", "--attachment-id", faults["in_use_missing"]["attachment_id"])
check(old_object["object"] is False, "old orphan object survived apply")
check(fresh_object["object"] is True, "fresh orphan object was deleted")
check(stale_meta["metadata"] is False, "stale unreferenced metadata survived apply")
check(used_meta["metadata"] is True, "in-use metadata was deleted")
downloaded = client.call("GET", "/api/v1/attachments/" + fresh["id"] + "/content", actor=bob)
check(downloaded["bytes"] == FIXTURE, "fresh in-use original changed")
check(hashlib.sha256(downloaded["bytes"]).hexdigest() == fresh["sha256"], "checksum changed")
client.call("GET", "/api/v1/attachments/" + loose["id"], actor=bob, schema="Attachment")
client.call("GET", "/api/v1/attachments/" + faults["in_use_missing"]["attachment_id"], actor=bob, schema="Attachment")
missing_bytes = client.call(
"GET",
"/api/v1/attachments/" + faults["in_use_missing"]["attachment_id"] + "/content",
404,
actor=bob,
)
check(missing_bytes["json"]["error"]["code"] == "not_found", "missing in-use bytes were not 404")
client.call("GET", "/api/v1/attachments/" + faults["stale_missing"]["attachment_id"], 404, actor=alice)
report = {
"hypothesis": "A full-scan reconciler with a 120s grace period removes only expired orphans and unreferenced metadata whose object is already gone.",
"method": "Inject a fresh orphan, a 2h orphan, a 2h missing unreferenced row, and a 2h missing row referenced by a message. Dry-run, apply, dry-run again.",
"dry_run": redact(dry),
"apply": redact(applied),
"second_dry_run": redact(again),
"survived": {
"fresh_orphan_object": True,
"in_use_metadata": True,
"fresh_message_attachment_sha256": fresh["sha256"],
"loose_unreferenced_attachment": loose["id"],
},
"limitations": [
"Walk cost is one full ListObjects plus one pass over attachment and ledger rows; this stand has one page.",
"Grace uses the ledger or metadata timestamp, not a backdated S3 LastModified, because MinIO does not let the lab rewrite LastModified.",
"In-use metadata whose bytes are gone is reported and kept; the reconciler does not invent replacement bytes.",
"A second dry-run being empty shows the repair is repeatable, not that a continuous watcher is running.",
],
}
dest = ROOT / "evidence" / "reconcile.json"
dest.parent.mkdir(exist_ok=True)
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"action_count": dry["action_count"], "second": again["action_count"], "duration_ms": dry["duration_ms"], "objects_seen": dry["objects_seen"], "list_pages": dry["list_pages"]}, ensure_ascii=False))
print("PASS: reconciler dry-run/apply/repeat kept fresh and in-use attachments")
if __name__ == "__main__":
try:
main()
except Exception as exc:
print("FAIL:", exc)
raise SystemExit(1)
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
# Restart API and every store used by the lab. Volumes are preserved.
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env up -d --no-recreate --no-build postgres minio api-1 api-2 lb
docker compose --env-file .env restart postgres minio api-1 api-2 lb
+85
View File
@@ -0,0 +1,85 @@
"""Stop MinIO, observe ready=503, start it again, and read the same bytes."""
import hashlib
import json
import subprocess
import time
import urllib.error
import urllib.request
from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
FIXTURE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
def status(path: str) -> int:
try:
with urllib.request.urlopen("http://127.0.0.1:8080" + path, timeout=15) as response:
return response.status
except urllib.error.HTTPError as exc:
return exc.code
def wait_ready(expected: int) -> None:
deadline = time.time() + 90
last = None
while time.time() < deadline:
try:
last = status("/health/ready")
except Exception:
last = None
if last == expected:
return
time.sleep(1)
raise SystemExit(f"ready did not become {expected}; last={last}")
def main() -> None:
client = Client("http://127.0.0.1:8080", timeout=20)
alice = client.user("out_alice")
bob = client.user("out_bob")
chat = client.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "Отказ S3", "member_ids": [alice["id"], bob["id"]]},
schema="Chat",
)["json"]
uploaded = client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=outage.png",
201,
actor=alice,
raw=FIXTURE,
headers={"Content-Type": "image/png"},
schema="Attachment",
)["json"]
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "minio"], cwd=ROOT)
time.sleep(2)
live = status("/health/live")
ready = status("/health/ready")
check(live == 200, "liveness failed while S3 was stopped")
check(ready == 503, "readiness stayed 200 while S3 was stopped")
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "minio"], cwd=ROOT)
wait_ready(200)
downloaded = client.call("GET", "/api/v1/attachments/" + uploaded["id"] + "/content", actor=bob)
check(downloaded["bytes"] == FIXTURE, "original changed after the S3 restart")
check(hashlib.sha256(downloaded["bytes"]).hexdigest() == uploaded["sha256"], "checksum changed after S3 restart")
report = {"live_while_s3_down": live, "ready_while_s3_down": ready, "download_after_start": "match"}
dest = ROOT / "evidence" / "s3_outage.json"
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False))
print("PASS: S3 outage fails readiness and preserves the original")
if __name__ == "__main__":
try:
main()
except Exception as exc:
print("FAIL:", exc)
raise SystemExit(1)
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
# Stop exactly one API replica and leave it stopped. Do not touch volumes or stores.
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?}"
case "$TARGET_INSTANCE_ID" in
api-1|api-2) ;;
*)
echo "unexpected instance" >&2
exit 2
;;
esac
docker compose --env-file .env stop -t 2 "$TARGET_INSTANCE_ID"