forked from mtusi2026/mtusi2026lab4
ЛР 4: вложения в S3 (MinIO), reconciler БД<->S3, отчёт и evidence
Перенесён минимум lab3. Добавлены загрузка/скачивание PNG/JPEG через API с потоковой проверкой целостности, метаданные в PostgreSQL и байты в приватном bucket, отдельный S3-пользователь с ограниченными правами, идемпотентный bootstrap bucket, компенсация и reconciler (трек 2), тесты отказов, эксперименты по памяти и согласованности.
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
import io, os, sys, tempfile, time
|
||||
sys.path.insert(0, "/app")
|
||||
from PIL import Image
|
||||
from app.errors import ApiError
|
||||
from app.imagecheck import inspect_image
|
||||
|
||||
def jpeg(w, h, progressive=False):
|
||||
img = Image.frombytes("RGB", (w, h), os.urandom(w * h * 3))
|
||||
b = io.BytesIO(); img.save(b, "JPEG", quality=80, progressive=progressive); return b.getvalue()
|
||||
|
||||
def verdict(data):
|
||||
f = tempfile.SpooledTemporaryFile(); f.write(data)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
inspect_image(f, len(data), "image/jpeg"); r = "accepted"
|
||||
except ApiError as e:
|
||||
r = "rejected %d" % e.status
|
||||
return r, time.monotonic() - started
|
||||
|
||||
ok = True
|
||||
for name, data, expect in [
|
||||
("valid 3000x2000 baseline", jpeg(3000, 2000), "accepted"),
|
||||
("valid 3000x2000 progressive", jpeg(3000, 2000, True), "accepted"),
|
||||
("27-byte stub", bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9"), "rejected 415"),
|
||||
]:
|
||||
r, t = verdict(data); ok &= r == expect; print("%-34s %-13s %.2fs %s" % (name, r, t, "ok" if r == expect else "FAIL"))
|
||||
big = jpeg(3000, 2000)
|
||||
sos = big.index(b"\xff\xda")
|
||||
cut = big[: len(big) // 2]
|
||||
r, t = verdict(cut); ok &= r == "rejected 415"; print("%-34s %-13s %.2fs %s" % ("scan cut in half, no EOI", r, t, "ok" if r == "rejected 415" else "FAIL"))
|
||||
noscan = big[:sos] + b"\xff\xd9"
|
||||
r, t = verdict(noscan); ok &= r == "rejected 415"; print("%-34s %-13s %.2fs %s" % ("headers only + EOI", r, t, "ok" if r == "rejected 415" else "FAIL"))
|
||||
sys.exit(0 if ok else 1)
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Versioned SQL migrations: applied once, checksum-verified, safe to re-run."""
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
import psycopg
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
MIGRATIONS = ROOT / "migrations"
|
||||
LOCK_ID = 2026_0004
|
||||
|
||||
|
||||
def database_url() -> str:
|
||||
path = os.environ.get("DATABASE_URL_FILE")
|
||||
if path:
|
||||
return pathlib.Path(path).read_text(encoding="utf-8").strip()
|
||||
url = os.environ.get("DATABASE_URL", "").strip()
|
||||
if not url:
|
||||
sys.exit("DATABASE_URL or DATABASE_URL_FILE is required")
|
||||
return url
|
||||
|
||||
|
||||
def main() -> None:
|
||||
files = sorted(MIGRATIONS.glob("*.sql"))
|
||||
if not files:
|
||||
sys.exit("no migration files found in %s" % MIGRATIONS)
|
||||
with psycopg.connect(database_url(), autocommit=True, connect_timeout=10) as conn:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
" version text PRIMARY KEY,"
|
||||
" checksum text NOT NULL,"
|
||||
" applied_at timestamptz NOT NULL DEFAULT now())"
|
||||
)
|
||||
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
|
||||
rows = conn.execute("SELECT version, checksum FROM schema_migrations").fetchall()
|
||||
applied = {row[0]: row[1] for row in rows}
|
||||
for path in files:
|
||||
version = path.stem
|
||||
body = path.read_text(encoding="utf-8")
|
||||
checksum = hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||||
if version in applied:
|
||||
if applied[version] != checksum:
|
||||
sys.exit("migration %s changed after it was applied" % version)
|
||||
print("skip %s" % version)
|
||||
continue
|
||||
with conn.transaction():
|
||||
conn.execute(body)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)",
|
||||
(version, checksum),
|
||||
)
|
||||
print("apply %s" % version)
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
|
||||
print("migrations complete: %d file(s)" % len(files))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run one DB<->S3 reconciliation pass. Default is dry-run; pass --apply to change anything."""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
|
||||
from starlette.concurrency import run_in_threadpool # noqa: E402
|
||||
|
||||
from app import config, reconcile # noqa: E402
|
||||
from app.db import Database # noqa: E402
|
||||
from app.s3 import S3Storage # noqa: E402
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--apply", action="store_true", help="delete/mark; without it only reports")
|
||||
parser.add_argument("--grace-seconds", type=int, default=config.RECONCILE_GRACE_SECONDS)
|
||||
args = parser.parse_args()
|
||||
db, s3 = Database(), S3Storage()
|
||||
await db.open()
|
||||
await run_in_threadpool(s3.open)
|
||||
try:
|
||||
print(json.dumps(await reconcile.run(db, s3, args.grace_seconds, dry_run=not args.apply), sort_keys=True))
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fault injection for the DB<->S3 reconciler. Run inside an api container:
|
||||
docker compose exec -T api-1 python scripts/reconcile_experiment.py [--bulk 1000]"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
|
||||
from starlette.concurrency import run_in_threadpool # noqa: E402
|
||||
|
||||
from app import config, reconcile # noqa: E402
|
||||
from app.db import Database # noqa: E402
|
||||
from app.s3 import S3Storage # noqa: E402
|
||||
|
||||
ZERO = "0" * 64
|
||||
checks = []
|
||||
|
||||
|
||||
def check(name, condition):
|
||||
checks.append((name, bool(condition)))
|
||||
print(("ok " if condition else "FAIL ") + name)
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--bulk", type=int, default=1000)
|
||||
args = parser.parse_args()
|
||||
db, s3 = Database(), S3Storage()
|
||||
await db.open()
|
||||
await run_in_threadpool(s3.open)
|
||||
put = lambda key: run_in_threadpool(s3.put, key, io.BytesIO(b"x"), 1, "image/png")
|
||||
exists = lambda key: run_in_threadpool(s3.exists, key)
|
||||
|
||||
baseline = await reconcile.run(db, s3, 900, dry_run=True)
|
||||
print("baseline (data left by earlier tests):", json.dumps(baseline, sort_keys=True))
|
||||
|
||||
async with db.connection() as conn:
|
||||
suffix = uuid.uuid4().hex[:10]
|
||||
user = (await (await conn.execute(
|
||||
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
|
||||
" VALUES (gen_random_uuid(), %s, %s, 'x', now()) RETURNING id", ("rx_" + suffix, b"rx"))).fetchone())["id"]
|
||||
chat = (await (await conn.execute(
|
||||
"INSERT INTO chats (id, title, created_at) VALUES (gen_random_uuid(), %s, now()) RETURNING id", (b"rx",))).fetchone())["id"]
|
||||
await conn.execute("INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", (chat, user))
|
||||
|
||||
async def row(status, age_s, with_object, linked=False):
|
||||
aid = str(uuid.uuid4())
|
||||
key = "attachments/" + aid
|
||||
await conn.execute(
|
||||
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key,"
|
||||
" status, created_at, updated_at) VALUES (%s,%s,%s,'f.png','image/png',1,%s,%s,%s,"
|
||||
" now() - make_interval(secs => %s), now() - make_interval(secs => %s))", (aid, chat, user, ZERO, key, status, age_s, age_s))
|
||||
if with_object:
|
||||
await put(key)
|
||||
if linked:
|
||||
mid = (await (await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
|
||||
" VALUES (gen_random_uuid(), %s, %s, ''::bytea, now()) RETURNING id", (chat, user))).fetchone())["id"]
|
||||
await conn.execute("INSERT INTO message_attachments VALUES (%s,%s,0)", (mid, aid))
|
||||
return aid, key
|
||||
|
||||
async def status_of(aid):
|
||||
r = await (await conn.execute("SELECT status, error_code FROM attachments WHERE id=%s", (aid,))).fetchone()
|
||||
return (r["status"], r["error_code"]) if r else None
|
||||
|
||||
old = 7200
|
||||
a_id, a_key = await row("uploading", old, True) # crash after PUT, before ready (object is young)
|
||||
b_id, b_key = await row("uploading", old, False) # crash before PUT
|
||||
c_id, c_key = await row("uploading", 5, True) # in-flight upload, must survive
|
||||
e_id, e_key = await row("ready", old, False) # object lost
|
||||
f_id, f_key = await row("ready", old, True, linked=True) # healthy, used
|
||||
g_id, g_key = await row("ready", old, False, linked=True) # object lost, used
|
||||
h_id, h_key = await row("uploading", old, False) # old intent row, but a NEW object under its key
|
||||
await put(h_key)
|
||||
i_id, i_key = await row("ready", 5, False) # freshly confirmed, object not yet visible: must not be marked
|
||||
d_key = "attachments/" + str(uuid.uuid4()) # PUT done, row never written
|
||||
await put(d_key)
|
||||
|
||||
print("== dry-run, grace=900 ==")
|
||||
r = await reconcile.run(db, s3, 900, dry_run=True)
|
||||
print(json.dumps(r, sort_keys=True))
|
||||
check("dry-run: 1 new stale row (b), 2 old rows protected by young objects (a, h)",
|
||||
r["stale_uploading"] - baseline["stale_uploading"] == 1
|
||||
and r["protected_fresh_objects"] - baseline["protected_fresh_objects"] == 2)
|
||||
check("dry-run: 2 new missing objects (e, g)", r["missing_objects"] - baseline["missing_objects"] == 2)
|
||||
check("dry-run changes nothing", await exists(a_key) and await status_of(a_id) == ("uploading", None)
|
||||
and await status_of(e_id) == ("ready", None))
|
||||
|
||||
print("== apply, grace=900 ==")
|
||||
r = await reconcile.run(db, s3, 900, dry_run=False)
|
||||
print(json.dumps(r, sort_keys=True))
|
||||
check("old row whose object is young is kept (a)", await status_of(a_id) == ("uploading", None) and await exists(a_key))
|
||||
check("stale row without object removed", await status_of(b_id) is None)
|
||||
check("fresh in-flight upload untouched", await status_of(c_id) == ("uploading", None) and await exists(c_key))
|
||||
check("fresh orphan object kept inside grace", await exists(d_key))
|
||||
check("old row with a fresh object is left alone", await status_of(h_id) == ("uploading", None) and await exists(h_key))
|
||||
check("recently confirmed row is not marked missing", await status_of(i_id) == ("ready", None))
|
||||
check("missing unused object marked failed", await status_of(e_id) == ("failed", "object_missing"))
|
||||
check("used attachment with object untouched", await status_of(f_id) == ("ready", None) and await exists(f_key))
|
||||
check("used attachment with lost object kept, marked failed", await status_of(g_id) == ("failed", "object_missing"))
|
||||
|
||||
print("== apply after grace expired (grace=2 after 3s) ==")
|
||||
await asyncio.sleep(3)
|
||||
r = await reconcile.run(db, s3, 2, dry_run=False)
|
||||
print(json.dumps(r, sort_keys=True))
|
||||
check("orphan object deleted after grace", not await exists(d_key))
|
||||
check("expired in-flight row collected after grace", await status_of(c_id) is None and not await exists(c_key))
|
||||
check("old row + object collected once the object is older than grace (a, h)",
|
||||
await status_of(a_id) is None and not await exists(a_key)
|
||||
and await status_of(h_id) is None and not await exists(h_key))
|
||||
check("used attachment still untouched", await status_of(f_id) == ("ready", None) and await exists(f_key))
|
||||
|
||||
print("== second apply is idempotent ==")
|
||||
r = await reconcile.run(db, s3, 2, dry_run=False)
|
||||
print(json.dumps(r, sort_keys=True))
|
||||
check("repeat run changes nothing", r["deleted_objects"] == 0 and r["removed_rows"] == 0 and r["marked_failed"] == 0)
|
||||
|
||||
print("== cost of a full walk, %d ready attachments ==" % args.bulk)
|
||||
keys = []
|
||||
for _ in range(args.bulk):
|
||||
aid, key = await row("ready", old, True)
|
||||
keys.append(aid)
|
||||
r = await reconcile.run(db, s3, 900, dry_run=True)
|
||||
print(json.dumps(r, sort_keys=True))
|
||||
check("bulk walk finds nothing wrong", r["missing_objects"] == 0 and r["orphan_objects"] == 0)
|
||||
|
||||
for aid in keys + [e_id, f_id, g_id, h_id, i_id]:
|
||||
await run_in_threadpool(s3.delete, "attachments/" + aid)
|
||||
await conn.execute("DELETE FROM chats WHERE id = %s", (chat,))
|
||||
await conn.execute("DELETE FROM users WHERE id = %s", (user,))
|
||||
await db.close()
|
||||
failed = [n for n, ok in checks if not ok]
|
||||
print("RESULT: %d/%d checks passed" % (len(checks) - len(failed), len(checks)))
|
||||
sys.exit(1 if failed else 0)
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart -t 5 postgres valkey minio api-1 api-2 nginx
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop -t 3 "$TARGET_INSTANCE_ID"
|
||||
Reference in New Issue
Block a user