""" Служебные проверки хранилища для тестов и защиты (не HTTP API). Запуск внутри контейнера api (он в сети storage): python -m app.tools private-check # bucket приватный, права API ограничены python -m app.tools stats # число объектов / attachments / upload_intents python -m app.tools state # объекты staging/originals, метаданные, журнал python -m app.tools job # lab5: job, попытки, события, варианты, объекты S3 python -m app.tools republish # lab5: отправить ДУБЛЬ сообщения job в брокер python -m app.tools queue # lab5: длина stream, pending, DLQ, outbox python -m app.tools chat-attachments # lab5: вложения чата и их статусы (для тестов) python -m app.tools limits-selftest # lab5: лимиты памяти/времени/пикселей worker python -m app.tools find-local # нет ли на диске реплики копии файла Секреты не печатаются. Ключи объектов печатаются только для служебного prefix. """ from __future__ import annotations import json import sys import urllib.error import urllib.request import uuid import psycopg from botocore.exceptions import ClientError from . import storage from .config import settings def _anon(method: str, path: str, data: bytes | None = None) -> int: """HTTP-запрос к MinIO БЕЗ подписи (как посторонний клиент).""" req = urllib.request.Request(settings.s3_endpoint.rstrip("/") + path, data=data, method=method) try: with urllib.request.urlopen(req, timeout=5) as r: return r.status except urllib.error.HTTPError as e: return e.code def _denied(fn) -> str: try: fn() return "ALLOWED" except ClientError as exc: return exc.response.get("Error", {}).get("Code", "error") def private_check() -> int: s3 = storage.client() bucket = settings.s3_bucket probe = f"{storage.ORIGINALS_PREFIX}privacy-probe-{uuid.uuid4()}" results: list[tuple[str, str, bool]] = [] def check(name: str, observed, ok: bool) -> None: results.append((name, str(observed), ok)) s3.put_object(Bucket=bucket, Key=probe, Body=b"probe", ContentType="text/plain") try: # Позитивный контроль: объект существует и API-пользователь его читает. body = s3.get_object(Bucket=bucket, Key=probe)["Body"].read() check("API credentials: GET own object", "200", body == b"probe") # Аноним: ни чтения, ни листинга, ни записи. st = _anon("GET", f"/{bucket}/{probe}") check("anonymous GET object", st, st == 403) st = _anon("GET", f"/{bucket}?list-type=2") check("anonymous LIST bucket", st, st == 403) st = _anon("PUT", f"/{bucket}/{storage.ORIGINALS_PREFIX}anon-{uuid.uuid4()}", b"x") check("anonymous PUT object", st, st == 403) # Ограниченные права API-пользователя: только объекты своего bucket. # MinIO без s3:ListAllMyBuckets либо отказывает, либо показывает только # бакеты, на которые есть права, — чужие бакеты API-пользователь не видит. try: names = sorted(b["Name"] for b in s3.list_buckets().get("Buckets", [])) r = "only " + ",".join(names) if names else "empty" ok = set(names) <= {bucket} except ClientError as exc: r = exc.response.get("Error", {}).get("Code", "error") ok = r == "AccessDenied" check("API credentials: ListBuckets (no foreign buckets)", r, ok) r = _denied(lambda: s3.create_bucket(Bucket="lab4-forbidden-" + uuid.uuid4().hex[:8])) check("API credentials: CreateBucket", r, r == "AccessDenied") public = json.dumps({"Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": {"AWS": ["*"]}, "Action": ["s3:GetObject"], "Resource": [f"arn:aws:s3:::{bucket}/*"]}]}) r = _denied(lambda: s3.put_bucket_policy(Bucket=bucket, Policy=public)) check("API credentials: PutBucketPolicy (make public)", r, r == "AccessDenied") r = _denied(lambda: s3.delete_bucket(Bucket=bucket)) check("API credentials: DeleteBucket", r, r == "AccessDenied") finally: s3.delete_object(Bucket=bucket, Key=probe) width = max(len(n) for n, _, _ in results) for name, observed, ok in results: print(f" {'ok ' if ok else 'FAIL'} {name.ljust(width)} -> {observed}") failed = sum(1 for _, _, ok in results if not ok) print(f"\n{len(results) - failed} passed, {failed} failed") return 1 if failed else 0 def _db(): return psycopg.connect(settings.database_url, connect_timeout=3) def stats() -> int: # Оригиналы и staging (lab5): сирота может остаться в любом из двух prefix. objects = sum(1 for _ in storage.list_sync(storage.ORIGINALS_PREFIX)) \ + sum(1 for _ in storage.list_sync(storage.STAGING_PREFIX)) with _db() as conn: attachments = conn.execute("SELECT count(*) FROM attachments").fetchone()[0] intents = conn.execute("SELECT count(*) FROM upload_intents").fetchone()[0] print(json.dumps({"objects": objects, "attachments": attachments, "upload_intents": intents})) return 0 def state(attachment_id: str) -> int: aid = uuid.UUID(attachment_id) original, staging = storage.object_key(aid), storage.staging_key(aid) with _db() as conn: att = conn.execute("SELECT status FROM attachments WHERE id = %s", (aid,)).fetchone() intent = conn.execute("SELECT 1 FROM upload_intents WHERE attachment_id = %s", (aid,)).fetchone() print(json.dumps({"staging_object_exists": storage.exists_sync(staging), "original_object_exists": storage.exists_sync(original), "attachment_row": att is not None, "attachment_status": att[0] if att else None, "upload_intent": intent is not None})) return 0 def job(attachment_id: str) -> int: """Полное состояние обработки одного вложения (без секретов).""" aid = uuid.UUID(attachment_id) with _db() as conn: j = conn.execute( "SELECT job_id, status, attempt, max_attempts, error_code, last_error, replays, " "lease_owner, lease_until FROM attachment_jobs WHERE attachment_id = %s", (aid,)).fetchone() a = conn.execute("SELECT status, error_code, staging_key IS NOT NULL FROM attachments " "WHERE id = %s", (aid,)).fetchone() variants = conn.execute( "SELECT variant, width, height, size_bytes, object_key FROM attachment_variants " "WHERE attachment_id = %s ORDER BY variant", (aid,)).fetchall() events = conn.execute( "SELECT to_char(at, 'HH24:MI:SS.MS'), event, attempt, actor, coalesce(detail, '') " "FROM job_events e WHERE job_id = %s ORDER BY id", (j[0],)).fetchall() if j else [] outbox = conn.execute("SELECT attempt, published_at IS NOT NULL FROM job_outbox WHERE job_id = %s " "ORDER BY id", (j[0],)).fetchall() if j else [] objs = sorted(k for k, _, _ in storage.list_sync(f"{storage.VARIANTS_PREFIX}{aid}/")) print(json.dumps({ "attachment": {"status": a[0], "error_code": a[1], "staging_present": a[2]} if a else None, "job": {"job_id": str(j[0]), "status": j[1], "attempt": j[2], "max_attempts": j[3], "error_code": j[4], "last_error": j[5], "replays": j[6], "lease_owner": j[7], "lease_until": j[8].isoformat() if j[8] else None} if j else None, "variant_rows": [{"name": v[0], "size": f"{v[1]}x{v[2]}", "bytes": v[3]} for v in variants], "variant_objects_in_s3": len(objs), "original_object_exists": storage.exists_sync(storage.object_key(aid)), "staging_object_exists": storage.exists_sync(storage.staging_key(aid)), "outbox": [{"attempt": o[0], "published": o[1]} for o in outbox], "events": [" ".join(str(x) for x in e if x not in (None, "")) for e in events], }, ensure_ascii=False, indent=2)) return 0 def republish(attachment_id: str) -> int: """Отправить в брокер копию сообщения ТЕКУЩЕЙ попытки job (проверка повторной доставки).""" from . import broker aid = uuid.UUID(attachment_id) with _db() as conn: j = conn.execute("SELECT job_id, pipeline_version, attempt, created_at FROM attachment_jobs " "WHERE attachment_id = %s", (aid,)).fetchone() if j is None: print("job not found") return 1 payload = broker.make_payload(j[0], aid, j[1], j[2], j[3]) print("published duplicate:", broker.publish_sync(broker.sync_client(), payload), json.dumps(payload)) return 0 def chat_attachments(chat_id: str) -> int: with _db() as conn: rows = conn.execute("SELECT id, status FROM attachments WHERE chat_id = %s ORDER BY created_at", (uuid.UUID(chat_id),)).fetchall() print(json.dumps([{"id": str(r[0]), "status": r[1]} for r in rows])) return 0 def limits_selftest() -> int: """Защита worker от «тяжёлых» изображений проверяется на настоящем дочернем процессе (тот же код и те же лимиты, что в worker), без API — API такие файлы и так отсекает.""" import multiprocessing import tempfile import time from pathlib import Path from PIL import Image from . import imaging ctx = multiprocessing.get_context("forkserver") results = [] def run(name, src, ctype, max_pixels, memory_mb, timeout, slow=0.0, expect=""): d = Path(tempfile.mkdtemp()) res = d / "r.json" (d / "out").mkdir() t0 = time.monotonic() p = ctx.Process(target=imaging.child_main, args=(str(src), str(d / "out"), ctype, max_pixels, memory_mb, int(timeout) + 5, str(res), slow)) p.start() p.join(timeout) killed = p.is_alive() if killed: p.kill() p.join() got = "timeout(killed)" if killed else (json.loads(res.read_text()) if res.exists() else f"exit {p.exitcode}") outcome = got if isinstance(got, str) else ("ok" if got["ok"] else f'{got["code"]}: {got["detail"]}') ok = expect in outcome results.append(ok) print(f" {'ok ' if ok else 'FAIL'} {name}: {outcome} ({time.monotonic() - t0:.1f}s)") with tempfile.TemporaryDirectory() as tmp: big = Path(tmp) / "big.png" # 6000×6000 = 36 Мп: в пределах 40 Мп, но ~100+ МБ после распаковки Image.new("RGB", (6000, 6000), (200, 30, 30)).save(big, optimize=False, compress_level=1) small = Path(tmp) / "small.png" Image.new("RGBA", (320, 200), (0, 0, 255, 128)).save(small) corrupt = Path(tmp) / "corrupt.png" corrupt.write_bytes(small.read_bytes()[:200]) run("normal 320x200 with alpha", small, "image/png", 40_000_000, 768, 30, expect="ok") run("pixel limit (36 Mp > limit 10 Mp)", big, "image/png", 10_000_000, 768, 30, expect="invalid_image: pixel limit") run("memory limit (36 Mp, RLIMIT_AS 120 MiB)", big, "image/png", 40_000_000, 120, 30, expect="invalid_image") run("corrupt/truncated PNG", corrupt, "image/png", 40_000_000, 768, 30, expect="invalid_image") run("time limit (hangs 10 s, timeout 2 s)", small, "image/png", 40_000_000, 768, 2, slow=10, expect="timeout") failed = results.count(False) print(f"\n{len(results) - failed} passed, {failed} failed") return 1 if failed else 0 def queue() -> int: from . import broker r = broker.sync_client() try: groups = r.xinfo_groups(settings.jobs_stream) except Exception: # noqa: BLE001 groups = [] with _db() as conn: pending_outbox = conn.execute("SELECT count(*) FROM job_outbox WHERE published_at IS NULL").fetchone()[0] by_status = dict(conn.execute("SELECT status, count(*) FROM attachment_jobs GROUP BY status").fetchall()) print(json.dumps({ "stream_length": r.xlen(settings.jobs_stream) if r.exists(settings.jobs_stream) else 0, "groups": [{"name": g["name"], "consumers": g["consumers"], "pending": g["pending"], "lag": g.get("lag")} for g in groups], "dlq_length": r.xlen(settings.dlq_stream) if r.exists(settings.dlq_stream) else 0, "outbox_unpublished": pending_outbox, "jobs_by_status": by_status, }, ensure_ascii=False)) return 0 def find_local(digest: str) -> int: """Обойти файловую систему контейнера API и найти файлы с таким SHA-256. Доказательство, что оригинал не хранится на реплике (только в S3).""" import hashlib import os skip = ("/proc", "/sys", "/dev", "/usr", "/lib", "/bin", "/sbin", "/etc", "/var/lib/dpkg") scanned, found = 0, [] for root, dirs, files in os.walk("/"): if root.startswith(skip): dirs[:] = [] continue for name in files: path = os.path.join(root, name) try: if not os.path.isfile(path) or os.path.getsize(path) > 32 * 1024 * 1024: continue h = hashlib.sha256() with open(path, "rb") as f: for block in iter(lambda: f.read(1 << 20), b""): h.update(block) scanned += 1 if h.hexdigest() == digest: found.append(path) except OSError: continue print(json.dumps({"files_scanned": scanned, "copies_found": found})) return 1 if found else 0 def main(argv: list[str]) -> int: if not argv: print(__doc__) return 2 cmd, args = argv[0], argv[1:] if cmd == "private-check": return private_check() if cmd == "stats": return stats() if cmd == "state" and len(args) == 1: return state(args[0]) if cmd == "job" and len(args) == 1: return job(args[0]) if cmd == "republish" and len(args) == 1: return republish(args[0]) if cmd == "queue": return queue() if cmd == "chat-attachments" and len(args) == 1: return chat_attachments(args[0]) if cmd == "limits-selftest": return limits_selftest() if cmd == "find-local" and len(args) == 1: return find_local(args[0]) print(__doc__) return 2 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))