Files
mtusi2026lab5/app/tools.py
T

314 lines
15 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
Служебные проверки хранилища для тестов и защиты (не HTTP API).
Запуск внутри контейнера api (он в сети storage):
python -m app.tools private-check # bucket приватный, права API ограничены
python -m app.tools stats # число объектов / attachments / upload_intents
python -m app.tools state <attachment_id> # объекты staging/originals, метаданные, журнал
python -m app.tools job <attachment_id> # lab5: job, попытки, события, варианты, объекты S3
python -m app.tools republish <attachment_id> # lab5: отправить ДУБЛЬ сообщения job в брокер
python -m app.tools queue # lab5: длина stream, pending, DLQ, outbox
python -m app.tools chat-attachments <chat_id> # lab5: вложения чата и их статусы (для тестов)
python -m app.tools limits-selftest # lab5: лимиты памяти/времени/пикселей worker
python -m app.tools find-local <sha256> # нет ли на диске реплики копии файла
Секреты не печатаются. Ключи объектов печатаются только для служебного prefix.
"""
from __future__ import annotations
import json
import sys
import urllib.error
import urllib.request
import uuid
import psycopg
from botocore.exceptions import ClientError
from . import storage
from .config import settings
def _anon(method: str, path: str, data: bytes | None = None) -> int:
"""HTTP-запрос к MinIO БЕЗ подписи (как посторонний клиент)."""
req = urllib.request.Request(settings.s3_endpoint.rstrip("/") + path, data=data, method=method)
try:
with urllib.request.urlopen(req, timeout=5) as r:
return r.status
except urllib.error.HTTPError as e:
return e.code
def _denied(fn) -> str:
try:
fn()
return "ALLOWED"
except ClientError as exc:
return exc.response.get("Error", {}).get("Code", "error")
def private_check() -> int:
s3 = storage.client()
bucket = settings.s3_bucket
probe = f"{storage.ORIGINALS_PREFIX}privacy-probe-{uuid.uuid4()}"
results: list[tuple[str, str, bool]] = []
def check(name: str, observed, ok: bool) -> None:
results.append((name, str(observed), ok))
s3.put_object(Bucket=bucket, Key=probe, Body=b"probe", ContentType="text/plain")
try:
# Позитивный контроль: объект существует и API-пользователь его читает.
body = s3.get_object(Bucket=bucket, Key=probe)["Body"].read()
check("API credentials: GET own object", "200", body == b"probe")
# Аноним: ни чтения, ни листинга, ни записи.
st = _anon("GET", f"/{bucket}/{probe}")
check("anonymous GET object", st, st == 403)
st = _anon("GET", f"/{bucket}?list-type=2")
check("anonymous LIST bucket", st, st == 403)
st = _anon("PUT", f"/{bucket}/{storage.ORIGINALS_PREFIX}anon-{uuid.uuid4()}", b"x")
check("anonymous PUT object", st, st == 403)
# Ограниченные права API-пользователя: только объекты своего bucket.
# MinIO без s3:ListAllMyBuckets либо отказывает, либо показывает только
# бакеты, на которые есть права, — чужие бакеты API-пользователь не видит.
try:
names = sorted(b["Name"] for b in s3.list_buckets().get("Buckets", []))
r = "only " + ",".join(names) if names else "empty"
ok = set(names) <= {bucket}
except ClientError as exc:
r = exc.response.get("Error", {}).get("Code", "error")
ok = r == "AccessDenied"
check("API credentials: ListBuckets (no foreign buckets)", r, ok)
r = _denied(lambda: s3.create_bucket(Bucket="lab4-forbidden-" + uuid.uuid4().hex[:8]))
check("API credentials: CreateBucket", r, r == "AccessDenied")
public = json.dumps({"Version": "2012-10-17", "Statement": [{
"Effect": "Allow", "Principal": {"AWS": ["*"]}, "Action": ["s3:GetObject"],
"Resource": [f"arn:aws:s3:::{bucket}/*"]}]})
r = _denied(lambda: s3.put_bucket_policy(Bucket=bucket, Policy=public))
check("API credentials: PutBucketPolicy (make public)", r, r == "AccessDenied")
r = _denied(lambda: s3.delete_bucket(Bucket=bucket))
check("API credentials: DeleteBucket", r, r == "AccessDenied")
finally:
s3.delete_object(Bucket=bucket, Key=probe)
width = max(len(n) for n, _, _ in results)
for name, observed, ok in results:
print(f" {'ok ' if ok else 'FAIL'} {name.ljust(width)} -> {observed}")
failed = sum(1 for _, _, ok in results if not ok)
print(f"\n{len(results) - failed} passed, {failed} failed")
return 1 if failed else 0
def _db():
return psycopg.connect(settings.database_url, connect_timeout=3)
def stats() -> int:
# Оригиналы и staging (lab5): сирота может остаться в любом из двух prefix.
objects = sum(1 for _ in storage.list_sync(storage.ORIGINALS_PREFIX)) \
+ sum(1 for _ in storage.list_sync(storage.STAGING_PREFIX))
with _db() as conn:
attachments = conn.execute("SELECT count(*) FROM attachments").fetchone()[0]
intents = conn.execute("SELECT count(*) FROM upload_intents").fetchone()[0]
print(json.dumps({"objects": objects, "attachments": attachments, "upload_intents": intents}))
return 0
def state(attachment_id: str) -> int:
aid = uuid.UUID(attachment_id)
original, staging = storage.object_key(aid), storage.staging_key(aid)
with _db() as conn:
att = conn.execute("SELECT status FROM attachments WHERE id = %s", (aid,)).fetchone()
intent = conn.execute("SELECT 1 FROM upload_intents WHERE attachment_id = %s", (aid,)).fetchone()
print(json.dumps({"staging_object_exists": storage.exists_sync(staging),
"original_object_exists": storage.exists_sync(original),
"attachment_row": att is not None,
"attachment_status": att[0] if att else None,
"upload_intent": intent is not None}))
return 0
def job(attachment_id: str) -> int:
"""Полное состояние обработки одного вложения (без секретов)."""
aid = uuid.UUID(attachment_id)
with _db() as conn:
j = conn.execute(
"SELECT job_id, status, attempt, max_attempts, error_code, last_error, replays, "
"lease_owner, lease_until FROM attachment_jobs WHERE attachment_id = %s", (aid,)).fetchone()
a = conn.execute("SELECT status, error_code, staging_key IS NOT NULL FROM attachments "
"WHERE id = %s", (aid,)).fetchone()
variants = conn.execute(
"SELECT variant, width, height, size_bytes, object_key FROM attachment_variants "
"WHERE attachment_id = %s ORDER BY variant", (aid,)).fetchall()
events = conn.execute(
"SELECT to_char(at, 'HH24:MI:SS.MS'), event, attempt, actor, coalesce(detail, '') "
"FROM job_events e WHERE job_id = %s ORDER BY id", (j[0],)).fetchall() if j else []
outbox = conn.execute("SELECT attempt, published_at IS NOT NULL FROM job_outbox WHERE job_id = %s "
"ORDER BY id", (j[0],)).fetchall() if j else []
objs = sorted(k for k, _, _ in storage.list_sync(f"{storage.VARIANTS_PREFIX}{aid}/"))
print(json.dumps({
"attachment": {"status": a[0], "error_code": a[1], "staging_present": a[2]} if a else None,
"job": {"job_id": str(j[0]), "status": j[1], "attempt": j[2], "max_attempts": j[3],
"error_code": j[4], "last_error": j[5], "replays": j[6], "lease_owner": j[7],
"lease_until": j[8].isoformat() if j[8] else None} if j else None,
"variant_rows": [{"name": v[0], "size": f"{v[1]}x{v[2]}", "bytes": v[3]} for v in variants],
"variant_objects_in_s3": len(objs),
"original_object_exists": storage.exists_sync(storage.object_key(aid)),
"staging_object_exists": storage.exists_sync(storage.staging_key(aid)),
"outbox": [{"attempt": o[0], "published": o[1]} for o in outbox],
"events": [" ".join(str(x) for x in e if x not in (None, "")) for e in events],
}, ensure_ascii=False, indent=2))
return 0
def republish(attachment_id: str) -> int:
"""Отправить в брокер копию сообщения ТЕКУЩЕЙ попытки job (проверка повторной доставки)."""
from . import broker
aid = uuid.UUID(attachment_id)
with _db() as conn:
j = conn.execute("SELECT job_id, pipeline_version, attempt, created_at FROM attachment_jobs "
"WHERE attachment_id = %s", (aid,)).fetchone()
if j is None:
print("job not found")
return 1
payload = broker.make_payload(j[0], aid, j[1], j[2], j[3])
print("published duplicate:", broker.publish_sync(broker.sync_client(), payload), json.dumps(payload))
return 0
def chat_attachments(chat_id: str) -> int:
with _db() as conn:
rows = conn.execute("SELECT id, status FROM attachments WHERE chat_id = %s ORDER BY created_at",
(uuid.UUID(chat_id),)).fetchall()
print(json.dumps([{"id": str(r[0]), "status": r[1]} for r in rows]))
return 0
def limits_selftest() -> int:
"""Защита worker от «тяжёлых» изображений проверяется на настоящем дочернем процессе
(тот же код и те же лимиты, что в worker), без API — API такие файлы и так отсекает."""
import multiprocessing
import tempfile
import time
from pathlib import Path
from PIL import Image
from . import imaging
ctx = multiprocessing.get_context("forkserver")
results = []
def run(name, src, ctype, max_pixels, memory_mb, timeout, slow=0.0, expect=""):
d = Path(tempfile.mkdtemp())
res = d / "r.json"
(d / "out").mkdir()
t0 = time.monotonic()
p = ctx.Process(target=imaging.child_main, args=(str(src), str(d / "out"), ctype, max_pixels,
memory_mb, int(timeout) + 5, str(res), slow))
p.start()
p.join(timeout)
killed = p.is_alive()
if killed:
p.kill()
p.join()
got = "timeout(killed)" if killed else (json.loads(res.read_text()) if res.exists() else f"exit {p.exitcode}")
outcome = got if isinstance(got, str) else ("ok" if got["ok"] else f'{got["code"]}: {got["detail"]}')
ok = expect in outcome
results.append(ok)
print(f" {'ok ' if ok else 'FAIL'} {name}: {outcome} ({time.monotonic() - t0:.1f}s)")
with tempfile.TemporaryDirectory() as tmp:
big = Path(tmp) / "big.png" # 6000×6000 = 36 Мп: в пределах 40 Мп, но ~100+ МБ после распаковки
Image.new("RGB", (6000, 6000), (200, 30, 30)).save(big, optimize=False, compress_level=1)
small = Path(tmp) / "small.png"
Image.new("RGBA", (320, 200), (0, 0, 255, 128)).save(small)
corrupt = Path(tmp) / "corrupt.png"
corrupt.write_bytes(small.read_bytes()[:200])
run("normal 320x200 with alpha", small, "image/png", 40_000_000, 768, 30, expect="ok")
run("pixel limit (36 Mp > limit 10 Mp)", big, "image/png", 10_000_000, 768, 30, expect="invalid_image: pixel limit")
run("memory limit (36 Mp, RLIMIT_AS 120 MiB)", big, "image/png", 40_000_000, 120, 30, expect="invalid_image")
run("corrupt/truncated PNG", corrupt, "image/png", 40_000_000, 768, 30, expect="invalid_image")
run("time limit (hangs 10 s, timeout 2 s)", small, "image/png", 40_000_000, 768, 2, slow=10, expect="timeout")
failed = results.count(False)
print(f"\n{len(results) - failed} passed, {failed} failed")
return 1 if failed else 0
def queue() -> int:
from . import broker
r = broker.sync_client()
try:
groups = r.xinfo_groups(settings.jobs_stream)
except Exception: # noqa: BLE001
groups = []
with _db() as conn:
pending_outbox = conn.execute("SELECT count(*) FROM job_outbox WHERE published_at IS NULL").fetchone()[0]
by_status = dict(conn.execute("SELECT status, count(*) FROM attachment_jobs GROUP BY status").fetchall())
print(json.dumps({
"stream_length": r.xlen(settings.jobs_stream) if r.exists(settings.jobs_stream) else 0,
"groups": [{"name": g["name"], "consumers": g["consumers"], "pending": g["pending"],
"lag": g.get("lag")} for g in groups],
"dlq_length": r.xlen(settings.dlq_stream) if r.exists(settings.dlq_stream) else 0,
"outbox_unpublished": pending_outbox,
"jobs_by_status": by_status,
}, ensure_ascii=False))
return 0
def find_local(digest: str) -> int:
"""Обойти файловую систему контейнера API и найти файлы с таким SHA-256.
Доказательство, что оригинал не хранится на реплике (только в S3)."""
import hashlib
import os
skip = ("/proc", "/sys", "/dev", "/usr", "/lib", "/bin", "/sbin", "/etc", "/var/lib/dpkg")
scanned, found = 0, []
for root, dirs, files in os.walk("/"):
if root.startswith(skip):
dirs[:] = []
continue
for name in files:
path = os.path.join(root, name)
try:
if not os.path.isfile(path) or os.path.getsize(path) > 32 * 1024 * 1024:
continue
h = hashlib.sha256()
with open(path, "rb") as f:
for block in iter(lambda: f.read(1 << 20), b""):
h.update(block)
scanned += 1
if h.hexdigest() == digest:
found.append(path)
except OSError:
continue
print(json.dumps({"files_scanned": scanned, "copies_found": found}))
return 1 if found else 0
def main(argv: list[str]) -> int:
if not argv:
print(__doc__)
return 2
cmd, args = argv[0], argv[1:]
if cmd == "private-check":
return private_check()
if cmd == "stats":
return stats()
if cmd == "state" and len(args) == 1:
return state(args[0])
if cmd == "job" and len(args) == 1:
return job(args[0])
if cmd == "republish" and len(args) == 1:
return republish(args[0])
if cmd == "queue":
return queue()
if cmd == "chat-attachments" and len(args) == 1:
return chat_attachments(args[0])
if cmd == "limits-selftest":
return limits_selftest()
if cmd == "find-local" and len(args) == 1:
return find_local(args[0])
print(__doc__)
return 2
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))