forked from mtusi2026/mtusi2026lab5
314 lines
15 KiB
Python
314 lines
15 KiB
Python
"""
|
||
Служебные проверки хранилища для тестов и защиты (не HTTP API).
|
||
Запуск внутри контейнера api (он в сети storage):
|
||
|
||
python -m app.tools private-check # bucket приватный, права API ограничены
|
||
python -m app.tools stats # число объектов / attachments / upload_intents
|
||
python -m app.tools state <attachment_id> # объекты staging/originals, метаданные, журнал
|
||
python -m app.tools job <attachment_id> # lab5: job, попытки, события, варианты, объекты S3
|
||
python -m app.tools republish <attachment_id> # lab5: отправить ДУБЛЬ сообщения job в брокер
|
||
python -m app.tools queue # lab5: длина stream, pending, DLQ, outbox
|
||
python -m app.tools chat-attachments <chat_id> # lab5: вложения чата и их статусы (для тестов)
|
||
python -m app.tools limits-selftest # lab5: лимиты памяти/времени/пикселей worker
|
||
python -m app.tools find-local <sha256> # нет ли на диске реплики копии файла
|
||
|
||
Секреты не печатаются. Ключи объектов печатаются только для служебного prefix.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import sys
|
||
import urllib.error
|
||
import urllib.request
|
||
import uuid
|
||
|
||
import psycopg
|
||
from botocore.exceptions import ClientError
|
||
|
||
from . import storage
|
||
from .config import settings
|
||
|
||
|
||
def _anon(method: str, path: str, data: bytes | None = None) -> int:
|
||
"""HTTP-запрос к MinIO БЕЗ подписи (как посторонний клиент)."""
|
||
req = urllib.request.Request(settings.s3_endpoint.rstrip("/") + path, data=data, method=method)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=5) as r:
|
||
return r.status
|
||
except urllib.error.HTTPError as e:
|
||
return e.code
|
||
|
||
|
||
def _denied(fn) -> str:
|
||
try:
|
||
fn()
|
||
return "ALLOWED"
|
||
except ClientError as exc:
|
||
return exc.response.get("Error", {}).get("Code", "error")
|
||
|
||
|
||
def private_check() -> int:
|
||
s3 = storage.client()
|
||
bucket = settings.s3_bucket
|
||
probe = f"{storage.ORIGINALS_PREFIX}privacy-probe-{uuid.uuid4()}"
|
||
results: list[tuple[str, str, bool]] = []
|
||
|
||
def check(name: str, observed, ok: bool) -> None:
|
||
results.append((name, str(observed), ok))
|
||
|
||
s3.put_object(Bucket=bucket, Key=probe, Body=b"probe", ContentType="text/plain")
|
||
try:
|
||
# Позитивный контроль: объект существует и API-пользователь его читает.
|
||
body = s3.get_object(Bucket=bucket, Key=probe)["Body"].read()
|
||
check("API credentials: GET own object", "200", body == b"probe")
|
||
# Аноним: ни чтения, ни листинга, ни записи.
|
||
st = _anon("GET", f"/{bucket}/{probe}")
|
||
check("anonymous GET object", st, st == 403)
|
||
st = _anon("GET", f"/{bucket}?list-type=2")
|
||
check("anonymous LIST bucket", st, st == 403)
|
||
st = _anon("PUT", f"/{bucket}/{storage.ORIGINALS_PREFIX}anon-{uuid.uuid4()}", b"x")
|
||
check("anonymous PUT object", st, st == 403)
|
||
# Ограниченные права API-пользователя: только объекты своего bucket.
|
||
# MinIO без s3:ListAllMyBuckets либо отказывает, либо показывает только
|
||
# бакеты, на которые есть права, — чужие бакеты API-пользователь не видит.
|
||
try:
|
||
names = sorted(b["Name"] for b in s3.list_buckets().get("Buckets", []))
|
||
r = "only " + ",".join(names) if names else "empty"
|
||
ok = set(names) <= {bucket}
|
||
except ClientError as exc:
|
||
r = exc.response.get("Error", {}).get("Code", "error")
|
||
ok = r == "AccessDenied"
|
||
check("API credentials: ListBuckets (no foreign buckets)", r, ok)
|
||
r = _denied(lambda: s3.create_bucket(Bucket="lab4-forbidden-" + uuid.uuid4().hex[:8]))
|
||
check("API credentials: CreateBucket", r, r == "AccessDenied")
|
||
public = json.dumps({"Version": "2012-10-17", "Statement": [{
|
||
"Effect": "Allow", "Principal": {"AWS": ["*"]}, "Action": ["s3:GetObject"],
|
||
"Resource": [f"arn:aws:s3:::{bucket}/*"]}]})
|
||
r = _denied(lambda: s3.put_bucket_policy(Bucket=bucket, Policy=public))
|
||
check("API credentials: PutBucketPolicy (make public)", r, r == "AccessDenied")
|
||
r = _denied(lambda: s3.delete_bucket(Bucket=bucket))
|
||
check("API credentials: DeleteBucket", r, r == "AccessDenied")
|
||
finally:
|
||
s3.delete_object(Bucket=bucket, Key=probe)
|
||
|
||
width = max(len(n) for n, _, _ in results)
|
||
for name, observed, ok in results:
|
||
print(f" {'ok ' if ok else 'FAIL'} {name.ljust(width)} -> {observed}")
|
||
failed = sum(1 for _, _, ok in results if not ok)
|
||
print(f"\n{len(results) - failed} passed, {failed} failed")
|
||
return 1 if failed else 0
|
||
|
||
|
||
def _db():
|
||
return psycopg.connect(settings.database_url, connect_timeout=3)
|
||
|
||
|
||
def stats() -> int:
|
||
# Оригиналы и staging (lab5): сирота может остаться в любом из двух prefix.
|
||
objects = sum(1 for _ in storage.list_sync(storage.ORIGINALS_PREFIX)) \
|
||
+ sum(1 for _ in storage.list_sync(storage.STAGING_PREFIX))
|
||
with _db() as conn:
|
||
attachments = conn.execute("SELECT count(*) FROM attachments").fetchone()[0]
|
||
intents = conn.execute("SELECT count(*) FROM upload_intents").fetchone()[0]
|
||
print(json.dumps({"objects": objects, "attachments": attachments, "upload_intents": intents}))
|
||
return 0
|
||
|
||
|
||
def state(attachment_id: str) -> int:
|
||
aid = uuid.UUID(attachment_id)
|
||
original, staging = storage.object_key(aid), storage.staging_key(aid)
|
||
with _db() as conn:
|
||
att = conn.execute("SELECT status FROM attachments WHERE id = %s", (aid,)).fetchone()
|
||
intent = conn.execute("SELECT 1 FROM upload_intents WHERE attachment_id = %s", (aid,)).fetchone()
|
||
print(json.dumps({"staging_object_exists": storage.exists_sync(staging),
|
||
"original_object_exists": storage.exists_sync(original),
|
||
"attachment_row": att is not None,
|
||
"attachment_status": att[0] if att else None,
|
||
"upload_intent": intent is not None}))
|
||
return 0
|
||
|
||
|
||
def job(attachment_id: str) -> int:
|
||
"""Полное состояние обработки одного вложения (без секретов)."""
|
||
aid = uuid.UUID(attachment_id)
|
||
with _db() as conn:
|
||
j = conn.execute(
|
||
"SELECT job_id, status, attempt, max_attempts, error_code, last_error, replays, "
|
||
"lease_owner, lease_until FROM attachment_jobs WHERE attachment_id = %s", (aid,)).fetchone()
|
||
a = conn.execute("SELECT status, error_code, staging_key IS NOT NULL FROM attachments "
|
||
"WHERE id = %s", (aid,)).fetchone()
|
||
variants = conn.execute(
|
||
"SELECT variant, width, height, size_bytes, object_key FROM attachment_variants "
|
||
"WHERE attachment_id = %s ORDER BY variant", (aid,)).fetchall()
|
||
events = conn.execute(
|
||
"SELECT to_char(at, 'HH24:MI:SS.MS'), event, attempt, actor, coalesce(detail, '') "
|
||
"FROM job_events e WHERE job_id = %s ORDER BY id", (j[0],)).fetchall() if j else []
|
||
outbox = conn.execute("SELECT attempt, published_at IS NOT NULL FROM job_outbox WHERE job_id = %s "
|
||
"ORDER BY id", (j[0],)).fetchall() if j else []
|
||
objs = sorted(k for k, _, _ in storage.list_sync(f"{storage.VARIANTS_PREFIX}{aid}/"))
|
||
print(json.dumps({
|
||
"attachment": {"status": a[0], "error_code": a[1], "staging_present": a[2]} if a else None,
|
||
"job": {"job_id": str(j[0]), "status": j[1], "attempt": j[2], "max_attempts": j[3],
|
||
"error_code": j[4], "last_error": j[5], "replays": j[6], "lease_owner": j[7],
|
||
"lease_until": j[8].isoformat() if j[8] else None} if j else None,
|
||
"variant_rows": [{"name": v[0], "size": f"{v[1]}x{v[2]}", "bytes": v[3]} for v in variants],
|
||
"variant_objects_in_s3": len(objs),
|
||
"original_object_exists": storage.exists_sync(storage.object_key(aid)),
|
||
"staging_object_exists": storage.exists_sync(storage.staging_key(aid)),
|
||
"outbox": [{"attempt": o[0], "published": o[1]} for o in outbox],
|
||
"events": [" ".join(str(x) for x in e if x not in (None, "")) for e in events],
|
||
}, ensure_ascii=False, indent=2))
|
||
return 0
|
||
|
||
|
||
def republish(attachment_id: str) -> int:
|
||
"""Отправить в брокер копию сообщения ТЕКУЩЕЙ попытки job (проверка повторной доставки)."""
|
||
from . import broker
|
||
aid = uuid.UUID(attachment_id)
|
||
with _db() as conn:
|
||
j = conn.execute("SELECT job_id, pipeline_version, attempt, created_at FROM attachment_jobs "
|
||
"WHERE attachment_id = %s", (aid,)).fetchone()
|
||
if j is None:
|
||
print("job not found")
|
||
return 1
|
||
payload = broker.make_payload(j[0], aid, j[1], j[2], j[3])
|
||
print("published duplicate:", broker.publish_sync(broker.sync_client(), payload), json.dumps(payload))
|
||
return 0
|
||
|
||
|
||
def chat_attachments(chat_id: str) -> int:
|
||
with _db() as conn:
|
||
rows = conn.execute("SELECT id, status FROM attachments WHERE chat_id = %s ORDER BY created_at",
|
||
(uuid.UUID(chat_id),)).fetchall()
|
||
print(json.dumps([{"id": str(r[0]), "status": r[1]} for r in rows]))
|
||
return 0
|
||
|
||
|
||
def limits_selftest() -> int:
|
||
"""Защита worker от «тяжёлых» изображений проверяется на настоящем дочернем процессе
|
||
(тот же код и те же лимиты, что в worker), без API — API такие файлы и так отсекает."""
|
||
import multiprocessing
|
||
import tempfile
|
||
import time
|
||
from pathlib import Path
|
||
from PIL import Image
|
||
from . import imaging
|
||
ctx = multiprocessing.get_context("forkserver")
|
||
results = []
|
||
|
||
def run(name, src, ctype, max_pixels, memory_mb, timeout, slow=0.0, expect=""):
|
||
d = Path(tempfile.mkdtemp())
|
||
res = d / "r.json"
|
||
(d / "out").mkdir()
|
||
t0 = time.monotonic()
|
||
p = ctx.Process(target=imaging.child_main, args=(str(src), str(d / "out"), ctype, max_pixels,
|
||
memory_mb, int(timeout) + 5, str(res), slow))
|
||
p.start()
|
||
p.join(timeout)
|
||
killed = p.is_alive()
|
||
if killed:
|
||
p.kill()
|
||
p.join()
|
||
got = "timeout(killed)" if killed else (json.loads(res.read_text()) if res.exists() else f"exit {p.exitcode}")
|
||
outcome = got if isinstance(got, str) else ("ok" if got["ok"] else f'{got["code"]}: {got["detail"]}')
|
||
ok = expect in outcome
|
||
results.append(ok)
|
||
print(f" {'ok ' if ok else 'FAIL'} {name}: {outcome} ({time.monotonic() - t0:.1f}s)")
|
||
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
big = Path(tmp) / "big.png" # 6000×6000 = 36 Мп: в пределах 40 Мп, но ~100+ МБ после распаковки
|
||
Image.new("RGB", (6000, 6000), (200, 30, 30)).save(big, optimize=False, compress_level=1)
|
||
small = Path(tmp) / "small.png"
|
||
Image.new("RGBA", (320, 200), (0, 0, 255, 128)).save(small)
|
||
corrupt = Path(tmp) / "corrupt.png"
|
||
corrupt.write_bytes(small.read_bytes()[:200])
|
||
run("normal 320x200 with alpha", small, "image/png", 40_000_000, 768, 30, expect="ok")
|
||
run("pixel limit (36 Mp > limit 10 Mp)", big, "image/png", 10_000_000, 768, 30, expect="invalid_image: pixel limit")
|
||
run("memory limit (36 Mp, RLIMIT_AS 120 MiB)", big, "image/png", 40_000_000, 120, 30, expect="invalid_image")
|
||
run("corrupt/truncated PNG", corrupt, "image/png", 40_000_000, 768, 30, expect="invalid_image")
|
||
run("time limit (hangs 10 s, timeout 2 s)", small, "image/png", 40_000_000, 768, 2, slow=10, expect="timeout")
|
||
failed = results.count(False)
|
||
print(f"\n{len(results) - failed} passed, {failed} failed")
|
||
return 1 if failed else 0
|
||
|
||
|
||
def queue() -> int:
|
||
from . import broker
|
||
r = broker.sync_client()
|
||
try:
|
||
groups = r.xinfo_groups(settings.jobs_stream)
|
||
except Exception: # noqa: BLE001
|
||
groups = []
|
||
with _db() as conn:
|
||
pending_outbox = conn.execute("SELECT count(*) FROM job_outbox WHERE published_at IS NULL").fetchone()[0]
|
||
by_status = dict(conn.execute("SELECT status, count(*) FROM attachment_jobs GROUP BY status").fetchall())
|
||
print(json.dumps({
|
||
"stream_length": r.xlen(settings.jobs_stream) if r.exists(settings.jobs_stream) else 0,
|
||
"groups": [{"name": g["name"], "consumers": g["consumers"], "pending": g["pending"],
|
||
"lag": g.get("lag")} for g in groups],
|
||
"dlq_length": r.xlen(settings.dlq_stream) if r.exists(settings.dlq_stream) else 0,
|
||
"outbox_unpublished": pending_outbox,
|
||
"jobs_by_status": by_status,
|
||
}, ensure_ascii=False))
|
||
return 0
|
||
|
||
|
||
def find_local(digest: str) -> int:
|
||
"""Обойти файловую систему контейнера API и найти файлы с таким SHA-256.
|
||
Доказательство, что оригинал не хранится на реплике (только в S3)."""
|
||
import hashlib
|
||
import os
|
||
skip = ("/proc", "/sys", "/dev", "/usr", "/lib", "/bin", "/sbin", "/etc", "/var/lib/dpkg")
|
||
scanned, found = 0, []
|
||
for root, dirs, files in os.walk("/"):
|
||
if root.startswith(skip):
|
||
dirs[:] = []
|
||
continue
|
||
for name in files:
|
||
path = os.path.join(root, name)
|
||
try:
|
||
if not os.path.isfile(path) or os.path.getsize(path) > 32 * 1024 * 1024:
|
||
continue
|
||
h = hashlib.sha256()
|
||
with open(path, "rb") as f:
|
||
for block in iter(lambda: f.read(1 << 20), b""):
|
||
h.update(block)
|
||
scanned += 1
|
||
if h.hexdigest() == digest:
|
||
found.append(path)
|
||
except OSError:
|
||
continue
|
||
print(json.dumps({"files_scanned": scanned, "copies_found": found}))
|
||
return 1 if found else 0
|
||
|
||
|
||
def main(argv: list[str]) -> int:
|
||
if not argv:
|
||
print(__doc__)
|
||
return 2
|
||
cmd, args = argv[0], argv[1:]
|
||
if cmd == "private-check":
|
||
return private_check()
|
||
if cmd == "stats":
|
||
return stats()
|
||
if cmd == "state" and len(args) == 1:
|
||
return state(args[0])
|
||
if cmd == "job" and len(args) == 1:
|
||
return job(args[0])
|
||
if cmd == "republish" and len(args) == 1:
|
||
return republish(args[0])
|
||
if cmd == "queue":
|
||
return queue()
|
||
if cmd == "chat-attachments" and len(args) == 1:
|
||
return chat_attachments(args[0])
|
||
if cmd == "limits-selftest":
|
||
return limits_selftest()
|
||
if cmd == "find-local" and len(args) == 1:
|
||
return find_local(args[0])
|
||
print(__doc__)
|
||
return 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main(sys.argv[1:]))
|