Вложения: приватный S3-bucket (RustFS 1.0.4) + метаданные в PostgreSQL, upload PNG/JPEG с лимитами до декодирования, авторизованный download, attachment_ids в сообщениях, идемпотентный bootstrap bucket, failover/persistence/S3-down тесты и REPORT.md. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
279 lines
15 KiB
Python
Executable File
279 lines
15 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Собственные проверки lab4 (уровень 3). Запуск: sh scripts/test-lab4.sh (внутри сервиса tester).
|
||
|
||
Покрывают критерии README «3»: PNG/JPEG upload -> 201 ready, побайтный download с SHA-256, доступ только
|
||
участникам, сообщение с attachment_ids, лимиты (10 MiB, 40 Мпикс до декодирования), MIME по содержимому,
|
||
опасные filename, failover загрузившей реплики, недоступность S3, приватность bucket вне API и отсутствие
|
||
локальной копии файла на API. Требуют доступ к docker: останавливают реплику и S3, затем возвращают.
|
||
"""
|
||
import hashlib
|
||
import os
|
||
import struct
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import uuid
|
||
import zlib
|
||
from pathlib import Path
|
||
from urllib.parse import quote
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / 'tests'))
|
||
from client import Client, Failure, check # noqa: E402
|
||
|
||
BASE_URL = os.environ.get('BASE_URL', 'http://lb:8080')
|
||
PROJECT = 'mtusi-lab4'
|
||
PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||
JPEG = (ROOT / 'student-tests/fixtures/sample.jpg').read_bytes()
|
||
|
||
|
||
def docker(*args: str) -> str:
|
||
return subprocess.run(['docker', *args], check=True, text=True, capture_output=True).stdout.strip()
|
||
|
||
|
||
def service_containers(service: str, running_only: bool = True) -> list[str]:
|
||
args = ['ps', '-q', '--filter', f'label=com.docker.compose.project={PROJECT}',
|
||
'--filter', f'label=com.docker.compose.service={service}']
|
||
if not running_only:
|
||
args.insert(1, '-a')
|
||
return docker(*args).split()
|
||
|
||
|
||
def wait_ready(c: Client, seconds: float = 60) -> None:
|
||
deadline = time.monotonic() + seconds
|
||
while True:
|
||
try:
|
||
c.call('GET', '/health/ready', schema='Health')
|
||
return
|
||
except Failure:
|
||
check(time.monotonic() < deadline, 'API did not become ready')
|
||
time.sleep(0.5)
|
||
|
||
|
||
def upload(c: Client, actor: dict, chat: dict, data: bytes, ctype: str, filename: str = 'photo.png',
|
||
expected=201) -> dict:
|
||
path = f"/api/v1/chats/{chat['id']}/attachments?filename={quote(filename, safe='')}"
|
||
return c.call('POST', path, expected, actor=actor, raw=data, headers={'Content-Type': ctype},
|
||
schema='Attachment' if expected == 201 else None)
|
||
|
||
|
||
def download(c: Client, actor: dict, att_id: str, expected=200) -> dict:
|
||
return c.call('GET', f'/api/v1/attachments/{att_id}/content', expected, actor=actor)
|
||
|
||
|
||
def png_chunk(kind: bytes, data: bytes) -> bytes:
|
||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
|
||
|
||
|
||
def png_header_only(width: int, height: int) -> bytes:
|
||
"""Валидная сигнатура и IHDR (с верным CRC) огромного изображения, без пиксельных данных."""
|
||
ihdr = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
|
||
idat = zlib.compress(b'\x00' * 64) # заведомо меньше, чем нужно: декодировать сервер не должен
|
||
return b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', ihdr) + png_chunk(b'IDAT', idat) + png_chunk(b'IEND', b'')
|
||
|
||
|
||
# ---------- тесты ----------
|
||
|
||
def test_png_and_jpeg_roundtrip(c: Client):
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
for data, ctype, name in ((PNG, 'image/png', 'Фото 1.png'), (JPEG, 'image/jpeg', 'кадр.jpg')):
|
||
att = upload(c, alice, chat, data, ctype, name)['json']
|
||
check(att['status'] == 'ready' and att['variants'] == [] and att['error_code'] is None, 'not ready')
|
||
check(att['filename'] == name and att['content_type'] == ctype, 'metadata mismatch')
|
||
check(att['sha256'] == hashlib.sha256(data).hexdigest() and att['size_bytes'] == len(data), 'checksum')
|
||
meta = c.call('GET', f"/api/v1/attachments/{att['id']}", actor=bob, schema='Attachment')['json']
|
||
check(meta == att, 'metadata via GET differs from upload response')
|
||
got = download(c, bob, att['id'])
|
||
check(got['bytes'] == data, 'downloaded bytes differ')
|
||
check(got['headers'].get_content_type() == ctype, 'wrong Content-Type on download')
|
||
check(hashlib.sha256(got['bytes']).hexdigest() == att['sha256'], 'sha256 mismatch')
|
||
for suffix in ('', '/content'):
|
||
c.call('GET', f"/api/v1/attachments/{att['id']}{suffix}", 404, actor=eve)
|
||
c.call('GET', f"/api/v1/attachments/{att['id']}/content", 401)
|
||
|
||
|
||
def test_message_with_attachments(c: Client):
|
||
alice, bob, _eve, chat = c.bootstrap()
|
||
a1 = upload(c, alice, chat, PNG, 'image/png')['json']
|
||
a2 = upload(c, bob, chat, JPEG, 'image/jpeg', 'b.jpg')['json']
|
||
key = uuid.uuid4().hex
|
||
msg = c.message(bob, chat, '', key=key, attachments=[a2['id'], a1['id']])['json'] # любой участник
|
||
check(msg['attachment_ids'] == [a2['id'], a1['id']] and msg['text'] == '', 'attachment order lost')
|
||
replay = c.message(bob, chat, '', key=key, attachments=[a2['id'], a1['id']])['json']
|
||
check(replay == msg, 'idempotent replay with attachments differs')
|
||
plain = c.message(alice, chat, 'только текст')['json']
|
||
check(plain['attachment_ids'] == [], 'text message must have attachment_ids=[]')
|
||
history = c.call('GET', f"/api/v1/chats/{chat['id']}/messages?limit=100", actor=alice,
|
||
schema='MessagePage')['json']['items']
|
||
check([m['attachment_ids'] for m in history] == [[a2['id'], a1['id']], []], 'history lost attachments')
|
||
h = {'Idempotency-Key': uuid.uuid4().hex}
|
||
c.call('POST', f"/api/v1/chats/{chat['id']}/messages", 400, actor=alice, body={'text': ' '}, headers=h)
|
||
c.call('POST', f"/api/v1/chats/{chat['id']}/messages", 400, actor=alice,
|
||
body={'text': '', 'attachment_ids': [a1['id'], a1['id']]}, headers=h)
|
||
|
||
|
||
def test_foreign_attachment_rejected(c: Client):
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
other = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'чужой', 'member_ids': [eve['id']]},
|
||
schema='Chat')['json']
|
||
foreign = upload(c, eve, other, PNG, 'image/png')['json']
|
||
h = {'Idempotency-Key': uuid.uuid4().hex}
|
||
# Вложение другого чата и несуществующее — 404, сообщение не создаётся.
|
||
for ids in ([foreign['id']], [str(uuid.uuid4())]):
|
||
c.call('POST', f"/api/v1/chats/{chat['id']}/messages", 404, actor=alice,
|
||
body={'text': 'x', 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex})
|
||
items = c.call('GET', f"/api/v1/chats/{chat['id']}/messages", actor=alice, schema='MessagePage')['json']['items']
|
||
check(items == [], 'message with foreign attachment was created')
|
||
# Посторонний не может загрузить файл в чужой чат.
|
||
upload(c, eve, chat, PNG, 'image/png', expected=404)
|
||
c.call('POST', f"/api/v1/chats/{chat['id']}/messages", 404, actor=eve,
|
||
body={'text': 'x'}, headers=h)
|
||
|
||
|
||
def test_limits_and_mime(c: Client):
|
||
alice, _bob, _eve, chat = c.bootstrap()
|
||
# 10 MiB + 1 байт: 413 (тело дальше лимита сервер не читает).
|
||
upload(c, alice, chat, PNG[:8] + b'\0' * (10 * 1024 * 1024 - 7), 'image/png', expected=413)
|
||
# Ровно 10 MiB проходит проверку размера (а затем отклоняется как повреждённый PNG — 415, не 413).
|
||
upload(c, alice, chat, PNG[:8] + b'\0' * (10 * 1024 * 1024 - 8), 'image/png', expected=415)
|
||
# 10000 x 5000 = 50 Мпикс по заголовку -> 413 до декодирования (файл всего ~100 байт).
|
||
upload(c, alice, chat, png_header_only(10000, 5000), 'image/png', expected=413)
|
||
# 6000 x 6000 = 36 Мпикс укладывается в лимит, но пикселей нет -> повреждённый -> 415.
|
||
upload(c, alice, chat, png_header_only(6000, 6000), 'image/png', expected=415)
|
||
# MIME по содержимому: PNG с Content-Type image/jpeg и наоборот -> 415.
|
||
upload(c, alice, chat, PNG, 'image/jpeg', expected=415)
|
||
upload(c, alice, chat, JPEG, 'image/png', expected=415)
|
||
# Не изображение, обрезанные файлы, пустое тело, чужие Content-Type.
|
||
upload(c, alice, chat, b'GIF89a' + b'\0' * 100, 'image/png', expected=415)
|
||
upload(c, alice, chat, PNG[:len(PNG) // 2], 'image/png', expected=415)
|
||
upload(c, alice, chat, JPEG[:len(JPEG) // 2], 'image/jpeg', expected=415)
|
||
upload(c, alice, chat, b'', 'image/png', expected=415)
|
||
upload(c, alice, chat, PNG, 'text/plain', expected=415)
|
||
upload(c, alice, chat, PNG, 'multipart/form-data; boundary=x', expected=415)
|
||
|
||
|
||
def test_dangerous_filenames(c: Client):
|
||
alice, _bob, _eve, chat = c.bootstrap()
|
||
for name in ('../etc/passwd', 'a/b.png', 'a\\b.png', 'nul\x00.png', 'tab\t.png', 'new\nline.png',
|
||
'\x7f.png', '', ' ', '..', 'x' * 256):
|
||
upload(c, alice, chat, PNG, 'image/png', name, expected=400)
|
||
c.call('POST', f"/api/v1/chats/{chat['id']}/attachments", 400, actor=alice, raw=PNG,
|
||
headers={'Content-Type': 'image/png'}) # без filename
|
||
ok = upload(c, alice, chat, PNG, 'image/png', 'отчёт (финал) ✓.png')['json']
|
||
check(ok['filename'] == 'отчёт (финал) ✓.png', 'unicode filename changed')
|
||
upload(c, alice, chat, PNG, 'image/png', 'x' * 255)
|
||
|
||
|
||
def test_failover_of_uploading_replica(c: Client):
|
||
"""Upload -> остановить реплику, которая его обслужила -> download через другую = тот же файл."""
|
||
alice, bob, _eve, chat = c.bootstrap()
|
||
response = upload(c, alice, chat, PNG, 'image/png')
|
||
att = response['json']
|
||
served_by = response['headers']['X-Instance-Id']
|
||
cid = served_by.rsplit('-', 1)[0]
|
||
check(cid in [x[:12] for x in service_containers('api')], 'uploader replica not found among containers')
|
||
docker('stop', '-t', '20', cid)
|
||
try:
|
||
time.sleep(2)
|
||
got = download(c, bob, att['id'])
|
||
check(got['headers']['X-Instance-Id'] != served_by, 'download served by stopped replica')
|
||
check(got['bytes'] == PNG and hashlib.sha256(got['bytes']).hexdigest() == att['sha256'],
|
||
'file differs after uploader replica stopped')
|
||
print(f' uploaded via {served_by}, downloaded via {got["headers"]["X-Instance-Id"]}')
|
||
finally:
|
||
docker('start', cid)
|
||
wait_ready(c)
|
||
|
||
|
||
def test_s3_unavailable(c: Client):
|
||
"""S3 остановлен: upload/download -> 503 без 500, текстовые сообщения работают, после старта — всё снова."""
|
||
alice, bob, _eve, chat = c.bootstrap()
|
||
att = upload(c, alice, chat, PNG, 'image/png')['json']
|
||
s3 = service_containers('s3')
|
||
check(len(s3) == 1, 'expected one s3 container')
|
||
docker('stop', s3[0])
|
||
try:
|
||
started = time.monotonic()
|
||
upload(c, alice, chat, PNG, 'image/png', expected=503)
|
||
download(c, bob, att['id'], expected=503)
|
||
check(time.monotonic() - started < 15, 'S3 failure is not bounded by timeouts')
|
||
c.message(alice, chat, 'S3 недоступен, а текст отправляется')
|
||
c.call('GET', f"/api/v1/attachments/{att['id']}", actor=bob, schema='Attachment') # метаданные в БД
|
||
c.call('GET', '/health/ready', schema='Health')
|
||
finally:
|
||
docker('start', s3[0])
|
||
deadline = time.monotonic() + 60
|
||
while True:
|
||
try:
|
||
check(download(c, bob, att['id'])['bytes'] == PNG, 'file changed after S3 restart')
|
||
break
|
||
except Failure:
|
||
check(time.monotonic() < deadline, 'S3 did not come back')
|
||
time.sleep(1)
|
||
upload(c, alice, chat, JPEG, 'image/jpeg', 'after.jpg')
|
||
|
||
|
||
def test_bucket_private_outside_api(c: Client):
|
||
"""Прямой анонимный GET объекта в S3 (в обход API) отклоняется; ключ в API не раскрывается."""
|
||
alice, _bob, _eve, chat = c.bootstrap()
|
||
att = upload(c, alice, chat, PNG, 'image/png')['json']
|
||
check('object_key' not in att and 'attachments/' not in str(att), 'object key leaked to API')
|
||
api = service_containers('api')[0]
|
||
code = docker('exec', api, 'python', '-c',
|
||
'import urllib.request,urllib.error,sys\n'
|
||
'try:\n'
|
||
f' urllib.request.urlopen("http://s3:9000/messenger-attachments/attachments/{att["id"]}", timeout=5)\n'
|
||
' print(200)\n'
|
||
'except urllib.error.HTTPError as e:\n'
|
||
' print(e.code)\n')
|
||
check(code in ('401', '403'), f'anonymous S3 GET returned {code}')
|
||
# Листинг bucket анонимно тоже запрещён.
|
||
code = docker('exec', api, 'python', '-c',
|
||
'import urllib.request,urllib.error\n'
|
||
'try:\n'
|
||
' urllib.request.urlopen("http://s3:9000/messenger-attachments", timeout=5); print(200)\n'
|
||
'except urllib.error.HTTPError as e:\n'
|
||
' print(e.code)\n')
|
||
check(code in ('401', '403'), f'anonymous S3 list returned {code}')
|
||
print(f' anonymous GET object/list -> {code}')
|
||
|
||
|
||
def test_no_local_copy_on_api(c: Client):
|
||
"""У API read-only корневая ФС и пустой tmpfs: оригинал не хранится локально."""
|
||
alice, _bob, _eve, chat = c.bootstrap()
|
||
upload(c, alice, chat, PNG, 'image/png')
|
||
for cid in service_containers('api'):
|
||
check(docker('inspect', '-f', '{{.HostConfig.ReadonlyRootfs}}', cid) == 'true', 'api rootfs is writable')
|
||
# Допустимы только read-only секреты compose (/run/secrets/*); volume и записываемых bind нет.
|
||
mounts = docker('inspect', '-f', '{{range .Mounts}}{{.Type}}:{{.Destination}}:{{.RW}} {{end}}', cid).split()
|
||
writable = [m for m in mounts if m.startswith('volume:') or
|
||
(m.startswith('bind:') and not (m.startswith('bind:/run/secrets/') and m.endswith(':false')))]
|
||
check(not writable, f'api has persistent/writable mounts: {writable}')
|
||
files = docker('exec', cid, 'find', '/tmp', '-type', 'f')
|
||
check(files == '', f'unexpected files in api /tmp: {files}')
|
||
|
||
|
||
TESTS = [test_png_and_jpeg_roundtrip, test_message_with_attachments, test_foreign_attachment_rejected,
|
||
test_limits_and_mime, test_dangerous_filenames, test_bucket_private_outside_api,
|
||
test_no_local_copy_on_api, test_failover_of_uploading_replica, test_s3_unavailable]
|
||
|
||
|
||
def main() -> int:
|
||
c = Client(BASE_URL, timeout=30)
|
||
wait_ready(c)
|
||
failed = 0
|
||
for test in TESTS:
|
||
started = time.monotonic()
|
||
try:
|
||
test(c)
|
||
print(f'PASS {test.__name__} ({time.monotonic() - started:.1f}s)', flush=True)
|
||
except (Failure, subprocess.CalledProcessError, OSError) as exc:
|
||
failed += 1
|
||
print(f'FAIL {test.__name__}: {exc}', flush=True)
|
||
print(f'{len(TESTS) - failed}/{len(TESTS)} passed')
|
||
return 1 if failed else 0
|
||
|
||
|
||
if __name__ == '__main__':
|
||
sys.exit(main())
|