Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
221 lines
13 KiB
Python
221 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""Собственные проверки вложений lab4 (уровень 4), только стандартная библиотека.
|
||
|
||
Покрывают: слишком большой файл, повреждённые PNG/JPEG, подмену MIME, decompression bomb,
|
||
опасные filename, чужое/неизвестное attachment_id, вложение другого чата, постороннего пользователя,
|
||
идемпотентность с вложениями. Ни одна пользовательская ошибка не должна давать 500.
|
||
|
||
--with-docker дополнительно (через docker compose из корня репозитория):
|
||
* останавливает S3: загрузка/скачивание -> 503, текстовые сообщения и /health/ready работают,
|
||
после неудачной загрузки не остаётся строк 'uploading' (компенсация);
|
||
* останавливает API-реплику, принявшую загрузку, и скачивает файл через другую.
|
||
Синтетический fixture tests/fixtures/sample.jpg получен из sample.png (sips, quality 85).
|
||
"""
|
||
import json
|
||
import os
|
||
import struct
|
||
import subprocess
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
import uuid
|
||
import zlib
|
||
from urllib.parse import quote
|
||
from client import ROOT, Client, arguments, check, run
|
||
|
||
PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||
JPEG = (ROOT / 'tests/fixtures/sample.jpg').read_bytes()
|
||
MIB = 1024 * 1024
|
||
|
||
|
||
def png(width, height, idat=None, depth=8, color=2):
|
||
def chunk(kind, data):
|
||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
|
||
if idat is None:
|
||
row = b'\x00' + bytes(width * 3)
|
||
idat = zlib.compress(row * height, 9)
|
||
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, depth, color, 0, 0, 0))
|
||
+ chunk(b'IDAT', idat) + chunk(b'IEND', b''))
|
||
|
||
|
||
def flip(data, at):
|
||
return data[:at] + bytes([data[at] ^ 0xff]) + data[at + 1:]
|
||
|
||
|
||
def compose(*args, check_rc=True):
|
||
return subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=check_rc, capture_output=True, text=True).stdout
|
||
|
||
|
||
def sql(query):
|
||
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-c', query).strip()
|
||
|
||
|
||
def main():
|
||
def configure(p):
|
||
p.add_argument('--with-docker', action='store_true', help='also stop S3 and the uploading replica via docker compose')
|
||
args = arguments('Negative attachment checks for lab4; creates synthetic data', configure)
|
||
c = Client(args.base_url, args.ca_file, timeout=20)
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'Другой чат', 'member_ids': [alice['id'], eve['id']]}, schema='Chat')['json']
|
||
passed = []
|
||
|
||
def upload(data, ctype='image/png', expected=201, actor=alice, target=chat, name='file.png'):
|
||
r = c.call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (target['id'], quote(name, safe='')), expected,
|
||
actor=actor, raw=data, headers={'Content-Type': ctype}, schema='Attachment' if expected == 201 else None)
|
||
if expected >= 400:
|
||
check(r['json']['error']['code'] != 'internal_error', 'user error turned into 500')
|
||
return r
|
||
|
||
def ok(name):
|
||
passed.append(name)
|
||
print('PASS', name)
|
||
|
||
# Валидные форматы и побайтовое скачивание.
|
||
att_png = upload(PNG)['json']
|
||
att_jpg = upload(JPEG, 'image/jpeg', name='photo.jpg')['json']
|
||
check(att_jpg['content_type'] == 'image/jpeg' and att_jpg['status'] == 'ready', 'jpeg metadata')
|
||
for att, data, ctype in ((att_png, PNG, 'image/png'), (att_jpg, JPEG, 'image/jpeg')):
|
||
got = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
|
||
check(got['bytes'] == data and got['headers'].get_content_type() == ctype, 'download differs')
|
||
ok('valid PNG and JPEG upload/download byte-for-byte')
|
||
|
||
# Размер: API отклоняет 10 MiB + 1 байт, nginx — всё больше 11 MiB, оба JSON 413.
|
||
upload(PNG[:8] + bytes(10 * MIB + 1 - 8), expected=413)
|
||
req = urllib.request.Request(c.base + '/api/v1/chats/%s/attachments?filename=big.png' % chat['id'], data=bytes(12 * MIB), method='POST',
|
||
headers={'Authorization': 'Bearer ' + alice['token'], 'Content-Type': 'image/png'})
|
||
try:
|
||
urllib.request.urlopen(req, timeout=20)
|
||
check(False, 'oversized body accepted')
|
||
except urllib.error.HTTPError as e:
|
||
body = json.loads(e.read())
|
||
check(e.code == 413 and body['error']['code'] == 'payload_too_large' and e.headers.get('X-Request-Id'), 'nginx 413 must be contract JSON')
|
||
# Почти 10 MiB: случайные пиксели не сжимаются; у каждой строки байт фильтра 0.
|
||
big = png(1800, 1900, idat=zlib.compress(b''.join(b'\x00' + os.urandom(1800 * 3) for _ in range(1900)), 1))
|
||
check(len(big) <= 10 * MIB, 'fixture too large')
|
||
upload(big)
|
||
ok('10 MiB + 1 byte -> 413 (API), 12 MiB -> 413 JSON (nginx), valid %.1f MiB accepted' % (len(big) / MIB))
|
||
|
||
# Повреждённые изображения.
|
||
idat_at = PNG.find(b'IDAT') + 8
|
||
for broken in (flip(PNG, idat_at), PNG[:len(PNG) // 2], PNG + b'trailing', PNG[:8] + b'garbage' * 10):
|
||
upload(broken, expected=415)
|
||
for broken in (JPEG[:len(JPEG) // 2], JPEG[:4] + b'\x00' * 100, JPEG[:2] + b'\xff\xda\x00\x02' + JPEG[2:]):
|
||
upload(broken, 'image/jpeg', expected=415)
|
||
ok('corrupted/truncated PNG and JPEG -> 415')
|
||
|
||
# Подмена MIME: проверяется содержимое, а не заголовок.
|
||
upload(PNG, 'image/jpeg', expected=415)
|
||
upload(JPEG, 'image/png', expected=415)
|
||
upload(b'GIF89a' + bytes(100), 'image/png', expected=415)
|
||
upload(b'<svg xmlns="http://www.w3.org/2000/svg"/>', 'image/png', expected=415)
|
||
upload(PNG, 'text/plain', expected=415)
|
||
upload(PNG, 'image/gif', expected=415)
|
||
ok('MIME spoofing (content vs Content-Type) -> 415')
|
||
|
||
# Decompression bomb: заголовок > 40 Мпикс -> 413 до распаковки; ложный маленький заголовок -> 415;
|
||
# честные 36 Мпикс (≈108 МБ пикселей) принимаются без декодирования в RAM.
|
||
upload(png(10000, 10000, idat=zlib.compress(bytes(1000))), expected=413)
|
||
upload(png(100, 100, idat=zlib.compress(bytes(50 * MIB), 9)), expected=415)
|
||
upload(png(6000, 6000))
|
||
ok('decompression bomb: >40 MP -> 413, lying header -> 415, honest 36 MP accepted')
|
||
|
||
# filename: только отображение; пути, NUL, управляющие символы и длина — 400.
|
||
for name in ('../evil.png', 'a/b.png', 'a\\b.png', 'a\x00b.png', 'a\x07b.png', 'line\nbreak.png', ' ', '..', 'x' * 256):
|
||
upload(PNG, name=name, expected=400)
|
||
c.call('POST', '/api/v1/chats/%s/attachments' % chat['id'], 400, actor=alice, raw=PNG, headers={'Content-Type': 'image/png'})
|
||
for name in ('фото 😀.png', '😀' * 255):
|
||
got = upload(PNG, name=name)['json']
|
||
check(got['filename'] == name, 'filename changed')
|
||
ok('dangerous filenames -> 400, Unicode up to 255 code points preserved')
|
||
|
||
# Доступ: посторонний не загружает и не читает, без сессии — 401, неизвестный id — 404.
|
||
upload(PNG, actor=eve, expected=404)
|
||
for suffix in ('', '/content'):
|
||
c.call('GET', '/api/v1/attachments/' + att_png['id'] + suffix, 404, actor=eve)
|
||
c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()) + suffix, 404, actor=alice)
|
||
c.call('GET', '/api/v1/attachments/not-a-uuid', 400, actor=alice)
|
||
c.call('POST', '/api/v1/chats/%s/attachments?filename=a.png' % chat['id'], 401, raw=PNG, headers={'Content-Type': 'image/png'})
|
||
ok('outsider/unknown attachment -> 404, no session -> 401')
|
||
|
||
# attachment_ids в сообщении.
|
||
path = '/api/v1/chats/%s/messages' % chat['id']
|
||
foreign = upload(PNG, target=other)['json']
|
||
|
||
def send(body, expected=201, key=None):
|
||
return c.call('POST', path, expected, actor=alice, body=body, headers={'Idempotency-Key': key or uuid.uuid4().hex},
|
||
schema='Message' if expected == 201 else None)
|
||
send({'text': 'чужое', 'attachment_ids': [foreign['id']]}, 404)
|
||
send({'text': 'неизвестное', 'attachment_ids': [str(uuid.uuid4())]}, 404)
|
||
send({'text': 'смесь', 'attachment_ids': [att_png['id'], foreign['id']]}, 404)
|
||
for bad in (['not-a-uuid'], [att_png['id'], att_png['id']], [str(uuid.uuid4()) for _ in range(11)], None, att_png['id'], [1]):
|
||
send({'text': 'x', 'attachment_ids': bad}, 400)
|
||
send({'text': ''}, 400)
|
||
send({'text': '', 'attachment_ids': []}, 400)
|
||
send({'text': ' ', 'attachment_ids': [att_png['id']]}, 400)
|
||
two = send({'text': '', 'attachment_ids': [att_jpg['id'], att_png['id']]})['json']
|
||
check(two['attachment_ids'] == [att_jpg['id'], att_png['id']] and two['text'] == '', 'attachment order or text changed')
|
||
history = c.call('GET', path + '?limit=100', actor=bob, schema='MessagePage')['json']['items']
|
||
check(any(m == two for m in history), 'history lost attachment_ids')
|
||
ok('foreign/unknown attachment -> 404, malformed attachment_ids -> 400, order preserved')
|
||
|
||
# Идемпотентность: отсутствующий attachment_ids == [], другой набор с тем же ключом — 409.
|
||
key = uuid.uuid4().hex
|
||
first = send({'text': 'ключ'}, key=key)['json']
|
||
check(send({'attachment_ids': [], 'text': 'ключ'}, key=key)['json'] == first, 'missing vs [] must be equal')
|
||
send({'text': 'ключ', 'attachment_ids': [att_png['id']]}, 409, key=key)
|
||
key = uuid.uuid4().hex
|
||
first = send({'text': 'с файлом', 'attachment_ids': [att_png['id']]}, key=key)['json']
|
||
check(send({'text': 'с файлом', 'attachment_ids': [att_png['id']]}, key=key)['json'] == first, 'replay with attachment differs')
|
||
send({'text': 'с файлом', 'attachment_ids': [att_jpg['id']]}, 409, key=key)
|
||
ok('idempotency with attachment_ids (missing == [], different set -> 409)')
|
||
|
||
# Варианты: в lab4 есть только original.
|
||
c.call('GET', '/api/v1/attachments/%s/content?variant=original' % att_png['id'], actor=bob)
|
||
c.call('GET', '/api/v1/attachments/%s/content?variant=thumbnail' % att_png['id'], 404, actor=bob)
|
||
c.call('GET', '/api/v1/attachments/%s/content?variant=../../etc' % att_png['id'], 400, actor=bob)
|
||
ok('variant: original 200, absent thumbnail 404, garbage 400')
|
||
|
||
if args.with_docker:
|
||
# Недоступный S3: вложения 503, текстовый путь живёт, после сбоя нет зависших строк.
|
||
compose('stop', 's3')
|
||
try:
|
||
r = upload(PNG, expected=503)
|
||
check(r['json']['error']['code'] == 'unavailable', 'S3 outage must be 503 unavailable')
|
||
c.call('GET', '/api/v1/attachments/' + att_png['id'] + '/content', 503, actor=bob)
|
||
c.call('GET', '/api/v1/attachments/' + att_png['id'], actor=bob, schema='Attachment')
|
||
send({'text': 'S3 лежит, текст идёт'})
|
||
c.call('GET', '/health/ready', schema='Health')
|
||
check(sql("select count(*) from attachments where state='uploading'") == '0', 'failed upload left an uploading row')
|
||
finally:
|
||
compose('start', 's3')
|
||
deadline = time.monotonic() + 60
|
||
while True:
|
||
try:
|
||
got = c.call('GET', '/api/v1/attachments/' + att_png['id'] + '/content', actor=bob)
|
||
break
|
||
except AssertionError:
|
||
check(time.monotonic() < deadline, 'downloads did not recover after S3 start')
|
||
time.sleep(1)
|
||
check(got['bytes'] == PNG, 'download after S3 recovery differs')
|
||
ok('S3 down: attachments 503, messages/ready OK, no uploading rows; recovered after start')
|
||
|
||
# Остановка реплики, принявшей загрузку: другая скачивает тот же файл.
|
||
r = upload(PNG)
|
||
served = r['headers']['X-Instance-Id']
|
||
cid = [x for x in compose('ps', '-q', 'api-1', 'api-2').split() if x.startswith(served)]
|
||
check(len(cid) == 1, 'cannot map X-Instance-Id to a container')
|
||
subprocess.run(['docker', 'stop', cid[0]], check=True, capture_output=True)
|
||
try:
|
||
for _ in range(5):
|
||
got = c.call('GET', '/api/v1/attachments/' + r['json']['id'] + '/content', actor=bob)
|
||
check(got['bytes'] == PNG and got['headers']['X-Instance-Id'] != served, 'download through other replica failed')
|
||
finally:
|
||
subprocess.run(['docker', 'start', cid[0]], check=True, capture_output=True)
|
||
ok('upload replica %s stopped: other replica serves identical bytes' % served)
|
||
|
||
print('PASS: %d attachment checks' % len(passed))
|
||
|
||
|
||
if __name__ == '__main__':
|
||
run(main)
|