Port final lab 5 validation, session and PNG recovery fixes
This commit is contained in:
@@ -0,0 +1,28 @@
|
||||
# Поздний перенос ЛР5 → ЛР6
|
||||
|
||||
Источник только immutable snapshot `lab5-c7b8feb805d151e878d77b0dff47320a9436b172`; диапазон `f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071..c7b8feb805d151e878d77b0dff47320a9436b172`. Исходный HEAD ЛР6 `237a7c93e507b984add5b49c51c6720a6bfca505`, дерево было чистым. Полный перечень всех изменённых путей, SHA-256 источника и решение по каждому: [source-inventory.json](forward-published/source-inventory.json).
|
||||
|
||||
## Перенос
|
||||
|
||||
| Группа | Изменение | Разрешение |
|
||||
|---|---|---|
|
||||
| identity handlers/postgres/valkey | Отзыв сессии, блокировка чтения, nonce заполнения кэша | Полный diff; SQLAlchemy и Lua, схема без изменений |
|
||||
| identity authentication/values, messaging values | NUL и не-ASCII Basic | Полный diff; ошибки по существующему контракту |
|
||||
| attachments images.py и png.py | Проверка IDAT, исходная tRNS глубина, tiles памяти | Полный diff; OTel S3/worker не изменён |
|
||||
| infra/haproxy/haproxy.cfg | JSON escaping request_id в 502/503/504 | Полный diff |
|
||||
| infra/rabbitmq/definitions.json | delivery-limit=-1 | Сохранено описание lab6 и существующая топология/DLX |
|
||||
| student-tests (все пути inventory) | Регрессии источника | Сохранены OTelJobObserver/provider ЛР6; прямые handle_job получают observer. Boundary PNG генерируются локально, без зависимости от evidence ЛР5; default project/port заменены на ЛР6 |
|
||||
| openspec/specs (10 capabilities) | 11 изменённых requirements | Полные delta blocks, исходные префиксы capabilities сохранены |
|
||||
| openspec/config.yaml | Версии SQLAlchemy/Alembic, provenance | Применимые строки влиты, контекст ЛР6 сохранён |
|
||||
|
||||
## Неприменимые артефакты
|
||||
|
||||
AGENTS.md, REPORT.md и evidence/README.md источника относятся к ЛР5 и не заменяют документы ЛР6. Результаты evidence источника относятся к другому приложению/SHA и не копируются как доказательство ЛР6. Исполняемая логика двух targeted.py адаптирована в student-tests/tools/forward_runtime.py; их файлы результатов не импортированы. Четыре новых source OpenSpec archives — история выполненных этапов ЛР5; их требования перенесены объединённым change `forward-lab5-published-fixes`, их завершение не выдаётся за проверку ЛР6. Каждый исключённый путь перечислен в inventory с причиной. Применимых runtime-изменений не пропущено; source diff не затрагивает Dockerfile, зависимости, Alembic или compose runtime.
|
||||
|
||||
## Документация
|
||||
|
||||
Context7 resolve для SQLAlchemy и RabbitMQ вернул `Monthly quota exceeded`. Использован официальный fallback: [SQLAlchemy 2.0 with_for_update](https://docs.sqlalchemy.org/en/20/core/selectable.html#sqlalchemy.sql.expression.GenerativeSelect.with_for_update) подтверждает read+key_share; [RabbitMQ 4.1 quorum delivery limit](https://www.rabbitmq.com/docs/4.1/quorum-queues#disabling-the-limit) содержит policy delivery-limit=-1. Новых зависимостей нет; версии остаются закреплёнными.
|
||||
|
||||
## Проверки и затронутые измерения
|
||||
|
||||
Изменение auth/cache влияет на HTTP/DB путь трека, поэтому повторяются все 15 серий с прежними параметрами; `evidence/track5` остаётся историческим результатом bc0b1620825b53fd3a3019623326ebf68cbbbeec. Новые PNG/доставка проверяются через runtime ЛР6 с включённой OTel, включая память, recovery и трассы. Статус и наблюдения будут дополнены после фактического завершения команд.
|
||||
@@ -0,0 +1,14 @@
|
||||
date: 2026-11-06T23:35:28Z commit: 97feb04b3f2a6008f636e1fdbdf284b1c97282a1
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make check validate lint
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
.venv/bin/ruff check src student-tests scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||
All checks passed!
|
||||
.venv/bin/ruff format --check src student-tests scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||
216 files already formatted
|
||||
.venv/bin/ty check src student-tests alembic
|
||||
All checks passed!
|
||||
exit_code: 0 elapsed_seconds: 0.590
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,78 @@
|
||||
date: 2026-11-06T23:32:01Z commit: 97feb04b3f2a6008f636e1fdbdf284b1c97282a1
|
||||
environment: {"COMPOSE_PROJECT_NAME": "mtusi2026lab6", "LB_HTTP_PORT": "18086", "LB_STATS_PORT": "26085", "BASE_URL": "http://127.0.0.1:18086", "NO_PROXY": "localhost,127.0.0.1,::1"}
|
||||
$ make student-test
|
||||
.venv/bin/pytest
|
||||
============================= test session starts ==============================
|
||||
platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0
|
||||
rootdir: <repo>
|
||||
configfile: pyproject.toml
|
||||
testpaths: student-tests
|
||||
plugins: asyncio-1.4.0, anyio-4.15.1
|
||||
asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function
|
||||
collected 1016 items / 19 deselected / 997 selected
|
||||
|
||||
student-tests/adapters/test_rabbitmq.py ..... [ 0%]
|
||||
student-tests/adapters/test_s3_storage.py ... [ 0%]
|
||||
student-tests/api/test_access.py ......... [ 1%]
|
||||
student-tests/api/test_attachments.py .................................. [ 5%]
|
||||
...... [ 5%]
|
||||
student-tests/api/test_bounded_contexts.py .................... [ 7%]
|
||||
student-tests/api/test_cache.py ............ [ 8%]
|
||||
student-tests/api/test_concurrency.py .... [ 9%]
|
||||
student-tests/api/test_consistency.py ........ [ 10%]
|
||||
student-tests/api/test_cost_experiment.py ... [ 10%]
|
||||
student-tests/api/test_drain.py .. [ 10%]
|
||||
student-tests/api/test_events.py ...... [ 11%]
|
||||
student-tests/api/test_http_telemetry.py . [ 11%]
|
||||
student-tests/api/test_idempotency.py ............... [ 12%]
|
||||
student-tests/api/test_instance.py .... [ 13%]
|
||||
student-tests/api/test_job_returning.py . [ 13%]
|
||||
student-tests/api/test_message_attachments.py ............. [ 14%]
|
||||
student-tests/api/test_migrations.py ................................... [ 18%]
|
||||
. [ 18%]
|
||||
student-tests/api/test_negative.py ..................................... [ 21%]
|
||||
[ 21%]
|
||||
student-tests/api/test_nul_validation.py ................. [ 23%]
|
||||
student-tests/api/test_operability.py .............. [ 25%]
|
||||
student-tests/api/test_pagination.py .................. [ 26%]
|
||||
student-tests/api/test_pipeline.py ............. [ 28%]
|
||||
student-tests/api/test_pool.py ............ [ 29%]
|
||||
student-tests/api/test_reconciler.py ........... [ 30%]
|
||||
student-tests/api/test_recovery.py ............ [ 31%]
|
||||
student-tests/api/test_sessions.py ...................... [ 33%]
|
||||
student-tests/api/test_timeout.py ........ [ 34%]
|
||||
student-tests/api/test_trace_context.py . [ 34%]
|
||||
student-tests/api/test_upload_consistency.py ......... [ 35%]
|
||||
student-tests/api/test_versioned_pipeline.py ........... [ 36%]
|
||||
student-tests/api/test_wire_format.py .. [ 37%]
|
||||
student-tests/unit/test_attachments_domain.py .......................... [ 39%]
|
||||
. [ 39%]
|
||||
student-tests/unit/test_authentication.py ............................. [ 42%]
|
||||
student-tests/unit/test_cursor.py ........... [ 43%]
|
||||
student-tests/unit/test_domain.py ...................................... [ 47%]
|
||||
.. [ 47%]
|
||||
student-tests/unit/test_haproxy_errors.py ............... [ 49%]
|
||||
student-tests/unit/test_image_inspection.py ............................ [ 52%]
|
||||
........................................................................ [ 59%]
|
||||
........................................................................ [ 66%]
|
||||
..................................................... [ 71%]
|
||||
student-tests/unit/test_job_observer.py . [ 71%]
|
||||
student-tests/unit/test_nul_inputs.py ...................... [ 74%]
|
||||
student-tests/unit/test_pipeline_domain.py ....................... [ 76%]
|
||||
student-tests/unit/test_png_memory.py ....s [ 76%]
|
||||
student-tests/unit/test_png_transparency.py ............................ [ 79%]
|
||||
........................................................................ [ 86%]
|
||||
.............................................. [ 91%]
|
||||
student-tests/unit/test_png_upload.py .. [ 91%]
|
||||
student-tests/unit/test_probe.py .............. [ 93%]
|
||||
student-tests/unit/test_reconcile_measurements.py ...... [ 93%]
|
||||
student-tests/unit/test_render_pool.py .....s. [ 94%]
|
||||
student-tests/unit/test_renderer.py .............................. [ 97%]
|
||||
student-tests/unit/test_scaling.py ..... [ 97%]
|
||||
student-tests/unit/test_session_cache.py .......... [ 98%]
|
||||
student-tests/unit/test_telemetry_platform.py ..... [ 99%]
|
||||
student-tests/unit/test_timeouts.py . [ 99%]
|
||||
student-tests/unit/test_worker_delivery.py .... [100%]
|
||||
|
||||
========== 995 passed, 2 skipped, 19 deselected in 176.45s (0:02:56) ===========
|
||||
exit_code: 0 elapsed_seconds: 179.184
|
||||
@@ -36,9 +36,9 @@ defaults
|
||||
timeout http-keep-alive 5s
|
||||
timeout queue 3s
|
||||
timeout check 1s
|
||||
http-error status 502 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"Upstream replica failed","request_id":"%[var(txn.request_id)]"}}'
|
||||
http-error status 503 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"No ready API replica","request_id":"%[var(txn.request_id)]"}}'
|
||||
http-error status 504 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"Upstream timeout","request_id":"%[var(txn.request_id)]"}}'
|
||||
http-error status 502 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"Upstream replica failed","request_id":"%[var(txn.request_id),json(ascii)]"}}'
|
||||
http-error status 503 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"No ready API replica","request_id":"%[var(txn.request_id),json(ascii)]"}}'
|
||||
http-error status 504 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"Upstream timeout","request_id":"%[var(txn.request_id),json(ascii)]"}}'
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
"dead-letter-exchange": "attachments.dlx",
|
||||
"dead-letter-strategy": "at-least-once",
|
||||
"overflow": "reject-publish",
|
||||
"delivery-limit": 10
|
||||
"delivery-limit": -1
|
||||
}
|
||||
},
|
||||
{
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
## 1. Наследуемый уровень 3
|
||||
- [ ] 1.1 Перенести identity validation/cache и regression tests; проверить cache races и logout failures.
|
||||
- [ ] 1.2 Перенести messaging NUL validation и regression tests.
|
||||
- [ ] 1.3 Перенести PNG validation/transparency/memory и тесты, с локальными fixtures.
|
||||
- [ ] 1.4 Перенести HAProxy/RabbitMQ, сохранив OTel; адаптировать вспомогательные проверки к ЛР6.
|
||||
- [ ] 1.5 Выполнить make check/validate/lint/student-test; создать коммит runtime.
|
||||
- [x] 1.1 Перенести identity validation/cache и regression tests; проверить cache races и logout failures.
|
||||
- [x] 1.2 Перенести messaging NUL validation и regression tests.
|
||||
- [x] 1.3 Перенести PNG validation/transparency/memory и тесты, с локальными fixtures.
|
||||
- [x] 1.4 Перенести HAProxy/RabbitMQ, сохранив OTel; адаптировать вспомогательные проверки к ЛР6.
|
||||
- [x] 1.5 Выполнить make check/validate/lint/student-test; создать коммит runtime.
|
||||
## 2. Затронутые гарантии уровня 4
|
||||
- [ ] 2.1 Реальный стенд: network, HAProxy errors, cache outage, PostgreSQL requeue более 10 раз.
|
||||
- [ ] 2.2 PNG HTTP, 40MP и concurrent worker, forced limits/recovery; smoke/persistence/failover.
|
||||
|
||||
@@ -41,7 +41,7 @@ context: |
|
||||
Python 3.14 (аннотации вычисляются лениво по PEP 649, `TYPE_CHECKING` и
|
||||
`from __future__ import annotations` не используются), FastAPI, gunicorn с несколькими
|
||||
`UvicornWorker` (пакет `uvicorn-worker`, `preload_app = False`), dishka (DI, интеграции FastAPI
|
||||
и FastStream), pydantic v2 + pydantic-settings, structlog, SQLAlchemy 2.x async (asyncpg), Alembic, redis-py для Valkey,
|
||||
и FastStream), pydantic v2 + pydantic-settings, structlog, SQLAlchemy 2.0.45 async (asyncpg), Alembic 1.16.5, redis-py для Valkey,
|
||||
argon2-cffi, aiobotocore, Pillow, FastStream 0.7 с `faststream[rabbit,otel]` поверх aio-pika,
|
||||
OpenTelemetry Python (API/SDK, OTLP/HTTP protobuf exporter без gRPC, инструментации FastAPI и
|
||||
asyncpg, версии в `uv.lock`), pytest + pytest-asyncio + httpx. Зависимости — uv и `uv.lock`;
|
||||
@@ -150,6 +150,8 @@ operations:
|
||||
- "Перед коммитом прогоняй `make lint`, `make student-test` и `make check`."
|
||||
- "Публичные `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh` и `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` запускаются против поднятого стенда с включённой телеметрией; после failover реплика возвращается `make restore`."
|
||||
- "Каждый заявленный критерий рубрики подтверждай командой и наблюдаемым результатом в `REPORT.md`; PASS без команды доказательством не считается."
|
||||
- "Фиксируй SHA кода API и обвязки отдельно; прежнее evidence сохраняй с исходным SHA. Числа сверяй с сырьём, неуспешные попытки сохраняй и объясняй; повторяй только затронутые или непригодные серии."
|
||||
- "Проверяй документацию закреплённых SQLAlchemy/Alembic через Context7. Не приравнивай микробенчмарк к HTTP throughput, пик RPS к SLO, а no-op миграцию в rolling к проверке изменения схемы."
|
||||
- "Отмечай выполнение в `tasks.md`; резюме держи кратким."
|
||||
archive:
|
||||
guidance:
|
||||
|
||||
@@ -5,10 +5,12 @@ import warnings
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from concurrent.futures.process import BrokenProcessPool
|
||||
from contextlib import closing
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
from PIL import Image, ImageChops, ImageOps
|
||||
from PIL.PngImagePlugin import PngImageFile
|
||||
|
||||
from messenger.attachments.application.ports import Rendition
|
||||
from messenger.attachments.domain.errors import (
|
||||
@@ -27,6 +29,7 @@ from messenger.attachments.domain.variants import (
|
||||
THUMBNAIL_BOX,
|
||||
fit,
|
||||
)
|
||||
from messenger.attachments.infrastructure.png import SIGNATURE, read_png_header, verify_png_pixels
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
|
||||
KINDS = {"PNG": ImageKind.PNG, "JPEG": ImageKind.JPEG}
|
||||
@@ -38,6 +41,8 @@ JPEG = {1: {"quality": 85, "optimize": True}, 2: {"quality": 90, "optimize": Tru
|
||||
WEBP = {1: {"quality": 80, "method": 4}, 2: {"quality": 85, "method": 4}}
|
||||
ENCODINGS = {"JPEG": (ImageKind.JPEG.value, "jpg"), "WEBP": ("image/webp", "webp")}
|
||||
REDUCING_GAP = 3.0
|
||||
PNG_WIDE_DEPTH = 16
|
||||
COLOR_TILE_SIDE = 256
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -60,6 +65,7 @@ class PillowImageInspector:
|
||||
|
||||
def _inspect(self, data: bytes, declared: ImageKind) -> ImageInfo:
|
||||
try:
|
||||
header = read_png_header(data) if data.startswith(SIGNATURE) else None
|
||||
image = Image.open(BytesIO(data), formats=tuple(KINDS))
|
||||
except BOMBS as error:
|
||||
raise PayloadTooLarge("изображение больше допустимого числа пикселей") from error
|
||||
@@ -67,28 +73,87 @@ class PillowImageInspector:
|
||||
raise UnsupportedImage("содержимое не является изображением PNG или JPEG") from error
|
||||
with image:
|
||||
kind = KINDS[str(image.format)]
|
||||
width, height = image.size
|
||||
width, height = image.size if header is None else header[:2]
|
||||
if kind is not declared:
|
||||
raise UnsupportedImage("содержимое не совпадает с заявленным Content-Type")
|
||||
if width * height > self._max_pixels:
|
||||
raise PayloadTooLarge("изображение больше допустимого числа пикселей")
|
||||
try:
|
||||
if kind is ImageKind.JPEG:
|
||||
if header is None:
|
||||
image.draft("RGB", (width // JPEG_SCALE or 1, height // JPEG_SCALE or 1))
|
||||
image.load()
|
||||
else:
|
||||
image.verify()
|
||||
verify_png_pixels(data, header)
|
||||
except Exception as error:
|
||||
raise UnsupportedImage("изображение повреждено или обрезано") from error
|
||||
return ImageInfo(kind, width, height)
|
||||
|
||||
|
||||
def map_tiles(
|
||||
image: Image.Image, mode: str, transform: Callable[[Image.Image], Image.Image]
|
||||
) -> Image.Image:
|
||||
result = Image.new(mode, image.size)
|
||||
for y in range(0, image.height, COLOR_TILE_SIDE):
|
||||
for x in range(0, image.width, COLOR_TILE_SIDE):
|
||||
box = (
|
||||
x,
|
||||
y,
|
||||
min(x + COLOR_TILE_SIDE, image.width),
|
||||
min(y + COLOR_TILE_SIDE, image.height),
|
||||
)
|
||||
with closing(image.crop(box)) as tile, closing(transform(tile)) as transformed:
|
||||
result.paste(transformed, box)
|
||||
return result
|
||||
|
||||
|
||||
def rgb_mask(image: Image.Image, color: tuple[int, ...]) -> Image.Image:
|
||||
return map_tiles(image, "L", lambda tile: rgb_tile_mask(tile, color))
|
||||
|
||||
|
||||
def rgb_tile_mask(image: Image.Image, color: tuple[int, ...]) -> Image.Image:
|
||||
mask = Image.new("L", image.size, 255)
|
||||
for channel, value in zip("RGB", color, strict=True):
|
||||
mask = ImageChops.multiply(
|
||||
mask, image.getchannel(channel).point([255 if i == value else 0 for i in range(256)])
|
||||
)
|
||||
return mask
|
||||
|
||||
|
||||
def apply_png_transparency(image: PngImageFile, data: bytes) -> None:
|
||||
if (transparent := image.info.get("transparency")) is None:
|
||||
return
|
||||
depth = read_png_header(data)[2]
|
||||
if image.mode == "L" and depth in (2, 4):
|
||||
image.info["transparency"] = transparent * 255 // ((1 << depth) - 1)
|
||||
elif image.mode == "RGB" and depth == PNG_WIDE_DEPTH:
|
||||
with closing(PngImageFile(BytesIO(data))) as low:
|
||||
low.tile = [tile._replace(args="RGB;16L") for tile in low.tile]
|
||||
mask = rgb_mask(low, tuple(value & 255 for value in transparent))
|
||||
with (
|
||||
closing(mask),
|
||||
closing(rgb_mask(image, tuple(value >> 8 for value in transparent))) as high,
|
||||
closing(ImageChops.multiply(mask, high)) as combined,
|
||||
):
|
||||
image.paste((255, 255, 255), mask=combined)
|
||||
image.info.pop("transparency")
|
||||
|
||||
|
||||
def flatten(image: Image.Image) -> Image.Image:
|
||||
if image.mode == "RGB" and "transparency" not in image.info:
|
||||
return image
|
||||
return map_tiles(image, "RGB", flatten_tile)
|
||||
|
||||
|
||||
def flatten_tile(image: Image.Image) -> Image.Image:
|
||||
if image.mode in WIDE_MODES:
|
||||
image = image.convert("I").point(lambda value: value / 256, "L")
|
||||
if image.mode == "P":
|
||||
image = image.convert("RGBA" if "transparency" in image.info else "RGB")
|
||||
if image.mode in ALPHA_MODES:
|
||||
image = image.convert("I")
|
||||
if (transparent := image.info.pop("transparency", None)) is not None:
|
||||
mask = [0] * 65536
|
||||
mask[transparent] = 255
|
||||
image.paste(65535, mask=image.point(mask, "L"))
|
||||
image = image.point(lambda value: value / 256, "L")
|
||||
if image.mode in ALPHA_MODES or "transparency" in image.info:
|
||||
image = image.convert("RGBA")
|
||||
if image.mode == "RGBA":
|
||||
background = Image.new("RGB", image.size, "white")
|
||||
@@ -116,12 +181,17 @@ def decoded(data: bytes, max_pixels: int) -> tuple[Image.Image, int, int]:
|
||||
width, height = image.size
|
||||
if width * height > max_pixels:
|
||||
raise UnreadableImage("изображение больше допустимого числа пикселей")
|
||||
if isinstance(image, PngImageFile):
|
||||
apply_png_transparency(image, data)
|
||||
image.draft(None, fit(width, height, OPTIMIZED_BOX))
|
||||
stored = image.size
|
||||
ImageOps.exif_transpose(image, in_place=True)
|
||||
if image.size != stored:
|
||||
width, height = height, width
|
||||
return flatten(image), width, height
|
||||
upright = flatten(image)
|
||||
if upright is not image:
|
||||
image.close()
|
||||
return upright, width, height
|
||||
except UnreadableImage, MemoryError:
|
||||
raise
|
||||
except Exception as error:
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import struct
|
||||
import zlib
|
||||
from collections.abc import Iterator
|
||||
|
||||
type PngHeader = tuple[int, int, int, int, int, int, int]
|
||||
|
||||
SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
HEADER_BYTES = 13
|
||||
CHUNK_OVERHEAD = 12
|
||||
BLOCK_BYTES = 65_536
|
||||
MAX_FILTER = 4
|
||||
PIXELS = {
|
||||
0: (1, (1, 2, 4, 8, 16)),
|
||||
2: (3, (8, 16)),
|
||||
3: (1, (1, 2, 4, 8)),
|
||||
4: (2, (8, 16)),
|
||||
6: (4, (8, 16)),
|
||||
}
|
||||
ADAM7 = (
|
||||
(0, 0, 8, 8),
|
||||
(4, 0, 8, 8),
|
||||
(0, 4, 4, 8),
|
||||
(2, 0, 4, 4),
|
||||
(0, 2, 2, 4),
|
||||
(1, 0, 2, 2),
|
||||
(0, 1, 1, 2),
|
||||
)
|
||||
|
||||
|
||||
def _chunks(data: bytes) -> Iterator[tuple[bytes, memoryview]]:
|
||||
view = memoryview(data)
|
||||
offset = len(SIGNATURE)
|
||||
while offset + CHUNK_OVERHEAD <= len(data):
|
||||
size = struct.unpack_from(">I", data, offset)[0]
|
||||
kind = data[offset + 4 : offset + 8]
|
||||
start, end = offset + 8, offset + 8 + size
|
||||
if end + 4 > len(data):
|
||||
raise ValueError("truncated PNG chunk")
|
||||
yield kind, view[start:end]
|
||||
if kind == b"IEND":
|
||||
return
|
||||
offset = end + 4
|
||||
raise ValueError("missing PNG end")
|
||||
|
||||
|
||||
def read_png_header(data: bytes) -> PngHeader:
|
||||
chunks = _chunks(data)
|
||||
kind, header = next(chunks)
|
||||
if kind != b"IHDR" or len(header) != HEADER_BYTES:
|
||||
raise ValueError("invalid first PNG header")
|
||||
if any(kind == b"IHDR" for kind, _ in chunks):
|
||||
raise ValueError("duplicate PNG header")
|
||||
return struct.unpack(">IIBBBBB", header)
|
||||
|
||||
|
||||
def _scanlines(header: PngHeader) -> Iterator[tuple[int, int]]:
|
||||
width, height, depth, color, compression, filtering, interlace = header
|
||||
samples, depths = PIXELS[color]
|
||||
if depth not in depths or compression or filtering or interlace not in (0, 1):
|
||||
raise ValueError("invalid PNG header")
|
||||
passes = ADAM7 if interlace else ((0, 0, 1, 1),)
|
||||
for x, y, dx, dy in passes:
|
||||
columns, rows = (width - x + dx - 1) // dx, (height - y + dy - 1) // dy
|
||||
if columns > 0 and rows > 0:
|
||||
stride = 1 + (columns * samples * depth + 7) // 8
|
||||
yield stride, stride * rows
|
||||
|
||||
|
||||
def _idat(data: bytes) -> Iterator[memoryview]:
|
||||
started = ended = False
|
||||
for kind, payload in _chunks(data):
|
||||
if kind == b"IDAT":
|
||||
if ended:
|
||||
raise ValueError("nonconsecutive PNG image data")
|
||||
started = True
|
||||
for position in range(0, len(payload), BLOCK_BYTES):
|
||||
yield payload[position : position + BLOCK_BYTES]
|
||||
elif started:
|
||||
ended = True
|
||||
|
||||
|
||||
def _inflate(data: bytes) -> Iterator[bytes]:
|
||||
decoder = zlib.decompressobj()
|
||||
for block in _idat(data):
|
||||
if decoder.eof:
|
||||
continue
|
||||
pending = block
|
||||
while True:
|
||||
decoded = decoder.decompress(pending, BLOCK_BYTES)
|
||||
yield decoded
|
||||
pending = decoder.unconsumed_tail
|
||||
if decoder.eof or (not pending and len(decoded) < BLOCK_BYTES):
|
||||
break
|
||||
if not decoder.eof:
|
||||
raise ValueError("incomplete PNG compressed data")
|
||||
|
||||
|
||||
def verify_png_pixels(data: bytes, header: PngHeader) -> None:
|
||||
passes = _scanlines(header)
|
||||
stride, remaining = next(passes)
|
||||
offset = 0
|
||||
for decoded in _inflate(data):
|
||||
cursor = 0
|
||||
while cursor < len(decoded):
|
||||
if not remaining:
|
||||
stride, remaining = next(passes, (0, 0))
|
||||
offset = 0
|
||||
if not remaining:
|
||||
raise ValueError("extra PNG pixel data")
|
||||
size = min(remaining, len(decoded) - cursor)
|
||||
filters = decoded[cursor + (-offset % stride) : cursor + size : stride]
|
||||
if max(filters, default=0) > MAX_FILTER:
|
||||
raise ValueError("invalid PNG filter")
|
||||
cursor += size
|
||||
offset += size
|
||||
remaining -= size
|
||||
if remaining or next(passes, None) is not None:
|
||||
raise ValueError("incomplete PNG pixel data")
|
||||
@@ -69,10 +69,10 @@ class CloseSessionHandler:
|
||||
self._revocations = revocations
|
||||
|
||||
async def __call__(self, command: CloseSession) -> None:
|
||||
await self._revocations.revoke(command.session)
|
||||
try:
|
||||
async with self._uow.transaction():
|
||||
await self._uow.sessions.remove(command.session.token_hash)
|
||||
await self._revocations.revoke(command.session)
|
||||
self._uow.record(SessionClosed(user_id=command.session.user_id))
|
||||
await self._uow.commit()
|
||||
except BaseException:
|
||||
|
||||
@@ -30,6 +30,8 @@ class DisplayName:
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.value, str):
|
||||
raise InvalidUserData("поле display_name должно быть строкой")
|
||||
if "\x00" in self.value:
|
||||
raise InvalidUserData("поле display_name не может содержать NUL")
|
||||
if not self.value.strip():
|
||||
raise InvalidUserData("поле display_name не может быть пустым или состоять из пробелов")
|
||||
if len(self.value) > DISPLAY_NAME_MAX:
|
||||
|
||||
@@ -30,14 +30,14 @@ def challenge_for(path: str) -> str:
|
||||
|
||||
def basic_credentials(value: str) -> tuple[str, str] | None:
|
||||
scheme, _, encoded = value.partition(" ")
|
||||
if scheme.lower() != "basic":
|
||||
if scheme.lower() != "basic" or not encoded.isascii():
|
||||
return None
|
||||
try:
|
||||
decoded = b64decode(encoded.strip(), validate=True).decode("utf-8")
|
||||
except Base64Error, UnicodeDecodeError:
|
||||
return None
|
||||
username, separator, password = decoded.partition(":")
|
||||
if not separator or not username.isascii():
|
||||
if not separator or not username.isascii() or "\x00" in username:
|
||||
return None
|
||||
return username, password
|
||||
|
||||
|
||||
@@ -166,10 +166,12 @@ class PostgresSessionDirectory:
|
||||
async with self._database.session() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(SessionRow).where(
|
||||
select(SessionRow)
|
||||
.where(
|
||||
SessionRow.token_hash == token_hash,
|
||||
SessionRow.expires_at > now,
|
||||
)
|
||||
.with_for_update(read=True, key_share=True)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import time
|
||||
from collections.abc import Awaitable
|
||||
from datetime import datetime, timedelta
|
||||
from uuid import UUID
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from redis.asyncio import Redis
|
||||
from redis.exceptions import RedisError
|
||||
@@ -13,6 +13,16 @@ from messenger.shared.application.ports import Logger
|
||||
|
||||
REVOKED = "-"
|
||||
FAILURES = (RedisError, OSError)
|
||||
FILL = """
|
||||
if redis.call('GET', KEYS[2]) ~= ARGV[1] then return false end
|
||||
redis.call('DEL', KEYS[2])
|
||||
if ARGV[2] == '' then return false end
|
||||
return redis.call('SET', KEYS[1], ARGV[2], 'PXAT', ARGV[3], 'NX')
|
||||
"""
|
||||
REVOKE = """
|
||||
redis.call('SET', KEYS[1], ARGV[1], 'PXAT', ARGV[2])
|
||||
return redis.call('DEL', KEYS[2])
|
||||
"""
|
||||
|
||||
|
||||
def key_of(token_hash: bytes) -> str:
|
||||
@@ -72,18 +82,26 @@ class CachedSessionDirectory:
|
||||
if cached is not None:
|
||||
session = decode(token_hash, cached)
|
||||
return session if session.expires_at > now else None
|
||||
marker, nonce = f"{key}:fill", uuid4().hex
|
||||
until = milliseconds(now + self._limit)
|
||||
reserved = self._closed() and await self._quietly(
|
||||
self._client.set(marker, nonce, pxat=until, nx=True)
|
||||
)
|
||||
session = await self._origin.find(token_hash, now)
|
||||
if session is not None and self._closed():
|
||||
until = milliseconds(min(session.expires_at, now + self._limit))
|
||||
await self._quietly(self._client.set(key, encode(session), pxat=until, nx=True))
|
||||
if reserved and self._closed():
|
||||
value = encode(session) if session is not None else ""
|
||||
if session is not None:
|
||||
until = min(until, milliseconds(session.expires_at))
|
||||
await self._quietly(self._client.eval(FILL, 2, key, marker, nonce, value, until))
|
||||
return session
|
||||
|
||||
async def revoke(self, session: Session) -> None:
|
||||
if not self._closed():
|
||||
raise Unavailable("кэш сессий недоступен")
|
||||
key = key_of(session.token_hash)
|
||||
try:
|
||||
await self._client.set(
|
||||
key_of(session.token_hash), REVOKED, pxat=milliseconds(session.expires_at)
|
||||
await self._client.eval(
|
||||
REVOKE, 2, key, f"{key}:fill", REVOKED, milliseconds(session.expires_at)
|
||||
)
|
||||
except FAILURES as error:
|
||||
self._trip(error)
|
||||
|
||||
@@ -15,6 +15,8 @@ class ChatTitle:
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.value, str):
|
||||
raise InvalidChatData("поле title должно быть строкой")
|
||||
if "\x00" in self.value:
|
||||
raise InvalidChatData("поле title не может содержать NUL")
|
||||
if not self.value.strip():
|
||||
raise InvalidChatData("поле title не может быть пустым или состоять из пробелов")
|
||||
if len(self.value) > CHAT_TITLE_MAX:
|
||||
@@ -31,6 +33,8 @@ class MessageText:
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.value, str):
|
||||
raise InvalidMessageText("поле text должно быть строкой")
|
||||
if "\x00" in self.value:
|
||||
raise InvalidMessageText("поле text не может содержать NUL")
|
||||
if self.value and not self.value.strip():
|
||||
raise InvalidMessageText("поле text не может состоять из пробелов")
|
||||
if len(self.value) > MESSAGE_TEXT_MAX:
|
||||
|
||||
@@ -248,14 +248,20 @@ async def test_message_off_the_schema_goes_to_the_dead_letter_queue(
|
||||
assert depth(rabbitmq, "attachments.process") == 0
|
||||
|
||||
|
||||
async def test_failed_processing_is_requeued_and_retried(rabbitmq: dict[str, Any]) -> None:
|
||||
@pytest.mark.parametrize("failures", [1, 12])
|
||||
async def test_failed_processing_is_requeued_and_retried(
|
||||
rabbitmq: dict[str, Any], failures: int
|
||||
) -> None:
|
||||
recorder, sent = Recorder(), job()
|
||||
recorder.failures = 1
|
||||
recorder.failures = failures
|
||||
before = depth(rabbitmq, "attachments.process.dlq")
|
||||
async with publisher(rabbitmq) as jobs:
|
||||
await jobs.publish(sent.id, body(sent.payload()))
|
||||
async with worker(rabbitmq, recorder):
|
||||
await wait_for(lambda: recorder.jobs)
|
||||
await wait_for(lambda: recorder.jobs, seconds=40)
|
||||
await wait_for(lambda: depth(rabbitmq, "attachments.process") == 0)
|
||||
assert recorder.jobs == [sent]
|
||||
assert depth(rabbitmq, "attachments.process.dlq") == before
|
||||
|
||||
|
||||
async def test_worker_account_cannot_publish(rabbitmq: dict[str, Any]) -> None:
|
||||
|
||||
@@ -12,7 +12,7 @@ from PIL import Image
|
||||
from conftest import actor, make_chat, register, resolve, running_app, scalar
|
||||
from fakes import MemoryObjectStorage
|
||||
from messenger.attachments.application.ports import ObjectStorage
|
||||
from pictures import GIF, JPEG, SAMPLE, WEBP, png
|
||||
from pictures import BROKEN_PNG, DUPLICATE_IHDR_PNG, GIF, JPEG, SAMPLE, WEBP, png
|
||||
from pipeline import settle
|
||||
|
||||
LIMIT = 10_485_760
|
||||
@@ -197,6 +197,8 @@ async def test_pixel_limit_is_read_from_header(
|
||||
@pytest.mark.parametrize(
|
||||
("content", "header"),
|
||||
[
|
||||
(BROKEN_PNG, "image/png"),
|
||||
(DUPLICATE_IHDR_PNG, "image/png"),
|
||||
(JPEG, "image/png"),
|
||||
(GIF, "image/png"),
|
||||
(WEBP, "image/png"),
|
||||
@@ -209,6 +211,8 @@ async def test_pixel_limit_is_read_from_header(
|
||||
(SAMPLE, "image/gif"),
|
||||
],
|
||||
ids=[
|
||||
"invalid-zlib",
|
||||
"duplicate-ihdr",
|
||||
"jpeg-as-png",
|
||||
"gif-as-png",
|
||||
"webp-as-png",
|
||||
|
||||
@@ -11,7 +11,12 @@ from sqlalchemy import select
|
||||
from conftest import actor, bearer, keyed, make_chat, register, resolve, running_app, scalar
|
||||
from messenger.identity.application.ports import SessionDirectory
|
||||
from messenger.identity.domain.models import Session
|
||||
from messenger.identity.infrastructure.postgres import PostgresSessionRepository, SessionRow
|
||||
from messenger.identity.infrastructure.postgres import (
|
||||
PostgresIdentityUnitOfWork,
|
||||
PostgresSessionDirectory,
|
||||
PostgresSessionRepository,
|
||||
SessionRow,
|
||||
)
|
||||
from messenger.identity.infrastructure.valkey import CachedSessionDirectory
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
|
||||
@@ -44,7 +49,7 @@ async def test_second_check_is_served_from_cache(valkey: dict[str, str]) -> None
|
||||
await cache.aclose()
|
||||
|
||||
|
||||
async def test_logout_writes_tombstone_before_database(valkey: dict[str, str]) -> None:
|
||||
async def test_logout_leaves_tombstone_after_commit(valkey: dict[str, str]) -> None:
|
||||
cache = Redis.from_url(valkey["url"])
|
||||
async with running_app(cache_url=valkey["url"]) as http:
|
||||
alice = await register(http, "alice")
|
||||
@@ -59,19 +64,26 @@ async def test_logout_writes_tombstone_before_database(valkey: dict[str, str]) -
|
||||
await cache.aclose()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["remove", "commit"])
|
||||
async def test_logout_rejected_by_database_changes_nothing(
|
||||
valkey: dict[str, str], monkeypatch: pytest.MonkeyPatch
|
||||
valkey: dict[str, str], monkeypatch: pytest.MonkeyPatch, failure: str
|
||||
) -> None:
|
||||
async def broken(self: PostgresSessionRepository, token_hash: bytes) -> None:
|
||||
raise Unavailable("PostgreSQL недоступен")
|
||||
|
||||
async def broken_commit(self: PostgresIdentityUnitOfWork) -> None:
|
||||
raise Unavailable("PostgreSQL недоступен")
|
||||
|
||||
cache = Redis.from_url(valkey["url"])
|
||||
logout = "/api/v1/auth/sessions/current"
|
||||
async with running_app(cache_url=valkey["url"]) as http:
|
||||
alice = await register(http, "alice")
|
||||
assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 200
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(PostgresSessionRepository, "remove", broken)
|
||||
if failure == "remove":
|
||||
patch.setattr(PostgresSessionRepository, "remove", broken)
|
||||
else:
|
||||
patch.setattr(PostgresIdentityUnitOfWork, "commit", broken_commit)
|
||||
failed = await http.delete(logout, headers=actor(alice))
|
||||
assert failed.status_code == 503
|
||||
assert failed.json()["error"]["code"] == "unavailable"
|
||||
@@ -82,8 +94,14 @@ async def test_logout_rejected_by_database_changes_nothing(
|
||||
await cache.aclose()
|
||||
|
||||
|
||||
async def test_fill_racing_with_logout_does_not_revive_token(valkey: dict[str, str]) -> None:
|
||||
async with running_app(cache_url=valkey["url"]) as http:
|
||||
@pytest.mark.parametrize("loss", ["none", "flushall", "evict", "replacement"])
|
||||
async def test_fill_racing_with_logout_does_not_revive_token(
|
||||
valkey: dict[str, str], loss: str
|
||||
) -> None:
|
||||
async with (
|
||||
Redis.from_url(valkey["url"]) as cache,
|
||||
running_app(cache_url=valkey["url"]) as http,
|
||||
):
|
||||
alice = await register(http, "alice")
|
||||
directory = await resolve(http, SessionDirectory)
|
||||
assert isinstance(directory, CachedSessionDirectory)
|
||||
@@ -99,14 +117,73 @@ async def test_fill_racing_with_logout_does_not_revive_token(valkey: dict[str, s
|
||||
return stale[-1]
|
||||
|
||||
directory._origin = SlowOrigin()
|
||||
reader = asyncio.create_task(http.get("/api/v1/users/me", headers=actor(alice)))
|
||||
await entered.wait()
|
||||
directory._origin = origin
|
||||
assert (
|
||||
await http.delete("/api/v1/auth/sessions/current", headers=actor(alice))
|
||||
).status_code == 204
|
||||
released.set()
|
||||
assert (await reader).status_code == 200
|
||||
async with asyncio.TaskGroup() as group:
|
||||
reader = group.create_task(http.get("/api/v1/users/me", headers=actor(alice)))
|
||||
try:
|
||||
await asyncio.wait_for(entered.wait(), 5)
|
||||
directory._origin = origin
|
||||
assert (
|
||||
await http.delete("/api/v1/auth/sessions/current", headers=actor(alice))
|
||||
).status_code == 204
|
||||
key = key_of(token_of(alice))
|
||||
assert await cache.get(f"{key}:fill") is None
|
||||
if loss == "flushall":
|
||||
await cache.flushall()
|
||||
elif loss in {"evict", "replacement"}:
|
||||
await cache.delete(key)
|
||||
if loss == "replacement":
|
||||
await cache.set(f"{key}:fill", "another-reader", px=1000)
|
||||
finally:
|
||||
released.set()
|
||||
assert reader.result().status_code == 200
|
||||
assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 401
|
||||
|
||||
|
||||
@pytest.mark.parametrize("loss", ["flushall", "evict"])
|
||||
async def test_cache_loss_before_logout_commit_cannot_refill_deleted_session(
|
||||
valkey: dict[str, str], monkeypatch: pytest.MonkeyPatch, loss: str
|
||||
) -> None:
|
||||
revoked, release_commit, reading = asyncio.Event(), asyncio.Event(), asyncio.Event()
|
||||
revoke = CachedSessionDirectory.revoke
|
||||
find = PostgresSessionDirectory.find
|
||||
|
||||
async def paused_revoke(self: CachedSessionDirectory, session: Session) -> None:
|
||||
await revoke(self, session)
|
||||
revoked.set()
|
||||
await release_commit.wait()
|
||||
|
||||
async def observed_find(
|
||||
self: PostgresSessionDirectory, token_hash: bytes, now: datetime
|
||||
) -> Session | None:
|
||||
reading.set()
|
||||
return await find(self, token_hash, now)
|
||||
|
||||
async with (
|
||||
Redis.from_url(valkey["url"]) as cache,
|
||||
running_app(cache_url=valkey["url"]) as http,
|
||||
):
|
||||
alice = await register(http, "alice")
|
||||
assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 200
|
||||
monkeypatch.setattr(CachedSessionDirectory, "revoke", paused_revoke)
|
||||
async with asyncio.TaskGroup() as group:
|
||||
logout = group.create_task(
|
||||
http.delete("/api/v1/auth/sessions/current", headers=actor(alice))
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(revoked.wait(), 5)
|
||||
if loss == "flushall":
|
||||
await cache.flushall()
|
||||
else:
|
||||
await cache.delete(key_of(token_of(alice)))
|
||||
monkeypatch.setattr(PostgresSessionDirectory, "find", observed_find)
|
||||
reader = group.create_task(http.get("/api/v1/users/me", headers=actor(alice)))
|
||||
await asyncio.wait_for(reading.wait(), 5)
|
||||
done, _ = await asyncio.wait({reader}, timeout=0.1)
|
||||
assert not done
|
||||
finally:
|
||||
release_commit.set()
|
||||
assert logout.result().status_code == 204
|
||||
assert reader.result().status_code == 401
|
||||
assert (await http.get("/api/v1/users/me", headers=actor(alice))).status_code == 401
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from httpx import AsyncClient, Response
|
||||
|
||||
from conftest import PASSWORD, actor, basic, keyed, make_chat, new_user, register
|
||||
|
||||
NUL_TEXTS = ["\x00", "\x00abc", "abc\x00def", "abc\x00"]
|
||||
|
||||
|
||||
def rejected(response: Response, status: int, code: str) -> None:
|
||||
assert response.status_code == status, response.text
|
||||
assert response.headers["content-type"].startswith("application/json")
|
||||
assert response.headers["X-Instance-Id"]
|
||||
assert response.headers["X-Request-Id"]
|
||||
assert set(response.json()) == {"error"}
|
||||
error = response.json()["error"]
|
||||
assert set(error) == {"code", "message", "request_id"}
|
||||
assert error["code"] == code
|
||||
assert error["message"]
|
||||
assert error["request_id"] == response.headers["X-Request-Id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("username", NUL_TEXTS)
|
||||
async def test_nul_basic_matches_wrong_password(client: AsyncClient, username: str) -> None:
|
||||
user = await register(client, "nul_login")
|
||||
responses = []
|
||||
for name, password in [(user["username"], "wrong-password"), (username, PASSWORD)]:
|
||||
response = await client.post(
|
||||
"/api/v1/auth/sessions",
|
||||
headers={**basic(name, password), "X-Request-Id": "nul-login"},
|
||||
)
|
||||
rejected(response, 401, "unauthorized")
|
||||
assert response.headers["WWW-Authenticate"] == 'Basic realm="messenger", charset="UTF-8"'
|
||||
responses.append(response.json())
|
||||
assert responses[0] == responses[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", NUL_TEXTS)
|
||||
async def test_nul_display_name_does_not_create_user(client: AsyncClient, raw: str) -> None:
|
||||
body = {**new_user("nul_display"), "display_name": raw}
|
||||
rejected(await client.post("/api/v1/users", json=body), 400, "invalid_request")
|
||||
response = await client.post("/api/v1/users", json={**body, "display_name": "Алиса 👋"})
|
||||
assert response.status_code == 201
|
||||
assert response.json()["display_name"] == "Алиса 👋"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", NUL_TEXTS)
|
||||
async def test_nul_title_does_not_create_chat(client: AsyncClient, raw: str) -> None:
|
||||
user = await register(client, "nul_title")
|
||||
body = {"title": raw, "member_ids": [user["id"]]}
|
||||
rejected(
|
||||
await client.post("/api/v1/chats", json=body, headers=actor(user)), 400, "invalid_request"
|
||||
)
|
||||
chats = await client.get("/api/v1/chats", headers=actor(user))
|
||||
assert chats.status_code == 200
|
||||
assert chats.json()["items"] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", NUL_TEXTS)
|
||||
async def test_nul_message_does_not_take_key(client: AsyncClient, raw: str) -> None:
|
||||
user = await register(client, "nul_message")
|
||||
chat = await make_chat(client, user)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
headers = keyed(user)
|
||||
rejected(await client.post(path, json={"text": raw}, headers=headers), 400, "invalid_request")
|
||||
history = await client.get(path, headers=actor(user))
|
||||
assert history.status_code == 200
|
||||
assert history.json()["items"] == []
|
||||
text = "Исправлено\tбез потерь\n👋"
|
||||
first = await client.post(path, json={"text": text}, headers=headers)
|
||||
again = await client.post(path, json={"text": text}, headers=headers)
|
||||
assert first.status_code == again.status_code == 201
|
||||
assert first.json()["text"] == text
|
||||
assert first.json() == again.json()
|
||||
assert (await client.get(path, headers=actor(user))).json()["items"] == [first.json()]
|
||||
|
||||
|
||||
async def test_nul_message_does_not_disclose_chat(client: AsyncClient) -> None:
|
||||
owner = await register(client, "nul_owner")
|
||||
outsider = await register(client, "nul_outsider")
|
||||
chat = await make_chat(client, owner)
|
||||
responses = []
|
||||
for chat_id in [chat["id"], str(uuid4())]:
|
||||
response = await client.post(
|
||||
f"/api/v1/chats/{chat_id}/messages",
|
||||
json={"text": "abc\x00def"},
|
||||
headers={**keyed(outsider), "X-Request-Id": "nul-access"},
|
||||
)
|
||||
rejected(response, 404, "not_found")
|
||||
responses.append(response.json())
|
||||
assert responses[0] == responses[1]
|
||||
history = await client.get(f"/api/v1/chats/{chat['id']}/messages", headers=actor(owner))
|
||||
assert history.status_code == 200
|
||||
assert history.json()["items"] == []
|
||||
@@ -86,6 +86,34 @@ async def test_login_without_valid_basic(client: AsyncClient, headers: dict) ->
|
||||
challenged(await client.post("/api/v1/auth/sessions", headers=headers), BASIC)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw", [b"Basic \x80", b"Basic \xff", b"Basic \xc3\xa9", b"Basic YWxpY2U6c2VjcmV0\xff"]
|
||||
)
|
||||
async def test_non_ascii_basic_matches_wrong_password(client: AsyncClient, raw: bytes) -> None:
|
||||
alice = await register(client, "non_ascii_basic")
|
||||
count = (
|
||||
select(func.count()).select_from(SessionRow).where(SessionRow.user_id == UUID(alice["id"]))
|
||||
)
|
||||
before = await scalar(count)
|
||||
wrong = await client.post(
|
||||
"/api/v1/auth/sessions",
|
||||
headers={**basic(alice["username"], "wrong-password"), "X-Request-Id": "non-ascii-basic"},
|
||||
)
|
||||
invalid = await client.post(
|
||||
"/api/v1/auth/sessions",
|
||||
headers={b"Authorization": raw, b"X-Request-Id": b"non-ascii-basic"},
|
||||
)
|
||||
for response in [wrong, invalid]:
|
||||
challenged(response, BASIC)
|
||||
assert response.headers["X-Instance-Id"]
|
||||
assert response.headers["content-type"].startswith("application/json")
|
||||
assert set(response.json()) == {"error"}
|
||||
assert set(response.json()["error"]) == {"code", "message", "request_id"}
|
||||
assert response.json()["error"]["message"]
|
||||
assert wrong.json() == invalid.json()
|
||||
assert await scalar(count) == before
|
||||
|
||||
|
||||
async def test_password_with_colon_logs_in(client: AsyncClient) -> None:
|
||||
body = new_user("colon", password="пароль:с:двоеточием")
|
||||
assert (await client.post("/api/v1/users", json=body)).status_code == 201
|
||||
|
||||
@@ -14,6 +14,33 @@ def chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", crc)
|
||||
|
||||
|
||||
def png_from_idat(
|
||||
data: bytes,
|
||||
width: int = 1,
|
||||
height: int = 1,
|
||||
*,
|
||||
depth: int = 8,
|
||||
color: int = 2,
|
||||
interlace: int = 0,
|
||||
chunk_size: int = 65_536,
|
||||
) -> bytes:
|
||||
header = struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace)
|
||||
palette = chunk(b"PLTE", bytes(range(256)) * 3) if color == 3 else b""
|
||||
blocks = [
|
||||
chunk(b"IDAT", data[start : start + chunk_size])
|
||||
for start in range(0, len(data), chunk_size)
|
||||
]
|
||||
return SIGNATURE + chunk(b"IHDR", header) + palette + b"".join(blocks) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
BROKEN_PNG = png_from_idat(b"not a zlib stream")
|
||||
DUPLICATE_IHDR_PNG = (
|
||||
SIGNATURE
|
||||
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))
|
||||
+ png_from_idat(zlib.compress(bytes(4)), 2, 2)[len(SIGNATURE) :]
|
||||
)
|
||||
|
||||
|
||||
def png(width: int, height: int, *, size: int | None = None, rows: bool = True) -> bytes:
|
||||
header = struct.pack(">IIBBBBB", width, height, 1, 0, 0, 0, 0)
|
||||
raw = (b"\x00" * (1 + (width + 7) // 8)) * height if rows else b"\x00"
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import struct
|
||||
import zlib
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from pictures import SIGNATURE, chunk
|
||||
|
||||
PASSES = (
|
||||
(0, 0, 8, 8),
|
||||
(4, 0, 8, 8),
|
||||
(0, 4, 4, 8),
|
||||
(2, 0, 4, 4),
|
||||
(0, 2, 2, 4),
|
||||
(1, 0, 2, 2),
|
||||
(0, 1, 1, 2),
|
||||
)
|
||||
|
||||
|
||||
def boundary_png(depth: int, transparent: tuple[int, ...]) -> bytes:
|
||||
width, height = 8000, 5000
|
||||
neighbor = (transparent[0] ^ 1, *transparent[1:])
|
||||
row = b"\x00" + packed(list(transparent) * (width // 2) + list(neighbor) * (width // 2), depth)
|
||||
compressor = zlib.compressobj()
|
||||
payload = b"".join(compressor.compress(row) for _ in range(height)) + compressor.flush()
|
||||
header = struct.pack(
|
||||
">IIBBBBB", width, height, depth, 0 if len(transparent) == 1 else 2, 0, 0, 0
|
||||
)
|
||||
return (
|
||||
SIGNATURE
|
||||
+ chunk(b"IHDR", header)
|
||||
+ chunk(b"tRNS", b"".join(value.to_bytes(2, "big") for value in transparent))
|
||||
+ chunk(b"IDAT", payload)
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def packed(samples: list[int], depth: int) -> bytes:
|
||||
if depth >= 8:
|
||||
return b"".join(value.to_bytes(depth // 8, "big") for value in samples)
|
||||
per_byte = 8 // depth
|
||||
padded = samples + [0] * (-len(samples) % per_byte)
|
||||
return bytes(
|
||||
sum(value << (8 - depth * (i + 1)) for i, value in enumerate(padded[j : j + per_byte]))
|
||||
for j in range(0, len(padded), per_byte)
|
||||
)
|
||||
|
||||
|
||||
def filtered(row: bytes, previous: bytes, bpp: int, method: int) -> bytes:
|
||||
result = bytearray([method])
|
||||
for i, value in enumerate(row):
|
||||
left = row[i - bpp] if i >= bpp else 0
|
||||
above = previous[i]
|
||||
corner = previous[i - bpp] if i >= bpp else 0
|
||||
estimate = left + above - corner
|
||||
paeth = min((left, above, corner), key=lambda candidate: abs(estimate - candidate))
|
||||
predictor = (0, left, above, (left + above) // 2, paeth)[method]
|
||||
result.append((value - predictor) % 256)
|
||||
return bytes(result)
|
||||
|
||||
|
||||
def png_with_trns(
|
||||
depth: int,
|
||||
transparent: tuple[int, ...],
|
||||
*,
|
||||
size: tuple[int, int] = (64, 64),
|
||||
filtering: int = 0,
|
||||
interlace: int = 0,
|
||||
orientation: int = 1,
|
||||
) -> tuple[bytes, bytes]:
|
||||
width, height = size
|
||||
neighbors = [
|
||||
tuple(value ^ (1 << bit) if i == channel else value for i, value in enumerate(transparent))
|
||||
for bit in ([0, 8] if depth == 16 else [0])
|
||||
for channel in range(len(transparent))
|
||||
]
|
||||
neighbors.insert(0, ((transparent[0] + 1) % (1 << depth), *transparent[1:]))
|
||||
pixels = [
|
||||
[
|
||||
transparent if x < (width + 1) // 2 else neighbors[(y // 8) % len(neighbors)]
|
||||
for x in range(width)
|
||||
]
|
||||
for y in range(height)
|
||||
]
|
||||
raw = bytearray()
|
||||
for x0, y0, dx, dy in PASSES if interlace else ((0, 0, 1, 1),):
|
||||
columns = range(x0, width, dx)
|
||||
if not columns:
|
||||
continue
|
||||
previous = bytes((len(columns) * len(transparent) * depth + 7) // 8)
|
||||
for y in range(y0, height, dy):
|
||||
row = packed([value for x in columns for value in pixels[y][x]], depth)
|
||||
raw.extend(filtered(row, previous, max(1, len(transparent) * depth // 8), filtering))
|
||||
previous = row
|
||||
header = struct.pack(
|
||||
">IIBBBBB", width, height, depth, 0 if len(transparent) == 1 else 2, 0, 0, interlace
|
||||
)
|
||||
exif = Image.Exif()
|
||||
exif[0x0112] = orientation
|
||||
compressed = zlib.compress(raw)
|
||||
data = (
|
||||
SIGNATURE
|
||||
+ chunk(b"IHDR", header)
|
||||
+ chunk(b"tRNS", b"".join(value.to_bytes(2, "big") for value in transparent))
|
||||
+ chunk(b"eXIf", exif.tobytes()[6:])
|
||||
+ b"".join(chunk(b"IDAT", compressed[i : i + 7]) for i in range(0, len(compressed), 7))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
expected = bytearray()
|
||||
for row in pixels:
|
||||
for pixel in row:
|
||||
channels = tuple(
|
||||
v >> 8 if depth == 16 else v * 255 // ((1 << depth) - 1) for v in pixel
|
||||
)
|
||||
expected.extend(
|
||||
(255, 255, 255)
|
||||
if pixel == transparent
|
||||
else channels * (3 if len(channels) == 1 else 1)
|
||||
)
|
||||
buffer = BytesIO()
|
||||
Image.frombytes("RGB", size, expected).save(buffer, "PNG", exif=exif)
|
||||
return data, buffer.getvalue()
|
||||
@@ -15,7 +15,7 @@ import httpx
|
||||
import pytest
|
||||
from httpx import AsyncClient, Response
|
||||
|
||||
from pictures import GIF, JPEG, SAMPLE, WEBP, png
|
||||
from pictures import BROKEN_PNG, DUPLICATE_IHDR_PNG, GIF, JPEG, SAMPLE, WEBP, png
|
||||
from stand import BASE_URL, bearer, compose, enroll, lb_up, psql, waited
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
@@ -313,6 +313,8 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None:
|
||||
)
|
||||
expect("PNG 8001×5000", await upload(http, alice, chat, png(8001, 5000, rows=False)), 413)
|
||||
for label, content, kind in (
|
||||
("PNG с корректными CRC и битым IDAT", BROKEN_PNG, "image/png"),
|
||||
("PNG с повторным IHDR", DUPLICATE_IHDR_PNG, "image/png"),
|
||||
("обрезанный PNG", SAMPLE[: len(SAMPLE) // 2], "image/png"),
|
||||
("обрезанный JPEG", JPEG[: len(JPEG) // 2], "image/jpeg"),
|
||||
("JPEG как image/png", JPEG, "image/png"),
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
|
||||
|
||||
def command(*args: str) -> str:
|
||||
return subprocess.check_output(args, text=True).strip()
|
||||
|
||||
|
||||
project = os.environ.get("COMPOSE_PROJECT_NAME", "mtusi2026lab6")
|
||||
containers = [
|
||||
json.loads(line)
|
||||
for line in command(
|
||||
"docker",
|
||||
"ps",
|
||||
"--format",
|
||||
'{"name":"{{.Names}}","project":"{{.Label "com.docker.compose.project"}}"}',
|
||||
).splitlines()
|
||||
]
|
||||
own = {row["name"] for row in containers if row["project"] == project}
|
||||
stats = [
|
||||
json.loads(line)
|
||||
for line in command("docker", "stats", "--no-stream", "--format", "{{json .}}").splitlines()
|
||||
]
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"at": datetime.now(UTC).isoformat(),
|
||||
"runtime_sha": command("git", "rev-parse", "HEAD"),
|
||||
"platform": platform.system(),
|
||||
"architecture": platform.machine(),
|
||||
"host_cpus": os.cpu_count(),
|
||||
"host_load": os.getloadavg(),
|
||||
"docker": command(
|
||||
"docker",
|
||||
"info",
|
||||
"--format",
|
||||
"{{.NCPU}} CPUs; {{.MemTotal}} bytes; {{.ServerVersion}}",
|
||||
),
|
||||
"own_project": project,
|
||||
"own": [
|
||||
{key: row[key] for key in ("Name", "CPUPerc", "MemUsage", "PIDs")}
|
||||
for row in stats
|
||||
if row["Name"] in own
|
||||
],
|
||||
"other_projects": dict(
|
||||
Counter(
|
||||
row["project"] or "unlabelled" for row in containers if row["name"] not in own
|
||||
)
|
||||
),
|
||||
"sampling": "Shared Docker snapshot; no throughput claim",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,130 @@
|
||||
import base64
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
base = urlsplit(os.environ.get("BASE_URL", "http://127.0.0.1:18086"))
|
||||
seen = set()
|
||||
checks = []
|
||||
password = "Пароль-проверки-01"
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None, status=200):
|
||||
request_id = "validation-wire"
|
||||
assert base.hostname
|
||||
conn = http.client.HTTPConnection(base.hostname, base.port, timeout=12)
|
||||
head = {"X-Request-Id": request_id, "Content-Type": "application/json", **(headers or {})}
|
||||
conn.request(method, path, None if body is None else json.dumps(body).encode(), head)
|
||||
response = conn.getresponse()
|
||||
data = response.read()
|
||||
instance = response.getheader("X-Instance-Id")
|
||||
assert response.status == status, (method, path, response.status, status)
|
||||
assert instance
|
||||
assert response.getheader("X-Request-Id") == request_id
|
||||
seen.add(instance)
|
||||
result = json.loads(data) if data else None
|
||||
if status >= 400:
|
||||
assert result is not None
|
||||
assert result["error"]["request_id"] == request_id
|
||||
if status == 401:
|
||||
assert response.getheader("WWW-Authenticate") == (
|
||||
'Basic realm="messenger", charset="UTF-8"'
|
||||
if path == "/api/v1/auth/sessions"
|
||||
else 'Bearer realm="messenger"'
|
||||
)
|
||||
conn.close()
|
||||
return result, instance
|
||||
|
||||
|
||||
def basic(name, secret=password):
|
||||
return {"Authorization": "Basic " + base64.b64encode(f"{name}:{secret}".encode()).decode()}
|
||||
|
||||
|
||||
def enroll():
|
||||
name = "wire_" + uuid.uuid4().hex[:15]
|
||||
user, _ = call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
{"username": name, "password": password, "display_name": "Алиса 👋"},
|
||||
status=201,
|
||||
)
|
||||
login, _ = call("POST", "/api/v1/auth/sessions", headers=basic(name), status=201)
|
||||
return user, {"Authorization": "Bearer " + login["token"]}
|
||||
|
||||
|
||||
user, auth = enroll()
|
||||
wrong, _ = call(
|
||||
"POST", "/api/v1/auth/sessions", headers=basic(user["username"], "wrong-password"), status=401
|
||||
)
|
||||
for raw in [b"Basic \x80", b"Basic \xff", b"Basic \xc3\xa9", b"Basic YWxpY2U6c2VjcmV0\xff"]:
|
||||
result, instance = call(
|
||||
"POST", "/api/v1/auth/sessions", headers={"Authorization": raw}, status=401
|
||||
)
|
||||
assert result == wrong
|
||||
checks.append(
|
||||
{"case": "non_ascii_basic", "raw_hex": raw.hex(), "status": 401, "instance": instance}
|
||||
)
|
||||
for text in ["\x00", "\x00abc", "abc\x00def", "abc\x00"]:
|
||||
result, instance = call("POST", "/api/v1/auth/sessions", headers=basic(text), status=401)
|
||||
assert result == wrong
|
||||
body = {"username": "wire_" + uuid.uuid4().hex[:15], "password": password, "display_name": text}
|
||||
call("POST", "/api/v1/users", body, status=400)
|
||||
call("POST", "/api/v1/users", {**body, "display_name": "Алиса 👋"}, status=201)
|
||||
call("POST", "/api/v1/chats", {"title": text, "member_ids": [user["id"]]}, auth, status=400)
|
||||
chats, _ = call("GET", "/api/v1/chats", headers=auth)
|
||||
assert not chats["items"]
|
||||
checks.append(
|
||||
{
|
||||
"case": "nul_basic_display_title",
|
||||
"position": repr(text),
|
||||
"status": [401, 400, 201, 400],
|
||||
"instance": instance,
|
||||
}
|
||||
)
|
||||
chat, _ = call(
|
||||
"POST", "/api/v1/chats", {"title": "Проверка 👋", "member_ids": [user["id"]]}, auth, status=201
|
||||
)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
messages = []
|
||||
for text in ["\x00", "\x00abc", "abc\x00def", "abc\x00"]:
|
||||
headers = {**auth, "Idempotency-Key": uuid.uuid4().hex}
|
||||
call("POST", path, {"text": text}, headers, status=400)
|
||||
page, _ = call("GET", path, headers=auth)
|
||||
assert page["items"] == messages
|
||||
first, one = call("POST", path, {"text": "Исправлено\tбез потерь\n👋"}, headers, status=201)
|
||||
again, two = call("POST", path, {"text": first["text"]}, headers, status=201)
|
||||
assert first == again and one != two
|
||||
messages.append(first)
|
||||
page, _ = call("GET", path, headers=auth)
|
||||
assert page["items"] == messages
|
||||
checks.append(
|
||||
{
|
||||
"case": "nul_message_corrected_retry",
|
||||
"position": repr(text),
|
||||
"instances": [one, two],
|
||||
"status": [400, 201, 201],
|
||||
"copies": 1,
|
||||
}
|
||||
)
|
||||
_, outsider = enroll()
|
||||
for target in [chat["id"], str(uuid.uuid4())]:
|
||||
call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{target}/messages",
|
||||
{"text": "abc\x00def"},
|
||||
{**outsider, "Idempotency-Key": uuid.uuid4().hex},
|
||||
status=404,
|
||||
)
|
||||
checks.append({"case": "nul_access", "status": [404, 404]})
|
||||
call("DELETE", "/api/v1/auth/sessions/current", headers=auth, status=204)
|
||||
for _ in range(4):
|
||||
call("GET", "/api/v1/users/me", headers=auth, status=401)
|
||||
checks.append({"case": "logout_across_replicas", "status": [204, 401, 401, 401, 401]})
|
||||
assert len(seen) >= 2
|
||||
print(
|
||||
json.dumps(
|
||||
{"checks": checks, "instances": sorted(seen), "pass": True}, ensure_ascii=False, indent=2
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,386 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image
|
||||
from scripts import pipeline_suite as ps
|
||||
|
||||
from messenger.attachments.infrastructure.images import RenderLimits, render_variants
|
||||
from png_trns_fixtures import boundary_png, png_with_trns
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
MODE = sys.argv.pop(1)
|
||||
OUT = Path(os.environ.get("FORWARD_EVIDENCE_DIR", "evidence/forward-published")) / (MODE + ".json")
|
||||
if OUT.exists():
|
||||
raise SystemExit("Result exists; do not repeat completed evidence")
|
||||
SUITE = ps.Suite(ps.arguments())
|
||||
STACK = SUITE.stack
|
||||
REPORT: dict[str, Any] = {
|
||||
"mode": MODE,
|
||||
"source_sha": ps.run(["git", "rev-parse", "HEAD"]).strip(),
|
||||
"started_at": ps.now(),
|
||||
"cases": [],
|
||||
}
|
||||
LIMITS = RenderLimits(10485760, 40000000)
|
||||
|
||||
|
||||
def save():
|
||||
REPORT["accepted_ids"] = SUITE.api.accepted if SUITE.api else []
|
||||
OUT.write_text(json.dumps(REPORT, ensure_ascii=False, indent=2) + "\n")
|
||||
|
||||
|
||||
def case(name, function):
|
||||
row = {"name": name, "started_at": ps.now()}
|
||||
REPORT["cases"].append(row)
|
||||
save()
|
||||
began = time.monotonic()
|
||||
try:
|
||||
row["observed"] = function()
|
||||
row["result"] = "PASS"
|
||||
except Exception as error:
|
||||
row.update(result="FAIL", error=str(error))
|
||||
row["elapsed_seconds"] = round(time.monotonic() - began, 3)
|
||||
save()
|
||||
print(name, row["result"], row.get("error", ""), flush=True)
|
||||
|
||||
|
||||
def containers():
|
||||
return STACK.dc("ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}").splitlines()
|
||||
|
||||
|
||||
def outage():
|
||||
assert SUITE.api is not None
|
||||
SUITE.drained()
|
||||
before_dlq = STACK.queues()[ps.DLQ]
|
||||
STACK.stop("worker")
|
||||
att = SUITE.api.fixture()
|
||||
ident = att["id"]
|
||||
ps.poll(
|
||||
lambda: all(r["published"] for r in STACK.sql("outbox", id=ident)), 30, "published outbox"
|
||||
)
|
||||
jobs = STACK.sql("jobs", id=ident)
|
||||
REPORT["outage_checkpoint"] = {
|
||||
"attachment_id": ident,
|
||||
"jobs_before": jobs,
|
||||
"dlq_before": before_dlq,
|
||||
}
|
||||
save()
|
||||
STACK.stop("postgres")
|
||||
REPORT["outage_checkpoint"]["stopped_containers"] = containers()
|
||||
save()
|
||||
since = ps.now()
|
||||
worker_id = STACK.dc("ps", "-aq", "worker").strip()
|
||||
assert worker_id
|
||||
ps.run(["docker", "start", worker_id])
|
||||
|
||||
def requeues():
|
||||
logs = STACK.dc("logs", "--no-color", "--since", since, "worker")
|
||||
rows = [
|
||||
line
|
||||
for line in logs.splitlines()
|
||||
if "job.requeued" in line and jobs[0]["job_id"] in line
|
||||
]
|
||||
return rows if len(rows) >= 12 else None
|
||||
|
||||
try:
|
||||
lines = ps.poll(requeues, 90, "12 real database failures", 1)
|
||||
OUT.with_name(OUT.stem + "-worker.txt").write_text("\n".join(lines) + "\n")
|
||||
during = STACK.queues()
|
||||
finally:
|
||||
STACK.up("postgres")
|
||||
ps.poll(lambda: ps.http_ready(SUITE.args.base_url), 30, "API after PostgreSQL")
|
||||
ready = SUITE.api.wait(ident)
|
||||
SUITE.verify(ready)
|
||||
SUITE.drained()
|
||||
after = STACK.sql("jobs", id=ident)
|
||||
assert (
|
||||
after[0]["job_id"] == jobs[0]["job_id"] and after[0]["attempt"] == jobs[0]["attempt"] == 1
|
||||
)
|
||||
assert after[0]["status"] == "done"
|
||||
assert STACK.queues()[ps.DLQ] == before_dlq
|
||||
assert len(STACK.sql("variants", id=ident)) == 3
|
||||
return {
|
||||
"attachment_id": ident,
|
||||
"jobs_before": jobs,
|
||||
"jobs_after": after,
|
||||
"requeues": len(lines),
|
||||
"queues_during": during,
|
||||
"queues_after": STACK.queues(),
|
||||
"replay": False,
|
||||
"stopped_containers": REPORT["outage_checkpoint"]["stopped_containers"],
|
||||
}
|
||||
|
||||
|
||||
def png(name, data, reference):
|
||||
assert SUITE.api is not None
|
||||
att = SUITE.api.upload(data, name=name + ".png")
|
||||
ident = att["id"]
|
||||
results = []
|
||||
for version in (1, 2):
|
||||
if version == 2:
|
||||
STACK.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2")
|
||||
ps.poll(lambda: SUITE.record(ident)["published_version"] == 2, 90, "published v2")
|
||||
ready = SUITE.api.wait(ident)
|
||||
SUITE.verify(ready)
|
||||
assert SUITE.api.content(ident)["bytes"] == data
|
||||
expected = {r.name: r for r in render_variants(reference, LIMITS, version)}
|
||||
variants = []
|
||||
for variant in ready["variants"]:
|
||||
content = SUITE.api.content(ident, variant["name"])["bytes"]
|
||||
assert content == expected[variant["name"]].data, (
|
||||
variant["name"] + " differs from independent white reference"
|
||||
)
|
||||
with Image.open(BytesIO(content)) as image:
|
||||
assert not image.getexif()
|
||||
assert not any(key in image.info for key in ("icc_profile", "xmp", "transparency"))
|
||||
variants.append(
|
||||
{
|
||||
"name": variant["name"],
|
||||
"sha256": hashlib.sha256(content).hexdigest(),
|
||||
"bytes": len(content),
|
||||
"dimensions": [variant["width"], variant["height"]],
|
||||
}
|
||||
)
|
||||
results.append({"version": version, "variants": variants})
|
||||
before = SUITE.snapshot(ident)
|
||||
STACK.publish(SUITE.payload(ident, 1))
|
||||
STACK.publish(SUITE.payload(ident, 2))
|
||||
SUITE.drained()
|
||||
assert SUITE.snapshot(ident) == before
|
||||
assert len(STACK.sql("variants", id=ident)) == 6
|
||||
return {
|
||||
"attachment_id": ident,
|
||||
"input_sha256": hashlib.sha256(data).hexdigest(),
|
||||
"input_bytes": len(data),
|
||||
"versions": results,
|
||||
"stale_and_duplicate_noop": True,
|
||||
}
|
||||
|
||||
|
||||
FIXTURES = ROOT / "build/forward-png"
|
||||
FIXTURES.mkdir(parents=True, exist_ok=True)
|
||||
for fixture_name, fixture_depth, fixture_key in (
|
||||
("gray2", 2, (1,)),
|
||||
("gray4", 4, (5,)),
|
||||
("gray16", 16, (12345,)),
|
||||
("rgb16", 16, (12345, 23456, 34567)),
|
||||
):
|
||||
(FIXTURES / (fixture_name + "-40mp.png")).write_bytes(boundary_png(fixture_depth, fixture_key))
|
||||
ps.SQL["timed_events"] = (
|
||||
"SELECT attachment_id,job_id,kind,at FROM job_events "
|
||||
"WHERE attachment_id = ANY(:'ids'::uuid[]) AND kind IN ('claimed','done') ORDER BY id"
|
||||
)
|
||||
|
||||
|
||||
def without_trns(data):
|
||||
result, offset = data[:8], 8
|
||||
while offset < len(data):
|
||||
size = struct.unpack(">I", data[offset : offset + 4])[0] + 12
|
||||
if data[offset + 4 : offset + 8] != b"tRNS":
|
||||
result += data[offset : offset + size]
|
||||
offset += size
|
||||
return result
|
||||
|
||||
|
||||
def verify_large(ident, data, transparent):
|
||||
assert SUITE.api is not None
|
||||
ready = SUITE.api.wait(ident)
|
||||
SUITE.verify(ready)
|
||||
assert SUITE.api.content(ident)["bytes"] == data
|
||||
variants = []
|
||||
for variant in ready["variants"]:
|
||||
content = SUITE.api.content(ident, variant["name"])["bytes"]
|
||||
with Image.open(BytesIO(content)) as image:
|
||||
left = image.getpixel((5, image.height // 2))
|
||||
right = image.getpixel((image.width - 6, image.height // 2))
|
||||
assert (left == (255, 255, 255)) == transparent
|
||||
assert right != (255, 255, 255)
|
||||
assert not image.getexif()
|
||||
variants.append({**variant, "left_pixel": left, "right_pixel": right})
|
||||
return variants
|
||||
|
||||
|
||||
def duplicate_check(ident):
|
||||
before = SUITE.snapshot(ident)
|
||||
STACK.publish(SUITE.payload(ident, 1))
|
||||
STACK.publish(SUITE.payload(ident, 2))
|
||||
SUITE.drained()
|
||||
assert SUITE.snapshot(ident) == before
|
||||
assert len(STACK.sql("variants", id=ident)) == 6
|
||||
|
||||
|
||||
def large(name, data, transparent=True):
|
||||
assert SUITE.api is not None
|
||||
att = SUITE.api.upload(data, name=name + ".png")
|
||||
ident = att["id"]
|
||||
versions = [{"version": 1, "variants": verify_large(ident, data, transparent)}]
|
||||
STACK.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2")
|
||||
ps.poll(lambda: SUITE.record(ident)["published_version"] == 2, 90, "v2 publication")
|
||||
versions.append({"version": 2, "variants": verify_large(ident, data, transparent)})
|
||||
duplicate_check(ident)
|
||||
return {
|
||||
"attachment_id": ident,
|
||||
"input_sha256": hashlib.sha256(data).hexdigest(),
|
||||
"bytes": len(data),
|
||||
"versions": versions,
|
||||
"memory_peak": int(STACK.cgroup("worker", "memory.peak")),
|
||||
"memory_max": int(STACK.cgroup("worker", "memory.max")),
|
||||
}
|
||||
|
||||
|
||||
def concurrent():
|
||||
assert SUITE.api is not None
|
||||
data = (FIXTURES / "rgb16-40mp.png").read_bytes()
|
||||
SUITE.drained()
|
||||
STACK.stop("worker")
|
||||
try:
|
||||
ids = [SUITE.api.upload(data, name="concurrent-rgb16.png")["id"] for _ in range(2)]
|
||||
SUITE.queued(2)
|
||||
finally:
|
||||
ps.run(["docker", "start", STACK.dc("ps", "-aq", "worker").strip()])
|
||||
versions = []
|
||||
for version in (1, 2):
|
||||
if version == 2:
|
||||
STACK.stop("worker")
|
||||
try:
|
||||
for ident in ids:
|
||||
STACK.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2")
|
||||
SUITE.queued(2)
|
||||
finally:
|
||||
ps.run(["docker", "start", STACK.dc("ps", "-aq", "worker").strip()])
|
||||
for ident in ids:
|
||||
ps.poll(
|
||||
lambda ident=ident, version=version: (
|
||||
SUITE.record(ident)["published_version"] == version
|
||||
),
|
||||
90,
|
||||
"concurrent publication",
|
||||
)
|
||||
variants = {ident: verify_large(ident, data, True) for ident in ids}
|
||||
events = STACK.sql("timed_events", ids="{" + ",".join(ids) + "}")
|
||||
job_ids = {
|
||||
row["job_id"]
|
||||
for ident in ids
|
||||
for row in STACK.sql("jobs", id=ident)
|
||||
if row["pipeline_version"] == version
|
||||
}
|
||||
selected = [e for e in events if e["job_id"] in job_ids]
|
||||
claims = [e["at"] for e in selected if e["kind"] == "claimed"]
|
||||
done = [e["at"] for e in selected if e["kind"] == "done"]
|
||||
assert len(claims) == len(done) == 2
|
||||
assert max(claims) < min(done), selected
|
||||
versions.append(
|
||||
{
|
||||
"version": version,
|
||||
"events": selected,
|
||||
"processing_overlapped": True,
|
||||
"variants": variants,
|
||||
}
|
||||
)
|
||||
for ident in ids:
|
||||
duplicate_check(ident)
|
||||
peak, limit = (int(STACK.cgroup("worker", name)) for name in ("memory.peak", "memory.max"))
|
||||
assert peak < limit
|
||||
return {
|
||||
"attachment_ids": ids,
|
||||
"versions": versions,
|
||||
"memory_peak": peak,
|
||||
"memory_max": limit,
|
||||
"memory_events": STACK.cgroup("worker", "memory.events"),
|
||||
}
|
||||
|
||||
|
||||
def forced(**overrides):
|
||||
data = (FIXTURES / "rgb16-40mp.png").read_bytes()
|
||||
result = SUITE.limited(data, **overrides)
|
||||
STACK.restore()
|
||||
result["next_normal"] = SUITE.smoke_pipeline()
|
||||
return result
|
||||
|
||||
|
||||
try:
|
||||
SUITE.detect_api()
|
||||
SUITE.api = ps.Api(SUITE.args)
|
||||
dead_before = STACK.queues()[ps.DLQ][0]
|
||||
REPORT["conditions"] = {
|
||||
"host_load": os.getloadavg(),
|
||||
"containers": containers(),
|
||||
"running": ps.run(["docker", "ps", "--format", "{{.Names}} {{.Status}}"]),
|
||||
"worker_limits": STACK.dc(
|
||||
"exec",
|
||||
"-T",
|
||||
"worker",
|
||||
"printenv",
|
||||
"RENDER_MEMORY_MB",
|
||||
"RENDER_TIMEOUT_SECONDS",
|
||||
"WORKER_CONCURRENCY",
|
||||
),
|
||||
}
|
||||
save()
|
||||
if MODE == "postgres-outage":
|
||||
case("published_outbox_database_outage", outage)
|
||||
elif MODE == "png-http":
|
||||
for name, depth, color in [
|
||||
("gray1", 1, (0,)),
|
||||
("gray2", 2, (1,)),
|
||||
("gray4", 4, (5,)),
|
||||
("gray8", 8, (85,)),
|
||||
("gray16", 16, (12345,)),
|
||||
("rgb8", 8, (255, 0, 0)),
|
||||
("rgb16", 16, (12345, 23456, 34567)),
|
||||
]:
|
||||
case(
|
||||
name,
|
||||
lambda name=name, depth=depth, color=color: png(name, *png_with_trns(depth, color)),
|
||||
)
|
||||
case(
|
||||
"rgb16-adam7-filter4-exif6",
|
||||
lambda: png(
|
||||
"adam7",
|
||||
*png_with_trns(
|
||||
16,
|
||||
(12345, 23456, 34567),
|
||||
size=(65, 63),
|
||||
interlace=1,
|
||||
filtering=4,
|
||||
orientation=6,
|
||||
),
|
||||
),
|
||||
)
|
||||
elif MODE == "png-boundary":
|
||||
for name in ("gray2", "gray4", "gray16", "rgb16"):
|
||||
case(
|
||||
name, lambda name=name: large(name, (FIXTURES / (name + "-40mp.png")).read_bytes())
|
||||
)
|
||||
case(
|
||||
"gray16-without-trns",
|
||||
lambda: large(
|
||||
"gray16-opaque", without_trns((FIXTURES / "gray16-40mp.png").read_bytes()), False
|
||||
),
|
||||
)
|
||||
case("concurrent-rgb16", concurrent)
|
||||
case("forced-timeout", lambda: forced(FAULT_RENDER_TIMEOUT_SECONDS="0.01"))
|
||||
case("forced-memory", lambda: forced(FAULT_RENDER_MEMORY_MB="200"))
|
||||
ids = "{" + ",".join(SUITE.api.accepted) + "}"
|
||||
ps.poll(lambda: not STACK.sql("staging_left", ids=ids), 45, "staging sweep")
|
||||
SUITE.drained()
|
||||
REPORT["invariants"] = SUITE.invariants(dead_before)
|
||||
finally:
|
||||
STACK.up("postgres")
|
||||
STACK.restore()
|
||||
REPORT["finished_at"] = ps.now()
|
||||
REPORT["result"] = (
|
||||
"PASS"
|
||||
if REPORT["cases"]
|
||||
and all(x.get("result") == "PASS" for x in REPORT["cases"] + REPORT.get("invariants", []))
|
||||
else "FAIL"
|
||||
)
|
||||
save()
|
||||
raise SystemExit(0 if REPORT["result"] == "PASS" else 1)
|
||||
@@ -0,0 +1,43 @@
|
||||
import http.client
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
|
||||
status = int(sys.argv[1])
|
||||
path = {502: "/bad", 503: "/none", 504: "/slow"}[status]
|
||||
if status != 503:
|
||||
for _ in range(60):
|
||||
conn = http.client.HTTPConnection("lb", 8080, timeout=2)
|
||||
conn.request("GET", "/health/ready")
|
||||
response = conn.getresponse()
|
||||
response.read()
|
||||
conn.close()
|
||||
if response.status == 200:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise RuntimeError("Backend never became ready")
|
||||
else:
|
||||
time.sleep(5)
|
||||
for value in ['a"b', "a\\b", "a\tb", "x" * 200]:
|
||||
conn = http.client.HTTPConnection("lb", 8080, timeout=20)
|
||||
start = time.monotonic()
|
||||
conn.request("GET", path, headers={"X-Request-Id": value})
|
||||
response = conn.getresponse()
|
||||
payload = json.loads(response.read())
|
||||
assert response.status == status, response.status
|
||||
assert response.getheader("X-Request-Id") == value
|
||||
assert payload["error"]["request_id"] == value
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": status,
|
||||
"request_id": value,
|
||||
"json_valid": True,
|
||||
"echo_matches": True,
|
||||
"seconds": round(time.monotonic() - start, 3),
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
conn.close()
|
||||
@@ -0,0 +1,25 @@
|
||||
services:
|
||||
lb:
|
||||
image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b
|
||||
volumes: [../../../infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
|
||||
init: true
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128m}
|
||||
api:
|
||||
image: ${API_IMAGE:?Set the image built by the lab6 stack}
|
||||
entrypoint: [python, /fixture/server.py]
|
||||
volumes: [./:/fixture:ro]
|
||||
init: true
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128m}
|
||||
probe:
|
||||
image: ${API_IMAGE:?Set the image built by the lab6 stack}
|
||||
entrypoint: [python, /fixture/client.py]
|
||||
volumes: [./:/fixture:ro]
|
||||
profiles: [probe]
|
||||
init: true
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128m}
|
||||
@@ -0,0 +1,20 @@
|
||||
import socketserver
|
||||
import time
|
||||
|
||||
|
||||
class Handler(socketserver.StreamRequestHandler):
|
||||
def handle(self) -> None:
|
||||
request = self.rfile.readline(8192)
|
||||
while self.rfile.readline(8192) not in {b"\r\n", b""}:
|
||||
pass
|
||||
path = request.split(b" ")[1]
|
||||
if path == b"/bad":
|
||||
self.wfile.write(b"NOT-HTTP\r\n\r\n")
|
||||
elif path == b"/slow":
|
||||
time.sleep(12)
|
||||
else:
|
||||
self.wfile.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK")
|
||||
|
||||
|
||||
with socketserver.ThreadingTCPServer(("0.0.0.0", 8080), Handler) as server:
|
||||
server.serve_forever()
|
||||
@@ -1,12 +1,24 @@
|
||||
import json
|
||||
from base64 import b64encode
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
from starlette.types import Scope
|
||||
|
||||
from messenger.identity.application.queries import VerifyCredentials
|
||||
from messenger.identity.entrypoints.authentication import (
|
||||
BASIC_CHALLENGE,
|
||||
LOGIN_PATH,
|
||||
basic_credentials,
|
||||
bearer_token,
|
||||
challenge_for,
|
||||
guard,
|
||||
)
|
||||
from messenger.shared.application.ports import Logger
|
||||
from messenger.shared.entrypoints.pipeline import RequestPipeline
|
||||
|
||||
NON_ASCII_BASIC = [b"Basic \x80", b"Basic \xff", b"Basic \xc3\xa9", b"Basic YWxpY2U6c2VjcmV0\xff"]
|
||||
|
||||
|
||||
def encoded(raw: bytes) -> str:
|
||||
@@ -43,6 +55,46 @@ def test_basic_rejects_malformed(raw: str) -> None:
|
||||
assert basic_credentials(raw) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", NON_ASCII_BASIC)
|
||||
def test_basic_rejects_non_ascii(raw: bytes) -> None:
|
||||
assert basic_credentials(raw.decode("latin-1")) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", NON_ASCII_BASIC)
|
||||
async def test_non_ascii_basic_returns_401_without_credential_lookup(raw: bytes) -> None:
|
||||
verify = AsyncMock(return_value=None)
|
||||
container = SimpleNamespace(get=AsyncMock(return_value=verify))
|
||||
scope: Scope = {
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": LOGIN_PATH,
|
||||
"headers": [(b"authorization", raw), (b"x-request-id", b"non-ascii-basic")],
|
||||
"app": SimpleNamespace(state=SimpleNamespace(dishka_container=container)),
|
||||
}
|
||||
endpoint, send = AsyncMock(), AsyncMock()
|
||||
pipeline = RequestPipeline(endpoint, Mock(spec=Logger), 1024, 1, guard, "test-instance")
|
||||
await pipeline(scope, AsyncMock(), send)
|
||||
start, body = [call.args[0] for call in send.await_args_list]
|
||||
assert start["status"] == 401
|
||||
headers = dict(start["headers"])
|
||||
assert headers[b"www-authenticate"] == BASIC_CHALLENGE.encode()
|
||||
assert headers[b"x-instance-id"] == b"test-instance"
|
||||
assert headers[b"x-request-id"] == b"non-ascii-basic"
|
||||
assert headers[b"content-type"] == b"application/json"
|
||||
assert json.loads(body["body"]) == {
|
||||
"error": {
|
||||
"code": "unauthorized",
|
||||
"message": "Аутентификация не предоставлена или недействительна",
|
||||
"request_id": "non-ascii-basic",
|
||||
}
|
||||
}
|
||||
container.get.assert_awaited_once_with(VerifyCredentials)
|
||||
verify.assert_not_awaited()
|
||||
endpoint.assert_not_awaited()
|
||||
assert "user" not in scope["state"]
|
||||
assert scope["state"]["actor"] is None
|
||||
|
||||
|
||||
def test_bearer_accepts_token68() -> None:
|
||||
assert bearer_token("Bearer abc-DEF_123.~+/==") == "abc-DEF_123.~+/=="
|
||||
assert bearer_token("bearer token") == "token"
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import json
|
||||
import shlex
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
CONFIG = Path(__file__).resolve().parents[2] / "infra/haproxy/haproxy.cfg"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", [502, 503, 504])
|
||||
@pytest.mark.parametrize("request_id", ['a"b', "a\\b", "a\tb", "aÿb", "x" * 200])
|
||||
def test_error_templates_escape_json_without_changing_header(status: int, request_id: str) -> None:
|
||||
line = next(
|
||||
line.strip()
|
||||
for line in CONFIG.read_text().splitlines()
|
||||
if line.strip().startswith(f"http-error status {status} ")
|
||||
)
|
||||
tokens = shlex.split(line)
|
||||
header = tokens[tokens.index("hdr") + 1 : tokens.index("lf-string")]
|
||||
assert header == ["X-Request-Id", "%[var(txn.request_id)]"]
|
||||
template = tokens[tokens.index("lf-string") + 1]
|
||||
escaped = json.dumps(request_id, ensure_ascii=True)[1:-1]
|
||||
rendered = template.replace("%[var(txn.request_id),json(ascii)]", escaped)
|
||||
body = json.loads(rendered)
|
||||
assert body["error"]["request_id"] == request_id
|
||||
assert body["error"]["code"] == "unavailable"
|
||||
@@ -0,0 +1,279 @@
|
||||
import struct
|
||||
import tracemalloc
|
||||
import zlib
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
from PIL import Image, ImageFile
|
||||
|
||||
from messenger.attachments.domain.errors import PayloadTooLarge, UnsupportedImage
|
||||
from messenger.attachments.domain.values import ImageInfo, ImageKind
|
||||
from messenger.attachments.infrastructure import png as png_decoder
|
||||
from messenger.attachments.infrastructure.images import PillowImageInspector
|
||||
from pictures import BROKEN_PNG, DUPLICATE_IHDR_PNG, JPEG, SAMPLE, chunk, png, png_from_idat
|
||||
|
||||
FORMATS = [(0, d, 1) for d in (1, 2, 4, 8, 16)] + [
|
||||
(color, depth, samples)
|
||||
for color, depths, samples in (
|
||||
(2, (8, 16), 3),
|
||||
(3, (1, 2, 4, 8), 1),
|
||||
(4, (8, 16), 2),
|
||||
(6, (8, 16), 4),
|
||||
)
|
||||
for depth in depths
|
||||
]
|
||||
ADAM7_GRID = (
|
||||
"16462646",
|
||||
"77777777",
|
||||
"56565656",
|
||||
"77777777",
|
||||
"36463646",
|
||||
"77777777",
|
||||
"56565656",
|
||||
"77777777",
|
||||
)
|
||||
PIXEL = zlib.compress(b"\x00\x11\x22\x33")
|
||||
BAD_STREAMS = [
|
||||
pytest.param(b"not a zlib stream", id="invalid-zlib"),
|
||||
pytest.param(PIXEL[:-1], id="truncated-zlib-footer"),
|
||||
pytest.param(PIXEL[:-1] + bytes([PIXEL[-1] ^ 1]), id="bad-adler32"),
|
||||
pytest.param(zlib.compress(b"\x00\x11\x22"), id="short-row"),
|
||||
pytest.param(zlib.compress(b"\x00\x11\x22\x33\x44"), id="long-row"),
|
||||
pytest.param(zlib.compress(b"\x05\x11\x22\x33"), id="invalid-filter"),
|
||||
]
|
||||
|
||||
|
||||
def scanlines(width: int, height: int, bits: int, interlace: int, filtering: int = 0) -> bytes:
|
||||
rows = []
|
||||
for number in range(1, 8) if interlace else (0,):
|
||||
for y in range(height):
|
||||
columns = (
|
||||
sum(int(ADAM7_GRID[y % 8][x % 8]) == number for x in range(width))
|
||||
if interlace
|
||||
else width
|
||||
)
|
||||
if columns:
|
||||
rows.append(bytes([filtering]) + b"\x00" * ((columns * bits + 7) // 8))
|
||||
return b"".join(rows)
|
||||
|
||||
|
||||
async def test_png_with_valid_crc_and_invalid_zlib_is_rejected() -> None:
|
||||
assert len(BROKEN_PNG) == 74
|
||||
with Image.open(BytesIO(BROKEN_PNG)) as image:
|
||||
image.verify()
|
||||
with Image.open(BytesIO(BROKEN_PNG)) as image, pytest.raises(OSError):
|
||||
image.load()
|
||||
with pytest.raises(UnsupportedImage):
|
||||
await PillowImageInspector(40_000_000, 1).inspect(BROKEN_PNG, ImageKind.PNG)
|
||||
|
||||
|
||||
async def test_duplicate_ihdr_cannot_disagree_with_pixel_validation() -> None:
|
||||
assert len(DUPLICATE_IHDR_PNG) == 94
|
||||
with Image.open(BytesIO(DUPLICATE_IHDR_PNG)) as image:
|
||||
assert image.size == (2, 2)
|
||||
image.verify()
|
||||
with Image.open(BytesIO(DUPLICATE_IHDR_PNG)) as image, pytest.raises(OSError):
|
||||
image.load()
|
||||
with pytest.raises(UnsupportedImage):
|
||||
await PillowImageInspector(40_000_000, 1).inspect(DUPLICATE_IHDR_PNG, ImageKind.PNG)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"header",
|
||||
[
|
||||
(1, 1, 8, 2, 0, 0, 0),
|
||||
(2, 2, 8, 2, 0, 0, 0),
|
||||
(1, 1, 16, 2, 0, 0, 0),
|
||||
(1, 1, 8, 6, 0, 0, 0),
|
||||
(1, 1, 8, 2, 0, 0, 1),
|
||||
(8001, 5000, 8, 2, 0, 0, 0),
|
||||
],
|
||||
ids=["identical", "dimensions", "depth", "color", "interlace", "over-pixel-limit"],
|
||||
)
|
||||
@pytest.mark.parametrize("position", ["before-idat", "between-idat", "after-idat"])
|
||||
async def test_duplicate_headers_are_rejected_before_inflating(
|
||||
header: tuple[int, ...], position: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
data = png_from_idat(PIXEL)
|
||||
duplicate = chunk(b"IHDR", struct.pack(">IIBBBBB", *header))
|
||||
if position == "before-idat":
|
||||
data = data[:33] + duplicate + data[33:]
|
||||
elif position == "after-idat":
|
||||
data = data[:-12] + duplicate + data[-12:]
|
||||
else:
|
||||
data = (
|
||||
data[:33]
|
||||
+ chunk(b"IDAT", PIXEL[:3])
|
||||
+ duplicate
|
||||
+ chunk(b"IDAT", PIXEL[3:])
|
||||
+ data[-12:]
|
||||
)
|
||||
|
||||
def forbidden_inflate(*args: object) -> None:
|
||||
pytest.fail("ambiguous PNG header must be rejected before decompression")
|
||||
|
||||
monkeypatch.setattr(png_decoder.zlib, "decompressobj", forbidden_inflate)
|
||||
with pytest.raises(UnsupportedImage):
|
||||
await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("damage", ["not-first", "missing", "short", "long"])
|
||||
async def test_ihdr_must_be_first_and_have_exact_length(damage: str) -> None:
|
||||
data = png_from_idat(PIXEL)
|
||||
if damage == "not-first":
|
||||
data = data[:8] + chunk(b"tEXt", b"name\x00value") + data[8:]
|
||||
elif damage == "missing":
|
||||
data = data[:8] + data[33:]
|
||||
else:
|
||||
header = data[16:29]
|
||||
header = header[:-1] if damage == "short" else header + b"\x00"
|
||||
data = data[:8] + chunk(b"IHDR", header) + data[33:]
|
||||
with pytest.raises(UnsupportedImage):
|
||||
await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", BAD_STREAMS)
|
||||
async def test_corrupt_pixels_are_rejected_even_with_valid_chunk_crc(stream: bytes) -> None:
|
||||
data = png_from_idat(stream, chunk_size=1)
|
||||
with Image.open(BytesIO(data)) as image:
|
||||
image.verify()
|
||||
with pytest.raises(UnsupportedImage):
|
||||
await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG)
|
||||
|
||||
|
||||
async def test_idat_chunks_must_be_consecutive() -> None:
|
||||
data = png_from_idat(PIXEL)
|
||||
data = (
|
||||
data[:33]
|
||||
+ chunk(b"IDAT", PIXEL[:3])
|
||||
+ chunk(b"tEXt", b"name\x00value")
|
||||
+ chunk(b"IDAT", PIXEL[3:])
|
||||
+ data[-12:]
|
||||
)
|
||||
with Image.open(BytesIO(data)) as image:
|
||||
image.verify()
|
||||
with pytest.raises(UnsupportedImage):
|
||||
await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("padding", [b"\x00" * 131_072, b"unused buffer", zlib.compress(b"")])
|
||||
async def test_unused_idat_trailing_bytes_are_ignored(padding: bytes) -> None:
|
||||
data = png_from_idat(PIXEL + padding, chunk_size=1_000_000)
|
||||
with Image.open(BytesIO(data)) as image:
|
||||
image.load()
|
||||
assert await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG) == ImageInfo(
|
||||
ImageKind.PNG, 1, 1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("color", "depth", "samples"), FORMATS)
|
||||
@pytest.mark.parametrize(("width", "height"), [(1, 1), (1, 9), (9, 1), (5, 5), (17, 13)])
|
||||
@pytest.mark.parametrize("interlace", [0, 1])
|
||||
async def test_supported_png_formats_match_pillow_decoding(
|
||||
*, color: int, depth: int, samples: int, width: int, height: int, interlace: int
|
||||
) -> None:
|
||||
raw = scanlines(width, height, depth * samples, interlace)
|
||||
data = png_from_idat(
|
||||
zlib.compress(raw),
|
||||
width,
|
||||
height,
|
||||
depth=depth,
|
||||
color=color,
|
||||
interlace=interlace,
|
||||
chunk_size=3,
|
||||
)
|
||||
with Image.open(BytesIO(data)) as image:
|
||||
image.load()
|
||||
assert image.size == (width, height)
|
||||
assert await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG) == ImageInfo(
|
||||
ImageKind.PNG, width, height
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("filtering", range(5))
|
||||
@pytest.mark.parametrize("interlace", [0, 1])
|
||||
@pytest.mark.parametrize("block_size", [3, 7, 31])
|
||||
async def test_filters_and_passes_cross_decompression_blocks(
|
||||
filtering: int, interlace: int, block_size: int, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(png_decoder, "BLOCK_BYTES", block_size)
|
||||
raw = scanlines(17, 13, 32, interlace, filtering)
|
||||
data = png_from_idat(zlib.compress(raw), 17, 13, color=6, interlace=interlace)
|
||||
with Image.open(BytesIO(data)) as image:
|
||||
image.load()
|
||||
assert (await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG)).width == 17
|
||||
|
||||
|
||||
@pytest.mark.parametrize("interlace", [0, 1])
|
||||
@pytest.mark.parametrize("damage", ["filter", "short", "long"])
|
||||
async def test_corruption_in_later_rows_or_passes(
|
||||
interlace: int, damage: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(png_decoder, "BLOCK_BYTES", 7)
|
||||
raw = bytearray(scanlines(17, 13, 32, interlace))
|
||||
if damage == "filter":
|
||||
raw[-69] = 255
|
||||
elif damage == "short":
|
||||
raw.pop()
|
||||
else:
|
||||
raw.append(0)
|
||||
data = png_from_idat(zlib.compress(raw), 17, 13, color=6, interlace=interlace, chunk_size=5)
|
||||
with pytest.raises(UnsupportedImage):
|
||||
await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("width", "height"), [(1_000_000, 1), (1000, 1000)])
|
||||
def test_png_validation_does_not_allocate_pixels_or_whole_rows(
|
||||
width: int, height: int, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
data = png_from_idat(
|
||||
zlib.compress(b"\x00" * ((width * 8 + 1) * height)), width, height, depth=16, color=6
|
||||
)
|
||||
|
||||
def forbidden_load(*args: object, **kwargs: object) -> None:
|
||||
pytest.fail("PNG validation must not allocate a Pillow pixel buffer")
|
||||
|
||||
monkeypatch.setattr(ImageFile.ImageFile, "load", forbidden_load)
|
||||
inspector = PillowImageInspector(40_000_000, 1)
|
||||
tracemalloc.start()
|
||||
try:
|
||||
assert inspector._inspect(data, ImageKind.PNG) == ImageInfo(ImageKind.PNG, width, height)
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
finally:
|
||||
tracemalloc.stop()
|
||||
assert peak < 2_000_000
|
||||
|
||||
|
||||
def test_excess_inflated_data_is_rejected_with_bounded_memory() -> None:
|
||||
data = png_from_idat(zlib.compress(b"\x00" * 8_000_000))
|
||||
inspector = PillowImageInspector(40_000_000, 1)
|
||||
tracemalloc.start()
|
||||
try:
|
||||
with pytest.raises(UnsupportedImage):
|
||||
inspector._inspect(data, ImageKind.PNG)
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
finally:
|
||||
tracemalloc.stop()
|
||||
assert peak < 2_000_000
|
||||
|
||||
|
||||
async def test_pixel_limit_precedes_pixel_validation(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def forbidden_validation(data: bytes) -> None:
|
||||
pytest.fail("oversized PNG must be rejected before decompression")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"messenger.attachments.infrastructure.images.verify_png_pixels", forbidden_validation
|
||||
)
|
||||
with pytest.raises(PayloadTooLarge):
|
||||
await PillowImageInspector(40_000_000, 1).inspect(
|
||||
png(8001, 5000, rows=False), ImageKind.PNG
|
||||
)
|
||||
|
||||
|
||||
async def test_exact_pixel_limit_and_existing_fixtures_still_work() -> None:
|
||||
inspector = PillowImageInspector(40_000_000, 1)
|
||||
assert await inspector.inspect(png(8000, 5000), ImageKind.PNG) == ImageInfo(
|
||||
ImageKind.PNG, 8000, 5000
|
||||
)
|
||||
assert (await inspector.inspect(SAMPLE, ImageKind.PNG)).kind is ImageKind.PNG
|
||||
assert (await inspector.inspect(JPEG, ImageKind.JPEG)).kind is ImageKind.JPEG
|
||||
@@ -0,0 +1,95 @@
|
||||
from base64 import b64encode
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from starlette.types import Scope
|
||||
|
||||
from messenger.identity.application.queries import VerifyCredentials
|
||||
from messenger.identity.domain.errors import InvalidUserData
|
||||
from messenger.identity.domain.values import DisplayName
|
||||
from messenger.identity.entrypoints.authentication import (
|
||||
BASIC_CHALLENGE,
|
||||
LOGIN_PATH,
|
||||
basic_credentials,
|
||||
guard,
|
||||
)
|
||||
from messenger.messaging.application.commands import PostMessage
|
||||
from messenger.messaging.application.handlers import PostMessageHandler
|
||||
from messenger.messaging.application.ports import ChatDirectory, MessageLog, MessagingUnitOfWork
|
||||
from messenger.messaging.domain.errors import ChatNotFound, InvalidChatData, InvalidMessageText
|
||||
from messenger.messaging.domain.values import ChatTitle, MessageText
|
||||
from messenger.shared.application.ports import Clock, IdGenerator
|
||||
from messenger.shared.domain.errors import DomainError
|
||||
from messenger.shared.entrypoints.pipeline import Rejection
|
||||
|
||||
NUL_TEXTS = ["\x00", "\x00abc", "abc\x00def", "abc\x00"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("username", NUL_TEXTS)
|
||||
def test_basic_rejects_nul_username(username: str) -> None:
|
||||
raw = b64encode(f"{username}:synthetic-password".encode()).decode()
|
||||
assert basic_credentials(f"Basic {raw}") is None
|
||||
|
||||
|
||||
async def test_nul_username_is_rejected_before_credential_lookup() -> None:
|
||||
verify = AsyncMock(return_value=None)
|
||||
container = SimpleNamespace(get=AsyncMock(return_value=verify))
|
||||
scope: Scope = {
|
||||
"method": "POST",
|
||||
"path": LOGIN_PATH,
|
||||
"headers": [(b"authorization", b"Basic " + b64encode(b"abc\x00def:synthetic-password"))],
|
||||
"app": SimpleNamespace(state=SimpleNamespace(dishka_container=container)),
|
||||
}
|
||||
assert await guard(scope) == Rejection(
|
||||
401, "unauthorized", (("www-authenticate", BASIC_CHALLENGE),)
|
||||
)
|
||||
container.get.assert_awaited_once_with(VerifyCredentials)
|
||||
verify.assert_not_awaited()
|
||||
assert "user" not in scope["state"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", NUL_TEXTS)
|
||||
@pytest.mark.parametrize(
|
||||
("kind", "error"),
|
||||
[
|
||||
(DisplayName, InvalidUserData),
|
||||
(ChatTitle, InvalidChatData),
|
||||
(MessageText, InvalidMessageText),
|
||||
],
|
||||
)
|
||||
def test_text_values_reject_nul(
|
||||
kind: type[DisplayName | ChatTitle | MessageText], error: type[DomainError], raw: str
|
||||
) -> None:
|
||||
with pytest.raises(error):
|
||||
kind(raw)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", [DisplayName, ChatTitle, MessageText])
|
||||
def test_text_values_preserve_unicode_and_whitespace(
|
||||
kind: type[DisplayName | ChatTitle | MessageText],
|
||||
) -> None:
|
||||
raw = " Привет\tмир\n👋 "
|
||||
assert kind(raw).value == raw
|
||||
|
||||
|
||||
@pytest.mark.parametrize("member", [False, True])
|
||||
async def test_nul_message_checks_access_without_writing(member: bool) -> None:
|
||||
chats = AsyncMock(spec=ChatDirectory)
|
||||
chats.chat_for_member.return_value = Mock() if member else None
|
||||
log = AsyncMock(spec=MessageLog)
|
||||
log.post.return_value = None
|
||||
uow = Mock(spec=MessagingUnitOfWork)
|
||||
clock = Mock(spec=Clock)
|
||||
clock.now.return_value = datetime(2026, 1, 1, tzinfo=UTC)
|
||||
handler = PostMessageHandler(
|
||||
uow, log, chats, clock, Mock(spec=IdGenerator), ttl=timedelta(days=1)
|
||||
)
|
||||
command = PostMessage(uuid4(), uuid4(), "abc\x00def", "same-key", b"fingerprint")
|
||||
with pytest.raises(InvalidMessageText if member else ChatNotFound):
|
||||
await handler(command)
|
||||
chats.chat_for_member.assert_awaited_once_with(command.chat_id, command.sender_id)
|
||||
log.post.assert_not_awaited()
|
||||
uow.record.assert_not_called()
|
||||
@@ -0,0 +1,110 @@
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import weakref
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from PIL import Image, ImageFile
|
||||
from PIL._util import DeferredError
|
||||
|
||||
from messenger.attachments.infrastructure.images import RenderLimits, render_variants
|
||||
from png_trns_fixtures import boundary_png, png_with_trns
|
||||
|
||||
CASES = [(2, (1,)), (4, (5,)), (16, (12345,)), (16, (12345, 23456, 34567))]
|
||||
LIMITS = RenderLimits(10_485_760, 40_000_000)
|
||||
SAMPLE = Path(__file__).parents[2] / "tests/fixtures/sample.png"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("depth,key", CASES)
|
||||
def test_live_pixel_buffers_are_bounded(
|
||||
monkeypatch: pytest.MonkeyPatch, depth: int, key: tuple[int, ...]
|
||||
) -> None:
|
||||
size = (1025, 769)
|
||||
data, reference = png_with_trns(depth, key, size=size)
|
||||
expected = render_variants(reference, LIMITS)
|
||||
references: dict[int, weakref.ReferenceType[Image.Image]] = {}
|
||||
peak = 0
|
||||
original_new = Image.Image._new
|
||||
original_load = ImageFile.ImageFile.load_prepare
|
||||
|
||||
def record(image: Image.Image) -> None:
|
||||
nonlocal peak
|
||||
references[id(image)] = weakref.ref(image)
|
||||
buffers = {}
|
||||
for ref in references.values():
|
||||
current = ref()
|
||||
if current is None or current._im is None or isinstance(current._im, DeferredError):
|
||||
continue
|
||||
stride = 1 if current.mode in {"1", "L", "P"} else 4
|
||||
if current.mode.startswith("I;16"):
|
||||
stride = 2
|
||||
buffers[id(current.im)] = current.width * current.height * stride
|
||||
peak = max(peak, sum(buffers.values()))
|
||||
|
||||
def allocated(image: Image.Image, core: Image.core.ImagingCore) -> Image.Image:
|
||||
result = original_new(image, core)
|
||||
record(result)
|
||||
return result
|
||||
|
||||
def loaded(image: ImageFile.ImageFile) -> None:
|
||||
original_load(image)
|
||||
record(image)
|
||||
|
||||
monkeypatch.setattr(Image.Image, "_new", allocated)
|
||||
monkeypatch.setattr(ImageFile.ImageFile, "load_prepare", loaded)
|
||||
assert render_variants(data, LIMITS) == expected
|
||||
assert peak < size[0] * size[1] * 8 + 2**20, peak
|
||||
print(json.dumps({"depth": depth, "key": key, "pixel_buffer_peak_bytes": peak}))
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "linux", reason="RLIMIT_AS проверяется в Linux")
|
||||
def test_boundary_pngs_fit_worker_address_space(tmp_path: Path) -> None:
|
||||
script = """
|
||||
import json
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import sys
|
||||
|
||||
import messenger.worker
|
||||
from PIL import Image
|
||||
from messenger.attachments.infrastructure.images import RenderLimits, render_variants
|
||||
|
||||
resource.setrlimit(resource.RLIMIT_AS, (448 << 20, 448 << 20))
|
||||
limits = RenderLimits(10_485_760, 40_000_000)
|
||||
for version in (1, 2):
|
||||
signal.alarm(30)
|
||||
variants = render_variants(Path(sys.argv[1]).read_bytes(), limits, version)
|
||||
signal.alarm(0)
|
||||
assert [(v.name, v.width, v.height) for v in variants] == [
|
||||
("thumbnail", 256, 160), ("crop", 128, 128), ("optimized", 1280, 800)
|
||||
]
|
||||
for variant in variants:
|
||||
with Image.open(BytesIO(variant.data)) as image:
|
||||
assert image.getpixel((5, image.height // 2)) == (255, 255, 255)
|
||||
assert image.getpixel((image.width - 6, image.height // 2)) != (255, 255, 255)
|
||||
assert len(render_variants(Path(sys.argv[2]).read_bytes(), limits)) == 3
|
||||
print(json.dumps({
|
||||
"versions": [1, 2], "max_rss_kib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
}))
|
||||
"""
|
||||
for name, (depth, key) in zip(("gray2", "gray4", "gray16", "rgb16"), CASES, strict=True):
|
||||
fixture = tmp_path / f"{name}-40mp.png"
|
||||
fixture.write_bytes(boundary_png(depth, key))
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
script,
|
||||
str(fixture),
|
||||
str(SAMPLE),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=65,
|
||||
check=True,
|
||||
)
|
||||
assert json.loads(result.stdout)["versions"] == [1, 2]
|
||||
print(name, result.stdout.strip())
|
||||
@@ -0,0 +1,67 @@
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from messenger.attachments.domain.values import ImageKind
|
||||
from messenger.attachments.infrastructure.images import (
|
||||
PillowImageInspector,
|
||||
RenderLimits,
|
||||
render_variants,
|
||||
)
|
||||
from png_trns_fixtures import png_with_trns
|
||||
|
||||
LIMITS = RenderLimits(10_485_760, 40_000_000)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [1, 2])
|
||||
@pytest.mark.parametrize("depth,transparent", [(2, (1,)), (4, (5,)), (16, (12345, 23456, 34567))])
|
||||
async def test_review_samples_have_white_transparency(
|
||||
version: int, depth: int, transparent: tuple[int, ...]
|
||||
) -> None:
|
||||
data, reference = png_with_trns(depth, transparent)
|
||||
info = await PillowImageInspector(LIMITS.max_pixels, 1).inspect(data, ImageKind.PNG)
|
||||
assert (info.width, info.height) == (64, 64)
|
||||
variants = render_variants(data, LIMITS, version)
|
||||
for variant in variants:
|
||||
with Image.open(BytesIO(variant.data)) as image:
|
||||
assert image.getpixel((5, image.height // 2)) == (255, 255, 255), variant.name
|
||||
assert variants == render_variants(reference, LIMITS, version)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [1, 2])
|
||||
@pytest.mark.parametrize(
|
||||
"depth,key",
|
||||
[(depth, key) for depth in (1, 2, 4) for key in range(1 << depth)]
|
||||
+ [(8, 0), (8, 85), (8, 255), (16, 0), (16, 12345), (16, 65535)],
|
||||
)
|
||||
def test_grayscale_keys_preserve_opaque_neighbors(version: int, depth: int, key: int) -> None:
|
||||
data, reference = png_with_trns(depth, (key,))
|
||||
assert render_variants(data, LIMITS, version) == render_variants(reference, LIMITS, version)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [1, 2])
|
||||
@pytest.mark.parametrize("interlace", [0, 1])
|
||||
@pytest.mark.parametrize("filtering", range(5))
|
||||
@pytest.mark.parametrize("depth,transparent", [(2, (1,)), (4, (5,)), (16, (12345, 23456, 34567))])
|
||||
async def test_png_filters_and_adam7_preserve_exact_samples(
|
||||
version: int, interlace: int, filtering: int, depth: int, transparent: tuple[int, ...]
|
||||
) -> None:
|
||||
data, reference = png_with_trns(
|
||||
depth, transparent, size=(65, 63), interlace=interlace, filtering=filtering
|
||||
)
|
||||
info = await PillowImageInspector(LIMITS.max_pixels, 1).inspect(data, ImageKind.PNG)
|
||||
assert (info.width, info.height) == (65, 63)
|
||||
assert render_variants(data, LIMITS, version) == render_variants(reference, LIMITS, version)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [1, 2])
|
||||
@pytest.mark.parametrize("size", [(1, 1), (1, 9), (9, 1), (65, 63)])
|
||||
@pytest.mark.parametrize("transparent", [(0, 0, 0), (123, 234, 45), (65535, 65535, 65535)])
|
||||
def test_rgb16_small_passes_orientation_and_key_extremes(
|
||||
version: int, size: tuple[int, int], transparent: tuple[int, ...]
|
||||
) -> None:
|
||||
data, reference = png_with_trns(
|
||||
16, transparent, size=size, interlace=1, filtering=4, orientation=6
|
||||
)
|
||||
assert render_variants(data, LIMITS, version) == render_variants(reference, LIMITS, version)
|
||||
@@ -0,0 +1,83 @@
|
||||
from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from starlette.types import Scope
|
||||
|
||||
from fakes import NullLogger, RecordingPublisher
|
||||
from messenger.attachments.application.bus import AttachmentsCommandBus
|
||||
from messenger.attachments.application.commands import UploadAttachment
|
||||
from messenger.attachments.application.handlers import UploadAttachmentHandler, UploadSlots
|
||||
from messenger.attachments.application.ports import (
|
||||
AttachmentRepository,
|
||||
AttachmentsUnitOfWork,
|
||||
ChatMembership,
|
||||
ObjectStorage,
|
||||
)
|
||||
from messenger.attachments.entrypoints.errors import ERROR_CODES
|
||||
from messenger.attachments.entrypoints.http import uploads_router
|
||||
from messenger.attachments.infrastructure.images import PillowImageInspector
|
||||
from messenger.shared.application.ports import Clock, IdGenerator
|
||||
from messenger.shared.entrypoints.errors import install
|
||||
from messenger.shared.entrypoints.pipeline import RequestPipeline
|
||||
from pictures import BROKEN_PNG, DUPLICATE_IHDR_PNG
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data", [BROKEN_PNG, DUPLICATE_IHDR_PNG], ids=["bad-zlib", "duplicate-ihdr"]
|
||||
)
|
||||
async def test_broken_png_returns_415_before_any_storage_write(data: bytes) -> None:
|
||||
uow = Mock(spec=AttachmentsUnitOfWork)
|
||||
uow.attachments = AsyncMock(spec=AttachmentRepository)
|
||||
uow.transaction.return_value = AsyncMock()
|
||||
uow.collect_new_events.return_value = []
|
||||
storage = AsyncMock(spec=ObjectStorage)
|
||||
membership = AsyncMock(spec=ChatMembership)
|
||||
membership.is_member.return_value = True
|
||||
logger = NullLogger()
|
||||
publisher = RecordingPublisher()
|
||||
handler = UploadAttachmentHandler(
|
||||
uow,
|
||||
membership,
|
||||
PillowImageInspector(40_000_000, 1),
|
||||
storage,
|
||||
clock=Mock(spec=Clock, now=Mock(return_value=datetime(2026, 11, 6, tzinfo=UTC))),
|
||||
ids=Mock(spec=IdGenerator, new_id=Mock(side_effect=uuid4)),
|
||||
limit=10_485_760,
|
||||
slots=UploadSlots(1),
|
||||
logger=logger,
|
||||
)
|
||||
bus = AttachmentsCommandBus({UploadAttachment: handler}, uow, publisher)
|
||||
container = SimpleNamespace(get=AsyncMock(return_value=bus))
|
||||
owner, chat = uuid4(), uuid4()
|
||||
|
||||
async def guard(scope: Scope) -> None:
|
||||
scope["state"].update(actor=owner, dishka_container=container)
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(uploads_router)
|
||||
install(app, logger, ERROR_CODES, lambda _: "Bearer", "offline-test")
|
||||
transport = ASGITransport(
|
||||
app=RequestPipeline(app, logger, 10_485_760, 5, guard, "offline-test")
|
||||
)
|
||||
async with AsyncClient(transport=transport, base_url="http://testserver") as http:
|
||||
response = await http.post(
|
||||
f"/api/v1/chats/{chat}/attachments?filename=broken.png",
|
||||
headers={"Content-Type": "image/png", "X-Request-Id": "png-regression"},
|
||||
content=data,
|
||||
)
|
||||
assert response.status_code == 415, response.text
|
||||
assert response.json()["error"]["code"] == "unsupported_media_type"
|
||||
assert response.json()["error"]["request_id"] == "png-regression"
|
||||
assert response.headers["X-Request-Id"] == "png-regression"
|
||||
assert response.headers["X-Instance-Id"] == "offline-test"
|
||||
membership.is_member.assert_awaited_once_with(chat, owner)
|
||||
uow.transaction.assert_not_called()
|
||||
uow.commit.assert_not_called()
|
||||
assert uow.attachments.mock_calls == []
|
||||
assert storage.mock_calls == []
|
||||
assert publisher.events == []
|
||||
@@ -5,7 +5,12 @@ from PIL import Image
|
||||
|
||||
from messenger.attachments.application.ports import Rendition
|
||||
from messenger.attachments.domain.errors import ImageTooDemanding, UnreadableImage
|
||||
from messenger.attachments.infrastructure.images import RenderLimits, render_variants
|
||||
from messenger.attachments.domain.values import ImageKind
|
||||
from messenger.attachments.infrastructure.images import (
|
||||
PillowImageInspector,
|
||||
RenderLimits,
|
||||
render_variants,
|
||||
)
|
||||
from pictures import SAMPLE
|
||||
|
||||
LIMITS = RenderLimits(max_bytes=10_485_760, max_pixels=40_000_000)
|
||||
@@ -22,8 +27,8 @@ def encoded(image: Image.Image, kind: str = "PNG", **options: object) -> bytes:
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def rendered(data: bytes) -> dict[str, Image.Image]:
|
||||
renditions: list[Rendition] = render_variants(data, LIMITS)
|
||||
def rendered(data: bytes, version: int = 1) -> dict[str, Image.Image]:
|
||||
renditions: list[Rendition] = render_variants(data, LIMITS, version)
|
||||
assert [rendition.name for rendition in renditions] == ["thumbnail", "crop", "optimized"]
|
||||
images = {}
|
||||
for rendition in renditions:
|
||||
@@ -87,6 +92,37 @@ def test_palette_transparency_is_composed_on_white() -> None:
|
||||
assert image.convert("RGB").getpixel((5, 5)) == (255, 255, 255)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [1, 2])
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "transparent", "opaque", "expected"),
|
||||
[
|
||||
("RGB", (255, 0, 0), (32, 96, 160), (32, 96, 160)),
|
||||
("L", 0, 96, (96, 96, 96)),
|
||||
("1", 0, 255, (255, 255, 255)),
|
||||
("1", 255, 0, (0, 0, 0)),
|
||||
("I;16", 12345, 12346, (48, 48, 48)),
|
||||
],
|
||||
)
|
||||
async def test_png_trns_is_white_without_changing_opaque_pixels(
|
||||
version: int,
|
||||
mode: str,
|
||||
transparent: int | tuple[int, int, int],
|
||||
opaque: int | tuple[int, int, int],
|
||||
expected: tuple[int, int, int],
|
||||
) -> None:
|
||||
source = Image.new(mode, (64, 64), transparent)
|
||||
source.paste(Image.new(mode, (32, 64), opaque), (32, 0))
|
||||
assert source.getpixel((58, 32)) == opaque
|
||||
data = encoded(source, transparency=transparent)
|
||||
info = await PillowImageInspector(LIMITS.max_pixels, 1).inspect(data, ImageKind.PNG)
|
||||
assert (info.width, info.height) == (64, 64)
|
||||
for image in rendered(data, version).values():
|
||||
assert image.getpixel((5, image.height // 2)) == (255, 255, 255)
|
||||
pixel = image.getpixel((image.width - 6, image.height // 2))
|
||||
assert isinstance(pixel, tuple)
|
||||
assert all(abs(a - b) <= 3 for a, b in zip(pixel, expected, strict=True))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("orientation", "thumbnail", "optimized"),
|
||||
[(6, (128, 256), (200, 400)), (3, (256, 128), (400, 200))],
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
import asyncio
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import cast
|
||||
from unittest.mock import AsyncMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from redis.asyncio import Redis
|
||||
from redis.exceptions import ConnectionError as CacheConnectionError
|
||||
|
||||
from fakes import NullLogger
|
||||
from messenger.identity.domain.models import Session
|
||||
from messenger.identity.infrastructure.valkey import (
|
||||
FILL,
|
||||
REVOKE,
|
||||
REVOKED,
|
||||
CachedSessionDirectory,
|
||||
key_of,
|
||||
milliseconds,
|
||||
)
|
||||
|
||||
NOW = datetime(2030, 1, 1, tzinfo=UTC)
|
||||
|
||||
|
||||
class CacheModel:
|
||||
def __init__(self) -> None:
|
||||
self.values: dict[str, str] = {}
|
||||
self.expires: dict[str, int] = {}
|
||||
|
||||
async def get(self, key: str) -> str | None:
|
||||
return self.values.get(key)
|
||||
|
||||
async def set(self, key: str, value: str, *, pxat: int, nx: bool = False) -> bool:
|
||||
if nx and key in self.values:
|
||||
return False
|
||||
self.values[key], self.expires[key] = value, pxat
|
||||
return True
|
||||
|
||||
async def delete(self, key: str) -> None:
|
||||
self.values.pop(key, None)
|
||||
self.expires.pop(key, None)
|
||||
|
||||
async def eval(self, script: str, count: int, key: str, marker: str, *args: str | int) -> bool:
|
||||
assert count == 2
|
||||
if script == REVOKE:
|
||||
value, until = args
|
||||
assert isinstance(value, str) and isinstance(until, int)
|
||||
await self.set(key, value, pxat=until)
|
||||
await self.delete(marker)
|
||||
return True
|
||||
assert script == FILL
|
||||
nonce, value, until = args
|
||||
assert isinstance(value, str) and isinstance(until, int)
|
||||
if await self.get(marker) != nonce:
|
||||
return False
|
||||
await self.delete(marker)
|
||||
return bool(value) and await self.set(key, value, pxat=until, nx=True)
|
||||
|
||||
|
||||
def directory(origin: AsyncMock, cache: CacheModel, ttl: int = 300) -> CachedSessionDirectory:
|
||||
return CachedSessionDirectory(
|
||||
origin, cast(Redis, cache), NullLogger(), limit=timedelta(seconds=ttl), cooldown=1
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def session() -> Session:
|
||||
return Session.issue(user_id=uuid4(), issued_at=NOW, ttl=timedelta(seconds=30))[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ttl", [10, 300])
|
||||
async def test_cache_hit_keeps_absolute_ttl_without_reading_origin(
|
||||
session: Session, ttl: int
|
||||
) -> None:
|
||||
origin, cache = AsyncMock(), CacheModel()
|
||||
origin.find.return_value = session
|
||||
cached = directory(origin, cache, ttl)
|
||||
assert await cached.find(session.token_hash, NOW) == session
|
||||
assert await directory(origin, cache, ttl).find(session.token_hash, NOW) == session
|
||||
origin.find.assert_awaited_once()
|
||||
assert cache.expires == {
|
||||
key_of(session.token_hash): milliseconds(
|
||||
min(session.expires_at, NOW + timedelta(seconds=ttl))
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("loss", ["none", "flushall", "evict", "replacement"])
|
||||
async def test_delayed_fill_cannot_revive_logged_out_session(session: Session, loss: str) -> None:
|
||||
origin, cache = AsyncMock(), CacheModel()
|
||||
origin.find.return_value = session
|
||||
entered, released = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def stale_find(token_hash: bytes, now: datetime) -> Session:
|
||||
entered.set()
|
||||
await released.wait()
|
||||
return session
|
||||
|
||||
slow = AsyncMock()
|
||||
slow.find.side_effect = stale_find
|
||||
async with asyncio.TaskGroup() as group:
|
||||
reader = group.create_task(directory(slow, cache).find(session.token_hash, NOW))
|
||||
await asyncio.wait_for(entered.wait(), 1)
|
||||
try:
|
||||
await directory(origin, cache).revoke(session)
|
||||
origin.find.return_value = None
|
||||
key = key_of(session.token_hash)
|
||||
if loss == "flushall":
|
||||
cache.values.clear()
|
||||
elif loss in {"evict", "replacement"}:
|
||||
await cache.delete(key)
|
||||
if loss == "replacement":
|
||||
await cache.set(f"{key}:fill", uuid4().hex, pxat=milliseconds(session.expires_at))
|
||||
finally:
|
||||
released.set()
|
||||
assert reader.result() == session
|
||||
assert await directory(origin, cache).find(session.token_hash, NOW) is None
|
||||
assert await cache.get(key) in {None, REVOKED}
|
||||
|
||||
|
||||
async def test_failed_reservation_never_fills_from_stale_database_result(session: Session) -> None:
|
||||
cache = CacheModel()
|
||||
origin = AsyncMock()
|
||||
key = key_of(session.token_hash)
|
||||
await cache.set(f"{key}:fill", "another-reader", pxat=milliseconds(session.expires_at))
|
||||
|
||||
async def find(token_hash: bytes, now: datetime) -> Session:
|
||||
cache.values.clear()
|
||||
return session
|
||||
|
||||
origin.find.side_effect = find
|
||||
assert await directory(origin, cache).find(session.token_hash, NOW) == session
|
||||
assert await cache.get(key) is None
|
||||
|
||||
|
||||
async def test_missing_session_releases_only_its_own_reservation(session: Session) -> None:
|
||||
origin, cache = AsyncMock(), CacheModel()
|
||||
origin.find.return_value = None
|
||||
assert await directory(origin, cache).find(session.token_hash, NOW) is None
|
||||
assert cache.values == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failed_operation", ["get", "set"])
|
||||
async def test_cache_recovery_during_database_read_does_not_allow_unreserved_fill(
|
||||
session: Session, failed_operation: str
|
||||
) -> None:
|
||||
origin, cache = AsyncMock(), AsyncMock()
|
||||
cache.get.return_value = None
|
||||
getattr(cache, failed_operation).side_effect = CacheConnectionError("offline")
|
||||
cached = CachedSessionDirectory(
|
||||
origin, cache, NullLogger(), limit=timedelta(seconds=300), cooldown=1
|
||||
)
|
||||
|
||||
async def find(token_hash: bytes, now: datetime) -> Session:
|
||||
cached._resume_at = 0.0
|
||||
return session
|
||||
|
||||
origin.find.side_effect = find
|
||||
assert await cached.find(session.token_hash, NOW) == session
|
||||
assert cache.set.await_count == (failed_operation == "set")
|
||||
cache.eval.assert_not_awaited()
|
||||
@@ -0,0 +1,63 @@
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, Mock, call
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from messenger.attachments.application.commands import ProcessJob
|
||||
from messenger.attachments.domain.jobs import Job, Outcome
|
||||
from messenger.attachments.entrypoints.worker import REQUEUE_DELAY_SECONDS, handle_job
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
|
||||
|
||||
def test_infrastructure_requeues_preserve_the_message_and_dead_letter_rejections() -> None:
|
||||
path = Path(__file__).resolve().parents[2] / "infra/rabbitmq/definitions.json"
|
||||
definitions = json.loads(path.read_text())
|
||||
policy = next(p for p in definitions["policies"] if p["name"] == "attachments-process")
|
||||
queue = next(q for q in definitions["queues"] if q["name"] == "attachments.process")
|
||||
assert policy["definition"]["delivery-limit"] == -1
|
||||
assert "x-delivery-limit" not in queue["arguments"]
|
||||
assert policy["definition"]["dead-letter-exchange"] == "attachments.dlx"
|
||||
assert policy["definition"]["dead-letter-strategy"] == "at-least-once"
|
||||
|
||||
|
||||
async def test_database_outage_never_acks_and_recovery_keeps_job_and_attempt(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
job = Job(uuid4(), uuid4(), 1, 1, datetime.now(UTC))
|
||||
message = AsyncMock(body=json.dumps(job.payload()).encode())
|
||||
bus, logger, sleep = AsyncMock(), Mock(), AsyncMock()
|
||||
monkeypatch.setattr("messenger.attachments.entrypoints.worker.asyncio.sleep", sleep)
|
||||
bus.execute.side_effect = [Unavailable("database unavailable")] * 12 + [Outcome.DONE]
|
||||
for _ in range(12):
|
||||
await handle_job(message, bus, logger, Mock())
|
||||
message.ack.assert_not_awaited()
|
||||
message.reject.assert_not_awaited()
|
||||
assert message.nack.await_args_list == [call(requeue=True)] * 12
|
||||
assert sleep.await_args_list == [call(REQUEUE_DELAY_SECONDS)] * 12
|
||||
await handle_job(message, bus, logger, Mock())
|
||||
message.ack.assert_awaited_once_with()
|
||||
assert bus.execute.await_args_list == [call(ProcessJob(job))] * 13
|
||||
|
||||
|
||||
async def test_invalid_message_is_dead_lettered_without_calling_the_bus() -> None:
|
||||
message, bus = AsyncMock(body=b"{}"), AsyncMock()
|
||||
await handle_job(message, bus, Mock(), Mock())
|
||||
message.reject.assert_awaited_once_with(requeue=False)
|
||||
message.ack.assert_not_awaited()
|
||||
message.nack.assert_not_awaited()
|
||||
bus.execute.assert_not_awaited()
|
||||
|
||||
|
||||
async def test_cancelled_handler_leaves_delivery_unacknowledged() -> None:
|
||||
job = Job(uuid4(), uuid4(), 1, 1, datetime.now(UTC))
|
||||
message, bus = AsyncMock(body=json.dumps(job.payload()).encode()), AsyncMock()
|
||||
bus.execute.side_effect = asyncio.CancelledError
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await handle_job(message, bus, Mock(), Mock())
|
||||
message.ack.assert_not_awaited()
|
||||
message.nack.assert_not_awaited()
|
||||
message.reject.assert_not_awaited()
|
||||
@@ -1,3 +1,4 @@
|
||||
import argparse
|
||||
import json
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
@@ -27,7 +28,10 @@ def values(rows, mode, key):
|
||||
|
||||
|
||||
def main():
|
||||
rows = json.loads((DIRECTORY / "comparison.json").read_text())["runs"]
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("directory", type=Path, nargs="?", default=DIRECTORY)
|
||||
directory = parser.parse_args().directory
|
||||
rows = json.loads((directory / "comparison.json").read_text())["runs"]
|
||||
figure, axes = plt.subplots(2, 2, figsize=(11, 7), constrained_layout=True)
|
||||
panels = (
|
||||
("latency", "Normal request p95", "milliseconds"),
|
||||
@@ -55,8 +59,8 @@ def main():
|
||||
"Dots: actual runs; bars: median; whiskers: observed range",
|
||||
fontsize=13,
|
||||
)
|
||||
figure.savefig(DIRECTORY / "comparison.png", dpi=180)
|
||||
figure.savefig(DIRECTORY / "comparison.svg")
|
||||
figure.savefig(directory / "comparison.png", dpi=180)
|
||||
figure.savefig(directory / "comparison.svg")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user