diff --git a/REPORT.md b/REPORT.md index 4f50f146..e119612f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,5 +1,22 @@ # Отчёт — лабораторная 5 +## Исправление по ревью: целостность PNG + +В `src/messenger/image_check.py` добавлен проход по структуре PNG: полные +заголовки, данные и CRC каждого блока, обязательный завершающий IEND нулевой +длины с корректным CRC. Проверка выполняется до S3 staging и создания задания. +Pillow verify/load сохранены для проверки формата, размеров и декодирования: +сами по себе они допускают усечение CRC блока IEND. +CRC считается потоково блоками до 64 KiB без выделения памяти по недоверенной +длине блока. Неполный или повреждённый PNG возвращает HTTP 415. + +Регрессия покрывает удаление последних 1–12 байт PNG, повреждение CRC IEND, +ошибочный CRC дополнительного блока и непустой IEND. HTTP-тест также проверяет, +что после отказов корректный PNG принимается и достигает ready. +Вывод повторного прогона: `evidence/png-integrity-offline.txt`, +`evidence/png-integrity-unit.txt`, `evidence/png-integrity-smoke.txt`, +`evidence/png-integrity-tests.txt`. + - Автор / группа: Черников Дмитрий, БВТ2304. - Целевая оценка: **4** — обязательные требования на 3 и все пункты на 4. Треки на 5 не заявляются. - Ветка: `solution/bvt2304-chernikov`, создана от официального skeleton `mtusi2026/mtusi2026lab5:main`, базовый commit `276dbda`. Commit решения/PR указывается после публикации в личный fork. diff --git a/evidence/png-integrity-offline.txt b/evidence/png-integrity-offline.txt new file mode 100644 index 00000000..af45ae2a --- /dev/null +++ b/evidence/png-integrity-offline.txt @@ -0,0 +1 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/png-integrity-smoke.txt b/evidence/png-integrity-smoke.txt new file mode 100644 index 00000000..be754951 --- /dev/null +++ b/evidence/png-integrity-smoke.txt @@ -0,0 +1 @@ +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/png-integrity-tests.txt b/evidence/png-integrity-tests.txt new file mode 100644 index 00000000..ce964592 --- /dev/null +++ b/evidence/png-integrity-tests.txt @@ -0,0 +1,104 @@ +test_authentication_is_checked_before_body_and_resource (test_access.AuthenticationOrderTest.test_authentication_is_checked_before_body_and_resource) ... ok +test_registration_does_not_need_a_token (test_access.AuthenticationOrderTest.test_registration_does_not_need_a_token) ... ok +test_chat_list_contains_only_own_chats (test_access.ForeignChatTest.test_chat_list_contains_only_own_chats) ... ok +test_every_member_can_read_and_write (test_access.ForeignChatTest.test_every_member_can_read_and_write) ... ok +test_foreign_chat_looks_like_missing_chat (test_access.ForeignChatTest.test_foreign_chat_looks_like_missing_chat) ... ok +test_invalid_chat_id_is_400 (test_access.ForeignChatTest.test_invalid_chat_id_is_400) ... ok +test_outsider_cannot_get_chat (test_access.ForeignChatTest.test_outsider_cannot_get_chat) ... ok +test_outsider_cannot_read_history (test_access.ForeignChatTest.test_outsider_cannot_read_history) ... ok +test_outsider_cannot_write (test_access.ForeignChatTest.test_outsider_cannot_write) ... ok +test_unknown_route_is_404 (test_access.ForeignChatTest.test_unknown_route_is_404) ... ok +test_x_user_id_does_not_open_a_foreign_chat (test_access.ForeignChatTest.test_x_user_id_does_not_open_a_foreign_chat) ... ok +test_attachment_only_and_idempotency (test_attachments.AttachmentsTest.test_attachment_only_and_idempotency) ... ok +test_byte_limit_before_body (test_attachments.AttachmentsTest.test_byte_limit_before_body) ... ok +test_corrupt_and_wrong_mime (test_attachments.AttachmentsTest.test_corrupt_and_wrong_mime) ... ok +test_exif_orientation_applies_to_variants_and_original_stays_identical (test_attachments.AttachmentsTest.test_exif_orientation_applies_to_variants_and_original_stays_identical) ... ok +test_filename (test_attachments.AttachmentsTest.test_filename) ... ok +test_foreign_chat_upload (test_attachments.AttachmentsTest.test_foreign_chat_upload) ... ok +test_jpeg_original (test_attachments.AttachmentsTest.test_jpeg_original) ... ok +test_large_optimized_image_is_bounded (test_attachments.AttachmentsTest.test_large_optimized_image_is_bounded) ... ok +test_original_and_metadata_and_acl (test_attachments.AttachmentsTest.test_original_and_metadata_and_acl) ... ok +test_pixel_limit_before_decode (test_attachments.AttachmentsTest.test_pixel_limit_before_decode) ... ok +test_png_requires_complete_iend_and_valid_crc (test_attachments.AttachmentsTest.test_png_requires_complete_iend_and_valid_crc) ... ok +test_small_transparent_image_does_not_upscale_thumbnail_or_optimized (test_attachments.AttachmentsTest.test_small_transparent_image_does_not_upscale_thumbnail_or_optimized) ... ok +test_unknown_foreign_and_duplicate_ids (test_attachments.AttachmentsTest.test_unknown_foreign_and_duplicate_ids) ... ok +test_errors_also_carry_instance_id (test_balancing.BalancingTest.test_errors_also_carry_instance_id) ... ok +test_message_written_on_one_replica_is_read_on_another (test_balancing.BalancingTest.test_message_written_on_one_replica_is_read_on_another) ... ok +test_one_session_is_served_by_several_replicas (test_balancing.BalancingTest.test_one_session_is_served_by_several_replicas) ... ok +test_100_messages_from_10_clients (test_concurrency.ConcurrencyTest.test_100_messages_from_10_clients) ... ok +test_chats_created_in_parallel_are_all_listed (test_concurrency.ConcurrencyTest.test_chats_created_in_parallel_are_all_listed) ... ok +test_same_username_registered_in_parallel_wins_once (test_concurrency.ConcurrencyTest.test_same_username_registered_in_parallel_wins_once) ... ok +test_failed_validation_is_not_remembered (test_idempotency.IdempotencyTest.test_failed_validation_is_not_remembered) ... ok +test_invalid_keys_are_400 (test_idempotency.IdempotencyTest.test_invalid_keys_are_400) ... ok +test_json_formatting_does_not_matter (test_idempotency.IdempotencyTest.test_json_formatting_does_not_matter) ... ok +test_key_format_limits_are_accepted (test_idempotency.IdempotencyTest.test_key_format_limits_are_accepted) ... ok +test_key_scope_is_user_and_chat (test_idempotency.IdempotencyTest.test_key_scope_is_user_and_chat) ... ok +test_missing_key_is_400 (test_idempotency.IdempotencyTest.test_missing_key_is_400) ... ok +test_parallel_requests_with_one_key_create_one_message (test_idempotency.IdempotencyTest.test_parallel_requests_with_one_key_create_one_message) ... ok +test_replay_checks_the_session_again (test_idempotency.IdempotencyTest.test_replay_checks_the_session_again) ... ok +test_replay_returns_the_same_message (test_idempotency.IdempotencyTest.test_replay_returns_the_same_message) ... ok +test_same_key_with_another_body_is_409 (test_idempotency.IdempotencyTest.test_same_key_with_another_body_is_409) ... ok +test_chats_are_paged_in_creation_order (test_pagination.ChatListPaginationTest.test_chats_are_paged_in_creation_order) ... ok +test_cursor_of_another_user_is_400 (test_pagination.ChatListPaginationTest.test_cursor_of_another_user_is_400) ... ok +test_default_limit_is_50 (test_pagination.DefaultLimitTest.test_default_limit_is_50) ... ok +test_empty_history (test_pagination.GrowingHistoryTest.test_empty_history) ... ok +test_messages_sent_during_walk_are_not_lost (test_pagination.GrowingHistoryTest.test_messages_sent_during_walk_are_not_lost) ... ok +test_cursor_from_another_chat_is_400 (test_pagination.MessagePaginationTest.test_cursor_from_another_chat_is_400) ... ok +test_default_and_maximum_limit (test_pagination.MessagePaginationTest.test_default_and_maximum_limit) ... ok +test_invalid_cursor_is_400 (test_pagination.MessagePaginationTest.test_invalid_cursor_is_400) ... ok +test_invalid_limit_is_400 (test_pagination.MessagePaginationTest.test_invalid_limit_is_400) ... ok +test_last_full_page_has_no_cursor (test_pagination.MessagePaginationTest.test_last_full_page_has_no_cursor) ... ok +test_limit_can_change_between_pages (test_pagination.MessagePaginationTest.test_limit_can_change_between_pages) ... ok +test_one_message_per_page (test_pagination.MessagePaginationTest.test_one_message_per_page) ... ok +test_walk_over_several_pages (test_pagination.MessagePaginationTest.test_walk_over_several_pages) ... ok +test_basic_is_rejected_on_business_endpoints (test_sessions.BearerTest.test_basic_is_rejected_on_business_endpoints) ... ok +test_protected_endpoints_need_a_token (test_sessions.BearerTest.test_protected_endpoints_need_a_token) ... ok +test_scheme_name_is_case_insensitive (test_sessions.BearerTest.test_scheme_name_is_case_insensitive) ... ok +test_token_selects_the_user (test_sessions.BearerTest.test_token_selects_the_user) ... ok +test_unknown_or_malformed_token (test_sessions.BearerTest.test_unknown_or_malformed_token) ... ok +test_x_user_id_is_ignored (test_sessions.BearerTest.test_x_user_id_is_ignored) ... ok +test_credentials_in_json_body_are_not_accepted (test_sessions.LoginTest.test_credentials_in_json_body_are_not_accepted) ... ok +test_each_login_gives_a_new_token (test_sessions.LoginTest.test_each_login_gives_a_new_token) ... ok +test_malformed_basic_header (test_sessions.LoginTest.test_malformed_basic_header) ... ok +test_password_with_colons_and_unicode (test_sessions.LoginTest.test_password_with_colons_and_unicode) ... ok +test_registration_never_returns_password (test_sessions.LoginTest.test_registration_never_returns_password) ... ok +test_successful_login (test_sessions.LoginTest.test_successful_login) ... ok +test_unknown_user_gets_the_same_answer (test_sessions.LoginTest.test_unknown_user_gets_the_same_answer) ... ok +test_wrong_password (test_sessions.LoginTest.test_wrong_password) ... ok +test_login_again_after_logout (test_sessions.LogoutTest.test_login_again_after_logout) ... ok +test_logout_revokes_only_the_current_token (test_sessions.LogoutTest.test_logout_revokes_only_the_current_token) ... ok +test_bad_member_ids (test_validation.ChatValidationTest.test_bad_member_ids) ... ok +test_bad_titles (test_validation.ChatValidationTest.test_bad_titles) ... ok +test_caller_must_be_a_member (test_validation.ChatValidationTest.test_caller_must_be_a_member) ... ok +test_duplicate_members (test_validation.ChatValidationTest.test_duplicate_members) ... ok +test_unknown_member_rolls_back_the_whole_chat (test_validation.ChatValidationTest.test_unknown_member_rolls_back_the_whole_chat) ... ok +test_valid_chat_keeps_member_order (test_validation.ChatValidationTest.test_valid_chat_keeps_member_order) ... ok +test_extra_fields (test_validation.InvalidJsonTest.test_extra_fields) ... ok +test_invalid_utf8 (test_validation.InvalidJsonTest.test_invalid_utf8) ... ok +test_json_that_is_not_an_object (test_validation.InvalidJsonTest.test_json_that_is_not_an_object) ... ok +test_lone_surrogate_is_rejected_not_crashed (test_validation.InvalidJsonTest.test_lone_surrogate_is_rejected_not_crashed) ... ok +test_malformed_json_on_message (test_validation.InvalidJsonTest.test_malformed_json_on_message) ... ok +test_malformed_json_on_register (test_validation.InvalidJsonTest.test_malformed_json_on_register) ... ok +test_nul_character_is_rejected_not_crashed (test_validation.InvalidJsonTest.test_nul_character_is_rejected_not_crashed) ... ok +test_wrong_content_type (test_validation.InvalidJsonTest.test_wrong_content_type) ... ok +test_wrong_types (test_validation.InvalidJsonTest.test_wrong_types) ... ok +test_empty_message_is_400 (test_validation.MessageValidationTest.test_empty_message_is_400) ... ok +test_missing_or_wrong_text (test_validation.MessageValidationTest.test_missing_or_wrong_text) ... ok +test_nul_in_text_is_400 (test_validation.MessageValidationTest.test_nul_in_text_is_400) ... ok +test_oversized_body_is_413 (test_validation.MessageValidationTest.test_oversized_body_is_413) ... ok +test_sender_id_in_body_is_rejected (test_validation.MessageValidationTest.test_sender_id_in_body_is_rejected) ... ok +test_text_length_limit (test_validation.MessageValidationTest.test_text_length_limit) ... ok +test_unicode_text_is_kept (test_validation.MessageValidationTest.test_unicode_text_is_kept) ... ok +test_bad_display_names (test_validation.UserValidationTest.test_bad_display_names) ... ok +test_bad_passwords (test_validation.UserValidationTest.test_bad_passwords) ... ok +test_bad_usernames (test_validation.UserValidationTest.test_bad_usernames) ... ok +test_duplicate_username_is_409 (test_validation.UserValidationTest.test_duplicate_username_is_409) ... ok +test_missing_fields (test_validation.UserValidationTest.test_missing_fields) ... ok +test_password_length_limits_are_accepted (test_validation.UserValidationTest.test_password_length_limits_are_accepted) ... ok +test_unicode_display_name_is_kept (test_validation.UserValidationTest.test_unicode_display_name_is_kept) ... ok +test_username_length_limits_are_accepted (test_validation.UserValidationTest.test_username_length_limits_are_accepted) ... ok + +---------------------------------------------------------------------- +Ran 99 tests in 50.949s + +OK diff --git a/evidence/png-integrity-unit.txt b/evidence/png-integrity-unit.txt new file mode 100644 index 00000000..9cccf8c2 --- /dev/null +++ b/evidence/png-integrity-unit.txt @@ -0,0 +1,24 @@ + Container mtusi-lab5-worker-run-4fa9ae19f8b5 Creating + Container mtusi-lab5-worker-run-4fa9ae19f8b5 Created +test_complete_chunk_integrity (test_image_check.ImageCheckTest.test_complete_chunk_integrity) ... ok +test_all_variants_and_original_checksum (test_image_process.ImagePipelineTests.test_all_variants_and_original_checksum) ... ok +test_center_crop_uses_center_pixels (test_image_process.ImagePipelineTests.test_center_crop_uses_center_pixels) ... ok +test_decoder_subprocess_under_production_limits (test_image_process.ImagePipelineTests.test_decoder_subprocess_under_production_limits) ... ok +test_exif_orientation_applied_and_metadata_removed (test_image_process.ImagePipelineTests.test_exif_orientation_applied_and_metadata_removed) ... ok +test_floor_dimensions_and_extreme_aspect_ratio (test_image_process.ImagePipelineTests.test_floor_dimensions_and_extreme_aspect_ratio) ... ok +test_invalid_pixels_and_mime_rejected (test_image_process.ImagePipelineTests.test_invalid_pixels_and_mime_rejected) ... ok +test_small_alpha_is_flattened_white_without_upscale (test_image_process.ImagePipelineTests.test_small_alpha_is_flattened_white_without_upscale) ... ok +test_ack_follows_committed_result (test_image_process.WorkerDurabilityTests.test_ack_follows_committed_result) ... ok +test_active_redelivery_does_not_ack_and_ready_is_noop (test_image_process.WorkerDurabilityTests.test_active_redelivery_does_not_ack_and_ready_is_noop) ... ok +test_backoff_bounds_and_cap (test_image_process.WorkerDurabilityTests.test_backoff_bounds_and_cap) ... ok +test_lost_fence_is_left_pending (test_image_process.WorkerDurabilityTests.test_lost_fence_is_left_pending) ... ok +test_permanent_failure_has_no_automatic_processing_retry (test_image_process.WorkerDurabilityTests.test_permanent_failure_has_no_automatic_processing_retry) ... event=job_error job_id=632e4298-0388-4a25-8ff0-0f4d332527f5 attempt=1 code=invalid_image permanent=True retry_delay=0.620 error_type=ProcessingError +ok +test_retry_ack_requires_durable_failure_record (test_image_process.WorkerDurabilityTests.test_retry_ack_requires_durable_failure_record) ... event=job_error job_id=706ee81d-986f-4435-b3d1-703e66acedf6 attempt=1 code=dependency_unavailable permanent=False retry_delay=0.510 error_type=ProcessingError +event=job_error job_id=706ee81d-986f-4435-b3d1-703e66acedf6 attempt=1 code=dependency_unavailable permanent=False retry_delay=0.519 error_type=ProcessingError +ok + +---------------------------------------------------------------------- +Ran 14 tests in 1.124s + +OK diff --git a/src/messenger/image_check.py b/src/messenger/image_check.py index 8df13fb4..794d3fa3 100644 --- a/src/messenger/image_check.py +++ b/src/messenger/image_check.py @@ -1,10 +1,39 @@ """Декодирование в отдельном процессе: RAM освобождается при выходе, время ограничено родителем.""" import sys import warnings +import zlib from PIL import Image, UnidentifiedImageError +def verify_png_integrity(path): + """Check every complete chunk, including IEND's CRC which Pillow skips.""" + with open(path, 'rb') as source: + if source.read(8) != b'\x89PNG\r\n\x1a\n': + raise ValueError('Invalid PNG signature') + while True: + header = source.read(8) + if len(header) != 8: + raise ValueError('Missing or truncated PNG chunk') + length = int.from_bytes(header[:4], 'big') + kind = header[4:] + checksum = zlib.crc32(kind) + remaining = length + while remaining: + block = source.read(min(remaining, 65536)) + if not block: + raise ValueError('Truncated PNG chunk data') + checksum = zlib.crc32(block, checksum) + remaining -= len(block) + crc = source.read(4) + if len(crc) != 4 or int.from_bytes(crc, 'big') != checksum: + raise ValueError('Missing or invalid PNG CRC') + if kind == b'IEND': + if length != 0: + raise ValueError('Invalid PNG IEND') + return + + def main(): path, expected, pixels = sys.argv[1:] Image.MAX_IMAGE_PIXELS = int(pixels) @@ -15,6 +44,8 @@ def main(): return 13 if Image.MIME[image.format] != expected: return 15 + if image.format == 'PNG': + verify_png_integrity(path) image.verify() with Image.open(path, formats=['PNG', 'JPEG']) as image: image.load() # verify одного JPEG недостаточно: проверяем и распаковку. diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py index 6ef5597c..d8f0568c 100644 --- a/student-tests/test_attachments.py +++ b/student-tests/test_attachments.py @@ -135,6 +135,16 @@ class AttachmentsTest(unittest.TestCase): original = self.c.call('GET', '/api/v1/attachments/' + a['id'] + '/content', actor=self.a) self.assertEqual(original['bytes'], source) + def test_png_requires_complete_iend_and_valid_crc(self): + png = small_png() + for missing in range(1, 13): + with self.subTest(missing_final_bytes=missing): + self.upload(raw=png[:-missing], status=415) + with self.subTest(corrupt_iend_crc=True): + self.upload(raw=png[:-1] + bytes([png[-1] ^ 1]), status=415) + # A rejected upload must not prevent subsequent valid processing. + self.ready(self.upload(raw=png)['json']) + def test_corrupt_and_wrong_mime(self): for raw, mime in ((b'not an image', 'image/png'), (self.png[:40], 'image/png'), (self.png, 'image/jpeg'), (self.png, 'text/plain'), (b'', 'image/png')): diff --git a/unit-tests/test_image_check.py b/unit-tests/test_image_check.py new file mode 100644 index 00000000..0c2cfe29 --- /dev/null +++ b/unit-tests/test_image_check.py @@ -0,0 +1,41 @@ +"""Regression: Pillow accepts truncated IEND, upload validator must reject it.""" +from pathlib import Path +import struct +import sys +import tempfile +import unittest +from unittest.mock import patch +import zlib + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src')) +from messenger.image_check import main + + +def chunk(kind, payload): + return (struct.pack('>I', len(payload)) + kind + payload + + struct.pack('>I', zlib.crc32(kind + payload))) + + +class ImageCheckTest(unittest.TestCase): + def test_complete_chunk_integrity(self): + png = (b'\x89PNG\r\n\x1a\n' + + chunk(b'IHDR', struct.pack('>IIBBBBB', 1, 1, 8, 2, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(b'\x00\xff\x00\x00')) + + chunk(b'IEND', b'')) + cases = [('valid', png, 0)] + cases += [(f'truncated-{n}', png[:-n], 15) for n in range(1, 13)] + cases += [('bad-iend-crc', png[:-1] + bytes([png[-1] ^ 1]), 15), + ('bad-ancillary-crc', png[:-12] + b'\x00\x00\x00\x00tEXt\x00\x00\x00\x00' + + png[-12:], 15), + ('nonempty-iend', png[:-12] + chunk(b'IEND', b'x'), 15)] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'image.png' + for label, data, expected in cases: + with self.subTest(case=label): + path.write_bytes(data) + with patch.object(sys, 'argv', ['image_check', str(path), 'image/png', '40000000']): + self.assertEqual(main(), expected) + + +if __name__ == '__main__': + unittest.main()