import hashlib import json from pathlib import Path import subprocess import time import uuid from client import Client, Failure, arguments, check, run from image_probe import jpeg_size ROOT = Path(__file__).resolve().parents[1] BASE = [ "docker", "compose", "-f", "compose.yml", "-f", "compose.override.yml", ] FAULT = BASE + [ "-f", "compose.faults.yml", ] FAULT_POINTS = { "hold_publish", "before_publish", "after_objects", "before_ack", "permanent_error", "transient_error", "image_timeout", } def command(args, required=True): try: result = subprocess.run( args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=240, ) except subprocess.TimeoutExpired: raise Failure("Docker command timed out") from None if required and result.returncode: raise Failure( "Docker command failed; " "inspect the corresponding service locally" ) return result def sql(query): result = command( BASE + [ "exec", "-T", "postgres", "sh", "-ec", 'exec psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" ' '-At -v ON_ERROR_STOP=1 -c "$1"', "sh", query, ] ) return result.stdout.strip() def wait(action, label, seconds=180): deadline = time.monotonic() + seconds while time.monotonic() < deadline: try: value = action() if value: return value except Failure: pass time.sleep(1) raise Failure("Timeout: " + label) def state(attachment_id): attachment_id = str(uuid.UUID(attachment_id)) raw = sql( f""" SELECT json_build_object( 'job_id', j.id, 'status', j.status, 'attempts', j.attempts, 'owner', j.lease_owner, 'staging', a.staging_key, 'original_key', a.object_key, 'variants', COALESCE( ( SELECT json_agg(v ORDER BY v.name) FROM attachment_variants v WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version ), '[]'::json ), 'events', COALESCE( ( SELECT json_agg(e ORDER BY e.id) FROM attachment_attempt_events e WHERE e.job_id = j.id ), '[]'::json ) ) FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE a.id = '{attachment_id}'::uuid AND j.pipeline_version = a.pipeline_version """ ) check(bool(raw), "attachment job not found") try: return json.loads(raw) except json.JSONDecodeError: raise Failure("Invalid job state returned by PostgreSQL") from None def arm(point, count=1): check(point in FAULT_POINTS, "unknown fault") check(isinstance(count, int) and count >= 0, "invalid fault count") sql( f""" INSERT INTO attachment_test_faults(point, remaining) VALUES ('{point}', {count}) ON CONFLICT (point) DO UPDATE SET remaining = EXCLUDED.remaining, hits = 0, job_id = NULL, hit_at = NULL """ ) def hit(point, job_id): check(point in FAULT_POINTS, "unknown fault") job_id = str(uuid.UUID(job_id)) return sql( f""" SELECT count(*) FROM attachment_test_faults WHERE point = '{point}' AND job_id = '{job_id}'::uuid AND hits > 0 """ ) == "1" def restarts(service): ids = command( BASE + ["ps", "-a", "-q", service] ).stdout.split() check(bool(ids), "service containers not found: " + service) return sum( int( command( [ "docker", "inspect", "--format", "{{.RestartCount}}", container, ] ).stdout.strip() ) for container in ids ) def object_exists(bucket, key): result = command( BASE + [ "exec", "-T", "minio", "sh", "-ec", "mc alias set grade4 http://localhost:9000 " '"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null; ' 'mc stat --json "grade4/$1/$2"', "sh", bucket, key, ], required=False, ) if result.returncode == 0: return True error = (result.stdout + result.stderr).lower() if any( value in error for value in [ "nosuchkey", "nosuchobject", "specified key does not exist", "object does not exist", ] ): return False raise Failure( "Cannot distinguish missing object from S3 failure" ) def webp_size(data): check( len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP", "invalid WebP signature", ) check( int.from_bytes(data[4:8], "little") + 8 == len(data), "invalid RIFF size", ) offset = 12 size = None while offset + 8 <= len(data): tag = data[offset:offset + 4] length = int.from_bytes( data[offset + 4:offset + 8], "little", ) end = offset + 8 + length check(end <= len(data), "truncated WebP chunk") block = data[offset + 8:end] check( tag not in (b"EXIF", b"XMP ", b"ICCP"), "metadata was retained in WebP", ) if tag == b"VP8X": check(len(block) == 10, "invalid VP8X chunk") size = ( 1 + int.from_bytes(block[4:7], "little"), 1 + int.from_bytes(block[7:10], "little"), ) elif tag == b"VP8 " and size is None: check(len(block) >= 10, "truncated VP8 frame") check( block[3:6] == b"\x9d\x01\x2a", "invalid VP8 frame", ) size = ( int.from_bytes(block[6:8], "little") & 0x3fff, int.from_bytes(block[8:10], "little") & 0x3fff, ) elif tag == b"VP8L" and size is None: check(len(block) >= 5, "truncated VP8L frame") check(block[0] == 0x2f, "invalid VP8L frame") bits = int.from_bytes(block[1:5], "little") size = ( 1 + (bits & 0x3fff), 1 + ((bits >> 14) & 0x3fff), ) offset = end + (length & 1) check(offset <= len(data), "missing WebP padding") check(offset == len(data), "invalid WebP trailing data") check(size is not None, "WebP dimensions missing") check(size[0] > 0 and size[1] > 0, "invalid WebP dimensions") return size def replay(job): result = command( FAULT + [ "run", "--rm", "--no-deps", "--pull", "never", "-e", "REPLAY_ENABLED=true", "-e", "FAULT_INJECTION_ENABLED=false", "-e", "REPLAY_JOB_ID=" + str(uuid.UUID(job)), "-e", "REPLAY_ACTOR=grade4_test", "-e", "REPLAY_REASON=verified_test_replay", "api", ] ) return result.stdout def restore_normal_configuration(): print( "Restoring normal API and worker configuration...", flush=True, ) try: sql( "UPDATE attachment_test_faults SET remaining = 0" ) finally: command( BASE + [ "up", "-d", "--no-build", "--pull", "never", "--no-deps", "--scale", "api=2", "--wait", "--wait-timeout", "180", "api", "worker", "lb", ] ) def wait_ready_after_fault(client, actor, attachment, seconds=180): deadline = time.monotonic() + seconds path = "/api/v1/attachments/" + attachment["id"] temporary_errors = 0 while time.monotonic() < deadline: try: result = client.call( "GET", path, actor=actor, schema="Attachment", ) except Failure as error: # Client проверяет HTTP-статус до разбора тела. # Повторяем только конкретные временные HTTP-ошибки. message = str(error) retryable = any( message == ( f"GET {path}: expected [200], received {status}" ) for status in (502, 503, 504) ) if not retryable: raise temporary_errors += 1 print( f"Waiting for API recovery: " f"temporary HTTP errors={temporary_errors}", flush=True, ) time.sleep(1) continue metadata = result["json"] check( metadata["status"] != "failed", "attachment reached failed during recovery", ) if metadata["status"] == "ready": print( "Attachment reached ready; " f"temporary HTTP errors={temporary_errors}", flush=True, ) return metadata check( metadata["status"] in ("queued", "processing"), "unexpected attachment status", ) time.sleep(0.5) raise Failure( "Attachment did not become ready after fault " f"within {seconds} seconds; " f"temporary HTTP errors={temporary_errors}" ) def main(): args = arguments( "Lab5 level4 fault tests", lambda parser: parser.add_argument( "--bucket", default="messenger-attachments", ), ) for filename in ( "compose.yml", "compose.override.yml", "compose.faults.yml", ): check( (ROOT / filename).is_file(), "Required Compose file not found: " + filename, ) client = Client(args.base_url, args.ca_file) evidence = [] try: print( "Starting API and worker with test fault injection enabled...", flush=True, ) command( FAULT + [ "up", "-d", "--no-build", "--pull", "never", "--no-deps", "--scale", "api=2", "--wait", "--wait-timeout", "180", "api", "worker", "lb", ] ) wait( lambda: client.call( "GET", "/health/ready", schema="Health", ), "API ready", ) sql( "UPDATE attachment_test_faults SET remaining = 0" ) wait( lambda: sql( """ SELECT count(*) FROM attachment_jobs WHERE status IN ('queued', 'processing') """ ) == "0", "existing work completed", ) alice, bob, _, chat = client.bootstrap() def upload(link=False): attachment, original = client.attachment( alice, chat, ) if link: client.message( alice, chat, "", attachments=[attachment["id"]], ) return attachment, original def verify(attachment, original): ready = wait_ready_after_fault( client, bob, attachment, seconds=180, ) path = ( "/api/v1/attachments/" + attachment["id"] + "/content" ) check( client.call( "GET", path, actor=bob, )["bytes"] == original, "original changed", ) expected = { "thumbnail": ("image/jpeg", (256, 160)), "crop": ("image/jpeg", (128, 128)), "optimized": ("image/webp", (320, 200)), } check( {v["name"] for v in ready["variants"]} == set(expected) and len(ready["variants"]) == 3, "expected three unique variants", ) for variant in ready["variants"]: mime, dimensions = expected[variant["name"]] result = client.call( "GET", path + "?variant=" + variant["name"], actor=bob, ) data = result["bytes"] actual = ( webp_size(data) if mime == "image/webp" else jpeg_size(data) ) check( actual == dimensions == ( variant["width"], variant["height"], ), "wrong actual dimensions", ) check( result["headers"].get_content_type() == mime == variant["content_type"], "wrong MIME", ) check( len(data) == variant["size_bytes"] and hashlib.sha256(data).hexdigest() == variant["sha256"], "variant checksum mismatch", ) if mime == "image/jpeg": check( b"Exif\x00\x00" not in data, "JPEG contains EXIF", ) return state(attachment["id"]) # Сценарий 1: # оригинал и задание уже сохранены, # но API падает до публикации в брокер. print( "Testing crash between COMMIT and publish...", flush=True, ) arm("hold_publish") attachment, original = upload(link=True) before = state(attachment["id"]) check( before["status"] == "queued", "publish hold did not work", ) baseline = restarts("api") arm("before_publish") arm("hold_publish", 0) wait( lambda: hit( "before_publish", before["job_id"], ), "crash before publish", ) wait( lambda: restarts("api") > baseline, "API actually restarted", ) after = verify(attachment, original) check( after["job_id"] == before["job_id"], "job was replaced", ) evidence.append( { "scenario": "commit_before_publish", **after, } ) print( "PASS: real API crash after COMMIT; " "the same job reached ready", flush=True, ) # Сценарии 2 и 3: # падение после записи объектов; # падение после фиксации ready, но до ACK. for point in ["after_objects", "before_ack"]: print( "Testing worker crash: " + point + "...", flush=True, ) baseline = restarts("worker") arm(point) attachment, original = upload() before = state(attachment["id"]) wait( lambda: hit(point, before["job_id"]), point, ) snapshot = state(attachment["id"]) old_keys = [] if point == "after_objects": check( snapshot["status"] == "processing" and not snapshot["variants"], "result committed before after_objects gate", ) check( snapshot["owner"] is not None, "processing lease owner missing", ) suffix = ( chat["id"] + "/" + attachment["id"] + "/v1/" + snapshot["owner"] ) old_keys = ["originals/" + suffix] + [ "variants/" + suffix + "/" + name for name in [ "thumbnail.jpg", "crop.jpg", "optimized.webp", ] ] check( all( object_exists(args.bucket, key) for key in old_keys ), "S3 writes missing", ) else: check( snapshot["status"] == "ready", "result not committed before before_ack gate", ) wait( lambda: restarts("worker") > baseline, "worker actually restarted", ) after = verify(attachment, original) if point == "before_ack": marker = ( "Duplicate job verified: job=" + after["job_id"] ) wait( lambda: marker in command( BASE + [ "logs", "--tail", "500", "worker", ] ).stdout, "duplicate checked after lost ACK", ) comparison = { "snapshot_status": snapshot["status"], "after_status": after["status"], "snapshot_attempts": snapshot["attempts"], "after_attempts": after["attempts"], "same_job": ( after["job_id"] == snapshot["job_id"] ), "same_original_key": ( after["original_key"] == snapshot["original_key"] ), "same_variants": ( after["variants"] == snapshot["variants"] ), "snapshot_variants": snapshot["variants"], "after_variants": after["variants"], "attempt_events": after["events"], } print( "BEFORE_ACK comparison:\n" + json.dumps( comparison, ensure_ascii=False, indent=2, ), flush=True, ) check( snapshot["status"] == "ready" and after["status"] == "ready" and after["attempts"] == snapshot["attempts"] and after["job_id"] == snapshot["job_id"] and after["variants"] == snapshot["variants"] and after["original_key"] == snapshot["original_key"] and after["events"] == snapshot["events"], "duplicate changed effect", ) else: check( after["attempts"] == 2, "lease recovery did not create one retry", ) staging = snapshot["staging"] check(bool(staging), "staging key missing") wait( lambda: state(attachment["id"])["staging"] is None and not any( object_exists(args.bucket, key) for key in old_keys + [staging] ), "safe staging/orphan cleanup", seconds=240, ) # После очистки подтверждённые результаты # должны оставаться доступными. after = verify(attachment, original) evidence.append( { "scenario": point, **after, } ) print( "PASS: " + point + "; durable result verified", flush=True, ) # Сценарии 4 и 5: # постоянная ошибка без повторов; # временная ошибка с ограниченным retry; # явный replay и повторный replay готовой задачи. for point, count, attempts in [ ("permanent_error", 1, 1), ("transient_error", 3, 3), ]: print( "Testing " + point + " and explicit replay...", flush=True, ) arm(point, count) attachment, original = upload() failed = wait( lambda: ( value if ( value := state(attachment["id"]) )["status"] == "failed" else None ), "bounded failure", ) check( failed["attempts"] == attempts, "incorrect retry count", ) check( bool(failed["events"]), "failure attempt events missing", ) for event in failed["events"]: cap = min( 60, 2 ** min(event["attempt"], 6), ) if event["state"] == "queued": valid_delay = ( 1 <= event["delay_seconds"] <= cap ) else: valid_delay = ( event["state"] == "failed" and event["delay_seconds"] == 0 ) check( valid_delay, "invalid backoff/jitter bounds", ) check( "REPLAY_RESULT=enqueued" in replay(failed["job_id"]), "replay failed", ) ready = verify(attachment, original) check( "REPLAY_RESULT=already_ready" in replay(failed["job_id"]), "ready replay not a no-op", ) repeated = state(attachment["id"]) check( repeated["attempts"] == ready["attempts"] and repeated["variants"] == ready["variants"] and repeated["original_key"] == ready["original_key"], "replay changed successful effect", ) evidence.append( { "scenario": point, "failed": failed, "ready": repeated, } ) print( "PASS: " + point + "; bounded attempts, explicit replay " "and ready no-op", flush=True, ) # Сценарий 6: # зависший процесс обработки должен быть остановлен # по таймауту, после чего задача повторяется. print( "Testing image-process timeout...", flush=True, ) arm("image_timeout") attachment, original = upload() result = verify(attachment, original) check( result["attempts"] == 2 and any( event["error_code"] == "processing_timeout" for event in result["events"] ), "hard image-process timeout not observed", ) evidence.append( { "scenario": "image_timeout", **result, } ) print( "PASS: stuck child process killed; retry succeeded", flush=True, ) finally: restore_normal_configuration() # Отчёт об успехе записывается только после прохождения # всех сценариев и восстановления обычной конфигурации. path = ROOT / "results" / "lab5-level4.json" path.parent.mkdir(parents=True, exist_ok=True) path.write_text( json.dumps( { "passed": True, "scenarios": evidence, }, ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) print( "PASS: level4 scenarios completed; " "results/lab5-level4.json saved", flush=True, ) if __name__ == "__main__": run(main)