forked from mtusi2026/mtusi2026lab5
1006 lines
26 KiB
Python
1006 lines
26 KiB
Python
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
|
|
from client import Client, Failure, arguments, check, run
|
|
from image_probe import jpeg_size
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
BASE = [
|
|
"docker",
|
|
"compose",
|
|
"-f",
|
|
"compose.yml",
|
|
"-f",
|
|
"compose.override.yml",
|
|
]
|
|
|
|
FAULT = BASE + [
|
|
"-f",
|
|
"compose.faults.yml",
|
|
]
|
|
|
|
FAULT_POINTS = {
|
|
"hold_publish",
|
|
"before_publish",
|
|
"after_objects",
|
|
"before_ack",
|
|
"permanent_error",
|
|
"transient_error",
|
|
"image_timeout",
|
|
}
|
|
|
|
|
|
def command(args, required=True):
|
|
try:
|
|
result = subprocess.run(
|
|
args,
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
timeout=240,
|
|
)
|
|
except subprocess.TimeoutExpired:
|
|
raise Failure("Docker command timed out") from None
|
|
|
|
if required and result.returncode:
|
|
raise Failure(
|
|
"Docker command failed; "
|
|
"inspect the corresponding service locally"
|
|
)
|
|
|
|
return result
|
|
|
|
|
|
def sql(query):
|
|
result = command(
|
|
BASE
|
|
+ [
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"sh",
|
|
"-ec",
|
|
'exec psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" '
|
|
'-At -v ON_ERROR_STOP=1 -c "$1"',
|
|
"sh",
|
|
query,
|
|
]
|
|
)
|
|
|
|
return result.stdout.strip()
|
|
|
|
|
|
def wait(action, label, seconds=180):
|
|
deadline = time.monotonic() + seconds
|
|
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
value = action()
|
|
|
|
if value:
|
|
return value
|
|
|
|
except Failure:
|
|
pass
|
|
|
|
time.sleep(1)
|
|
|
|
raise Failure("Timeout: " + label)
|
|
|
|
|
|
def state(attachment_id):
|
|
attachment_id = str(uuid.UUID(attachment_id))
|
|
|
|
raw = sql(
|
|
f"""
|
|
SELECT json_build_object(
|
|
'job_id', j.id,
|
|
'status', j.status,
|
|
'attempts', j.attempts,
|
|
'owner', j.lease_owner,
|
|
'staging', a.staging_key,
|
|
'original_key', a.object_key,
|
|
|
|
'variants', COALESCE(
|
|
(
|
|
SELECT json_agg(v ORDER BY v.name)
|
|
FROM attachment_variants v
|
|
WHERE v.attachment_id = a.id
|
|
AND v.pipeline_version = a.pipeline_version
|
|
),
|
|
'[]'::json
|
|
),
|
|
|
|
'events', COALESCE(
|
|
(
|
|
SELECT json_agg(e ORDER BY e.id)
|
|
FROM attachment_attempt_events e
|
|
WHERE e.job_id = j.id
|
|
),
|
|
'[]'::json
|
|
)
|
|
)
|
|
FROM attachment_jobs j
|
|
JOIN attachments a ON a.id = j.attachment_id
|
|
WHERE a.id = '{attachment_id}'::uuid
|
|
AND j.pipeline_version = a.pipeline_version
|
|
"""
|
|
)
|
|
|
|
check(bool(raw), "attachment job not found")
|
|
|
|
try:
|
|
return json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
raise Failure("Invalid job state returned by PostgreSQL") from None
|
|
|
|
|
|
def arm(point, count=1):
|
|
check(point in FAULT_POINTS, "unknown fault")
|
|
check(isinstance(count, int) and count >= 0, "invalid fault count")
|
|
|
|
sql(
|
|
f"""
|
|
INSERT INTO attachment_test_faults(point, remaining)
|
|
VALUES ('{point}', {count})
|
|
|
|
ON CONFLICT (point)
|
|
DO UPDATE SET
|
|
remaining = EXCLUDED.remaining,
|
|
hits = 0,
|
|
job_id = NULL,
|
|
hit_at = NULL
|
|
"""
|
|
)
|
|
|
|
|
|
def hit(point, job_id):
|
|
check(point in FAULT_POINTS, "unknown fault")
|
|
job_id = str(uuid.UUID(job_id))
|
|
|
|
return sql(
|
|
f"""
|
|
SELECT count(*)
|
|
FROM attachment_test_faults
|
|
WHERE point = '{point}'
|
|
AND job_id = '{job_id}'::uuid
|
|
AND hits > 0
|
|
"""
|
|
) == "1"
|
|
|
|
|
|
def restarts(service):
|
|
ids = command(
|
|
BASE + ["ps", "-a", "-q", service]
|
|
).stdout.split()
|
|
|
|
check(bool(ids), "service containers not found: " + service)
|
|
|
|
return sum(
|
|
int(
|
|
command(
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
"{{.RestartCount}}",
|
|
container,
|
|
]
|
|
).stdout.strip()
|
|
)
|
|
for container in ids
|
|
)
|
|
|
|
|
|
def object_exists(bucket, key):
|
|
result = command(
|
|
BASE
|
|
+ [
|
|
"exec",
|
|
"-T",
|
|
"minio",
|
|
"sh",
|
|
"-ec",
|
|
"mc alias set grade4 http://localhost:9000 "
|
|
'"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null; '
|
|
'mc stat --json "grade4/$1/$2"',
|
|
"sh",
|
|
bucket,
|
|
key,
|
|
],
|
|
required=False,
|
|
)
|
|
|
|
if result.returncode == 0:
|
|
return True
|
|
|
|
error = (result.stdout + result.stderr).lower()
|
|
|
|
if any(
|
|
value in error
|
|
for value in [
|
|
"nosuchkey",
|
|
"nosuchobject",
|
|
"specified key does not exist",
|
|
"object does not exist",
|
|
]
|
|
):
|
|
return False
|
|
|
|
raise Failure(
|
|
"Cannot distinguish missing object from S3 failure"
|
|
)
|
|
|
|
|
|
def webp_size(data):
|
|
check(
|
|
len(data) >= 12
|
|
and data[:4] == b"RIFF"
|
|
and data[8:12] == b"WEBP",
|
|
"invalid WebP signature",
|
|
)
|
|
|
|
check(
|
|
int.from_bytes(data[4:8], "little") + 8 == len(data),
|
|
"invalid RIFF size",
|
|
)
|
|
|
|
offset = 12
|
|
size = None
|
|
|
|
while offset + 8 <= len(data):
|
|
tag = data[offset:offset + 4]
|
|
|
|
length = int.from_bytes(
|
|
data[offset + 4:offset + 8],
|
|
"little",
|
|
)
|
|
|
|
end = offset + 8 + length
|
|
check(end <= len(data), "truncated WebP chunk")
|
|
|
|
block = data[offset + 8:end]
|
|
|
|
check(
|
|
tag not in (b"EXIF", b"XMP ", b"ICCP"),
|
|
"metadata was retained in WebP",
|
|
)
|
|
|
|
if tag == b"VP8X":
|
|
check(len(block) == 10, "invalid VP8X chunk")
|
|
|
|
size = (
|
|
1 + int.from_bytes(block[4:7], "little"),
|
|
1 + int.from_bytes(block[7:10], "little"),
|
|
)
|
|
|
|
elif tag == b"VP8 " and size is None:
|
|
check(len(block) >= 10, "truncated VP8 frame")
|
|
|
|
check(
|
|
block[3:6] == b"\x9d\x01\x2a",
|
|
"invalid VP8 frame",
|
|
)
|
|
|
|
size = (
|
|
int.from_bytes(block[6:8], "little") & 0x3fff,
|
|
int.from_bytes(block[8:10], "little") & 0x3fff,
|
|
)
|
|
|
|
elif tag == b"VP8L" and size is None:
|
|
check(len(block) >= 5, "truncated VP8L frame")
|
|
check(block[0] == 0x2f, "invalid VP8L frame")
|
|
|
|
bits = int.from_bytes(block[1:5], "little")
|
|
|
|
size = (
|
|
1 + (bits & 0x3fff),
|
|
1 + ((bits >> 14) & 0x3fff),
|
|
)
|
|
|
|
offset = end + (length & 1)
|
|
check(offset <= len(data), "missing WebP padding")
|
|
|
|
check(offset == len(data), "invalid WebP trailing data")
|
|
check(size is not None, "WebP dimensions missing")
|
|
check(size[0] > 0 and size[1] > 0, "invalid WebP dimensions")
|
|
|
|
return size
|
|
|
|
|
|
def replay(job):
|
|
result = command(
|
|
FAULT
|
|
+ [
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"--pull",
|
|
"never",
|
|
"-e",
|
|
"REPLAY_ENABLED=true",
|
|
"-e",
|
|
"FAULT_INJECTION_ENABLED=false",
|
|
"-e",
|
|
"REPLAY_JOB_ID=" + str(uuid.UUID(job)),
|
|
"-e",
|
|
"REPLAY_ACTOR=grade4_test",
|
|
"-e",
|
|
"REPLAY_REASON=verified_test_replay",
|
|
"api",
|
|
]
|
|
)
|
|
|
|
return result.stdout
|
|
|
|
|
|
def restore_normal_configuration():
|
|
print(
|
|
"Restoring normal API and worker configuration...",
|
|
flush=True,
|
|
)
|
|
|
|
try:
|
|
sql(
|
|
"UPDATE attachment_test_faults SET remaining = 0"
|
|
)
|
|
finally:
|
|
command(
|
|
BASE
|
|
+ [
|
|
"up",
|
|
"-d",
|
|
"--no-build",
|
|
"--pull",
|
|
"never",
|
|
"--no-deps",
|
|
"--scale",
|
|
"api=2",
|
|
"--wait",
|
|
"--wait-timeout",
|
|
"180",
|
|
"api",
|
|
"worker",
|
|
"lb",
|
|
]
|
|
)
|
|
def wait_ready_after_fault(client, actor, attachment, seconds=180):
|
|
deadline = time.monotonic() + seconds
|
|
path = "/api/v1/attachments/" + attachment["id"]
|
|
temporary_errors = 0
|
|
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
result = client.call(
|
|
"GET",
|
|
path,
|
|
actor=actor,
|
|
schema="Attachment",
|
|
)
|
|
|
|
except Failure as error:
|
|
# Client проверяет HTTP-статус до разбора тела.
|
|
# Повторяем только конкретные временные HTTP-ошибки.
|
|
message = str(error)
|
|
|
|
retryable = any(
|
|
message == (
|
|
f"GET {path}: expected [200], received {status}"
|
|
)
|
|
for status in (502, 503, 504)
|
|
)
|
|
|
|
if not retryable:
|
|
raise
|
|
|
|
temporary_errors += 1
|
|
|
|
print(
|
|
f"Waiting for API recovery: "
|
|
f"temporary HTTP errors={temporary_errors}",
|
|
flush=True,
|
|
)
|
|
|
|
time.sleep(1)
|
|
continue
|
|
|
|
metadata = result["json"]
|
|
|
|
check(
|
|
metadata["status"] != "failed",
|
|
"attachment reached failed during recovery",
|
|
)
|
|
|
|
if metadata["status"] == "ready":
|
|
print(
|
|
"Attachment reached ready; "
|
|
f"temporary HTTP errors={temporary_errors}",
|
|
flush=True,
|
|
)
|
|
return metadata
|
|
|
|
check(
|
|
metadata["status"] in ("queued", "processing"),
|
|
"unexpected attachment status",
|
|
)
|
|
|
|
time.sleep(0.5)
|
|
|
|
raise Failure(
|
|
"Attachment did not become ready after fault "
|
|
f"within {seconds} seconds; "
|
|
f"temporary HTTP errors={temporary_errors}"
|
|
)
|
|
|
|
def main():
|
|
args = arguments(
|
|
"Lab5 level4 fault tests",
|
|
lambda parser: parser.add_argument(
|
|
"--bucket",
|
|
default="messenger-attachments",
|
|
),
|
|
)
|
|
|
|
for filename in (
|
|
"compose.yml",
|
|
"compose.override.yml",
|
|
"compose.faults.yml",
|
|
):
|
|
check(
|
|
(ROOT / filename).is_file(),
|
|
"Required Compose file not found: " + filename,
|
|
)
|
|
|
|
client = Client(args.base_url, args.ca_file)
|
|
evidence = []
|
|
|
|
try:
|
|
print(
|
|
"Starting API and worker with test fault injection enabled...",
|
|
flush=True,
|
|
)
|
|
|
|
command(
|
|
FAULT
|
|
+ [
|
|
"up",
|
|
"-d",
|
|
"--no-build",
|
|
"--pull",
|
|
"never",
|
|
"--no-deps",
|
|
"--scale",
|
|
"api=2",
|
|
"--wait",
|
|
"--wait-timeout",
|
|
"180",
|
|
"api",
|
|
"worker",
|
|
"lb",
|
|
]
|
|
)
|
|
|
|
wait(
|
|
lambda: client.call(
|
|
"GET",
|
|
"/health/ready",
|
|
schema="Health",
|
|
),
|
|
"API ready",
|
|
)
|
|
|
|
sql(
|
|
"UPDATE attachment_test_faults SET remaining = 0"
|
|
)
|
|
|
|
wait(
|
|
lambda: sql(
|
|
"""
|
|
SELECT count(*)
|
|
FROM attachment_jobs
|
|
WHERE status IN ('queued', 'processing')
|
|
"""
|
|
) == "0",
|
|
"existing work completed",
|
|
)
|
|
|
|
alice, bob, _, chat = client.bootstrap()
|
|
|
|
def upload(link=False):
|
|
attachment, original = client.attachment(
|
|
alice,
|
|
chat,
|
|
)
|
|
|
|
if link:
|
|
client.message(
|
|
alice,
|
|
chat,
|
|
"",
|
|
attachments=[attachment["id"]],
|
|
)
|
|
|
|
return attachment, original
|
|
|
|
def verify(attachment, original):
|
|
ready = wait_ready_after_fault(
|
|
client,
|
|
bob,
|
|
attachment,
|
|
seconds=180,
|
|
)
|
|
|
|
path = (
|
|
"/api/v1/attachments/"
|
|
+ attachment["id"]
|
|
+ "/content"
|
|
)
|
|
|
|
check(
|
|
client.call(
|
|
"GET",
|
|
path,
|
|
actor=bob,
|
|
)["bytes"] == original,
|
|
"original changed",
|
|
)
|
|
|
|
expected = {
|
|
"thumbnail": ("image/jpeg", (256, 160)),
|
|
"crop": ("image/jpeg", (128, 128)),
|
|
"optimized": ("image/webp", (320, 200)),
|
|
}
|
|
|
|
check(
|
|
{v["name"] for v in ready["variants"]}
|
|
== set(expected)
|
|
and len(ready["variants"]) == 3,
|
|
"expected three unique variants",
|
|
)
|
|
|
|
for variant in ready["variants"]:
|
|
mime, dimensions = expected[variant["name"]]
|
|
|
|
result = client.call(
|
|
"GET",
|
|
path + "?variant=" + variant["name"],
|
|
actor=bob,
|
|
)
|
|
|
|
data = result["bytes"]
|
|
|
|
actual = (
|
|
webp_size(data)
|
|
if mime == "image/webp"
|
|
else jpeg_size(data)
|
|
)
|
|
|
|
check(
|
|
actual
|
|
== dimensions
|
|
== (
|
|
variant["width"],
|
|
variant["height"],
|
|
),
|
|
"wrong actual dimensions",
|
|
)
|
|
|
|
check(
|
|
result["headers"].get_content_type()
|
|
== mime
|
|
== variant["content_type"],
|
|
"wrong MIME",
|
|
)
|
|
|
|
check(
|
|
len(data) == variant["size_bytes"]
|
|
and hashlib.sha256(data).hexdigest()
|
|
== variant["sha256"],
|
|
"variant checksum mismatch",
|
|
)
|
|
|
|
if mime == "image/jpeg":
|
|
check(
|
|
b"Exif\x00\x00" not in data,
|
|
"JPEG contains EXIF",
|
|
)
|
|
|
|
return state(attachment["id"])
|
|
|
|
# Сценарий 1:
|
|
# оригинал и задание уже сохранены,
|
|
# но API падает до публикации в брокер.
|
|
print(
|
|
"Testing crash between COMMIT and publish...",
|
|
flush=True,
|
|
)
|
|
|
|
arm("hold_publish")
|
|
|
|
attachment, original = upload(link=True)
|
|
before = state(attachment["id"])
|
|
|
|
check(
|
|
before["status"] == "queued",
|
|
"publish hold did not work",
|
|
)
|
|
|
|
baseline = restarts("api")
|
|
|
|
arm("before_publish")
|
|
arm("hold_publish", 0)
|
|
|
|
wait(
|
|
lambda: hit(
|
|
"before_publish",
|
|
before["job_id"],
|
|
),
|
|
"crash before publish",
|
|
)
|
|
|
|
wait(
|
|
lambda: restarts("api") > baseline,
|
|
"API actually restarted",
|
|
)
|
|
|
|
after = verify(attachment, original)
|
|
|
|
check(
|
|
after["job_id"] == before["job_id"],
|
|
"job was replaced",
|
|
)
|
|
|
|
evidence.append(
|
|
{
|
|
"scenario": "commit_before_publish",
|
|
**after,
|
|
}
|
|
)
|
|
|
|
print(
|
|
"PASS: real API crash after COMMIT; "
|
|
"the same job reached ready",
|
|
flush=True,
|
|
)
|
|
|
|
# Сценарии 2 и 3:
|
|
# падение после записи объектов;
|
|
# падение после фиксации ready, но до ACK.
|
|
for point in ["after_objects", "before_ack"]:
|
|
print(
|
|
"Testing worker crash: " + point + "...",
|
|
flush=True,
|
|
)
|
|
|
|
baseline = restarts("worker")
|
|
|
|
arm(point)
|
|
|
|
attachment, original = upload()
|
|
before = state(attachment["id"])
|
|
|
|
wait(
|
|
lambda: hit(point, before["job_id"]),
|
|
point,
|
|
)
|
|
|
|
snapshot = state(attachment["id"])
|
|
old_keys = []
|
|
|
|
if point == "after_objects":
|
|
check(
|
|
snapshot["status"] == "processing"
|
|
and not snapshot["variants"],
|
|
"result committed before after_objects gate",
|
|
)
|
|
|
|
check(
|
|
snapshot["owner"] is not None,
|
|
"processing lease owner missing",
|
|
)
|
|
|
|
suffix = (
|
|
chat["id"]
|
|
+ "/"
|
|
+ attachment["id"]
|
|
+ "/v1/"
|
|
+ snapshot["owner"]
|
|
)
|
|
|
|
old_keys = ["originals/" + suffix] + [
|
|
"variants/" + suffix + "/" + name
|
|
for name in [
|
|
"thumbnail.jpg",
|
|
"crop.jpg",
|
|
"optimized.webp",
|
|
]
|
|
]
|
|
|
|
check(
|
|
all(
|
|
object_exists(args.bucket, key)
|
|
for key in old_keys
|
|
),
|
|
"S3 writes missing",
|
|
)
|
|
|
|
else:
|
|
check(
|
|
snapshot["status"] == "ready",
|
|
"result not committed before before_ack gate",
|
|
)
|
|
|
|
wait(
|
|
lambda: restarts("worker") > baseline,
|
|
"worker actually restarted",
|
|
)
|
|
|
|
after = verify(attachment, original)
|
|
|
|
if point == "before_ack":
|
|
marker = (
|
|
"Duplicate job verified: job="
|
|
+ after["job_id"]
|
|
)
|
|
|
|
wait(
|
|
lambda: marker in command(
|
|
BASE
|
|
+ [
|
|
"logs",
|
|
"--tail",
|
|
"500",
|
|
"worker",
|
|
]
|
|
).stdout,
|
|
"duplicate checked after lost ACK",
|
|
)
|
|
|
|
comparison = {
|
|
"snapshot_status": snapshot["status"],
|
|
"after_status": after["status"],
|
|
"snapshot_attempts": snapshot["attempts"],
|
|
"after_attempts": after["attempts"],
|
|
"same_job": (
|
|
after["job_id"] == snapshot["job_id"]
|
|
),
|
|
"same_original_key": (
|
|
after["original_key"]
|
|
== snapshot["original_key"]
|
|
),
|
|
"same_variants": (
|
|
after["variants"] == snapshot["variants"]
|
|
),
|
|
"snapshot_variants": snapshot["variants"],
|
|
"after_variants": after["variants"],
|
|
"attempt_events": after["events"],
|
|
}
|
|
|
|
print(
|
|
"BEFORE_ACK comparison:\n"
|
|
+ json.dumps(
|
|
comparison,
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
),
|
|
flush=True,
|
|
)
|
|
|
|
check(
|
|
snapshot["status"] == "ready"
|
|
and after["status"] == "ready"
|
|
and after["attempts"] == snapshot["attempts"]
|
|
and after["job_id"] == snapshot["job_id"]
|
|
and after["variants"] == snapshot["variants"]
|
|
and after["original_key"] == snapshot["original_key"]
|
|
and after["events"] == snapshot["events"],
|
|
"duplicate changed effect",
|
|
)
|
|
|
|
else:
|
|
check(
|
|
after["attempts"] == 2,
|
|
"lease recovery did not create one retry",
|
|
)
|
|
|
|
staging = snapshot["staging"]
|
|
check(bool(staging), "staging key missing")
|
|
|
|
wait(
|
|
lambda: state(attachment["id"])["staging"]
|
|
is None
|
|
and not any(
|
|
object_exists(args.bucket, key)
|
|
for key in old_keys + [staging]
|
|
),
|
|
"safe staging/orphan cleanup",
|
|
seconds=240,
|
|
)
|
|
|
|
# После очистки подтверждённые результаты
|
|
# должны оставаться доступными.
|
|
after = verify(attachment, original)
|
|
|
|
evidence.append(
|
|
{
|
|
"scenario": point,
|
|
**after,
|
|
}
|
|
)
|
|
|
|
print(
|
|
"PASS: "
|
|
+ point
|
|
+ "; durable result verified",
|
|
flush=True,
|
|
)
|
|
|
|
# Сценарии 4 и 5:
|
|
# постоянная ошибка без повторов;
|
|
# временная ошибка с ограниченным retry;
|
|
# явный replay и повторный replay готовой задачи.
|
|
for point, count, attempts in [
|
|
("permanent_error", 1, 1),
|
|
("transient_error", 3, 3),
|
|
]:
|
|
print(
|
|
"Testing " + point + " and explicit replay...",
|
|
flush=True,
|
|
)
|
|
|
|
arm(point, count)
|
|
|
|
attachment, original = upload()
|
|
|
|
failed = wait(
|
|
lambda: (
|
|
value
|
|
if (
|
|
value := state(attachment["id"])
|
|
)["status"] == "failed"
|
|
else None
|
|
),
|
|
"bounded failure",
|
|
)
|
|
|
|
check(
|
|
failed["attempts"] == attempts,
|
|
"incorrect retry count",
|
|
)
|
|
|
|
check(
|
|
bool(failed["events"]),
|
|
"failure attempt events missing",
|
|
)
|
|
|
|
for event in failed["events"]:
|
|
cap = min(
|
|
60,
|
|
2 ** min(event["attempt"], 6),
|
|
)
|
|
|
|
if event["state"] == "queued":
|
|
valid_delay = (
|
|
1 <= event["delay_seconds"] <= cap
|
|
)
|
|
else:
|
|
valid_delay = (
|
|
event["state"] == "failed"
|
|
and event["delay_seconds"] == 0
|
|
)
|
|
|
|
check(
|
|
valid_delay,
|
|
"invalid backoff/jitter bounds",
|
|
)
|
|
|
|
check(
|
|
"REPLAY_RESULT=enqueued"
|
|
in replay(failed["job_id"]),
|
|
"replay failed",
|
|
)
|
|
|
|
ready = verify(attachment, original)
|
|
|
|
check(
|
|
"REPLAY_RESULT=already_ready"
|
|
in replay(failed["job_id"]),
|
|
"ready replay not a no-op",
|
|
)
|
|
|
|
repeated = state(attachment["id"])
|
|
|
|
check(
|
|
repeated["attempts"] == ready["attempts"]
|
|
and repeated["variants"] == ready["variants"]
|
|
and repeated["original_key"] == ready["original_key"],
|
|
"replay changed successful effect",
|
|
)
|
|
|
|
evidence.append(
|
|
{
|
|
"scenario": point,
|
|
"failed": failed,
|
|
"ready": repeated,
|
|
}
|
|
)
|
|
|
|
print(
|
|
"PASS: "
|
|
+ point
|
|
+ "; bounded attempts, explicit replay "
|
|
"and ready no-op",
|
|
flush=True,
|
|
)
|
|
|
|
# Сценарий 6:
|
|
# зависший процесс обработки должен быть остановлен
|
|
# по таймауту, после чего задача повторяется.
|
|
print(
|
|
"Testing image-process timeout...",
|
|
flush=True,
|
|
)
|
|
|
|
arm("image_timeout")
|
|
|
|
attachment, original = upload()
|
|
result = verify(attachment, original)
|
|
|
|
check(
|
|
result["attempts"] == 2
|
|
and any(
|
|
event["error_code"] == "processing_timeout"
|
|
for event in result["events"]
|
|
),
|
|
"hard image-process timeout not observed",
|
|
)
|
|
|
|
evidence.append(
|
|
{
|
|
"scenario": "image_timeout",
|
|
**result,
|
|
}
|
|
)
|
|
|
|
print(
|
|
"PASS: stuck child process killed; retry succeeded",
|
|
flush=True,
|
|
)
|
|
|
|
finally:
|
|
restore_normal_configuration()
|
|
|
|
# Отчёт об успехе записывается только после прохождения
|
|
# всех сценариев и восстановления обычной конфигурации.
|
|
path = ROOT / "results" / "lab5-level4.json"
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
path.write_text(
|
|
json.dumps(
|
|
{
|
|
"passed": True,
|
|
"scenarios": evidence,
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
print(
|
|
"PASS: level4 scenarios completed; "
|
|
"results/lab5-level4.json saved",
|
|
flush=True,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main) |