Files
mtusi2026lab5/tests/level4_check.py
2026-11-11 23:05:16 +03:00

1006 lines
26 KiB
Python

import hashlib
import json
from pathlib import Path
import subprocess
import time
import uuid
from client import Client, Failure, arguments, check, run
from image_probe import jpeg_size
ROOT = Path(__file__).resolve().parents[1]
BASE = [
"docker",
"compose",
"-f",
"compose.yml",
"-f",
"compose.override.yml",
]
FAULT = BASE + [
"-f",
"compose.faults.yml",
]
FAULT_POINTS = {
"hold_publish",
"before_publish",
"after_objects",
"before_ack",
"permanent_error",
"transient_error",
"image_timeout",
}
def command(args, required=True):
try:
result = subprocess.run(
args,
cwd=ROOT,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=240,
)
except subprocess.TimeoutExpired:
raise Failure("Docker command timed out") from None
if required and result.returncode:
raise Failure(
"Docker command failed; "
"inspect the corresponding service locally"
)
return result
def sql(query):
result = command(
BASE
+ [
"exec",
"-T",
"postgres",
"sh",
"-ec",
'exec psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" '
'-At -v ON_ERROR_STOP=1 -c "$1"',
"sh",
query,
]
)
return result.stdout.strip()
def wait(action, label, seconds=180):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
try:
value = action()
if value:
return value
except Failure:
pass
time.sleep(1)
raise Failure("Timeout: " + label)
def state(attachment_id):
attachment_id = str(uuid.UUID(attachment_id))
raw = sql(
f"""
SELECT json_build_object(
'job_id', j.id,
'status', j.status,
'attempts', j.attempts,
'owner', j.lease_owner,
'staging', a.staging_key,
'original_key', a.object_key,
'variants', COALESCE(
(
SELECT json_agg(v ORDER BY v.name)
FROM attachment_variants v
WHERE v.attachment_id = a.id
AND v.pipeline_version = a.pipeline_version
),
'[]'::json
),
'events', COALESCE(
(
SELECT json_agg(e ORDER BY e.id)
FROM attachment_attempt_events e
WHERE e.job_id = j.id
),
'[]'::json
)
)
FROM attachment_jobs j
JOIN attachments a ON a.id = j.attachment_id
WHERE a.id = '{attachment_id}'::uuid
AND j.pipeline_version = a.pipeline_version
"""
)
check(bool(raw), "attachment job not found")
try:
return json.loads(raw)
except json.JSONDecodeError:
raise Failure("Invalid job state returned by PostgreSQL") from None
def arm(point, count=1):
check(point in FAULT_POINTS, "unknown fault")
check(isinstance(count, int) and count >= 0, "invalid fault count")
sql(
f"""
INSERT INTO attachment_test_faults(point, remaining)
VALUES ('{point}', {count})
ON CONFLICT (point)
DO UPDATE SET
remaining = EXCLUDED.remaining,
hits = 0,
job_id = NULL,
hit_at = NULL
"""
)
def hit(point, job_id):
check(point in FAULT_POINTS, "unknown fault")
job_id = str(uuid.UUID(job_id))
return sql(
f"""
SELECT count(*)
FROM attachment_test_faults
WHERE point = '{point}'
AND job_id = '{job_id}'::uuid
AND hits > 0
"""
) == "1"
def restarts(service):
ids = command(
BASE + ["ps", "-a", "-q", service]
).stdout.split()
check(bool(ids), "service containers not found: " + service)
return sum(
int(
command(
[
"docker",
"inspect",
"--format",
"{{.RestartCount}}",
container,
]
).stdout.strip()
)
for container in ids
)
def object_exists(bucket, key):
result = command(
BASE
+ [
"exec",
"-T",
"minio",
"sh",
"-ec",
"mc alias set grade4 http://localhost:9000 "
'"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null; '
'mc stat --json "grade4/$1/$2"',
"sh",
bucket,
key,
],
required=False,
)
if result.returncode == 0:
return True
error = (result.stdout + result.stderr).lower()
if any(
value in error
for value in [
"nosuchkey",
"nosuchobject",
"specified key does not exist",
"object does not exist",
]
):
return False
raise Failure(
"Cannot distinguish missing object from S3 failure"
)
def webp_size(data):
check(
len(data) >= 12
and data[:4] == b"RIFF"
and data[8:12] == b"WEBP",
"invalid WebP signature",
)
check(
int.from_bytes(data[4:8], "little") + 8 == len(data),
"invalid RIFF size",
)
offset = 12
size = None
while offset + 8 <= len(data):
tag = data[offset:offset + 4]
length = int.from_bytes(
data[offset + 4:offset + 8],
"little",
)
end = offset + 8 + length
check(end <= len(data), "truncated WebP chunk")
block = data[offset + 8:end]
check(
tag not in (b"EXIF", b"XMP ", b"ICCP"),
"metadata was retained in WebP",
)
if tag == b"VP8X":
check(len(block) == 10, "invalid VP8X chunk")
size = (
1 + int.from_bytes(block[4:7], "little"),
1 + int.from_bytes(block[7:10], "little"),
)
elif tag == b"VP8 " and size is None:
check(len(block) >= 10, "truncated VP8 frame")
check(
block[3:6] == b"\x9d\x01\x2a",
"invalid VP8 frame",
)
size = (
int.from_bytes(block[6:8], "little") & 0x3fff,
int.from_bytes(block[8:10], "little") & 0x3fff,
)
elif tag == b"VP8L" and size is None:
check(len(block) >= 5, "truncated VP8L frame")
check(block[0] == 0x2f, "invalid VP8L frame")
bits = int.from_bytes(block[1:5], "little")
size = (
1 + (bits & 0x3fff),
1 + ((bits >> 14) & 0x3fff),
)
offset = end + (length & 1)
check(offset <= len(data), "missing WebP padding")
check(offset == len(data), "invalid WebP trailing data")
check(size is not None, "WebP dimensions missing")
check(size[0] > 0 and size[1] > 0, "invalid WebP dimensions")
return size
def replay(job):
result = command(
FAULT
+ [
"run",
"--rm",
"--no-deps",
"--pull",
"never",
"-e",
"REPLAY_ENABLED=true",
"-e",
"FAULT_INJECTION_ENABLED=false",
"-e",
"REPLAY_JOB_ID=" + str(uuid.UUID(job)),
"-e",
"REPLAY_ACTOR=grade4_test",
"-e",
"REPLAY_REASON=verified_test_replay",
"api",
]
)
return result.stdout
def restore_normal_configuration():
print(
"Restoring normal API and worker configuration...",
flush=True,
)
try:
sql(
"UPDATE attachment_test_faults SET remaining = 0"
)
finally:
command(
BASE
+ [
"up",
"-d",
"--no-build",
"--pull",
"never",
"--no-deps",
"--scale",
"api=2",
"--wait",
"--wait-timeout",
"180",
"api",
"worker",
"lb",
]
)
def wait_ready_after_fault(client, actor, attachment, seconds=180):
deadline = time.monotonic() + seconds
path = "/api/v1/attachments/" + attachment["id"]
temporary_errors = 0
while time.monotonic() < deadline:
try:
result = client.call(
"GET",
path,
actor=actor,
schema="Attachment",
)
except Failure as error:
# Client проверяет HTTP-статус до разбора тела.
# Повторяем только конкретные временные HTTP-ошибки.
message = str(error)
retryable = any(
message == (
f"GET {path}: expected [200], received {status}"
)
for status in (502, 503, 504)
)
if not retryable:
raise
temporary_errors += 1
print(
f"Waiting for API recovery: "
f"temporary HTTP errors={temporary_errors}",
flush=True,
)
time.sleep(1)
continue
metadata = result["json"]
check(
metadata["status"] != "failed",
"attachment reached failed during recovery",
)
if metadata["status"] == "ready":
print(
"Attachment reached ready; "
f"temporary HTTP errors={temporary_errors}",
flush=True,
)
return metadata
check(
metadata["status"] in ("queued", "processing"),
"unexpected attachment status",
)
time.sleep(0.5)
raise Failure(
"Attachment did not become ready after fault "
f"within {seconds} seconds; "
f"temporary HTTP errors={temporary_errors}"
)
def main():
args = arguments(
"Lab5 level4 fault tests",
lambda parser: parser.add_argument(
"--bucket",
default="messenger-attachments",
),
)
for filename in (
"compose.yml",
"compose.override.yml",
"compose.faults.yml",
):
check(
(ROOT / filename).is_file(),
"Required Compose file not found: " + filename,
)
client = Client(args.base_url, args.ca_file)
evidence = []
try:
print(
"Starting API and worker with test fault injection enabled...",
flush=True,
)
command(
FAULT
+ [
"up",
"-d",
"--no-build",
"--pull",
"never",
"--no-deps",
"--scale",
"api=2",
"--wait",
"--wait-timeout",
"180",
"api",
"worker",
"lb",
]
)
wait(
lambda: client.call(
"GET",
"/health/ready",
schema="Health",
),
"API ready",
)
sql(
"UPDATE attachment_test_faults SET remaining = 0"
)
wait(
lambda: sql(
"""
SELECT count(*)
FROM attachment_jobs
WHERE status IN ('queued', 'processing')
"""
) == "0",
"existing work completed",
)
alice, bob, _, chat = client.bootstrap()
def upload(link=False):
attachment, original = client.attachment(
alice,
chat,
)
if link:
client.message(
alice,
chat,
"",
attachments=[attachment["id"]],
)
return attachment, original
def verify(attachment, original):
ready = wait_ready_after_fault(
client,
bob,
attachment,
seconds=180,
)
path = (
"/api/v1/attachments/"
+ attachment["id"]
+ "/content"
)
check(
client.call(
"GET",
path,
actor=bob,
)["bytes"] == original,
"original changed",
)
expected = {
"thumbnail": ("image/jpeg", (256, 160)),
"crop": ("image/jpeg", (128, 128)),
"optimized": ("image/webp", (320, 200)),
}
check(
{v["name"] for v in ready["variants"]}
== set(expected)
and len(ready["variants"]) == 3,
"expected three unique variants",
)
for variant in ready["variants"]:
mime, dimensions = expected[variant["name"]]
result = client.call(
"GET",
path + "?variant=" + variant["name"],
actor=bob,
)
data = result["bytes"]
actual = (
webp_size(data)
if mime == "image/webp"
else jpeg_size(data)
)
check(
actual
== dimensions
== (
variant["width"],
variant["height"],
),
"wrong actual dimensions",
)
check(
result["headers"].get_content_type()
== mime
== variant["content_type"],
"wrong MIME",
)
check(
len(data) == variant["size_bytes"]
and hashlib.sha256(data).hexdigest()
== variant["sha256"],
"variant checksum mismatch",
)
if mime == "image/jpeg":
check(
b"Exif\x00\x00" not in data,
"JPEG contains EXIF",
)
return state(attachment["id"])
# Сценарий 1:
# оригинал и задание уже сохранены,
# но API падает до публикации в брокер.
print(
"Testing crash between COMMIT and publish...",
flush=True,
)
arm("hold_publish")
attachment, original = upload(link=True)
before = state(attachment["id"])
check(
before["status"] == "queued",
"publish hold did not work",
)
baseline = restarts("api")
arm("before_publish")
arm("hold_publish", 0)
wait(
lambda: hit(
"before_publish",
before["job_id"],
),
"crash before publish",
)
wait(
lambda: restarts("api") > baseline,
"API actually restarted",
)
after = verify(attachment, original)
check(
after["job_id"] == before["job_id"],
"job was replaced",
)
evidence.append(
{
"scenario": "commit_before_publish",
**after,
}
)
print(
"PASS: real API crash after COMMIT; "
"the same job reached ready",
flush=True,
)
# Сценарии 2 и 3:
# падение после записи объектов;
# падение после фиксации ready, но до ACK.
for point in ["after_objects", "before_ack"]:
print(
"Testing worker crash: " + point + "...",
flush=True,
)
baseline = restarts("worker")
arm(point)
attachment, original = upload()
before = state(attachment["id"])
wait(
lambda: hit(point, before["job_id"]),
point,
)
snapshot = state(attachment["id"])
old_keys = []
if point == "after_objects":
check(
snapshot["status"] == "processing"
and not snapshot["variants"],
"result committed before after_objects gate",
)
check(
snapshot["owner"] is not None,
"processing lease owner missing",
)
suffix = (
chat["id"]
+ "/"
+ attachment["id"]
+ "/v1/"
+ snapshot["owner"]
)
old_keys = ["originals/" + suffix] + [
"variants/" + suffix + "/" + name
for name in [
"thumbnail.jpg",
"crop.jpg",
"optimized.webp",
]
]
check(
all(
object_exists(args.bucket, key)
for key in old_keys
),
"S3 writes missing",
)
else:
check(
snapshot["status"] == "ready",
"result not committed before before_ack gate",
)
wait(
lambda: restarts("worker") > baseline,
"worker actually restarted",
)
after = verify(attachment, original)
if point == "before_ack":
marker = (
"Duplicate job verified: job="
+ after["job_id"]
)
wait(
lambda: marker in command(
BASE
+ [
"logs",
"--tail",
"500",
"worker",
]
).stdout,
"duplicate checked after lost ACK",
)
comparison = {
"snapshot_status": snapshot["status"],
"after_status": after["status"],
"snapshot_attempts": snapshot["attempts"],
"after_attempts": after["attempts"],
"same_job": (
after["job_id"] == snapshot["job_id"]
),
"same_original_key": (
after["original_key"]
== snapshot["original_key"]
),
"same_variants": (
after["variants"] == snapshot["variants"]
),
"snapshot_variants": snapshot["variants"],
"after_variants": after["variants"],
"attempt_events": after["events"],
}
print(
"BEFORE_ACK comparison:\n"
+ json.dumps(
comparison,
ensure_ascii=False,
indent=2,
),
flush=True,
)
check(
snapshot["status"] == "ready"
and after["status"] == "ready"
and after["attempts"] == snapshot["attempts"]
and after["job_id"] == snapshot["job_id"]
and after["variants"] == snapshot["variants"]
and after["original_key"] == snapshot["original_key"]
and after["events"] == snapshot["events"],
"duplicate changed effect",
)
else:
check(
after["attempts"] == 2,
"lease recovery did not create one retry",
)
staging = snapshot["staging"]
check(bool(staging), "staging key missing")
wait(
lambda: state(attachment["id"])["staging"]
is None
and not any(
object_exists(args.bucket, key)
for key in old_keys + [staging]
),
"safe staging/orphan cleanup",
seconds=240,
)
# После очистки подтверждённые результаты
# должны оставаться доступными.
after = verify(attachment, original)
evidence.append(
{
"scenario": point,
**after,
}
)
print(
"PASS: "
+ point
+ "; durable result verified",
flush=True,
)
# Сценарии 4 и 5:
# постоянная ошибка без повторов;
# временная ошибка с ограниченным retry;
# явный replay и повторный replay готовой задачи.
for point, count, attempts in [
("permanent_error", 1, 1),
("transient_error", 3, 3),
]:
print(
"Testing " + point + " and explicit replay...",
flush=True,
)
arm(point, count)
attachment, original = upload()
failed = wait(
lambda: (
value
if (
value := state(attachment["id"])
)["status"] == "failed"
else None
),
"bounded failure",
)
check(
failed["attempts"] == attempts,
"incorrect retry count",
)
check(
bool(failed["events"]),
"failure attempt events missing",
)
for event in failed["events"]:
cap = min(
60,
2 ** min(event["attempt"], 6),
)
if event["state"] == "queued":
valid_delay = (
1 <= event["delay_seconds"] <= cap
)
else:
valid_delay = (
event["state"] == "failed"
and event["delay_seconds"] == 0
)
check(
valid_delay,
"invalid backoff/jitter bounds",
)
check(
"REPLAY_RESULT=enqueued"
in replay(failed["job_id"]),
"replay failed",
)
ready = verify(attachment, original)
check(
"REPLAY_RESULT=already_ready"
in replay(failed["job_id"]),
"ready replay not a no-op",
)
repeated = state(attachment["id"])
check(
repeated["attempts"] == ready["attempts"]
and repeated["variants"] == ready["variants"]
and repeated["original_key"] == ready["original_key"],
"replay changed successful effect",
)
evidence.append(
{
"scenario": point,
"failed": failed,
"ready": repeated,
}
)
print(
"PASS: "
+ point
+ "; bounded attempts, explicit replay "
"and ready no-op",
flush=True,
)
# Сценарий 6:
# зависший процесс обработки должен быть остановлен
# по таймауту, после чего задача повторяется.
print(
"Testing image-process timeout...",
flush=True,
)
arm("image_timeout")
attachment, original = upload()
result = verify(attachment, original)
check(
result["attempts"] == 2
and any(
event["error_code"] == "processing_timeout"
for event in result["events"]
),
"hard image-process timeout not observed",
)
evidence.append(
{
"scenario": "image_timeout",
**result,
}
)
print(
"PASS: stuck child process killed; retry succeeded",
flush=True,
)
finally:
restore_normal_configuration()
# Отчёт об успехе записывается только после прохождения
# всех сценариев и восстановления обычной конфигурации.
path = ROOT / "results" / "lab5-level4.json"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(
{
"passed": True,
"scenarios": evidence,
},
ensure_ascii=False,
indent=2,
) + "\n",
encoding="utf-8",
)
print(
"PASS: level4 scenarios completed; "
"results/lab5-level4.json saved",
flush=True,
)
if __name__ == "__main__":
run(main)