Files
mtusi2026lab5/student-tests/after_objects.py
T

267 lines
8.7 KiB
Python

from __future__ import annotations
import base64
import hashlib
import json
import secrets
import time
from uuid import UUID, uuid4
from queued_restart import ROOT, api, dc, http, wait_ready
from outbox_gap import snapshot
from before_ack import sql, object_keys
EVIDENCE = ROOT / "evidence" / "after_objects.json"
def intermediate_state(job_id):
return json.loads(sql(f"""
SELECT json_build_object(
'job_id', j.id,
'status', j.status,
'attempt', j.attempt,
'lease_remaining_seconds',
extract(epoch FROM (j.lease_until - clock_timestamp())),
'fault_consumed', f.consumed_at IS NOT NULL,
'variant_rows', (
SELECT count(*)
FROM attachment_variants v
WHERE v.attachment_id = j.attachment_id
)
)
FROM attachment_jobs j
JOIN pipeline_fault_injections f ON f.job_id = j.id
WHERE j.id = '{job_id}'::uuid
AND f.stage = 'after_objects';
"""))
def main():
evidence = {
"scenario": "worker crash after S3 writes, before result COMMIT",
}
job_id = None
restore = False
try:
wait_ready()
for service in ("publisher", "worker"):
if not dc("ps", "--status", "running", "-q", service):
raise AssertionError(f"Start {service} before this test")
username = "obj_" + uuid4().hex[:16]
password = secrets.token_hex(16)
user = api(
"POST", "/api/v1/users", 201,
payload={
"username": username,
"display_name": "After objects test",
"password": password,
},
)
basic = base64.b64encode(
f"{username}:{password}".encode()
).decode()
token = api(
"POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": "Basic " + basic},
)["token"]
chat = api(
"POST", "/api/v1/chats", 201,
token=token,
payload={
"title": "After objects",
"member_ids": [user["id"]],
},
)
restore = True
dc("stop", "worker")
if dc("ps", "--status", "running", "-q", "worker"):
raise AssertionError("Worker is still running")
original = (ROOT / "tests/fixtures/sample.png").read_bytes()
attachment = api(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png",
202,
token=token,
data=original,
headers={"Content-Type": "image/png"},
)
attachment_id = str(UUID(attachment["id"]))
path = f"/api/v1/attachments/{attachment_id}"
initial = snapshot(attachment_id)
job_id = str(UUID(initial["job_id"]))
sql(f"""
INSERT INTO pipeline_fault_injections (job_id, stage)
VALUES ('{job_id}'::uuid, 'after_objects');
""")
evidence["attachment_id"] = attachment_id
evidence["initial_job"] = initial
print(f"Armed one-shot after_objects fault for job {job_id}")
dc("start", "worker")
started = time.monotonic()
deadline = started + 45
while time.monotonic() < deadline:
middle = intermediate_state(job_id)
if middle["fault_consumed"]:
break
time.sleep(0.5)
else:
raise AssertionError("after_objects fault was not consumed")
assert middle["status"] == "processing"
assert middle["attempt"] == 1
assert middle["lease_remaining_seconds"] > 0
assert middle["variant_rows"] == 0
evidence["after_crash"] = middle
current = api("GET", path, 200, token=token)
assert current["status"] == "processing"
assert current["variants"] == []
api("GET", path + "/content", 409, token=token)
expected_keys = sorted([
f"originals/{attachment_id}",
f"variants/{attachment_id}/v1/thumbnail",
f"variants/{attachment_id}/v1/crop",
f"variants/{attachment_id}/v1/optimized",
])
# Во время автоматического рестарта docker exec может быть недоступен.
deadline = time.monotonic() + 25
while time.monotonic() < deadline:
try:
keys_before_recovery = object_keys(attachment_id)
break
except RuntimeError:
time.sleep(1)
else:
raise AssertionError("Could not inspect S3 after worker restart")
assert keys_before_recovery == expected_keys
# Подтверждаем, что осмотр S3 всё ещё был до восстановления lease.
confirmed_middle = intermediate_state(job_id)
assert confirmed_middle["status"] == "processing"
assert confirmed_middle["attempt"] == 1
assert confirmed_middle["lease_remaining_seconds"] > 0
assert confirmed_middle["variant_rows"] == 0
evidence["objects_before_recovery"] = keys_before_recovery
evidence["confirmed_intermediate_state"] = confirmed_middle
print(
"Fault consumed: processing, attempt=1, active lease, "
"S3 objects=4, variant rows=0, content=409"
)
print("Waiting for automatic lease recovery...")
deadline = started + 240
while time.monotonic() < deadline:
current = api("GET", path, 200, token=token)
if current["status"] == "ready":
break
if current["status"] == "failed":
raise AssertionError(
"Attachment failed: " + str(current["error_code"])
)
time.sleep(1)
else:
raise AssertionError("Lease recovery did not reach ready")
final = snapshot(attachment_id)
assert final["job_id"] == job_id
assert final["status"] == "ready"
assert final["attempt"] == 2
assert final["outbox_count"] == 2
assert current["error_code"] is None
status, body = http("GET", path + "/content", token=token)
assert status == 200 and body == original
variants = {item["name"]: item for item in current["variants"]}
assert len(current["variants"]) == 3
assert set(variants) == {"thumbnail", "crop", "optimized"}
for name, metadata in variants.items():
status, body = http(
"GET", path + "/content?variant=" + name,
token=token,
)
assert status == 200
assert len(body) == metadata["size_bytes"]
assert hashlib.sha256(body).hexdigest() == metadata["sha256"]
count = int(sql(f"""
SELECT count(*)
FROM attachment_variants
WHERE attachment_id = '{attachment_id}'::uuid;
"""))
assert count == 3
keys_after_recovery = object_keys(attachment_id)
assert keys_after_recovery == keys_before_recovery
logs = dc("logs", "--no-color", "--tail=400", "worker")
events = [
line for line in logs.splitlines()
if job_id in line
]
assert any(
"child_fault" in line and "stage=after_objects" in line
for line in events
)
assert any(
"status=queued" in line and "reason=lease_expired" in line
for line in events
)
evidence["final_job"] = final
evidence["variant_rows"] = count
evidence["objects_after_recovery"] = keys_after_recovery
evidence["worker_events"] = events
evidence["elapsed_s"] = round(time.monotonic() - started, 3)
evidence["result"] = "PASS"
except BaseException as exc:
evidence["result"] = "FAIL"
evidence["error_type"] = type(exc).__name__
raise
finally:
try:
if job_id is not None:
sql(f"""
UPDATE pipeline_fault_injections
SET consumed_at = clock_timestamp()
WHERE job_id = '{job_id}'::uuid
AND stage = 'after_objects'
AND consumed_at IS NULL;
""")
finally:
try:
if restore:
dc("start", "worker")
finally:
EVIDENCE.parent.mkdir(exist_ok=True)
EVIDENCE.write_text(
json.dumps(evidence, ensure_ascii=False, indent=2) + "\n"
)
print(
"PASS: crash after S3 writes recovered via expired lease; "
"same job, attempt=2, variants=3, objects=4"
)
if __name__ == "__main__":
main()