267 lines
8.7 KiB
Python
267 lines
8.7 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import secrets
|
|
import time
|
|
from uuid import UUID, uuid4
|
|
|
|
from queued_restart import ROOT, api, dc, http, wait_ready
|
|
from outbox_gap import snapshot
|
|
from before_ack import sql, object_keys
|
|
|
|
|
|
EVIDENCE = ROOT / "evidence" / "after_objects.json"
|
|
|
|
|
|
def intermediate_state(job_id):
|
|
return json.loads(sql(f"""
|
|
SELECT json_build_object(
|
|
'job_id', j.id,
|
|
'status', j.status,
|
|
'attempt', j.attempt,
|
|
'lease_remaining_seconds',
|
|
extract(epoch FROM (j.lease_until - clock_timestamp())),
|
|
'fault_consumed', f.consumed_at IS NOT NULL,
|
|
'variant_rows', (
|
|
SELECT count(*)
|
|
FROM attachment_variants v
|
|
WHERE v.attachment_id = j.attachment_id
|
|
)
|
|
)
|
|
FROM attachment_jobs j
|
|
JOIN pipeline_fault_injections f ON f.job_id = j.id
|
|
WHERE j.id = '{job_id}'::uuid
|
|
AND f.stage = 'after_objects';
|
|
"""))
|
|
|
|
|
|
def main():
|
|
evidence = {
|
|
"scenario": "worker crash after S3 writes, before result COMMIT",
|
|
}
|
|
job_id = None
|
|
restore = False
|
|
|
|
try:
|
|
wait_ready()
|
|
for service in ("publisher", "worker"):
|
|
if not dc("ps", "--status", "running", "-q", service):
|
|
raise AssertionError(f"Start {service} before this test")
|
|
|
|
username = "obj_" + uuid4().hex[:16]
|
|
password = secrets.token_hex(16)
|
|
user = api(
|
|
"POST", "/api/v1/users", 201,
|
|
payload={
|
|
"username": username,
|
|
"display_name": "After objects test",
|
|
"password": password,
|
|
},
|
|
)
|
|
basic = base64.b64encode(
|
|
f"{username}:{password}".encode()
|
|
).decode()
|
|
token = api(
|
|
"POST", "/api/v1/auth/sessions", 201,
|
|
headers={"Authorization": "Basic " + basic},
|
|
)["token"]
|
|
|
|
chat = api(
|
|
"POST", "/api/v1/chats", 201,
|
|
token=token,
|
|
payload={
|
|
"title": "After objects",
|
|
"member_ids": [user["id"]],
|
|
},
|
|
)
|
|
|
|
restore = True
|
|
dc("stop", "worker")
|
|
if dc("ps", "--status", "running", "-q", "worker"):
|
|
raise AssertionError("Worker is still running")
|
|
|
|
original = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
attachment = api(
|
|
"POST",
|
|
f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png",
|
|
202,
|
|
token=token,
|
|
data=original,
|
|
headers={"Content-Type": "image/png"},
|
|
)
|
|
|
|
attachment_id = str(UUID(attachment["id"]))
|
|
path = f"/api/v1/attachments/{attachment_id}"
|
|
initial = snapshot(attachment_id)
|
|
job_id = str(UUID(initial["job_id"]))
|
|
|
|
sql(f"""
|
|
INSERT INTO pipeline_fault_injections (job_id, stage)
|
|
VALUES ('{job_id}'::uuid, 'after_objects');
|
|
""")
|
|
|
|
evidence["attachment_id"] = attachment_id
|
|
evidence["initial_job"] = initial
|
|
print(f"Armed one-shot after_objects fault for job {job_id}")
|
|
|
|
dc("start", "worker")
|
|
started = time.monotonic()
|
|
|
|
deadline = started + 45
|
|
while time.monotonic() < deadline:
|
|
middle = intermediate_state(job_id)
|
|
if middle["fault_consumed"]:
|
|
break
|
|
time.sleep(0.5)
|
|
else:
|
|
raise AssertionError("after_objects fault was not consumed")
|
|
|
|
assert middle["status"] == "processing"
|
|
assert middle["attempt"] == 1
|
|
assert middle["lease_remaining_seconds"] > 0
|
|
assert middle["variant_rows"] == 0
|
|
evidence["after_crash"] = middle
|
|
|
|
current = api("GET", path, 200, token=token)
|
|
assert current["status"] == "processing"
|
|
assert current["variants"] == []
|
|
api("GET", path + "/content", 409, token=token)
|
|
|
|
expected_keys = sorted([
|
|
f"originals/{attachment_id}",
|
|
f"variants/{attachment_id}/v1/thumbnail",
|
|
f"variants/{attachment_id}/v1/crop",
|
|
f"variants/{attachment_id}/v1/optimized",
|
|
])
|
|
|
|
# Во время автоматического рестарта docker exec может быть недоступен.
|
|
deadline = time.monotonic() + 25
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
keys_before_recovery = object_keys(attachment_id)
|
|
break
|
|
except RuntimeError:
|
|
time.sleep(1)
|
|
else:
|
|
raise AssertionError("Could not inspect S3 after worker restart")
|
|
|
|
assert keys_before_recovery == expected_keys
|
|
|
|
# Подтверждаем, что осмотр S3 всё ещё был до восстановления lease.
|
|
confirmed_middle = intermediate_state(job_id)
|
|
assert confirmed_middle["status"] == "processing"
|
|
assert confirmed_middle["attempt"] == 1
|
|
assert confirmed_middle["lease_remaining_seconds"] > 0
|
|
assert confirmed_middle["variant_rows"] == 0
|
|
|
|
evidence["objects_before_recovery"] = keys_before_recovery
|
|
evidence["confirmed_intermediate_state"] = confirmed_middle
|
|
print(
|
|
"Fault consumed: processing, attempt=1, active lease, "
|
|
"S3 objects=4, variant rows=0, content=409"
|
|
)
|
|
print("Waiting for automatic lease recovery...")
|
|
|
|
deadline = started + 240
|
|
while time.monotonic() < deadline:
|
|
current = api("GET", path, 200, token=token)
|
|
if current["status"] == "ready":
|
|
break
|
|
if current["status"] == "failed":
|
|
raise AssertionError(
|
|
"Attachment failed: " + str(current["error_code"])
|
|
)
|
|
time.sleep(1)
|
|
else:
|
|
raise AssertionError("Lease recovery did not reach ready")
|
|
|
|
final = snapshot(attachment_id)
|
|
assert final["job_id"] == job_id
|
|
assert final["status"] == "ready"
|
|
assert final["attempt"] == 2
|
|
assert final["outbox_count"] == 2
|
|
assert current["error_code"] is None
|
|
|
|
status, body = http("GET", path + "/content", token=token)
|
|
assert status == 200 and body == original
|
|
|
|
variants = {item["name"]: item for item in current["variants"]}
|
|
assert len(current["variants"]) == 3
|
|
assert set(variants) == {"thumbnail", "crop", "optimized"}
|
|
|
|
for name, metadata in variants.items():
|
|
status, body = http(
|
|
"GET", path + "/content?variant=" + name,
|
|
token=token,
|
|
)
|
|
assert status == 200
|
|
assert len(body) == metadata["size_bytes"]
|
|
assert hashlib.sha256(body).hexdigest() == metadata["sha256"]
|
|
|
|
count = int(sql(f"""
|
|
SELECT count(*)
|
|
FROM attachment_variants
|
|
WHERE attachment_id = '{attachment_id}'::uuid;
|
|
"""))
|
|
assert count == 3
|
|
|
|
keys_after_recovery = object_keys(attachment_id)
|
|
assert keys_after_recovery == keys_before_recovery
|
|
|
|
logs = dc("logs", "--no-color", "--tail=400", "worker")
|
|
events = [
|
|
line for line in logs.splitlines()
|
|
if job_id in line
|
|
]
|
|
assert any(
|
|
"child_fault" in line and "stage=after_objects" in line
|
|
for line in events
|
|
)
|
|
assert any(
|
|
"status=queued" in line and "reason=lease_expired" in line
|
|
for line in events
|
|
)
|
|
|
|
evidence["final_job"] = final
|
|
evidence["variant_rows"] = count
|
|
evidence["objects_after_recovery"] = keys_after_recovery
|
|
evidence["worker_events"] = events
|
|
evidence["elapsed_s"] = round(time.monotonic() - started, 3)
|
|
evidence["result"] = "PASS"
|
|
|
|
except BaseException as exc:
|
|
evidence["result"] = "FAIL"
|
|
evidence["error_type"] = type(exc).__name__
|
|
raise
|
|
|
|
finally:
|
|
try:
|
|
if job_id is not None:
|
|
sql(f"""
|
|
UPDATE pipeline_fault_injections
|
|
SET consumed_at = clock_timestamp()
|
|
WHERE job_id = '{job_id}'::uuid
|
|
AND stage = 'after_objects'
|
|
AND consumed_at IS NULL;
|
|
""")
|
|
finally:
|
|
try:
|
|
if restore:
|
|
dc("start", "worker")
|
|
finally:
|
|
EVIDENCE.parent.mkdir(exist_ok=True)
|
|
EVIDENCE.write_text(
|
|
json.dumps(evidence, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
|
|
print(
|
|
"PASS: crash after S3 writes recovered via expired lease; "
|
|
"same job, attempt=2, variants=3, objects=4"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|