forked from mtusi2026/mtusi2026lab5
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
502 lines
28 KiB
Python
502 lines
28 KiB
Python
"""Lab 5 pipeline against the running stand: 202 -> queued -> ready, variants by the contract,
|
|
the failure windows of contracts/attachment-states.md, retries, the failed registry and replay,
|
|
lease expiry of a hung worker, cleanup that never deletes accepted work.
|
|
|
|
Failures are injected with the real mechanisms - docker stop/pause/restart of a service, and
|
|
FAULT_POINTS of ONE service recreated with an override (conftest.service_with_env) - and every
|
|
test restores the normal stand in `finally`. Each test checks exact HTTP codes AND the rows in
|
|
PostgreSQL AND the objects in S3, then the read-only invariant audit (I1-I7) of the whole stand."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from PIL import Image
|
|
|
|
import imagegen
|
|
from conftest import (attachment_row, audit, compose, events_of, job_rows, jobctl, psql, queue_depth, s3_keys,
|
|
s3_size, service_down, service_with_env, wait_healthy, wait_stand)
|
|
|
|
pytestmark = pytest.mark.stack
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
|
FAST_LEASE = {"JOB_LEASE_SECONDS": "6"} # the reaper notices a dead attempt within ~11 s
|
|
|
|
|
|
@pytest.fixture()
|
|
def pair(api):
|
|
alice, bob = api.user("alice"), api.user("bob")
|
|
return alice, bob, api.chat(alice, bob)
|
|
|
|
|
|
def wait_for(condition, timeout: float, what: str):
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
value = condition()
|
|
if value:
|
|
return value
|
|
assert time.monotonic() < deadline, what
|
|
time.sleep(0.5)
|
|
|
|
|
|
def check_variants(api, actor, att: dict, source: bytes) -> dict[str, bytes]:
|
|
"""Download every variant: real format and size decoded with Pillow, size/sha256 = metadata."""
|
|
expected = {"thumbnail": ("image/jpeg", "JPEG"), "crop": ("image/jpeg", "JPEG"), "optimized": ("image/webp", "WEBP")}
|
|
assert [v["name"] for v in att["variants"]] == list(expected)
|
|
src = Image.open(io.BytesIO(source))
|
|
w, h = src.size
|
|
out = {}
|
|
for v in att["variants"]:
|
|
got = api.content(actor, att["id"], v["name"])
|
|
assert got.headers["content-type"] == expected[v["name"]][0] == v["content_type"]
|
|
assert len(got.content) == v["size_bytes"] and hashlib.sha256(got.content).hexdigest() == v["sha256"]
|
|
img = Image.open(io.BytesIO(got.content))
|
|
assert img.format == expected[v["name"]][1] and img.size == (v["width"], v["height"])
|
|
out[v["name"]] = got.content
|
|
def fit(box):
|
|
if w <= box and h <= box:
|
|
return w, h
|
|
return (box, max(1, h * box // w)) if w >= h else (max(1, w * box // h), box)
|
|
sizes = {v["name"]: (v["width"], v["height"]) for v in att["variants"]}
|
|
assert sizes == {"thumbnail": fit(256), "crop": (128, 128), "optimized": fit(1280)}
|
|
original = api.content(actor, att["id"])
|
|
assert original.content == source # the original is published byte for byte
|
|
return out
|
|
|
|
|
|
def published_keys(attachment_id: str) -> list[str]:
|
|
return psql(f"SELECT object_key FROM attachment_variants WHERE attachment_id = '{attachment_id}' "
|
|
f"UNION ALL SELECT object_key FROM attachments WHERE id = '{attachment_id}' ORDER BY 1").split()
|
|
|
|
|
|
def attempts(attachment_id: str) -> list[dict]:
|
|
out = psql("SELECT coalesce(json_agg(json_build_object('attempt', a.attempt, 'outcome', a.outcome, "
|
|
"'error_code', a.error_code, 'token', a.token, 'cleaned', a.objects_cleaned_at IS NOT NULL, "
|
|
"'keys', a.planned_keys) ORDER BY a.started_at), '[]') FROM job_attempts a JOIN jobs j ON j.id = a.job_id "
|
|
f"WHERE j.attachment_id = '{attachment_id}'")
|
|
return json.loads(out)
|
|
|
|
|
|
def outbox_of(attachment_id: str) -> list[dict]:
|
|
out = psql("SELECT coalesce(json_agg(json_build_object('attempt', o.attempt, 'published', o.published_at IS NOT NULL) "
|
|
f"ORDER BY o.id), '[]') FROM outbox o JOIN jobs j ON j.id = o.job_id WHERE j.attachment_id = '{attachment_id}'")
|
|
return json.loads(out)
|
|
|
|
|
|
def names(events: list[dict]) -> list[str]:
|
|
return [e["event"] for e in events]
|
|
|
|
|
|
# ------------------------------------------------------------------ level 3: the happy path
|
|
def test_upload_is_202_queued_then_ready_with_contract_variants(api, pair):
|
|
alice, bob, chat = pair
|
|
data = imagegen.png(900, 600, noise=True)
|
|
response = api.upload(alice, chat, data)
|
|
assert response.status_code == 202
|
|
att = response.json()
|
|
assert att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None
|
|
ready = api.wait_status(bob, att["id"], "ready")
|
|
assert ready["error_code"] is None and ready["sha256"] == hashlib.sha256(data).hexdigest()
|
|
check_variants(api, bob, ready, data)
|
|
row = attachment_row(att["id"])
|
|
assert row["published_version"] == 1 and row["staging_state"] == "deleted" # staging cleaned after ready
|
|
assert s3_size(row["staging_key"], "staging") is None
|
|
[job] = job_rows(att["id"])
|
|
assert job["state"] == "succeeded" and job["attempt"] == 1 and job["kind"] == "upload"
|
|
assert names(events_of(att["id"])) == ["created", "claimed", "succeeded"]
|
|
|
|
|
|
def test_queued_attachment_is_attachable_failed_is_not(api, pair):
|
|
alice, bob, chat = pair
|
|
with service_down("worker"): # stays queued
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
msg = api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]}).json()
|
|
assert msg["attachment_ids"] == [att["id"]]
|
|
api.wait_status(alice, att["id"], "ready")
|
|
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"):
|
|
bad = api.upload(alice, chat, SAMPLE).json()
|
|
failed = api.wait_status(bob, bad["id"], "failed")
|
|
assert failed["error_code"] == "invalid_image" and failed["variants"] == []
|
|
api.post_message(alice, chat, {"text": "", "attachment_ids": [bad["id"]]}, 409)
|
|
api.content(bob, bad["id"], expected=409)
|
|
api.content(bob, bad["id"], "thumbnail", expected=409)
|
|
|
|
|
|
def test_worker_stopped_upload_202_content_409_restart_api_and_broker_then_ready(api, pair):
|
|
"""The defence scenario: worker stopped -> upload 202 -> content 409, other messages keep working
|
|
-> the message with the attachment is created -> API and broker restarted WITH volumes ->
|
|
the job is still in the queue -> worker started -> ready, thumbnail downloadable."""
|
|
alice, bob, chat = pair
|
|
compose("stop", "worker")
|
|
try:
|
|
response = api.upload(alice, chat, SAMPLE)
|
|
att = response.json()
|
|
assert response.status_code == 202 and att["status"] == "queued"
|
|
for variant in (None, "thumbnail"):
|
|
assert api.content(bob, att["id"], variant, expected=409).json()["error"]["code"] == "conflict"
|
|
api.send(bob, chat, "текст работает без worker")
|
|
msg = api.post_message(alice, chat, {"text": "фото", "attachment_ids": [att["id"]]}).json()
|
|
wait_for(lambda: all(o["published"] for o in outbox_of(att["id"])), 15, "relay did not publish the job")
|
|
# rabbitmqctl counters are refreshed every ~5 s (statistics interval): wait, do not sample once.
|
|
wait_for(lambda: queue_depth() >= 1, 20, "no message in the queue") # durable message waits in the queue
|
|
compose("restart", "api", "broker") # volumes preserved
|
|
wait_healthy("api", "broker", "relay")
|
|
wait_stand()
|
|
wait_for(lambda: queue_depth() >= 1, 20, "the accepted job was lost by the broker restart")
|
|
assert api.attachment(bob, att["id"])["status"] == "queued"
|
|
finally:
|
|
compose("start", "worker")
|
|
wait_healthy("worker")
|
|
ready = api.wait_status(bob, att["id"], "ready")
|
|
check_variants(api, bob, ready, SAMPLE)
|
|
items, _ = api.history(bob, chat, 50)
|
|
assert any(m["id"] == msg["id"] and m["attachment_ids"] == [att["id"]] for m in items)
|
|
[job] = job_rows(att["id"])
|
|
assert job["attempt"] == 1 # no duplicate work
|
|
|
|
|
|
# ------------------------------------------------------------------ failure windows
|
|
def test_broker_down_commit_to_publish_gap_is_closed_by_the_outbox(api, pair):
|
|
"""Window "after COMMIT, before publish": the broker is down, the API still answers 202 (it never
|
|
talks to the broker), the outbox row stays unpublished. No manual action: when the broker is
|
|
back the relay publishes it and the worker finishes the job."""
|
|
alice, bob, chat = pair
|
|
with service_down("broker"):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
assert att["status"] == "queued"
|
|
time.sleep(3)
|
|
assert outbox_of(att["id"]) == [{"attempt": 1, "published": False}]
|
|
api.send(alice, chat, "сообщения работают без брокера")
|
|
api.wait_status(bob, att["id"], "ready", timeout=90)
|
|
assert outbox_of(att["id"]) == [{"attempt": 1, "published": True}]
|
|
assert [j["attempt"] for j in job_rows(att["id"])] == [1]
|
|
|
|
|
|
def test_relay_down_job_waits_in_outbox_not_lost(api, pair):
|
|
alice, bob, chat = pair
|
|
with service_down("relay"):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
time.sleep(3)
|
|
assert outbox_of(att["id"]) == [{"attempt": 1, "published": False}]
|
|
assert api.attachment(bob, att["id"])["status"] == "queued"
|
|
api.wait_status(bob, att["id"], "ready")
|
|
|
|
|
|
def test_crash_after_publish_before_confirm_is_recorded_gives_a_harmless_duplicate(api, pair):
|
|
"""Window "after publish, before confirm": the relay dies after the broker confirmed the message
|
|
but before outbox.published_at is committed. The restarted relay publishes it AGAIN; the
|
|
duplicate is a no-op (one claim, one set of variants)."""
|
|
alice, bob, chat = pair
|
|
with service_with_env("relay", RELAY_FAULT_POINTS="crash_after_publish"):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
api.wait_status(bob, att["id"], "ready", timeout=90)
|
|
wait_for(lambda: all(o["published"] for o in outbox_of(att["id"])), 30, "outbox not marked after restart")
|
|
events = events_of(att["id"])
|
|
assert names(events).count("fault_injected") == 1 and names(events).count("claimed") == 1
|
|
assert [j["attempt"] for j in job_rows(att["id"])] == [1]
|
|
assert len(published_keys(att["id"])) == 4 # original + 3 variants, once
|
|
|
|
|
|
def test_duplicate_and_reordered_deliveries_are_noops(api, pair):
|
|
alice, bob, chat = pair
|
|
with service_with_env("relay", RELAY_FAULT_POINTS="duplicate_publish,reverse_order"):
|
|
atts = [api.upload(alice, chat, imagegen.png(50 + i, 40)).json() for i in range(4)]
|
|
for att in atts:
|
|
api.wait_status(bob, att["id"], "ready")
|
|
time.sleep(2) # let the duplicates be consumed
|
|
for att in atts:
|
|
assert names(events_of(att["id"])) == ["created", "claimed", "succeeded"]
|
|
assert len(published_keys(att["id"])) == 4
|
|
logs = compose("logs", "--no-log-prefix", "worker").stdout
|
|
assert '"noop_finished"' in logs or '"noop_leased"' in logs # duplicates were seen and acked
|
|
|
|
|
|
def test_worker_crash_before_side_effect_lease_expires_and_job_completes(api, pair):
|
|
"""The worker dies right after the claim. The redelivered message finds a valid lease (no-op);
|
|
the reaper returns the job to queued when the lease expires; attempt 2 publishes."""
|
|
alice, bob, chat = pair
|
|
with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_claim", **FAST_LEASE):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
api.wait_status(bob, att["id"], "ready", timeout=90)
|
|
ev = names(events_of(att["id"]))
|
|
assert ev.count("fault_injected") == 1 and "lease_expired" in ev and ev[-1] == "succeeded"
|
|
tries = attempts(att["id"])
|
|
assert [(a["attempt"], a["outcome"]) for a in tries] == [(1, "abandoned"), (2, "succeeded")]
|
|
assert tries[0]["keys"] and all(s3_size(k) is None for k in tries[0]["keys"]) # nothing written
|
|
|
|
|
|
def test_worker_crash_after_s3_write_before_commit_republishes_without_duplicates(api, pair):
|
|
"""Window "after the S3 write, before the result COMMIT": objects of attempt 1 exist under ITS keys
|
|
but are never published; attempt 2 publishes its own; the cleanup deletes attempt 1's objects
|
|
after ORPHAN_GRACE_SECONDS (they can never become published: the token is fenced)."""
|
|
alice, bob, chat = pair
|
|
with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_s3_write", ORPHAN_GRACE_SECONDS="2",
|
|
CLEANUP_INTERVAL_SECONDS="1", **FAST_LEASE):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
ready = api.wait_status(bob, att["id"], "ready", timeout=90)
|
|
first, second = attempts(att["id"])
|
|
assert (first["outcome"], second["outcome"]) == ("abandoned", "succeeded")
|
|
assert set(published_keys(att["id"])) == set(second["keys"])
|
|
wait_for(lambda: attempts(att["id"])[0]["cleaned"], 30, "orphans of attempt 1 not cleaned")
|
|
assert all(s3_size(k) is None for k in first["keys"])
|
|
assert all(s3_size(k) is not None for k in second["keys"])
|
|
assert s3_keys(f"a/{att['id']}/") == set(second["keys"]) # exactly one set of objects
|
|
check_variants(api, bob, ready, SAMPLE)
|
|
|
|
|
|
def test_worker_crash_after_commit_before_ack_redelivery_is_noop(api, pair):
|
|
"""Window "after the result COMMIT, before the ack": the message comes back; the job is
|
|
succeeded -> ack without any work; the same result stays published."""
|
|
alice, bob, chat = pair
|
|
with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_commit"):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
ready = api.wait_status(bob, att["id"], "ready", timeout=60)
|
|
before = published_keys(att["id"])
|
|
wait_healthy("worker")
|
|
wait_for(lambda: '"noop_finished"' in compose("logs", "--no-log-prefix", "worker").stdout, 30,
|
|
"the redelivered message was not acknowledged as a no-op")
|
|
ev = names(events_of(att["id"]))
|
|
assert ev == ["created", "claimed", "fault_injected", "succeeded"] or ev == ["created", "claimed", "succeeded", "fault_injected"]
|
|
assert published_keys(att["id"]) == before and [a["outcome"] for a in attempts(att["id"])] == ["succeeded"]
|
|
assert api.attachment(bob, att["id"]) == ready
|
|
|
|
|
|
# ------------------------------------------------------------------ retries and failures
|
|
def test_transient_errors_are_retried_with_jittered_backoff(api, pair):
|
|
alice, bob, chat = pair
|
|
with service_with_env("worker", WORKER_FAULT_POINTS="transient_error=2", JOB_RETRY_BASE_SECONDS="1",
|
|
JOB_RETRY_MAX_SECONDS="8"):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
api.wait_status(bob, att["id"], "ready", timeout=60)
|
|
events = events_of(att["id"])
|
|
retries = [e for e in events if e["event"] == "retry_scheduled"]
|
|
assert [(e["attempt"], e["error_code"]) for e in retries] == [(1, "injected_transient"), (2, "injected_transient")]
|
|
for e in retries:
|
|
assert 0 <= e["detail"]["delay_s"] <= 1 * 2 ** (e["attempt"] - 1) # full jitter window
|
|
assert [(a["attempt"], a["outcome"]) for a in attempts(att["id"])] == [(1, "retry"), (2, "retry"), (3, "succeeded")]
|
|
assert [o["attempt"] for o in outbox_of(att["id"])] == [1, 2, 3] # every retry went through the outbox
|
|
|
|
|
|
def test_retry_exhausted_failed_registry_then_replay(api, pair):
|
|
alice, bob, chat = pair
|
|
with service_with_env("worker", WORKER_FAULT_POINTS="transient_error", JOB_RETRY_BASE_SECONDS="0.2",
|
|
JOB_RETRY_MAX_SECONDS="0.5"):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
failed = api.wait_status(bob, att["id"], "failed", timeout=60)
|
|
assert failed["error_code"] == "retry_exhausted" and failed["variants"] == []
|
|
[job] = job_rows(att["id"])
|
|
assert (job["state"], job["attempt"], job["error_code"]) == ("failed", 5, "retry_exhausted") # JOB_MAX_ATTEMPTS
|
|
registry = jobctl("list-failed", "--limit", "200")["failed"]
|
|
assert any(r["attachment_id"] == att["id"] and r["error_code"] == "retry_exhausted" for r in registry)
|
|
assert attachment_row(att["id"])["staging_state"] == "present" # kept for the replay
|
|
# Operator replay: a NEW job (replay_of = the failed one), audit with operator and reason.
|
|
result = jobctl("replay", att["id"], "--operator", "alice", "--reason", "fault removed")
|
|
assert result["result"] == "queued" and result["replay_of"] == job["id"]
|
|
ready = api.wait_status(bob, att["id"], "ready", timeout=60)
|
|
check_variants(api, bob, ready, SAMPLE)
|
|
jobs = job_rows(att["id"])
|
|
assert [(j["kind"], j["state"]) for j in jobs] == [("upload", "failed"), ("replay", "succeeded")]
|
|
ev = events_of(att["id"])
|
|
req = next(e for e in ev if e["event"] == "replay_requested")
|
|
assert req["actor"] == "jobctl:alice" and req["detail"]["reason"] == "fault removed"
|
|
# "Repeat" never re-runs a finished job: a replay of a ready attachment is refused, nothing created.
|
|
refused = jobctl("replay", att["id"], "--operator", "alice", "--reason", "again", expect_code=3)
|
|
assert refused == {"result": "refused", "reason": "already_ready"}
|
|
assert len(job_rows(att["id"])) == 2
|
|
|
|
|
|
def test_permanent_error_fails_at_once_without_retries(api, pair):
|
|
alice, bob, chat = pair
|
|
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
failed = api.wait_status(bob, att["id"], "failed")
|
|
assert failed["error_code"] == "invalid_image"
|
|
assert [(a["attempt"], a["outcome"]) for a in attempts(att["id"])] == [(1, "failed")]
|
|
|
|
|
|
def test_memory_and_time_limits_of_the_worker(api, pair):
|
|
"""40 Mpx PNG (valid, accepted by the API) with IMAGE_MAX_MEMORY_MB=200: the child process cannot
|
|
allocate the raster -> image_too_large (permanent); the worker itself keeps running.
|
|
JOB_TIMEOUT_SECONDS=3 with a 10 s stall -> processing_timeout (transient, retried)."""
|
|
alice, bob, chat = pair
|
|
with service_with_env("worker", IMAGE_MAX_MEMORY_MB="200"):
|
|
att = api.upload(alice, chat, imagegen.png(8000, 5000, compress_level=9)).json()
|
|
failed = api.wait_status(bob, att["id"], "failed", timeout=90)
|
|
assert failed["error_code"] == "image_too_large"
|
|
ok = api.upload(alice, chat, SAMPLE).json() # the same worker still works
|
|
api.wait_status(bob, ok["id"], "ready")
|
|
with service_with_env("worker", WORKER_FAULT_POINTS="slow=10", JOB_TIMEOUT_SECONDS="3"):
|
|
slow = api.upload(alice, chat, SAMPLE).json()
|
|
wait_for(lambda: any(a["error_code"] == "processing_timeout" for a in attempts(slow["id"])), 60,
|
|
"no processing_timeout")
|
|
api.wait_status(bob, slow["id"], "ready", timeout=120) # normal worker finishes it
|
|
|
|
|
|
def test_bad_message_is_dead_lettered_not_processed(api, pair):
|
|
"""A message that does not match jobs.schema.json (published here with the relay's own broker
|
|
user) is rejected by the worker into the DLQ; nothing is processed."""
|
|
before = queue_depth("messenger.attachment-jobs.dlq")
|
|
script = ("import asyncio, aio_pika\n"
|
|
"async def m():\n"
|
|
" url = open('/run/secrets/broker_url').read().strip()\n"
|
|
" c = await aio_pika.connect(url)\n"
|
|
" ch = await c.channel(publisher_confirms=True)\n"
|
|
" ex = await ch.get_exchange('messenger.jobs', ensure=False)\n"
|
|
" await ex.publish(aio_pika.Message(b'{\"url\":\"http://evil/x.png\"}'), routing_key='attachment.process.v1')\n"
|
|
" await c.close()\n"
|
|
"asyncio.run(m())\n")
|
|
compose("exec", "-T", "relay", "python", "-c", script)
|
|
wait_for(lambda: queue_depth("messenger.attachment-jobs.dlq") == before + 1, 20, "not dead-lettered")
|
|
assert '"bad_message"' in compose("logs", "--no-log-prefix", "worker").stdout
|
|
|
|
|
|
# ------------------------------------------------------------------ lease, hung worker, shutdown, concurrency
|
|
def worker_containers() -> dict[str, str]:
|
|
"""hostname (= lease_owner) -> container id of every running worker."""
|
|
ids = compose("ps", "-q", "worker").stdout.split()
|
|
out = subprocess.run(["docker", "inspect", "-f", "{{.Config.Hostname}} {{.Id}}", *ids], capture_output=True,
|
|
text=True, check=True).stdout
|
|
return dict(line.split() for line in out.splitlines())
|
|
|
|
|
|
def test_hung_worker_loses_its_lease_and_cannot_publish(api, pair):
|
|
"""Two workers. The one processing the job is frozen (docker pause: no heartbeat). Its lease
|
|
expires, the reaper re-queues the job, the other worker publishes it. When the frozen worker
|
|
resumes, its result is fenced by the token: the published result does not change."""
|
|
alice, bob, chat = pair
|
|
with service_with_env("worker", scale=2, WORKER_FAULT_POINTS="slow=8", **FAST_LEASE):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
owner = wait_for(lambda: psql(f"SELECT lease_owner FROM jobs WHERE attachment_id = '{att['id']}' "
|
|
"AND state = 'processing'"), 30, "job never claimed")
|
|
frozen = worker_containers()[owner]
|
|
subprocess.run(["docker", "pause", frozen], check=True, capture_output=True)
|
|
try:
|
|
ready = api.wait_status(bob, att["id"], "ready", timeout=120)
|
|
published = published_keys(att["id"])
|
|
finally:
|
|
subprocess.run(["docker", "unpause", frozen], check=True, capture_output=True)
|
|
time.sleep(12) # the resumed attempt finishes its sleep and tries to publish
|
|
assert published_keys(att["id"]) == published and api.attachment(bob, att["id"]) == ready
|
|
tries = attempts(att["id"])
|
|
assert [(a["attempt"], a["outcome"]) for a in tries] == [(1, "abandoned"), (2, "succeeded")]
|
|
assert "lease_expired" in names(events_of(att["id"]))
|
|
check_variants(api, bob, ready, SAMPLE)
|
|
|
|
|
|
def test_graceful_worker_stop_finishes_the_running_attempt(api, pair):
|
|
alice, bob, chat = pair
|
|
with service_with_env("worker", WORKER_FAULT_POINTS="slow=5"):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
wait_for(lambda: api.attachment(bob, att["id"])["status"] == "processing", 30, "never processing")
|
|
started = time.monotonic()
|
|
compose("stop", "worker") # SIGTERM: drain, not kill
|
|
assert time.monotonic() - started < 30
|
|
assert attachment_row(att["id"])["status"] == "ready" # committed before the exit
|
|
compose("start", "worker")
|
|
wait_healthy("worker")
|
|
assert [a["outcome"] for a in attempts(att["id"])] == ["succeeded"]
|
|
|
|
|
|
def test_concurrency_is_bounded_by_prefetch(api, pair):
|
|
"""WORKER_CONCURRENCY=2: never more than 2 jobs processing and 2 unacked messages at once."""
|
|
alice, bob, chat = pair
|
|
with service_with_env("worker", WORKER_FAULT_POINTS="slow=3"):
|
|
atts = [api.upload(alice, chat, imagegen.png(30 + i, 30)).json() for i in range(6)]
|
|
peak = 0
|
|
deadline = time.monotonic() + 90
|
|
while time.monotonic() < deadline:
|
|
states = [api.attachment(bob, a["id"])["status"] for a in atts]
|
|
peak = max(peak, states.count("processing"))
|
|
unacked = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name",
|
|
"messages_unacknowledged").stdout
|
|
for line in unacked.splitlines():
|
|
if line.startswith("messenger.attachment-jobs\t"):
|
|
assert int(line.split("\t")[1]) <= 2, line
|
|
if all(s == "ready" for s in states):
|
|
break
|
|
time.sleep(0.3)
|
|
assert all(api.attachment(bob, a["id"])["status"] == "ready" for a in atts)
|
|
assert peak == 2
|
|
|
|
|
|
# ------------------------------------------------------------------ cleanup never deletes accepted work
|
|
def test_queued_staging_is_never_deleted_even_after_retention(api, pair):
|
|
"""Reviewer's case in another PR: the worker was down longer than the retention and the cleanup
|
|
deleted the original of a job that was still queued. Here: the job waits queued (relay down)
|
|
while the cleanup runs every second with retention 0 - the staging copy stays, and the
|
|
database itself refuses marking it (CHECK attachments_active_has_source)."""
|
|
alice, bob, chat = pair
|
|
with service_with_env("worker", STAGING_RETENTION_SECONDS="0", CLEANUP_INTERVAL_SECONDS="1"):
|
|
with service_down("relay"):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
time.sleep(5) # several cleanup passes
|
|
row = attachment_row(att["id"])
|
|
assert row["status"] == "queued" and row["staging_state"] == "present"
|
|
assert s3_size(row["staging_key"], "staging") == len(SAMPLE)
|
|
refused = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c",
|
|
f"UPDATE attachments SET staging_state = 'deleting' WHERE id = '{att['id']}'",
|
|
check=False)
|
|
assert refused.returncode != 0 and "attachments_active_has_source" in refused.stderr
|
|
ready = api.wait_status(bob, att["id"], "ready", timeout=60)
|
|
check_variants(api, bob, ready, SAMPLE)
|
|
|
|
|
|
def test_retention_deletes_failed_staging_then_replay_is_refused(api, pair):
|
|
alice, bob, chat = pair
|
|
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error", STAGING_RETENTION_SECONDS="1",
|
|
CLEANUP_INTERVAL_SECONDS="1"):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
api.wait_status(bob, att["id"], "failed")
|
|
wait_for(lambda: attachment_row(att["id"])["staging_state"] == "deleted", 30, "staging not cleaned")
|
|
assert s3_size(attachment_row(att["id"])["staging_key"], "staging") is None
|
|
refused = jobctl("replay", att["id"], "--operator", "alice", "--reason", "too late", expect_code=3)
|
|
assert refused == {"result": "refused", "reason": "source_missing"}
|
|
assert api.attachment(bob, att["id"])["status"] == "failed" and len(job_rows(att["id"])) == 1
|
|
|
|
|
|
def test_cleanup_racing_a_replay_never_deletes_the_replayed_source(api, pair):
|
|
"""Reviewer's interleaving in another PR: the cleanup READ its candidates (an expired failed
|
|
attachment), a replay committed queued, the cleanup went on and deleted the staging copy.
|
|
Here the cleanup pauses exactly there (pause_cleanup); the replay commits during the pause;
|
|
the cleanup then re-checks under the row lock and skips the row. The replay finishes ready."""
|
|
alice, bob, chat = pair
|
|
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
api.wait_status(bob, att["id"], "failed")
|
|
with service_with_env("worker", WORKER_FAULT_POINTS="pause_cleanup=8", STAGING_RETENTION_SECONDS="1",
|
|
CLEANUP_INTERVAL_SECONDS="1"):
|
|
wait_for(lambda: '"cleanup_paused"' in compose("logs", "--no-log-prefix", "--since", "60s", "worker").stdout
|
|
and att["id"] in compose("logs", "--no-log-prefix", "--since", "60s", "worker").stdout,
|
|
30, "cleanup did not pick the failed attachment")
|
|
result = jobctl("replay", att["id"], "--operator", "alice", "--reason", "race test")
|
|
assert result["result"] == "queued"
|
|
ready = api.wait_status(bob, att["id"], "ready", timeout=90)
|
|
check_variants(api, bob, ready, SAMPLE)
|
|
assert attachment_row(att["id"])["staging_state"] == "deleted" # deleted only AFTER ready
|
|
|
|
|
|
def test_reprocess_of_a_not_ready_upload_is_refused(api, pair):
|
|
"""Reviewer's case in another PR: worker stopped -> upload 202 -> reprocess with pipeline 2 ->
|
|
the attachment stayed queued forever. Here the command is refused and v1 completes."""
|
|
alice, bob, chat = pair
|
|
with service_down("worker"):
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
refused = jobctl("reprocess", att["id"], "--pipeline", "2", "--operator", "alice", "--reason", "x",
|
|
expect_code=3)
|
|
assert refused == {"result": "refused", "reason": "not_ready:queued"}
|
|
assert len(job_rows(att["id"])) == 1
|
|
api.wait_status(bob, att["id"], "ready")
|
|
assert attachment_row(att["id"])["published_version"] == 1
|
|
|
|
|
|
def test_zz_invariants_hold_after_all_scenarios():
|
|
report = audit("--verify-checksums")
|
|
assert report["objects"]["published"] > 0
|