fix(pipeline): reclaim pending entries after the live lease budget

This commit is contained in:
2026-11-15 23:47:51 +03:00
parent eb6d229afb
commit 442163b0f0
4 changed files with 8835 additions and 5 deletions
+16
View File
@@ -304,3 +304,19 @@ API failovers (writes/errors/recovery seconds): 214/6/3.086, 216/8/2.762.
Rolling: 730 confirmed writes, 18 transient errors; все исходныеAPI заменены, данные/replay сохранены.
Public failover/persistence PASS; StartedAt PG/cache/S3/broker/LB изменились,
прежний token и file сохранились. Это не zero-error rolling.
## Уточнение строгого окна COMMIT→ACK
Первый чистый повтор [clean-clone-first.json](evidence/clean-clone-first.json)
sourceeb6d229 прошёл112HTTP379.34s/22unit/19cases/cluster/cleanup/hashes.
Однако в after_commit snapshot уже был duplicate_decision: второйconsumer
XAUTOCLAIM после3s успевал ACK раньшеSIGKILL. Это безопасно для данных,
но не доказывает конкретное окно COMMIT-before-ACK. Первый receipt сохранён
как результат прежней версии проверок; строгая граница требует нового повтора.
Reclaim idle теперь равен lease(23s normal/15s test), превышая child timeout
20s/12s. Это предотвращает ранний захват чужого live entry. Новый тест
снимком XPENDING+XRANGE подтверждает конкретные job entries доSIGKILL и
отсутствие child_finished; после restart требует ready duplicate decision
и исчезновение notifications. Scope=all, данные и все прежние assertions
сохранены. Финальным доказательством станет clean-clone-final.json.
File diff suppressed because it is too large Load Diff
+19 -3
View File
@@ -86,14 +86,20 @@ def duplicate(job, count=5):
)
def inspect_effect(job):
def inspect_effect(job, *, pending=False):
return inside(f"""with db.connection() as c:
j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()
v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall()
p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n']
e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall()
keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")]
print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e}},default=str))""")
pending_entries=[]
if {pending!r}:
for item in queue.xpending_range(STREAM,GROUP,'-','+',1000):
entry=item['message_id']
messages=queue.xrange(STREAM,min=entry,max=entry)
if messages and json.loads(messages[0][1].get('job','{{}}')).get('job_id')==str(j['id']): pending_entries.append(entry)
print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e,'pending_stream_entries':pending_entries}},default=str))""")
def main():
@@ -366,7 +372,9 @@ def main():
marker = inject(j, phase, "wait")
start("worker")
wait(lambda marker=marker: hit(marker))
at_crash = inspect_effect(j)
at_crash = inspect_effect(
j, pending=phase == "after_commit"
)
assert at_crash["job"]["state"] == (
"ready"
if phase == "after_commit"
@@ -378,6 +386,14 @@ def main():
assert len(at_crash["physical_keys"]) == (
0 if phase == "before_effect" else 4
)
if phase == "after_commit":
assert at_crash["pending_stream_entries"], (
"Result must be committed while notification is still unacknowledged"
)
assert not any(
e["event"] == "child_finished"
for e in at_crash["events"]
)
compose("kill", "-s", "SIGKILL", "worker")
start("worker")
effect = confirm(a, j)
+4 -2
View File
@@ -315,7 +315,8 @@ def consume(slot):
context = multiprocessing.get_context("spawn")
cursor = "0-0"
timeout = integer("JOB_TIMEOUT_SECONDS", 20, 2, 117)
if integer("JOB_LEASE_SECONDS", 23, 3, 120) <= timeout:
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
if lease <= timeout:
raise ValueError("Lease must exceed child timeout")
try:
while True:
@@ -329,7 +330,8 @@ def consume(slot):
STREAM,
GROUP,
consumer,
min_idle_time=3000,
# Do not steal an entry from a consumer inside its live budget.
min_idle_time=lease * 1000,
start_id=cursor,
count=1,
)