fix(pipeline): reclaim pending entries after the live lease budget
This commit is contained in:
@@ -304,3 +304,19 @@ API failovers (writes/errors/recovery seconds): 214/6/3.086, 216/8/2.762.
|
||||
Rolling: 730 confirmed writes, 18 transient errors; все исходныеAPI заменены, данные/replay сохранены.
|
||||
Public failover/persistence PASS; StartedAt PG/cache/S3/broker/LB изменились,
|
||||
прежний token и file сохранились. Это не zero-error rolling.
|
||||
|
||||
## Уточнение строгого окна COMMIT→ACK
|
||||
|
||||
Первый чистый повтор [clean-clone-first.json](evidence/clean-clone-first.json)
|
||||
sourceeb6d229 прошёл112HTTP379.34s/22unit/19cases/cluster/cleanup/hashes.
|
||||
Однако в after_commit snapshot уже был duplicate_decision: второйconsumer
|
||||
XAUTOCLAIM после3s успевал ACK раньшеSIGKILL. Это безопасно для данных,
|
||||
но не доказывает конкретное окно COMMIT-before-ACK. Первый receipt сохранён
|
||||
как результат прежней версии проверок; строгая граница требует нового повтора.
|
||||
|
||||
Reclaim idle теперь равен lease(23s normal/15s test), превышая child timeout
|
||||
20s/12s. Это предотвращает ранний захват чужого live entry. Новый тест
|
||||
снимком XPENDING+XRANGE подтверждает конкретные job entries доSIGKILL и
|
||||
отсутствие child_finished; после restart требует ready duplicate decision
|
||||
и исчезновение notifications. Scope=all, данные и все прежние assertions
|
||||
сохранены. Финальным доказательством станет clean-clone-final.json.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -86,14 +86,20 @@ def duplicate(job, count=5):
|
||||
)
|
||||
|
||||
|
||||
def inspect_effect(job):
|
||||
def inspect_effect(job, *, pending=False):
|
||||
return inside(f"""with db.connection() as c:
|
||||
j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()
|
||||
v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall()
|
||||
p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n']
|
||||
e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall()
|
||||
keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")]
|
||||
print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e}},default=str))""")
|
||||
pending_entries=[]
|
||||
if {pending!r}:
|
||||
for item in queue.xpending_range(STREAM,GROUP,'-','+',1000):
|
||||
entry=item['message_id']
|
||||
messages=queue.xrange(STREAM,min=entry,max=entry)
|
||||
if messages and json.loads(messages[0][1].get('job','{{}}')).get('job_id')==str(j['id']): pending_entries.append(entry)
|
||||
print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e,'pending_stream_entries':pending_entries}},default=str))""")
|
||||
|
||||
|
||||
def main():
|
||||
@@ -366,7 +372,9 @@ def main():
|
||||
marker = inject(j, phase, "wait")
|
||||
start("worker")
|
||||
wait(lambda marker=marker: hit(marker))
|
||||
at_crash = inspect_effect(j)
|
||||
at_crash = inspect_effect(
|
||||
j, pending=phase == "after_commit"
|
||||
)
|
||||
assert at_crash["job"]["state"] == (
|
||||
"ready"
|
||||
if phase == "after_commit"
|
||||
@@ -378,6 +386,14 @@ def main():
|
||||
assert len(at_crash["physical_keys"]) == (
|
||||
0 if phase == "before_effect" else 4
|
||||
)
|
||||
if phase == "after_commit":
|
||||
assert at_crash["pending_stream_entries"], (
|
||||
"Result must be committed while notification is still unacknowledged"
|
||||
)
|
||||
assert not any(
|
||||
e["event"] == "child_finished"
|
||||
for e in at_crash["events"]
|
||||
)
|
||||
compose("kill", "-s", "SIGKILL", "worker")
|
||||
start("worker")
|
||||
effect = confirm(a, j)
|
||||
|
||||
+4
-2
@@ -315,7 +315,8 @@ def consume(slot):
|
||||
context = multiprocessing.get_context("spawn")
|
||||
cursor = "0-0"
|
||||
timeout = integer("JOB_TIMEOUT_SECONDS", 20, 2, 117)
|
||||
if integer("JOB_LEASE_SECONDS", 23, 3, 120) <= timeout:
|
||||
lease = integer("JOB_LEASE_SECONDS", 23, 3, 120)
|
||||
if lease <= timeout:
|
||||
raise ValueError("Lease must exceed child timeout")
|
||||
try:
|
||||
while True:
|
||||
@@ -329,7 +330,8 @@ def consume(slot):
|
||||
STREAM,
|
||||
GROUP,
|
||||
consumer,
|
||||
min_idle_time=3000,
|
||||
# Do not steal an entry from a consumer inside its live budget.
|
||||
min_idle_time=lease * 1000,
|
||||
start_id=cursor,
|
||||
count=1,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user