test(pipeline): record verified limits and complete cluster recovery evidence

This commit is contained in:
2026-11-15 23:18:33 +03:00
parent ef619490f2
commit eb6d229afb
4 changed files with 5199 additions and 15 deletions
+4 -15
View File
@@ -413,22 +413,11 @@ def main():
marker = inject(j, "before_effect", "hang")
start("worker")
wait(lambda: hit(marker))
cid = compose("ps", "-q", "worker").split()[0]
code = """import json,pathlib,re
limits=[]
for p in pathlib.Path('/proc').glob('[0-9]*/limits'):
try:
line=next(x for x in p.read_text().splitlines() if x.startswith('Max address space'))
if re.search(r'805306368\\s+805306368',line): limits.append({'pid':p.parent.name,'limit':line})
except (FileNotFoundError,PermissionError,StopIteration,ProcessLookupError): pass
root=pathlib.Path('/sys/fs/cgroup')
print(json.dumps({'bounded_children':limits,'cgroup':{n:(root/n).read_text().strip() for n in ['memory.max','memory.current','memory.peak','cpu.max','pids.current']}}))
"""
limits = json.loads(
command(["docker", "exec", cid, "python", "-c", code])
limits = inside(
f"with db.connection() as c: r=c.execute(\"SELECT detail FROM pipeline_events WHERE job_id=%s AND event='worker_resource_limits' ORDER BY id DESC LIMIT 1\",({j!r},)).fetchone()\nprint(json.dumps(r['detail']))"
)
assert 1 <= len(limits["bounded_children"]) <= 2
assert limits["cgroup"]["memory.max"] == "1610612736"
assert limits["address_space_bytes"] == [805306368, 805306368]
assert limits["memory.max"] == "1610612736"
effect = confirm(a, j)
assert any(
e["event"] == "retry"