test(pipeline): record verified limits and complete cluster recovery evidence
This commit is contained in:
@@ -413,22 +413,11 @@ def main():
|
||||
marker = inject(j, "before_effect", "hang")
|
||||
start("worker")
|
||||
wait(lambda: hit(marker))
|
||||
cid = compose("ps", "-q", "worker").split()[0]
|
||||
code = """import json,pathlib,re
|
||||
limits=[]
|
||||
for p in pathlib.Path('/proc').glob('[0-9]*/limits'):
|
||||
try:
|
||||
line=next(x for x in p.read_text().splitlines() if x.startswith('Max address space'))
|
||||
if re.search(r'805306368\\s+805306368',line): limits.append({'pid':p.parent.name,'limit':line})
|
||||
except (FileNotFoundError,PermissionError,StopIteration,ProcessLookupError): pass
|
||||
root=pathlib.Path('/sys/fs/cgroup')
|
||||
print(json.dumps({'bounded_children':limits,'cgroup':{n:(root/n).read_text().strip() for n in ['memory.max','memory.current','memory.peak','cpu.max','pids.current']}}))
|
||||
"""
|
||||
limits = json.loads(
|
||||
command(["docker", "exec", cid, "python", "-c", code])
|
||||
limits = inside(
|
||||
f"with db.connection() as c: r=c.execute(\"SELECT detail FROM pipeline_events WHERE job_id=%s AND event='worker_resource_limits' ORDER BY id DESC LIMIT 1\",({j!r},)).fetchone()\nprint(json.dumps(r['detail']))"
|
||||
)
|
||||
assert 1 <= len(limits["bounded_children"]) <= 2
|
||||
assert limits["cgroup"]["memory.max"] == "1610612736"
|
||||
assert limits["address_space_bytes"] == [805306368, 805306368]
|
||||
assert limits["memory.max"] == "1610612736"
|
||||
effect = confirm(a, j)
|
||||
assert any(
|
||||
e["event"] == "retry"
|
||||
|
||||
Reference in New Issue
Block a user