formatted project by python-black
This commit is contained in:
+340
-122
@@ -16,7 +16,7 @@ from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compos
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, check
|
||||
|
||||
PROBE = '''
|
||||
PROBE = """
|
||||
import asyncio, hashlib, json
|
||||
from uuid import UUID
|
||||
from sqlalchemy import select
|
||||
@@ -43,169 +43,387 @@ async def main():
|
||||
result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
|
||||
finally: await storage.aclose();await engine.dispose()
|
||||
asyncio.run(main())
|
||||
'''
|
||||
"""
|
||||
|
||||
|
||||
def probe(attachment_id):
|
||||
code = 'ATTACHMENT_ID = ' + repr(str(UUID(attachment_id))) + '\n' + PROBE
|
||||
result = subprocess.run(COMPOSE + ['exec', '-T', 'api', 'python', '-'], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30)
|
||||
check(result.returncode == 0, 'Probe failed; diagnostic output hidden')
|
||||
code = "ATTACHMENT_ID = " + repr(str(UUID(attachment_id))) + "\n" + PROBE
|
||||
result = subprocess.run(
|
||||
COMPOSE + ["exec", "-T", "api", "python", "-"],
|
||||
input=code,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
cwd=ROOT,
|
||||
timeout=30,
|
||||
)
|
||||
check(result.returncode == 0, "Probe failed; diagnostic output hidden")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def admin(*arguments, expected=0):
|
||||
result = compose('run', '--rm', '--no-deps', 'worker', 'python', '-m', 'app.core.job_admin', *arguments)
|
||||
check(result.returncode == expected, 'Unexpected operator command exit status')
|
||||
result = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"app.core.job_admin",
|
||||
*arguments
|
||||
)
|
||||
check(result.returncode == expected, "Unexpected operator command exit status")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def verify_result(client, actor, attachment, original):
|
||||
ready = client.wait_ready(actor, attachment, wait_seconds=180)
|
||||
check(ready['id'] == attachment['id'], 'Attachment identity changed')
|
||||
path = '/api/v1/attachments/' + ready['id'] + '/content'
|
||||
check(client.call('GET', path, actor=actor)['bytes'] == original, 'Original changed')
|
||||
check({v['name'] for v in ready['variants']} == {'thumbnail','crop','optimized'}, 'Incomplete pipeline')
|
||||
for variant in ready['variants']:
|
||||
data = client.call('GET', path+'?variant='+variant['name'], actor=actor)['bytes']
|
||||
check(hashlib.sha256(data).hexdigest() == variant['sha256'], 'Checksum mismatch')
|
||||
check(ready["id"] == attachment["id"], "Attachment identity changed")
|
||||
path = "/api/v1/attachments/" + ready["id"] + "/content"
|
||||
check(
|
||||
client.call("GET", path, actor=actor)["bytes"] == original, "Original changed"
|
||||
)
|
||||
check(
|
||||
{v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"},
|
||||
"Incomplete pipeline",
|
||||
)
|
||||
for variant in ready["variants"]:
|
||||
data = client.call("GET", path + "?variant=" + variant["name"], actor=actor)[
|
||||
"bytes"
|
||||
]
|
||||
check(
|
||||
hashlib.sha256(data).hexdigest() == variant["sha256"], "Checksum mismatch"
|
||||
)
|
||||
with Image.open(BytesIO(data)) as image:
|
||||
check(image.size == (variant['width'],variant['height']), 'Metadata dimensions mismatch')
|
||||
check(image.format == ('WEBP' if variant['name']=='optimized' else 'JPEG'), 'Wrong format')
|
||||
check(not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp'), 'Metadata leaked')
|
||||
if variant['name']=='crop':check(image.size==(128,128),'Wrong crop size')
|
||||
snapshot=probe(ready['id'])
|
||||
check(len(snapshot['objects']) == 4, 'Expected exactly one original and three variants')
|
||||
check(
|
||||
image.size == (variant["width"], variant["height"]),
|
||||
"Metadata dimensions mismatch",
|
||||
)
|
||||
check(
|
||||
image.format == ("WEBP" if variant["name"] == "optimized" else "JPEG"),
|
||||
"Wrong format",
|
||||
)
|
||||
check(
|
||||
not image.getexif()
|
||||
and not image.info.get("icc_profile")
|
||||
and not image.info.get("xmp"),
|
||||
"Metadata leaked",
|
||||
)
|
||||
if variant["name"] == "crop":
|
||||
check(image.size == (128, 128), "Wrong crop size")
|
||||
snapshot = probe(ready["id"])
|
||||
check(
|
||||
len(snapshot["objects"]) == 4,
|
||||
"Expected exactly one original and three variants",
|
||||
)
|
||||
return snapshot
|
||||
|
||||
|
||||
def crash(client, actor, chat, mode):
|
||||
name='messenger-worker-fault-'+uuid4().hex
|
||||
name = "messenger-worker-fault-" + uuid4().hex
|
||||
try:
|
||||
require_compose('stop','worker')
|
||||
attachment,original=client.attachment(actor,chat)
|
||||
result=compose('run','--rm','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
|
||||
phase='after_object:optimized' if mode=='crash-after-object' else 'before_ack'
|
||||
marker=json.dumps({'fault':phase,'attachment_id':attachment['id']})
|
||||
check(result.returncode==137 and marker in result.stdout,'Targeted crash not observed')
|
||||
before=probe(attachment['id'])
|
||||
check(before['status']==('processing' if mode=='crash-after-object' else 'ready'),'Unexpected crash state')
|
||||
check(len(before['objects'])==4,'Crash must occur after all four PUTs')
|
||||
require_compose("stop", "worker")
|
||||
attachment, original = client.attachment(actor, chat)
|
||||
result = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"--name",
|
||||
name,
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"app.core.job_worker",
|
||||
"--" + mode,
|
||||
attachment["id"],
|
||||
)
|
||||
phase = (
|
||||
"after_object:optimized" if mode == "crash-after-object" else "before_ack"
|
||||
)
|
||||
marker = json.dumps({"fault": phase, "attachment_id": attachment["id"]})
|
||||
check(
|
||||
result.returncode == 137 and marker in result.stdout,
|
||||
"Targeted crash not observed",
|
||||
)
|
||||
before = probe(attachment["id"])
|
||||
check(
|
||||
before["status"]
|
||||
== ("processing" if mode == "crash-after-object" else "ready"),
|
||||
"Unexpected crash state",
|
||||
)
|
||||
check(len(before["objects"]) == 4, "Crash must occur after all four PUTs")
|
||||
finally:
|
||||
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
|
||||
require_compose('start','worker')
|
||||
after=verify_result(client,actor,attachment,original)
|
||||
if mode=='crash-before-ack':
|
||||
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
|
||||
require_compose("start", "worker")
|
||||
after = verify_result(client, actor, attachment, original)
|
||||
if mode == "crash-before-ack":
|
||||
# Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
|
||||
wait_queue_empty()
|
||||
after=probe(attachment['id'])
|
||||
check(after==before,'Ready duplicate rewrote result')
|
||||
after = probe(attachment["id"])
|
||||
check(after == before, "Ready duplicate rewrote result")
|
||||
else:
|
||||
check(after['attempt']==before['attempt']+1,'Lease did not recover the interrupted attempt')
|
||||
check([(x['name'],x['sha256']) for x in before['objects']]==[(x['name'],x['sha256']) for x in after['objects']],'Replay changed object content')
|
||||
return dict(scenario=mode,attachment_id=attachment['id'],exit_code=137,before=before,after=after)
|
||||
check(
|
||||
after["attempt"] == before["attempt"] + 1,
|
||||
"Lease did not recover the interrupted attempt",
|
||||
)
|
||||
check(
|
||||
[(x["name"], x["sha256"]) for x in before["objects"]]
|
||||
== [(x["name"], x["sha256"]) for x in after["objects"]],
|
||||
"Replay changed object content",
|
||||
)
|
||||
return dict(
|
||||
scenario=mode,
|
||||
attachment_id=attachment["id"],
|
||||
exit_code=137,
|
||||
before=before,
|
||||
after=after,
|
||||
)
|
||||
|
||||
|
||||
def retry_replay(client,actor,chat,permanent):
|
||||
name='messenger-retry-fault-'+uuid4().hex
|
||||
samples=[]
|
||||
configuration=ComposeProject().config['services']['worker']['environment']
|
||||
retry_base=float(configuration['JOB_RETRY_SECONDS'])
|
||||
retry_max=float(configuration['JOB_RETRY_MAX_SECONDS'])
|
||||
def retry_replay(client, actor, chat, permanent):
|
||||
name = "messenger-retry-fault-" + uuid4().hex
|
||||
samples = []
|
||||
configuration = ComposeProject().config["services"]["worker"]["environment"]
|
||||
retry_base = float(configuration["JOB_RETRY_SECONDS"])
|
||||
retry_max = float(configuration["JOB_RETRY_MAX_SECONDS"])
|
||||
try:
|
||||
require_compose('stop','worker')
|
||||
attachment,original=client.attachment(actor,chat)
|
||||
mode='fail-permanent' if permanent else 'fail-transient'
|
||||
require_compose('run','-d','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
|
||||
deadline=time.monotonic()+180
|
||||
while time.monotonic()<deadline:
|
||||
sample=probe(attachment['id'])
|
||||
if sample['attempt'] and sample['status'] in {'queued', 'failed'} and (not samples or samples[-1]['attempt']!=sample['attempt']):samples.append(sample)
|
||||
if sample['status']=='failed':break
|
||||
time.sleep(.5)
|
||||
check(sample['status']=='failed','Retry never reached failed')
|
||||
check(sample['attempt']==(1 if permanent else sample['limit']),'Incorrect attempt budget')
|
||||
check(sample['permanent']==permanent,'Incorrect error classification')
|
||||
require_compose("stop", "worker")
|
||||
attachment, original = client.attachment(actor, chat)
|
||||
mode = "fail-permanent" if permanent else "fail-transient"
|
||||
require_compose(
|
||||
"run",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--name",
|
||||
name,
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"app.core.job_worker",
|
||||
"--" + mode,
|
||||
attachment["id"],
|
||||
)
|
||||
deadline = time.monotonic() + 180
|
||||
while time.monotonic() < deadline:
|
||||
sample = probe(attachment["id"])
|
||||
if (
|
||||
sample["attempt"]
|
||||
and sample["status"] in {"queued", "failed"}
|
||||
and (not samples or samples[-1]["attempt"] != sample["attempt"])
|
||||
):
|
||||
samples.append(sample)
|
||||
if sample["status"] == "failed":
|
||||
break
|
||||
time.sleep(0.5)
|
||||
check(sample["status"] == "failed", "Retry never reached failed")
|
||||
check(
|
||||
sample["attempt"] == (1 if permanent else sample["limit"]),
|
||||
"Incorrect attempt budget",
|
||||
)
|
||||
check(sample["permanent"] == permanent, "Incorrect error classification")
|
||||
if not permanent:
|
||||
check(len(samples)==sample['limit'],'Missed retry samples')
|
||||
check(len(samples) == sample["limit"], "Missed retry samples")
|
||||
for entry in samples:
|
||||
delay=(datetime.fromisoformat(entry['next_attempt_at'])-datetime.fromisoformat(entry['updated_at'])).total_seconds()
|
||||
ceiling=min(retry_max,retry_base*2**(entry['attempt']-1))
|
||||
check(ceiling/2<=delay<=ceiling,'Retry delay outside configured jitter bounds')
|
||||
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
|
||||
job_id=sample['job_id'];request_id=str(uuid4())
|
||||
listing=admin('list-failed')
|
||||
check(any(row['job_id']==job_id for row in listing),'Job missing from failed registry')
|
||||
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30,check=True)
|
||||
first=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
|
||||
check(first['status']=='queued','Replay did not queue job')
|
||||
delay = (
|
||||
datetime.fromisoformat(entry["next_attempt_at"])
|
||||
- datetime.fromisoformat(entry["updated_at"])
|
||||
).total_seconds()
|
||||
ceiling = min(retry_max, retry_base * 2 ** (entry["attempt"] - 1))
|
||||
check(
|
||||
ceiling / 2 <= delay <= ceiling,
|
||||
"Retry delay outside configured jitter bounds",
|
||||
)
|
||||
client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=actor,
|
||||
expected=409,
|
||||
)
|
||||
job_id = sample["job_id"]
|
||||
request_id = str(uuid4())
|
||||
listing = admin("list-failed")
|
||||
check(
|
||||
any(row["job_id"] == job_id for row in listing),
|
||||
"Job missing from failed registry",
|
||||
)
|
||||
subprocess.run(
|
||||
["docker", "rm", "-f", name], capture_output=True, timeout=30, check=True
|
||||
)
|
||||
first = admin(
|
||||
"replay",
|
||||
job_id,
|
||||
"--request-id",
|
||||
request_id,
|
||||
"--actor",
|
||||
"verification",
|
||||
"--reason",
|
||||
"dependency_restored",
|
||||
)
|
||||
check(first["status"] == "queued", "Replay did not queue job")
|
||||
finally:
|
||||
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
|
||||
require_compose('start','worker')
|
||||
after=verify_result(client,actor,attachment,original)
|
||||
repeat=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
|
||||
check(repeat['already_applied'],'Replay request id not idempotent')
|
||||
refused=admin('replay',job_id,'--request-id',str(uuid4()),'--actor','verification','--reason','manual_review',expected=2)
|
||||
check(refused['error_code']=='job_not_failed','Successful job was replayed')
|
||||
check(probe(attachment['id'])==after,'Operator replay altered ready job')
|
||||
check(len(admin('history',job_id))==1,'Incorrect audit count')
|
||||
return dict(scenario=mode,attachment_id=attachment['id'],samples=samples,after=after,replay_idempotent=True)
|
||||
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
|
||||
require_compose("start", "worker")
|
||||
after = verify_result(client, actor, attachment, original)
|
||||
repeat = admin(
|
||||
"replay",
|
||||
job_id,
|
||||
"--request-id",
|
||||
request_id,
|
||||
"--actor",
|
||||
"verification",
|
||||
"--reason",
|
||||
"dependency_restored",
|
||||
)
|
||||
check(repeat["already_applied"], "Replay request id not idempotent")
|
||||
refused = admin(
|
||||
"replay",
|
||||
job_id,
|
||||
"--request-id",
|
||||
str(uuid4()),
|
||||
"--actor",
|
||||
"verification",
|
||||
"--reason",
|
||||
"manual_review",
|
||||
expected=2,
|
||||
)
|
||||
check(refused["error_code"] == "job_not_failed", "Successful job was replayed")
|
||||
check(probe(attachment["id"]) == after, "Operator replay altered ready job")
|
||||
check(len(admin("history", job_id)) == 1, "Incorrect audit count")
|
||||
return dict(
|
||||
scenario=mode,
|
||||
attachment_id=attachment["id"],
|
||||
samples=samples,
|
||||
after=after,
|
||||
replay_idempotent=True,
|
||||
)
|
||||
|
||||
|
||||
def queue_state():
|
||||
output=require_compose('exec','-T','broker','rabbitmqctl','list_queues','--formatter=json','name','messages_ready','messages_unacknowledged','consumers','durable')
|
||||
return next(row for row in json.loads(output) if row['name']=='attachment-jobs')
|
||||
output = require_compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"broker",
|
||||
"rabbitmqctl",
|
||||
"list_queues",
|
||||
"--formatter=json",
|
||||
"name",
|
||||
"messages_ready",
|
||||
"messages_unacknowledged",
|
||||
"consumers",
|
||||
"durable",
|
||||
)
|
||||
return next(row for row in json.loads(output) if row["name"] == "attachment-jobs")
|
||||
|
||||
|
||||
def wait_queue_empty():
|
||||
deadline=time.monotonic()+20
|
||||
while time.monotonic()<deadline:
|
||||
state=queue_state()
|
||||
if state['messages_ready']==0 and state['messages_unacknowledged']==0:
|
||||
deadline = time.monotonic() + 20
|
||||
while time.monotonic() < deadline:
|
||||
state = queue_state()
|
||||
if state["messages_ready"] == 0 and state["messages_unacknowledged"] == 0:
|
||||
return state
|
||||
time.sleep(.5)
|
||||
check(False,'RabbitMQ did not acknowledge all completed deliveries')
|
||||
time.sleep(0.5)
|
||||
check(False, "RabbitMQ did not acknowledge all completed deliveries")
|
||||
|
||||
|
||||
def persistence(client,actor,chat):
|
||||
def persistence(client, actor, chat):
|
||||
try:
|
||||
require_compose('stop','worker')
|
||||
attachment,original=client.attachment(actor,chat)
|
||||
client.message(actor,chat,'',attachments=[attachment['id']])
|
||||
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
|
||||
require_compose("stop", "worker")
|
||||
attachment, original = client.attachment(actor, chat)
|
||||
client.message(actor, chat, "", attachments=[attachment["id"]])
|
||||
client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=actor,
|
||||
expected=409,
|
||||
)
|
||||
time.sleep(2)
|
||||
queued_before=queue_state()
|
||||
check(queued_before['messages_ready']>=1 and queued_before['durable'], 'No durable queued message before restart')
|
||||
result=subprocess.run([str(ROOT/'scripts/restart.sh')],cwd=ROOT,capture_output=True,timeout=180)
|
||||
check(result.returncode==0,'Restart hook failed')
|
||||
check('worker' not in require_compose('ps','--status','running','--services').splitlines(),'Restart started deliberately stopped worker')
|
||||
queued_after=queue_state()
|
||||
check(queued_after['messages_ready']>=1 and queued_after['durable'], 'Broker lost queued delivery across restart')
|
||||
client.message(actor,chat,'session and chat survived broker/API restart')
|
||||
finally:require_compose('start','worker')
|
||||
after=verify_result(client,actor,attachment,original)
|
||||
return dict(scenario='worker_stopped_restart_all',attachment_id=attachment['id'],queue_before=queued_before,queue_after=queued_after,after=after)
|
||||
queued_before = queue_state()
|
||||
check(
|
||||
queued_before["messages_ready"] >= 1 and queued_before["durable"],
|
||||
"No durable queued message before restart",
|
||||
)
|
||||
result = subprocess.run(
|
||||
[str(ROOT / "scripts/restart.sh")],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
timeout=180,
|
||||
)
|
||||
check(result.returncode == 0, "Restart hook failed")
|
||||
check(
|
||||
"worker"
|
||||
not in require_compose(
|
||||
"ps", "--status", "running", "--services"
|
||||
).splitlines(),
|
||||
"Restart started deliberately stopped worker",
|
||||
)
|
||||
queued_after = queue_state()
|
||||
check(
|
||||
queued_after["messages_ready"] >= 1 and queued_after["durable"],
|
||||
"Broker lost queued delivery across restart",
|
||||
)
|
||||
client.message(actor, chat, "session and chat survived broker/API restart")
|
||||
finally:
|
||||
require_compose("start", "worker")
|
||||
after = verify_result(client, actor, attachment, original)
|
||||
return dict(
|
||||
scenario="worker_stopped_restart_all",
|
||||
attachment_id=attachment["id"],
|
||||
queue_before=queued_before,
|
||||
queue_after=queued_after,
|
||||
after=after,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url',required=True)
|
||||
parser.add_argument('--ca-file')
|
||||
parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all')
|
||||
parser.add_argument('--output',default='evidence/jobs.json')
|
||||
args=parser.parse_args()
|
||||
client=Client(args.base_url,args.ca_file,timeout=40)
|
||||
aliases={r.instance_id for r in ComposeProject().replicas() if r.running}
|
||||
check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL')
|
||||
actor,_,_,chat=client.bootstrap()
|
||||
results=[]
|
||||
operations={'persistence':persistence,'crash-after-object':lambda *x:crash(*x,'crash-after-object'),'crash-before-ack':lambda *x:crash(*x,'crash-before-ack'),'fail-transient':lambda *x:retry_replay(*x,False),'fail-permanent':lambda *x:retry_replay(*x,True)}
|
||||
selected=operations.values() if args.scenario=='all' else [operations[args.scenario]]
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--ca-file")
|
||||
parser.add_argument(
|
||||
"--scenario",
|
||||
choices=[
|
||||
"all",
|
||||
"persistence",
|
||||
"crash-after-object",
|
||||
"crash-before-ack",
|
||||
"fail-transient",
|
||||
"fail-permanent",
|
||||
],
|
||||
default="all",
|
||||
)
|
||||
parser.add_argument("--output", default="evidence/jobs.json")
|
||||
args = parser.parse_args()
|
||||
client = Client(args.base_url, args.ca_file, timeout=40)
|
||||
aliases = {r.instance_id for r in ComposeProject().replicas() if r.running}
|
||||
check(
|
||||
client.call("GET", "/health/live", schema="Health")["headers"].get(
|
||||
"X-Instance-Id"
|
||||
)
|
||||
in aliases,
|
||||
"Wrong Compose project for BASE_URL",
|
||||
)
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
results = []
|
||||
operations = {
|
||||
"persistence": persistence,
|
||||
"crash-after-object": lambda *x: crash(*x, "crash-after-object"),
|
||||
"crash-before-ack": lambda *x: crash(*x, "crash-before-ack"),
|
||||
"fail-transient": lambda *x: retry_replay(*x, False),
|
||||
"fail-permanent": lambda *x: retry_replay(*x, True),
|
||||
}
|
||||
selected = (
|
||||
operations.values() if args.scenario == "all" else [operations[args.scenario]]
|
||||
)
|
||||
for operation in selected:
|
||||
result=operation(client,actor,chat);results.append(result)
|
||||
Path(args.output).parent.mkdir(parents=True,exist_ok=True)
|
||||
Path(args.output).write_text(json.dumps(dict(status='in_progress',results=results),indent=2)+'\n')
|
||||
print('PASS: '+result['scenario'],flush=True)
|
||||
Path(args.output).write_text(json.dumps(dict(status='passed',results=results),indent=2)+'\n')
|
||||
result = operation(client, actor, chat)
|
||||
results.append(result)
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.output).write_text(
|
||||
json.dumps(dict(status="in_progress", results=results), indent=2) + "\n"
|
||||
)
|
||||
print("PASS: " + result["scenario"], flush=True)
|
||||
Path(args.output).write_text(
|
||||
json.dumps(dict(status="passed", results=results), indent=2) + "\n"
|
||||
)
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user