formatted project by python-black

This commit is contained in:
2026-11-10 09:52:42 +03:00
parent 7cd02da51d
commit 9899ee4676
66 changed files with 2517 additions and 944 deletions
+340 -122
View File
@@ -16,7 +16,7 @@ from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compos
from scripts.deployment import ComposeProject
from tests.client import Client, check
PROBE = '''
PROBE = """
import asyncio, hashlib, json
from uuid import UUID
from sqlalchemy import select
@@ -43,169 +43,387 @@ async def main():
result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
finally: await storage.aclose();await engine.dispose()
asyncio.run(main())
'''
"""
def probe(attachment_id):
code = 'ATTACHMENT_ID = ' + repr(str(UUID(attachment_id))) + '\n' + PROBE
result = subprocess.run(COMPOSE + ['exec', '-T', 'api', 'python', '-'], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30)
check(result.returncode == 0, 'Probe failed; diagnostic output hidden')
code = "ATTACHMENT_ID = " + repr(str(UUID(attachment_id))) + "\n" + PROBE
result = subprocess.run(
COMPOSE + ["exec", "-T", "api", "python", "-"],
input=code,
text=True,
capture_output=True,
cwd=ROOT,
timeout=30,
)
check(result.returncode == 0, "Probe failed; diagnostic output hidden")
return json.loads(result.stdout)
def admin(*arguments, expected=0):
result = compose('run', '--rm', '--no-deps', 'worker', 'python', '-m', 'app.core.job_admin', *arguments)
check(result.returncode == expected, 'Unexpected operator command exit status')
result = compose(
"run",
"--rm",
"--no-deps",
"worker",
"python",
"-m",
"app.core.job_admin",
*arguments
)
check(result.returncode == expected, "Unexpected operator command exit status")
return json.loads(result.stdout)
def verify_result(client, actor, attachment, original):
ready = client.wait_ready(actor, attachment, wait_seconds=180)
check(ready['id'] == attachment['id'], 'Attachment identity changed')
path = '/api/v1/attachments/' + ready['id'] + '/content'
check(client.call('GET', path, actor=actor)['bytes'] == original, 'Original changed')
check({v['name'] for v in ready['variants']} == {'thumbnail','crop','optimized'}, 'Incomplete pipeline')
for variant in ready['variants']:
data = client.call('GET', path+'?variant='+variant['name'], actor=actor)['bytes']
check(hashlib.sha256(data).hexdigest() == variant['sha256'], 'Checksum mismatch')
check(ready["id"] == attachment["id"], "Attachment identity changed")
path = "/api/v1/attachments/" + ready["id"] + "/content"
check(
client.call("GET", path, actor=actor)["bytes"] == original, "Original changed"
)
check(
{v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"},
"Incomplete pipeline",
)
for variant in ready["variants"]:
data = client.call("GET", path + "?variant=" + variant["name"], actor=actor)[
"bytes"
]
check(
hashlib.sha256(data).hexdigest() == variant["sha256"], "Checksum mismatch"
)
with Image.open(BytesIO(data)) as image:
check(image.size == (variant['width'],variant['height']), 'Metadata dimensions mismatch')
check(image.format == ('WEBP' if variant['name']=='optimized' else 'JPEG'), 'Wrong format')
check(not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp'), 'Metadata leaked')
if variant['name']=='crop':check(image.size==(128,128),'Wrong crop size')
snapshot=probe(ready['id'])
check(len(snapshot['objects']) == 4, 'Expected exactly one original and three variants')
check(
image.size == (variant["width"], variant["height"]),
"Metadata dimensions mismatch",
)
check(
image.format == ("WEBP" if variant["name"] == "optimized" else "JPEG"),
"Wrong format",
)
check(
not image.getexif()
and not image.info.get("icc_profile")
and not image.info.get("xmp"),
"Metadata leaked",
)
if variant["name"] == "crop":
check(image.size == (128, 128), "Wrong crop size")
snapshot = probe(ready["id"])
check(
len(snapshot["objects"]) == 4,
"Expected exactly one original and three variants",
)
return snapshot
def crash(client, actor, chat, mode):
name='messenger-worker-fault-'+uuid4().hex
name = "messenger-worker-fault-" + uuid4().hex
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
result=compose('run','--rm','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
phase='after_object:optimized' if mode=='crash-after-object' else 'before_ack'
marker=json.dumps({'fault':phase,'attachment_id':attachment['id']})
check(result.returncode==137 and marker in result.stdout,'Targeted crash not observed')
before=probe(attachment['id'])
check(before['status']==('processing' if mode=='crash-after-object' else 'ready'),'Unexpected crash state')
check(len(before['objects'])==4,'Crash must occur after all four PUTs')
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
result = compose(
"run",
"--rm",
"--no-deps",
"--name",
name,
"worker",
"python",
"-m",
"app.core.job_worker",
"--" + mode,
attachment["id"],
)
phase = (
"after_object:optimized" if mode == "crash-after-object" else "before_ack"
)
marker = json.dumps({"fault": phase, "attachment_id": attachment["id"]})
check(
result.returncode == 137 and marker in result.stdout,
"Targeted crash not observed",
)
before = probe(attachment["id"])
check(
before["status"]
== ("processing" if mode == "crash-after-object" else "ready"),
"Unexpected crash state",
)
check(len(before["objects"]) == 4, "Crash must occur after all four PUTs")
finally:
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
if mode=='crash-before-ack':
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
if mode == "crash-before-ack":
# Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
wait_queue_empty()
after=probe(attachment['id'])
check(after==before,'Ready duplicate rewrote result')
after = probe(attachment["id"])
check(after == before, "Ready duplicate rewrote result")
else:
check(after['attempt']==before['attempt']+1,'Lease did not recover the interrupted attempt')
check([(x['name'],x['sha256']) for x in before['objects']]==[(x['name'],x['sha256']) for x in after['objects']],'Replay changed object content')
return dict(scenario=mode,attachment_id=attachment['id'],exit_code=137,before=before,after=after)
check(
after["attempt"] == before["attempt"] + 1,
"Lease did not recover the interrupted attempt",
)
check(
[(x["name"], x["sha256"]) for x in before["objects"]]
== [(x["name"], x["sha256"]) for x in after["objects"]],
"Replay changed object content",
)
return dict(
scenario=mode,
attachment_id=attachment["id"],
exit_code=137,
before=before,
after=after,
)
def retry_replay(client,actor,chat,permanent):
name='messenger-retry-fault-'+uuid4().hex
samples=[]
configuration=ComposeProject().config['services']['worker']['environment']
retry_base=float(configuration['JOB_RETRY_SECONDS'])
retry_max=float(configuration['JOB_RETRY_MAX_SECONDS'])
def retry_replay(client, actor, chat, permanent):
name = "messenger-retry-fault-" + uuid4().hex
samples = []
configuration = ComposeProject().config["services"]["worker"]["environment"]
retry_base = float(configuration["JOB_RETRY_SECONDS"])
retry_max = float(configuration["JOB_RETRY_MAX_SECONDS"])
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
mode='fail-permanent' if permanent else 'fail-transient'
require_compose('run','-d','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
deadline=time.monotonic()+180
while time.monotonic()<deadline:
sample=probe(attachment['id'])
if sample['attempt'] and sample['status'] in {'queued', 'failed'} and (not samples or samples[-1]['attempt']!=sample['attempt']):samples.append(sample)
if sample['status']=='failed':break
time.sleep(.5)
check(sample['status']=='failed','Retry never reached failed')
check(sample['attempt']==(1 if permanent else sample['limit']),'Incorrect attempt budget')
check(sample['permanent']==permanent,'Incorrect error classification')
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
mode = "fail-permanent" if permanent else "fail-transient"
require_compose(
"run",
"-d",
"--no-deps",
"--name",
name,
"worker",
"python",
"-m",
"app.core.job_worker",
"--" + mode,
attachment["id"],
)
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
sample = probe(attachment["id"])
if (
sample["attempt"]
and sample["status"] in {"queued", "failed"}
and (not samples or samples[-1]["attempt"] != sample["attempt"])
):
samples.append(sample)
if sample["status"] == "failed":
break
time.sleep(0.5)
check(sample["status"] == "failed", "Retry never reached failed")
check(
sample["attempt"] == (1 if permanent else sample["limit"]),
"Incorrect attempt budget",
)
check(sample["permanent"] == permanent, "Incorrect error classification")
if not permanent:
check(len(samples)==sample['limit'],'Missed retry samples')
check(len(samples) == sample["limit"], "Missed retry samples")
for entry in samples:
delay=(datetime.fromisoformat(entry['next_attempt_at'])-datetime.fromisoformat(entry['updated_at'])).total_seconds()
ceiling=min(retry_max,retry_base*2**(entry['attempt']-1))
check(ceiling/2<=delay<=ceiling,'Retry delay outside configured jitter bounds')
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
job_id=sample['job_id'];request_id=str(uuid4())
listing=admin('list-failed')
check(any(row['job_id']==job_id for row in listing),'Job missing from failed registry')
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30,check=True)
first=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
check(first['status']=='queued','Replay did not queue job')
delay = (
datetime.fromisoformat(entry["next_attempt_at"])
- datetime.fromisoformat(entry["updated_at"])
).total_seconds()
ceiling = min(retry_max, retry_base * 2 ** (entry["attempt"] - 1))
check(
ceiling / 2 <= delay <= ceiling,
"Retry delay outside configured jitter bounds",
)
client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=actor,
expected=409,
)
job_id = sample["job_id"]
request_id = str(uuid4())
listing = admin("list-failed")
check(
any(row["job_id"] == job_id for row in listing),
"Job missing from failed registry",
)
subprocess.run(
["docker", "rm", "-f", name], capture_output=True, timeout=30, check=True
)
first = admin(
"replay",
job_id,
"--request-id",
request_id,
"--actor",
"verification",
"--reason",
"dependency_restored",
)
check(first["status"] == "queued", "Replay did not queue job")
finally:
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
repeat=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
check(repeat['already_applied'],'Replay request id not idempotent')
refused=admin('replay',job_id,'--request-id',str(uuid4()),'--actor','verification','--reason','manual_review',expected=2)
check(refused['error_code']=='job_not_failed','Successful job was replayed')
check(probe(attachment['id'])==after,'Operator replay altered ready job')
check(len(admin('history',job_id))==1,'Incorrect audit count')
return dict(scenario=mode,attachment_id=attachment['id'],samples=samples,after=after,replay_idempotent=True)
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
repeat = admin(
"replay",
job_id,
"--request-id",
request_id,
"--actor",
"verification",
"--reason",
"dependency_restored",
)
check(repeat["already_applied"], "Replay request id not idempotent")
refused = admin(
"replay",
job_id,
"--request-id",
str(uuid4()),
"--actor",
"verification",
"--reason",
"manual_review",
expected=2,
)
check(refused["error_code"] == "job_not_failed", "Successful job was replayed")
check(probe(attachment["id"]) == after, "Operator replay altered ready job")
check(len(admin("history", job_id)) == 1, "Incorrect audit count")
return dict(
scenario=mode,
attachment_id=attachment["id"],
samples=samples,
after=after,
replay_idempotent=True,
)
def queue_state():
output=require_compose('exec','-T','broker','rabbitmqctl','list_queues','--formatter=json','name','messages_ready','messages_unacknowledged','consumers','durable')
return next(row for row in json.loads(output) if row['name']=='attachment-jobs')
output = require_compose(
"exec",
"-T",
"broker",
"rabbitmqctl",
"list_queues",
"--formatter=json",
"name",
"messages_ready",
"messages_unacknowledged",
"consumers",
"durable",
)
return next(row for row in json.loads(output) if row["name"] == "attachment-jobs")
def wait_queue_empty():
deadline=time.monotonic()+20
while time.monotonic()<deadline:
state=queue_state()
if state['messages_ready']==0 and state['messages_unacknowledged']==0:
deadline = time.monotonic() + 20
while time.monotonic() < deadline:
state = queue_state()
if state["messages_ready"] == 0 and state["messages_unacknowledged"] == 0:
return state
time.sleep(.5)
check(False,'RabbitMQ did not acknowledge all completed deliveries')
time.sleep(0.5)
check(False, "RabbitMQ did not acknowledge all completed deliveries")
def persistence(client,actor,chat):
def persistence(client, actor, chat):
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
client.message(actor,chat,'',attachments=[attachment['id']])
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
client.message(actor, chat, "", attachments=[attachment["id"]])
client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=actor,
expected=409,
)
time.sleep(2)
queued_before=queue_state()
check(queued_before['messages_ready']>=1 and queued_before['durable'], 'No durable queued message before restart')
result=subprocess.run([str(ROOT/'scripts/restart.sh')],cwd=ROOT,capture_output=True,timeout=180)
check(result.returncode==0,'Restart hook failed')
check('worker' not in require_compose('ps','--status','running','--services').splitlines(),'Restart started deliberately stopped worker')
queued_after=queue_state()
check(queued_after['messages_ready']>=1 and queued_after['durable'], 'Broker lost queued delivery across restart')
client.message(actor,chat,'session and chat survived broker/API restart')
finally:require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
return dict(scenario='worker_stopped_restart_all',attachment_id=attachment['id'],queue_before=queued_before,queue_after=queued_after,after=after)
queued_before = queue_state()
check(
queued_before["messages_ready"] >= 1 and queued_before["durable"],
"No durable queued message before restart",
)
result = subprocess.run(
[str(ROOT / "scripts/restart.sh")],
cwd=ROOT,
capture_output=True,
timeout=180,
)
check(result.returncode == 0, "Restart hook failed")
check(
"worker"
not in require_compose(
"ps", "--status", "running", "--services"
).splitlines(),
"Restart started deliberately stopped worker",
)
queued_after = queue_state()
check(
queued_after["messages_ready"] >= 1 and queued_after["durable"],
"Broker lost queued delivery across restart",
)
client.message(actor, chat, "session and chat survived broker/API restart")
finally:
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
return dict(
scenario="worker_stopped_restart_all",
attachment_id=attachment["id"],
queue_before=queued_before,
queue_after=queued_after,
after=after,
)
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--base-url',required=True)
parser.add_argument('--ca-file')
parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all')
parser.add_argument('--output',default='evidence/jobs.json')
args=parser.parse_args()
client=Client(args.base_url,args.ca_file,timeout=40)
aliases={r.instance_id for r in ComposeProject().replicas() if r.running}
check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL')
actor,_,_,chat=client.bootstrap()
results=[]
operations={'persistence':persistence,'crash-after-object':lambda *x:crash(*x,'crash-after-object'),'crash-before-ack':lambda *x:crash(*x,'crash-before-ack'),'fail-transient':lambda *x:retry_replay(*x,False),'fail-permanent':lambda *x:retry_replay(*x,True)}
selected=operations.values() if args.scenario=='all' else [operations[args.scenario]]
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--ca-file")
parser.add_argument(
"--scenario",
choices=[
"all",
"persistence",
"crash-after-object",
"crash-before-ack",
"fail-transient",
"fail-permanent",
],
default="all",
)
parser.add_argument("--output", default="evidence/jobs.json")
args = parser.parse_args()
client = Client(args.base_url, args.ca_file, timeout=40)
aliases = {r.instance_id for r in ComposeProject().replicas() if r.running}
check(
client.call("GET", "/health/live", schema="Health")["headers"].get(
"X-Instance-Id"
)
in aliases,
"Wrong Compose project for BASE_URL",
)
actor, _, _, chat = client.bootstrap()
results = []
operations = {
"persistence": persistence,
"crash-after-object": lambda *x: crash(*x, "crash-after-object"),
"crash-before-ack": lambda *x: crash(*x, "crash-before-ack"),
"fail-transient": lambda *x: retry_replay(*x, False),
"fail-permanent": lambda *x: retry_replay(*x, True),
}
selected = (
operations.values() if args.scenario == "all" else [operations[args.scenario]]
)
for operation in selected:
result=operation(client,actor,chat);results.append(result)
Path(args.output).parent.mkdir(parents=True,exist_ok=True)
Path(args.output).write_text(json.dumps(dict(status='in_progress',results=results),indent=2)+'\n')
print('PASS: '+result['scenario'],flush=True)
Path(args.output).write_text(json.dumps(dict(status='passed',results=results),indent=2)+'\n')
result = operation(client, actor, chat)
results.append(result)
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text(
json.dumps(dict(status="in_progress", results=results), indent=2) + "\n"
)
print("PASS: " + result["scenario"], flush=True)
Path(args.output).write_text(
json.dumps(dict(status="passed", results=results), indent=2) + "\n"
)
if __name__=='__main__':main()
if __name__ == "__main__":
main()