formatted project by python-black

This commit is contained in:
2026-11-10 09:52:42 +03:00
parent 7cd02da51d
commit 9899ee4676
66 changed files with 2517 additions and 944 deletions
+56 -32
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Offline structural checks; full schema validation is a separate target."""
import ast
import json
from pathlib import Path
@@ -7,48 +8,71 @@ import re
ROOT = Path(__file__).resolve().parents[1]
def main():
lab = json.loads((ROOT/'lab.json').read_text())['number']
spec = json.loads((ROOT/'contracts/openapi.json').read_text())
assert spec['openapi']=='3.1.0'
assert spec['info']['version']=='1.%d.0'%lab
lab = json.loads((ROOT / "lab.json").read_text())["number"]
spec = json.loads((ROOT / "contracts/openapi.json").read_text())
assert spec["openapi"] == "3.1.0"
assert spec["info"]["version"] == "1.%d.0" % lab
operation_ids = set()
def walk(value):
if isinstance(value,dict):
if '$ref' in value:
pointer = value['$ref']
assert pointer.startswith('#/'), 'unexpected external reference'
if isinstance(value, dict):
if "$ref" in value:
pointer = value["$ref"]
assert pointer.startswith("#/"), "unexpected external reference"
target = spec
for part in pointer[2:].split('/'):
target = target[part.replace('~1','/').replace('~0','~')]
for part in pointer[2:].split("/"):
target = target[part.replace("~1", "/").replace("~0", "~")]
for v in value.values():
walk(v)
elif isinstance(value,list):
elif isinstance(value, list):
for v in value:
walk(v)
walk(spec)
for route,methods in spec['paths'].items():
placeholders = set(re.findall(r'\{([^}]+)\}',route))
for method,op in methods.items():
assert op['operationId'] not in operation_ids, 'duplicate operationId'
operation_ids.add(op['operationId'])
params = {p['name'] for p in op.get('parameters',[]) if p['in']=='path' and p.get('required')}
assert placeholders==params, 'path parameter mismatch: '+route
for response in op['responses'].values():
assert 'X-Request-Id' in response['headers']
if lab>=3:
assert 'X-Instance-Id' in response['headers']
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
ast.parse(file.read_text(),filename=str(file))
for file in ROOT.rglob('*.json'):
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
for route, methods in spec["paths"].items():
placeholders = set(re.findall(r"\{([^}]+)\}", route))
for method, op in methods.items():
assert op["operationId"] not in operation_ids, "duplicate operationId"
operation_ids.add(op["operationId"])
params = {
p["name"]
for p in op.get("parameters", [])
if p["in"] == "path" and p.get("required")
}
assert placeholders == params, "path parameter mismatch: " + route
for response in op["responses"].values():
assert "X-Request-Id" in response["headers"]
if lab >= 3:
assert "X-Instance-Id" in response["headers"]
assert ("/api/v1/auth/sessions" in spec["paths"]) == (lab >= 2)
assert ("/api/v1/attachments/{attachment_id}" in spec["paths"]) == (lab >= 4)
for file in list((ROOT / "tests").glob("*.py")) + list(
(ROOT / "scripts").glob("*.py")
):
ast.parse(file.read_text(), filename=str(file))
for file in ROOT.rglob("*.json"):
if any(
part in {".git", ".venv", "node_modules", "vendor", "output"}
for part in file.parts
):
continue
json.loads(file.read_text())
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
assert (ROOT/required).is_file(), 'missing '+required
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
for required in (
"README.md",
"COURSE.md",
"CONTRIBUTING.md",
"REPORT.md",
"contracts/README.md",
"tests/smoke.py",
"tests/README.md",
):
assert (ROOT / required).is_file(), "missing " + required
print(
"PASS: offline skeleton structure, local refs and Python syntax; application NOT tested."
)
if __name__=='__main__':
if __name__ == "__main__":
main()
+340 -122
View File
@@ -16,7 +16,7 @@ from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compos
from scripts.deployment import ComposeProject
from tests.client import Client, check
PROBE = '''
PROBE = """
import asyncio, hashlib, json
from uuid import UUID
from sqlalchemy import select
@@ -43,169 +43,387 @@ async def main():
result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
finally: await storage.aclose();await engine.dispose()
asyncio.run(main())
'''
"""
def probe(attachment_id):
code = 'ATTACHMENT_ID = ' + repr(str(UUID(attachment_id))) + '\n' + PROBE
result = subprocess.run(COMPOSE + ['exec', '-T', 'api', 'python', '-'], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30)
check(result.returncode == 0, 'Probe failed; diagnostic output hidden')
code = "ATTACHMENT_ID = " + repr(str(UUID(attachment_id))) + "\n" + PROBE
result = subprocess.run(
COMPOSE + ["exec", "-T", "api", "python", "-"],
input=code,
text=True,
capture_output=True,
cwd=ROOT,
timeout=30,
)
check(result.returncode == 0, "Probe failed; diagnostic output hidden")
return json.loads(result.stdout)
def admin(*arguments, expected=0):
result = compose('run', '--rm', '--no-deps', 'worker', 'python', '-m', 'app.core.job_admin', *arguments)
check(result.returncode == expected, 'Unexpected operator command exit status')
result = compose(
"run",
"--rm",
"--no-deps",
"worker",
"python",
"-m",
"app.core.job_admin",
*arguments
)
check(result.returncode == expected, "Unexpected operator command exit status")
return json.loads(result.stdout)
def verify_result(client, actor, attachment, original):
ready = client.wait_ready(actor, attachment, wait_seconds=180)
check(ready['id'] == attachment['id'], 'Attachment identity changed')
path = '/api/v1/attachments/' + ready['id'] + '/content'
check(client.call('GET', path, actor=actor)['bytes'] == original, 'Original changed')
check({v['name'] for v in ready['variants']} == {'thumbnail','crop','optimized'}, 'Incomplete pipeline')
for variant in ready['variants']:
data = client.call('GET', path+'?variant='+variant['name'], actor=actor)['bytes']
check(hashlib.sha256(data).hexdigest() == variant['sha256'], 'Checksum mismatch')
check(ready["id"] == attachment["id"], "Attachment identity changed")
path = "/api/v1/attachments/" + ready["id"] + "/content"
check(
client.call("GET", path, actor=actor)["bytes"] == original, "Original changed"
)
check(
{v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"},
"Incomplete pipeline",
)
for variant in ready["variants"]:
data = client.call("GET", path + "?variant=" + variant["name"], actor=actor)[
"bytes"
]
check(
hashlib.sha256(data).hexdigest() == variant["sha256"], "Checksum mismatch"
)
with Image.open(BytesIO(data)) as image:
check(image.size == (variant['width'],variant['height']), 'Metadata dimensions mismatch')
check(image.format == ('WEBP' if variant['name']=='optimized' else 'JPEG'), 'Wrong format')
check(not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp'), 'Metadata leaked')
if variant['name']=='crop':check(image.size==(128,128),'Wrong crop size')
snapshot=probe(ready['id'])
check(len(snapshot['objects']) == 4, 'Expected exactly one original and three variants')
check(
image.size == (variant["width"], variant["height"]),
"Metadata dimensions mismatch",
)
check(
image.format == ("WEBP" if variant["name"] == "optimized" else "JPEG"),
"Wrong format",
)
check(
not image.getexif()
and not image.info.get("icc_profile")
and not image.info.get("xmp"),
"Metadata leaked",
)
if variant["name"] == "crop":
check(image.size == (128, 128), "Wrong crop size")
snapshot = probe(ready["id"])
check(
len(snapshot["objects"]) == 4,
"Expected exactly one original and three variants",
)
return snapshot
def crash(client, actor, chat, mode):
name='messenger-worker-fault-'+uuid4().hex
name = "messenger-worker-fault-" + uuid4().hex
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
result=compose('run','--rm','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
phase='after_object:optimized' if mode=='crash-after-object' else 'before_ack'
marker=json.dumps({'fault':phase,'attachment_id':attachment['id']})
check(result.returncode==137 and marker in result.stdout,'Targeted crash not observed')
before=probe(attachment['id'])
check(before['status']==('processing' if mode=='crash-after-object' else 'ready'),'Unexpected crash state')
check(len(before['objects'])==4,'Crash must occur after all four PUTs')
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
result = compose(
"run",
"--rm",
"--no-deps",
"--name",
name,
"worker",
"python",
"-m",
"app.core.job_worker",
"--" + mode,
attachment["id"],
)
phase = (
"after_object:optimized" if mode == "crash-after-object" else "before_ack"
)
marker = json.dumps({"fault": phase, "attachment_id": attachment["id"]})
check(
result.returncode == 137 and marker in result.stdout,
"Targeted crash not observed",
)
before = probe(attachment["id"])
check(
before["status"]
== ("processing" if mode == "crash-after-object" else "ready"),
"Unexpected crash state",
)
check(len(before["objects"]) == 4, "Crash must occur after all four PUTs")
finally:
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
if mode=='crash-before-ack':
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
if mode == "crash-before-ack":
# Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
wait_queue_empty()
after=probe(attachment['id'])
check(after==before,'Ready duplicate rewrote result')
after = probe(attachment["id"])
check(after == before, "Ready duplicate rewrote result")
else:
check(after['attempt']==before['attempt']+1,'Lease did not recover the interrupted attempt')
check([(x['name'],x['sha256']) for x in before['objects']]==[(x['name'],x['sha256']) for x in after['objects']],'Replay changed object content')
return dict(scenario=mode,attachment_id=attachment['id'],exit_code=137,before=before,after=after)
check(
after["attempt"] == before["attempt"] + 1,
"Lease did not recover the interrupted attempt",
)
check(
[(x["name"], x["sha256"]) for x in before["objects"]]
== [(x["name"], x["sha256"]) for x in after["objects"]],
"Replay changed object content",
)
return dict(
scenario=mode,
attachment_id=attachment["id"],
exit_code=137,
before=before,
after=after,
)
def retry_replay(client,actor,chat,permanent):
name='messenger-retry-fault-'+uuid4().hex
samples=[]
configuration=ComposeProject().config['services']['worker']['environment']
retry_base=float(configuration['JOB_RETRY_SECONDS'])
retry_max=float(configuration['JOB_RETRY_MAX_SECONDS'])
def retry_replay(client, actor, chat, permanent):
name = "messenger-retry-fault-" + uuid4().hex
samples = []
configuration = ComposeProject().config["services"]["worker"]["environment"]
retry_base = float(configuration["JOB_RETRY_SECONDS"])
retry_max = float(configuration["JOB_RETRY_MAX_SECONDS"])
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
mode='fail-permanent' if permanent else 'fail-transient'
require_compose('run','-d','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
deadline=time.monotonic()+180
while time.monotonic()<deadline:
sample=probe(attachment['id'])
if sample['attempt'] and sample['status'] in {'queued', 'failed'} and (not samples or samples[-1]['attempt']!=sample['attempt']):samples.append(sample)
if sample['status']=='failed':break
time.sleep(.5)
check(sample['status']=='failed','Retry never reached failed')
check(sample['attempt']==(1 if permanent else sample['limit']),'Incorrect attempt budget')
check(sample['permanent']==permanent,'Incorrect error classification')
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
mode = "fail-permanent" if permanent else "fail-transient"
require_compose(
"run",
"-d",
"--no-deps",
"--name",
name,
"worker",
"python",
"-m",
"app.core.job_worker",
"--" + mode,
attachment["id"],
)
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
sample = probe(attachment["id"])
if (
sample["attempt"]
and sample["status"] in {"queued", "failed"}
and (not samples or samples[-1]["attempt"] != sample["attempt"])
):
samples.append(sample)
if sample["status"] == "failed":
break
time.sleep(0.5)
check(sample["status"] == "failed", "Retry never reached failed")
check(
sample["attempt"] == (1 if permanent else sample["limit"]),
"Incorrect attempt budget",
)
check(sample["permanent"] == permanent, "Incorrect error classification")
if not permanent:
check(len(samples)==sample['limit'],'Missed retry samples')
check(len(samples) == sample["limit"], "Missed retry samples")
for entry in samples:
delay=(datetime.fromisoformat(entry['next_attempt_at'])-datetime.fromisoformat(entry['updated_at'])).total_seconds()
ceiling=min(retry_max,retry_base*2**(entry['attempt']-1))
check(ceiling/2<=delay<=ceiling,'Retry delay outside configured jitter bounds')
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
job_id=sample['job_id'];request_id=str(uuid4())
listing=admin('list-failed')
check(any(row['job_id']==job_id for row in listing),'Job missing from failed registry')
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30,check=True)
first=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
check(first['status']=='queued','Replay did not queue job')
delay = (
datetime.fromisoformat(entry["next_attempt_at"])
- datetime.fromisoformat(entry["updated_at"])
).total_seconds()
ceiling = min(retry_max, retry_base * 2 ** (entry["attempt"] - 1))
check(
ceiling / 2 <= delay <= ceiling,
"Retry delay outside configured jitter bounds",
)
client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=actor,
expected=409,
)
job_id = sample["job_id"]
request_id = str(uuid4())
listing = admin("list-failed")
check(
any(row["job_id"] == job_id for row in listing),
"Job missing from failed registry",
)
subprocess.run(
["docker", "rm", "-f", name], capture_output=True, timeout=30, check=True
)
first = admin(
"replay",
job_id,
"--request-id",
request_id,
"--actor",
"verification",
"--reason",
"dependency_restored",
)
check(first["status"] == "queued", "Replay did not queue job")
finally:
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
repeat=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
check(repeat['already_applied'],'Replay request id not idempotent')
refused=admin('replay',job_id,'--request-id',str(uuid4()),'--actor','verification','--reason','manual_review',expected=2)
check(refused['error_code']=='job_not_failed','Successful job was replayed')
check(probe(attachment['id'])==after,'Operator replay altered ready job')
check(len(admin('history',job_id))==1,'Incorrect audit count')
return dict(scenario=mode,attachment_id=attachment['id'],samples=samples,after=after,replay_idempotent=True)
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
repeat = admin(
"replay",
job_id,
"--request-id",
request_id,
"--actor",
"verification",
"--reason",
"dependency_restored",
)
check(repeat["already_applied"], "Replay request id not idempotent")
refused = admin(
"replay",
job_id,
"--request-id",
str(uuid4()),
"--actor",
"verification",
"--reason",
"manual_review",
expected=2,
)
check(refused["error_code"] == "job_not_failed", "Successful job was replayed")
check(probe(attachment["id"]) == after, "Operator replay altered ready job")
check(len(admin("history", job_id)) == 1, "Incorrect audit count")
return dict(
scenario=mode,
attachment_id=attachment["id"],
samples=samples,
after=after,
replay_idempotent=True,
)
def queue_state():
output=require_compose('exec','-T','broker','rabbitmqctl','list_queues','--formatter=json','name','messages_ready','messages_unacknowledged','consumers','durable')
return next(row for row in json.loads(output) if row['name']=='attachment-jobs')
output = require_compose(
"exec",
"-T",
"broker",
"rabbitmqctl",
"list_queues",
"--formatter=json",
"name",
"messages_ready",
"messages_unacknowledged",
"consumers",
"durable",
)
return next(row for row in json.loads(output) if row["name"] == "attachment-jobs")
def wait_queue_empty():
deadline=time.monotonic()+20
while time.monotonic()<deadline:
state=queue_state()
if state['messages_ready']==0 and state['messages_unacknowledged']==0:
deadline = time.monotonic() + 20
while time.monotonic() < deadline:
state = queue_state()
if state["messages_ready"] == 0 and state["messages_unacknowledged"] == 0:
return state
time.sleep(.5)
check(False,'RabbitMQ did not acknowledge all completed deliveries')
time.sleep(0.5)
check(False, "RabbitMQ did not acknowledge all completed deliveries")
def persistence(client,actor,chat):
def persistence(client, actor, chat):
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
client.message(actor,chat,'',attachments=[attachment['id']])
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
client.message(actor, chat, "", attachments=[attachment["id"]])
client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=actor,
expected=409,
)
time.sleep(2)
queued_before=queue_state()
check(queued_before['messages_ready']>=1 and queued_before['durable'], 'No durable queued message before restart')
result=subprocess.run([str(ROOT/'scripts/restart.sh')],cwd=ROOT,capture_output=True,timeout=180)
check(result.returncode==0,'Restart hook failed')
check('worker' not in require_compose('ps','--status','running','--services').splitlines(),'Restart started deliberately stopped worker')
queued_after=queue_state()
check(queued_after['messages_ready']>=1 and queued_after['durable'], 'Broker lost queued delivery across restart')
client.message(actor,chat,'session and chat survived broker/API restart')
finally:require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
return dict(scenario='worker_stopped_restart_all',attachment_id=attachment['id'],queue_before=queued_before,queue_after=queued_after,after=after)
queued_before = queue_state()
check(
queued_before["messages_ready"] >= 1 and queued_before["durable"],
"No durable queued message before restart",
)
result = subprocess.run(
[str(ROOT / "scripts/restart.sh")],
cwd=ROOT,
capture_output=True,
timeout=180,
)
check(result.returncode == 0, "Restart hook failed")
check(
"worker"
not in require_compose(
"ps", "--status", "running", "--services"
).splitlines(),
"Restart started deliberately stopped worker",
)
queued_after = queue_state()
check(
queued_after["messages_ready"] >= 1 and queued_after["durable"],
"Broker lost queued delivery across restart",
)
client.message(actor, chat, "session and chat survived broker/API restart")
finally:
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
return dict(
scenario="worker_stopped_restart_all",
attachment_id=attachment["id"],
queue_before=queued_before,
queue_after=queued_after,
after=after,
)
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--base-url',required=True)
parser.add_argument('--ca-file')
parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all')
parser.add_argument('--output',default='evidence/jobs.json')
args=parser.parse_args()
client=Client(args.base_url,args.ca_file,timeout=40)
aliases={r.instance_id for r in ComposeProject().replicas() if r.running}
check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL')
actor,_,_,chat=client.bootstrap()
results=[]
operations={'persistence':persistence,'crash-after-object':lambda *x:crash(*x,'crash-after-object'),'crash-before-ack':lambda *x:crash(*x,'crash-before-ack'),'fail-transient':lambda *x:retry_replay(*x,False),'fail-permanent':lambda *x:retry_replay(*x,True)}
selected=operations.values() if args.scenario=='all' else [operations[args.scenario]]
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--ca-file")
parser.add_argument(
"--scenario",
choices=[
"all",
"persistence",
"crash-after-object",
"crash-before-ack",
"fail-transient",
"fail-permanent",
],
default="all",
)
parser.add_argument("--output", default="evidence/jobs.json")
args = parser.parse_args()
client = Client(args.base_url, args.ca_file, timeout=40)
aliases = {r.instance_id for r in ComposeProject().replicas() if r.running}
check(
client.call("GET", "/health/live", schema="Health")["headers"].get(
"X-Instance-Id"
)
in aliases,
"Wrong Compose project for BASE_URL",
)
actor, _, _, chat = client.bootstrap()
results = []
operations = {
"persistence": persistence,
"crash-after-object": lambda *x: crash(*x, "crash-after-object"),
"crash-before-ack": lambda *x: crash(*x, "crash-before-ack"),
"fail-transient": lambda *x: retry_replay(*x, False),
"fail-permanent": lambda *x: retry_replay(*x, True),
}
selected = (
operations.values() if args.scenario == "all" else [operations[args.scenario]]
)
for operation in selected:
result=operation(client,actor,chat);results.append(result)
Path(args.output).parent.mkdir(parents=True,exist_ok=True)
Path(args.output).write_text(json.dumps(dict(status='in_progress',results=results),indent=2)+'\n')
print('PASS: '+result['scenario'],flush=True)
Path(args.output).write_text(json.dumps(dict(status='passed',results=results),indent=2)+'\n')
result = operation(client, actor, chat)
results.append(result)
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text(
json.dumps(dict(status="in_progress", results=results), indent=2) + "\n"
)
print("PASS: " + result["scenario"], flush=True)
Path(args.output).write_text(
json.dumps(dict(status="passed", results=results), indent=2) + "\n"
)
if __name__=='__main__':main()
if __name__ == "__main__":
main()
+89 -25
View File
@@ -19,7 +19,13 @@ COMPOSE = ["docker", "compose", "--env-file", ".env"]
def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess:
return subprocess.run(COMPOSE + list(arguments), cwd=ROOT, text=True, capture_output=True, timeout=timeout)
return subprocess.run(
COMPOSE + list(arguments),
cwd=ROOT,
text=True,
capture_output=True,
timeout=timeout,
)
def require_compose(*arguments: str) -> str:
@@ -29,54 +35,112 @@ def require_compose(*arguments: str) -> str:
def main() -> None:
parser = argparse.ArgumentParser(description="Отказ publisher после COMMIT перед RabbitMQ publish")
parser = argparse.ArgumentParser(
description="Отказ publisher после COMMIT перед RabbitMQ publish"
)
parser.add_argument("--base-url", required=True)
parser.add_argument("--ca-file")
args = parser.parse_args()
running = set(require_compose("ps", "--status", "running", "--services").splitlines())
check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the application stack before this scenario")
running = set(
require_compose("ps", "--status", "running", "--services").splitlines()
)
check(
{"api", "postgres", "minio", "broker", "publisher", "worker"} <= running,
"Start the application stack before this scenario",
)
client = Client(args.base_url, args.ca_file, timeout=40)
project = ComposeProject()
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
live = client.call("GET", "/health/live", schema="Health")
check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project")
check(
live["headers"].get("X-Instance-Id") in aliases,
"BASE_URL does not belong to the selected Compose project",
)
alice, _, _, chat = client.bootstrap()
attachment = None
fault_container = "messenger-outbox-fault-" + uuid4().hex
try:
require_compose("stop", "publisher")
attachment, original = client.attachment(alice, chat)
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected=409, actor=alice)
killed = compose(
"run", "--rm", "--no-deps", "--name", fault_container, "publisher", "python", "-m",
"app.core.outbox_publisher", "--crash-before-publish", attachment["id"],
message = client.message(alice, chat, "", attachments=[attachment["id"]])[
"json"
]
client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}/content",
expected=409,
actor=alice,
)
marker = json.dumps({"fault": "before_publish", "attachment_id": attachment["id"]})
check(killed.returncode == 137 and marker in killed.stdout, "Expected targeted SIGKILL before publish was not observed")
pending = client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=alice, schema="Attachment")["json"]
killed = compose(
"run",
"--rm",
"--no-deps",
"--name",
fault_container,
"publisher",
"python",
"-m",
"app.core.outbox_publisher",
"--crash-before-publish",
attachment["id"],
)
marker = json.dumps(
{"fault": "before_publish", "attachment_id": attachment["id"]}
)
check(
killed.returncode == 137 and marker in killed.stdout,
"Expected targeted SIGKILL before publish was not observed",
)
pending = client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}",
actor=alice,
schema="Attachment",
)["json"]
check(pending["status"] == "queued", "Crash must leave accepted job queued")
finally:
# Если docker compose run оборвался по timeout, одноразовый контейнер тоже удаляется.
try:
subprocess.run(["docker", "rm", "-f", fault_container], capture_output=True, timeout=30)
subprocess.run(
["docker", "rm", "-f", fault_container], capture_output=True, timeout=30
)
finally:
require_compose("start", "publisher")
ready = client.wait_ready(alice, attachment, wait_seconds=60)
check(ready["id"] == attachment["id"], "Recovered a different attachment")
check({variant["name"] for variant in ready["variants"]} == {"thumbnail", "crop", "optimized"}, "Missing required variants")
check(
{variant["name"] for variant in ready["variants"]}
== {"thumbnail", "crop", "optimized"},
"Missing required variants",
)
path = f"/api/v1/attachments/{attachment['id']}/content"
check(client.call("GET", path, actor=alice)["bytes"] == original, "Original changed after recovery")
check(
client.call("GET", path, actor=alice)["bytes"] == original,
"Original changed after recovery",
)
for variant in ready["variants"]:
downloaded = client.call("GET", path + "?variant=" + variant["name"], actor=alice)["bytes"]
check(len(downloaded) == variant["size_bytes"] and hashlib.sha256(downloaded).hexdigest() == variant["sha256"], "Recovered variant checksum mismatch")
print(json.dumps({
"scenario": "commit_before_publish", "attachment_id": ready["id"],
"message_id": message["id"], "publisher_exit_code": killed.returncode,
"status_after_crash": pending["status"], "status_after_restart": ready["status"],
"variants": [variant["name"] for variant in ready["variants"]],
"reuploaded": False,
}))
downloaded = client.call(
"GET", path + "?variant=" + variant["name"], actor=alice
)["bytes"]
check(
len(downloaded) == variant["size_bytes"]
and hashlib.sha256(downloaded).hexdigest() == variant["sha256"],
"Recovered variant checksum mismatch",
)
print(
json.dumps(
{
"scenario": "commit_before_publish",
"attachment_id": ready["id"],
"message_id": message["id"],
"publisher_exit_code": killed.returncode,
"status_after_crash": pending["status"],
"status_after_restart": ready["status"],
"variants": [variant["name"] for variant in ready["variants"]],
"reuploaded": False,
}
)
)
if __name__ == "__main__":
+61 -30
View File
@@ -7,7 +7,7 @@ import time
from scripts.deployment import ComposeProject
from tests.client import Client, arguments, check, run
PROBE = r'''
PROBE = r"""
import asyncio, json, sys
from pathlib import Path
from uuid import UUID
@@ -62,60 +62,91 @@ async def main():
await storage.aclose()
await engine.dispose()
asyncio.run(main())
'''
"""
def main():
def configure(parser):
parser.add_argument('--with-outage', action='store_true', help='временно остановить MinIO выбранного проекта')
args = arguments('MinIO private bucket and failure checks', configure)
parser.add_argument(
"--with-outage",
action="store_true",
help="временно остановить MinIO выбранного проекта",
)
args = arguments("MinIO private bucket and failure checks", configure)
project = ComposeProject()
client = Client(args.base_url, args.ca_file, timeout=40)
alice, _, _, chat = client.bootstrap()
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
live = client.call('GET', '/health/live', schema='Health')
check(live['headers'].get('X-Instance-Id') in aliases, 'BASE_URL does not belong to the selected Compose project')
live = client.call("GET", "/health/live", schema="Health")
check(
live["headers"].get("X-Instance-Id") in aliases,
"BASE_URL does not belong to the selected Compose project",
)
attachment, original = client.attachment(alice, chat)
attachment = client.wait_ready(alice, attachment)
# Повторный bootstrap не должен очистить bucket или сменить используемые credentials.
for _ in range(2):
project.run('run', '--rm', '--no-deps', 'minio-bootstrap', timeout=100)
project.run("run", "--rm", "--no-deps", "minio-bootstrap", timeout=100)
replicas = project.replicas()
check(bool(replicas), 'API not running')
result = subprocess.run(['docker', 'exec', '-i', replicas[0].id, 'python', '-c', PROBE],
input=json.dumps({'attachment_id': attachment['id']}),
text=True, capture_output=True, timeout=20)
check(result.returncode == 0, 'private bucket probe failed (details hidden)')
check(bool(replicas), "API not running")
result = subprocess.run(
["docker", "exec", "-i", replicas[0].id, "python", "-c", PROBE],
input=json.dumps({"attachment_id": attachment["id"]}),
text=True,
capture_output=True,
timeout=20,
)
check(result.returncode == 0, "private bucket probe failed (details hidden)")
statuses = json.loads(result.stdout)
check(statuses['GET'] == statuses['PUT'] == 403, 'anonymous S3 access allowed')
check(statuses['admin_denied'] and statuses['other_buckets_denied'] and statuses['root_files_absent'], 'API has excessive access')
check(statuses['console_unreachable'], 'console reachable from API network')
check(statuses['temp_originals_absent'], 'API retained temporary originals')
check(not project.config['services']['minio'].get('ports'), 'MinIO publishes host ports')
env = project.config['services']['minio']['environment']
check(env['MINIO_BROWSER'] == 'off', 'browser enabled')
check('127.0.0.1:9001' in project.config['services']['minio']['command'], 'console is not bound to loopback')
path = '/api/v1/attachments/' + attachment['id'] + '/content'
check(client.call('GET', path, actor=alice)['bytes'] == original, 'bootstrap changed original')
check(statuses["GET"] == statuses["PUT"] == 403, "anonymous S3 access allowed")
check(
statuses["admin_denied"]
and statuses["other_buckets_denied"]
and statuses["root_files_absent"],
"API has excessive access",
)
check(statuses["console_unreachable"], "console reachable from API network")
check(statuses["temp_originals_absent"], "API retained temporary originals")
check(
not project.config["services"]["minio"].get("ports"),
"MinIO publishes host ports",
)
env = project.config["services"]["minio"]["environment"]
check(env["MINIO_BROWSER"] == "off", "browser enabled")
check(
"127.0.0.1:9001" in project.config["services"]["minio"]["command"],
"console is not bound to loopback",
)
path = "/api/v1/attachments/" + attachment["id"] + "/content"
check(
client.call("GET", path, actor=alice)["bytes"] == original,
"bootstrap changed original",
)
if args.with_outage:
try:
project.run('stop', 'minio')
client.call('GET', path, expected=503, actor=alice)
project.run("stop", "minio")
client.call("GET", path, expected=503, actor=alice)
finally:
project.run('start', 'minio')
project.run("start", "minio")
deadline = time.monotonic() + 60
while True:
try:
client.call('GET', '/health/ready', schema='Health')
client.call("GET", "/health/ready", schema="Health")
break
except Exception:
if time.monotonic() >= deadline:
raise
time.sleep(0.5)
check(client.call('GET', path, actor=alice)['bytes'] == original, 'original lost after MinIO restart')
print('PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console'
+ (', MinIO outage/recovery' if args.with_outage else ''))
check(
client.call("GET", path, actor=alice)["bytes"] == original,
"original lost after MinIO restart",
)
print(
"PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console"
+ (", MinIO outage/recovery" if args.with_outage else "")
)
if __name__ == '__main__':
if __name__ == "__main__":
run(main)
+17 -4
View File
@@ -31,8 +31,12 @@ class ComposeProject:
def docker(self, *args: str, timeout: int = 90, env=None) -> str:
"""Выполняет Docker-команду с ограничением времени."""
result = subprocess.run(
["docker", *args], cwd=ROOT, text=True, capture_output=True,
timeout=timeout, env={**os.environ, **(env or {})},
["docker", *args],
cwd=ROOT,
text=True,
capture_output=True,
timeout=timeout,
env={**os.environ, **(env or {})},
)
if result.returncode:
raise RuntimeError(result.stderr.strip() or result.stdout.strip())
@@ -53,8 +57,17 @@ class ComposeProject:
'"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}}'
)
rows = self.docker("inspect", "--format", template, *ids).splitlines()
configured = str(self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID") or "").strip()
configured = str(
self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID")
or ""
).strip()
return [
Replica(row["id"], row["image"], configured or row["hostname"], row["running"], row["health"])
Replica(
row["id"],
row["image"],
configured or row["hostname"],
row["running"],
row["health"],
)
for row in map(json.loads, rows)
]
+9 -1
View File
@@ -14,7 +14,15 @@ def main() -> None:
destination.chmod(0o600)
directory = root / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
for name in ("db_password", "redis_password", "broker_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"):
for name in (
"db_password",
"redis_password",
"broker_password",
"minio_root_user",
"minio_root_password",
"s3_access_key_id",
"s3_secret_access_key",
):
path = directory / name
try:
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444)
+12 -8
View File
@@ -1,23 +1,27 @@
#!/usr/bin/env python3
"""Full OpenAPI and JSON Schema checks; install requirements-dev.txt first."""
import json
from pathlib import Path
from openapi_spec_validator import validate
from jsonschema import Draft202012Validator
root = Path(__file__).resolve().parents[1]
spec = json.loads((root/'contracts/openapi.json').read_text())
spec = json.loads((root / "contracts/openapi.json").read_text())
validate(spec)
from referencing import Registry, Resource
# The base URI resolves local #/components references inside examples.
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
registry = Registry().with_resource('urn:messenger:openapi', Resource.from_contents(resource))
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
schema = {'$ref': 'urn:messenger:openapi#/components/schemas/' + name}
resource = dict(spec, **{"$schema": "https://json-schema.org/draft/2020-12/schema"})
registry = Registry().with_resource(
"urn:messenger:openapi", Resource.from_contents(resource)
)
for name, example in json.loads((root / "contracts/examples.json").read_text()).items():
schema = {"$ref": "urn:messenger:openapi#/components/schemas/" + name}
Draft202012Validator(schema, registry=registry).validate(example)
for path in (root/'contracts').glob('*.schema.json'):
for path in (root / "contracts").glob("*.schema.json"):
schema = json.loads(path.read_text())
Draft202012Validator.check_schema(schema)
for example in schema.get('examples',[]):
for example in schema.get("examples", []):
Draft202012Validator(schema).validate(example)
print('PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.')
print("PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.")