688 lines
29 KiB
Python
688 lines
29 KiB
Python
"""Repeatable fault injection for grade 5, track 2; synthetic data only."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
from PIL import Image
|
|
|
|
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
|
|
os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1")
|
|
os.environ.setdefault("no_proxy", "localhost,127.0.0.1")
|
|
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
|
|
|
|
PREFIX = """import io,json,hashlib,time,urllib.request
|
|
from uuid import UUID,uuid4
|
|
from psycopg.types.json import Jsonb
|
|
from src.pipeline import database,broker,envelope,STREAM,GROUP
|
|
from src.object_store import ObjectStore,BUCKET
|
|
from src.operator import change
|
|
from src.reconcile import reconcile
|
|
db,store,queue=database(),ObjectStore(),broker()
|
|
"""
|
|
|
|
|
|
def inside(code):
|
|
cid = compose("ps", "-q", "controller").split()[0]
|
|
return json.loads(
|
|
command(
|
|
["docker", "exec", "-i", cid, "python", "-"], input=PREFIX + code
|
|
)
|
|
)
|
|
|
|
|
|
def wait(fn, timeout=90):
|
|
end = time.monotonic() + timeout
|
|
while time.monotonic() < end:
|
|
value = fn()
|
|
if value:
|
|
return value
|
|
time.sleep(0.15)
|
|
raise TimeoutError("Fault scenario condition not reached")
|
|
|
|
|
|
def stop(role):
|
|
compose("stop", "--timeout", "1", role)
|
|
|
|
|
|
def start(role):
|
|
compose("up", "-d", "--no-deps", "--no-recreate", role)
|
|
|
|
|
|
def row(job):
|
|
return inside(
|
|
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\nprint(json.dumps(r,default=str))"
|
|
)
|
|
|
|
|
|
def inject(job, phase, action, remaining=1):
|
|
return inside(
|
|
f"with db.connection() as c: r=c.execute('INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES(%s,%s,%s,%s) RETURNING id',({job!r},{phase!r},{action!r},{remaining})).fetchone()\nprint(json.dumps(r['id']))"
|
|
)
|
|
|
|
|
|
def hit(identity):
|
|
return inside(
|
|
f"with db.connection() as c: r=c.execute('SELECT hit_at IS NOT NULL hit FROM pipeline_faults WHERE id=%s',({identity},)).fetchone()\nprint(json.dumps(r['hit']))"
|
|
)
|
|
|
|
|
|
def operator(job, version=None):
|
|
return inside(
|
|
f"print(json.dumps(change(db,store,job_id=UUID({job!r}),operator='synthetic-verifier',reason='Explicit fault matrix replay',version={version!r})))"
|
|
)
|
|
|
|
|
|
def duplicate(job, count=5):
|
|
return inside(
|
|
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\ne=envelope(r)\nids=[queue.xadd(STREAM,{{'job':json.dumps(e)}}) for _ in range({count})]\nprint(json.dumps(ids))"
|
|
)
|
|
|
|
|
|
def inspect_effect(job, *, pending=False):
|
|
return inside(f"""with db.connection() as c:
|
|
j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()
|
|
v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall()
|
|
p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n']
|
|
e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall()
|
|
keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")]
|
|
pending_entries=[]
|
|
if {pending!r}:
|
|
for item in queue.xpending_range(STREAM,GROUP,'-','+',1000):
|
|
entry=item['message_id']
|
|
messages=queue.xrange(STREAM,min=entry,max=entry)
|
|
if messages and json.loads(messages[0][1].get('job','{{}}')).get('job_id')==str(j['id']): pending_entries.append(entry)
|
|
print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e,'pending_stream_entries':pending_entries}},default=str))""")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--output", default="evidence/pipeline-repeat.json")
|
|
parser.add_argument("--repeats", type=int, default=2)
|
|
parser.add_argument(
|
|
"--start-at", choices=["all", "versions"], default="all"
|
|
)
|
|
args = parser.parse_args()
|
|
output = ROOT / args.output
|
|
if output.exists():
|
|
raise ValueError("Choose a new output path")
|
|
report = {
|
|
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
|
"date": datetime.now(timezone.utc).isoformat(),
|
|
"track": 2,
|
|
"scope": args.start_at,
|
|
"complete": False,
|
|
"cases": [],
|
|
}
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
def save(name, details):
|
|
report["cases"].append({"case": name, **details})
|
|
output.write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
print("PASS:", name, flush=True)
|
|
|
|
try:
|
|
ready()
|
|
# Check instrumentation is really enabled and production default is off.
|
|
cid = compose("ps", "-q", "api").split()[0]
|
|
assert (
|
|
command(
|
|
[
|
|
"docker",
|
|
"exec",
|
|
cid,
|
|
"python",
|
|
"-c",
|
|
"import os;print(os.getenv('PIPELINE_FAULTS'))",
|
|
]
|
|
)
|
|
== "1"
|
|
)
|
|
stop("reconciler")
|
|
with httpx.Client(
|
|
base_url=BASE, trust_env=False, timeout=25
|
|
) as client:
|
|
user, _, session = actor(client)
|
|
h = auth(session)
|
|
c = client.post(
|
|
"/api/v1/chats",
|
|
headers=h,
|
|
json={
|
|
"title": "Pipeline synthetic faults",
|
|
"member_ids": [user["id"]],
|
|
},
|
|
)
|
|
assert c.status_code == 201
|
|
chat = c.json()["id"]
|
|
raw_io = io.BytesIO()
|
|
Image.new("RGBA", (320, 200), (20, 50, 170, 120)).save(
|
|
raw_io, format="PNG"
|
|
)
|
|
raw = raw_io.getvalue()
|
|
|
|
def upload():
|
|
r = client.post(
|
|
f"/api/v1/chats/{chat}/attachments",
|
|
headers={**h, "Content-Type": "image/png"},
|
|
params={"filename": "synthetic.png"},
|
|
content=raw,
|
|
)
|
|
assert r.status_code == 202, r.text
|
|
a = r.json()
|
|
assert a["status"] == "queued" and not a["variants"]
|
|
j = inside(
|
|
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
|
|
)
|
|
return a, j
|
|
|
|
def metadata(a):
|
|
r = client.get("/api/v1/attachments/" + a["id"], headers=h)
|
|
assert r.status_code == 200, r.text
|
|
return r.json()
|
|
|
|
def until(a, state="ready", timeout=90):
|
|
return wait(
|
|
lambda: (
|
|
r if (r := metadata(a))["status"] == state else None
|
|
),
|
|
timeout,
|
|
)
|
|
|
|
def confirm(a, j):
|
|
until(a)
|
|
downloaded = client.get(
|
|
f"/api/v1/attachments/{a['id']}/content", headers=h
|
|
)
|
|
assert (
|
|
downloaded.status_code == 200 and downloaded.content == raw
|
|
)
|
|
m = metadata(a)
|
|
assert (
|
|
m["sha256"] == hashlib.sha256(raw).hexdigest()
|
|
and m["error_code"] is None
|
|
)
|
|
assert {v["name"] for v in m["variants"]} == {
|
|
"thumbnail",
|
|
"crop",
|
|
"optimized",
|
|
}
|
|
for v in m["variants"]:
|
|
r = client.get(
|
|
f"/api/v1/attachments/{a['id']}/content",
|
|
headers=h,
|
|
params={"variant": v["name"]},
|
|
)
|
|
assert (
|
|
r.status_code == 200
|
|
and hashlib.sha256(r.content).hexdigest()
|
|
== v["sha256"]
|
|
)
|
|
with Image.open(io.BytesIO(r.content)) as image:
|
|
assert (
|
|
image.size
|
|
== {
|
|
"thumbnail": (256, 160),
|
|
"crop": (128, 128),
|
|
"optimized": (320, 200),
|
|
}[v["name"]]
|
|
)
|
|
assert not image.getexif()
|
|
effect = inspect_effect(j)
|
|
assert (
|
|
effect["publications"] == 1
|
|
and len(effect["variants"]) == 4
|
|
and len(effect["physical_keys"]) == 4
|
|
)
|
|
return effect
|
|
|
|
if args.start_at == "all":
|
|
# Source-only crash before the DB handoff must never be reported as accepted.
|
|
stop("worker")
|
|
stop("relay")
|
|
marker = inject(chat, "api_after_staging", "wait")
|
|
before = inside(
|
|
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
|
|
)
|
|
pool = ThreadPoolExecutor(max_workers=1)
|
|
future = pool.submit(
|
|
lambda: client.post(
|
|
f"/api/v1/chats/{chat}/attachments",
|
|
headers={**h, "Content-Type": "image/png"},
|
|
params={"filename": "unconfirmed.png"},
|
|
content=raw,
|
|
)
|
|
)
|
|
wait(lambda marker=marker: hit(marker))
|
|
compose("kill", "-s", "SIGKILL", "api")
|
|
try:
|
|
response = future.result(15)
|
|
assert response.status_code != 202
|
|
except httpx.HTTPError:
|
|
pass
|
|
pool.shutdown()
|
|
start("api")
|
|
ready()
|
|
after = inside(
|
|
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
|
|
)
|
|
assert before == after
|
|
save(
|
|
"staging_before_handoff_crash",
|
|
{
|
|
"no_202": True,
|
|
"no_attachment_or_job_created": True,
|
|
"orphan_retained_for_cleanup": True,
|
|
},
|
|
)
|
|
# API durable commit/outbox survives restart without a broker or worker.
|
|
stop("broker")
|
|
a, j = upload()
|
|
linked = client.post(
|
|
f"/api/v1/chats/{chat}/messages",
|
|
headers={**h, "Idempotency-Key": uuid4().hex},
|
|
json={"text": "", "attachment_ids": [a["id"]]},
|
|
)
|
|
assert linked.status_code == 201
|
|
assert (
|
|
client.get(
|
|
f"/api/v1/attachments/{a['id']}/content", headers=h
|
|
).status_code
|
|
== 409
|
|
)
|
|
compose("restart", "api")
|
|
ready()
|
|
start("broker")
|
|
start("relay")
|
|
start("worker")
|
|
effect = confirm(a, j)
|
|
save(
|
|
"commit_before_publish_and_worker_stopped",
|
|
{
|
|
"attachment": a["id"],
|
|
"effect": effect,
|
|
"queued_link_allowed": True,
|
|
"api_and_broker_restarted": True,
|
|
"broker_stopped_during_upload": True,
|
|
},
|
|
)
|
|
# Broker-volume survival with a confirmed, still-unconsumed stream entry.
|
|
stop("worker")
|
|
a, j = upload()
|
|
wait(
|
|
lambda: inside(
|
|
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))"
|
|
)
|
|
)
|
|
stop("relay")
|
|
ids = inside("print(json.dumps(queue.xrange(STREAM)))")
|
|
compose("kill", "-s", "SIGKILL", "broker")
|
|
start("broker")
|
|
wait(lambda: inside("print(json.dumps(queue.ping()))"))
|
|
restored = inside("print(json.dumps(queue.xrange(STREAM)))")
|
|
assert restored == ids, {"before": ids, "after": restored}
|
|
start("relay")
|
|
start("worker")
|
|
save(
|
|
"confirmed_broker_entry_survives_sigkill",
|
|
{
|
|
"effect": confirm(a, j),
|
|
"same_stream_entries": True,
|
|
"entry_count": len(ids),
|
|
},
|
|
)
|
|
# Confirm succeeded, PG marker did not: relay recovery may publish a duplicate.
|
|
stop("worker")
|
|
stop("relay")
|
|
a, j = upload()
|
|
marker = inject(j, "relay_after_publish", "crash")
|
|
start("relay")
|
|
wait(lambda marker=marker: hit(marker))
|
|
time.sleep(0.5)
|
|
unpublished = inside(
|
|
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']))"
|
|
)
|
|
assert unpublished >= 1
|
|
start("relay")
|
|
start("worker")
|
|
save(
|
|
"relay_crash_after_xadd_before_pg_marker",
|
|
{
|
|
"unpublished_after_crash": unpublished,
|
|
"effect": confirm(a, j),
|
|
},
|
|
)
|
|
# Repeated before/after effect and commit/ack windows.
|
|
for repeat in range(args.repeats):
|
|
for phase in [
|
|
"before_effect",
|
|
"after_effect",
|
|
"after_commit",
|
|
]:
|
|
stop("worker")
|
|
a, j = upload()
|
|
marker = inject(j, phase, "wait")
|
|
start("worker")
|
|
wait(lambda marker=marker: hit(marker))
|
|
at_crash = inspect_effect(
|
|
j, pending=phase == "after_commit"
|
|
)
|
|
assert at_crash["job"]["state"] == (
|
|
"ready"
|
|
if phase == "after_commit"
|
|
else "processing"
|
|
)
|
|
assert at_crash["publications"] == (
|
|
1 if phase == "after_commit" else 0
|
|
)
|
|
assert len(at_crash["physical_keys"]) == (
|
|
0 if phase == "before_effect" else 4
|
|
)
|
|
if phase == "after_commit":
|
|
assert at_crash["pending_stream_entries"], (
|
|
"Result must be committed while notification is still unacknowledged"
|
|
)
|
|
assert not any(
|
|
e["event"] == "child_finished"
|
|
for e in at_crash["events"]
|
|
)
|
|
compose("kill", "-s", "SIGKILL", "worker")
|
|
start("worker")
|
|
effect = confirm(a, j)
|
|
if phase == "after_commit":
|
|
wait(
|
|
lambda j=j: any(
|
|
e["event"] == "duplicate_decision"
|
|
and e["detail"]["state"] == "ready"
|
|
for e in inspect_effect(j)["events"]
|
|
)
|
|
)
|
|
wait(
|
|
lambda j=j: inside(
|
|
f"print(json.dumps(not any(json.loads(fields.get('job','{{}}')).get('job_id')=={j!r} for _,fields in queue.xrange(STREAM))))"
|
|
)
|
|
)
|
|
effect = inspect_effect(j)
|
|
else:
|
|
assert effect["job"]["attempt"] >= 2
|
|
save(
|
|
f"{phase}_sigkill_repeat_{repeat + 1}",
|
|
{"at_crash": at_crash, "effect": effect},
|
|
)
|
|
stop("worker")
|
|
a, j = upload()
|
|
inject(j, "before_effect", "transient")
|
|
start("worker")
|
|
effect = confirm(a, j)
|
|
assert effect["job"]["attempt"] == 2
|
|
save("transient_retry_with_jitter", {"effect": effect})
|
|
stop("worker")
|
|
a, j = upload()
|
|
marker = inject(j, "before_effect", "hang")
|
|
start("worker")
|
|
wait(lambda: hit(marker))
|
|
limits = inside(
|
|
f"with db.connection() as c: r=c.execute(\"SELECT detail FROM pipeline_events WHERE job_id=%s AND event='worker_resource_limits' ORDER BY id DESC LIMIT 1\",({j!r},)).fetchone()\nprint(json.dumps(r['detail']))"
|
|
)
|
|
assert limits["address_space_bytes"] == [805306368, 805306368]
|
|
assert limits["memory.max"] == "1610612736"
|
|
effect = confirm(a, j)
|
|
assert any(
|
|
e["event"] == "retry"
|
|
and e["detail"]["code"] == "worker_timeout"
|
|
for e in effect["events"]
|
|
)
|
|
save(
|
|
"hung_child_killed_and_retried",
|
|
{"effect": effect, "observed_limits": limits},
|
|
)
|
|
stop("worker")
|
|
a, j = upload()
|
|
inject(j, "before_effect", "permanent")
|
|
start("worker")
|
|
failed = until(a, "failed")
|
|
assert failed["error_code"] == "invalid_image"
|
|
assert (
|
|
client.get(
|
|
f"/api/v1/attachments/{a['id']}/content", headers=h
|
|
).status_code
|
|
== 409
|
|
)
|
|
r = client.post(
|
|
f"/api/v1/chats/{chat}/messages",
|
|
headers={**h, "Idempotency-Key": uuid4().hex},
|
|
json={"text": "", "attachment_ids": [a["id"]]},
|
|
)
|
|
assert r.status_code == 409
|
|
assert operator(j) == j
|
|
effect = confirm(a, j)
|
|
assert effect["job"]["cycle"] == 2
|
|
try:
|
|
operator(j)
|
|
except RuntimeError:
|
|
pass
|
|
else:
|
|
raise AssertionError("Replay of ready must be rejected")
|
|
save(
|
|
"permanent_failed_registry_and_audited_replay",
|
|
{
|
|
"failed": failed,
|
|
"effect": effect,
|
|
"ready_replay_rejected": True,
|
|
},
|
|
)
|
|
stop("worker")
|
|
a, j = upload()
|
|
inject(j, "before_effect", "transient", remaining=5)
|
|
start("worker")
|
|
failed = until(a, "failed", timeout=150)
|
|
assert (
|
|
failed["error_code"] == "retry_exhausted"
|
|
and row(j)["attempt"] == 5
|
|
)
|
|
operator(j)
|
|
save(
|
|
"finite_retry_exhaustion_then_explicit_replay",
|
|
{"failed": failed, "effect": confirm(a, j)},
|
|
)
|
|
# Reverse notification order and add concurrent copies: one publication/key.
|
|
stop("worker")
|
|
stop("relay")
|
|
a, j = upload()
|
|
b, k = upload()
|
|
duplicate(k, 8)
|
|
duplicate(j, 8)
|
|
start("worker")
|
|
start("relay")
|
|
ea = confirm(a, j)
|
|
confirm(b, k)
|
|
duplicate(j, 10)
|
|
duplicate(k, 10)
|
|
wait(
|
|
lambda: any(
|
|
e["event"] == "duplicate_decision"
|
|
for e in inspect_effect(j)["events"]
|
|
)
|
|
)
|
|
assert (
|
|
inspect_effect(j)["job"]["attempt"] == ea["job"]["attempt"]
|
|
)
|
|
save(
|
|
"reordered_and_duplicate_notifications",
|
|
{
|
|
"effects": [inspect_effect(j), inspect_effect(k)],
|
|
"added_duplicates": 36,
|
|
"no_extra_attempt_after_ready": True,
|
|
},
|
|
)
|
|
# Hold v1 after its PUTs, complete v2 on the other bounded slot, release old CAS.
|
|
for repeat in range(args.repeats):
|
|
stop("worker")
|
|
a, j = upload()
|
|
marker = inject(j, "after_effect", "wait")
|
|
# Keep one controller process hot throughout the narrow stale-writer window.
|
|
code = f"""from src.pipeline import event
|
|
with db.connection() as c: event(c,UUID({j!r}),'version_controller_waiting')
|
|
end=time.monotonic()+45
|
|
while True:
|
|
with db.connection() as c: hit=c.execute('SELECT hit_at FROM pipeline_faults WHERE id=%s',({marker},)).fetchone()['hit_at']
|
|
if hit: break
|
|
if time.monotonic()>end: raise TimeoutError('v1 hook never hit')
|
|
time.sleep(.05)
|
|
k=change(db,store,job_id=UUID({j!r}),operator='synthetic-verifier',reason='Held stale writer race',version=2)
|
|
end=time.monotonic()+10
|
|
while True:
|
|
with db.connection() as c: ready=c.execute('SELECT state FROM jobs WHERE id=%s',(k,)).fetchone()['state']=='ready'
|
|
if ready: break
|
|
if time.monotonic()>end: raise TimeoutError('v2 did not finish inside live v1 budget')
|
|
time.sleep(.05)
|
|
with db.connection() as c:
|
|
hashes=c.execute('SELECT name,sha256 FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=2 ORDER BY name',({a["id"]!r},)).fetchall()
|
|
c.execute('UPDATE pipeline_faults SET released_at=clock_timestamp() WHERE id=%s',({marker},))
|
|
end=time.monotonic()+10
|
|
while True:
|
|
with db.connection() as c: stale=c.execute("SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='stale_completion_rejected'",({j!r},)).fetchone()['n']
|
|
if stale: break
|
|
if time.monotonic()>end: raise TimeoutError('Old completion was not rejected')
|
|
time.sleep(.05)
|
|
print(json.dumps({{'job_id':k,'hashes':{{r['name']:r['sha256'] for r in hashes}}}}))
|
|
"""
|
|
with ThreadPoolExecutor(max_workers=1) as hot:
|
|
future = hot.submit(inside, code)
|
|
wait(
|
|
lambda j=j: inside(
|
|
f"with db.connection() as c: n=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='version_controller_waiting'\",({j!r},)).fetchone()['n']\nprint(json.dumps(n>0))"
|
|
)
|
|
)
|
|
start("worker")
|
|
race = future.result(timeout=45)
|
|
k = race["job_id"]
|
|
hashes = race["hashes"]
|
|
confirm(a, k)
|
|
duplicate(j, 12)
|
|
wait(
|
|
lambda j=j: any(
|
|
e["event"] == "duplicate_decision"
|
|
for e in inspect_effect(j)["events"]
|
|
)
|
|
)
|
|
assert hashes == {
|
|
v["name"]: v["sha256"] for v in confirm(a, k)["variants"]
|
|
}
|
|
assert inspect_effect(j)["publications"] == 0
|
|
save(
|
|
f"stale_v1_after_v2_repeat_{repeat + 1}",
|
|
{
|
|
"v1": inspect_effect(j),
|
|
"v2": inspect_effect(k),
|
|
"new_hashes_unchanged": True,
|
|
},
|
|
)
|
|
# No arbitrary URL/bytes fields in queue envelopes.
|
|
invalid = inside(
|
|
"bad={'schema_version':1,'url':'http://example.invalid/raw'}\nqueue.xadd(STREAM,{'job':json.dumps(bad)})\nprint('true')"
|
|
)
|
|
assert invalid
|
|
wait(
|
|
lambda: inside(
|
|
"with db.connection() as c: r=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE event='invalid_envelope'\").fetchone()\nprint(json.dumps(r['n']>0))"
|
|
)
|
|
)
|
|
save(
|
|
"malformed_notification_rejected", {"durable_diagnostic": True}
|
|
)
|
|
# Private bucket, bounded IAM, and download after uploader process dies.
|
|
a, j = upload()
|
|
confirm(a, j)
|
|
key = f"objects/{a['id']}/v1/original"
|
|
security = inside(f"""from minio.error import S3Error
|
|
from urllib.error import HTTPError
|
|
try: urllib.request.urlopen('http://s3:9000/'+BUCKET+'/' + {key!r},timeout=3)
|
|
except HTTPError as e: anonymous=e.code
|
|
else: raise AssertionError('Anonymous access allowed')
|
|
denials={{}}
|
|
for name,fn in [('create_bucket',lambda:store.client.make_bucket('forbidden-'+uuid4().hex)),('write_outside_prefix',lambda:store.client.put_object(BUCKET,'outside-'+uuid4().hex,io.BytesIO(b'x'),1)),('list_other_bucket',lambda:list(store.client.list_objects('forbidden-'+uuid4().hex)))]:
|
|
try: fn()
|
|
except S3Error as e: denials[name]=e.code
|
|
else: raise AssertionError('IAM allowed forbidden action')
|
|
print(json.dumps({{'anonymous':anonymous,'iam':denials}}))""")
|
|
assert security["anonymous"] == 403 and set(
|
|
security["iam"].values()
|
|
) == {"AccessDenied"}
|
|
selected = None
|
|
for cid in compose("ps", "-q", "api").split():
|
|
alias = command(
|
|
[
|
|
"docker",
|
|
"exec",
|
|
cid,
|
|
"python",
|
|
"-c",
|
|
"import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
|
|
]
|
|
)
|
|
uploader = inside(
|
|
f"with db.connection() as c: r=c.execute('SELECT uploader_instance FROM attachments WHERE id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(r['uploader_instance']))"
|
|
)
|
|
if alias == uploader:
|
|
selected = cid
|
|
assert selected
|
|
command(["docker", "stop", "--time", "1", selected])
|
|
ready()
|
|
wait(
|
|
lambda: (
|
|
client.get(
|
|
f"/api/v1/attachments/{a['id']}/content", headers=h
|
|
).status_code
|
|
== 200
|
|
)
|
|
)
|
|
downloaded = client.get(
|
|
f"/api/v1/attachments/{a['id']}/content", headers=h
|
|
)
|
|
assert (
|
|
downloaded.content == raw
|
|
and downloaded.headers["X-Instance-Id"] != uploader
|
|
)
|
|
start("api")
|
|
ready()
|
|
save(
|
|
"private_iam_and_other_api_download",
|
|
{
|
|
"security": security,
|
|
"attachment": a["id"],
|
|
"uploader": uploader,
|
|
"downloader": downloaded.headers["X-Instance-Id"],
|
|
"sha256": hashlib.sha256(raw).hexdigest(),
|
|
},
|
|
)
|
|
report["worker_metrics"] = inside(
|
|
"with db.connection() as c: r=c.execute(\"SELECT job_id,detail FROM pipeline_events WHERE event='child_finished' ORDER BY id\").fetchall()\nprint(json.dumps(r,default=str))"
|
|
)
|
|
report["audit"] = inside(
|
|
"with db.connection() as c: r=c.execute('SELECT job_id,action,operator,reason,created_at FROM pipeline_audit ORDER BY id').fetchall()\nprint(json.dumps(r,default=str))"
|
|
)
|
|
report["complete"] = True
|
|
finally:
|
|
for role in ["api", "broker", "relay", "worker", "reconciler"]:
|
|
try:
|
|
start(role)
|
|
except Exception:
|
|
pass
|
|
output.write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
print("PASS: versioned pipeline scope=" + args.start_at, flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|