109 lines
3.6 KiB
Python
109 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Accepted jobs survive worker identity revoke + working-secret rotation."""
|
|
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, check # noqa: E402
|
|
|
|
|
|
def docker(*args):
|
|
return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True).strip()
|
|
|
|
|
|
def images():
|
|
ids = docker("ps", "-q", "api", "worker").split()
|
|
return sorted(
|
|
set(
|
|
subprocess.check_output(
|
|
["docker", "inspect", "--format", "{{.Image}}", *ids], text=True
|
|
).split()
|
|
)
|
|
)
|
|
|
|
|
|
def main():
|
|
c = Client("https://localhost:8447", str(ROOT / "secrets/ca.pem"))
|
|
alice, bob, _, chat = c.bootstrap()
|
|
before = images()
|
|
docker("stop", "worker")
|
|
accepted = [c.attachment(alice, chat) for _ in range(3)]
|
|
start = time.monotonic()
|
|
rotate = subprocess.run(
|
|
[sys.executable, "scripts/rotate_identity.py"], cwd=ROOT, text=True, capture_output=True
|
|
)
|
|
check(rotate.returncode == 0, "operator rotation failed; inspect private operator execution")
|
|
errors = 0
|
|
deadline = time.monotonic() + 90
|
|
pending = {a[0]["id"] for a in accepted}
|
|
while pending and time.monotonic() < deadline:
|
|
for att, _ in accepted:
|
|
if att["id"] not in pending:
|
|
continue
|
|
result = c.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + att["id"],
|
|
expected=[200, 503],
|
|
actor=alice,
|
|
schema=None,
|
|
)
|
|
if result["status"] == 503:
|
|
errors += 1
|
|
else:
|
|
body = json.loads(result["bytes"])
|
|
check(body["status"] != "failed", "accepted task became failed")
|
|
if body["status"] == "ready":
|
|
pending.remove(att["id"])
|
|
time.sleep(0.2)
|
|
check(not pending, "accepted jobs did not recover")
|
|
rows = []
|
|
for att, original in accepted:
|
|
downloaded = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob)[
|
|
"bytes"
|
|
]
|
|
thumbnail = c.call(
|
|
"GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob
|
|
)["bytes"]
|
|
sha = hashlib.sha256(original).hexdigest()
|
|
rows.append(
|
|
{
|
|
"attachment_id": att["id"],
|
|
"original_sha256": sha,
|
|
"download_sha256": hashlib.sha256(downloaded).hexdigest(),
|
|
"thumbnail_sha256": hashlib.sha256(thumbnail).hexdigest(),
|
|
"thumbnail_nonempty": bool(thumbnail),
|
|
"accepted_original_sha_matches": sha == att["sha256"],
|
|
}
|
|
)
|
|
after = images()
|
|
result = {
|
|
"accepted_jobs": 3,
|
|
"ready_jobs": len(rows),
|
|
"recovery_seconds": round(time.monotonic() - start, 3),
|
|
"observed_api_503": errors,
|
|
"images_before": before,
|
|
"images_after": after,
|
|
"image_unchanged": before == after,
|
|
"rows": rows,
|
|
"passed": before == after
|
|
and all(
|
|
r["original_sha256"] == r["download_sha256"]
|
|
and r["thumbnail_nonempty"]
|
|
and r["accepted_original_sha_matches"]
|
|
for r in rows
|
|
),
|
|
}
|
|
(ROOT / "evidence/compromise-recovery.json").write_text(json.dumps(result, indent=2) + "\n")
|
|
check(result["passed"], "recovery invariant failed")
|
|
print(json.dumps({k: v for k, v in result.items() if k != "rows"}))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|