feat(observability): add bounded OTLP signals, causal jobs and SLO rules
This commit is contained in:
@@ -0,0 +1,64 @@
|
||||
"""Создаёт только отсутствующие локальные файлы; секреты не печатает."""
|
||||
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def create(path, value):
|
||||
try:
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
||||
except FileExistsError:
|
||||
return
|
||||
with os.fdopen(fd, "w") as stream:
|
||||
stream.write(value + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
directory = ROOT / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
pg = directory / "postgres_password"
|
||||
cache = directory / "cache_password"
|
||||
create(pg, secrets.token_urlsafe(32))
|
||||
create(cache, secrets.token_urlsafe(32))
|
||||
create(
|
||||
directory / "database_url",
|
||||
"postgresql://messenger:"
|
||||
+ pg.read_text().strip()
|
||||
+ "@postgres:5432/messenger",
|
||||
)
|
||||
create(
|
||||
directory / "cache_url",
|
||||
"redis://:" + cache.read_text().strip() + "@cache:6379/0",
|
||||
)
|
||||
create(
|
||||
directory / "valkey.conf",
|
||||
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nmaxmemory 64mb\n"
|
||||
'maxmemory-policy allkeys-lru\nsave ""\nappendonly no\nrequirepass '
|
||||
+ cache.read_text().strip(),
|
||||
)
|
||||
create(directory / "s3_root_user", secrets.token_hex(10))
|
||||
create(directory / "s3_root_password", secrets.token_urlsafe(32))
|
||||
create(directory / "s3_access_key", secrets.token_hex(10))
|
||||
create(directory / "s3_secret_key", secrets.token_urlsafe(32))
|
||||
create(directory / "grafana_password", secrets.token_urlsafe(32))
|
||||
create(directory / "broker_password", secrets.token_urlsafe(32))
|
||||
create(
|
||||
directory / "broker_url",
|
||||
"redis://:"
|
||||
+ (directory / "broker_password").read_text().strip()
|
||||
+ "@broker:6379/0",
|
||||
)
|
||||
create(
|
||||
directory / "broker.conf",
|
||||
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nappendonly yes\nappendfsync always\nmaxmemory 192mb\nmaxmemory-policy noeviction\ndir /data\nrequirepass "
|
||||
+ (directory / "broker_password").read_text().strip(),
|
||||
)
|
||||
create(ROOT / ".env", (ROOT / ".env.example").read_text().rstrip())
|
||||
print("Local configuration ready; existing files preserved")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Явные операторские действия только над API текущего Compose-проекта."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(*args):
|
||||
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def containers():
|
||||
ids = command("docker", "compose", "ps", "-q", "api").split()
|
||||
return json.loads(command("docker", "inspect", *ids)) if ids else []
|
||||
|
||||
|
||||
def alias(cid):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=2); "
|
||||
"print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
return command("docker", "exec", cid, "python", "-c", code)
|
||||
|
||||
|
||||
def wait_count(count):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
data = containers()
|
||||
if len(data) == count and all(
|
||||
c["State"].get("Health", {}).get("Status") == "healthy"
|
||||
for c in data
|
||||
):
|
||||
# DNS и проверки HAProxy должны успеть увидеть новый адрес.
|
||||
time.sleep(3)
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError("Replica readiness")
|
||||
|
||||
|
||||
def main():
|
||||
action = sys.argv[1]
|
||||
data = containers()
|
||||
if action == "stop":
|
||||
target = os.environ["TARGET_INSTANCE_ID"]
|
||||
matches = [c for c in data if alias(c["Id"]) == target]
|
||||
if len(matches) != 1 or len(data) < 2:
|
||||
raise RuntimeError("Target must identify one running replica")
|
||||
command("docker", "stop", "--time", "10", matches[0]["Id"])
|
||||
print("Selected API stopped")
|
||||
elif action == "rolling":
|
||||
count = len(data)
|
||||
if count < 2:
|
||||
raise RuntimeError("Need at least two replicas")
|
||||
# Сначала добавляется готовая реплика, затем удаляется одна старая.
|
||||
for old in data:
|
||||
command(
|
||||
"docker",
|
||||
"compose",
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--no-recreate",
|
||||
"--scale",
|
||||
f"api={count + 1}",
|
||||
"api",
|
||||
)
|
||||
wait_count(count + 1)
|
||||
command("docker", "stop", "--time", "10", old["Id"])
|
||||
command("docker", "rm", old["Id"])
|
||||
wait_count(count)
|
||||
print("Sequential replacement completed")
|
||||
else:
|
||||
raise ValueError("Unknown action")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
docker compose restart api postgres cache s3 broker relay worker lb
|
||||
docker compose up -d --no-build --no-recreate --wait
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
exec python3 scripts/cluster.py rolling
|
||||
@@ -0,0 +1,137 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8086")
|
||||
|
||||
|
||||
def command(args, *, input=None):
|
||||
result = subprocess.run(
|
||||
args, cwd=ROOT, input=input, capture_output=True, text=True
|
||||
)
|
||||
if result.returncode:
|
||||
raise RuntimeError(
|
||||
"Command failed: "
|
||||
+ args[0]
|
||||
+ "; "
|
||||
+ result.stderr.replace(str(ROOT), "<repository>")
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return command(["docker", "compose", *args])
|
||||
|
||||
|
||||
def sql(query, *, database="messenger"):
|
||||
return compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
database,
|
||||
"-A",
|
||||
"-t",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
"-c",
|
||||
query,
|
||||
)
|
||||
|
||||
|
||||
def cache(command_name, *args):
|
||||
script = (
|
||||
'export REDISCLI_AUTH="$(cat /run/secrets/cache_password)"; '
|
||||
'exec valkey-cli "$@"'
|
||||
)
|
||||
return compose(
|
||||
"exec", "-T", "cache", "sh", "-c", script, "sh", command_name, *args
|
||||
)
|
||||
|
||||
|
||||
def ready(base=BASE, *, timeout=30):
|
||||
end = time.monotonic() + timeout
|
||||
with httpx.Client(base_url=base, trust_env=False, timeout=6) as client:
|
||||
while time.monotonic() < end:
|
||||
try:
|
||||
if client.get("/health/ready").status_code == 200:
|
||||
return
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
raise TimeoutError("API readiness")
|
||||
|
||||
|
||||
def actor(client):
|
||||
username = "verify_" + uuid4().hex[:20]
|
||||
password = "Synthetic-" + uuid4().hex + ":🌍"
|
||||
result = client.post(
|
||||
"/api/v1/users",
|
||||
json={
|
||||
"username": username,
|
||||
"display_name": "Synthetic",
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
assert result.status_code == 201
|
||||
user = result.json()
|
||||
session = client.post(
|
||||
"/api/v1/auth/sessions", auth=httpx.BasicAuth(username, password)
|
||||
)
|
||||
assert session.status_code == 201
|
||||
return user, password, session.json()
|
||||
|
||||
|
||||
def auth(session):
|
||||
return {"Authorization": "Bearer " + session["token"]}
|
||||
|
||||
|
||||
def extra_api(*, ttl=3600, pool_size=1):
|
||||
name = "mtusi-lab6-test-" + uuid4().hex[:10]
|
||||
compose(
|
||||
"run",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--name",
|
||||
name,
|
||||
"-e",
|
||||
"SESSION_TTL_SECONDS=" + str(ttl),
|
||||
"-e",
|
||||
"DB_POOL_SIZE=" + str(pool_size),
|
||||
"-p",
|
||||
"127.0.0.1::8080",
|
||||
"api",
|
||||
)
|
||||
try:
|
||||
info = json.loads(command(["docker", "inspect", name]))[0]
|
||||
base = (
|
||||
"http://127.0.0.1:"
|
||||
+ info["NetworkSettings"]["Ports"]["8080/tcp"][0]["HostPort"]
|
||||
)
|
||||
ready(base)
|
||||
return name, base
|
||||
except Exception:
|
||||
command(["docker", "rm", "-f", name])
|
||||
raise
|
||||
|
||||
|
||||
def instances():
|
||||
data = {}
|
||||
for role in ["api", "postgres", "cache"]:
|
||||
cid = compose("ps", "-q", role)
|
||||
info = json.loads(command(["docker", "inspect", cid]))[0]
|
||||
data[role] = {
|
||||
"container": cid[:12],
|
||||
"pid": info["State"]["Pid"],
|
||||
"started_at": info["State"]["StartedAt"],
|
||||
}
|
||||
return data
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
exec python3 scripts/cluster.py stop
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
docker compose stop --timeout 1 worker
|
||||
@@ -0,0 +1,254 @@
|
||||
import argparse
|
||||
import atexit
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROGRESS = {}
|
||||
|
||||
|
||||
def save_incomplete():
|
||||
receipt = PROGRESS.get("receipt")
|
||||
if receipt and receipt.get("result") != "PASS":
|
||||
target = PROGRESS["output"].with_name(
|
||||
PROGRESS["output"].stem + "-failed.json"
|
||||
)
|
||||
if target.exists():
|
||||
target = target.with_name(
|
||||
target.stem + "-" + uuid4().hex[:8] + target.suffix
|
||||
)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(
|
||||
json.dumps(
|
||||
{**receipt, "result": "INCOMPLETE"},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
atexit.register(save_incomplete)
|
||||
|
||||
|
||||
def run(args, *, cwd, env, expected=0):
|
||||
result = subprocess.run(
|
||||
args, cwd=cwd, env=env, capture_output=True, text=True
|
||||
)
|
||||
|
||||
def redact(text):
|
||||
return (
|
||||
text.replace(str(cwd), "<clean-clone>")
|
||||
.replace(str(ROOT), "<repository>")
|
||||
.replace(str(Path.home()), "<home>")
|
||||
)
|
||||
|
||||
if result.returncode != expected:
|
||||
if "receipt" in PROGRESS:
|
||||
PROGRESS["receipt"]["failed_action"] = redact(" ".join(args))
|
||||
raise RuntimeError(redact(result.stdout + result.stderr))
|
||||
return redact(result.stdout.strip())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/clean-clone.json")
|
||||
args = parser.parse_args()
|
||||
output = ROOT / args.output
|
||||
if output.exists():
|
||||
raise SystemExit("Choose new output path")
|
||||
assert not subprocess.check_output(
|
||||
["git", "status", "--porcelain"], cwd=ROOT
|
||||
)
|
||||
sha = subprocess.check_output(
|
||||
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
|
||||
).strip()
|
||||
identity = uuid4().hex[:10]
|
||||
project = "mtusi-lab6-clean-" + identity
|
||||
with socket.socket() as reservation:
|
||||
reservation.bind(("127.0.0.1", 0))
|
||||
public_port = str(reservation.getsockname()[1])
|
||||
env = {
|
||||
**os.environ,
|
||||
"COMPOSE_PROJECT_NAME": project,
|
||||
"API_IMAGE": "mtusi-messenger:lab6-clean-" + identity,
|
||||
"PUBLIC_PORT": public_port,
|
||||
"VCS_REF": sha,
|
||||
"COMPOSE_FILE": "compose.yaml:compose.test.yaml",
|
||||
"NO_PROXY": "localhost,127.0.0.1",
|
||||
"no_proxy": "localhost,127.0.0.1",
|
||||
}
|
||||
receipt = {
|
||||
"date": datetime.now(UTC).isoformat(),
|
||||
"source_commit": sha,
|
||||
"fresh_secrets_environment_and_volumes": True,
|
||||
"command": "python scripts/verify_clean.py",
|
||||
"fixed_host_port_for_restart": int(public_port),
|
||||
}
|
||||
PROGRESS.update(receipt=receipt, output=output)
|
||||
with tempfile.TemporaryDirectory(prefix="mtusi-lab6-clean-") as temporary:
|
||||
clone = Path(temporary) / "lab6"
|
||||
run(
|
||||
[
|
||||
"git",
|
||||
"clone",
|
||||
"--no-hardlinks",
|
||||
"--branch",
|
||||
"solution/bvt2301-maslov",
|
||||
str(ROOT),
|
||||
str(clone),
|
||||
],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
)
|
||||
for name in [".env", ".venv", "secrets"]:
|
||||
assert not (clone / name).exists()
|
||||
run(["uv", "sync", "--locked"], cwd=clone, env=env)
|
||||
receipt["static_checks"] = run(
|
||||
["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
receipt["unit_tests"] = run(["make", "unit"], cwd=clone, env=env)
|
||||
run(["make", "bootstrap"], cwd=clone, env=env)
|
||||
try:
|
||||
run(
|
||||
["docker", "compose", "up", "-d", "--build", "--wait"],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
env["BASE_URL"] = "http://127.0.0.1:" + public_port
|
||||
receipt["smoke"] = run(
|
||||
["make", "test", "BASE_URL=" + env["BASE_URL"]],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
receipt["http_tests"] = run(
|
||||
[".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env
|
||||
)
|
||||
run(
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_cluster",
|
||||
"--output",
|
||||
"evidence/clean-cluster.json",
|
||||
],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
for command_args in [
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_cleanup",
|
||||
"seed",
|
||||
"--seed",
|
||||
"evidence/clean-gc-seed.json",
|
||||
],
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_pipeline",
|
||||
"--output",
|
||||
"evidence/clean-pipeline.json",
|
||||
],
|
||||
[
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.verify_cleanup",
|
||||
"verify",
|
||||
"--seed",
|
||||
"evidence/clean-gc-seed.json",
|
||||
"--output",
|
||||
"evidence/clean-cleanup.json",
|
||||
],
|
||||
]:
|
||||
run(command_args, cwd=clone, env=env)
|
||||
receipt["pipeline"] = json.loads(
|
||||
(clone / "evidence/clean-pipeline.json").read_text()
|
||||
)
|
||||
receipt["cleanup"] = json.loads(
|
||||
(clone / "evidence/clean-cleanup.json").read_text()
|
||||
)
|
||||
receipt["cluster"] = json.loads(
|
||||
(clone / "evidence/clean-cluster.json").read_text()
|
||||
)
|
||||
import hashlib
|
||||
|
||||
expected = {
|
||||
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in (clone / "src").glob("*.py")
|
||||
}
|
||||
ids = run(
|
||||
["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env
|
||||
).split()
|
||||
assert len(ids) == 2
|
||||
code = (
|
||||
"import hashlib,json,pathlib; print(json.dumps({p.name:"
|
||||
"hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))"
|
||||
)
|
||||
ids += run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"ps",
|
||||
"-q",
|
||||
"worker",
|
||||
"relay",
|
||||
"reconciler",
|
||||
"controller",
|
||||
],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
).split()
|
||||
assert len(ids) == 6
|
||||
for cid in ids:
|
||||
actual = json.loads(
|
||||
run(
|
||||
["docker", "exec", cid, "python", "-c", code],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
)
|
||||
assert actual == expected
|
||||
info = json.loads(
|
||||
run(["docker", "inspect", cid], cwd=clone, env=env)
|
||||
)[0]
|
||||
assert (
|
||||
info["Config"]["Labels"][
|
||||
"org.opencontainers.image.revision"
|
||||
]
|
||||
== sha
|
||||
)
|
||||
assert not info["HostConfig"]["PortBindings"]
|
||||
receipt[
|
||||
"api_worker_relay_reconciler_runtime_hashes_and_revision_match"
|
||||
] = True
|
||||
receipt["fsck"] = run(
|
||||
["git", "fsck", "--full"], cwd=clone, env=env
|
||||
)
|
||||
receipt["checks_completed"] = True
|
||||
finally:
|
||||
assert project.startswith("mtusi-lab6-clean-")
|
||||
run(
|
||||
["docker", "compose", "down", "--volumes", "--remove-orphans"],
|
||||
cwd=clone,
|
||||
env=env,
|
||||
)
|
||||
receipt["result"] = "PASS"
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
|
||||
print(
|
||||
"PASS: fresh clone, build, HTTP tests, failover, rolling, persistence and runtime hashes"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Real S3 retention test: seed is separate from verification output."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
|
||||
from scripts.verify_pipeline import inside, operator, wait
|
||||
|
||||
|
||||
def seed(path):
|
||||
if path.exists():
|
||||
raise ValueError("Choose a new seed file")
|
||||
ready()
|
||||
compose("stop", "--timeout", "1", "reconciler")
|
||||
raw = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
with httpx.Client(base_url=BASE, trust_env=False, timeout=20) as c:
|
||||
user, _, session = actor(c)
|
||||
headers = auth(session)
|
||||
result = c.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers,
|
||||
json={
|
||||
"title": "Cleanup synthetic seed",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert result.status_code == 201
|
||||
chat = result.json()["id"]
|
||||
r = c.post(
|
||||
f"/api/v1/chats/{chat}/attachments",
|
||||
headers={**headers, "Content-Type": "image/png"},
|
||||
params={"filename": "retained.png"},
|
||||
content=raw,
|
||||
)
|
||||
assert r.status_code == 202
|
||||
attachment = r.json()["id"]
|
||||
wait(
|
||||
lambda: (
|
||||
c.get(
|
||||
"/api/v1/attachments/" + attachment, headers=headers
|
||||
).json()["status"]
|
||||
== "ready"
|
||||
)
|
||||
)
|
||||
job = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({attachment!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
|
||||
)
|
||||
data = {
|
||||
"seed_id": uuid4().hex,
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"date": datetime.now(timezone.utc).isoformat(),
|
||||
"ready_attachment": attachment,
|
||||
"ready_job": job,
|
||||
"chat": chat,
|
||||
"owner": user["id"],
|
||||
}
|
||||
# Real old v1 objects will become obsolete after v2 is ready.
|
||||
second = operator(job, version=2)
|
||||
wait(
|
||||
lambda: inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT state FROM jobs WHERE id=%s',({second!r},)).fetchone()\nprint(json.dumps(r['state']=='ready'))"
|
||||
)
|
||||
)
|
||||
data["version2_job"] = second
|
||||
data["orphan"] = f"objects/cleanup-{data['seed_id']}/old-orphan"
|
||||
data["failed_id"] = str(uuid4())
|
||||
data["active_id"] = str(uuid4())
|
||||
data["reservation_id"] = str(uuid4())
|
||||
data["expired_reservation_id"] = str(uuid4())
|
||||
data["failed_source"] = f"objects/staging/{data['failed_id']}/original"
|
||||
data["active_source"] = f"objects/staging/{data['active_id']}/original"
|
||||
data["reservation_key"] = (
|
||||
f"objects/staging/{data['reservation_id']}/original"
|
||||
)
|
||||
data["expired_reservation_key"] = (
|
||||
f"objects/staging/{data['expired_reservation_id']}/original"
|
||||
)
|
||||
inside(f"""from pathlib import Path
|
||||
raw=Path('/app/src/__init__.py').read_bytes() or b'fixture'
|
||||
keys={json.dumps([data[k] for k in ["orphan", "failed_source", "active_source", "reservation_key", "expired_reservation_key"]])}
|
||||
for key in keys: store.put(key=key,stream=io.BytesIO(raw),size=len(raw),content_type='image/png',sha256=hashlib.sha256(raw).hexdigest())
|
||||
with db.connection() as c, c.transaction():
|
||||
for identity,state in [({data["failed_id"]!r},'failed'),({data["active_id"]!r},'processing')]:
|
||||
key='objects/staging/'+identity+'/original'
|
||||
c.execute(\"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,error_code,uploader_instance,expires_at) VALUES(%s,%s,%s,'synthetic.png','image/png',%s,%s,%s,%s,%s,'cleanup-fixture',clock_timestamp())\",(identity,{chat!r},{user["id"]!r},len(raw),hashlib.sha256(raw).hexdigest(),key,state,'invalid_image' if state=='failed' else None))
|
||||
c.execute(\"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state,attempt,lease_until,updated_at,error_code) VALUES(%s,%s,1,%s,%s,1,clock_timestamp()+interval '1 hour',clock_timestamp()-interval '2 days',%s)\",(uuid4(),identity,key,state,'invalid_image' if state=='failed' else None))
|
||||
c.execute(\"INSERT INTO upload_reservations(id,object_key,expires_at) VALUES(%s,%s,clock_timestamp()+interval '1 hour'),(%s,%s,clock_timestamp()-interval '1 second')\",({data["reservation_id"]!r},{data["reservation_key"]!r},{data["expired_reservation_id"]!r},{data["expired_reservation_key"]!r}))
|
||||
print('true')""")
|
||||
data["seeded_unix"] = time.time()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
|
||||
print(
|
||||
"PASS: retention seed created; real S3 age must reach300s", flush=True
|
||||
)
|
||||
|
||||
|
||||
def verify(seed_path, output):
|
||||
if output.exists():
|
||||
raise ValueError("Choose a new output path")
|
||||
data = json.loads(seed_path.read_text())
|
||||
report = {
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"seed": seed_path.name,
|
||||
"complete": False,
|
||||
}
|
||||
try:
|
||||
compose("stop", "--timeout", "1", "reconciler")
|
||||
|
||||
def age():
|
||||
return inside(
|
||||
f"h=store.head(key={data['orphan']!r})\nassert h is not None\nwith db.connection() as c: now=c.execute('SELECT clock_timestamp() now').fetchone()['now']\nprint(json.dumps((now-h.last_modified).total_seconds()))"
|
||||
)
|
||||
|
||||
while age() < 302:
|
||||
time.sleep(min(5, 303 - age()))
|
||||
report["real_s3_age_seconds"] = age()
|
||||
fresh = f"objects/cleanup-{data['seed_id']}/fresh-orphan"
|
||||
inside(
|
||||
f"b=b'fresh'\nstore.put(key={fresh!r},stream=io.BytesIO(b),size=len(b),content_type='image/png',sha256=hashlib.sha256(b).hexdigest())\nprint('true')"
|
||||
)
|
||||
oldkeys = [
|
||||
data["orphan"],
|
||||
data["failed_source"],
|
||||
data["expired_reservation_key"],
|
||||
] + [
|
||||
f"objects/{data['ready_attachment']}/v1/{v}"
|
||||
for v in ["original", "thumbnail", "crop", "optimized"]
|
||||
]
|
||||
protected = [data["active_source"], data["reservation_key"], fresh] + [
|
||||
f"objects/{data['ready_attachment']}/v2/{v}"
|
||||
for v in ["original", "thumbnail", "crop", "optimized"]
|
||||
]
|
||||
before = inside(
|
||||
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
||||
)
|
||||
assert all(before.values())
|
||||
dry = inside(
|
||||
"print(json.dumps(reconcile(db,store,apply=False,grace=300)))"
|
||||
)
|
||||
candidates = {a["key"] for a in dry["actions"]}
|
||||
assert set(oldkeys) <= candidates and not set(protected) & candidates
|
||||
afterdry = inside(
|
||||
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
||||
)
|
||||
assert afterdry == before
|
||||
applied = inside(
|
||||
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
|
||||
)
|
||||
repeat = inside(
|
||||
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
|
||||
)
|
||||
assert not repeat["actions"]
|
||||
after = inside(
|
||||
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
|
||||
)
|
||||
assert all(not after[k] for k in oldkeys) and all(
|
||||
after[k] for k in protected
|
||||
)
|
||||
failed = inside(
|
||||
f"with db.connection() as c: j=c.execute('SELECT id,error_code FROM jobs WHERE attachment_id=%s',({data['failed_id']!r},)).fetchone()\nprint(json.dumps(j,default=str))"
|
||||
)
|
||||
assert failed["error_code"] == "source_expired"
|
||||
try:
|
||||
operator(failed["id"])
|
||||
except RuntimeError:
|
||||
replay_refused = True
|
||||
else:
|
||||
raise AssertionError("Expired source replay must fail")
|
||||
# Expired reservation was removed, so a delayed API cannot commit its handoff.
|
||||
fenced = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp()',({data['expired_reservation_id']!r},)).fetchone()\nprint(json.dumps(r['n']==0))"
|
||||
)
|
||||
assert fenced
|
||||
report.update(
|
||||
dry_run=dry,
|
||||
apply=applied,
|
||||
repeat=repeat,
|
||||
before=before,
|
||||
after=after,
|
||||
dry_run_unchanged=True,
|
||||
active_current_and_fresh_protected=True,
|
||||
expired_source_replay_refused=replay_refused,
|
||||
expired_writer_fenced=fenced,
|
||||
complete=True,
|
||||
)
|
||||
finally:
|
||||
# Remove only this fixture's artificial active reservation/job; never real jobs.
|
||||
inside(
|
||||
f"with db.connection() as c, c.transaction():\n c.execute(\"UPDATE jobs SET state='failed',error_code='fixture_complete',lease_until=NULL WHERE attachment_id=%s\",({data['active_id']!r},))\n c.execute(\"UPDATE attachments SET state='failed',error_code='fixture_complete' WHERE id=%s\",({data['active_id']!r},))\n c.execute('DELETE FROM upload_reservations WHERE id=%s',({data['reservation_id']!r},))\nprint('true')"
|
||||
)
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "reconciler")
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(
|
||||
"PASS: real retention, dry-run/apply/repeat, current/active/fresh protection",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("action", choices=["seed", "verify"])
|
||||
p.add_argument("--seed", required=True, type=Path)
|
||||
p.add_argument("--output", type=Path)
|
||||
a = p.parse_args()
|
||||
if a.action == "seed":
|
||||
seed(a.seed)
|
||||
else:
|
||||
if a.output is None:
|
||||
p.error("output required")
|
||||
verify(a.seed, a.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,347 @@
|
||||
"""Проверки реальных процессов, failover, rolling и общей идемпотентности."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts import cluster
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
|
||||
|
||||
|
||||
def snapshot():
|
||||
ids = compose("ps", "-q").split()
|
||||
result = []
|
||||
for c in json.loads(command(["docker", "inspect", *ids])):
|
||||
role = c["Config"]["Labels"]["com.docker.compose.service"]
|
||||
entry = {
|
||||
"role": role,
|
||||
"container": c["Id"][:12],
|
||||
"pid": c["State"]["Pid"],
|
||||
"started_at": c["State"]["StartedAt"],
|
||||
"ports": c["HostConfig"]["PortBindings"],
|
||||
"cpu_nano": c["HostConfig"]["NanoCpus"],
|
||||
"memory_limit": c["HostConfig"]["Memory"],
|
||||
}
|
||||
if role == "api":
|
||||
entry["instance"] = cluster.alias(c["Id"])
|
||||
assert not entry["ports"]
|
||||
result.append(entry)
|
||||
return result
|
||||
|
||||
|
||||
def exercise(client, session, chat, action, *, seconds=15):
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
observations, confirmed, errors, reads = [], {}, 0, 0
|
||||
started = time.monotonic()
|
||||
finished = threading.Event()
|
||||
action_result = {}
|
||||
|
||||
def operate():
|
||||
time.sleep(0.5)
|
||||
try:
|
||||
action()
|
||||
except BaseException as exc:
|
||||
action_result["error"] = type(exc).__name__
|
||||
finally:
|
||||
action_result["finished"] = round(time.monotonic() - started, 3)
|
||||
finished.set()
|
||||
|
||||
thread = threading.Thread(target=operate)
|
||||
thread.start()
|
||||
key = uuid4().hex
|
||||
try:
|
||||
while (
|
||||
time.monotonic() - started < seconds
|
||||
or not finished.is_set()
|
||||
or time.monotonic() - started
|
||||
< action_result.get("finished", 0) + 2
|
||||
):
|
||||
h = {**auth(session), "Idempotency-Key": key}
|
||||
try:
|
||||
r = client.post(path, headers=h, json={"text": key})
|
||||
if r.status_code != 201:
|
||||
errors += 1
|
||||
continue
|
||||
m = r.json()
|
||||
confirmed[m["id"]] = m
|
||||
observations.append(
|
||||
{
|
||||
"at": round(time.monotonic() - started, 3),
|
||||
"instance": r.headers["x-instance-id"],
|
||||
}
|
||||
)
|
||||
key = uuid4().hex
|
||||
read = client.get(
|
||||
path, headers=auth(session), params={"limit": 100}
|
||||
)
|
||||
reads += read.status_code == 200
|
||||
errors += read.status_code != 200
|
||||
except httpx.HTTPError:
|
||||
errors += 1
|
||||
time.sleep(0.04)
|
||||
finally:
|
||||
thread.join(timeout=90)
|
||||
assert not action_result.get("error"), action_result
|
||||
found, cursor = [], None
|
||||
while True:
|
||||
r = client.get(
|
||||
path,
|
||||
headers=auth(session),
|
||||
params={"limit": 100, **({"cursor": cursor} if cursor else {})},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
page = r.json()
|
||||
found.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
by_id = {m["id"]: m for m in found}
|
||||
assert all(by_id.get(k) == v for k, v in confirmed.items())
|
||||
texts = [m["text"] for m in found]
|
||||
assert len(texts) == len(set(texts))
|
||||
recovery = next(
|
||||
(
|
||||
x["at"]
|
||||
for x in observations
|
||||
if x["at"] >= action_result["finished"]
|
||||
),
|
||||
None,
|
||||
)
|
||||
assert recovery is not None and (seconds > 20 or recovery <= 10)
|
||||
elapsed = time.monotonic() - started
|
||||
return {
|
||||
"confirmed_writes": len(confirmed),
|
||||
"successful_reads": reads,
|
||||
"errors": errors,
|
||||
"successful_rps": round((len(confirmed) + reads) / elapsed, 2),
|
||||
"first_success_after_action_seconds": recovery,
|
||||
"action": action_result,
|
||||
"observations": observations,
|
||||
"confirmed_preserved": True,
|
||||
"duplicate_texts": False,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/cluster-repeat.json")
|
||||
args = parser.parse_args()
|
||||
report = {
|
||||
"date": datetime.now(timezone.utc).isoformat(),
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"complete": False,
|
||||
}
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
ready()
|
||||
initial = snapshot()
|
||||
apis = [c for c in initial if c["role"] == "api"]
|
||||
assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2
|
||||
report["before"] = initial
|
||||
with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client:
|
||||
user, _, session = actor(client)
|
||||
r = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=auth(session),
|
||||
json={
|
||||
"title": "Cluster verification",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
chat = r.json()
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
key = uuid4().hex
|
||||
|
||||
def repeat(_):
|
||||
with httpx.Client(
|
||||
base_url=BASE, trust_env=False, timeout=8
|
||||
) as c:
|
||||
r = c.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": key},
|
||||
json={"text": "Same concurrent request"},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
return r.json(), r.headers["x-instance-id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
repeated = list(pool.map(repeat, range(30)))
|
||||
assert len({m[0]["id"] for m in repeated}) == 1
|
||||
assert len({m[1] for m in repeated}) == 2
|
||||
report["parallel_replay"] = {
|
||||
"requests": 30,
|
||||
"messages": 1,
|
||||
"instances": sorted({m[1] for m in repeated}),
|
||||
}
|
||||
# Сбой второй записи транзакции не должен оставить сообщение.
|
||||
fault_key = uuid4().hex
|
||||
fault_text = "Atomic rollback " + fault_key
|
||||
sql(
|
||||
"CREATE FUNCTION student_idempo_fault() RETURNS trigger "
|
||||
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='"
|
||||
+ fault_key
|
||||
+ "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; "
|
||||
"CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency "
|
||||
"FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()"
|
||||
)
|
||||
try:
|
||||
failed = client.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": fault_key},
|
||||
json={"text": fault_text},
|
||||
)
|
||||
assert failed.status_code == 500
|
||||
assert (
|
||||
sql(
|
||||
"SELECT count(*) FROM messages WHERE text='"
|
||||
+ fault_text
|
||||
+ "'"
|
||||
)
|
||||
== "0"
|
||||
)
|
||||
finally:
|
||||
sql(
|
||||
"DROP TRIGGER student_idempo_fault ON idempotency; "
|
||||
"DROP FUNCTION student_idempo_fault()"
|
||||
)
|
||||
retried = client.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": fault_key},
|
||||
json={"text": fault_text},
|
||||
)
|
||||
assert retried.status_code == 201
|
||||
report["atomic_rollback_and_5xx_retry"] = True
|
||||
sql(
|
||||
"UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' "
|
||||
"WHERE key='" + fault_key + "'"
|
||||
)
|
||||
expired = client.post(
|
||||
path,
|
||||
headers={**auth(session), "Idempotency-Key": fault_key},
|
||||
json={"text": "New retention window"},
|
||||
)
|
||||
assert expired.status_code == 201
|
||||
assert expired.json()["id"] != retried.json()["id"]
|
||||
report["expired_key_can_be_reused"] = True
|
||||
# Отдельный чат каждой серии: проверки дубликатов не смешивают тексты.
|
||||
report["failovers"] = []
|
||||
for api in apis:
|
||||
|
||||
def stop(target=api["instance"]):
|
||||
env = {**os.environ, "TARGET_INSTANCE_ID": target}
|
||||
subprocess.run(
|
||||
["scripts/stop-one.sh"],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
r = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=auth(session),
|
||||
json={"title": "Failover", "member_ids": [user["id"]]},
|
||||
)
|
||||
result = exercise(client, session, r.json(), stop)
|
||||
assert api["instance"] not in [
|
||||
o["instance"] for o in result["observations"][-5:]
|
||||
]
|
||||
assert (
|
||||
command(
|
||||
[
|
||||
"docker",
|
||||
"inspect",
|
||||
"-f",
|
||||
"{{.State.Running}}",
|
||||
api["container"],
|
||||
]
|
||||
)
|
||||
== "false"
|
||||
)
|
||||
result["stopped"] = api
|
||||
report["failovers"].append(result)
|
||||
compose("up", "-d", "--no-build", "--wait")
|
||||
ready()
|
||||
r = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=auth(session),
|
||||
json={"title": "Rolling", "member_ids": [user["id"]]},
|
||||
)
|
||||
before_roll = snapshot()
|
||||
report["rolling"] = exercise(
|
||||
client,
|
||||
session,
|
||||
r.json(),
|
||||
lambda: command(["scripts/rolling-update.sh"]),
|
||||
seconds=45,
|
||||
)
|
||||
after_roll = snapshot()
|
||||
assert not (
|
||||
{c["container"] for c in before_roll if c["role"] == "api"}
|
||||
& {c["container"] for c in after_roll if c["role"] == "api"}
|
||||
)
|
||||
report["rolling"]["before"] = before_roll
|
||||
report["rolling"]["after"] = after_roll
|
||||
replay = repeat(0)[0]
|
||||
assert replay == repeated[0][0]
|
||||
report["replay_after_replacement"] = True
|
||||
before = snapshot()
|
||||
env = {
|
||||
**os.environ,
|
||||
"NO_PROXY": "localhost,127.0.0.1",
|
||||
"no_proxy": "localhost,127.0.0.1",
|
||||
}
|
||||
for mode, hook in [
|
||||
("failover", "scripts/stop-one.sh"),
|
||||
("persistence", "scripts/restart.sh"),
|
||||
]:
|
||||
if mode == "persistence":
|
||||
compose("up", "-d", "--no-build", "--wait")
|
||||
ready()
|
||||
before = snapshot()
|
||||
run = subprocess.run(
|
||||
[
|
||||
".venv/bin/python",
|
||||
"tests/resilience.py",
|
||||
"--mode",
|
||||
mode,
|
||||
"--action-script",
|
||||
hook,
|
||||
"--base-url",
|
||||
BASE,
|
||||
],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert run.returncode == 0, run.stdout + run.stderr
|
||||
report["public_" + mode] = run.stdout.strip()
|
||||
after = snapshot()
|
||||
for role in ["postgres", "cache", "s3", "broker", "lb"]:
|
||||
old = next(c for c in before if c["role"] == role)
|
||||
new = next(c for c in after if c["role"] == role)
|
||||
assert old["started_at"] != new["started_at"]
|
||||
report["after_persistence"] = after
|
||||
report["complete"] = True
|
||||
finally:
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print("PASS: process failover, rolling, replay and full persistence")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,687 @@
|
||||
"""Repeatable fault injection for grade 5, track 2; synthetic data only."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
|
||||
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
|
||||
os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1")
|
||||
os.environ.setdefault("no_proxy", "localhost,127.0.0.1")
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
|
||||
|
||||
PREFIX = """import io,json,hashlib,time,urllib.request
|
||||
from uuid import UUID,uuid4
|
||||
from psycopg.types.json import Jsonb
|
||||
from src.pipeline import database,broker,envelope,STREAM,GROUP
|
||||
from src.object_store import ObjectStore,BUCKET
|
||||
from src.operator import change
|
||||
from src.reconcile import reconcile
|
||||
db,store,queue=database(),ObjectStore(),broker()
|
||||
"""
|
||||
|
||||
|
||||
def inside(code):
|
||||
cid = compose("ps", "-q", "controller").split()[0]
|
||||
return json.loads(
|
||||
command(
|
||||
["docker", "exec", "-i", cid, "python", "-"], input=PREFIX + code
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def wait(fn, timeout=90):
|
||||
end = time.monotonic() + timeout
|
||||
while time.monotonic() < end:
|
||||
value = fn()
|
||||
if value:
|
||||
return value
|
||||
time.sleep(0.15)
|
||||
raise TimeoutError("Fault scenario condition not reached")
|
||||
|
||||
|
||||
def stop(role):
|
||||
compose("stop", "--timeout", "1", role)
|
||||
|
||||
|
||||
def start(role):
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", role)
|
||||
|
||||
|
||||
def row(job):
|
||||
return inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\nprint(json.dumps(r,default=str))"
|
||||
)
|
||||
|
||||
|
||||
def inject(job, phase, action, remaining=1):
|
||||
return inside(
|
||||
f"with db.connection() as c: r=c.execute('INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES(%s,%s,%s,%s) RETURNING id',({job!r},{phase!r},{action!r},{remaining})).fetchone()\nprint(json.dumps(r['id']))"
|
||||
)
|
||||
|
||||
|
||||
def hit(identity):
|
||||
return inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT hit_at IS NOT NULL hit FROM pipeline_faults WHERE id=%s',({identity},)).fetchone()\nprint(json.dumps(r['hit']))"
|
||||
)
|
||||
|
||||
|
||||
def operator(job, version=None):
|
||||
return inside(
|
||||
f"print(json.dumps(change(db,store,job_id=UUID({job!r}),operator='synthetic-verifier',reason='Explicit fault matrix replay',version={version!r})))"
|
||||
)
|
||||
|
||||
|
||||
def duplicate(job, count=5):
|
||||
return inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\ne=envelope(r)\nids=[queue.xadd(STREAM,{{'job':json.dumps(e)}}) for _ in range({count})]\nprint(json.dumps(ids))"
|
||||
)
|
||||
|
||||
|
||||
def inspect_effect(job, *, pending=False):
|
||||
return inside(f"""with db.connection() as c:
|
||||
j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()
|
||||
v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall()
|
||||
p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n']
|
||||
e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall()
|
||||
keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")]
|
||||
pending_entries=[]
|
||||
if {pending!r}:
|
||||
for item in queue.xpending_range(STREAM,GROUP,'-','+',1000):
|
||||
entry=item['message_id']
|
||||
messages=queue.xrange(STREAM,min=entry,max=entry)
|
||||
if messages and json.loads(messages[0][1].get('job','{{}}')).get('job_id')==str(j['id']): pending_entries.append(entry)
|
||||
print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e,'pending_stream_entries':pending_entries}},default=str))""")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/pipeline-repeat.json")
|
||||
parser.add_argument("--repeats", type=int, default=2)
|
||||
parser.add_argument(
|
||||
"--start-at", choices=["all", "versions"], default="all"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
output = ROOT / args.output
|
||||
if output.exists():
|
||||
raise ValueError("Choose a new output path")
|
||||
report = {
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"date": datetime.now(timezone.utc).isoformat(),
|
||||
"track": 2,
|
||||
"scope": args.start_at,
|
||||
"complete": False,
|
||||
"cases": [],
|
||||
}
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def save(name, details):
|
||||
report["cases"].append({"case": name, **details})
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print("PASS:", name, flush=True)
|
||||
|
||||
try:
|
||||
ready()
|
||||
# Check instrumentation is really enabled and production default is off.
|
||||
cid = compose("ps", "-q", "api").split()[0]
|
||||
assert (
|
||||
command(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cid,
|
||||
"python",
|
||||
"-c",
|
||||
"import os;print(os.getenv('PIPELINE_FAULTS'))",
|
||||
]
|
||||
)
|
||||
== "1"
|
||||
)
|
||||
stop("reconciler")
|
||||
with httpx.Client(
|
||||
base_url=BASE, trust_env=False, timeout=25
|
||||
) as client:
|
||||
user, _, session = actor(client)
|
||||
h = auth(session)
|
||||
c = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=h,
|
||||
json={
|
||||
"title": "Pipeline synthetic faults",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert c.status_code == 201
|
||||
chat = c.json()["id"]
|
||||
raw_io = io.BytesIO()
|
||||
Image.new("RGBA", (320, 200), (20, 50, 170, 120)).save(
|
||||
raw_io, format="PNG"
|
||||
)
|
||||
raw = raw_io.getvalue()
|
||||
|
||||
def upload():
|
||||
r = client.post(
|
||||
f"/api/v1/chats/{chat}/attachments",
|
||||
headers={**h, "Content-Type": "image/png"},
|
||||
params={"filename": "synthetic.png"},
|
||||
content=raw,
|
||||
)
|
||||
assert r.status_code == 202, r.text
|
||||
a = r.json()
|
||||
assert a["status"] == "queued" and not a["variants"]
|
||||
j = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
|
||||
)
|
||||
return a, j
|
||||
|
||||
def metadata(a):
|
||||
r = client.get("/api/v1/attachments/" + a["id"], headers=h)
|
||||
assert r.status_code == 200, r.text
|
||||
return r.json()
|
||||
|
||||
def until(a, state="ready", timeout=90):
|
||||
return wait(
|
||||
lambda: (
|
||||
r if (r := metadata(a))["status"] == state else None
|
||||
),
|
||||
timeout,
|
||||
)
|
||||
|
||||
def confirm(a, j):
|
||||
until(a)
|
||||
downloaded = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
)
|
||||
assert (
|
||||
downloaded.status_code == 200 and downloaded.content == raw
|
||||
)
|
||||
m = metadata(a)
|
||||
assert (
|
||||
m["sha256"] == hashlib.sha256(raw).hexdigest()
|
||||
and m["error_code"] is None
|
||||
)
|
||||
assert {v["name"] for v in m["variants"]} == {
|
||||
"thumbnail",
|
||||
"crop",
|
||||
"optimized",
|
||||
}
|
||||
for v in m["variants"]:
|
||||
r = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content",
|
||||
headers=h,
|
||||
params={"variant": v["name"]},
|
||||
)
|
||||
assert (
|
||||
r.status_code == 200
|
||||
and hashlib.sha256(r.content).hexdigest()
|
||||
== v["sha256"]
|
||||
)
|
||||
with Image.open(io.BytesIO(r.content)) as image:
|
||||
assert (
|
||||
image.size
|
||||
== {
|
||||
"thumbnail": (256, 160),
|
||||
"crop": (128, 128),
|
||||
"optimized": (320, 200),
|
||||
}[v["name"]]
|
||||
)
|
||||
assert not image.getexif()
|
||||
effect = inspect_effect(j)
|
||||
assert (
|
||||
effect["publications"] == 1
|
||||
and len(effect["variants"]) == 4
|
||||
and len(effect["physical_keys"]) == 4
|
||||
)
|
||||
return effect
|
||||
|
||||
if args.start_at == "all":
|
||||
# Source-only crash before the DB handoff must never be reported as accepted.
|
||||
stop("worker")
|
||||
stop("relay")
|
||||
marker = inject(chat, "api_after_staging", "wait")
|
||||
before = inside(
|
||||
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
|
||||
)
|
||||
pool = ThreadPoolExecutor(max_workers=1)
|
||||
future = pool.submit(
|
||||
lambda: client.post(
|
||||
f"/api/v1/chats/{chat}/attachments",
|
||||
headers={**h, "Content-Type": "image/png"},
|
||||
params={"filename": "unconfirmed.png"},
|
||||
content=raw,
|
||||
)
|
||||
)
|
||||
wait(lambda marker=marker: hit(marker))
|
||||
compose("kill", "-s", "SIGKILL", "api")
|
||||
try:
|
||||
response = future.result(15)
|
||||
assert response.status_code != 202
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
pool.shutdown()
|
||||
start("api")
|
||||
ready()
|
||||
after = inside(
|
||||
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
|
||||
)
|
||||
assert before == after
|
||||
save(
|
||||
"staging_before_handoff_crash",
|
||||
{
|
||||
"no_202": True,
|
||||
"no_attachment_or_job_created": True,
|
||||
"orphan_retained_for_cleanup": True,
|
||||
},
|
||||
)
|
||||
# API durable commit/outbox survives restart without a broker or worker.
|
||||
stop("broker")
|
||||
a, j = upload()
|
||||
linked = client.post(
|
||||
f"/api/v1/chats/{chat}/messages",
|
||||
headers={**h, "Idempotency-Key": uuid4().hex},
|
||||
json={"text": "", "attachment_ids": [a["id"]]},
|
||||
)
|
||||
assert linked.status_code == 201
|
||||
assert (
|
||||
client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
).status_code
|
||||
== 409
|
||||
)
|
||||
compose("restart", "api")
|
||||
ready()
|
||||
start("broker")
|
||||
start("relay")
|
||||
start("worker")
|
||||
effect = confirm(a, j)
|
||||
save(
|
||||
"commit_before_publish_and_worker_stopped",
|
||||
{
|
||||
"attachment": a["id"],
|
||||
"effect": effect,
|
||||
"queued_link_allowed": True,
|
||||
"api_and_broker_restarted": True,
|
||||
"broker_stopped_during_upload": True,
|
||||
},
|
||||
)
|
||||
# Broker-volume survival with a confirmed, still-unconsumed stream entry.
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
wait(
|
||||
lambda: inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))"
|
||||
)
|
||||
)
|
||||
stop("relay")
|
||||
ids = inside("print(json.dumps(queue.xrange(STREAM)))")
|
||||
compose("kill", "-s", "SIGKILL", "broker")
|
||||
start("broker")
|
||||
wait(lambda: inside("print(json.dumps(queue.ping()))"))
|
||||
restored = inside("print(json.dumps(queue.xrange(STREAM)))")
|
||||
assert restored == ids, {"before": ids, "after": restored}
|
||||
start("relay")
|
||||
start("worker")
|
||||
save(
|
||||
"confirmed_broker_entry_survives_sigkill",
|
||||
{
|
||||
"effect": confirm(a, j),
|
||||
"same_stream_entries": True,
|
||||
"entry_count": len(ids),
|
||||
},
|
||||
)
|
||||
# Confirm succeeded, PG marker did not: relay recovery may publish a duplicate.
|
||||
stop("worker")
|
||||
stop("relay")
|
||||
a, j = upload()
|
||||
marker = inject(j, "relay_after_publish", "crash")
|
||||
start("relay")
|
||||
wait(lambda marker=marker: hit(marker))
|
||||
time.sleep(0.5)
|
||||
unpublished = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']))"
|
||||
)
|
||||
assert unpublished >= 1
|
||||
start("relay")
|
||||
start("worker")
|
||||
save(
|
||||
"relay_crash_after_xadd_before_pg_marker",
|
||||
{
|
||||
"unpublished_after_crash": unpublished,
|
||||
"effect": confirm(a, j),
|
||||
},
|
||||
)
|
||||
# Repeated before/after effect and commit/ack windows.
|
||||
for repeat in range(args.repeats):
|
||||
for phase in [
|
||||
"before_effect",
|
||||
"after_effect",
|
||||
"after_commit",
|
||||
]:
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
marker = inject(j, phase, "wait")
|
||||
start("worker")
|
||||
wait(lambda marker=marker: hit(marker))
|
||||
at_crash = inspect_effect(
|
||||
j, pending=phase == "after_commit"
|
||||
)
|
||||
assert at_crash["job"]["state"] == (
|
||||
"ready"
|
||||
if phase == "after_commit"
|
||||
else "processing"
|
||||
)
|
||||
assert at_crash["publications"] == (
|
||||
1 if phase == "after_commit" else 0
|
||||
)
|
||||
assert len(at_crash["physical_keys"]) == (
|
||||
0 if phase == "before_effect" else 4
|
||||
)
|
||||
if phase == "after_commit":
|
||||
assert at_crash["pending_stream_entries"], (
|
||||
"Result must be committed while notification is still unacknowledged"
|
||||
)
|
||||
assert not any(
|
||||
e["event"] == "child_finished"
|
||||
for e in at_crash["events"]
|
||||
)
|
||||
compose("kill", "-s", "SIGKILL", "worker")
|
||||
start("worker")
|
||||
effect = confirm(a, j)
|
||||
if phase == "after_commit":
|
||||
wait(
|
||||
lambda j=j: any(
|
||||
e["event"] == "duplicate_decision"
|
||||
and e["detail"]["state"] == "ready"
|
||||
for e in inspect_effect(j)["events"]
|
||||
)
|
||||
)
|
||||
wait(
|
||||
lambda j=j: inside(
|
||||
f"print(json.dumps(not any(json.loads(fields.get('job','{{}}')).get('job_id')=={j!r} for _,fields in queue.xrange(STREAM))))"
|
||||
)
|
||||
)
|
||||
effect = inspect_effect(j)
|
||||
else:
|
||||
assert effect["job"]["attempt"] >= 2
|
||||
save(
|
||||
f"{phase}_sigkill_repeat_{repeat + 1}",
|
||||
{"at_crash": at_crash, "effect": effect},
|
||||
)
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
inject(j, "before_effect", "transient")
|
||||
start("worker")
|
||||
effect = confirm(a, j)
|
||||
assert effect["job"]["attempt"] == 2
|
||||
save("transient_retry_with_jitter", {"effect": effect})
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
marker = inject(j, "before_effect", "hang")
|
||||
start("worker")
|
||||
wait(lambda: hit(marker))
|
||||
limits = inside(
|
||||
f"with db.connection() as c: r=c.execute(\"SELECT detail FROM pipeline_events WHERE job_id=%s AND event='worker_resource_limits' ORDER BY id DESC LIMIT 1\",({j!r},)).fetchone()\nprint(json.dumps(r['detail']))"
|
||||
)
|
||||
assert limits["address_space_bytes"] == [805306368, 805306368]
|
||||
assert limits["memory.max"] == "1610612736"
|
||||
effect = confirm(a, j)
|
||||
assert any(
|
||||
e["event"] == "retry"
|
||||
and e["detail"]["code"] == "worker_timeout"
|
||||
for e in effect["events"]
|
||||
)
|
||||
save(
|
||||
"hung_child_killed_and_retried",
|
||||
{"effect": effect, "observed_limits": limits},
|
||||
)
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
inject(j, "before_effect", "permanent")
|
||||
start("worker")
|
||||
failed = until(a, "failed")
|
||||
assert failed["error_code"] == "invalid_image"
|
||||
assert (
|
||||
client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
).status_code
|
||||
== 409
|
||||
)
|
||||
r = client.post(
|
||||
f"/api/v1/chats/{chat}/messages",
|
||||
headers={**h, "Idempotency-Key": uuid4().hex},
|
||||
json={"text": "", "attachment_ids": [a["id"]]},
|
||||
)
|
||||
assert r.status_code == 409
|
||||
assert operator(j) == j
|
||||
effect = confirm(a, j)
|
||||
assert effect["job"]["cycle"] == 2
|
||||
try:
|
||||
operator(j)
|
||||
except RuntimeError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("Replay of ready must be rejected")
|
||||
save(
|
||||
"permanent_failed_registry_and_audited_replay",
|
||||
{
|
||||
"failed": failed,
|
||||
"effect": effect,
|
||||
"ready_replay_rejected": True,
|
||||
},
|
||||
)
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
inject(j, "before_effect", "transient", remaining=5)
|
||||
start("worker")
|
||||
failed = until(a, "failed", timeout=150)
|
||||
assert (
|
||||
failed["error_code"] == "retry_exhausted"
|
||||
and row(j)["attempt"] == 5
|
||||
)
|
||||
operator(j)
|
||||
save(
|
||||
"finite_retry_exhaustion_then_explicit_replay",
|
||||
{"failed": failed, "effect": confirm(a, j)},
|
||||
)
|
||||
# Reverse notification order and add concurrent copies: one publication/key.
|
||||
stop("worker")
|
||||
stop("relay")
|
||||
a, j = upload()
|
||||
b, k = upload()
|
||||
duplicate(k, 8)
|
||||
duplicate(j, 8)
|
||||
start("worker")
|
||||
start("relay")
|
||||
ea = confirm(a, j)
|
||||
confirm(b, k)
|
||||
duplicate(j, 10)
|
||||
duplicate(k, 10)
|
||||
wait(
|
||||
lambda: any(
|
||||
e["event"] == "duplicate_decision"
|
||||
for e in inspect_effect(j)["events"]
|
||||
)
|
||||
)
|
||||
assert (
|
||||
inspect_effect(j)["job"]["attempt"] == ea["job"]["attempt"]
|
||||
)
|
||||
save(
|
||||
"reordered_and_duplicate_notifications",
|
||||
{
|
||||
"effects": [inspect_effect(j), inspect_effect(k)],
|
||||
"added_duplicates": 36,
|
||||
"no_extra_attempt_after_ready": True,
|
||||
},
|
||||
)
|
||||
# Hold v1 after its PUTs, complete v2 on the other bounded slot, release old CAS.
|
||||
for repeat in range(args.repeats):
|
||||
stop("worker")
|
||||
a, j = upload()
|
||||
marker = inject(j, "after_effect", "wait")
|
||||
# Keep one controller process hot throughout the narrow stale-writer window.
|
||||
code = f"""from src.pipeline import event
|
||||
with db.connection() as c: event(c,UUID({j!r}),'version_controller_waiting')
|
||||
end=time.monotonic()+45
|
||||
while True:
|
||||
with db.connection() as c: hit=c.execute('SELECT hit_at FROM pipeline_faults WHERE id=%s',({marker},)).fetchone()['hit_at']
|
||||
if hit: break
|
||||
if time.monotonic()>end: raise TimeoutError('v1 hook never hit')
|
||||
time.sleep(.05)
|
||||
k=change(db,store,job_id=UUID({j!r}),operator='synthetic-verifier',reason='Held stale writer race',version=2)
|
||||
end=time.monotonic()+10
|
||||
while True:
|
||||
with db.connection() as c: ready=c.execute('SELECT state FROM jobs WHERE id=%s',(k,)).fetchone()['state']=='ready'
|
||||
if ready: break
|
||||
if time.monotonic()>end: raise TimeoutError('v2 did not finish inside live v1 budget')
|
||||
time.sleep(.05)
|
||||
with db.connection() as c:
|
||||
hashes=c.execute('SELECT name,sha256 FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=2 ORDER BY name',({a["id"]!r},)).fetchall()
|
||||
c.execute('UPDATE pipeline_faults SET released_at=clock_timestamp() WHERE id=%s',({marker},))
|
||||
end=time.monotonic()+10
|
||||
while True:
|
||||
with db.connection() as c: stale=c.execute("SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='stale_completion_rejected'",({j!r},)).fetchone()['n']
|
||||
if stale: break
|
||||
if time.monotonic()>end: raise TimeoutError('Old completion was not rejected')
|
||||
time.sleep(.05)
|
||||
print(json.dumps({{'job_id':k,'hashes':{{r['name']:r['sha256'] for r in hashes}}}}))
|
||||
"""
|
||||
with ThreadPoolExecutor(max_workers=1) as hot:
|
||||
future = hot.submit(inside, code)
|
||||
wait(
|
||||
lambda j=j: inside(
|
||||
f"with db.connection() as c: n=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='version_controller_waiting'\",({j!r},)).fetchone()['n']\nprint(json.dumps(n>0))"
|
||||
)
|
||||
)
|
||||
start("worker")
|
||||
race = future.result(timeout=45)
|
||||
k = race["job_id"]
|
||||
hashes = race["hashes"]
|
||||
confirm(a, k)
|
||||
duplicate(j, 12)
|
||||
wait(
|
||||
lambda j=j: any(
|
||||
e["event"] == "duplicate_decision"
|
||||
for e in inspect_effect(j)["events"]
|
||||
)
|
||||
)
|
||||
assert hashes == {
|
||||
v["name"]: v["sha256"] for v in confirm(a, k)["variants"]
|
||||
}
|
||||
assert inspect_effect(j)["publications"] == 0
|
||||
save(
|
||||
f"stale_v1_after_v2_repeat_{repeat + 1}",
|
||||
{
|
||||
"v1": inspect_effect(j),
|
||||
"v2": inspect_effect(k),
|
||||
"new_hashes_unchanged": True,
|
||||
},
|
||||
)
|
||||
# No arbitrary URL/bytes fields in queue envelopes.
|
||||
invalid = inside(
|
||||
"bad={'schema_version':1,'url':'http://example.invalid/raw'}\nqueue.xadd(STREAM,{'job':json.dumps(bad)})\nprint('true')"
|
||||
)
|
||||
assert invalid
|
||||
wait(
|
||||
lambda: inside(
|
||||
"with db.connection() as c: r=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE event='invalid_envelope'\").fetchone()\nprint(json.dumps(r['n']>0))"
|
||||
)
|
||||
)
|
||||
save(
|
||||
"malformed_notification_rejected", {"durable_diagnostic": True}
|
||||
)
|
||||
# Private bucket, bounded IAM, and download after uploader process dies.
|
||||
a, j = upload()
|
||||
confirm(a, j)
|
||||
key = f"objects/{a['id']}/v1/original"
|
||||
security = inside(f"""from minio.error import S3Error
|
||||
from urllib.error import HTTPError
|
||||
try: urllib.request.urlopen('http://s3:9000/'+BUCKET+'/' + {key!r},timeout=3)
|
||||
except HTTPError as e: anonymous=e.code
|
||||
else: raise AssertionError('Anonymous access allowed')
|
||||
denials={{}}
|
||||
for name,fn in [('create_bucket',lambda:store.client.make_bucket('forbidden-'+uuid4().hex)),('write_outside_prefix',lambda:store.client.put_object(BUCKET,'outside-'+uuid4().hex,io.BytesIO(b'x'),1)),('list_other_bucket',lambda:list(store.client.list_objects('forbidden-'+uuid4().hex)))]:
|
||||
try: fn()
|
||||
except S3Error as e: denials[name]=e.code
|
||||
else: raise AssertionError('IAM allowed forbidden action')
|
||||
print(json.dumps({{'anonymous':anonymous,'iam':denials}}))""")
|
||||
assert security["anonymous"] == 403 and set(
|
||||
security["iam"].values()
|
||||
) == {"AccessDenied"}
|
||||
selected = None
|
||||
for cid in compose("ps", "-q", "api").split():
|
||||
alias = command(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cid,
|
||||
"python",
|
||||
"-c",
|
||||
"import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
|
||||
]
|
||||
)
|
||||
uploader = inside(
|
||||
f"with db.connection() as c: r=c.execute('SELECT uploader_instance FROM attachments WHERE id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(r['uploader_instance']))"
|
||||
)
|
||||
if alias == uploader:
|
||||
selected = cid
|
||||
assert selected
|
||||
command(["docker", "stop", "--time", "1", selected])
|
||||
ready()
|
||||
wait(
|
||||
lambda: (
|
||||
client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
).status_code
|
||||
== 200
|
||||
)
|
||||
)
|
||||
downloaded = client.get(
|
||||
f"/api/v1/attachments/{a['id']}/content", headers=h
|
||||
)
|
||||
assert (
|
||||
downloaded.content == raw
|
||||
and downloaded.headers["X-Instance-Id"] != uploader
|
||||
)
|
||||
start("api")
|
||||
ready()
|
||||
save(
|
||||
"private_iam_and_other_api_download",
|
||||
{
|
||||
"security": security,
|
||||
"attachment": a["id"],
|
||||
"uploader": uploader,
|
||||
"downloader": downloaded.headers["X-Instance-Id"],
|
||||
"sha256": hashlib.sha256(raw).hexdigest(),
|
||||
},
|
||||
)
|
||||
report["worker_metrics"] = inside(
|
||||
"with db.connection() as c: r=c.execute(\"SELECT job_id,detail FROM pipeline_events WHERE event='child_finished' ORDER BY id\").fetchall()\nprint(json.dumps(r,default=str))"
|
||||
)
|
||||
report["audit"] = inside(
|
||||
"with db.connection() as c: r=c.execute('SELECT job_id,action,operator,reason,created_at FROM pipeline_audit ORDER BY id').fetchall()\nprint(json.dumps(r,default=str))"
|
||||
)
|
||||
report["complete"] = True
|
||||
finally:
|
||||
for role in ["api", "broker", "relay", "worker", "reconciler"]:
|
||||
try:
|
||||
start(role)
|
||||
except Exception:
|
||||
pass
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print("PASS: versioned pipeline scope=" + args.start_at, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user