feat(observability): add bounded OTLP signals, causal jobs and SLO rules

This commit is contained in:
2026-11-20 15:41:53 +03:00
parent 470d9d6f1e
commit 8f3ff33b2d
76 changed files with 9995 additions and 33 deletions
+64
View File
@@ -0,0 +1,64 @@
"""Создаёт только отсутствующие локальные файлы; секреты не печатает."""
import os
import secrets
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def create(path, value):
try:
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
except FileExistsError:
return
with os.fdopen(fd, "w") as stream:
stream.write(value + "\n")
def main():
directory = ROOT / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
pg = directory / "postgres_password"
cache = directory / "cache_password"
create(pg, secrets.token_urlsafe(32))
create(cache, secrets.token_urlsafe(32))
create(
directory / "database_url",
"postgresql://messenger:"
+ pg.read_text().strip()
+ "@postgres:5432/messenger",
)
create(
directory / "cache_url",
"redis://:" + cache.read_text().strip() + "@cache:6379/0",
)
create(
directory / "valkey.conf",
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nmaxmemory 64mb\n"
'maxmemory-policy allkeys-lru\nsave ""\nappendonly no\nrequirepass '
+ cache.read_text().strip(),
)
create(directory / "s3_root_user", secrets.token_hex(10))
create(directory / "s3_root_password", secrets.token_urlsafe(32))
create(directory / "s3_access_key", secrets.token_hex(10))
create(directory / "s3_secret_key", secrets.token_urlsafe(32))
create(directory / "grafana_password", secrets.token_urlsafe(32))
create(directory / "broker_password", secrets.token_urlsafe(32))
create(
directory / "broker_url",
"redis://:"
+ (directory / "broker_password").read_text().strip()
+ "@broker:6379/0",
)
create(
directory / "broker.conf",
"bind 0.0.0.0\nprotected-mode yes\nport 6379\nappendonly yes\nappendfsync always\nmaxmemory 192mb\nmaxmemory-policy noeviction\ndir /data\nrequirepass "
+ (directory / "broker_password").read_text().strip(),
)
create(ROOT / ".env", (ROOT / ".env.example").read_text().rstrip())
print("Local configuration ready; existing files preserved")
if __name__ == "__main__":
main()
+83
View File
@@ -0,0 +1,83 @@
"""Явные операторские действия только над API текущего Compose-проекта."""
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def command(*args):
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
def containers():
ids = command("docker", "compose", "ps", "-q", "api").split()
return json.loads(command("docker", "inspect", *ids)) if ids else []
def alias(cid):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=2); "
"print(r.headers['X-Instance-Id'])"
)
return command("docker", "exec", cid, "python", "-c", code)
def wait_count(count):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
data = containers()
if len(data) == count and all(
c["State"].get("Health", {}).get("Status") == "healthy"
for c in data
):
# DNS и проверки HAProxy должны успеть увидеть новый адрес.
time.sleep(3)
return
time.sleep(0.5)
raise TimeoutError("Replica readiness")
def main():
action = sys.argv[1]
data = containers()
if action == "stop":
target = os.environ["TARGET_INSTANCE_ID"]
matches = [c for c in data if alias(c["Id"]) == target]
if len(matches) != 1 or len(data) < 2:
raise RuntimeError("Target must identify one running replica")
command("docker", "stop", "--time", "10", matches[0]["Id"])
print("Selected API stopped")
elif action == "rolling":
count = len(data)
if count < 2:
raise RuntimeError("Need at least two replicas")
# Сначала добавляется готовая реплика, затем удаляется одна старая.
for old in data:
command(
"docker",
"compose",
"up",
"-d",
"--no-deps",
"--no-recreate",
"--scale",
f"api={count + 1}",
"api",
)
wait_count(count + 1)
command("docker", "stop", "--time", "10", old["Id"])
command("docker", "rm", old["Id"])
wait_count(count)
print("Sequential replacement completed")
else:
raise ValueError("Unknown action")
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
docker compose restart api postgres cache s3 broker relay worker lb
docker compose up -d --no-build --no-recreate --wait
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
exec python3 scripts/cluster.py rolling
+137
View File
@@ -0,0 +1,137 @@
import json
import os
import subprocess
import time
from pathlib import Path
from uuid import uuid4
import httpx
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8086")
def command(args, *, input=None):
result = subprocess.run(
args, cwd=ROOT, input=input, capture_output=True, text=True
)
if result.returncode:
raise RuntimeError(
"Command failed: "
+ args[0]
+ "; "
+ result.stderr.replace(str(ROOT), "<repository>")
)
return result.stdout.strip()
def compose(*args):
return command(["docker", "compose", *args])
def sql(query, *, database="messenger"):
return compose(
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
database,
"-A",
"-t",
"-v",
"ON_ERROR_STOP=1",
"-c",
query,
)
def cache(command_name, *args):
script = (
'export REDISCLI_AUTH="$(cat /run/secrets/cache_password)"; '
'exec valkey-cli "$@"'
)
return compose(
"exec", "-T", "cache", "sh", "-c", script, "sh", command_name, *args
)
def ready(base=BASE, *, timeout=30):
end = time.monotonic() + timeout
with httpx.Client(base_url=base, trust_env=False, timeout=6) as client:
while time.monotonic() < end:
try:
if client.get("/health/ready").status_code == 200:
return
except httpx.HTTPError:
pass
time.sleep(0.2)
raise TimeoutError("API readiness")
def actor(client):
username = "verify_" + uuid4().hex[:20]
password = "Synthetic-" + uuid4().hex + ":🌍"
result = client.post(
"/api/v1/users",
json={
"username": username,
"display_name": "Synthetic",
"password": password,
},
)
assert result.status_code == 201
user = result.json()
session = client.post(
"/api/v1/auth/sessions", auth=httpx.BasicAuth(username, password)
)
assert session.status_code == 201
return user, password, session.json()
def auth(session):
return {"Authorization": "Bearer " + session["token"]}
def extra_api(*, ttl=3600, pool_size=1):
name = "mtusi-lab6-test-" + uuid4().hex[:10]
compose(
"run",
"-d",
"--no-deps",
"--name",
name,
"-e",
"SESSION_TTL_SECONDS=" + str(ttl),
"-e",
"DB_POOL_SIZE=" + str(pool_size),
"-p",
"127.0.0.1::8080",
"api",
)
try:
info = json.loads(command(["docker", "inspect", name]))[0]
base = (
"http://127.0.0.1:"
+ info["NetworkSettings"]["Ports"]["8080/tcp"][0]["HostPort"]
)
ready(base)
return name, base
except Exception:
command(["docker", "rm", "-f", name])
raise
def instances():
data = {}
for role in ["api", "postgres", "cache"]:
cid = compose("ps", "-q", role)
info = json.loads(command(["docker", "inspect", cid]))[0]
data[role] = {
"container": cid[:12],
"pid": info["State"]["Pid"],
"started_at": info["State"]["StartedAt"],
}
return data
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
exec python3 scripts/cluster.py stop
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
docker compose stop --timeout 1 worker
+254
View File
@@ -0,0 +1,254 @@
import argparse
import atexit
import json
import os
import socket
import subprocess
import tempfile
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[1]
PROGRESS = {}
def save_incomplete():
receipt = PROGRESS.get("receipt")
if receipt and receipt.get("result") != "PASS":
target = PROGRESS["output"].with_name(
PROGRESS["output"].stem + "-failed.json"
)
if target.exists():
target = target.with_name(
target.stem + "-" + uuid4().hex[:8] + target.suffix
)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(
json.dumps(
{**receipt, "result": "INCOMPLETE"},
ensure_ascii=False,
indent=2,
)
+ "\n"
)
atexit.register(save_incomplete)
def run(args, *, cwd, env, expected=0):
result = subprocess.run(
args, cwd=cwd, env=env, capture_output=True, text=True
)
def redact(text):
return (
text.replace(str(cwd), "<clean-clone>")
.replace(str(ROOT), "<repository>")
.replace(str(Path.home()), "<home>")
)
if result.returncode != expected:
if "receipt" in PROGRESS:
PROGRESS["receipt"]["failed_action"] = redact(" ".join(args))
raise RuntimeError(redact(result.stdout + result.stderr))
return redact(result.stdout.strip())
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/clean-clone.json")
args = parser.parse_args()
output = ROOT / args.output
if output.exists():
raise SystemExit("Choose new output path")
assert not subprocess.check_output(
["git", "status", "--porcelain"], cwd=ROOT
)
sha = subprocess.check_output(
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
).strip()
identity = uuid4().hex[:10]
project = "mtusi-lab6-clean-" + identity
with socket.socket() as reservation:
reservation.bind(("127.0.0.1", 0))
public_port = str(reservation.getsockname()[1])
env = {
**os.environ,
"COMPOSE_PROJECT_NAME": project,
"API_IMAGE": "mtusi-messenger:lab6-clean-" + identity,
"PUBLIC_PORT": public_port,
"VCS_REF": sha,
"COMPOSE_FILE": "compose.yaml:compose.test.yaml",
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
receipt = {
"date": datetime.now(UTC).isoformat(),
"source_commit": sha,
"fresh_secrets_environment_and_volumes": True,
"command": "python scripts/verify_clean.py",
"fixed_host_port_for_restart": int(public_port),
}
PROGRESS.update(receipt=receipt, output=output)
with tempfile.TemporaryDirectory(prefix="mtusi-lab6-clean-") as temporary:
clone = Path(temporary) / "lab6"
run(
[
"git",
"clone",
"--no-hardlinks",
"--branch",
"solution/bvt2301-maslov",
str(ROOT),
str(clone),
],
cwd=ROOT,
env=env,
)
for name in [".env", ".venv", "secrets"]:
assert not (clone / name).exists()
run(["uv", "sync", "--locked"], cwd=clone, env=env)
receipt["static_checks"] = run(
["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"],
cwd=clone,
env=env,
)
receipt["unit_tests"] = run(["make", "unit"], cwd=clone, env=env)
run(["make", "bootstrap"], cwd=clone, env=env)
try:
run(
["docker", "compose", "up", "-d", "--build", "--wait"],
cwd=clone,
env=env,
)
env["BASE_URL"] = "http://127.0.0.1:" + public_port
receipt["smoke"] = run(
["make", "test", "BASE_URL=" + env["BASE_URL"]],
cwd=clone,
env=env,
)
receipt["http_tests"] = run(
[".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env
)
run(
[
".venv/bin/python",
"-m",
"scripts.verify_cluster",
"--output",
"evidence/clean-cluster.json",
],
cwd=clone,
env=env,
)
for command_args in [
[
".venv/bin/python",
"-m",
"scripts.verify_cleanup",
"seed",
"--seed",
"evidence/clean-gc-seed.json",
],
[
".venv/bin/python",
"-m",
"scripts.verify_pipeline",
"--output",
"evidence/clean-pipeline.json",
],
[
".venv/bin/python",
"-m",
"scripts.verify_cleanup",
"verify",
"--seed",
"evidence/clean-gc-seed.json",
"--output",
"evidence/clean-cleanup.json",
],
]:
run(command_args, cwd=clone, env=env)
receipt["pipeline"] = json.loads(
(clone / "evidence/clean-pipeline.json").read_text()
)
receipt["cleanup"] = json.loads(
(clone / "evidence/clean-cleanup.json").read_text()
)
receipt["cluster"] = json.loads(
(clone / "evidence/clean-cluster.json").read_text()
)
import hashlib
expected = {
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in (clone / "src").glob("*.py")
}
ids = run(
["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env
).split()
assert len(ids) == 2
code = (
"import hashlib,json,pathlib; print(json.dumps({p.name:"
"hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))"
)
ids += run(
[
"docker",
"compose",
"ps",
"-q",
"worker",
"relay",
"reconciler",
"controller",
],
cwd=clone,
env=env,
).split()
assert len(ids) == 6
for cid in ids:
actual = json.loads(
run(
["docker", "exec", cid, "python", "-c", code],
cwd=clone,
env=env,
)
)
assert actual == expected
info = json.loads(
run(["docker", "inspect", cid], cwd=clone, env=env)
)[0]
assert (
info["Config"]["Labels"][
"org.opencontainers.image.revision"
]
== sha
)
assert not info["HostConfig"]["PortBindings"]
receipt[
"api_worker_relay_reconciler_runtime_hashes_and_revision_match"
] = True
receipt["fsck"] = run(
["git", "fsck", "--full"], cwd=clone, env=env
)
receipt["checks_completed"] = True
finally:
assert project.startswith("mtusi-lab6-clean-")
run(
["docker", "compose", "down", "--volumes", "--remove-orphans"],
cwd=clone,
env=env,
)
receipt["result"] = "PASS"
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n")
print(
"PASS: fresh clone, build, HTTP tests, failover, rolling, persistence and runtime hashes"
)
if __name__ == "__main__":
main()
+223
View File
@@ -0,0 +1,223 @@
"""Real S3 retention test: seed is separate from verification output."""
import argparse
import json
import time
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
import httpx
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
from scripts.verify_pipeline import inside, operator, wait
def seed(path):
if path.exists():
raise ValueError("Choose a new seed file")
ready()
compose("stop", "--timeout", "1", "reconciler")
raw = (ROOT / "tests/fixtures/sample.png").read_bytes()
with httpx.Client(base_url=BASE, trust_env=False, timeout=20) as c:
user, _, session = actor(c)
headers = auth(session)
result = c.post(
"/api/v1/chats",
headers=headers,
json={
"title": "Cleanup synthetic seed",
"member_ids": [user["id"]],
},
)
assert result.status_code == 201
chat = result.json()["id"]
r = c.post(
f"/api/v1/chats/{chat}/attachments",
headers={**headers, "Content-Type": "image/png"},
params={"filename": "retained.png"},
content=raw,
)
assert r.status_code == 202
attachment = r.json()["id"]
wait(
lambda: (
c.get(
"/api/v1/attachments/" + attachment, headers=headers
).json()["status"]
== "ready"
)
)
job = inside(
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({attachment!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
)
data = {
"seed_id": uuid4().hex,
"source_commit": command(["git", "rev-parse", "HEAD"]),
"date": datetime.now(timezone.utc).isoformat(),
"ready_attachment": attachment,
"ready_job": job,
"chat": chat,
"owner": user["id"],
}
# Real old v1 objects will become obsolete after v2 is ready.
second = operator(job, version=2)
wait(
lambda: inside(
f"with db.connection() as c: r=c.execute('SELECT state FROM jobs WHERE id=%s',({second!r},)).fetchone()\nprint(json.dumps(r['state']=='ready'))"
)
)
data["version2_job"] = second
data["orphan"] = f"objects/cleanup-{data['seed_id']}/old-orphan"
data["failed_id"] = str(uuid4())
data["active_id"] = str(uuid4())
data["reservation_id"] = str(uuid4())
data["expired_reservation_id"] = str(uuid4())
data["failed_source"] = f"objects/staging/{data['failed_id']}/original"
data["active_source"] = f"objects/staging/{data['active_id']}/original"
data["reservation_key"] = (
f"objects/staging/{data['reservation_id']}/original"
)
data["expired_reservation_key"] = (
f"objects/staging/{data['expired_reservation_id']}/original"
)
inside(f"""from pathlib import Path
raw=Path('/app/src/__init__.py').read_bytes() or b'fixture'
keys={json.dumps([data[k] for k in ["orphan", "failed_source", "active_source", "reservation_key", "expired_reservation_key"]])}
for key in keys: store.put(key=key,stream=io.BytesIO(raw),size=len(raw),content_type='image/png',sha256=hashlib.sha256(raw).hexdigest())
with db.connection() as c, c.transaction():
for identity,state in [({data["failed_id"]!r},'failed'),({data["active_id"]!r},'processing')]:
key='objects/staging/'+identity+'/original'
c.execute(\"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,error_code,uploader_instance,expires_at) VALUES(%s,%s,%s,'synthetic.png','image/png',%s,%s,%s,%s,%s,'cleanup-fixture',clock_timestamp())\",(identity,{chat!r},{user["id"]!r},len(raw),hashlib.sha256(raw).hexdigest(),key,state,'invalid_image' if state=='failed' else None))
c.execute(\"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state,attempt,lease_until,updated_at,error_code) VALUES(%s,%s,1,%s,%s,1,clock_timestamp()+interval '1 hour',clock_timestamp()-interval '2 days',%s)\",(uuid4(),identity,key,state,'invalid_image' if state=='failed' else None))
c.execute(\"INSERT INTO upload_reservations(id,object_key,expires_at) VALUES(%s,%s,clock_timestamp()+interval '1 hour'),(%s,%s,clock_timestamp()-interval '1 second')\",({data["reservation_id"]!r},{data["reservation_key"]!r},{data["expired_reservation_id"]!r},{data["expired_reservation_key"]!r}))
print('true')""")
data["seeded_unix"] = time.time()
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
print(
"PASS: retention seed created; real S3 age must reach300s", flush=True
)
def verify(seed_path, output):
if output.exists():
raise ValueError("Choose a new output path")
data = json.loads(seed_path.read_text())
report = {
"source_commit": command(["git", "rev-parse", "HEAD"]),
"seed": seed_path.name,
"complete": False,
}
try:
compose("stop", "--timeout", "1", "reconciler")
def age():
return inside(
f"h=store.head(key={data['orphan']!r})\nassert h is not None\nwith db.connection() as c: now=c.execute('SELECT clock_timestamp() now').fetchone()['now']\nprint(json.dumps((now-h.last_modified).total_seconds()))"
)
while age() < 302:
time.sleep(min(5, 303 - age()))
report["real_s3_age_seconds"] = age()
fresh = f"objects/cleanup-{data['seed_id']}/fresh-orphan"
inside(
f"b=b'fresh'\nstore.put(key={fresh!r},stream=io.BytesIO(b),size=len(b),content_type='image/png',sha256=hashlib.sha256(b).hexdigest())\nprint('true')"
)
oldkeys = [
data["orphan"],
data["failed_source"],
data["expired_reservation_key"],
] + [
f"objects/{data['ready_attachment']}/v1/{v}"
for v in ["original", "thumbnail", "crop", "optimized"]
]
protected = [data["active_source"], data["reservation_key"], fresh] + [
f"objects/{data['ready_attachment']}/v2/{v}"
for v in ["original", "thumbnail", "crop", "optimized"]
]
before = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert all(before.values())
dry = inside(
"print(json.dumps(reconcile(db,store,apply=False,grace=300)))"
)
candidates = {a["key"] for a in dry["actions"]}
assert set(oldkeys) <= candidates and not set(protected) & candidates
afterdry = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert afterdry == before
applied = inside(
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
)
repeat = inside(
"print(json.dumps(reconcile(db,store,apply=True,grace=300)))"
)
assert not repeat["actions"]
after = inside(
f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))"
)
assert all(not after[k] for k in oldkeys) and all(
after[k] for k in protected
)
failed = inside(
f"with db.connection() as c: j=c.execute('SELECT id,error_code FROM jobs WHERE attachment_id=%s',({data['failed_id']!r},)).fetchone()\nprint(json.dumps(j,default=str))"
)
assert failed["error_code"] == "source_expired"
try:
operator(failed["id"])
except RuntimeError:
replay_refused = True
else:
raise AssertionError("Expired source replay must fail")
# Expired reservation was removed, so a delayed API cannot commit its handoff.
fenced = inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp()',({data['expired_reservation_id']!r},)).fetchone()\nprint(json.dumps(r['n']==0))"
)
assert fenced
report.update(
dry_run=dry,
apply=applied,
repeat=repeat,
before=before,
after=after,
dry_run_unchanged=True,
active_current_and_fresh_protected=True,
expired_source_replay_refused=replay_refused,
expired_writer_fenced=fenced,
complete=True,
)
finally:
# Remove only this fixture's artificial active reservation/job; never real jobs.
inside(
f"with db.connection() as c, c.transaction():\n c.execute(\"UPDATE jobs SET state='failed',error_code='fixture_complete',lease_until=NULL WHERE attachment_id=%s\",({data['active_id']!r},))\n c.execute(\"UPDATE attachments SET state='failed',error_code='fixture_complete' WHERE id=%s\",({data['active_id']!r},))\n c.execute('DELETE FROM upload_reservations WHERE id=%s',({data['reservation_id']!r},))\nprint('true')"
)
compose("up", "-d", "--no-deps", "--no-recreate", "reconciler")
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(
"PASS: real retention, dry-run/apply/repeat, current/active/fresh protection",
flush=True,
)
def main():
p = argparse.ArgumentParser()
p.add_argument("action", choices=["seed", "verify"])
p.add_argument("--seed", required=True, type=Path)
p.add_argument("--output", type=Path)
a = p.parse_args()
if a.action == "seed":
seed(a.seed)
else:
if a.output is None:
p.error("output required")
verify(a.seed, a.output)
if __name__ == "__main__":
main()
+347
View File
@@ -0,0 +1,347 @@
"""Проверки реальных процессов, failover, rolling и общей идемпотентности."""
import argparse
import json
import os
import subprocess
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
import httpx
from scripts import cluster
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
def snapshot():
ids = compose("ps", "-q").split()
result = []
for c in json.loads(command(["docker", "inspect", *ids])):
role = c["Config"]["Labels"]["com.docker.compose.service"]
entry = {
"role": role,
"container": c["Id"][:12],
"pid": c["State"]["Pid"],
"started_at": c["State"]["StartedAt"],
"ports": c["HostConfig"]["PortBindings"],
"cpu_nano": c["HostConfig"]["NanoCpus"],
"memory_limit": c["HostConfig"]["Memory"],
}
if role == "api":
entry["instance"] = cluster.alias(c["Id"])
assert not entry["ports"]
result.append(entry)
return result
def exercise(client, session, chat, action, *, seconds=15):
path = f"/api/v1/chats/{chat['id']}/messages"
observations, confirmed, errors, reads = [], {}, 0, 0
started = time.monotonic()
finished = threading.Event()
action_result = {}
def operate():
time.sleep(0.5)
try:
action()
except BaseException as exc:
action_result["error"] = type(exc).__name__
finally:
action_result["finished"] = round(time.monotonic() - started, 3)
finished.set()
thread = threading.Thread(target=operate)
thread.start()
key = uuid4().hex
try:
while (
time.monotonic() - started < seconds
or not finished.is_set()
or time.monotonic() - started
< action_result.get("finished", 0) + 2
):
h = {**auth(session), "Idempotency-Key": key}
try:
r = client.post(path, headers=h, json={"text": key})
if r.status_code != 201:
errors += 1
continue
m = r.json()
confirmed[m["id"]] = m
observations.append(
{
"at": round(time.monotonic() - started, 3),
"instance": r.headers["x-instance-id"],
}
)
key = uuid4().hex
read = client.get(
path, headers=auth(session), params={"limit": 100}
)
reads += read.status_code == 200
errors += read.status_code != 200
except httpx.HTTPError:
errors += 1
time.sleep(0.04)
finally:
thread.join(timeout=90)
assert not action_result.get("error"), action_result
found, cursor = [], None
while True:
r = client.get(
path,
headers=auth(session),
params={"limit": 100, **({"cursor": cursor} if cursor else {})},
)
assert r.status_code == 200
page = r.json()
found.extend(page["items"])
cursor = page["next_cursor"]
if not cursor:
break
by_id = {m["id"]: m for m in found}
assert all(by_id.get(k) == v for k, v in confirmed.items())
texts = [m["text"] for m in found]
assert len(texts) == len(set(texts))
recovery = next(
(
x["at"]
for x in observations
if x["at"] >= action_result["finished"]
),
None,
)
assert recovery is not None and (seconds > 20 or recovery <= 10)
elapsed = time.monotonic() - started
return {
"confirmed_writes": len(confirmed),
"successful_reads": reads,
"errors": errors,
"successful_rps": round((len(confirmed) + reads) / elapsed, 2),
"first_success_after_action_seconds": recovery,
"action": action_result,
"observations": observations,
"confirmed_preserved": True,
"duplicate_texts": False,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/cluster-repeat.json")
args = parser.parse_args()
report = {
"date": datetime.now(timezone.utc).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"complete": False,
}
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
try:
ready()
initial = snapshot()
apis = [c for c in initial if c["role"] == "api"]
assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2
report["before"] = initial
with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client:
user, _, session = actor(client)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={
"title": "Cluster verification",
"member_ids": [user["id"]],
},
)
assert r.status_code == 201
chat = r.json()
path = f"/api/v1/chats/{chat['id']}/messages"
key = uuid4().hex
def repeat(_):
with httpx.Client(
base_url=BASE, trust_env=False, timeout=8
) as c:
r = c.post(
path,
headers={**auth(session), "Idempotency-Key": key},
json={"text": "Same concurrent request"},
)
assert r.status_code == 201
return r.json(), r.headers["x-instance-id"]
with ThreadPoolExecutor(max_workers=10) as pool:
repeated = list(pool.map(repeat, range(30)))
assert len({m[0]["id"] for m in repeated}) == 1
assert len({m[1] for m in repeated}) == 2
report["parallel_replay"] = {
"requests": 30,
"messages": 1,
"instances": sorted({m[1] for m in repeated}),
}
# Сбой второй записи транзакции не должен оставить сообщение.
fault_key = uuid4().hex
fault_text = "Atomic rollback " + fault_key
sql(
"CREATE FUNCTION student_idempo_fault() RETURNS trigger "
"LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='"
+ fault_key
+ "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; "
"CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency "
"FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()"
)
try:
failed = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": fault_text},
)
assert failed.status_code == 500
assert (
sql(
"SELECT count(*) FROM messages WHERE text='"
+ fault_text
+ "'"
)
== "0"
)
finally:
sql(
"DROP TRIGGER student_idempo_fault ON idempotency; "
"DROP FUNCTION student_idempo_fault()"
)
retried = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": fault_text},
)
assert retried.status_code == 201
report["atomic_rollback_and_5xx_retry"] = True
sql(
"UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' "
"WHERE key='" + fault_key + "'"
)
expired = client.post(
path,
headers={**auth(session), "Idempotency-Key": fault_key},
json={"text": "New retention window"},
)
assert expired.status_code == 201
assert expired.json()["id"] != retried.json()["id"]
report["expired_key_can_be_reused"] = True
# Отдельный чат каждой серии: проверки дубликатов не смешивают тексты.
report["failovers"] = []
for api in apis:
def stop(target=api["instance"]):
env = {**os.environ, "TARGET_INSTANCE_ID": target}
subprocess.run(
["scripts/stop-one.sh"],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={"title": "Failover", "member_ids": [user["id"]]},
)
result = exercise(client, session, r.json(), stop)
assert api["instance"] not in [
o["instance"] for o in result["observations"][-5:]
]
assert (
command(
[
"docker",
"inspect",
"-f",
"{{.State.Running}}",
api["container"],
]
)
== "false"
)
result["stopped"] = api
report["failovers"].append(result)
compose("up", "-d", "--no-build", "--wait")
ready()
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={"title": "Rolling", "member_ids": [user["id"]]},
)
before_roll = snapshot()
report["rolling"] = exercise(
client,
session,
r.json(),
lambda: command(["scripts/rolling-update.sh"]),
seconds=45,
)
after_roll = snapshot()
assert not (
{c["container"] for c in before_roll if c["role"] == "api"}
& {c["container"] for c in after_roll if c["role"] == "api"}
)
report["rolling"]["before"] = before_roll
report["rolling"]["after"] = after_roll
replay = repeat(0)[0]
assert replay == repeated[0][0]
report["replay_after_replacement"] = True
before = snapshot()
env = {
**os.environ,
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
for mode, hook in [
("failover", "scripts/stop-one.sh"),
("persistence", "scripts/restart.sh"),
]:
if mode == "persistence":
compose("up", "-d", "--no-build", "--wait")
ready()
before = snapshot()
run = subprocess.run(
[
".venv/bin/python",
"tests/resilience.py",
"--mode",
mode,
"--action-script",
hook,
"--base-url",
BASE,
],
cwd=ROOT,
env=env,
capture_output=True,
text=True,
)
assert run.returncode == 0, run.stdout + run.stderr
report["public_" + mode] = run.stdout.strip()
after = snapshot()
for role in ["postgres", "cache", "s3", "broker", "lb"]:
old = next(c for c in before if c["role"] == role)
new = next(c for c in after if c["role"] == role)
assert old["started_at"] != new["started_at"]
report["after_persistence"] = after
report["complete"] = True
finally:
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS: process failover, rolling, replay and full persistence")
if __name__ == "__main__":
main()
+687
View File
@@ -0,0 +1,687 @@
"""Repeatable fault injection for grade 5, track 2; synthetic data only."""
import argparse
import hashlib
import io
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from uuid import uuid4
import httpx
from PIL import Image
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1")
os.environ.setdefault("no_proxy", "localhost,127.0.0.1")
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
PREFIX = """import io,json,hashlib,time,urllib.request
from uuid import UUID,uuid4
from psycopg.types.json import Jsonb
from src.pipeline import database,broker,envelope,STREAM,GROUP
from src.object_store import ObjectStore,BUCKET
from src.operator import change
from src.reconcile import reconcile
db,store,queue=database(),ObjectStore(),broker()
"""
def inside(code):
cid = compose("ps", "-q", "controller").split()[0]
return json.loads(
command(
["docker", "exec", "-i", cid, "python", "-"], input=PREFIX + code
)
)
def wait(fn, timeout=90):
end = time.monotonic() + timeout
while time.monotonic() < end:
value = fn()
if value:
return value
time.sleep(0.15)
raise TimeoutError("Fault scenario condition not reached")
def stop(role):
compose("stop", "--timeout", "1", role)
def start(role):
compose("up", "-d", "--no-deps", "--no-recreate", role)
def row(job):
return inside(
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\nprint(json.dumps(r,default=str))"
)
def inject(job, phase, action, remaining=1):
return inside(
f"with db.connection() as c: r=c.execute('INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES(%s,%s,%s,%s) RETURNING id',({job!r},{phase!r},{action!r},{remaining})).fetchone()\nprint(json.dumps(r['id']))"
)
def hit(identity):
return inside(
f"with db.connection() as c: r=c.execute('SELECT hit_at IS NOT NULL hit FROM pipeline_faults WHERE id=%s',({identity},)).fetchone()\nprint(json.dumps(r['hit']))"
)
def operator(job, version=None):
return inside(
f"print(json.dumps(change(db,store,job_id=UUID({job!r}),operator='synthetic-verifier',reason='Explicit fault matrix replay',version={version!r})))"
)
def duplicate(job, count=5):
return inside(
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\ne=envelope(r)\nids=[queue.xadd(STREAM,{{'job':json.dumps(e)}}) for _ in range({count})]\nprint(json.dumps(ids))"
)
def inspect_effect(job, *, pending=False):
return inside(f"""with db.connection() as c:
j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()
v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall()
p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n']
e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall()
keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")]
pending_entries=[]
if {pending!r}:
for item in queue.xpending_range(STREAM,GROUP,'-','+',1000):
entry=item['message_id']
messages=queue.xrange(STREAM,min=entry,max=entry)
if messages and json.loads(messages[0][1].get('job','{{}}')).get('job_id')==str(j['id']): pending_entries.append(entry)
print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e,'pending_stream_entries':pending_entries}},default=str))""")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/pipeline-repeat.json")
parser.add_argument("--repeats", type=int, default=2)
parser.add_argument(
"--start-at", choices=["all", "versions"], default="all"
)
args = parser.parse_args()
output = ROOT / args.output
if output.exists():
raise ValueError("Choose a new output path")
report = {
"source_commit": command(["git", "rev-parse", "HEAD"]),
"date": datetime.now(timezone.utc).isoformat(),
"track": 2,
"scope": args.start_at,
"complete": False,
"cases": [],
}
output.parent.mkdir(parents=True, exist_ok=True)
def save(name, details):
report["cases"].append({"case": name, **details})
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS:", name, flush=True)
try:
ready()
# Check instrumentation is really enabled and production default is off.
cid = compose("ps", "-q", "api").split()[0]
assert (
command(
[
"docker",
"exec",
cid,
"python",
"-c",
"import os;print(os.getenv('PIPELINE_FAULTS'))",
]
)
== "1"
)
stop("reconciler")
with httpx.Client(
base_url=BASE, trust_env=False, timeout=25
) as client:
user, _, session = actor(client)
h = auth(session)
c = client.post(
"/api/v1/chats",
headers=h,
json={
"title": "Pipeline synthetic faults",
"member_ids": [user["id"]],
},
)
assert c.status_code == 201
chat = c.json()["id"]
raw_io = io.BytesIO()
Image.new("RGBA", (320, 200), (20, 50, 170, 120)).save(
raw_io, format="PNG"
)
raw = raw_io.getvalue()
def upload():
r = client.post(
f"/api/v1/chats/{chat}/attachments",
headers={**h, "Content-Type": "image/png"},
params={"filename": "synthetic.png"},
content=raw,
)
assert r.status_code == 202, r.text
a = r.json()
assert a["status"] == "queued" and not a["variants"]
j = inside(
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
)
return a, j
def metadata(a):
r = client.get("/api/v1/attachments/" + a["id"], headers=h)
assert r.status_code == 200, r.text
return r.json()
def until(a, state="ready", timeout=90):
return wait(
lambda: (
r if (r := metadata(a))["status"] == state else None
),
timeout,
)
def confirm(a, j):
until(a)
downloaded = client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
)
assert (
downloaded.status_code == 200 and downloaded.content == raw
)
m = metadata(a)
assert (
m["sha256"] == hashlib.sha256(raw).hexdigest()
and m["error_code"] is None
)
assert {v["name"] for v in m["variants"]} == {
"thumbnail",
"crop",
"optimized",
}
for v in m["variants"]:
r = client.get(
f"/api/v1/attachments/{a['id']}/content",
headers=h,
params={"variant": v["name"]},
)
assert (
r.status_code == 200
and hashlib.sha256(r.content).hexdigest()
== v["sha256"]
)
with Image.open(io.BytesIO(r.content)) as image:
assert (
image.size
== {
"thumbnail": (256, 160),
"crop": (128, 128),
"optimized": (320, 200),
}[v["name"]]
)
assert not image.getexif()
effect = inspect_effect(j)
assert (
effect["publications"] == 1
and len(effect["variants"]) == 4
and len(effect["physical_keys"]) == 4
)
return effect
if args.start_at == "all":
# Source-only crash before the DB handoff must never be reported as accepted.
stop("worker")
stop("relay")
marker = inject(chat, "api_after_staging", "wait")
before = inside(
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
)
pool = ThreadPoolExecutor(max_workers=1)
future = pool.submit(
lambda: client.post(
f"/api/v1/chats/{chat}/attachments",
headers={**h, "Content-Type": "image/png"},
params={"filename": "unconfirmed.png"},
content=raw,
)
)
wait(lambda marker=marker: hit(marker))
compose("kill", "-s", "SIGKILL", "api")
try:
response = future.result(15)
assert response.status_code != 202
except httpx.HTTPError:
pass
pool.shutdown()
start("api")
ready()
after = inside(
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
)
assert before == after
save(
"staging_before_handoff_crash",
{
"no_202": True,
"no_attachment_or_job_created": True,
"orphan_retained_for_cleanup": True,
},
)
# API durable commit/outbox survives restart without a broker or worker.
stop("broker")
a, j = upload()
linked = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**h, "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
)
assert linked.status_code == 201
assert (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 409
)
compose("restart", "api")
ready()
start("broker")
start("relay")
start("worker")
effect = confirm(a, j)
save(
"commit_before_publish_and_worker_stopped",
{
"attachment": a["id"],
"effect": effect,
"queued_link_allowed": True,
"api_and_broker_restarted": True,
"broker_stopped_during_upload": True,
},
)
# Broker-volume survival with a confirmed, still-unconsumed stream entry.
stop("worker")
a, j = upload()
wait(
lambda: inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))"
)
)
stop("relay")
ids = inside("print(json.dumps(queue.xrange(STREAM)))")
compose("kill", "-s", "SIGKILL", "broker")
start("broker")
wait(lambda: inside("print(json.dumps(queue.ping()))"))
restored = inside("print(json.dumps(queue.xrange(STREAM)))")
assert restored == ids, {"before": ids, "after": restored}
start("relay")
start("worker")
save(
"confirmed_broker_entry_survives_sigkill",
{
"effect": confirm(a, j),
"same_stream_entries": True,
"entry_count": len(ids),
},
)
# Confirm succeeded, PG marker did not: relay recovery may publish a duplicate.
stop("worker")
stop("relay")
a, j = upload()
marker = inject(j, "relay_after_publish", "crash")
start("relay")
wait(lambda marker=marker: hit(marker))
time.sleep(0.5)
unpublished = inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']))"
)
assert unpublished >= 1
start("relay")
start("worker")
save(
"relay_crash_after_xadd_before_pg_marker",
{
"unpublished_after_crash": unpublished,
"effect": confirm(a, j),
},
)
# Repeated before/after effect and commit/ack windows.
for repeat in range(args.repeats):
for phase in [
"before_effect",
"after_effect",
"after_commit",
]:
stop("worker")
a, j = upload()
marker = inject(j, phase, "wait")
start("worker")
wait(lambda marker=marker: hit(marker))
at_crash = inspect_effect(
j, pending=phase == "after_commit"
)
assert at_crash["job"]["state"] == (
"ready"
if phase == "after_commit"
else "processing"
)
assert at_crash["publications"] == (
1 if phase == "after_commit" else 0
)
assert len(at_crash["physical_keys"]) == (
0 if phase == "before_effect" else 4
)
if phase == "after_commit":
assert at_crash["pending_stream_entries"], (
"Result must be committed while notification is still unacknowledged"
)
assert not any(
e["event"] == "child_finished"
for e in at_crash["events"]
)
compose("kill", "-s", "SIGKILL", "worker")
start("worker")
effect = confirm(a, j)
if phase == "after_commit":
wait(
lambda j=j: any(
e["event"] == "duplicate_decision"
and e["detail"]["state"] == "ready"
for e in inspect_effect(j)["events"]
)
)
wait(
lambda j=j: inside(
f"print(json.dumps(not any(json.loads(fields.get('job','{{}}')).get('job_id')=={j!r} for _,fields in queue.xrange(STREAM))))"
)
)
effect = inspect_effect(j)
else:
assert effect["job"]["attempt"] >= 2
save(
f"{phase}_sigkill_repeat_{repeat + 1}",
{"at_crash": at_crash, "effect": effect},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "transient")
start("worker")
effect = confirm(a, j)
assert effect["job"]["attempt"] == 2
save("transient_retry_with_jitter", {"effect": effect})
stop("worker")
a, j = upload()
marker = inject(j, "before_effect", "hang")
start("worker")
wait(lambda: hit(marker))
limits = inside(
f"with db.connection() as c: r=c.execute(\"SELECT detail FROM pipeline_events WHERE job_id=%s AND event='worker_resource_limits' ORDER BY id DESC LIMIT 1\",({j!r},)).fetchone()\nprint(json.dumps(r['detail']))"
)
assert limits["address_space_bytes"] == [805306368, 805306368]
assert limits["memory.max"] == "1610612736"
effect = confirm(a, j)
assert any(
e["event"] == "retry"
and e["detail"]["code"] == "worker_timeout"
for e in effect["events"]
)
save(
"hung_child_killed_and_retried",
{"effect": effect, "observed_limits": limits},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "permanent")
start("worker")
failed = until(a, "failed")
assert failed["error_code"] == "invalid_image"
assert (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 409
)
r = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**h, "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
)
assert r.status_code == 409
assert operator(j) == j
effect = confirm(a, j)
assert effect["job"]["cycle"] == 2
try:
operator(j)
except RuntimeError:
pass
else:
raise AssertionError("Replay of ready must be rejected")
save(
"permanent_failed_registry_and_audited_replay",
{
"failed": failed,
"effect": effect,
"ready_replay_rejected": True,
},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "transient", remaining=5)
start("worker")
failed = until(a, "failed", timeout=150)
assert (
failed["error_code"] == "retry_exhausted"
and row(j)["attempt"] == 5
)
operator(j)
save(
"finite_retry_exhaustion_then_explicit_replay",
{"failed": failed, "effect": confirm(a, j)},
)
# Reverse notification order and add concurrent copies: one publication/key.
stop("worker")
stop("relay")
a, j = upload()
b, k = upload()
duplicate(k, 8)
duplicate(j, 8)
start("worker")
start("relay")
ea = confirm(a, j)
confirm(b, k)
duplicate(j, 10)
duplicate(k, 10)
wait(
lambda: any(
e["event"] == "duplicate_decision"
for e in inspect_effect(j)["events"]
)
)
assert (
inspect_effect(j)["job"]["attempt"] == ea["job"]["attempt"]
)
save(
"reordered_and_duplicate_notifications",
{
"effects": [inspect_effect(j), inspect_effect(k)],
"added_duplicates": 36,
"no_extra_attempt_after_ready": True,
},
)
# Hold v1 after its PUTs, complete v2 on the other bounded slot, release old CAS.
for repeat in range(args.repeats):
stop("worker")
a, j = upload()
marker = inject(j, "after_effect", "wait")
# Keep one controller process hot throughout the narrow stale-writer window.
code = f"""from src.pipeline import event
with db.connection() as c: event(c,UUID({j!r}),'version_controller_waiting')
end=time.monotonic()+45
while True:
with db.connection() as c: hit=c.execute('SELECT hit_at FROM pipeline_faults WHERE id=%s',({marker},)).fetchone()['hit_at']
if hit: break
if time.monotonic()>end: raise TimeoutError('v1 hook never hit')
time.sleep(.05)
k=change(db,store,job_id=UUID({j!r}),operator='synthetic-verifier',reason='Held stale writer race',version=2)
end=time.monotonic()+10
while True:
with db.connection() as c: ready=c.execute('SELECT state FROM jobs WHERE id=%s',(k,)).fetchone()['state']=='ready'
if ready: break
if time.monotonic()>end: raise TimeoutError('v2 did not finish inside live v1 budget')
time.sleep(.05)
with db.connection() as c:
hashes=c.execute('SELECT name,sha256 FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=2 ORDER BY name',({a["id"]!r},)).fetchall()
c.execute('UPDATE pipeline_faults SET released_at=clock_timestamp() WHERE id=%s',({marker},))
end=time.monotonic()+10
while True:
with db.connection() as c: stale=c.execute("SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='stale_completion_rejected'",({j!r},)).fetchone()['n']
if stale: break
if time.monotonic()>end: raise TimeoutError('Old completion was not rejected')
time.sleep(.05)
print(json.dumps({{'job_id':k,'hashes':{{r['name']:r['sha256'] for r in hashes}}}}))
"""
with ThreadPoolExecutor(max_workers=1) as hot:
future = hot.submit(inside, code)
wait(
lambda j=j: inside(
f"with db.connection() as c: n=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='version_controller_waiting'\",({j!r},)).fetchone()['n']\nprint(json.dumps(n>0))"
)
)
start("worker")
race = future.result(timeout=45)
k = race["job_id"]
hashes = race["hashes"]
confirm(a, k)
duplicate(j, 12)
wait(
lambda j=j: any(
e["event"] == "duplicate_decision"
for e in inspect_effect(j)["events"]
)
)
assert hashes == {
v["name"]: v["sha256"] for v in confirm(a, k)["variants"]
}
assert inspect_effect(j)["publications"] == 0
save(
f"stale_v1_after_v2_repeat_{repeat + 1}",
{
"v1": inspect_effect(j),
"v2": inspect_effect(k),
"new_hashes_unchanged": True,
},
)
# No arbitrary URL/bytes fields in queue envelopes.
invalid = inside(
"bad={'schema_version':1,'url':'http://example.invalid/raw'}\nqueue.xadd(STREAM,{'job':json.dumps(bad)})\nprint('true')"
)
assert invalid
wait(
lambda: inside(
"with db.connection() as c: r=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE event='invalid_envelope'\").fetchone()\nprint(json.dumps(r['n']>0))"
)
)
save(
"malformed_notification_rejected", {"durable_diagnostic": True}
)
# Private bucket, bounded IAM, and download after uploader process dies.
a, j = upload()
confirm(a, j)
key = f"objects/{a['id']}/v1/original"
security = inside(f"""from minio.error import S3Error
from urllib.error import HTTPError
try: urllib.request.urlopen('http://s3:9000/'+BUCKET+'/' + {key!r},timeout=3)
except HTTPError as e: anonymous=e.code
else: raise AssertionError('Anonymous access allowed')
denials={{}}
for name,fn in [('create_bucket',lambda:store.client.make_bucket('forbidden-'+uuid4().hex)),('write_outside_prefix',lambda:store.client.put_object(BUCKET,'outside-'+uuid4().hex,io.BytesIO(b'x'),1)),('list_other_bucket',lambda:list(store.client.list_objects('forbidden-'+uuid4().hex)))]:
try: fn()
except S3Error as e: denials[name]=e.code
else: raise AssertionError('IAM allowed forbidden action')
print(json.dumps({{'anonymous':anonymous,'iam':denials}}))""")
assert security["anonymous"] == 403 and set(
security["iam"].values()
) == {"AccessDenied"}
selected = None
for cid in compose("ps", "-q", "api").split():
alias = command(
[
"docker",
"exec",
cid,
"python",
"-c",
"import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
]
)
uploader = inside(
f"with db.connection() as c: r=c.execute('SELECT uploader_instance FROM attachments WHERE id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(r['uploader_instance']))"
)
if alias == uploader:
selected = cid
assert selected
command(["docker", "stop", "--time", "1", selected])
ready()
wait(
lambda: (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 200
)
)
downloaded = client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
)
assert (
downloaded.content == raw
and downloaded.headers["X-Instance-Id"] != uploader
)
start("api")
ready()
save(
"private_iam_and_other_api_download",
{
"security": security,
"attachment": a["id"],
"uploader": uploader,
"downloader": downloaded.headers["X-Instance-Id"],
"sha256": hashlib.sha256(raw).hexdigest(),
},
)
report["worker_metrics"] = inside(
"with db.connection() as c: r=c.execute(\"SELECT job_id,detail FROM pipeline_events WHERE event='child_finished' ORDER BY id\").fetchall()\nprint(json.dumps(r,default=str))"
)
report["audit"] = inside(
"with db.connection() as c: r=c.execute('SELECT job_id,action,operator,reason,created_at FROM pipeline_audit ORDER BY id').fetchall()\nprint(json.dumps(r,default=str))"
)
report["complete"] = True
finally:
for role in ["api", "broker", "relay", "worker", "reconciler"]:
try:
start(role)
except Exception:
pass
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS: versioned pipeline scope=" + args.start_at, flush=True)
if __name__ == "__main__":
main()