Реализовать наблюдаемость мессенджера и проверить её под нагрузкой
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local synthetic credentials once, without printing their values."""
|
||||
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
directory = root / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
password_file = directory / "postgres_password"
|
||||
if not password_file.exists():
|
||||
password_file.write_text(secrets.token_urlsafe(32) + "\n")
|
||||
password = password_file.read_text().strip()
|
||||
for name, value in {
|
||||
"database_url": f"postgresql://messenger:{password}@postgres:5432/messenger",
|
||||
"grafana_password": secrets.token_urlsafe(24),
|
||||
"cache_url": "redis://valkey:6379/0",
|
||||
"broker_url": "redis://broker:6379/0",
|
||||
}.items():
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(value + "\n")
|
||||
# Compose bind-mount secrets must be readable by the non-root container process.
|
||||
# The parent directory remains 0700 on the host.
|
||||
for role in ("root", "api", "worker", "reconciler", "test"):
|
||||
names = (
|
||||
("s3_root_user", "s3_root_password")
|
||||
if role == "root"
|
||||
else ("s3_" + role + "_access_key_id", "s3_" + role + "_secret_access_key")
|
||||
)
|
||||
for index, name in enumerate(names):
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(
|
||||
(secrets.token_hex(10) if index == 0 else secrets.token_urlsafe(32)) + "\n"
|
||||
)
|
||||
for name in (
|
||||
"postgres_password",
|
||||
"grafana_password",
|
||||
"database_url",
|
||||
"cache_url",
|
||||
"broker_url",
|
||||
*[p.name for p in directory.glob("s3_*")],
|
||||
):
|
||||
os.chmod(directory / name, 0o644)
|
||||
env = root / ".env"
|
||||
if not env.exists():
|
||||
env.write_text((root / ".env.example").read_text())
|
||||
print("Local secret files and environment are ready; existing values preserved.")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Read-only discovery and scoped Compose operations for the local lab5 stack."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", *args],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=90,
|
||||
).stdout
|
||||
|
||||
|
||||
def inspect(container):
|
||||
template = (
|
||||
'{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
|
||||
'"pid":{{.State.Pid}},"running":{{json .State.Running}},'
|
||||
'"image":{{json .Image}},"memory":{{.HostConfig.Memory}},'
|
||||
'"nano_cpus":{{.HostConfig.NanoCpus}},'
|
||||
'"published_ports":{{json .HostConfig.PortBindings}}}'
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "inspect", "--format", template, container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=5,
|
||||
)
|
||||
return json.loads(r.stdout)
|
||||
|
||||
|
||||
def instance(container):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "exec", container, "python", "-c", code],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=3,
|
||||
)
|
||||
value = inspect(container)
|
||||
value["instance"] = r.stdout.strip()
|
||||
return value
|
||||
|
||||
|
||||
def instances():
|
||||
ids = compose("ps", "-q", "api").split()
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
return list(pool.map(instance, ids))
|
||||
|
||||
|
||||
def wait_lb(base="http://127.0.0.1:8086", count=2, timeout=45):
|
||||
deadline, seen = time.monotonic() + timeout, set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
current = {value["instance"] for value in instances()}
|
||||
if len(current) != count:
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
with urllib.request.urlopen(base + "/health/ready", timeout=2) as response:
|
||||
seen.add(response.headers["X-Instance-Id"])
|
||||
if current <= seen:
|
||||
return sorted(current)
|
||||
except (OSError, urllib.error.URLError, subprocess.SubprocessError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("Expected healthy API replicas were not observed through LB")
|
||||
@@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Collect sanitized evidence from the local lab6 stand."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import urlopen
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
|
||||
def compose(*args, **kwargs):
|
||||
return subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, **kwargs)
|
||||
|
||||
|
||||
def api_json(port, path, **params):
|
||||
url = f"http://127.0.0.1:{port}{path}"
|
||||
if params:
|
||||
url += "?" + urlencode(params)
|
||||
with urlopen(url, timeout=5) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def prom(query):
|
||||
data = api_json(9096, "/api/v1/query", query=query)["data"]["result"]
|
||||
return float(data[0]["value"][1]) if data else 0
|
||||
|
||||
|
||||
def until(check, timeout=45):
|
||||
end = time.monotonic() + timeout
|
||||
while time.monotonic() < end:
|
||||
try:
|
||||
result = check()
|
||||
if result:
|
||||
return result
|
||||
except (HTTPError, URLError):
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError(f"Timed out waiting for {check.__name__}")
|
||||
|
||||
|
||||
def log_for(request_id):
|
||||
data = api_json(
|
||||
3106,
|
||||
"/loki/api/v1/query_range",
|
||||
query='{service_name="messenger-api"} | request_id="' + request_id + '"',
|
||||
start=str(time.time_ns() - 180_000_000_000),
|
||||
limit=5,
|
||||
)["data"]["result"]
|
||||
for stream in data:
|
||||
for value in stream["values"]:
|
||||
return value[2] if len(value) > 2 else stream["stream"]
|
||||
return None
|
||||
|
||||
|
||||
def span_names(trace_id):
|
||||
data = api_json(3206, "/api/traces/" + trace_id)
|
||||
return [
|
||||
(span["name"], span.get("kind"), span["spanId"], span.get("parentSpanId"))
|
||||
for batch in data.get("batches", data.get("resourceSpans", []))
|
||||
for group in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", []))
|
||||
for span in group["spans"]
|
||||
]
|
||||
|
||||
|
||||
def alerts():
|
||||
return {
|
||||
a["labels"]["alertname"]: a["state"]
|
||||
for a in api_json(9096, "/api/v1/alerts")["data"]["alerts"]
|
||||
}
|
||||
|
||||
|
||||
def correlation(client, actor, chat, retry=False):
|
||||
if retry:
|
||||
compose("stop", "worker", stdout=subprocess.DEVNULL)
|
||||
try:
|
||||
attachment = client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=observability.png",
|
||||
202,
|
||||
actor=actor,
|
||||
raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
if retry:
|
||||
job_attachment = attachment["json"]["id"]
|
||||
compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-Xq",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
"-v",
|
||||
"attachment=" + job_attachment,
|
||||
input=(
|
||||
"INSERT INTO pipeline_faults(job_id,phase,action,remaining) "
|
||||
"SELECT job_id,'before_effect','transient',1 FROM attachment_jobs "
|
||||
"WHERE attachment_id=:'attachment';\n"
|
||||
),
|
||||
text=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
)
|
||||
finally:
|
||||
if retry:
|
||||
compose("start", "worker", stdout=subprocess.DEVNULL)
|
||||
client.wait_ready(actor, attachment["json"])
|
||||
request_id = attachment["headers"]["X-Request-Id"]
|
||||
log = until(lambda: log_for(request_id))
|
||||
trace_id = log["trace_id"]
|
||||
expected = {"jobs.publish", "jobs.consume", "s3.put", "s3.download", "db.transaction"}
|
||||
|
||||
def connected():
|
||||
spans = span_names(trace_id)
|
||||
names = {row[0] for row in spans}
|
||||
consumers = sum(row[0] == "jobs.consume" for row in spans)
|
||||
return spans if expected <= names and consumers >= (2 if retry else 1) else None
|
||||
|
||||
spans = until(connected)
|
||||
by_id = {row[2]: row for row in spans}
|
||||
consumers = [row for row in spans if row[0] == "jobs.consume"]
|
||||
assert all(by_id[row[3]][0] == "jobs.publish" for row in consumers)
|
||||
logs = until(
|
||||
lambda: api_json(
|
||||
3106,
|
||||
"/loki/api/v1/query_range",
|
||||
query='{service_name="messenger-worker"} | trace_id="' + trace_id + '"',
|
||||
start=str(time.time_ns() - 180_000_000_000),
|
||||
limit=5,
|
||||
)["data"]["result"]
|
||||
)
|
||||
assert any(value[1] == "job_processed" for row in logs for value in row["values"])
|
||||
return {
|
||||
"request_id": request_id,
|
||||
"trace_id": trace_id,
|
||||
"span_names": sorted({r[0] for r in spans}),
|
||||
"worker_parent": by_id[consumers[0][3]][0],
|
||||
"worker_attempts": len(consumers),
|
||||
"api_and_worker_loki": True,
|
||||
}
|
||||
|
||||
|
||||
def worker_incident(client, actor, chat):
|
||||
compose("stop", "worker", stdout=subprocess.DEVNULL)
|
||||
try:
|
||||
uploads = [client.attachment(actor, chat)[0] for _ in range(3)]
|
||||
until(lambda: alerts().get("MessengerQueueLag") == "firing")
|
||||
record = {
|
||||
"depth_at_firing": prom("max(messenger_queue_depth)"),
|
||||
"oldest_seconds_at_firing": prom("max(messenger_queue_oldest_seconds)"),
|
||||
}
|
||||
assert record["depth_at_firing"] >= 3
|
||||
finally:
|
||||
compose("start", "worker", stdout=subprocess.DEVNULL)
|
||||
for attachment in uploads:
|
||||
client.wait_ready(actor, attachment)
|
||||
until(lambda: not prom("max(messenger_queue_depth)") and "MessengerQueueLag" not in alerts())
|
||||
return {**record, "state": ["firing", "resolved"], "uploads_ready": len(uploads)}
|
||||
|
||||
|
||||
def database_incident(client, actor):
|
||||
# Independent, bounded transaction. Lock sessions only; readiness can still use the schema.
|
||||
command = [
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
"messenger",
|
||||
"-XqAt",
|
||||
"-v",
|
||||
"ON_ERROR_STOP=1",
|
||||
]
|
||||
start = time.monotonic()
|
||||
lock = subprocess.Popen(
|
||||
command, cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
|
||||
)
|
||||
lock.stdin.write(
|
||||
"BEGIN;\nLOCK TABLE sessions IN ACCESS EXCLUSIVE MODE;\n"
|
||||
"\\echo locked\nSELECT pg_sleep(42);\nROLLBACK;\n"
|
||||
)
|
||||
lock.stdin.close()
|
||||
try:
|
||||
assert lock.stdout.readline().strip() == "locked"
|
||||
attempts = []
|
||||
while time.monotonic() - start < 34:
|
||||
attempt = client.call("GET", "/api/v1/users/me", [200, 503], actor=actor, schema="User")
|
||||
attempts.append(attempt)
|
||||
states = alerts()
|
||||
if all(
|
||||
states.get(a) == "firing"
|
||||
for a in ("MessengerHttpErrors", "MessengerDatabaseErrors")
|
||||
):
|
||||
break
|
||||
assert any(item["status"] == 503 for item in attempts)
|
||||
active = alerts()
|
||||
assert active.get("MessengerDatabaseErrors") == "firing", active
|
||||
assert active.get("MessengerHttpErrors") == "firing", active
|
||||
finally:
|
||||
lock.wait(timeout=46)
|
||||
lock.stdout.close()
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
|
||||
# Keep successful traffic to publish fresh data while the alert window slides.
|
||||
def resolved():
|
||||
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
return not ({"MessengerDatabaseErrors", "MessengerHttpErrors"} & alerts().keys())
|
||||
|
||||
until(resolved)
|
||||
failure = next(item for item in attempts if item["status"] == 503)
|
||||
failure_log = until(lambda: log_for(failure["headers"]["X-Request-Id"]))
|
||||
names = span_names(failure_log["trace_id"])
|
||||
assert "db.read" in {row[0] for row in names}
|
||||
return {
|
||||
"statuses": [item["status"] for item in attempts],
|
||||
"max_latency_ms": max(item["elapsed"] for item in attempts) * 1000,
|
||||
"trace_id": failure_log["trace_id"],
|
||||
"state": ["firing", "resolved"],
|
||||
"spans": sorted({row[0] for row in names}),
|
||||
}
|
||||
|
||||
|
||||
def backend_incident(client, actor, chat):
|
||||
lost = "sum(otelcol_exporter_send_failed_spans_total)"
|
||||
baseline = prom(lost)
|
||||
compose("stop", "tempo", stdout=subprocess.DEVNULL)
|
||||
try:
|
||||
for _ in range(15):
|
||||
client.message(actor, chat, "synthetic telemetry outage probe")
|
||||
time.sleep(0.5)
|
||||
delta = until(lambda: value if (value := prom(lost) - baseline) > 0 else None)
|
||||
memory = prom("max(otelcol_process_memory_rss_bytes)")
|
||||
assert memory > 0
|
||||
finally:
|
||||
compose("start", "tempo", stdout=subprocess.DEVNULL)
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
return {"successful_messages": 15, "failed_export_spans": delta, "collector_rss_bytes": memory}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--incidents", action="store_true")
|
||||
parser.add_argument("--retry", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
parser.error("choose a new evidence file")
|
||||
client = Client(args.base_url)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
result = {"correlation": correlation(client, alice, chat, retry=args.retry)}
|
||||
print("PASS: same Tempo trace and Loki logs for upload and worker", flush=True)
|
||||
if args.incidents:
|
||||
result["worker_stop"] = worker_incident(client, alice, chat)
|
||||
print("PASS: queue alert firing and resolved", flush=True)
|
||||
result["database_lock"] = database_incident(client, alice)
|
||||
print("PASS: database and HTTP alerts firing and resolved", flush=True)
|
||||
result["tempo_stop"] = backend_incident(client, alice, chat)
|
||||
print("PASS: messages persisted during Tempo outage", flush=True)
|
||||
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Local test operator configuration. Credentials stay in ignored files/in memory."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from psycopg.conninfo import make_conninfo
|
||||
|
||||
from messenger.settings import Settings
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def settings(role="worker"):
|
||||
prefix = "s3_root" if role == "root" else "s3_" + role
|
||||
access, secret = (
|
||||
("user", "password") if role == "root" else ("access_key_id", "secret_access_key")
|
||||
)
|
||||
return Settings(
|
||||
make_conninfo(
|
||||
host="127.0.0.1",
|
||||
port=int(os.getenv("TEST_PG_PORT", "55467")),
|
||||
user="messenger",
|
||||
dbname="messenger",
|
||||
password=(ROOT / "secrets/postgres_password").read_text().strip(),
|
||||
),
|
||||
"redis://127.0.0.1:" + os.getenv("TEST_CACHE_PORT", "56386") + "/0",
|
||||
s3_endpoint="http://127.0.0.1:" + os.getenv("TEST_S3_PORT", "59006"),
|
||||
s3_access_key=(ROOT / ("secrets/" + prefix + "_" + access)).read_text().strip(),
|
||||
s3_secret_key=(ROOT / ("secrets/" + prefix + "_" + secret)).read_text().strip(),
|
||||
broker_url="redis://127.0.0.1:" + os.getenv("TEST_BROKER_PORT", "56387") + "/0",
|
||||
job_timeout=int(os.getenv("TEST_JOB_TIMEOUT_SECONDS", "5")),
|
||||
fault_injection=True,
|
||||
)
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# All running application/state services; no migration, down -v, flush or seed.
|
||||
docker compose --env-file .env restart --timeout 7 api postgres valkey s3 broker relay worker lb
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sequential process replacement; at least one existing API stays running."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
from cluster import compose, instances, wait_lb
|
||||
|
||||
|
||||
def replace_all(base="http://127.0.0.1:8084"):
|
||||
original = instances()
|
||||
if len(original) < 2:
|
||||
raise RuntimeError("Rolling replacement requires at least two healthy replicas")
|
||||
wait_lb(base, len(original))
|
||||
replacements = []
|
||||
for current in original:
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "5", current["id"]],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=8,
|
||||
)
|
||||
subprocess.run(["docker", "rm", current["id"]], check=True, capture_output=True)
|
||||
compose(
|
||||
"up", "-d", "--no-deps", "--no-recreate", "--scale", "api=" + str(len(original)), "api"
|
||||
)
|
||||
aliases = wait_lb(base, len(original))
|
||||
if current["instance"] in aliases:
|
||||
raise RuntimeError("Old API process is still serving after replacement")
|
||||
replacements.append(
|
||||
{
|
||||
"old_container": current["id"],
|
||||
"old_instance": current["instance"],
|
||||
"live_instances": aliases,
|
||||
}
|
||||
)
|
||||
return replacements
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8084")
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(replace_all(args.base_url)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Map a process alias to ONE container belonging to this Compose service."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def alias(container):
|
||||
code = (
|
||||
"import urllib.request; r=urllib.request.urlopen("
|
||||
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
r = subprocess.run(
|
||||
["docker", "exec", container, "python", "-c", code],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=3,
|
||||
check=True,
|
||||
)
|
||||
return container, r.stdout.strip()
|
||||
|
||||
|
||||
def main():
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "")
|
||||
if not re.fullmatch(r"[a-f0-9]{32}", target):
|
||||
raise SystemExit("TARGET_INSTANCE_ID must name a live API process")
|
||||
ids = subprocess.check_output(
|
||||
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
|
||||
).split()
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
matches = [cid for cid, identity in pool.map(alias, ids) if identity == target]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit("Expected exactly one matching API in this project")
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "5", matches[0]], check=True, capture_output=True, timeout=7
|
||||
)
|
||||
print(json.dumps({"stopped_container": matches[0], "instance": target}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/stop-one.py
|
||||
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fixed offered-load comparison. Stops only named services in the current lab6 project."""
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.request import urlopen
|
||||
|
||||
from observe import Client, compose, prom, until
|
||||
|
||||
MODES = (("off", "0", "0"), ("full", "1", "1"), ("half", "1", "0.5"), ("quarter", "1", "0.25"))
|
||||
COUNTERS = {
|
||||
"accepted_spans": "sum(otelcol_receiver_accepted_spans_total)",
|
||||
"sent_spans": "sum(otelcol_exporter_sent_spans_total)",
|
||||
"failed_spans": "sum(otelcol_exporter_send_failed_spans_total)",
|
||||
"accepted_logs": "sum(otelcol_receiver_accepted_log_records_total)",
|
||||
"accepted_metric_points": "sum(otelcol_receiver_accepted_metric_points_total)",
|
||||
}
|
||||
|
||||
|
||||
def configure(enabled, ratio):
|
||||
compose(
|
||||
"up",
|
||||
"-d",
|
||||
"--no-build",
|
||||
"--no-deps",
|
||||
"--force-recreate",
|
||||
"api",
|
||||
"worker",
|
||||
"relay",
|
||||
env={**os.environ, "TELEMETRY_ENABLED": enabled, "OTEL_SAMPLE_RATIO": ratio},
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
def ready():
|
||||
try:
|
||||
with urlopen("http://127.0.0.1:8086/health/ready", timeout=2) as response:
|
||||
return response.status == 200
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
until(ready)
|
||||
|
||||
|
||||
def snapshot():
|
||||
with urlopen("http://127.0.0.1:3206/metrics", timeout=4) as response:
|
||||
metrics = response.read().decode()
|
||||
received_bytes = sum(
|
||||
float(value)
|
||||
for value in re.findall(
|
||||
r"^tempo_distributor_bytes_received_total\{[^}]+\} (\S+)", metrics, re.M
|
||||
)
|
||||
)
|
||||
return {
|
||||
"trace_ingest_proto_bytes": received_bytes,
|
||||
**{name: prom(expression) for name, expression in COUNTERS.items()},
|
||||
}
|
||||
|
||||
|
||||
def workload(client, actor, chat, seconds, rate):
|
||||
def operation(n):
|
||||
if n % 45 == 0:
|
||||
started = time.perf_counter()
|
||||
client.attachment(actor, chat)
|
||||
return (time.perf_counter() - started) * 1000
|
||||
if n % 5 == 0:
|
||||
response = client.message(actor, chat, "synthetic cost experiment")
|
||||
else:
|
||||
response = client.call(
|
||||
"GET",
|
||||
f"/api/v1/chats/{chat['id']}/messages?limit=20",
|
||||
actor=actor,
|
||||
schema="MessagePage",
|
||||
)
|
||||
return response["elapsed"] * 1000
|
||||
|
||||
start = time.perf_counter()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
|
||||
futures = []
|
||||
for number in range(seconds * rate):
|
||||
delay = start + number / rate - time.perf_counter()
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
futures.append(pool.submit(operation, number))
|
||||
observations = sorted(future.result() for future in futures)
|
||||
elapsed = time.perf_counter() - start
|
||||
return {
|
||||
"requests": len(observations),
|
||||
"elapsed_s": elapsed,
|
||||
"latency_ms": {
|
||||
f"p{int(q * 100)}": observations[math.ceil(q * len(observations)) - 1]
|
||||
for q in (0.50, 0.95, 0.99)
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def stats():
|
||||
services = ("api", "worker", "relay", "otel-collector", "prometheus", "loki", "tempo")
|
||||
ids = compose("ps", "-q", *services, capture_output=True, text=True).stdout.split()
|
||||
raw = subprocess.check_output(
|
||||
["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids], text=True
|
||||
)
|
||||
result = []
|
||||
for line in raw.splitlines():
|
||||
row = json.loads(line)
|
||||
result.append(
|
||||
{
|
||||
"container": row["Name"],
|
||||
"cpu_percent": float(row["CPUPerc"].rstrip("%")),
|
||||
"memory": row["MemUsage"].split(" / ")[0],
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def run_one(mode, seconds, rate):
|
||||
label, enabled, ratio = mode
|
||||
configure(enabled, ratio)
|
||||
client = Client("http://127.0.0.1:8086")
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
for _ in range(25):
|
||||
client.message(actor, chat, "seed")
|
||||
workload(client, actor, chat, 3, rate)
|
||||
time.sleep(3)
|
||||
initial = snapshot()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
run = pool.submit(workload, client, actor, chat, seconds, rate)
|
||||
samples = []
|
||||
for _ in range(2):
|
||||
time.sleep(2)
|
||||
samples.append(stats())
|
||||
receipt = run.result()
|
||||
# Collector emits batches and Prometheus scrapes every two seconds.
|
||||
time.sleep(4)
|
||||
final = snapshot()
|
||||
return {
|
||||
"mode": label,
|
||||
"sample_ratio": float(ratio),
|
||||
"load": receipt,
|
||||
"container_samples": samples,
|
||||
"ingest": {k: final[k] - v for k, v in initial.items()},
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--seconds", type=int, default=10)
|
||||
parser.add_argument("--rate", type=int, default=18)
|
||||
parser.add_argument("--repeats", type=int, default=2)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.output.exists() or args.seconds < 8 or args.rate < 1 or args.repeats < 1:
|
||||
parser.error("use a new output and a positive load for at least 8 s")
|
||||
results = []
|
||||
try:
|
||||
for repeat in range(args.repeats):
|
||||
for mode in MODES if repeat % 2 == 0 else MODES[::-1]:
|
||||
row = run_one(mode, args.seconds, args.rate)
|
||||
print(row["mode"], row["load"]["latency_ms"], row["ingest"], flush=True)
|
||||
results.append(row)
|
||||
args.output.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"seconds": args.seconds,
|
||||
"offered_requests_per_s": args.rate,
|
||||
"runs": results,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
finally:
|
||||
configure("1", "1")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Two-phase real-age GC check: seed, run other checks, then verify after >=300 seconds."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from cluster import ROOT, compose
|
||||
from pipeline_lab import settings
|
||||
|
||||
from messenger.cleanup import collect
|
||||
from messenger.pipeline import Pipeline, intent
|
||||
from messenger.repository import PostgresRepository
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(__doc__)
|
||||
parser.add_argument("action", choices=("seed", "verify"))
|
||||
parser.add_argument("--seed", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
|
||||
args = parser.parse_args()
|
||||
config = settings("reconciler")
|
||||
db = PostgresRepository(config.database_url, pool_size=3)
|
||||
db.open()
|
||||
store = ObjectStore(config)
|
||||
try:
|
||||
if args.action == "seed":
|
||||
check(not args.seed.exists(), "Seed already exists")
|
||||
compose("stop", "--timeout", "2", "worker", "relay")
|
||||
try:
|
||||
c = Client(args.base_url)
|
||||
alice, _, _, room = c.bootstrap()
|
||||
active, data = c.attachment(alice, room)
|
||||
failed, _ = c.attachment(alice, room)
|
||||
orphan = str(uuid4())
|
||||
Pipeline(db, config).reserve(orphan, "staging/" + orphan)
|
||||
uploader = ObjectStore(settings("api"))
|
||||
try:
|
||||
uploader.put(
|
||||
"staging/" + orphan,
|
||||
io.BytesIO(data),
|
||||
len(data),
|
||||
"image/png",
|
||||
hashlib.sha256(data).hexdigest(),
|
||||
)
|
||||
finally:
|
||||
uploader.close()
|
||||
with db.connection(write=True) as conn:
|
||||
j = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (active["id"],)
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"UPDATE attachment_jobs SET "
|
||||
"next_attempt_at=clock_timestamp()+interval '1 hour' "
|
||||
"WHERE job_id=%s",
|
||||
(j["job_id"],),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE job_outbox SET due_at=clock_timestamp()+interval '1 "
|
||||
"hour' WHERE job_id=%s",
|
||||
(j["job_id"],),
|
||||
)
|
||||
f = conn.execute(
|
||||
"SELECT job_id FROM attachment_jobs WHERE attachment_id=%s", (failed["id"],)
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"INSERT INTO pipeline_faults(job_id,phase,action) VALUES "
|
||||
"(%s,'before_effect','permanent')",
|
||||
(f["job_id"],),
|
||||
)
|
||||
seed = {
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"active_id": active["id"],
|
||||
"failed_id": failed["id"],
|
||||
"orphan_id": orphan,
|
||||
}
|
||||
args.seed.write_text(json.dumps(seed, indent=2) + "\n")
|
||||
print(
|
||||
"PASS: active job, failed candidate and unaccepted orphan seeded; no "
|
||||
"credentials saved"
|
||||
)
|
||||
finally:
|
||||
compose("start", "worker", "relay")
|
||||
else:
|
||||
check(args.output is not None and not args.output.exists(), "Choose a new output file")
|
||||
seed = json.loads(args.seed.read_text())
|
||||
# Real S3 timestamps and failed retention; no modified clocks.
|
||||
while True:
|
||||
with db.connection() as conn:
|
||||
f = conn.execute(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s", (seed["failed_id"],)
|
||||
).fetchone()
|
||||
check(f["state"] == "failed", "Failed seed did not reach terminal state")
|
||||
age = (datetime.now(UTC) - f["updated_at"]).total_seconds()
|
||||
if age >= 301:
|
||||
break
|
||||
check(age >= 0, "Clock mismatch")
|
||||
time.sleep(min(1, 301 - age))
|
||||
with db.connection() as conn:
|
||||
ready = conn.execute(
|
||||
"SELECT * FROM attachments WHERE storage_state='ready' LIMIT 1"
|
||||
).fetchone()
|
||||
check(ready is not None, "Need a published attachment to protect")
|
||||
keys = [ready["object_key"], *[v["object_key"] for v in ready["variants"]]]
|
||||
before = {key: store.head(key)["ETag"] for key in keys}
|
||||
preview = collect(db, store, 300, False)
|
||||
result = collect(db, store, 300, True)
|
||||
check(
|
||||
store.head("staging/" + seed["active_id"]) is not None, "Active staging was deleted"
|
||||
)
|
||||
check(store.head("staging/" + seed["orphan_id"]) is None, "Orphan not deleted")
|
||||
check(
|
||||
store.head("staging/" + seed["failed_id"]) is None,
|
||||
"Expired failed source not deleted",
|
||||
)
|
||||
check(
|
||||
all(store.head(k)["ETag"] == tag for k, tag in before.items()),
|
||||
"Published result changed",
|
||||
)
|
||||
with db.connection(write=True) as conn:
|
||||
expired = conn.execute(
|
||||
"SELECT source_expired FROM attachment_jobs WHERE attachment_id=%s",
|
||||
(seed["failed_id"],),
|
||||
).fetchone()
|
||||
check(expired["source_expired"], "Expired source not registered")
|
||||
j = conn.execute(
|
||||
"UPDATE attachment_jobs SET next_attempt_at=clock_timestamp() WHERE "
|
||||
"attachment_id=%s RETURNING job_id",
|
||||
(seed["active_id"],),
|
||||
).fetchone()
|
||||
intent(conn, j["job_id"])
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
if db.attachment(seed["active_id"])["storage_state"] == "ready":
|
||||
break
|
||||
time.sleep(0.2)
|
||||
check(
|
||||
db.attachment(seed["active_id"])["storage_state"] == "ready",
|
||||
"Protected work failed to complete",
|
||||
)
|
||||
proof = {
|
||||
"retention_seconds": 300,
|
||||
"failed_real_age_seconds": age,
|
||||
"seed": seed,
|
||||
"dry_run": preview,
|
||||
"apply": result,
|
||||
"active_protected_then_ready": True,
|
||||
"orphan_deleted": True,
|
||||
"failed_source_expired": True,
|
||||
"published_objects_unchanged": len(keys),
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(proof, indent=2) + "\n")
|
||||
print("PASS: real-age cleanup preserves active work and published results")
|
||||
finally:
|
||||
store.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real LB requests during each replica failure and sequential replacement."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from cluster import ROOT, compose, inspect, instance, instances, wait_lb
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
from resilience import history # noqa: E402
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
values = sorted(values)
|
||||
return values[min(int((len(values) - 1) * fraction), len(values) - 1)] if values else None
|
||||
|
||||
|
||||
class Traffic:
|
||||
def __init__(self, base, actor, room, workers=6):
|
||||
self.base, self.actor, self.room = base, actor, room
|
||||
self.prefix = uuid4().hex
|
||||
self.stop = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.events = []
|
||||
self.executor = ThreadPoolExecutor(max_workers=workers)
|
||||
self.workers = workers
|
||||
|
||||
def worker(self, worker):
|
||||
client, iteration = Client(self.base, timeout=3), 0
|
||||
while not self.stop.is_set():
|
||||
started = time.monotonic()
|
||||
write = iteration % 10 == 9
|
||||
try:
|
||||
if write:
|
||||
key = f"{self.prefix}:{worker}:{iteration}"
|
||||
response = client.message(self.actor, self.room, key, key=key)
|
||||
else:
|
||||
response = client.call(
|
||||
"GET", "/api/v1/users/me", actor=self.actor, schema="User"
|
||||
)
|
||||
event = {
|
||||
"at": time.monotonic(),
|
||||
"ok": True,
|
||||
"instance": response["headers"]["X-Instance-Id"],
|
||||
"latency_ms": (time.monotonic() - started) * 1000,
|
||||
"message": response["json"] if write else None,
|
||||
}
|
||||
iteration += 1
|
||||
except Failure:
|
||||
event = {
|
||||
"at": time.monotonic(),
|
||||
"ok": False,
|
||||
"latency_ms": (time.monotonic() - started) * 1000,
|
||||
}
|
||||
with self.lock:
|
||||
self.events.append(event)
|
||||
|
||||
def __enter__(self):
|
||||
self.started = time.monotonic()
|
||||
self.futures = [self.executor.submit(self.worker, i) for i in range(self.workers)]
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.stop.set()
|
||||
self.executor.shutdown(wait=True)
|
||||
for future in self.futures:
|
||||
future.result()
|
||||
self.finished = time.monotonic()
|
||||
|
||||
def report(self, action_start, action_end):
|
||||
events = sorted(self.events, key=lambda row: row["at"])
|
||||
ok = [row for row in events if row["ok"]]
|
||||
after = [row for row in ok if row["at"] >= action_end]
|
||||
check(bool(after), "No successful request after action")
|
||||
elapsed = self.finished - self.started
|
||||
latency = [row["latency_ms"] for row in ok]
|
||||
return {
|
||||
"duration_seconds": elapsed,
|
||||
"workers": self.workers,
|
||||
"requests": len(events),
|
||||
"successes": len(ok),
|
||||
"errors": len(events) - len(ok),
|
||||
"successful_rps": len(ok) / elapsed,
|
||||
"error_fraction": (len(events) - len(ok)) / len(events),
|
||||
"first_success_after_action_seconds": after[0]["at"] - action_start,
|
||||
"action_seconds": action_end - action_start,
|
||||
"p50_ms": percentile(latency, 0.5),
|
||||
"p95_ms": percentile(latency, 0.95),
|
||||
"p99_ms": percentile(latency, 0.99),
|
||||
"instances": sorted({row["instance"] for row in ok}),
|
||||
"tail_instances": sorted({row["instance"] for row in after[-10:]}),
|
||||
"confirmed_writes": sum(row["message"] is not None for row in ok),
|
||||
}
|
||||
|
||||
def verify_messages(self, client, actor, room):
|
||||
actual = history(client, actor, room)
|
||||
by_id = {value["id"]: value for value in actual}
|
||||
check(len(by_id) == len(actual), "History has duplicate IDs")
|
||||
texts = [row["text"] for row in actual if row["text"].startswith(self.prefix)]
|
||||
check(len(texts) == len(set(texts)), "An uncertain retry created a duplicate effect")
|
||||
for event in self.events:
|
||||
if event["ok"] and event["message"]:
|
||||
check(
|
||||
by_id.get(event["message"]["id"]) == event["message"],
|
||||
"A confirmed message was lost or changed",
|
||||
)
|
||||
|
||||
|
||||
def public_test(target, base):
|
||||
result = subprocess.run(
|
||||
[
|
||||
"make",
|
||||
target,
|
||||
"ACTION_SCRIPT=scripts/"
|
||||
+ ("restart.sh" if target == "test-persistence" else "stop-one.sh"),
|
||||
"BASE_URL=" + base,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=150,
|
||||
)
|
||||
return "\n".join(line.rstrip() for line in result.stdout.splitlines()) + "\n"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
|
||||
parser.add_argument(
|
||||
"--output", type=Path, default=ROOT / f"evidence/cluster-{time.time_ns()}.json"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
raise SystemExit("Use a new evidence filename")
|
||||
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost"
|
||||
baseline = instances()
|
||||
check(len(baseline) == 2, "This verification expects the default two API replicas")
|
||||
check(not any(row["published_ports"] for row in baseline), "API must not publish host ports")
|
||||
wait_lb(args.base_url, 2)
|
||||
result = {"initial_instances": baseline, "failures": []}
|
||||
client = Client(args.base_url, timeout=4)
|
||||
alice, bob, eve, room = client.bootstrap()
|
||||
key = uuid4().hex
|
||||
|
||||
def same_request(_):
|
||||
response = Client(args.base_url).message(alice, room, "concurrent replay", key=key)
|
||||
return response["json"], response["headers"]["X-Instance-Id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
responses = list(pool.map(same_request, range(30)))
|
||||
check(
|
||||
len({row[0]["id"] for row in responses}) == 1, "Cross-replica replay duplicated a message"
|
||||
)
|
||||
check(len({row[1] for row in responses}) == 2, "Concurrent replay did not cross real replicas")
|
||||
result["concurrent_replays"] = 30
|
||||
result["concurrent_effects"] = 1
|
||||
try:
|
||||
for selected in baseline:
|
||||
selected = instance(selected["id"])
|
||||
with Traffic(args.base_url, alice, room) as traffic:
|
||||
time.sleep(1)
|
||||
action_start = time.monotonic()
|
||||
env = {**os.environ, "TARGET_INSTANCE_ID": selected["instance"]}
|
||||
subprocess.run(
|
||||
[str(ROOT / "scripts/stop-one.sh")],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
)
|
||||
action_end = time.monotonic()
|
||||
check(not inspect(selected["id"])["running"], "Chosen process was not stopped")
|
||||
time.sleep(3)
|
||||
report = traffic.report(action_start, action_end)
|
||||
check(report["first_success_after_action_seconds"] <= 10, "Failover took too long")
|
||||
check(selected["instance"] not in report["tail_instances"], "Stopped API still serves")
|
||||
report["stopped"] = selected
|
||||
traffic.verify_messages(client, bob, room)
|
||||
result["failures"].append(report)
|
||||
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
|
||||
wait_lb(args.base_url, 2)
|
||||
print("Verified failure and restoration of one API", flush=True)
|
||||
with Traffic(args.base_url, alice, room) as traffic:
|
||||
time.sleep(1)
|
||||
action_start = time.monotonic()
|
||||
replaced = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / "scripts/rolling-update.py"),
|
||||
"--base-url",
|
||||
args.base_url,
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
action_end = time.monotonic()
|
||||
time.sleep(3)
|
||||
traffic.verify_messages(client, bob, room)
|
||||
result["rolling"] = traffic.report(action_start, action_end)
|
||||
result["rolling"]["replacements"] = json.loads(replaced.stdout)
|
||||
result["rolling"]["confirmed_messages_preserved"] = True
|
||||
print("Verified sequential replacement under traffic", flush=True)
|
||||
(ROOT / "evidence/failover.txt").write_text(public_test("test-failover", args.base_url))
|
||||
for value in compose("ps", "-aq", "api").split():
|
||||
if not inspect(value)["running"]:
|
||||
subprocess.run(["docker", "start", value], check=True, capture_output=True)
|
||||
wait_lb(args.base_url, 2)
|
||||
before = {
|
||||
service: inspect(compose("ps", "-q", service).strip())
|
||||
for service in ("postgres", "valkey", "s3", "broker", "lb")
|
||||
}
|
||||
(ROOT / "evidence/persistence.txt").write_text(
|
||||
public_test("test-persistence", args.base_url)
|
||||
)
|
||||
after = {service: inspect(compose("ps", "-q", service).strip()) for service in before}
|
||||
check(
|
||||
all(before[s]["started_at"] != after[s]["started_at"] for s in before),
|
||||
"Persistence hook did not restart all state/LB services",
|
||||
)
|
||||
result["persistence"] = {"before": before, "after": after, "same_token_public_test": True}
|
||||
result["final_instances"] = instances()
|
||||
logs = compose("logs", "--no-color", "api", "lb")
|
||||
check(
|
||||
not any(
|
||||
a["token"] in logs or a["registration"]["password"] in logs
|
||||
for a in (alice, bob, eve)
|
||||
),
|
||||
"Credential appeared in logs",
|
||||
)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print("PASS: real replica failures, replay, rolling replacement and persistence")
|
||||
finally:
|
||||
for value in compose("ps", "-aq", "api").split():
|
||||
if not inspect(value)["running"]:
|
||||
subprocess.run(["docker", "start", value], check=True, capture_output=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,513 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real-process fault matrix for the versioned pipeline; test Compose override required."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
from cluster import ROOT, compose, inspect, instances, wait_lb
|
||||
from PIL import Image
|
||||
from pipeline_lab import settings
|
||||
|
||||
from messenger.broker import STREAM, Broker, Job
|
||||
from messenger.pipeline import Pipeline
|
||||
from messenger.repository import PostgresRepository, stamp
|
||||
from messenger.storage import ObjectStore
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
|
||||
def until(fn, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
result = fn()
|
||||
if result:
|
||||
return result
|
||||
time.sleep(0.2)
|
||||
raise AssertionError("Timed out waiting for a durable pipeline condition")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
|
||||
parser.add_argument("--output", type=Path, default=ROOT / "evidence/pipeline.json")
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
parser.error("Output already exists; choose a new evidence file")
|
||||
config = settings()
|
||||
db = PostgresRepository(config.database_url, pool_size=4)
|
||||
db.open()
|
||||
broker, store = Broker(config), ObjectStore(config)
|
||||
c = Client(args.base_url)
|
||||
alice, bob, eve, room = c.bootstrap()
|
||||
records, accepted = [], []
|
||||
data = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
def query(sql, params=()):
|
||||
with db.connection() as conn:
|
||||
return conn.execute(sql, params).fetchall()
|
||||
|
||||
def job(a, version=1):
|
||||
return query(
|
||||
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s",
|
||||
(a["id"], version),
|
||||
)[0]
|
||||
|
||||
def audit(j):
|
||||
return query(
|
||||
"SELECT event,details,created_at FROM pipeline_audit WHERE job_id=%s ORDER BY sequence",
|
||||
(j["job_id"],),
|
||||
)
|
||||
|
||||
def injected(j, phase):
|
||||
return any(
|
||||
e["event"] == "fault_injected" and e["details"]["phase"] == phase for e in audit(j)
|
||||
)
|
||||
|
||||
def fault(j, phase, action, count=1):
|
||||
with db.connection(write=True) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO pipeline_faults(job_id,phase,action,remaining) VALUES (%s,%s,%s,%s)",
|
||||
(j["job_id"], phase, action, count),
|
||||
)
|
||||
|
||||
def upload():
|
||||
a, original = c.attachment(alice, room)
|
||||
check(original == data, "Unexpected fixture")
|
||||
accepted.append(a["id"])
|
||||
return a
|
||||
|
||||
def ready(a):
|
||||
value = c.wait_ready(bob, a)
|
||||
path = "/api/v1/attachments/" + a["id"]
|
||||
check(c.call("GET", path + "/content", actor=bob)["bytes"] == data, "Original changed")
|
||||
for v in value["variants"]:
|
||||
content = c.call("GET", path + "/content?variant=" + v["name"], actor=bob)["bytes"]
|
||||
check(hashlib.sha256(content).hexdigest() == v["sha256"], "Variant checksum mismatch")
|
||||
with Image.open(io.BytesIO(content)) as im:
|
||||
check(im.size == (v["width"], v["height"]), "Variant dimensions mismatch")
|
||||
check(len(value["variants"]) == 3, "Missing variants")
|
||||
check(
|
||||
len(query("SELECT 1 FROM message_attachments WHERE attachment_id=%s", (a["id"],))) <= 1,
|
||||
"Unexpected duplicate link",
|
||||
)
|
||||
return value
|
||||
|
||||
def saved(name, a, start, **details):
|
||||
j = job(a, details.get("version", 1))
|
||||
current = query(
|
||||
"SELECT storage_state,pipeline_version FROM attachments WHERE id=%s", (a["id"],)
|
||||
)[0]
|
||||
keys = [
|
||||
o["Key"]
|
||||
for o in store.client.list_objects_v2(
|
||||
Bucket=store.bucket,
|
||||
Prefix="objects/" + a["id"] + "/v" + str(current["pipeline_version"]) + "/",
|
||||
).get("Contents", [])
|
||||
]
|
||||
check(len(keys) == 4, "Repeated deliveries multiplied the published variant set")
|
||||
record = {
|
||||
"scenario": name,
|
||||
"attachment_id": a["id"],
|
||||
"job_id": str(j["job_id"]),
|
||||
"elapsed_seconds": time.monotonic() - start,
|
||||
"final_state": current["storage_state"],
|
||||
"attempts": j["attempt"],
|
||||
"final_version": current["pipeline_version"],
|
||||
"published_objects": len(keys),
|
||||
"audit": audit(j),
|
||||
**details,
|
||||
}
|
||||
records.append(record)
|
||||
print("PASS: " + name, flush=True)
|
||||
|
||||
def envelope(j, attempt=None):
|
||||
return Job(
|
||||
schema_version=1,
|
||||
type="attachment.process.v1",
|
||||
job_id=str(j["job_id"]),
|
||||
attachment_id=str(j["attachment_id"]),
|
||||
pipeline_version=j["pipeline_version"],
|
||||
attempt=attempt or max(1, j["attempt"]),
|
||||
created_at=stamp(j["created_at"]),
|
||||
)
|
||||
|
||||
try:
|
||||
# Actual COMMIT/publish gap: no relay exists while the acknowledged upload is stored.
|
||||
compose("stop", "--timeout", "2", "worker", "relay")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
j = job(a)
|
||||
check(
|
||||
query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
|
||||
"published_at"
|
||||
]
|
||||
is None,
|
||||
"Gap was not established",
|
||||
)
|
||||
message = c.message(alice, room, "", attachments=[a["id"]])["json"]
|
||||
c.call("GET", "/api/v1/attachments/" + a["id"] + "/content", 409, actor=bob)
|
||||
before = instances()
|
||||
compose("restart", "--timeout", "3", "api")
|
||||
wait_lb(args.base_url)
|
||||
after = instances()
|
||||
check(
|
||||
{r["instance"] for r in before}.isdisjoint({r["instance"] for r in after}),
|
||||
"API did not restart",
|
||||
)
|
||||
compose("start", "relay")
|
||||
until(
|
||||
lambda: query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
|
||||
"published_at"
|
||||
]
|
||||
)
|
||||
entry_ids = [
|
||||
id
|
||||
for id, fields in broker.client.xrange(STREAM)
|
||||
if json.loads(fields["job"])["job_id"] == str(j["job_id"])
|
||||
]
|
||||
check(bool(entry_ids), "Broker has no accepted work")
|
||||
broker_id = compose("ps", "-q", "broker").strip()
|
||||
broker_before = inspect(broker_id)
|
||||
subprocess.run(
|
||||
["docker", "kill", "--signal", "KILL", broker_id], check=True, capture_output=True
|
||||
)
|
||||
subprocess.run(["docker", "start", broker_id], check=True, capture_output=True)
|
||||
|
||||
def broker_restored():
|
||||
try:
|
||||
return broker.client.ping()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
until(broker_restored)
|
||||
check(
|
||||
set(entry_ids) <= {id for id, _ in broker.client.xrange(STREAM)},
|
||||
"Durable stream lost confirmed work",
|
||||
)
|
||||
compose("start", "worker")
|
||||
ready(a)
|
||||
history = c.call(
|
||||
"GET", "/api/v1/chats/" + room["id"] + "/messages", actor=bob, schema="MessagePage"
|
||||
)["json"]
|
||||
check(message in history["items"], "Queued attachment message lost")
|
||||
saved(
|
||||
"commit_publish_gap_api_restart_broker_sigkill",
|
||||
a,
|
||||
start,
|
||||
api_before=before,
|
||||
api_after=after,
|
||||
broker_before=broker_before,
|
||||
broker_after=inspect(broker_id),
|
||||
broker_entry_ids=entry_ids,
|
||||
message_id=message["id"],
|
||||
)
|
||||
|
||||
# A publisher can lose its COMMIT after XADD; same logical ID is sent twice.
|
||||
compose("stop", "--timeout", "2", "worker", "relay")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
j = job(a)
|
||||
fault(j, "after_publish", "crash")
|
||||
compose("start", "relay")
|
||||
until(lambda: injected(j, "after_publish"))
|
||||
until(
|
||||
lambda: (
|
||||
sum(
|
||||
json.loads(f["job"])["job_id"] == str(j["job_id"])
|
||||
for _, f in broker.client.xrange(STREAM)
|
||||
)
|
||||
>= 2
|
||||
)
|
||||
)
|
||||
compose("start", "worker")
|
||||
ready(a)
|
||||
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
|
||||
saved("publish_before_outbox_commit_duplicate", a, start)
|
||||
|
||||
for phase in ("before_effect", "after_effect", "after_commit"):
|
||||
compose("stop", "--timeout", "2", "worker")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
j = job(a)
|
||||
fault(j, phase, "crash")
|
||||
compose("start", "worker")
|
||||
ready(a)
|
||||
until(lambda j=j, phase=phase: injected(j, phase))
|
||||
if phase == "after_commit":
|
||||
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
|
||||
check(job(a)["attempt"] == 1, "Committed job was processed twice")
|
||||
else:
|
||||
check(job(a)["attempt"] == 2, "Expired lease did not recover exactly once")
|
||||
saved("worker_crash_" + phase, a, start)
|
||||
|
||||
for action, count, expected in (
|
||||
("transient", 2, "ready"),
|
||||
("hang", 1, "ready"),
|
||||
("permanent", 1, "failed"),
|
||||
("transient", 5, "failed"),
|
||||
):
|
||||
compose("stop", "--timeout", "2", "worker")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
j = job(a)
|
||||
fault(j, "before_effect", action, count)
|
||||
compose("start", "worker")
|
||||
path = "/api/v1/attachments/" + a["id"]
|
||||
if expected == "failed":
|
||||
until(
|
||||
lambda path=path: (
|
||||
c.call("GET", path, actor=bob, schema="Attachment")["json"]["status"]
|
||||
== "failed"
|
||||
)
|
||||
)
|
||||
terminal = job(a)
|
||||
check(terminal["attempt"] == count, "Wrong retry budget")
|
||||
c.call("GET", path + "/content", 409, actor=bob)
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + room["id"] + "/messages",
|
||||
409,
|
||||
actor=alice,
|
||||
body={"text": "", "attachment_ids": [a["id"]]},
|
||||
headers={"Idempotency-Key": uuid4().hex},
|
||||
)
|
||||
# Explicit CLI replay, same logical job; no public administrator endpoint.
|
||||
replay = json.loads(
|
||||
compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"messenger.operator",
|
||||
"replay",
|
||||
a["id"],
|
||||
)
|
||||
)
|
||||
check(replay["job_id"] == str(j["job_id"]), "Replay changed logical job")
|
||||
else:
|
||||
terminal = None
|
||||
ready(a)
|
||||
events = audit(j)
|
||||
delays = [e["details"]["delay_seconds"] for e in events if e["event"] == "queued"]
|
||||
check(all(0.25 <= delay <= 30 for delay in delays), "Invalid bounded jitter")
|
||||
rejected = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"exec",
|
||||
"-T",
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"messenger.operator",
|
||||
"replay",
|
||||
a["id"],
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
)
|
||||
check(rejected.returncode != 0, "Ready replay was accepted")
|
||||
saved(
|
||||
action + "_" + str(count) + ("_failed_then_explicit_replay" if terminal else ""),
|
||||
a,
|
||||
start,
|
||||
terminal_before_replay={k: terminal[k] for k in ("state", "attempt", "error_code")}
|
||||
if terminal
|
||||
else None,
|
||||
backoff_delays=delays,
|
||||
)
|
||||
|
||||
# Reorder and stale v1 delivered after v2; exercise the DB publication fence directly too.
|
||||
compose("stop", "--timeout", "2", "worker", "relay")
|
||||
start = time.monotonic()
|
||||
a = upload()
|
||||
old = job(a)
|
||||
old_envelope = envelope(old, 1)
|
||||
outcome, old_claim = Pipeline(db, config).claim(old_envelope)
|
||||
check(outcome == "claimed", "Cannot establish old writer")
|
||||
Pipeline(db, config).replay(a["id"], new_version=True)
|
||||
new = job(a, 2)
|
||||
# Publish v2 first, then v1, plus duplicates through both worker slots.
|
||||
for value in (envelope(new, 1), old_envelope, envelope(new, 1), old_envelope):
|
||||
broker.publish(value)
|
||||
compose("start", "worker", "relay")
|
||||
current = ready(a)
|
||||
row = db.attachment(a["id"])
|
||||
check(
|
||||
not Pipeline(db, config).publish(old_claim, "objects/old-writer", []),
|
||||
"Old writer overwrote v2",
|
||||
)
|
||||
for _ in range(20):
|
||||
broker.publish(old_envelope)
|
||||
broker.publish(envelope(new, 1))
|
||||
until(lambda: sum(e["event"] == "stale_delivery" for e in audit(old)) >= 21)
|
||||
until(lambda: sum(e["event"] == "duplicate_ready" for e in audit(new)) >= 20)
|
||||
check(
|
||||
db.attachment(a["id"])["object_key"] == row["object_key"],
|
||||
"Stale delivery changed current pointer",
|
||||
)
|
||||
check(
|
||||
c.call("GET", "/api/v1/attachments/" + a["id"], actor=bob, schema="Attachment")["json"]
|
||||
== current,
|
||||
"Old version mutated ready metadata",
|
||||
)
|
||||
saved(
|
||||
"reorder_stale_v1_after_v2_and_publication_fence",
|
||||
a,
|
||||
start,
|
||||
version=2,
|
||||
old_audit=audit(old),
|
||||
duplicates=44,
|
||||
)
|
||||
|
||||
# ACL remains enforced through both API replicas; S3 is private and role-scoped.
|
||||
a = {"id": accepted[0]}
|
||||
for suffix in ("", "/content", "/content?variant=crop"):
|
||||
c.call("GET", "/api/v1/attachments/" + a["id"] + suffix, 404, actor=eve)
|
||||
# Direct bucket access and write/delete privilege checks use no public credentials.
|
||||
import boto3
|
||||
from botocore import UNSIGNED
|
||||
from botocore.config import Config
|
||||
|
||||
first = db.attachment(accepted[0])
|
||||
unsigned = boto3.client(
|
||||
"s3", endpoint_url=config.s3_endpoint, config=Config(signature_version=UNSIGNED)
|
||||
)
|
||||
try:
|
||||
unsigned.get_object(Bucket=store.bucket, Key=first["object_key"])
|
||||
raise AssertionError("Anonymous S3 read succeeded")
|
||||
except ClientError as exc:
|
||||
check(
|
||||
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Wrong anonymous status"
|
||||
)
|
||||
finally:
|
||||
unsigned.close()
|
||||
denied = []
|
||||
for role, action in (
|
||||
("api", "put_object"),
|
||||
("api", "delete_object"),
|
||||
("worker", "delete_object"),
|
||||
("reconciler", "put_object"),
|
||||
):
|
||||
scoped = ObjectStore(settings(role))
|
||||
try:
|
||||
kw = {"Bucket": store.bucket, "Key": "objects/" + uuid4().hex + "/forbidden"}
|
||||
if action == "put_object":
|
||||
kw["Body"] = b"forbidden"
|
||||
try:
|
||||
getattr(scoped.client, action)(**kw)
|
||||
raise AssertionError("IAM action unexpectedly allowed")
|
||||
except ClientError as exc:
|
||||
check(
|
||||
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403,
|
||||
"Wrong IAM status",
|
||||
)
|
||||
denied.append(role + ":" + action)
|
||||
finally:
|
||||
scoped.close()
|
||||
# Stop the actual uploader, then read original and thumbnail through another replica.
|
||||
response = c.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + room["id"] + "/attachments?filename=cross.png",
|
||||
202,
|
||||
actor=alice,
|
||||
raw=data,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
cross = response["json"]
|
||||
accepted.append(cross["id"])
|
||||
ready(cross)
|
||||
uploader = response["headers"]["X-Instance-Id"]
|
||||
process = next(v for v in instances() if v["instance"] == uploader)
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "3", process["id"]], check=True, capture_output=True
|
||||
)
|
||||
try:
|
||||
wait_lb(args.base_url, count=1)
|
||||
download = c.call("GET", "/api/v1/attachments/" + cross["id"] + "/content", actor=bob)
|
||||
check(
|
||||
download["bytes"] == data and download["headers"]["X-Instance-Id"] != uploader,
|
||||
"Other replica could not serve the original",
|
||||
)
|
||||
c.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + cross["id"] + "/content?variant=thumbnail",
|
||||
actor=bob,
|
||||
)
|
||||
cross_proof = {
|
||||
"uploader": uploader,
|
||||
"stopped": inspect(process["id"]),
|
||||
"reader": download["headers"]["X-Instance-Id"],
|
||||
}
|
||||
finally:
|
||||
subprocess.run(["docker", "start", process["id"]], check=True, capture_output=True)
|
||||
wait_lb(args.base_url)
|
||||
logs = compose("logs", "--no-color", "api", "worker", "relay", "lb")
|
||||
check(
|
||||
not any(
|
||||
actor["token"] in logs or actor["registration"]["password"] in logs
|
||||
for actor in (alice, bob, eve)
|
||||
),
|
||||
"Credential in logs",
|
||||
)
|
||||
states = query(
|
||||
"SELECT id,storage_state FROM attachments WHERE id=ANY(%s::uuid[])", (accepted,)
|
||||
)
|
||||
check(
|
||||
len(states) == len(accepted) and all(s["storage_state"] == "ready" for s in states),
|
||||
"Confirmed work lost",
|
||||
)
|
||||
stats = compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"worker",
|
||||
"python",
|
||||
"-c",
|
||||
"import json;from pathlib import Path;print(json.dumps({"
|
||||
"n:Path('/sys/fs/cgroup/'+n).read_text().strip() "
|
||||
"for n in ('memory.current','memory.peak','memory.events','cpu.stat')}))",
|
||||
)
|
||||
result = {
|
||||
"anonymous_s3_status": 403,
|
||||
"denied_iam_actions": denied,
|
||||
"cross_replica": cross_proof,
|
||||
"track": 2,
|
||||
"delivery": "at-least-once",
|
||||
"effect_key": ["attachment_id", "pipeline_version", "variant"],
|
||||
"worker_concurrency": config.worker_concurrency,
|
||||
"prefetch_per_slot": 1,
|
||||
"job_timeout_seconds": config.job_timeout,
|
||||
"lease_seconds": config.lease_seconds,
|
||||
"max_attempts": config.job_max_attempts,
|
||||
"scenarios": records,
|
||||
"accepted": len(accepted),
|
||||
"lost_confirmed": 0,
|
||||
"stale_overwrites": 0,
|
||||
"worker_cgroup": json.loads(stats),
|
||||
"logs_credentials_absent": True,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2, default=str) + "\n")
|
||||
print("PASS: all acknowledged work ready; no stale overwrites", flush=True)
|
||||
finally:
|
||||
compose("start", "worker", "relay", "broker", "api")
|
||||
broker.close()
|
||||
store.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user