Добавить автоматический эксперимент и изоляцию повторов
This commit is contained in:
@@ -0,0 +1,693 @@
|
||||
"""Controlled delivery/fault cases. Called only by an explicit experiment run."""
|
||||
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
from support import ROOT, digest, dump
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure
|
||||
from image_probe import jpeg_size
|
||||
|
||||
|
||||
class Violation(AssertionError):
|
||||
pass
|
||||
|
||||
|
||||
class HarnessError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise Violation(message)
|
||||
|
||||
|
||||
class Stack:
|
||||
def __init__(self, project, config, private, protocol):
|
||||
self.project, self.config, self.private, self.protocol = (
|
||||
project,
|
||||
config,
|
||||
private,
|
||||
protocol,
|
||||
)
|
||||
self.prefix = [
|
||||
"docker",
|
||||
"compose",
|
||||
"--project-name",
|
||||
project,
|
||||
"-f",
|
||||
str(config),
|
||||
]
|
||||
|
||||
def call(self, *args):
|
||||
result = subprocess.run(
|
||||
self.prefix + list(args),
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=60,
|
||||
)
|
||||
if result.returncode:
|
||||
error = result.stderr
|
||||
for path in (self.private / "secrets").iterdir():
|
||||
value = path.read_text().strip()
|
||||
if len(value) >= 12:
|
||||
error = error.replace(value, "[REDACTED]")
|
||||
with (self.private.parent.parent / "command-errors.jsonl").open(
|
||||
"a"
|
||||
) as output:
|
||||
output.write(
|
||||
json.dumps(
|
||||
{
|
||||
"project": self.project,
|
||||
"args": args,
|
||||
"exit_code": result.returncode,
|
||||
"stderr": error,
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
raise HarnessError("Compose command failed; see command-errors.jsonl")
|
||||
return result.stdout.strip()
|
||||
|
||||
def wait_broker_ready(self, timeout=60):
|
||||
"""Wait only during setup; runtime queue failures remain fatal."""
|
||||
deadline = time.monotonic() + timeout
|
||||
args = ["exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues",
|
||||
"name", "messages_ready", "messages_unacknowledged",
|
||||
"--formatter", "json", "--timeout", "5"]
|
||||
attempts = 0
|
||||
while time.monotonic() < deadline:
|
||||
attempts += 1
|
||||
try:
|
||||
result = subprocess.run(
|
||||
self.prefix + args, cwd=ROOT, text=True, capture_output=True,
|
||||
timeout=min(15, max(0.1, deadline-time.monotonic())),
|
||||
)
|
||||
if result.returncode == 0:
|
||||
rows = json.loads(result.stdout)
|
||||
if not isinstance(rows, list):
|
||||
raise HarnessError("Unexpected RabbitMQ queue response")
|
||||
dump(self.private.parent.parent / ("broker-readiness-"+self.private.name+".json"), {
|
||||
"project": self.project, "attempts": attempts,
|
||||
"ready": True,
|
||||
})
|
||||
return
|
||||
error = result.stderr
|
||||
# Auth, command syntax and stopped-container errors are not startup races.
|
||||
if "requires the 'rabbit' app to be running" not in error:
|
||||
raise HarnessError("RabbitMQ readiness probe failed: " + error.strip())
|
||||
except subprocess.TimeoutExpired:
|
||||
error = "RabbitMQ startup probe timed out"
|
||||
with (self.private.parent.parent / "broker-startup.jsonl").open("a") as output:
|
||||
output.write(json.dumps({"project": self.project, "attempt": attempts,
|
||||
"stderr": error}) + "\n")
|
||||
time.sleep(min(1, max(0, deadline-time.monotonic())))
|
||||
raise HarnessError("RabbitMQ did not finish startup before the setup deadline")
|
||||
|
||||
def sql(self, query):
|
||||
return self.call(
|
||||
"exec",
|
||||
"-T",
|
||||
"db",
|
||||
"sh",
|
||||
"-c",
|
||||
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"',
|
||||
"sh",
|
||||
query,
|
||||
)
|
||||
|
||||
def workers(self):
|
||||
return self.call("ps", "-aq", "worker").splitlines()
|
||||
|
||||
def worker_action(self, action, ids):
|
||||
for ident in ids:
|
||||
owner = subprocess.check_output(
|
||||
[
|
||||
"docker",
|
||||
"inspect",
|
||||
"--format",
|
||||
'{{index .Config.Labels "com.docker.compose.project"}}',
|
||||
ident,
|
||||
],
|
||||
text=True,
|
||||
).strip()
|
||||
if owner != self.project:
|
||||
raise HarnessError("Container belongs to another project")
|
||||
if ids:
|
||||
subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
action,
|
||||
*(["--time", "5"] if action == "stop" else []),
|
||||
*ids,
|
||||
],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
def queue_depth(self):
|
||||
rows = json.loads(
|
||||
self.call(
|
||||
"exec",
|
||||
"-T",
|
||||
"broker",
|
||||
"rabbitmqctl",
|
||||
"-q",
|
||||
"list_queues",
|
||||
"name",
|
||||
"messages_ready",
|
||||
"messages_unacknowledged",
|
||||
"--formatter",
|
||||
"json",
|
||||
)
|
||||
)
|
||||
return sum(
|
||||
row["messages_ready"] + row["messages_unacknowledged"]
|
||||
for row in rows
|
||||
if row["name"] == "attachments.v1"
|
||||
)
|
||||
|
||||
def operator(self, *args):
|
||||
running = self.call("ps", "--status", "running", "-q", "dispatcher")
|
||||
mode = (
|
||||
["exec", "-T", "dispatcher"]
|
||||
if running
|
||||
else ["run", "--rm", "--no-deps", "-T", "dispatcher"]
|
||||
)
|
||||
return self.call(*mode, "./messenger", *args)
|
||||
|
||||
def admin(self, *args):
|
||||
return json.loads(self.operator("jobs", *args))
|
||||
|
||||
def snapshot(self, attachment):
|
||||
result = self.sql(
|
||||
"SELECT json_build_object('desired',a.pipeline_version,'published',a.published_pipeline_version,'status',a.status,'jobs',(SELECT json_agg(json_build_object('job_id',j.id,'version',j.pipeline_version,'attempts',j.attempts,'superseded',j.superseded_at,'lease_until',j.lease_until) ORDER BY j.pipeline_version) FROM attachment_jobs j WHERE j.attachment_id=a.id),'attempts_audit',(SELECT COALESCE(json_agg(json_build_object('job_id',t.job_id,'generation',t.generation,'attempt',t.attempt,'started_at',t.started_at,'finished_at',t.finished_at,'error_code',t.error_code,'next_attempt_at',t.next_attempt_at) ORDER BY t.started_at),'[]') FROM attachment_job_attempts t JOIN attachment_jobs x ON x.id=t.job_id WHERE x.attachment_id=a.id),'publications',(SELECT COALESCE(json_agg(json_build_object('id',p.id,'version',p.pipeline_version,'variants',p.variants,'published_at',p.published_at) ORDER BY p.id),'[]') FROM attachment_publications p WHERE p.attachment_id=a.id)) FROM attachments a WHERE a.id='"
|
||||
+ attachment
|
||||
+ "'"
|
||||
)
|
||||
return json.loads(result)
|
||||
|
||||
def messages(self, attachment):
|
||||
return json.loads(
|
||||
self.sql(
|
||||
"SELECT json_agg(json_build_object('schema_version',1,'job_id',id,'type','attachment.process.v1','attachment_id',attachment_id,'pipeline_version',pipeline_version,'attempt',1,'created_at',to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')) ORDER BY pipeline_version) FROM attachment_jobs WHERE attachment_id='"
|
||||
+ attachment
|
||||
+ "'"
|
||||
)
|
||||
)
|
||||
|
||||
def publish(self, job):
|
||||
self.operator("broker-publish", json.dumps(job))
|
||||
|
||||
def containers(self):
|
||||
ids = self.call("ps", "-aq").splitlines()
|
||||
if not ids:
|
||||
return []
|
||||
# Never save Config.Env: it contains the experiment DB password.
|
||||
rows = json.loads(
|
||||
subprocess.check_output(["docker", "inspect", *ids], text=True)
|
||||
)
|
||||
return [
|
||||
{
|
||||
"id": row["Id"],
|
||||
"image": row["Image"],
|
||||
"service": row["Config"]["Labels"].get("com.docker.compose.service"),
|
||||
"state": {
|
||||
k: row["State"].get(k)
|
||||
for k in (
|
||||
"Status",
|
||||
"ExitCode",
|
||||
"OOMKilled",
|
||||
"StartedAt",
|
||||
"FinishedAt",
|
||||
)
|
||||
},
|
||||
"restarts": row["RestartCount"],
|
||||
"memory": row["HostConfig"]["Memory"],
|
||||
"nano_cpus": row["HostConfig"]["NanoCpus"],
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def s3(self, method, key="", query=None):
|
||||
secret_dir = self.private / "secrets"
|
||||
access = (secret_dir / "s3_api_access_key_id").read_text().strip()
|
||||
secret = (secret_dir / "s3_api_secret_access_key").read_text().strip()
|
||||
host = "127.0.0.1:" + str(self.protocol["ports"]["s3"])
|
||||
uri = "/messenger-attachments/" + key
|
||||
query_string = urllib.parse.urlencode(
|
||||
sorted((query or {}).items()), quote_via=urllib.parse.quote
|
||||
)
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
timestamp, day = now.strftime("%Y%m%dT%H%M%SZ"), now.strftime("%Y%m%d")
|
||||
payload = digest(b"")
|
||||
signed = "host;x-amz-content-sha256;x-amz-date"
|
||||
headers = (
|
||||
"host:"
|
||||
+ host
|
||||
+ "\nx-amz-content-sha256:"
|
||||
+ payload
|
||||
+ "\nx-amz-date:"
|
||||
+ timestamp
|
||||
+ "\n"
|
||||
)
|
||||
canonical = "\n".join([method, uri, query_string, headers, signed, payload])
|
||||
scope = day + "/us-east-1/s3/aws4_request"
|
||||
text = (
|
||||
"AWS4-HMAC-SHA256\n"
|
||||
+ timestamp
|
||||
+ "\n"
|
||||
+ scope
|
||||
+ "\n"
|
||||
+ digest(canonical.encode())
|
||||
)
|
||||
signing = ("AWS4" + secret).encode()
|
||||
for part in (day, "us-east-1", "s3", "aws4_request"):
|
||||
signing = hmac.new(signing, part.encode(), hashlib.sha256).digest()
|
||||
signature = hmac.new(signing, text.encode(), hashlib.sha256).hexdigest()
|
||||
request = urllib.request.Request(
|
||||
"http://" + host + uri + ("?" + query_string if query_string else ""),
|
||||
method=method,
|
||||
headers={
|
||||
"x-amz-date": timestamp,
|
||||
"x-amz-content-sha256": payload,
|
||||
"Authorization": "AWS4-HMAC-SHA256 Credential="
|
||||
+ access
|
||||
+ "/"
|
||||
+ scope
|
||||
+ ", SignedHeaders="
|
||||
+ signed
|
||||
+ ", Signature="
|
||||
+ signature,
|
||||
},
|
||||
)
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=15)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response = exc
|
||||
with response:
|
||||
code, body = response.code, response.read()
|
||||
if code in (401, 403):
|
||||
raise HarnessError("S3 measurement identity could not authenticate")
|
||||
return code, body
|
||||
|
||||
|
||||
def webp_size(data):
|
||||
require(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "invalid WebP container")
|
||||
offset = 12
|
||||
dimensions = None
|
||||
while offset + 8 <= len(data):
|
||||
kind = data[offset : offset + 4]
|
||||
length = int.from_bytes(data[offset + 4 : offset + 8], "little")
|
||||
body = data[offset + 8 : offset + 8 + length]
|
||||
require(kind not in (b"EXIF", b"XMP ", b"ICCP"), "WebP metadata leaked")
|
||||
if kind == b"VP8 ":
|
||||
dimensions = (
|
||||
int.from_bytes(body[6:8], "little") & 16383,
|
||||
int.from_bytes(body[8:10], "little") & 16383,
|
||||
)
|
||||
if kind == b"VP8L":
|
||||
bits = int.from_bytes(body[1:5], "little")
|
||||
dimensions = ((bits & 16383) + 1, ((bits >> 14) & 16383) + 1)
|
||||
if kind == b"VP8X":
|
||||
dimensions = (
|
||||
int.from_bytes(body[4:7], "little") + 1,
|
||||
int.from_bytes(body[7:10], "little") + 1,
|
||||
)
|
||||
offset += 8 + length + (length % 2)
|
||||
require(dimensions is not None, "WebP dimensions absent")
|
||||
return dimensions
|
||||
|
||||
|
||||
class Case:
|
||||
def __init__(self, stack, profile, scenario, data, directory):
|
||||
self.s, self.profile, self.scenario, self.data, self.directory = (
|
||||
stack,
|
||||
profile,
|
||||
scenario,
|
||||
data,
|
||||
directory,
|
||||
)
|
||||
directory.mkdir(parents=True)
|
||||
self.events = (directory / "events.jsonl").open("x")
|
||||
self.start = time.monotonic()
|
||||
self.last = None
|
||||
self.gaps = []
|
||||
self.attachment = None
|
||||
self.verifications = 0
|
||||
self.c = Client("http://127.0.0.1:" + str(stack.protocol["ports"]["api"]))
|
||||
|
||||
def event(self, kind, **values):
|
||||
now = time.monotonic()
|
||||
record = {
|
||||
"utc": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"elapsed": now - self.start,
|
||||
"kind": kind,
|
||||
**values,
|
||||
}
|
||||
self.events.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
self.events.flush()
|
||||
|
||||
def wait(self, fn, timeout=None, invariant=True):
|
||||
deadline = time.monotonic() + (
|
||||
timeout or self.s.protocol["case_timeout_seconds"]
|
||||
)
|
||||
while time.monotonic() < deadline:
|
||||
value = fn()
|
||||
if value and time.monotonic() <= deadline:
|
||||
return value
|
||||
time.sleep(self.s.protocol["poll_interval_seconds"])
|
||||
if invariant:
|
||||
raise Violation("completion/quiescence deadline exceeded")
|
||||
raise HarnessError("required fault/precondition did not occur before deadline")
|
||||
|
||||
def observe(self):
|
||||
now = time.monotonic()
|
||||
if self.last is not None and now - self.last > 2:
|
||||
self.gaps.append(now - self.last)
|
||||
self.last = now
|
||||
value = self.c.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + self.attachment,
|
||||
actor=self.alice,
|
||||
schema="Attachment",
|
||||
)["json"]
|
||||
snapshot = self.s.snapshot(self.attachment)
|
||||
self.event("observation", api=value, database=snapshot)
|
||||
return value, snapshot
|
||||
|
||||
def ready(self, version):
|
||||
def check():
|
||||
value, snapshot = self.observe()
|
||||
require(value["status"] != "failed", "valid fixture reached failed")
|
||||
return (
|
||||
(value, snapshot)
|
||||
if value["status"] == "ready" and snapshot["published"] == version
|
||||
else None
|
||||
)
|
||||
|
||||
return self.wait(check)
|
||||
|
||||
def arm(self, point):
|
||||
marker = self.s.private / "faults" / (point + "." + self.attachment)
|
||||
marker.write_text("experiment " + self.scenario + "\n")
|
||||
self.event("arm", point=point)
|
||||
return Path(str(marker) + ".hit")
|
||||
|
||||
def upload(self):
|
||||
self.alice, self.bob, self.eve, chat = self.c.bootstrap()
|
||||
value = self.c.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/attachments?filename=fixture.png",
|
||||
202,
|
||||
actor=self.alice,
|
||||
raw=self.data,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)["json"]
|
||||
self.attachment = value["id"]
|
||||
require(
|
||||
value["sha256"] == digest(self.data), "accepted original checksum mismatch"
|
||||
)
|
||||
self.event("accepted_202", attachment_id=self.attachment)
|
||||
require(
|
||||
len(self.s.messages(self.attachment)) == 1,
|
||||
"accepted upload has no unique durable job",
|
||||
)
|
||||
|
||||
def upgrade(self):
|
||||
require(
|
||||
self.s.admin(
|
||||
"upgrade", self.attachment, "2", "Protocol v1 " + self.scenario
|
||||
)["changed"],
|
||||
"upgrade intent was not persisted",
|
||||
)
|
||||
self.event("upgrade_accepted", version=2)
|
||||
|
||||
def publish(self, job, count):
|
||||
for _ in range(count):
|
||||
self.s.publish(job)
|
||||
self.event(
|
||||
"explicit_delivery",
|
||||
job_id=job["job_id"],
|
||||
version=job["pipeline_version"],
|
||||
count=count,
|
||||
)
|
||||
|
||||
def verify(self, version):
|
||||
value, snapshot = self.ready(version)
|
||||
self.verifications += 1
|
||||
observation = "verify-" + str(self.verifications).zfill(3) + "-v" + str(version)
|
||||
objects = self.directory / "objects" / observation
|
||||
objects.mkdir(parents=True)
|
||||
require(
|
||||
snapshot["desired"] == version, "desired version differs from publication"
|
||||
)
|
||||
original = self.c.call(
|
||||
"GET", "/api/v1/attachments/" + self.attachment + "/content", actor=self.bob
|
||||
)["bytes"]
|
||||
require(original == self.data, "original changed")
|
||||
require(
|
||||
len(value["variants"]) == 3
|
||||
and {v["name"] for v in value["variants"]}
|
||||
== {"thumbnail", "crop", "optimized"},
|
||||
"incomplete or duplicate variant set",
|
||||
)
|
||||
manifest = {}
|
||||
for item in value["variants"]:
|
||||
name = item["name"]
|
||||
key = (
|
||||
"variants/"
|
||||
+ self.attachment
|
||||
+ "/v"
|
||||
+ str(version)
|
||||
+ "/"
|
||||
+ name
|
||||
+ (".webp" if name == "optimized" else ".jpg")
|
||||
)
|
||||
response = self.c.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + self.attachment + "/content?variant=" + name,
|
||||
actor=self.bob,
|
||||
)
|
||||
require(
|
||||
digest(response["bytes"]) == item["sha256"]
|
||||
and len(response["bytes"]) == item["size_bytes"],
|
||||
"variant checksum mismatch",
|
||||
)
|
||||
require(
|
||||
response["headers"].get_content_type() == item["content_type"],
|
||||
"variant MIME mismatch",
|
||||
)
|
||||
try:
|
||||
dimensions = (
|
||||
webp_size(response["bytes"])
|
||||
if name == "optimized"
|
||||
else jpeg_size(response["bytes"])
|
||||
)
|
||||
except Failure as exc:
|
||||
raise Violation("actual variant decode failed") from exc
|
||||
w, h = self.profile["width"], self.profile["height"]
|
||||
limit = 1280 if name == "optimized" else 256
|
||||
expected = (
|
||||
(128, 128)
|
||||
if name == "crop"
|
||||
else (min(w, limit), max(1, h * limit // w))
|
||||
if w >= h and w > limit
|
||||
else (max(1, w * limit // h), limit)
|
||||
if h > w and h > limit
|
||||
else (w, h)
|
||||
)
|
||||
require(
|
||||
dimensions == expected == (item["width"], item["height"]),
|
||||
"actual variant dimensions mismatch",
|
||||
)
|
||||
code, body = self.s.s3("GET", key)
|
||||
require(
|
||||
code == 200 and body == response["bytes"],
|
||||
"API exposes a different version than S3",
|
||||
)
|
||||
(
|
||||
objects / (name + (".webp" if name == "optimized" else ".jpg"))
|
||||
).write_bytes(body)
|
||||
manifest[key] = digest(body)
|
||||
code, body = self.s.s3(
|
||||
"GET",
|
||||
query={
|
||||
"list-type": "2",
|
||||
"prefix": "variants/" + self.attachment + "/v" + str(version) + "/",
|
||||
},
|
||||
)
|
||||
require(code == 200, "cannot list variant set")
|
||||
require(
|
||||
len(ET.fromstring(body).findall("{*}Contents")) == 3,
|
||||
"extra/missing object set",
|
||||
)
|
||||
history = snapshot["publications"]
|
||||
versions = [p["version"] for p in history]
|
||||
require(
|
||||
versions == sorted(versions) and len(versions) == len(set(versions)),
|
||||
"publication duplicated or rolled back",
|
||||
)
|
||||
require(versions[-1] == version, "publication history missing active version")
|
||||
require(
|
||||
all(
|
||||
j["version"] == version or j["superseded"] is not None
|
||||
for j in snapshot["jobs"]
|
||||
),
|
||||
"older accepted job has no durable disposition",
|
||||
)
|
||||
dump(self.directory / (observation + "-manifest.json"), manifest)
|
||||
return manifest, snapshot
|
||||
|
||||
def execute(self):
|
||||
self.s.worker_action("stop", self.s.workers())
|
||||
self.s.call("stop", "dispatcher")
|
||||
self.upload()
|
||||
workers = self.s.workers()
|
||||
require(len(workers) == 2, "two workers are required by protocol")
|
||||
scenario = self.scenario
|
||||
version = 1
|
||||
first = None
|
||||
if scenario == "commit_publish_gap":
|
||||
marker = self.arm("after_commit_before_publish")
|
||||
if scenario != "reorder":
|
||||
self.s.call("start", "dispatcher")
|
||||
if scenario == "commit_publish_gap":
|
||||
self.wait(marker.exists, invariant=False)
|
||||
if scenario == "duplicates":
|
||||
self.publish(
|
||||
self.s.messages(self.attachment)[0],
|
||||
self.s.protocol["duplicates_before_ready"],
|
||||
)
|
||||
if scenario.startswith("crash_"):
|
||||
point = {
|
||||
"crash_before_effect": "before_effect",
|
||||
"crash_after_effect": "after_object_before_commit",
|
||||
"crash_before_ack": "after_commit_before_ack",
|
||||
}[scenario]
|
||||
marker = self.arm(point)
|
||||
self.s.worker_action("start", [workers[0]])
|
||||
self.wait(marker.exists, invariant=False)
|
||||
self.event("fault_hit", point=point)
|
||||
elif scenario == "reorder":
|
||||
# No consumer is running; broker order is deliberately v2 then old v1.
|
||||
self.upgrade()
|
||||
jobs = self.s.messages(self.attachment)
|
||||
self.publish(jobs[1], 1)
|
||||
self.publish(jobs[0], 1)
|
||||
version = 2
|
||||
self.s.worker_action("start", workers)
|
||||
elif scenario == "stale_lease":
|
||||
marker = self.arm("after_claim")
|
||||
self.s.worker_action("start", [workers[0]])
|
||||
self.wait(marker.exists, invariant=False)
|
||||
self.s.call("start", "dispatcher")
|
||||
self.upgrade()
|
||||
self.s.worker_action("start", [workers[1]])
|
||||
first, _ = self.verify(2)
|
||||
version = 2
|
||||
release = (
|
||||
self.s.private
|
||||
/ "faults"
|
||||
/ ("after_claim." + self.attachment + ".release")
|
||||
)
|
||||
release.write_text("release\n")
|
||||
self.event("release_old_lease")
|
||||
self.wait(
|
||||
Path(str(release) + ".observed").exists, timeout=5, invariant=False
|
||||
)
|
||||
self.wait(
|
||||
(
|
||||
self.s.private
|
||||
/ "faults"
|
||||
/ ("stale_rejected." + self.attachment + ".observed")
|
||||
).exists,
|
||||
timeout=5,
|
||||
invariant=False,
|
||||
)
|
||||
else:
|
||||
self.s.worker_action("start", workers)
|
||||
if scenario == "old_after_v2":
|
||||
old_manifest, _ = self.verify(1)
|
||||
old = self.s.messages(self.attachment)[0]
|
||||
self.upgrade()
|
||||
version = 2
|
||||
first, _ = self.verify(2)
|
||||
if not any(
|
||||
old_manifest.get(key.replace("/v2/", "/v1/")) != value
|
||||
for key, value in first.items()
|
||||
):
|
||||
raise HarnessError(
|
||||
"Version fingerprints are indistinguishable for this fixture"
|
||||
)
|
||||
self.publish(old, self.s.protocol["duplicates_after_ready"])
|
||||
for key, checksum in old_manifest.items():
|
||||
code, body = self.s.s3("GET", key)
|
||||
require(
|
||||
code == 200 and digest(body) == checksum,
|
||||
"v1 immutable object changed",
|
||||
)
|
||||
if scenario == "duplicates":
|
||||
first, before = self.verify(1)
|
||||
self.publish(
|
||||
self.s.messages(self.attachment)[0],
|
||||
self.s.protocol["duplicates_after_ready"],
|
||||
)
|
||||
if scenario != "reorder":
|
||||
self.s.call("start", "dispatcher")
|
||||
self.s.worker_action("start", workers)
|
||||
self.wait(lambda: self.s.queue_depth() == 0)
|
||||
# Let an already-delivered stale attempt leave its pause and hit the fence.
|
||||
if scenario == "stale_lease":
|
||||
time.sleep(1)
|
||||
final, snapshot = self.verify(version)
|
||||
if first is not None:
|
||||
require(first == final, "old delivery changed published bytes")
|
||||
if scenario in ("duplicates", "old_after_v2"):
|
||||
require(
|
||||
all(j["attempts"] == 1 for j in snapshot["jobs"]),
|
||||
"terminal duplicate started another attempt",
|
||||
)
|
||||
if scenario == "reorder":
|
||||
self.s.call("start", "dispatcher")
|
||||
self.event("case_complete", version=version)
|
||||
return {
|
||||
"attachment_id": self.attachment,
|
||||
"version": version,
|
||||
"snapshot": snapshot,
|
||||
"manifest": final,
|
||||
"observation_gaps": self.gaps,
|
||||
"elapsed_seconds": time.monotonic() - self.start,
|
||||
}
|
||||
|
||||
def recover(self):
|
||||
for marker in (self.s.private / "faults").glob("after_claim.*.hit"):
|
||||
Path(str(marker)[:-4] + ".release").write_text("cleanup release\n")
|
||||
self.s.call("start", "dispatcher")
|
||||
self.s.worker_action("start", self.s.workers())
|
||||
self.wait(
|
||||
lambda: (
|
||||
self.s.sql(
|
||||
"SELECT count(*) FROM attachments WHERE status IN ('queued','processing')"
|
||||
)
|
||||
== "0"
|
||||
and self.s.queue_depth() == 0
|
||||
)
|
||||
)
|
||||
|
||||
def close(self):
|
||||
self.events.close()
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Whitelisted Docker lifecycle events; never serialize environment or exec args."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
|
||||
|
||||
class LifecycleEvents:
|
||||
def __init__(self, project, path):
|
||||
self.closing = False
|
||||
self.oom = 0
|
||||
self.failed = False
|
||||
self.path = path
|
||||
self.process = subprocess.Popen(
|
||||
[
|
||||
"docker",
|
||||
"events",
|
||||
"--filter",
|
||||
"label=com.docker.compose.project=" + project,
|
||||
"--format",
|
||||
"{{json .}}",
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
)
|
||||
self.thread = threading.Thread(target=self.read, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def read(self):
|
||||
try:
|
||||
with self.path.open("x") as output:
|
||||
for line in self.process.stdout:
|
||||
try:
|
||||
event = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
if self.closing:
|
||||
break
|
||||
raise
|
||||
action = event.get("Action", "")
|
||||
if event.get("Type") != "container" or action not in (
|
||||
"create",
|
||||
"start",
|
||||
"stop",
|
||||
"kill",
|
||||
"die",
|
||||
"restart",
|
||||
"destroy",
|
||||
"oom",
|
||||
"pause",
|
||||
"unpause",
|
||||
):
|
||||
continue
|
||||
actor = event.get("Actor", {})
|
||||
attributes = actor.get("Attributes", {})
|
||||
value = {
|
||||
"time_nano": event.get("timeNano"),
|
||||
"action": action,
|
||||
"container_id": actor.get("ID"),
|
||||
"service": attributes.get("com.docker.compose.service"),
|
||||
"exit_code": attributes.get("exitCode"),
|
||||
"signal": attributes.get("signal"),
|
||||
}
|
||||
if action == "oom":
|
||||
self.oom += 1
|
||||
output.write(json.dumps(value) + "\n")
|
||||
output.flush()
|
||||
except Exception:
|
||||
self.failed = True
|
||||
finally:
|
||||
if not self.closing:
|
||||
self.failed = True
|
||||
|
||||
def close(self):
|
||||
self.closing = True
|
||||
if self.process.poll() is None:
|
||||
self.process.terminate()
|
||||
try:
|
||||
self.process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.process.kill()
|
||||
self.process.wait()
|
||||
self.thread.join(timeout=5)
|
||||
if self.thread.is_alive():
|
||||
self.failed = True
|
||||
@@ -0,0 +1,293 @@
|
||||
"""Deterministic inputs, source sealing and private isolated Compose configuration."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import struct
|
||||
import subprocess
|
||||
import zipfile
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
PROTOCOL = ROOT / "experiments/config/protocol.json"
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def ensure_local_ignore():
|
||||
path = Path(
|
||||
subprocess.check_output(
|
||||
["git", "rev-parse", "--git-path", "info/exclude"], cwd=ROOT, text=True
|
||||
).strip()
|
||||
)
|
||||
if not path.is_absolute():
|
||||
path = ROOT / path
|
||||
text = path.read_text() if path.exists() else ""
|
||||
# Migrate the old blanket exclusion; public results belong in Git.
|
||||
lines = [line for line in text.splitlines() if line.strip() != "evidence/idempotency/"]
|
||||
for pattern in ("experiments/results/", "evidence/idempotency/**/.private/", "evidence/idempotency/.run.lock"):
|
||||
if pattern not in lines:
|
||||
lines.append(pattern)
|
||||
updated = "\n".join(lines).rstrip() + "\n"
|
||||
if updated != text:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(updated)
|
||||
|
||||
|
||||
def dump(path, value):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def source_files():
|
||||
names = (
|
||||
subprocess.check_output(
|
||||
["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"],
|
||||
cwd=ROOT,
|
||||
)
|
||||
.decode()
|
||||
.split("\0")
|
||||
)
|
||||
return sorted(
|
||||
{
|
||||
name
|
||||
for name in names
|
||||
if name
|
||||
and not name.startswith(
|
||||
(
|
||||
"experiments/results/",
|
||||
"evidence/",
|
||||
".memory/",
|
||||
"secrets/",
|
||||
".git/",
|
||||
)
|
||||
)
|
||||
and (ROOT / name).is_file()
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def source_manifest():
|
||||
return {name: digest((ROOT / name).read_bytes()) for name in source_files()}
|
||||
|
||||
|
||||
def verify_seal(seal):
|
||||
value = json.loads((seal / "manifest.json").read_text())
|
||||
if value["files"] != source_manifest():
|
||||
raise RuntimeError("Source changed after seal; create a new seal/series.")
|
||||
if value["protocol_sha256"] != digest(PROTOCOL.read_bytes()):
|
||||
raise RuntimeError("Protocol changed after seal.")
|
||||
return value
|
||||
|
||||
|
||||
def png(profile, seed):
|
||||
w, h = profile["width"], profile["height"]
|
||||
encoder = zlib.compressobj(6)
|
||||
parts = []
|
||||
for y in range(h):
|
||||
row = bytearray([0])
|
||||
for x in range(w):
|
||||
v = (x * 17 + y * 31 + seed + (x * y) % 251) & 255
|
||||
row.extend(
|
||||
(
|
||||
v,
|
||||
(v * 13 + x) & 255,
|
||||
(v * 7 + y) & 255,
|
||||
((x + y) * 11) & 255 if profile["alpha"] else 255,
|
||||
)
|
||||
)
|
||||
parts.append(encoder.compress(row))
|
||||
parts.append(encoder.flush())
|
||||
|
||||
def chunk(kind, data):
|
||||
return (
|
||||
struct.pack(">I", len(data))
|
||||
+ kind
|
||||
+ data
|
||||
+ struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
)
|
||||
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 6, 0, 0, 0))
|
||||
+ chunk(b"IDAT", b"".join(parts))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def private_environment(directory, protocol):
|
||||
directory.mkdir(parents=True, exist_ok=False)
|
||||
os.chmod(directory, 0o700)
|
||||
secret_dir = directory / "secrets"
|
||||
secret_dir.mkdir()
|
||||
password = secrets.token_hex(24)
|
||||
values = {
|
||||
"POSTGRES_DB": "messenger",
|
||||
"POSTGRES_USER": "messenger",
|
||||
"POSTGRES_PASSWORD": password,
|
||||
"APP_HOST": "0.0.0.0",
|
||||
"APP_PORT": "8080",
|
||||
"APP_ENV": "experiment",
|
||||
"API_REPLICAS": "2",
|
||||
"API_GOMAXPROCS": "1",
|
||||
"SESSION_TTL_SECONDS": "7200",
|
||||
"IDEMPOTENCY_TTL_SECONDS": "86400",
|
||||
"S3_ENDPOINT": "http://s3:9000",
|
||||
"S3_PUBLIC_ENDPOINT": "http://127.0.0.1:" + str(protocol["ports"]["s3"]),
|
||||
"S3_BUCKET": "messenger-attachments",
|
||||
"S3_REGION": "us-east-1",
|
||||
"DATABASE_URL_FILE": "/run/secrets/database_url",
|
||||
"CACHE_URL_FILE": "/run/secrets/cache_url",
|
||||
"BROKER_URL_FILE": "/run/secrets/broker_url",
|
||||
"S3_ACCESS_KEY_ID_FILE": "/run/secrets/s3_access_key_id",
|
||||
"S3_SECRET_ACCESS_KEY_FILE": "/run/secrets/s3_secret_access_key",
|
||||
"JOB_TIMEOUT_SECONDS": str(protocol["job_timeout_seconds"]),
|
||||
"JOB_MAX_ATTEMPTS": str(protocol["job_max_attempts"]),
|
||||
"WORKER_MAX_PIXELS": "8000000",
|
||||
"UPLOAD_CLEANUP_GRACE_SECONDS": "600",
|
||||
}
|
||||
credentials = {
|
||||
"postgres_password": password,
|
||||
"database_url": "postgres://messenger:"
|
||||
+ password
|
||||
+ "@db:5432/messenger?sslmode=disable",
|
||||
"cache_url": "redis://redis:6379/0",
|
||||
"s3_access_key_id": secrets.token_hex(12),
|
||||
"s3_secret_access_key": secrets.token_hex(32),
|
||||
"s3_api_access_key_id": secrets.token_hex(12),
|
||||
"s3_api_secret_access_key": secrets.token_hex(32),
|
||||
"broker_user": "messenger",
|
||||
"broker_password": secrets.token_hex(32),
|
||||
}
|
||||
credentials["broker_url"] = (
|
||||
"amqp://messenger:" + credentials["broker_password"] + "@broker:5672/%2F"
|
||||
)
|
||||
for name, value in credentials.items():
|
||||
(secret_dir / name).write_text(value + "\n")
|
||||
os.chmod(secret_dir / name, 0o644)
|
||||
env_file = directory / "private.env"
|
||||
env_file.write_text("".join(k + "=" + v + "\n" for k, v in values.items()))
|
||||
os.chmod(env_file, 0o600)
|
||||
faults = directory / "faults"
|
||||
faults.mkdir()
|
||||
os.chmod(faults, 0o777)
|
||||
return values
|
||||
|
||||
|
||||
def isolated_config(base, project, directory, protocol, app_image, s3_image):
|
||||
# config --format json resolves names: remove them to avoid sharing lab volumes.
|
||||
base["name"] = project
|
||||
for group in ("volumes", "networks"):
|
||||
for value in base.get(group, {}).values():
|
||||
if value.get("external"):
|
||||
raise RuntimeError("External resources are forbidden in experiment.")
|
||||
value.pop("name", None)
|
||||
services = base["services"]
|
||||
for name in ("frontend", "backend-tests"):
|
||||
services.pop(name, None)
|
||||
for name, service in services.items():
|
||||
if "build" in service:
|
||||
service.pop("build")
|
||||
service["image"] = s3_image if name == "s3" else app_image
|
||||
for mount in service.get("volumes", []):
|
||||
if (
|
||||
mount.get("type") == "bind"
|
||||
and Path(mount["source"]).parent == ROOT / "secrets"
|
||||
):
|
||||
mount["source"] = str(
|
||||
directory / "secrets" / Path(mount["source"]).name
|
||||
)
|
||||
services["s3"]["ports"] = [
|
||||
{
|
||||
"target": 9000,
|
||||
"published": str(protocol["ports"]["s3"]),
|
||||
"host_ip": "127.0.0.1",
|
||||
"protocol": "tcp",
|
||||
}
|
||||
]
|
||||
services["haproxy"]["ports"] = [
|
||||
{
|
||||
"target": 8080,
|
||||
"published": str(protocol["ports"]["api"]),
|
||||
"host_ip": "127.0.0.1",
|
||||
"protocol": "tcp",
|
||||
}
|
||||
]
|
||||
services["worker"].setdefault("deploy", {})["replicas"] = protocol["workers"]
|
||||
for name in ("worker", "dispatcher"):
|
||||
services[name]["environment"]["PIPELINE_FAULT_DIR"] = "/run/pipeline-faults"
|
||||
services[name].setdefault("volumes", []).append(
|
||||
{
|
||||
"type": "bind",
|
||||
"source": str(directory / "faults"),
|
||||
"target": "/run/pipeline-faults",
|
||||
}
|
||||
)
|
||||
return base
|
||||
|
||||
|
||||
def build_images(public):
|
||||
# Only builds images; does not start a container or collect an experiment case.
|
||||
subprocess.run(
|
||||
["docker", "build", "-t", "lab5-idem-app:prepared", str(ROOT / "src")],
|
||||
check=True,
|
||||
stdout=(public / "app-build.log").open("w"),
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"build",
|
||||
"-f",
|
||||
str(ROOT / "infra/minio.Dockerfile"),
|
||||
"-t",
|
||||
"lab5-idem-s3:prepared",
|
||||
str(ROOT / "infra"),
|
||||
],
|
||||
check=True,
|
||||
stdout=(public / "s3-build.log").open("w"),
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
|
||||
def image(tag):
|
||||
return subprocess.check_output(
|
||||
["docker", "image", "inspect", "--format", "{{.Id}}", tag], text=True
|
||||
).strip()
|
||||
|
||||
return image("lab5-idem-app:prepared"), image("lab5-idem-s3:prepared")
|
||||
|
||||
|
||||
def archive_public(run):
|
||||
target = run.parent / (run.name + "-public.zip")
|
||||
secret_values = []
|
||||
for directory in (run / ".private").glob("*/secrets"):
|
||||
for path in directory.iterdir():
|
||||
if any(
|
||||
word in path.name
|
||||
for word in (
|
||||
"password",
|
||||
"secret",
|
||||
"access_key",
|
||||
"database_url",
|
||||
"broker_url",
|
||||
)
|
||||
):
|
||||
value = path.read_bytes().strip()
|
||||
if len(value) >= 16:
|
||||
secret_values.append(value)
|
||||
with zipfile.ZipFile(target, "x", zipfile.ZIP_DEFLATED) as archive:
|
||||
for path in sorted(run.rglob("*")):
|
||||
relative = path.relative_to(run)
|
||||
if path.is_file() and ".private" not in relative.parts:
|
||||
if any(value in path.read_bytes() for value in secret_values):
|
||||
raise RuntimeError(
|
||||
"Secret detected; do not share archive: " + str(relative)
|
||||
)
|
||||
archive.write(path, relative)
|
||||
return target
|
||||
Reference in New Issue
Block a user