forked from mtusi2026/mtusi2026lab5
694 lines
26 KiB
Python
694 lines
26 KiB
Python
"""Controlled delivery/fault cases. Called only by an explicit experiment run."""
|
|
|
|
import datetime as dt
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import xml.etree.ElementTree as ET
|
|
from pathlib import Path
|
|
|
|
from support import ROOT, digest, dump
|
|
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure
|
|
from image_probe import jpeg_size
|
|
|
|
|
|
class Violation(AssertionError):
|
|
pass
|
|
|
|
|
|
class HarnessError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def require(condition, message):
|
|
if not condition:
|
|
raise Violation(message)
|
|
|
|
|
|
class Stack:
|
|
def __init__(self, project, config, private, protocol):
|
|
self.project, self.config, self.private, self.protocol = (
|
|
project,
|
|
config,
|
|
private,
|
|
protocol,
|
|
)
|
|
self.prefix = [
|
|
"docker",
|
|
"compose",
|
|
"--project-name",
|
|
project,
|
|
"-f",
|
|
str(config),
|
|
]
|
|
|
|
def call(self, *args):
|
|
result = subprocess.run(
|
|
self.prefix + list(args),
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=60,
|
|
)
|
|
if result.returncode:
|
|
error = result.stderr
|
|
for path in (self.private / "secrets").iterdir():
|
|
value = path.read_text().strip()
|
|
if len(value) >= 12:
|
|
error = error.replace(value, "[REDACTED]")
|
|
with (self.private.parent.parent / "command-errors.jsonl").open(
|
|
"a"
|
|
) as output:
|
|
output.write(
|
|
json.dumps(
|
|
{
|
|
"project": self.project,
|
|
"args": args,
|
|
"exit_code": result.returncode,
|
|
"stderr": error,
|
|
}
|
|
)
|
|
+ "\n"
|
|
)
|
|
raise HarnessError("Compose command failed; see command-errors.jsonl")
|
|
return result.stdout.strip()
|
|
|
|
def wait_broker_ready(self, timeout=60):
|
|
"""Wait only during setup; runtime queue failures remain fatal."""
|
|
deadline = time.monotonic() + timeout
|
|
args = ["exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues",
|
|
"name", "messages_ready", "messages_unacknowledged",
|
|
"--formatter", "json", "--timeout", "5"]
|
|
attempts = 0
|
|
while time.monotonic() < deadline:
|
|
attempts += 1
|
|
try:
|
|
result = subprocess.run(
|
|
self.prefix + args, cwd=ROOT, text=True, capture_output=True,
|
|
timeout=min(15, max(0.1, deadline-time.monotonic())),
|
|
)
|
|
if result.returncode == 0:
|
|
rows = json.loads(result.stdout)
|
|
if not isinstance(rows, list):
|
|
raise HarnessError("Unexpected RabbitMQ queue response")
|
|
dump(self.private.parent.parent / ("broker-readiness-"+self.private.name+".json"), {
|
|
"project": self.project, "attempts": attempts,
|
|
"ready": True,
|
|
})
|
|
return
|
|
error = result.stderr
|
|
# Auth, command syntax and stopped-container errors are not startup races.
|
|
if "requires the 'rabbit' app to be running" not in error:
|
|
raise HarnessError("RabbitMQ readiness probe failed: " + error.strip())
|
|
except subprocess.TimeoutExpired:
|
|
error = "RabbitMQ startup probe timed out"
|
|
with (self.private.parent.parent / "broker-startup.jsonl").open("a") as output:
|
|
output.write(json.dumps({"project": self.project, "attempt": attempts,
|
|
"stderr": error}) + "\n")
|
|
time.sleep(min(1, max(0, deadline-time.monotonic())))
|
|
raise HarnessError("RabbitMQ did not finish startup before the setup deadline")
|
|
|
|
def sql(self, query):
|
|
return self.call(
|
|
"exec",
|
|
"-T",
|
|
"db",
|
|
"sh",
|
|
"-c",
|
|
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"',
|
|
"sh",
|
|
query,
|
|
)
|
|
|
|
def workers(self):
|
|
return self.call("ps", "-aq", "worker").splitlines()
|
|
|
|
def worker_action(self, action, ids):
|
|
for ident in ids:
|
|
owner = subprocess.check_output(
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
'{{index .Config.Labels "com.docker.compose.project"}}',
|
|
ident,
|
|
],
|
|
text=True,
|
|
).strip()
|
|
if owner != self.project:
|
|
raise HarnessError("Container belongs to another project")
|
|
if ids:
|
|
subprocess.run(
|
|
[
|
|
"docker",
|
|
action,
|
|
*(["--time", "5"] if action == "stop" else []),
|
|
*ids,
|
|
],
|
|
check=True,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
|
|
def queue_depth(self):
|
|
rows = json.loads(
|
|
self.call(
|
|
"exec",
|
|
"-T",
|
|
"broker",
|
|
"rabbitmqctl",
|
|
"-q",
|
|
"list_queues",
|
|
"name",
|
|
"messages_ready",
|
|
"messages_unacknowledged",
|
|
"--formatter",
|
|
"json",
|
|
)
|
|
)
|
|
return sum(
|
|
row["messages_ready"] + row["messages_unacknowledged"]
|
|
for row in rows
|
|
if row["name"] == "attachments.v1"
|
|
)
|
|
|
|
def operator(self, *args):
|
|
running = self.call("ps", "--status", "running", "-q", "dispatcher")
|
|
mode = (
|
|
["exec", "-T", "dispatcher"]
|
|
if running
|
|
else ["run", "--rm", "--no-deps", "-T", "dispatcher"]
|
|
)
|
|
return self.call(*mode, "./messenger", *args)
|
|
|
|
def admin(self, *args):
|
|
return json.loads(self.operator("jobs", *args))
|
|
|
|
def snapshot(self, attachment):
|
|
result = self.sql(
|
|
"SELECT json_build_object('desired',a.pipeline_version,'published',a.published_pipeline_version,'status',a.status,'jobs',(SELECT json_agg(json_build_object('job_id',j.id,'version',j.pipeline_version,'attempts',j.attempts,'superseded',j.superseded_at,'lease_until',j.lease_until) ORDER BY j.pipeline_version) FROM attachment_jobs j WHERE j.attachment_id=a.id),'attempts_audit',(SELECT COALESCE(json_agg(json_build_object('job_id',t.job_id,'generation',t.generation,'attempt',t.attempt,'started_at',t.started_at,'finished_at',t.finished_at,'error_code',t.error_code,'next_attempt_at',t.next_attempt_at) ORDER BY t.started_at),'[]') FROM attachment_job_attempts t JOIN attachment_jobs x ON x.id=t.job_id WHERE x.attachment_id=a.id),'publications',(SELECT COALESCE(json_agg(json_build_object('id',p.id,'version',p.pipeline_version,'variants',p.variants,'published_at',p.published_at) ORDER BY p.id),'[]') FROM attachment_publications p WHERE p.attachment_id=a.id)) FROM attachments a WHERE a.id='"
|
|
+ attachment
|
|
+ "'"
|
|
)
|
|
return json.loads(result)
|
|
|
|
def messages(self, attachment):
|
|
return json.loads(
|
|
self.sql(
|
|
"SELECT json_agg(json_build_object('schema_version',1,'job_id',id,'type','attachment.process.v1','attachment_id',attachment_id,'pipeline_version',pipeline_version,'attempt',1,'created_at',to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"')) ORDER BY pipeline_version) FROM attachment_jobs WHERE attachment_id='"
|
|
+ attachment
|
|
+ "'"
|
|
)
|
|
)
|
|
|
|
def publish(self, job):
|
|
self.operator("broker-publish", json.dumps(job))
|
|
|
|
def containers(self):
|
|
ids = self.call("ps", "-aq").splitlines()
|
|
if not ids:
|
|
return []
|
|
# Never save Config.Env: it contains the experiment DB password.
|
|
rows = json.loads(
|
|
subprocess.check_output(["docker", "inspect", *ids], text=True)
|
|
)
|
|
return [
|
|
{
|
|
"id": row["Id"],
|
|
"image": row["Image"],
|
|
"service": row["Config"]["Labels"].get("com.docker.compose.service"),
|
|
"state": {
|
|
k: row["State"].get(k)
|
|
for k in (
|
|
"Status",
|
|
"ExitCode",
|
|
"OOMKilled",
|
|
"StartedAt",
|
|
"FinishedAt",
|
|
)
|
|
},
|
|
"restarts": row["RestartCount"],
|
|
"memory": row["HostConfig"]["Memory"],
|
|
"nano_cpus": row["HostConfig"]["NanoCpus"],
|
|
}
|
|
for row in rows
|
|
]
|
|
|
|
def s3(self, method, key="", query=None):
|
|
secret_dir = self.private / "secrets"
|
|
access = (secret_dir / "s3_api_access_key_id").read_text().strip()
|
|
secret = (secret_dir / "s3_api_secret_access_key").read_text().strip()
|
|
host = "127.0.0.1:" + str(self.protocol["ports"]["s3"])
|
|
uri = "/messenger-attachments/" + key
|
|
query_string = urllib.parse.urlencode(
|
|
sorted((query or {}).items()), quote_via=urllib.parse.quote
|
|
)
|
|
now = dt.datetime.now(dt.timezone.utc)
|
|
timestamp, day = now.strftime("%Y%m%dT%H%M%SZ"), now.strftime("%Y%m%d")
|
|
payload = digest(b"")
|
|
signed = "host;x-amz-content-sha256;x-amz-date"
|
|
headers = (
|
|
"host:"
|
|
+ host
|
|
+ "\nx-amz-content-sha256:"
|
|
+ payload
|
|
+ "\nx-amz-date:"
|
|
+ timestamp
|
|
+ "\n"
|
|
)
|
|
canonical = "\n".join([method, uri, query_string, headers, signed, payload])
|
|
scope = day + "/us-east-1/s3/aws4_request"
|
|
text = (
|
|
"AWS4-HMAC-SHA256\n"
|
|
+ timestamp
|
|
+ "\n"
|
|
+ scope
|
|
+ "\n"
|
|
+ digest(canonical.encode())
|
|
)
|
|
signing = ("AWS4" + secret).encode()
|
|
for part in (day, "us-east-1", "s3", "aws4_request"):
|
|
signing = hmac.new(signing, part.encode(), hashlib.sha256).digest()
|
|
signature = hmac.new(signing, text.encode(), hashlib.sha256).hexdigest()
|
|
request = urllib.request.Request(
|
|
"http://" + host + uri + ("?" + query_string if query_string else ""),
|
|
method=method,
|
|
headers={
|
|
"x-amz-date": timestamp,
|
|
"x-amz-content-sha256": payload,
|
|
"Authorization": "AWS4-HMAC-SHA256 Credential="
|
|
+ access
|
|
+ "/"
|
|
+ scope
|
|
+ ", SignedHeaders="
|
|
+ signed
|
|
+ ", Signature="
|
|
+ signature,
|
|
},
|
|
)
|
|
try:
|
|
response = urllib.request.urlopen(request, timeout=15)
|
|
except urllib.error.HTTPError as exc:
|
|
response = exc
|
|
with response:
|
|
code, body = response.code, response.read()
|
|
if code in (401, 403):
|
|
raise HarnessError("S3 measurement identity could not authenticate")
|
|
return code, body
|
|
|
|
|
|
def webp_size(data):
|
|
require(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "invalid WebP container")
|
|
offset = 12
|
|
dimensions = None
|
|
while offset + 8 <= len(data):
|
|
kind = data[offset : offset + 4]
|
|
length = int.from_bytes(data[offset + 4 : offset + 8], "little")
|
|
body = data[offset + 8 : offset + 8 + length]
|
|
require(kind not in (b"EXIF", b"XMP ", b"ICCP"), "WebP metadata leaked")
|
|
if kind == b"VP8 ":
|
|
dimensions = (
|
|
int.from_bytes(body[6:8], "little") & 16383,
|
|
int.from_bytes(body[8:10], "little") & 16383,
|
|
)
|
|
if kind == b"VP8L":
|
|
bits = int.from_bytes(body[1:5], "little")
|
|
dimensions = ((bits & 16383) + 1, ((bits >> 14) & 16383) + 1)
|
|
if kind == b"VP8X":
|
|
dimensions = (
|
|
int.from_bytes(body[4:7], "little") + 1,
|
|
int.from_bytes(body[7:10], "little") + 1,
|
|
)
|
|
offset += 8 + length + (length % 2)
|
|
require(dimensions is not None, "WebP dimensions absent")
|
|
return dimensions
|
|
|
|
|
|
class Case:
|
|
def __init__(self, stack, profile, scenario, data, directory):
|
|
self.s, self.profile, self.scenario, self.data, self.directory = (
|
|
stack,
|
|
profile,
|
|
scenario,
|
|
data,
|
|
directory,
|
|
)
|
|
directory.mkdir(parents=True)
|
|
self.events = (directory / "events.jsonl").open("x")
|
|
self.start = time.monotonic()
|
|
self.last = None
|
|
self.gaps = []
|
|
self.attachment = None
|
|
self.verifications = 0
|
|
self.c = Client("http://127.0.0.1:" + str(stack.protocol["ports"]["api"]))
|
|
|
|
def event(self, kind, **values):
|
|
now = time.monotonic()
|
|
record = {
|
|
"utc": dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
"elapsed": now - self.start,
|
|
"kind": kind,
|
|
**values,
|
|
}
|
|
self.events.write(json.dumps(record, ensure_ascii=False) + "\n")
|
|
self.events.flush()
|
|
|
|
def wait(self, fn, timeout=None, invariant=True):
|
|
deadline = time.monotonic() + (
|
|
timeout or self.s.protocol["case_timeout_seconds"]
|
|
)
|
|
while time.monotonic() < deadline:
|
|
value = fn()
|
|
if value and time.monotonic() <= deadline:
|
|
return value
|
|
time.sleep(self.s.protocol["poll_interval_seconds"])
|
|
if invariant:
|
|
raise Violation("completion/quiescence deadline exceeded")
|
|
raise HarnessError("required fault/precondition did not occur before deadline")
|
|
|
|
def observe(self):
|
|
now = time.monotonic()
|
|
if self.last is not None and now - self.last > 2:
|
|
self.gaps.append(now - self.last)
|
|
self.last = now
|
|
value = self.c.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + self.attachment,
|
|
actor=self.alice,
|
|
schema="Attachment",
|
|
)["json"]
|
|
snapshot = self.s.snapshot(self.attachment)
|
|
self.event("observation", api=value, database=snapshot)
|
|
return value, snapshot
|
|
|
|
def ready(self, version):
|
|
def check():
|
|
value, snapshot = self.observe()
|
|
require(value["status"] != "failed", "valid fixture reached failed")
|
|
return (
|
|
(value, snapshot)
|
|
if value["status"] == "ready" and snapshot["published"] == version
|
|
else None
|
|
)
|
|
|
|
return self.wait(check)
|
|
|
|
def arm(self, point):
|
|
marker = self.s.private / "faults" / (point + "." + self.attachment)
|
|
marker.write_text("experiment " + self.scenario + "\n")
|
|
self.event("arm", point=point)
|
|
return Path(str(marker) + ".hit")
|
|
|
|
def upload(self):
|
|
self.alice, self.bob, self.eve, chat = self.c.bootstrap()
|
|
value = self.c.call(
|
|
"POST",
|
|
"/api/v1/chats/" + chat["id"] + "/attachments?filename=fixture.png",
|
|
202,
|
|
actor=self.alice,
|
|
raw=self.data,
|
|
headers={"Content-Type": "image/png"},
|
|
schema="Attachment",
|
|
)["json"]
|
|
self.attachment = value["id"]
|
|
require(
|
|
value["sha256"] == digest(self.data), "accepted original checksum mismatch"
|
|
)
|
|
self.event("accepted_202", attachment_id=self.attachment)
|
|
require(
|
|
len(self.s.messages(self.attachment)) == 1,
|
|
"accepted upload has no unique durable job",
|
|
)
|
|
|
|
def upgrade(self):
|
|
require(
|
|
self.s.admin(
|
|
"upgrade", self.attachment, "2", "Protocol v1 " + self.scenario
|
|
)["changed"],
|
|
"upgrade intent was not persisted",
|
|
)
|
|
self.event("upgrade_accepted", version=2)
|
|
|
|
def publish(self, job, count):
|
|
for _ in range(count):
|
|
self.s.publish(job)
|
|
self.event(
|
|
"explicit_delivery",
|
|
job_id=job["job_id"],
|
|
version=job["pipeline_version"],
|
|
count=count,
|
|
)
|
|
|
|
def verify(self, version):
|
|
value, snapshot = self.ready(version)
|
|
self.verifications += 1
|
|
observation = "verify-" + str(self.verifications).zfill(3) + "-v" + str(version)
|
|
objects = self.directory / "objects" / observation
|
|
objects.mkdir(parents=True)
|
|
require(
|
|
snapshot["desired"] == version, "desired version differs from publication"
|
|
)
|
|
original = self.c.call(
|
|
"GET", "/api/v1/attachments/" + self.attachment + "/content", actor=self.bob
|
|
)["bytes"]
|
|
require(original == self.data, "original changed")
|
|
require(
|
|
len(value["variants"]) == 3
|
|
and {v["name"] for v in value["variants"]}
|
|
== {"thumbnail", "crop", "optimized"},
|
|
"incomplete or duplicate variant set",
|
|
)
|
|
manifest = {}
|
|
for item in value["variants"]:
|
|
name = item["name"]
|
|
key = (
|
|
"variants/"
|
|
+ self.attachment
|
|
+ "/v"
|
|
+ str(version)
|
|
+ "/"
|
|
+ name
|
|
+ (".webp" if name == "optimized" else ".jpg")
|
|
)
|
|
response = self.c.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + self.attachment + "/content?variant=" + name,
|
|
actor=self.bob,
|
|
)
|
|
require(
|
|
digest(response["bytes"]) == item["sha256"]
|
|
and len(response["bytes"]) == item["size_bytes"],
|
|
"variant checksum mismatch",
|
|
)
|
|
require(
|
|
response["headers"].get_content_type() == item["content_type"],
|
|
"variant MIME mismatch",
|
|
)
|
|
try:
|
|
dimensions = (
|
|
webp_size(response["bytes"])
|
|
if name == "optimized"
|
|
else jpeg_size(response["bytes"])
|
|
)
|
|
except Failure as exc:
|
|
raise Violation("actual variant decode failed") from exc
|
|
w, h = self.profile["width"], self.profile["height"]
|
|
limit = 1280 if name == "optimized" else 256
|
|
expected = (
|
|
(128, 128)
|
|
if name == "crop"
|
|
else (min(w, limit), max(1, h * limit // w))
|
|
if w >= h and w > limit
|
|
else (max(1, w * limit // h), limit)
|
|
if h > w and h > limit
|
|
else (w, h)
|
|
)
|
|
require(
|
|
dimensions == expected == (item["width"], item["height"]),
|
|
"actual variant dimensions mismatch",
|
|
)
|
|
code, body = self.s.s3("GET", key)
|
|
require(
|
|
code == 200 and body == response["bytes"],
|
|
"API exposes a different version than S3",
|
|
)
|
|
(
|
|
objects / (name + (".webp" if name == "optimized" else ".jpg"))
|
|
).write_bytes(body)
|
|
manifest[key] = digest(body)
|
|
code, body = self.s.s3(
|
|
"GET",
|
|
query={
|
|
"list-type": "2",
|
|
"prefix": "variants/" + self.attachment + "/v" + str(version) + "/",
|
|
},
|
|
)
|
|
require(code == 200, "cannot list variant set")
|
|
require(
|
|
len(ET.fromstring(body).findall("{*}Contents")) == 3,
|
|
"extra/missing object set",
|
|
)
|
|
history = snapshot["publications"]
|
|
versions = [p["version"] for p in history]
|
|
require(
|
|
versions == sorted(versions) and len(versions) == len(set(versions)),
|
|
"publication duplicated or rolled back",
|
|
)
|
|
require(versions[-1] == version, "publication history missing active version")
|
|
require(
|
|
all(
|
|
j["version"] == version or j["superseded"] is not None
|
|
for j in snapshot["jobs"]
|
|
),
|
|
"older accepted job has no durable disposition",
|
|
)
|
|
dump(self.directory / (observation + "-manifest.json"), manifest)
|
|
return manifest, snapshot
|
|
|
|
def execute(self):
|
|
self.s.worker_action("stop", self.s.workers())
|
|
self.s.call("stop", "dispatcher")
|
|
self.upload()
|
|
workers = self.s.workers()
|
|
require(len(workers) == 2, "two workers are required by protocol")
|
|
scenario = self.scenario
|
|
version = 1
|
|
first = None
|
|
if scenario == "commit_publish_gap":
|
|
marker = self.arm("after_commit_before_publish")
|
|
if scenario != "reorder":
|
|
self.s.call("start", "dispatcher")
|
|
if scenario == "commit_publish_gap":
|
|
self.wait(marker.exists, invariant=False)
|
|
if scenario == "duplicates":
|
|
self.publish(
|
|
self.s.messages(self.attachment)[0],
|
|
self.s.protocol["duplicates_before_ready"],
|
|
)
|
|
if scenario.startswith("crash_"):
|
|
point = {
|
|
"crash_before_effect": "before_effect",
|
|
"crash_after_effect": "after_object_before_commit",
|
|
"crash_before_ack": "after_commit_before_ack",
|
|
}[scenario]
|
|
marker = self.arm(point)
|
|
self.s.worker_action("start", [workers[0]])
|
|
self.wait(marker.exists, invariant=False)
|
|
self.event("fault_hit", point=point)
|
|
elif scenario == "reorder":
|
|
# No consumer is running; broker order is deliberately v2 then old v1.
|
|
self.upgrade()
|
|
jobs = self.s.messages(self.attachment)
|
|
self.publish(jobs[1], 1)
|
|
self.publish(jobs[0], 1)
|
|
version = 2
|
|
self.s.worker_action("start", workers)
|
|
elif scenario == "stale_lease":
|
|
marker = self.arm("after_claim")
|
|
self.s.worker_action("start", [workers[0]])
|
|
self.wait(marker.exists, invariant=False)
|
|
self.s.call("start", "dispatcher")
|
|
self.upgrade()
|
|
self.s.worker_action("start", [workers[1]])
|
|
first, _ = self.verify(2)
|
|
version = 2
|
|
release = (
|
|
self.s.private
|
|
/ "faults"
|
|
/ ("after_claim." + self.attachment + ".release")
|
|
)
|
|
release.write_text("release\n")
|
|
self.event("release_old_lease")
|
|
self.wait(
|
|
Path(str(release) + ".observed").exists, timeout=5, invariant=False
|
|
)
|
|
self.wait(
|
|
(
|
|
self.s.private
|
|
/ "faults"
|
|
/ ("stale_rejected." + self.attachment + ".observed")
|
|
).exists,
|
|
timeout=5,
|
|
invariant=False,
|
|
)
|
|
else:
|
|
self.s.worker_action("start", workers)
|
|
if scenario == "old_after_v2":
|
|
old_manifest, _ = self.verify(1)
|
|
old = self.s.messages(self.attachment)[0]
|
|
self.upgrade()
|
|
version = 2
|
|
first, _ = self.verify(2)
|
|
if not any(
|
|
old_manifest.get(key.replace("/v2/", "/v1/")) != value
|
|
for key, value in first.items()
|
|
):
|
|
raise HarnessError(
|
|
"Version fingerprints are indistinguishable for this fixture"
|
|
)
|
|
self.publish(old, self.s.protocol["duplicates_after_ready"])
|
|
for key, checksum in old_manifest.items():
|
|
code, body = self.s.s3("GET", key)
|
|
require(
|
|
code == 200 and digest(body) == checksum,
|
|
"v1 immutable object changed",
|
|
)
|
|
if scenario == "duplicates":
|
|
first, before = self.verify(1)
|
|
self.publish(
|
|
self.s.messages(self.attachment)[0],
|
|
self.s.protocol["duplicates_after_ready"],
|
|
)
|
|
if scenario != "reorder":
|
|
self.s.call("start", "dispatcher")
|
|
self.s.worker_action("start", workers)
|
|
self.wait(lambda: self.s.queue_depth() == 0)
|
|
# Let an already-delivered stale attempt leave its pause and hit the fence.
|
|
if scenario == "stale_lease":
|
|
time.sleep(1)
|
|
final, snapshot = self.verify(version)
|
|
if first is not None:
|
|
require(first == final, "old delivery changed published bytes")
|
|
if scenario in ("duplicates", "old_after_v2"):
|
|
require(
|
|
all(j["attempts"] == 1 for j in snapshot["jobs"]),
|
|
"terminal duplicate started another attempt",
|
|
)
|
|
if scenario == "reorder":
|
|
self.s.call("start", "dispatcher")
|
|
self.event("case_complete", version=version)
|
|
return {
|
|
"attachment_id": self.attachment,
|
|
"version": version,
|
|
"snapshot": snapshot,
|
|
"manifest": final,
|
|
"observation_gaps": self.gaps,
|
|
"elapsed_seconds": time.monotonic() - self.start,
|
|
}
|
|
|
|
def recover(self):
|
|
for marker in (self.s.private / "faults").glob("after_claim.*.hit"):
|
|
Path(str(marker)[:-4] + ".release").write_text("cleanup release\n")
|
|
self.s.call("start", "dispatcher")
|
|
self.s.worker_action("start", self.s.workers())
|
|
self.wait(
|
|
lambda: (
|
|
self.s.sql(
|
|
"SELECT count(*) FROM attachments WHERE status IN ('queued','processing')"
|
|
)
|
|
== "0"
|
|
and self.s.queue_depth() == 0
|
|
)
|
|
)
|
|
|
|
def close(self):
|
|
self.events.close()
|