docs(storage): document consistency boundaries and reproducible checks

This commit is contained in:
2026-11-15 20:33:49 +03:00
parent 8e120ec937
commit fba6b6334e
9 changed files with 364 additions and 45 deletions
+33 -3
View File
@@ -182,6 +182,9 @@ def main():
"complete": False,
}
ready()
from scripts.verify_cluster import snapshot
result["runtime"] = snapshot()
compose("stop", "reconciler")
orphan = "objects/verification-" + uuid4().hex + "/orphan"
fresh = "objects/verification-" + uuid4().hex + "/fresh"
@@ -209,6 +212,14 @@ def main():
r = upload(client, session, chat)
assert r.status_code == 201
saved, uploader = r.json(), r.headers["x-instance-id"]
failed_old = upload(client, session, chat).json()
sql(
"UPDATE attachments SET state='failed',error_code='synthetic' WHERE id='"
+ failed_old["id"]
+ "'"
)
age(failed_old["id"])
grace_started = time.monotonic()
target = next(
c
for c in cluster.containers()
@@ -241,7 +252,12 @@ def main():
)
ready()
private_code = f"""
from src.errors import DomainError
checks={{}}
try:
s.head(key='forbidden'); raise AssertionError('HEAD outside IAM prefix')
except DomainError as e:
assert e.status==503; checks['denied_head_is_not_missing']=e.status
try:
urllib.request.urlopen('http://s3:9000/messenger/{key(saved["id"])}',timeout=3)
raise AssertionError('public object')
@@ -367,17 +383,26 @@ except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':j
age(used["id"])
age(saved["id"])
result["concurrent_uploads"] = concurrency(client, session, chat)
while time.monotonic() - orphan_started < 302:
while time.monotonic() - grace_started < 302:
print(
"Waiting for genuine S3 LastModified grace: "
+ str(round(302 - (time.monotonic() - orphan_started)))
+ str(round(302 - (time.monotonic() - grace_started)))
+ "s",
flush=True,
)
time.sleep(min(20, 302 - (time.monotonic() - orphan_started)))
time.sleep(
min(20, max(0, 302 - (time.monotonic() - grace_started)))
)
inside(
f"s.put(key={fresh!r},stream=io.BytesIO(b'fresh'),size=5,content_type='application/octet-stream',sha256=hashlib.sha256(b'fresh').hexdigest()); print('true')"
)
failed_fresh = upload(client, session, chat).json()
sql(
"UPDATE attachments SET state='failed',error_code='synthetic' WHERE id='"
+ failed_fresh["id"]
+ "'"
)
age(failed_fresh["id"])
before_states = [
state(i)
for i in [pending, missing_pending, used["id"], saved["id"]]
@@ -395,6 +420,7 @@ except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':j
"fail_pending_missing",
"mark_missing",
"delete_orphan",
"delete_failed_object",
]:
assert dry["actions"].get(action, 0) >= 1, dry
applied = run_reconcile(True)
@@ -430,6 +456,9 @@ except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':j
assert inside(
f"print(json.dumps(s.head(key={orphan!r}) is None and s.head(key={fresh!r}) is not None and s.head(key={key(saved['id'])!r}) is not None))"
)
assert inside(
f"print(json.dumps(s.head(key={key(failed_old['id'])!r}) is None and s.head(key={key(failed_fresh['id'])!r}) is not None))"
)
repeat = run_reconcile(True)
assert not repeat["applied"].get("delete_orphan")
assert repeat["linked_protected"] >= 1
@@ -441,6 +470,7 @@ except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':j
time.monotonic() - orphan_started, 3
),
"fresh_and_ready_unlinked_protected": True,
"old_failed_unlinked_deleted_and_fresh_failed_protected": True,
"missing_used_id_and_link_preserved": True,
"dry_run_unchanged": True,
}