docs(storage): document consistency boundaries and reproducible checks
This commit is contained in:
@@ -182,6 +182,9 @@ def main():
|
||||
"complete": False,
|
||||
}
|
||||
ready()
|
||||
from scripts.verify_cluster import snapshot
|
||||
|
||||
result["runtime"] = snapshot()
|
||||
compose("stop", "reconciler")
|
||||
orphan = "objects/verification-" + uuid4().hex + "/orphan"
|
||||
fresh = "objects/verification-" + uuid4().hex + "/fresh"
|
||||
@@ -209,6 +212,14 @@ def main():
|
||||
r = upload(client, session, chat)
|
||||
assert r.status_code == 201
|
||||
saved, uploader = r.json(), r.headers["x-instance-id"]
|
||||
failed_old = upload(client, session, chat).json()
|
||||
sql(
|
||||
"UPDATE attachments SET state='failed',error_code='synthetic' WHERE id='"
|
||||
+ failed_old["id"]
|
||||
+ "'"
|
||||
)
|
||||
age(failed_old["id"])
|
||||
grace_started = time.monotonic()
|
||||
target = next(
|
||||
c
|
||||
for c in cluster.containers()
|
||||
@@ -241,7 +252,12 @@ def main():
|
||||
)
|
||||
ready()
|
||||
private_code = f"""
|
||||
from src.errors import DomainError
|
||||
checks={{}}
|
||||
try:
|
||||
s.head(key='forbidden'); raise AssertionError('HEAD outside IAM prefix')
|
||||
except DomainError as e:
|
||||
assert e.status==503; checks['denied_head_is_not_missing']=e.status
|
||||
try:
|
||||
urllib.request.urlopen('http://s3:9000/messenger/{key(saved["id"])}',timeout=3)
|
||||
raise AssertionError('public object')
|
||||
@@ -367,17 +383,26 @@ except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':j
|
||||
age(used["id"])
|
||||
age(saved["id"])
|
||||
result["concurrent_uploads"] = concurrency(client, session, chat)
|
||||
while time.monotonic() - orphan_started < 302:
|
||||
while time.monotonic() - grace_started < 302:
|
||||
print(
|
||||
"Waiting for genuine S3 LastModified grace: "
|
||||
+ str(round(302 - (time.monotonic() - orphan_started)))
|
||||
+ str(round(302 - (time.monotonic() - grace_started)))
|
||||
+ "s",
|
||||
flush=True,
|
||||
)
|
||||
time.sleep(min(20, 302 - (time.monotonic() - orphan_started)))
|
||||
time.sleep(
|
||||
min(20, max(0, 302 - (time.monotonic() - grace_started)))
|
||||
)
|
||||
inside(
|
||||
f"s.put(key={fresh!r},stream=io.BytesIO(b'fresh'),size=5,content_type='application/octet-stream',sha256=hashlib.sha256(b'fresh').hexdigest()); print('true')"
|
||||
)
|
||||
failed_fresh = upload(client, session, chat).json()
|
||||
sql(
|
||||
"UPDATE attachments SET state='failed',error_code='synthetic' WHERE id='"
|
||||
+ failed_fresh["id"]
|
||||
+ "'"
|
||||
)
|
||||
age(failed_fresh["id"])
|
||||
before_states = [
|
||||
state(i)
|
||||
for i in [pending, missing_pending, used["id"], saved["id"]]
|
||||
@@ -395,6 +420,7 @@ except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':j
|
||||
"fail_pending_missing",
|
||||
"mark_missing",
|
||||
"delete_orphan",
|
||||
"delete_failed_object",
|
||||
]:
|
||||
assert dry["actions"].get(action, 0) >= 1, dry
|
||||
applied = run_reconcile(True)
|
||||
@@ -430,6 +456,9 @@ except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':j
|
||||
assert inside(
|
||||
f"print(json.dumps(s.head(key={orphan!r}) is None and s.head(key={fresh!r}) is not None and s.head(key={key(saved['id'])!r}) is not None))"
|
||||
)
|
||||
assert inside(
|
||||
f"print(json.dumps(s.head(key={key(failed_old['id'])!r}) is None and s.head(key={key(failed_fresh['id'])!r}) is not None))"
|
||||
)
|
||||
repeat = run_reconcile(True)
|
||||
assert not repeat["applied"].get("delete_orphan")
|
||||
assert repeat["linked_protected"] >= 1
|
||||
@@ -441,6 +470,7 @@ except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':j
|
||||
time.monotonic() - orphan_started, 3
|
||||
),
|
||||
"fresh_and_ready_unlinked_protected": True,
|
||||
"old_failed_unlinked_deleted_and_fresh_failed_protected": True,
|
||||
"missing_used_id_and_link_preserved": True,
|
||||
"dry_run_unchanged": True,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user