204 lines
7.8 KiB
Python
204 lines
7.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Exercise real S3/PostgreSQL reconciliation using disposable synthetic objects."""
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
import tempfile
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, arguments, check, run
|
|
|
|
|
|
STORE_CODE = """
|
|
import sys
|
|
from pathlib import Path
|
|
from src.store import Store
|
|
|
|
store = Store()
|
|
store.open()
|
|
try:
|
|
action = sys.argv[1]
|
|
if action == "key":
|
|
attachment_id = sys.argv[2]
|
|
matches = [
|
|
key for key, row in store.reconciliation_snapshot().items()
|
|
if str(row["id"]) == attachment_id
|
|
]
|
|
if len(matches) != 1:
|
|
raise RuntimeError("attachment metadata lookup failed")
|
|
print(matches[0])
|
|
elif action == "put":
|
|
key, file_path = sys.argv[2:4]
|
|
content = Path(file_path).read_bytes()
|
|
store.objects.put(key, content, "image/png", len(content))
|
|
elif action == "delete":
|
|
store.objects.delete(sys.argv[2])
|
|
else:
|
|
raise RuntimeError("unsupported test operation")
|
|
finally:
|
|
store.close()
|
|
"""
|
|
|
|
PRIVATE_CHECK_CODE = """
|
|
import socket
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
try:
|
|
socket.getaddrinfo("s3-core", 8888)
|
|
except socket.gaierror:
|
|
print("PASS: SeaweedFS master/filer network is isolated")
|
|
else:
|
|
raise RuntimeError("API network can resolve the SeaweedFS core service")
|
|
|
|
for port in (8181, 9101):
|
|
try:
|
|
socket.create_connection(("s3", port), timeout=1).close()
|
|
except OSError:
|
|
continue
|
|
raise RuntimeError("unexpected SeaweedFS gateway interface is reachable on port " + str(port))
|
|
|
|
try:
|
|
urllib.request.urlopen("http://s3:8333/messenger-attachments/", timeout=3)
|
|
except urllib.error.HTTPError as error:
|
|
if error.code == 403:
|
|
print("PASS: unauthenticated bucket access denied")
|
|
else:
|
|
raise RuntimeError("unexpected anonymous S3 status: " + str(error.code))
|
|
else:
|
|
raise RuntimeError("private bucket allowed an unauthenticated GET")
|
|
"""
|
|
|
|
|
|
def docker_exec(container_id, *args):
|
|
return subprocess.run(
|
|
["docker", "exec", container_id, *args],
|
|
cwd=ROOT,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
|
|
def reconciliation(container_id, *args):
|
|
result = docker_exec(container_id, "python3", "src/reconcile.py", "--grace-seconds", "60", *args)
|
|
return json.loads(result.stdout)
|
|
|
|
|
|
def main():
|
|
args = arguments("Real PostgreSQL/SeaweedFS reconciliation and private-bucket experiment")
|
|
client = Client(args.base_url, args.ca_file)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
attachment, content = client.attachment(alice, chat)
|
|
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
|
|
|
|
result = subprocess.run(
|
|
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
|
|
cwd=ROOT,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
containers = result.stdout.split()
|
|
check(len(containers) >= 2, "at least two running API replicas are required")
|
|
container_id = containers[0]
|
|
key = docker_exec(
|
|
container_id, "python3", "-c", STORE_CODE, "key", attachment["id"]
|
|
).stdout.strip()
|
|
orphan_key = "original/" + str(uuid.uuid4())
|
|
|
|
with tempfile.TemporaryDirectory() as backup:
|
|
source = Path(backup) / "original.png"
|
|
source.write_bytes(content)
|
|
container_file = "/tmp/lab4-orphan-" + uuid.uuid4().hex + ".png"
|
|
recovery_dir = "/tmp/lab4-recovery-" + uuid.uuid4().hex
|
|
recovery_file = recovery_dir + "/" + attachment["id"]
|
|
orphan_created = False
|
|
orphan_removed = False
|
|
missing_deleted = False
|
|
missing_restored = False
|
|
try:
|
|
subprocess.run(["docker", "cp", str(source), container_id + ":" + container_file], check=True)
|
|
docker_exec(container_id, "python3", "-c", STORE_CODE, "put", orphan_key, container_file)
|
|
orphan_created = True
|
|
docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", key)
|
|
missing_deleted = True
|
|
|
|
preview = reconciliation(container_id)
|
|
check(
|
|
preview["orphan_objects"] >= 1 and preview["missing_objects"] == 1,
|
|
"dry-run missed injected inconsistencies",
|
|
)
|
|
check(preview["deleted_orphans"] == 0, "dry-run modified S3")
|
|
immediate_apply = reconciliation(container_id, "--apply")
|
|
check(immediate_apply["stale_orphan_candidates"] == 0, "fresh orphan bypassed grace period")
|
|
check(immediate_apply["deleted_orphans"] == 0, "fresh orphan was deleted")
|
|
|
|
docker_exec(container_id, "mkdir", "-p", recovery_dir)
|
|
subprocess.run(
|
|
["docker", "cp", str(source), container_id + ":" + recovery_file],
|
|
check=True,
|
|
)
|
|
restore_preview = reconciliation(container_id, "--restore-dir", recovery_dir)
|
|
check(restore_preview["restorable_missing_objects"] == 1, "exact backup was not identified as restorable")
|
|
restored = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir)
|
|
check(restored["restored_missing_objects"] == 1, "missing S3 object was not restored")
|
|
missing_restored = True
|
|
|
|
repeated = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir)
|
|
check(repeated["missing_objects"] == 0, "repeat reconciliation still reports missing object")
|
|
check(repeated["stale_orphan_candidates"] == 0, "recent orphan did not respect grace period")
|
|
print("Waiting for the 60-second orphan grace period.")
|
|
time.sleep(61)
|
|
|
|
cleanup_preview = reconciliation(container_id)
|
|
check(cleanup_preview["stale_orphan_candidates"] == 1, "stale orphan was not identified after grace")
|
|
cleaned = reconciliation(container_id, "--apply")
|
|
check(cleaned["deleted_orphans"] == 1, "stale orphan was not removed")
|
|
orphan_removed = True
|
|
final_scan = reconciliation(container_id, "--apply")
|
|
check(
|
|
final_scan["orphan_objects"] == 0 and final_scan["missing_objects"] == 0,
|
|
"repeat scan is not clean",
|
|
)
|
|
finally:
|
|
if missing_deleted and not missing_restored:
|
|
docker_exec(container_id, "python3", "-c", STORE_CODE, "put", key, container_file)
|
|
if orphan_created and not orphan_removed:
|
|
docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", orphan_key)
|
|
|
|
private = docker_exec(container_id, "python3", "-c", PRIVATE_CHECK_CODE).stdout.strip()
|
|
check("PASS:" in private, "unauthenticated S3 access check failed")
|
|
downloaded = client.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + attachment["id"] + "/content",
|
|
actor=bob,
|
|
)["bytes"]
|
|
check(downloaded == content, "reconciler changed the original of an attachment used by a message")
|
|
check(message["attachment_ids"] == [attachment["id"]], "test message did not retain the attachment reference")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"result": "PASS",
|
|
"dataset_objects_scanned": final_scan["objects_seen"],
|
|
"metadata_records_scanned": final_scan["metadata_seen"],
|
|
"scan_duration_ms": final_scan["duration_ms"],
|
|
"orphan_detected_and_removed": cleaned["deleted_orphans"],
|
|
"missing_detected_and_restored": restored["restored_missing_objects"],
|
|
"fresh_orphan_preserved_until_grace": True,
|
|
"message_attachment_download_checksum": "unchanged",
|
|
"private_bucket": "anonymous request denied",
|
|
},
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|