forked from mtusi2026/mtusi2026lab5
276 lines
16 KiB
Python
276 lines
16 KiB
Python
"""Реальные отказы/очистка/память уровня 4; только временный тестовый стенд."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import socket
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from urllib.parse import urlsplit
|
|
from uuid import uuid4
|
|
|
|
from PIL import Image
|
|
|
|
from helpers import Client, ROOT
|
|
from client import Failure
|
|
|
|
|
|
def run_checks(base, env, compose, apis, script, sql):
|
|
result = {}
|
|
client = Client(base, timeout=30)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
png = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
upload_path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png"
|
|
|
|
def upload():
|
|
return client.call("POST", upload_path, 201, actor=alice, raw=png,
|
|
headers={"Content-Type": "image/png"}, schema="Attachment")
|
|
|
|
def inside(code, data=None):
|
|
completed = subprocess.run(["docker", "exec", "-i", apis()[0]["Id"], "python", "-c", code],
|
|
input=json.dumps(data), capture_output=True, text=True, timeout=45, check=True)
|
|
return json.loads(completed.stdout)
|
|
|
|
def cleanup():
|
|
return inside("from src import db; from src.cleanup import cleanup_once; import json; "
|
|
"db.pool.open(wait=True); print(json.dumps(cleanup_once())); db.pool.close()")
|
|
|
|
def stat(key):
|
|
return inside("import json,sys; from src import object_store; from minio.error import S3Error\n"
|
|
"key=json.load(sys.stdin)\n"
|
|
"try:\n r=object_store.client.stat_object(object_store.BUCKET,key); print(json.dumps({'size':r.size}))\n"
|
|
"except S3Error as e:\n print(json.dumps({'error':e.code}))", key)
|
|
|
|
def age(key):
|
|
sql("UPDATE attachment_uploads SET created_at=clock_timestamp()-interval '2 hours' WHERE object_key='" + key + "'")
|
|
|
|
compose("stop", "cleanup")
|
|
try:
|
|
for _ in range(2):
|
|
compose("run", "--rm", "s3-bootstrap")
|
|
ready = upload()["json"]
|
|
key = sql("SELECT object_key FROM attachments WHERE id='" + ready["id"] + "'")
|
|
backend_id = compose("ps", "-q", "s3-storage")
|
|
backend = json.loads(subprocess.check_output(["docker", "inspect", backend_id], text=True))[0]
|
|
assert not set(backend["NetworkSettings"]["Networks"]) & set(apis()[0]["NetworkSettings"]["Networks"])
|
|
backend_ip = next(iter(backend["NetworkSettings"]["Networks"].values()))["IPAddress"]
|
|
# Вызов S3 напрямую из контейнера клиента, минуя HTTP API приложения.
|
|
result["private_bucket"] = inside("""
|
|
import io,json,socket,sys
|
|
from urllib.error import HTTPError
|
|
from urllib.request import urlopen
|
|
from minio.error import S3Error
|
|
from src import object_store
|
|
data=json.load(sys.stdin); key=data['key']
|
|
try:
|
|
urlopen('http://s3:9000/'+object_store.BUCKET+'/'+key,timeout=3)
|
|
raise AssertionError('public original')
|
|
except HTTPError as e:
|
|
assert e.code == 403
|
|
for operation in (lambda: object_store.client.make_bucket('forbidden-bucket'),
|
|
lambda: object_store.client.put_object(object_store.BUCKET,'outside-prefix',io.BytesIO(b'x'),1)):
|
|
try:
|
|
operation()
|
|
raise AssertionError('excess application permissions')
|
|
except S3Error as e:
|
|
assert e.code == 'AccessDenied', e.code
|
|
for port in (9333,8888,8080):
|
|
with socket.socket() as connection:
|
|
connection.settimeout(1)
|
|
assert connection.connect_ex(('s3',port)) != 0, 'admin/backend port open'
|
|
with socket.socket() as connection:
|
|
connection.settimeout(.5)
|
|
assert connection.connect_ex((data['backend_ip'],port)) != 0, 'storage network reachable from API'
|
|
print(json.dumps({'anonymous_get':403,'create_bucket':'AccessDenied','write_outside_prefix':'AccessDenied',
|
|
'admin_ports_unreachable':[9333,8888,8080],'storage_network_isolated':True,'bootstrap_runs':2}))
|
|
""", {"key":key, "backend_ip":backend_ip})
|
|
print("PASS: private S3, limited credentials, isolated administration, repeated bootstrap", flush=True)
|
|
|
|
# Останавливаем именно реплику, которая ответила на upload, а затем читаем другой.
|
|
uploaded = upload()
|
|
target = uploaded["headers"]["X-Instance-Id"]
|
|
stopped_at = time.monotonic()
|
|
subprocess.run([str(ROOT / "scripts/stop-one.sh")], cwd=ROOT,
|
|
env=dict(env, TARGET_INSTANCE_ID=target), check=True, timeout=15)
|
|
script("wait-ready.sh")
|
|
while True:
|
|
try:
|
|
response = client.call("GET", "/api/v1/attachments/" + uploaded["json"]["id"] + "/content", actor=bob)
|
|
break
|
|
except Failure:
|
|
assert time.monotonic() - stopped_at < 10, "download failover exceeded 10 seconds"
|
|
time.sleep(.1)
|
|
assert response["bytes"] == png and response["headers"]["X-Instance-Id"] != target
|
|
stopped = next(info for info in apis() if info["Config"]["Hostname"] == target)
|
|
assert not stopped["State"]["Running"]
|
|
result["upload_replica_failover"] = {"uploaded_by": target, "stopped_container": stopped["Id"],
|
|
"downloaded_by": response["headers"]["X-Instance-Id"], "sha256": hashlib.sha256(png).hexdigest(),
|
|
"recovery_seconds":round(time.monotonic()-stopped_at,3)}
|
|
compose("start", "api")
|
|
script("wait-ready.sh")
|
|
print("PASS: original downloaded through another replica after stopping uploader", flush=True)
|
|
|
|
# Незавершённое HTTP-тело не достигает S3 и не создаёт метаданных/журнала.
|
|
before = sql("SELECT (SELECT count(*) FROM attachments)||':'||(SELECT count(*) FROM attachment_uploads)")
|
|
address = urlsplit(base)
|
|
with socket.create_connection((address.hostname, address.port), timeout=3) as connection:
|
|
headers = ("POST " + upload_path + " HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer " + alice["token"] +
|
|
"\r\nContent-Type: image/png\r\nContent-Length: 1000000\r\n\r\n").encode()
|
|
connection.sendall(headers + png[:30])
|
|
time.sleep(0.5)
|
|
assert sql("SELECT (SELECT count(*) FROM attachments)||':'||(SELECT count(*) FROM attachment_uploads)") == before
|
|
result["interrupted_body"] = {"metadata_or_journal_created": False}
|
|
|
|
# Детерминированный реальный отказ PostgreSQL ПОСЛЕ успешного S3 PUT.
|
|
sql("""CREATE FUNCTION lab4_reject_attachment() RETURNS trigger LANGUAGE plpgsql AS $$
|
|
BEGIN RAISE EXCEPTION 'injected storage failure' USING ERRCODE='08006'; END $$;
|
|
CREATE TRIGGER lab4_reject BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION lab4_reject_attachment();""")
|
|
try:
|
|
failure = client.call("POST", upload_path, 503, actor=alice, raw=png, headers={"Content-Type": "image/png"})
|
|
assert failure["json"]["error"]["code"] == "unavailable"
|
|
finally:
|
|
sql("DROP TRIGGER lab4_reject ON attachments; DROP FUNCTION lab4_reject_attachment()")
|
|
pending_key = sql("SELECT object_key FROM attachment_uploads ORDER BY created_at DESC LIMIT 1")
|
|
assert stat(pending_key)["size"] == len(png)
|
|
assert sql("SELECT count(*) FROM attachments WHERE object_key='" + pending_key + "'") == "0"
|
|
assert cleanup()["removed"] == 0
|
|
age(pending_key)
|
|
lock_code = ("from src import db; from src.attachments import lock_upload; import sys\n"
|
|
"db.pool.open(wait=True)\nwith db.connection() as c:\n"
|
|
" lock_upload(c,'" + pending_key + "'); print('locked',flush=True); sys.stdin.readline()\n"
|
|
"db.pool.close()")
|
|
holder = subprocess.Popen(["docker", "exec", "-i", apis()[0]["Id"], "python", "-u", "-c", lock_code],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
try:
|
|
assert holder.stdout.readline().strip() == "locked"
|
|
assert cleanup()["skipped_active"] == 1
|
|
assert stat(pending_key)["size"] == len(png)
|
|
finally:
|
|
holder.communicate("\n", timeout=10)
|
|
assert cleanup()["removed"] == 1
|
|
assert stat(pending_key).get("error") == "NoSuchKey"
|
|
assert cleanup()["removed"] == 0
|
|
# Даже устаревшая ошибочная запись журнала не позволяет удалить ready-файл.
|
|
sql("INSERT INTO attachment_uploads(id,object_key,created_at) VALUES ('" + str(uuid4()) + "','" + key +
|
|
"',clock_timestamp()-interval '2 hours')")
|
|
assert cleanup()["kept_ready"] == 1
|
|
assert client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=bob)["bytes"] == png
|
|
result["compensation"] = {"failure_after_put":503, "orphan_recorded":True, "fresh_kept":True,
|
|
"active_locked_kept":True, "expired_removed":True, "ready_kept":True, "repeat_safe":True}
|
|
print("PASS: post-PUT DB failure recorded; cleanup protects fresh, active and ready uploads", flush=True)
|
|
|
|
try:
|
|
compose("stop", "--timeout", "8", "s3")
|
|
result["s3_outage"] = inside("""
|
|
import json,sys
|
|
from urllib.request import Request,urlopen
|
|
from urllib.error import HTTPError
|
|
data=json.load(sys.stdin)
|
|
statuses={}
|
|
for name,method,path,body in [('live','GET','/health/live',None),('ready','GET','/health/ready',None),
|
|
('download','GET','/api/v1/attachments/'+data['id']+'/content',None),
|
|
('upload','POST',data['upload'],bytes.fromhex(data['png']))]:
|
|
req=Request('http://127.0.0.1:8080'+path,data=body,method=method,
|
|
headers={'Authorization':'Bearer '+data['token'],'Content-Type':'image/png'})
|
|
try: response=urlopen(req,timeout=8)
|
|
except HTTPError as error: response=error
|
|
with response:
|
|
statuses[name]=response.status
|
|
payload=json.load(response)
|
|
if name!='live':
|
|
assert response.status==503 and payload['error']['code']=='unavailable'
|
|
assert payload['error']['request_id']==response.headers['X-Request-Id']
|
|
assert statuses['live']==200
|
|
print(json.dumps(statuses))
|
|
""", {"id":ready["id"], "token":alice["token"], "upload":upload_path, "png":png.hex()})
|
|
finally:
|
|
compose("start", "s3")
|
|
script("wait-ready.sh")
|
|
assert client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=bob)["bytes"] == png
|
|
sql("UPDATE attachment_uploads SET created_at=clock_timestamp()-interval '2 hours'")
|
|
assert cleanup()["removed"] >= 1
|
|
result["s3_outage"]["original_preserved_after_recovery"] = True
|
|
print("PASS: S3 outage gives API 503; recovery preserves original and session", flush=True)
|
|
|
|
# Доказываем работу постоянного worker, а не только ручного вызова cleanup_once.
|
|
inside("import io,json; from src import object_store; "
|
|
"object_store.put('attachments/cleanup-worker-test',io.BytesIO(b'x'),1,'application/octet-stream'); print('{}')")
|
|
sql("INSERT INTO attachment_uploads(id,object_key,created_at) VALUES ('" + str(uuid4()) +
|
|
"','attachments/cleanup-worker-test',clock_timestamp()-interval '2 hours')")
|
|
compose("start", "cleanup")
|
|
deadline = time.monotonic() + 40
|
|
while sql("SELECT count(*) FROM attachment_uploads WHERE object_key='attachments/cleanup-worker-test'") != "0":
|
|
assert time.monotonic() < deadline, "automatic cleanup not running"
|
|
time.sleep(0.2)
|
|
assert stat('attachments/cleanup-worker-test').get("error") == "NoSuchKey"
|
|
result["compensation"]["automatic_worker"] = True
|
|
|
|
# 10 одновременно отправляемых реальных PNG ~5.5 MiB; монитор cgroup каждые 20 ms.
|
|
image = Image.frombytes("RGB", (1600, 1200), os.urandom(1600 * 1200 * 3))
|
|
buffer = io.BytesIO()
|
|
image.save(buffer, format="PNG")
|
|
image.close()
|
|
data = buffer.getvalue()
|
|
expected_sha = hashlib.sha256(data).hexdigest()
|
|
monitor_code = """
|
|
from pathlib import Path
|
|
import json,sys,threading,time
|
|
root=Path('/sys/fs/cgroup'); stop=threading.Event(); samples=[]
|
|
threading.Thread(target=lambda:(sys.stdin.readline(),stop.set()),daemon=True).start()
|
|
start=time.monotonic(); before=int((root/'memory.current').read_text())
|
|
print('ready',flush=True)
|
|
while not stop.is_set():
|
|
samples.append([round(time.monotonic()-start,3),int((root/'memory.current').read_text())])
|
|
stop.wait(.02)
|
|
print(json.dumps({'baseline_bytes':before,'sample_peak_bytes':max(s[1] for s in samples),
|
|
'lifetime_peak_bytes':int((root/'memory.peak').read_text()),'samples':samples}),flush=True)
|
|
"""
|
|
monitors = []
|
|
responses = []
|
|
try:
|
|
for info in apis():
|
|
process = subprocess.Popen(["docker", "exec", "-i", info["Id"], "python", "-u", "-c", monitor_code],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
assert process.stdout.readline().strip() == "ready"
|
|
monitors.append((info, process))
|
|
barrier = threading.Barrier(10)
|
|
def send(_):
|
|
concurrent_client = Client(base, timeout=60)
|
|
barrier.wait(timeout=15)
|
|
return concurrent_client.call("POST", upload_path, 201, actor=alice, raw=data,
|
|
headers={"Content-Type":"image/png"}, schema="Attachment")
|
|
started = time.monotonic()
|
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
|
responses = list(pool.map(send, range(10)))
|
|
elapsed = time.monotonic() - started
|
|
finally:
|
|
memory = []
|
|
for info, process in monitors:
|
|
output, error = process.communicate("\n", timeout=10)
|
|
assert process.returncode == 0, error
|
|
measurement = json.loads(output)
|
|
measurement.update({"instance": info["Config"]["Hostname"], "limit_bytes": info["HostConfig"]["Memory"]})
|
|
assert measurement["sample_peak_bytes"] < measurement["limit_bytes"]
|
|
memory.append(measurement)
|
|
assert len(responses) == 10
|
|
assert len({r["headers"]["X-Instance-Id"] for r in responses}) >= 2
|
|
for response in responses:
|
|
assert response["json"]["sha256"] == expected_sha and response["json"]["size_bytes"] == len(data)
|
|
original = client.call("GET", "/api/v1/attachments/" + response["json"]["id"] + "/content", actor=bob)
|
|
assert hashlib.sha256(original["bytes"]).hexdigest() == expected_sha
|
|
assert all(not i["State"]["OOMKilled"] for i in apis())
|
|
timings = sorted(r["elapsed"] * 1000 for r in responses)
|
|
result["parallel_uploads"] = {"concurrency":10, "successful":10, "errors":0,
|
|
"dimensions":[1600,1200], "size_bytes":len(data), "sha256":expected_sha,
|
|
"duration_seconds":round(elapsed,3), "successful_rps":round(10/elapsed,3),
|
|
"p50_ms":round(timings[4],3), "p95_ms":round(timings[-1],3),
|
|
"memory_sample_interval_seconds":0.02, "memory_includes_monitor_process":True,
|
|
"memory":memory, "all_original_checksums_verified":True, "oom_killed":False}
|
|
print("PASS: 10 concurrent real PNG uploads; all checksums match; memory peaks:",
|
|
[m["sample_peak_bytes"] for m in memory], flush=True)
|
|
return result
|
|
finally:
|
|
compose("start", "s3", "api", "cleanup")
|