Files
mtusi2026lab4/student-tests/attachment_checks.py

276 lines
16 KiB
Python

"""Реальные отказы/очистка/память уровня 4; только временный тестовый стенд."""
from concurrent.futures import ThreadPoolExecutor
import hashlib
import io
import json
import os
import socket
import subprocess
import threading
import time
from urllib.parse import urlsplit
from uuid import uuid4
from PIL import Image
from helpers import Client, ROOT
from client import Failure
def run_checks(base, env, compose, apis, script, sql):
result = {}
client = Client(base, timeout=30)
alice, bob, _, chat = client.bootstrap()
png = (ROOT / "tests/fixtures/sample.png").read_bytes()
upload_path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png"
def upload():
return client.call("POST", upload_path, 201, actor=alice, raw=png,
headers={"Content-Type": "image/png"}, schema="Attachment")
def inside(code, data=None):
completed = subprocess.run(["docker", "exec", "-i", apis()[0]["Id"], "python", "-c", code],
input=json.dumps(data), capture_output=True, text=True, timeout=45, check=True)
return json.loads(completed.stdout)
def cleanup():
return inside("from src import db; from src.cleanup import cleanup_once; import json; "
"db.pool.open(wait=True); print(json.dumps(cleanup_once())); db.pool.close()")
def stat(key):
return inside("import json,sys; from src import object_store; from minio.error import S3Error\n"
"key=json.load(sys.stdin)\n"
"try:\n r=object_store.client.stat_object(object_store.BUCKET,key); print(json.dumps({'size':r.size}))\n"
"except S3Error as e:\n print(json.dumps({'error':e.code}))", key)
def age(key):
sql("UPDATE attachment_uploads SET created_at=clock_timestamp()-interval '2 hours' WHERE object_key='" + key + "'")
compose("stop", "cleanup")
try:
for _ in range(2):
compose("run", "--rm", "s3-bootstrap")
ready = upload()["json"]
key = sql("SELECT object_key FROM attachments WHERE id='" + ready["id"] + "'")
backend_id = compose("ps", "-q", "s3-storage")
backend = json.loads(subprocess.check_output(["docker", "inspect", backend_id], text=True))[0]
assert not set(backend["NetworkSettings"]["Networks"]) & set(apis()[0]["NetworkSettings"]["Networks"])
backend_ip = next(iter(backend["NetworkSettings"]["Networks"].values()))["IPAddress"]
# Вызов S3 напрямую из контейнера клиента, минуя HTTP API приложения.
result["private_bucket"] = inside("""
import io,json,socket,sys
from urllib.error import HTTPError
from urllib.request import urlopen
from minio.error import S3Error
from src import object_store
data=json.load(sys.stdin); key=data['key']
try:
urlopen('http://s3:9000/'+object_store.BUCKET+'/'+key,timeout=3)
raise AssertionError('public original')
except HTTPError as e:
assert e.code == 403
for operation in (lambda: object_store.client.make_bucket('forbidden-bucket'),
lambda: object_store.client.put_object(object_store.BUCKET,'outside-prefix',io.BytesIO(b'x'),1)):
try:
operation()
raise AssertionError('excess application permissions')
except S3Error as e:
assert e.code == 'AccessDenied', e.code
for port in (9333,8888,8080):
with socket.socket() as connection:
connection.settimeout(1)
assert connection.connect_ex(('s3',port)) != 0, 'admin/backend port open'
with socket.socket() as connection:
connection.settimeout(.5)
assert connection.connect_ex((data['backend_ip'],port)) != 0, 'storage network reachable from API'
print(json.dumps({'anonymous_get':403,'create_bucket':'AccessDenied','write_outside_prefix':'AccessDenied',
'admin_ports_unreachable':[9333,8888,8080],'storage_network_isolated':True,'bootstrap_runs':2}))
""", {"key":key, "backend_ip":backend_ip})
print("PASS: private S3, limited credentials, isolated administration, repeated bootstrap", flush=True)
# Останавливаем именно реплику, которая ответила на upload, а затем читаем другой.
uploaded = upload()
target = uploaded["headers"]["X-Instance-Id"]
stopped_at = time.monotonic()
subprocess.run([str(ROOT / "scripts/stop-one.sh")], cwd=ROOT,
env=dict(env, TARGET_INSTANCE_ID=target), check=True, timeout=15)
script("wait-ready.sh")
while True:
try:
response = client.call("GET", "/api/v1/attachments/" + uploaded["json"]["id"] + "/content", actor=bob)
break
except Failure:
assert time.monotonic() - stopped_at < 10, "download failover exceeded 10 seconds"
time.sleep(.1)
assert response["bytes"] == png and response["headers"]["X-Instance-Id"] != target
stopped = next(info for info in apis() if info["Config"]["Hostname"] == target)
assert not stopped["State"]["Running"]
result["upload_replica_failover"] = {"uploaded_by": target, "stopped_container": stopped["Id"],
"downloaded_by": response["headers"]["X-Instance-Id"], "sha256": hashlib.sha256(png).hexdigest(),
"recovery_seconds":round(time.monotonic()-stopped_at,3)}
compose("start", "api")
script("wait-ready.sh")
print("PASS: original downloaded through another replica after stopping uploader", flush=True)
# Незавершённое HTTP-тело не достигает S3 и не создаёт метаданных/журнала.
before = sql("SELECT (SELECT count(*) FROM attachments)||':'||(SELECT count(*) FROM attachment_uploads)")
address = urlsplit(base)
with socket.create_connection((address.hostname, address.port), timeout=3) as connection:
headers = ("POST " + upload_path + " HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer " + alice["token"] +
"\r\nContent-Type: image/png\r\nContent-Length: 1000000\r\n\r\n").encode()
connection.sendall(headers + png[:30])
time.sleep(0.5)
assert sql("SELECT (SELECT count(*) FROM attachments)||':'||(SELECT count(*) FROM attachment_uploads)") == before
result["interrupted_body"] = {"metadata_or_journal_created": False}
# Детерминированный реальный отказ PostgreSQL ПОСЛЕ успешного S3 PUT.
sql("""CREATE FUNCTION lab4_reject_attachment() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN RAISE EXCEPTION 'injected storage failure' USING ERRCODE='08006'; END $$;
CREATE TRIGGER lab4_reject BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION lab4_reject_attachment();""")
try:
failure = client.call("POST", upload_path, 503, actor=alice, raw=png, headers={"Content-Type": "image/png"})
assert failure["json"]["error"]["code"] == "unavailable"
finally:
sql("DROP TRIGGER lab4_reject ON attachments; DROP FUNCTION lab4_reject_attachment()")
pending_key = sql("SELECT object_key FROM attachment_uploads ORDER BY created_at DESC LIMIT 1")
assert stat(pending_key)["size"] == len(png)
assert sql("SELECT count(*) FROM attachments WHERE object_key='" + pending_key + "'") == "0"
assert cleanup()["removed"] == 0
age(pending_key)
lock_code = ("from src import db; from src.attachments import lock_upload; import sys\n"
"db.pool.open(wait=True)\nwith db.connection() as c:\n"
" lock_upload(c,'" + pending_key + "'); print('locked',flush=True); sys.stdin.readline()\n"
"db.pool.close()")
holder = subprocess.Popen(["docker", "exec", "-i", apis()[0]["Id"], "python", "-u", "-c", lock_code],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
try:
assert holder.stdout.readline().strip() == "locked"
assert cleanup()["skipped_active"] == 1
assert stat(pending_key)["size"] == len(png)
finally:
holder.communicate("\n", timeout=10)
assert cleanup()["removed"] == 1
assert stat(pending_key).get("error") == "NoSuchKey"
assert cleanup()["removed"] == 0
# Даже устаревшая ошибочная запись журнала не позволяет удалить ready-файл.
sql("INSERT INTO attachment_uploads(id,object_key,created_at) VALUES ('" + str(uuid4()) + "','" + key +
"',clock_timestamp()-interval '2 hours')")
assert cleanup()["kept_ready"] == 1
assert client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=bob)["bytes"] == png
result["compensation"] = {"failure_after_put":503, "orphan_recorded":True, "fresh_kept":True,
"active_locked_kept":True, "expired_removed":True, "ready_kept":True, "repeat_safe":True}
print("PASS: post-PUT DB failure recorded; cleanup protects fresh, active and ready uploads", flush=True)
try:
compose("stop", "--timeout", "8", "s3")
result["s3_outage"] = inside("""
import json,sys
from urllib.request import Request,urlopen
from urllib.error import HTTPError
data=json.load(sys.stdin)
statuses={}
for name,method,path,body in [('live','GET','/health/live',None),('ready','GET','/health/ready',None),
('download','GET','/api/v1/attachments/'+data['id']+'/content',None),
('upload','POST',data['upload'],bytes.fromhex(data['png']))]:
req=Request('http://127.0.0.1:8080'+path,data=body,method=method,
headers={'Authorization':'Bearer '+data['token'],'Content-Type':'image/png'})
try: response=urlopen(req,timeout=8)
except HTTPError as error: response=error
with response:
statuses[name]=response.status
payload=json.load(response)
if name!='live':
assert response.status==503 and payload['error']['code']=='unavailable'
assert payload['error']['request_id']==response.headers['X-Request-Id']
assert statuses['live']==200
print(json.dumps(statuses))
""", {"id":ready["id"], "token":alice["token"], "upload":upload_path, "png":png.hex()})
finally:
compose("start", "s3")
script("wait-ready.sh")
assert client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=bob)["bytes"] == png
sql("UPDATE attachment_uploads SET created_at=clock_timestamp()-interval '2 hours'")
assert cleanup()["removed"] >= 1
result["s3_outage"]["original_preserved_after_recovery"] = True
print("PASS: S3 outage gives API 503; recovery preserves original and session", flush=True)
# Доказываем работу постоянного worker, а не только ручного вызова cleanup_once.
inside("import io,json; from src import object_store; "
"object_store.put('attachments/cleanup-worker-test',io.BytesIO(b'x'),1,'application/octet-stream'); print('{}')")
sql("INSERT INTO attachment_uploads(id,object_key,created_at) VALUES ('" + str(uuid4()) +
"','attachments/cleanup-worker-test',clock_timestamp()-interval '2 hours')")
compose("start", "cleanup")
deadline = time.monotonic() + 40
while sql("SELECT count(*) FROM attachment_uploads WHERE object_key='attachments/cleanup-worker-test'") != "0":
assert time.monotonic() < deadline, "automatic cleanup not running"
time.sleep(0.2)
assert stat('attachments/cleanup-worker-test').get("error") == "NoSuchKey"
result["compensation"]["automatic_worker"] = True
# 10 одновременно отправляемых реальных PNG ~5.5 MiB; монитор cgroup каждые 20 ms.
image = Image.frombytes("RGB", (1600, 1200), os.urandom(1600 * 1200 * 3))
buffer = io.BytesIO()
image.save(buffer, format="PNG")
image.close()
data = buffer.getvalue()
expected_sha = hashlib.sha256(data).hexdigest()
monitor_code = """
from pathlib import Path
import json,sys,threading,time
root=Path('/sys/fs/cgroup'); stop=threading.Event(); samples=[]
threading.Thread(target=lambda:(sys.stdin.readline(),stop.set()),daemon=True).start()
start=time.monotonic(); before=int((root/'memory.current').read_text())
print('ready',flush=True)
while not stop.is_set():
samples.append([round(time.monotonic()-start,3),int((root/'memory.current').read_text())])
stop.wait(.02)
print(json.dumps({'baseline_bytes':before,'sample_peak_bytes':max(s[1] for s in samples),
'lifetime_peak_bytes':int((root/'memory.peak').read_text()),'samples':samples}),flush=True)
"""
monitors = []
responses = []
try:
for info in apis():
process = subprocess.Popen(["docker", "exec", "-i", info["Id"], "python", "-u", "-c", monitor_code],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
assert process.stdout.readline().strip() == "ready"
monitors.append((info, process))
barrier = threading.Barrier(10)
def send(_):
concurrent_client = Client(base, timeout=60)
barrier.wait(timeout=15)
return concurrent_client.call("POST", upload_path, 201, actor=alice, raw=data,
headers={"Content-Type":"image/png"}, schema="Attachment")
started = time.monotonic()
with ThreadPoolExecutor(max_workers=10) as pool:
responses = list(pool.map(send, range(10)))
elapsed = time.monotonic() - started
finally:
memory = []
for info, process in monitors:
output, error = process.communicate("\n", timeout=10)
assert process.returncode == 0, error
measurement = json.loads(output)
measurement.update({"instance": info["Config"]["Hostname"], "limit_bytes": info["HostConfig"]["Memory"]})
assert measurement["sample_peak_bytes"] < measurement["limit_bytes"]
memory.append(measurement)
assert len(responses) == 10
assert len({r["headers"]["X-Instance-Id"] for r in responses}) >= 2
for response in responses:
assert response["json"]["sha256"] == expected_sha and response["json"]["size_bytes"] == len(data)
original = client.call("GET", "/api/v1/attachments/" + response["json"]["id"] + "/content", actor=bob)
assert hashlib.sha256(original["bytes"]).hexdigest() == expected_sha
assert all(not i["State"]["OOMKilled"] for i in apis())
timings = sorted(r["elapsed"] * 1000 for r in responses)
result["parallel_uploads"] = {"concurrency":10, "successful":10, "errors":0,
"dimensions":[1600,1200], "size_bytes":len(data), "sha256":expected_sha,
"duration_seconds":round(elapsed,3), "successful_rps":round(10/elapsed,3),
"p50_ms":round(timings[4],3), "p95_ms":round(timings[-1],3),
"memory_sample_interval_seconds":0.02, "memory_includes_monitor_process":True,
"memory":memory, "all_original_checksums_verified":True, "oom_killed":False}
print("PASS: 10 concurrent real PNG uploads; all checksums match; memory peaks:",
[m["sample_peak_bytes"] for m in memory], flush=True)
return result
finally:
compose("start", "s3", "api", "cleanup")