"""Реальные отказы/очистка/память уровня 4; только временный тестовый стенд.""" from concurrent.futures import ThreadPoolExecutor import hashlib import io import json import os import socket import subprocess import threading import time from urllib.parse import urlsplit from uuid import uuid4 from PIL import Image from helpers import Client, ROOT from client import Failure def run_checks(base, env, compose, apis, script, sql): result = {} client = Client(base, timeout=30) alice, bob, _, chat = client.bootstrap() png = (ROOT / "tests/fixtures/sample.png").read_bytes() upload_path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png" def upload(): return client.call("POST", upload_path, 201, actor=alice, raw=png, headers={"Content-Type": "image/png"}, schema="Attachment") def inside(code, data=None): completed = subprocess.run(["docker", "exec", "-i", apis()[0]["Id"], "python", "-c", code], input=json.dumps(data), capture_output=True, text=True, timeout=45, check=True) return json.loads(completed.stdout) def cleanup(): return inside("from src import db; from src.cleanup import cleanup_once; import json; " "db.pool.open(wait=True); print(json.dumps(cleanup_once())); db.pool.close()") def stat(key): return inside("import json,sys; from src import object_store; from minio.error import S3Error\n" "key=json.load(sys.stdin)\n" "try:\n r=object_store.client.stat_object(object_store.BUCKET,key); print(json.dumps({'size':r.size}))\n" "except S3Error as e:\n print(json.dumps({'error':e.code}))", key) def age(key): sql("UPDATE attachment_uploads SET created_at=clock_timestamp()-interval '2 hours' WHERE object_key='" + key + "'") compose("stop", "cleanup") try: for _ in range(2): compose("run", "--rm", "s3-bootstrap") ready = upload()["json"] key = sql("SELECT object_key FROM attachments WHERE id='" + ready["id"] + "'") backend_id = compose("ps", "-q", "s3-storage") backend = json.loads(subprocess.check_output(["docker", "inspect", backend_id], text=True))[0] assert not set(backend["NetworkSettings"]["Networks"]) & set(apis()[0]["NetworkSettings"]["Networks"]) backend_ip = next(iter(backend["NetworkSettings"]["Networks"].values()))["IPAddress"] # Вызов S3 напрямую из контейнера клиента, минуя HTTP API приложения. result["private_bucket"] = inside(""" import io,json,socket,sys from urllib.error import HTTPError from urllib.request import urlopen from minio.error import S3Error from src import object_store data=json.load(sys.stdin); key=data['key'] try: urlopen('http://s3:9000/'+object_store.BUCKET+'/'+key,timeout=3) raise AssertionError('public original') except HTTPError as e: assert e.code == 403 for operation in (lambda: object_store.client.make_bucket('forbidden-bucket'), lambda: object_store.client.put_object(object_store.BUCKET,'outside-prefix',io.BytesIO(b'x'),1)): try: operation() raise AssertionError('excess application permissions') except S3Error as e: assert e.code == 'AccessDenied', e.code for port in (9333,8888,8080): with socket.socket() as connection: connection.settimeout(1) assert connection.connect_ex(('s3',port)) != 0, 'admin/backend port open' with socket.socket() as connection: connection.settimeout(.5) assert connection.connect_ex((data['backend_ip'],port)) != 0, 'storage network reachable from API' print(json.dumps({'anonymous_get':403,'create_bucket':'AccessDenied','write_outside_prefix':'AccessDenied', 'admin_ports_unreachable':[9333,8888,8080],'storage_network_isolated':True,'bootstrap_runs':2})) """, {"key":key, "backend_ip":backend_ip}) print("PASS: private S3, limited credentials, isolated administration, repeated bootstrap", flush=True) # Останавливаем именно реплику, которая ответила на upload, а затем читаем другой. uploaded = upload() target = uploaded["headers"]["X-Instance-Id"] stopped_at = time.monotonic() subprocess.run([str(ROOT / "scripts/stop-one.sh")], cwd=ROOT, env=dict(env, TARGET_INSTANCE_ID=target), check=True, timeout=15) script("wait-ready.sh") while True: try: response = client.call("GET", "/api/v1/attachments/" + uploaded["json"]["id"] + "/content", actor=bob) break except Failure: assert time.monotonic() - stopped_at < 10, "download failover exceeded 10 seconds" time.sleep(.1) assert response["bytes"] == png and response["headers"]["X-Instance-Id"] != target stopped = next(info for info in apis() if info["Config"]["Hostname"] == target) assert not stopped["State"]["Running"] result["upload_replica_failover"] = {"uploaded_by": target, "stopped_container": stopped["Id"], "downloaded_by": response["headers"]["X-Instance-Id"], "sha256": hashlib.sha256(png).hexdigest(), "recovery_seconds":round(time.monotonic()-stopped_at,3)} compose("start", "api") script("wait-ready.sh") print("PASS: original downloaded through another replica after stopping uploader", flush=True) # Незавершённое HTTP-тело не достигает S3 и не создаёт метаданных/журнала. before = sql("SELECT (SELECT count(*) FROM attachments)||':'||(SELECT count(*) FROM attachment_uploads)") address = urlsplit(base) with socket.create_connection((address.hostname, address.port), timeout=3) as connection: headers = ("POST " + upload_path + " HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer " + alice["token"] + "\r\nContent-Type: image/png\r\nContent-Length: 1000000\r\n\r\n").encode() connection.sendall(headers + png[:30]) time.sleep(0.5) assert sql("SELECT (SELECT count(*) FROM attachments)||':'||(SELECT count(*) FROM attachment_uploads)") == before result["interrupted_body"] = {"metadata_or_journal_created": False} # Детерминированный реальный отказ PostgreSQL ПОСЛЕ успешного S3 PUT. sql("""CREATE FUNCTION lab4_reject_attachment() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected storage failure' USING ERRCODE='08006'; END $$; CREATE TRIGGER lab4_reject BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION lab4_reject_attachment();""") try: failure = client.call("POST", upload_path, 503, actor=alice, raw=png, headers={"Content-Type": "image/png"}) assert failure["json"]["error"]["code"] == "unavailable" finally: sql("DROP TRIGGER lab4_reject ON attachments; DROP FUNCTION lab4_reject_attachment()") pending_key = sql("SELECT object_key FROM attachment_uploads ORDER BY created_at DESC LIMIT 1") assert stat(pending_key)["size"] == len(png) assert sql("SELECT count(*) FROM attachments WHERE object_key='" + pending_key + "'") == "0" assert cleanup()["removed"] == 0 age(pending_key) lock_code = ("from src import db; from src.attachments import lock_upload; import sys\n" "db.pool.open(wait=True)\nwith db.connection() as c:\n" " lock_upload(c,'" + pending_key + "'); print('locked',flush=True); sys.stdin.readline()\n" "db.pool.close()") holder = subprocess.Popen(["docker", "exec", "-i", apis()[0]["Id"], "python", "-u", "-c", lock_code], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) try: assert holder.stdout.readline().strip() == "locked" assert cleanup()["skipped_active"] == 1 assert stat(pending_key)["size"] == len(png) finally: holder.communicate("\n", timeout=10) assert cleanup()["removed"] == 1 assert stat(pending_key).get("error") == "NoSuchKey" assert cleanup()["removed"] == 0 # Даже устаревшая ошибочная запись журнала не позволяет удалить ready-файл. sql("INSERT INTO attachment_uploads(id,object_key,created_at) VALUES ('" + str(uuid4()) + "','" + key + "',clock_timestamp()-interval '2 hours')") assert cleanup()["kept_ready"] == 1 assert client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=bob)["bytes"] == png result["compensation"] = {"failure_after_put":503, "orphan_recorded":True, "fresh_kept":True, "active_locked_kept":True, "expired_removed":True, "ready_kept":True, "repeat_safe":True} print("PASS: post-PUT DB failure recorded; cleanup protects fresh, active and ready uploads", flush=True) try: compose("stop", "--timeout", "8", "s3") result["s3_outage"] = inside(""" import json,sys from urllib.request import Request,urlopen from urllib.error import HTTPError data=json.load(sys.stdin) statuses={} for name,method,path,body in [('live','GET','/health/live',None),('ready','GET','/health/ready',None), ('download','GET','/api/v1/attachments/'+data['id']+'/content',None), ('upload','POST',data['upload'],bytes.fromhex(data['png']))]: req=Request('http://127.0.0.1:8080'+path,data=body,method=method, headers={'Authorization':'Bearer '+data['token'],'Content-Type':'image/png'}) try: response=urlopen(req,timeout=8) except HTTPError as error: response=error with response: statuses[name]=response.status payload=json.load(response) if name!='live': assert response.status==503 and payload['error']['code']=='unavailable' assert payload['error']['request_id']==response.headers['X-Request-Id'] assert statuses['live']==200 print(json.dumps(statuses)) """, {"id":ready["id"], "token":alice["token"], "upload":upload_path, "png":png.hex()}) finally: compose("start", "s3") script("wait-ready.sh") assert client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=bob)["bytes"] == png sql("UPDATE attachment_uploads SET created_at=clock_timestamp()-interval '2 hours'") assert cleanup()["removed"] >= 1 result["s3_outage"]["original_preserved_after_recovery"] = True print("PASS: S3 outage gives API 503; recovery preserves original and session", flush=True) # Доказываем работу постоянного worker, а не только ручного вызова cleanup_once. inside("import io,json; from src import object_store; " "object_store.put('attachments/cleanup-worker-test',io.BytesIO(b'x'),1,'application/octet-stream'); print('{}')") sql("INSERT INTO attachment_uploads(id,object_key,created_at) VALUES ('" + str(uuid4()) + "','attachments/cleanup-worker-test',clock_timestamp()-interval '2 hours')") compose("start", "cleanup") deadline = time.monotonic() + 40 while sql("SELECT count(*) FROM attachment_uploads WHERE object_key='attachments/cleanup-worker-test'") != "0": assert time.monotonic() < deadline, "automatic cleanup not running" time.sleep(0.2) assert stat('attachments/cleanup-worker-test').get("error") == "NoSuchKey" result["compensation"]["automatic_worker"] = True # 10 одновременно отправляемых реальных PNG ~5.5 MiB; монитор cgroup каждые 20 ms. image = Image.frombytes("RGB", (1600, 1200), os.urandom(1600 * 1200 * 3)) buffer = io.BytesIO() image.save(buffer, format="PNG") image.close() data = buffer.getvalue() expected_sha = hashlib.sha256(data).hexdigest() monitor_code = """ from pathlib import Path import json,sys,threading,time root=Path('/sys/fs/cgroup'); stop=threading.Event(); samples=[] threading.Thread(target=lambda:(sys.stdin.readline(),stop.set()),daemon=True).start() start=time.monotonic(); before=int((root/'memory.current').read_text()) print('ready',flush=True) while not stop.is_set(): samples.append([round(time.monotonic()-start,3),int((root/'memory.current').read_text())]) stop.wait(.02) print(json.dumps({'baseline_bytes':before,'sample_peak_bytes':max(s[1] for s in samples), 'lifetime_peak_bytes':int((root/'memory.peak').read_text()),'samples':samples}),flush=True) """ monitors = [] responses = [] try: for info in apis(): process = subprocess.Popen(["docker", "exec", "-i", info["Id"], "python", "-u", "-c", monitor_code], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) assert process.stdout.readline().strip() == "ready" monitors.append((info, process)) barrier = threading.Barrier(10) def send(_): concurrent_client = Client(base, timeout=60) barrier.wait(timeout=15) return concurrent_client.call("POST", upload_path, 201, actor=alice, raw=data, headers={"Content-Type":"image/png"}, schema="Attachment") started = time.monotonic() with ThreadPoolExecutor(max_workers=10) as pool: responses = list(pool.map(send, range(10))) elapsed = time.monotonic() - started finally: memory = [] for info, process in monitors: output, error = process.communicate("\n", timeout=10) assert process.returncode == 0, error measurement = json.loads(output) measurement.update({"instance": info["Config"]["Hostname"], "limit_bytes": info["HostConfig"]["Memory"]}) assert measurement["sample_peak_bytes"] < measurement["limit_bytes"] memory.append(measurement) assert len(responses) == 10 assert len({r["headers"]["X-Instance-Id"] for r in responses}) >= 2 for response in responses: assert response["json"]["sha256"] == expected_sha and response["json"]["size_bytes"] == len(data) original = client.call("GET", "/api/v1/attachments/" + response["json"]["id"] + "/content", actor=bob) assert hashlib.sha256(original["bytes"]).hexdigest() == expected_sha assert all(not i["State"]["OOMKilled"] for i in apis()) timings = sorted(r["elapsed"] * 1000 for r in responses) result["parallel_uploads"] = {"concurrency":10, "successful":10, "errors":0, "dimensions":[1600,1200], "size_bytes":len(data), "sha256":expected_sha, "duration_seconds":round(elapsed,3), "successful_rps":round(10/elapsed,3), "p50_ms":round(timings[4],3), "p95_ms":round(timings[-1],3), "memory_sample_interval_seconds":0.02, "memory_includes_monitor_process":True, "memory":memory, "all_original_checksums_verified":True, "oom_killed":False} print("PASS: 10 concurrent real PNG uploads; all checksums match; memory peaks:", [m["sample_peak_bytes"] for m in memory], flush=True) return result finally: compose("start", "s3", "api", "cleanup")