all for lab4

This commit is contained in:
2026-11-08 18:17:48 +03:00
parent c1bd817abf
commit fc22dcc4a4
72 changed files with 12880 additions and 102 deletions
+321
View File
@@ -0,0 +1,321 @@
#!/usr/bin/env python3
import hashlib
import os
import struct
import uuid
import zlib
from pathlib import Path
from urllib.parse import quote
from client import Client, arguments, check, run
MAX_BYTES = 10 * 1024 * 1024
FIXTURE = (
Path(__file__).resolve().parent
/ "fixtures"
/ "sample.png"
)
def make_png(width, height, noisy=False):
"""Отдельное изображение для проверки лимита байтов или пикселей."""
def chunk(kind, data):
return (
struct.pack(">I", len(data))
+ kind
+ data
+ struct.pack(
">I",
zlib.crc32(kind + data) & 0xFFFFFFFF,
)
)
compressor = zlib.compressobj(level=1)
compressed = bytearray()
for _ in range(height):
row = os.urandom(width) if noisy else bytes(width)
compressed.extend(compressor.compress(b"\x00" + row))
compressed.extend(compressor.flush())
header = struct.pack(
">IIBBBBB",
width, height,
8, 0, 0, 0, 0,
)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", bytes(compressed))
+ chunk(b"IEND", b"")
)
def main():
args = arguments("Attachment negative tests")
c = Client(args.base_url, args.ca_file, timeout=30)
alice, bob, eve, chat = c.bootstrap()
chat_path = "/api/v1/chats/" + chat["id"]
original = FIXTURE.read_bytes()
def expect_error(label, method, path, status, code, **kwargs):
response = c.call(method, path, status, **kwargs)
check(
response["json"]["error"]["code"] == code,
label + ": incorrect error code",
)
print("PASS:", label, flush=True)
def reject_upload(
label,
content,
status,
code,
mime="image/png",
filename="sample.png",
actor=None,
):
expect_error(
label,
"POST",
chat_path + "/attachments?filename="
+ quote(filename, safe=""),
status,
code,
actor=alice if actor is None else actor,
raw=content,
headers={"Content-Type": mime},
)
def reject_message(label, target_chat, ids, status, code):
expect_error(
label,
"POST",
"/api/v1/chats/" + target_chat["id"] + "/messages",
status,
code,
actor=alice,
body={"text": "", "attachment_ids": ids},
headers={"Idempotency-Key": uuid.uuid4().hex},
)
# 1. Контрольная загрузка исходного sample.png.
attachment, uploaded = c.attachment(alice, chat)
check(uploaded == original, "Unexpected fixture content")
downloaded = c.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=bob,
)
check(downloaded["bytes"] == original, "Original bytes changed")
check(
attachment["sha256"] == hashlib.sha256(original).hexdigest(),
"SHA-256 mismatch",
)
print("PASS: original PNG upload and download", flush=True)
# 2. Настоящий PNG больше 10 MiB, но меньше 40 млн пикселей.
large = make_png(4096, 2700, noisy=True)
check(len(large) > MAX_BYTES, "Fixture must exceed 10 MiB")
reject_upload(
"file larger than 10 MiB",
large,
413,
"payload_too_large",
)
del large
# 3. Небольшой по байтам PNG с 40 960 000 пикселями.
many_pixels = make_png(6400, 6400)
check(len(many_pixels) < MAX_BYTES, "Fixture exceeds byte limit")
reject_upload(
"image larger than 40 million pixels",
many_pixels,
413,
"payload_too_large",
)
del many_pixels
# 4. Повреждение создаётся в памяти, sample.png не меняется.
reject_upload(
"truncated PNG",
original[:len(original) // 2],
415,
"unsupported_media_type",
)
reject_upload(
"text disguised as PNG",
b"This is not an image",
415,
"unsupported_media_type",
)
# 5. Реальный PNG объявлен как JPEG.
reject_upload(
"MIME spoofing",
original,
415,
"unsupported_media_type",
mime="image/jpeg",
filename="fake.jpg",
)
reject_upload(
"empty file",
b"",
415,
"unsupported_media_type",
)
# 6. Опасные и недопустимые имена файлов.
for filename in (
"../sample.png",
"folder\\sample.png",
"bad\x00name.png",
"bad\nname.png",
"",
"x" * 256,
):
reject_upload(
"invalid filename " + repr(filename),
original,
400,
"invalid_request",
filename=filename,
)
# 7. Посторонний не может читать метаданные и оригинал.
for suffix in ("", "/content"):
expect_error(
"outsider GET " + (suffix or "metadata"),
"GET",
"/api/v1/attachments/" + attachment["id"] + suffix,
404,
"not_found",
actor=eve,
)
reject_upload(
"outsider upload",
original,
404,
"not_found",
actor=eve,
)
# 8. Участник чата может прикрепить файл другого участника.
message = c.message(
bob,
chat,
"",
attachments=[attachment["id"]],
)["json"]
check(
message["attachment_ids"] == [attachment["id"]],
"Chat member could not attach the file",
)
print("PASS: chat member can attach the file", flush=True)
# 9. Alice состоит в обоих чатах, но перенос вложения запрещён.
second_chat = c.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={
"title": "Second test chat",
"member_ids": [alice["id"], bob["id"]],
},
schema="Chat",
)["json"]
reject_message(
"attachment from another chat",
second_chat,
[attachment["id"]],
404,
"not_found",
)
reject_message(
"unknown attachment",
chat,
[str(uuid.uuid4())],
404,
"not_found",
)
reject_message(
"duplicate attachment IDs",
chat,
[attachment["id"], attachment["id"]],
400,
"invalid_request",
)
# 10. Идемпотентность учитывает вложения.
key = uuid.uuid4().hex
first = c.message(
alice, chat, "Photo",
key=key,
attachments=[attachment["id"]],
)["json"]
replay = c.message(
alice, chat, "Photo",
key=key,
attachments=[attachment["id"]],
)["json"]
check(first == replay, "Idempotent replay differs")
expect_error(
"same key with different attachments",
"POST",
chat_path + "/messages",
409,
"conflict",
actor=alice,
body={"text": "Photo", "attachment_ids": []},
headers={"Idempotency-Key": key},
)
text_key = uuid.uuid4().hex
first = c.message(
alice, chat, "Text only", key=text_key,
)["json"]
replay = c.message(
alice, chat, "Text only",
key=text_key,
attachments=[],
)["json"]
check(first == replay, "Missing attachment_ids differs from []")
print("PASS: missing attachment_ids equals []", flush=True)
print("PASS: attachment negative tests completed", flush=True)
if __name__ == "__main__":
run(main)
+240
View File
@@ -0,0 +1,240 @@
#!/usr/bin/env python3
import argparse
import hashlib
import json
import re
import subprocess
import time
import uuid
from client import Client
def check(condition, message):
if not condition:
raise RuntimeError(message)
def docker_script(script, *arguments, input_bytes=None):
command = [
"docker", "compose", "exec", "-T",
"minio",
"sh", "-ec", script,
"cleanup-check",
*arguments,
]
result = subprocess.run(
command,
input=input_bytes,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=30,
)
if result.returncode != 0:
raise RuntimeError(
"Команда MinIO завершилась ошибкой. "
"Проверь доступность minio и название bucket."
)
return result.stdout.decode("utf-8")
def configure_minio():
docker_script(
'''
mc alias set cleanupcheck http://localhost:9000 \
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
'''
)
def put_orphan(bucket, key):
docker_script(
'mc pipe "cleanupcheck/$1/$2" >/dev/null',
bucket,
key,
input_bytes=b"synthetic cleanup test object\n",
)
def list_test_objects(bucket, prefix):
output = docker_script(
'mc ls --json "cleanupcheck/$1/$2"',
bucket,
prefix,
)
names = set()
for line in output.splitlines():
if not line.strip():
continue
item = json.loads(line)
check(
item.get("status") != "error",
"MinIO вернул ошибку при получении списка объектов",
)
key = item.get("key")
if key:
names.add(key.rstrip("/").split("/")[-1])
return names
def remove_test_object(bucket, key):
docker_script(
'mc rm --force "cleanupcheck/$1/$2" >/dev/null',
bucket,
key,
)
def check_download(client, actor, attachment, original):
result = client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=actor,
)
check(
result["bytes"] == original,
"Корректное вложение потеряно или изменилось",
)
check(
hashlib.sha256(result["bytes"]).hexdigest()
== attachment["sha256"],
"SHA-256 корректного вложения не совпадает",
)
def wait_until(deadline):
remaining = deadline - time.monotonic()
if remaining > 0:
time.sleep(remaining)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--base-url",
default="http://localhost:8080",
)
parser.add_argument(
"--bucket",
default="messenger-attachments",
)
args = parser.parse_args()
check(
re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", args.bucket),
"Некорректное название bucket",
)
client = Client(args.base_url, None)
alice, bob, eve, chat = client.bootstrap()
# Создаётся через API: в S3 и PostgreSQL.
attachment, original = client.attachment(alice, chat)
check_download(client, bob, attachment, original)
configure_minio()
prefix = "originals/cleanup-test-" + uuid.uuid4().hex + "/"
old_key = prefix + "old.bin"
fresh_key = prefix + "fresh.bin"
try:
put_orphan(args.bucket, old_key)
old_created = time.monotonic()
names = list_test_objects(args.bucket, prefix)
check(
"old.bin" in names,
"Тестовый объект не появился в S3",
)
print("Created API attachment and an untracked S3 object", flush=True)
wait_until(old_created + 25)
names = list_test_objects(args.bucket, prefix)
check(
"old.bin" in names,
"Объект удалён раньше grace period",
)
print("PASS: object younger than grace period is preserved", flush=True)
wait_until(old_created + 65)
put_orphan(args.bucket, fresh_key)
fresh_created = time.monotonic()
print("Waiting for cleanup of the old object...", flush=True)
deadline = fresh_created + 30
while True:
names = list_test_objects(args.bucket, prefix)
check(
"fresh.bin" in names,
"Свежий объект удалён раньше grace period",
)
if "old.bin" not in names:
break
check(
time.monotonic() < deadline,
"Старый объект не удалён. Проверь настройки и логи очистки",
)
time.sleep(2)
print("PASS: old object without metadata is deleted", flush=True)
print("PASS: fresh object without metadata is preserved", flush=True)
client.call(
"GET",
"/api/v1/attachments/" + attachment["id"],
actor=bob,
schema="Attachment",
)
check_download(client, bob, attachment, original)
print("PASS: attachment with metadata and its bytes are preserved", flush=True)
print("PASS: cleanup scenario completed", flush=True)
finally:
for key in (old_key, fresh_key):
try:
remove_test_object(args.bucket, key)
except Exception as error:
print(
f"WARNING: final cleanup failed for {key}: {error}",
flush=True,
)
if __name__ == "__main__":
try:
main()
except Exception as error:
print("FAIL:", error, flush=True)
raise SystemExit(1)
+686
View File
@@ -0,0 +1,686 @@
import json
from pathlib import Path
import subprocess
import sys
import time
import uuid
from client import Client, arguments, check, run
ROOT = Path(__file__).resolve().parents[1]
BUCKET = None
def command(args, data=None, timeout=30):
result = subprocess.run(
args,
cwd=ROOT,
input=data,
capture_output=True,
timeout=timeout,
)
check(
result.returncode == 0,
"Command failed: " + " ".join(args[:3]),
)
return result.stdout
def sql(statement):
result = command(
[
"docker",
"compose",
"exec",
"-T",
"postgres",
"sh",
"-ec",
(
'exec psql -X -A -t -v ON_ERROR_STOP=1 '
'-U "$POSTGRES_USER" '
'-d "${POSTGRES_DB:-$POSTGRES_USER}"'
),
],
statement.encode(),
)
return result.decode().strip()
def mc(*args, data=None):
return command(
[
"docker",
"compose",
"exec",
"-T",
"minio",
"sh",
"-ec",
(
"mc alias set reconcile_test "
"http://localhost:9000 "
'"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" '
'>/dev/null; exec mc "$@"'
),
"sh",
*args,
],
data,
)
def target(key):
return "reconcile_test/" + BUCKET + "/" + key
def object_names(chat_id):
raw = mc(
"ls",
"--recursive",
"--json",
target("originals/" + chat_id + "/"),
)
names = set()
for line in raw.decode().splitlines():
if not line.strip():
continue
item = json.loads(line)
check(
item.get("status") == "success",
"Cannot list test S3 prefix",
)
if item.get("type") == "file":
names.add(
item["key"].rstrip("/").rsplit("/", 1)[-1]
)
return names
def inspect_api():
ids = command([
"docker",
"compose",
"ps",
"-q",
"api",
]).decode().split()
check(
len(ids) == 2,
"Start exactly two API replicas",
)
containers = json.loads(command([
"docker",
"inspect",
*ids,
]))
buckets = set()
for container in containers:
env = dict(
value.split("=", 1)
for value in container["Config"]["Env"]
if "=" in value
)
check(
env.get("APP_CLEANUP_ENABLED", "").lower()
== "false",
"Disable AttachmentCleanupJob "
"on both replicas first",
)
check(
bool(env.get("S3_BUCKET")),
"S3_BUCKET is missing",
)
buckets.add(env["S3_BUCKET"])
check(
len(buckets) == 1,
"API replicas use different buckets",
)
return buckets.pop()
def snapshot(chat_id):
# chat_id предварительно проверен через uuid.UUID.
return sql(
"SELECT coalesce("
"json_agg(a ORDER BY id), '[]'::json"
") FROM attachments a "
"WHERE chat_id='" + chat_id + "'::uuid;"
)
def reconcile(
chat_id,
backups,
directory,
label,
apply=False,
):
path = directory / (label + ".json")
result = subprocess.run(
[
sys.executable,
"-X",
"utf8",
str(ROOT / "scripts/reconcile.py"),
"--apply" if apply else "--dry-run",
"--chat-id",
chat_id,
"--grace-seconds",
"60",
"--max-seconds",
"120",
"--backup-dir",
str(backups),
"--output",
str(path),
],
cwd=ROOT,
capture_output=True,
text=True,
timeout=180,
)
check(
result.returncode in (0, 2),
"Reconciler failed at "
+ label
+ "; inspect "
+ str(path),
)
check(
path.exists(),
"Reconciler did not save report",
)
report = json.loads(
path.read_text(encoding="utf-8")
)
check(
report["status"] in (
"completed",
"needs_attention",
),
"Scan failed",
)
check(
report["counts"].get("busy", 0) == 0,
"Object lock was busy; repeat experiment",
)
check(
report["counts"].get("size_mismatches", 0) == 0,
"Unexpected size mismatch",
)
print(
label + ": " + json.dumps(report["counts"]),
flush=True,
)
return report
def count(report, name):
return report["counts"].get(name, 0)
def main():
global BUCKET
args = arguments(
"Lab4 reconciler: dry-run, repair "
"and repeated execution"
)
BUCKET = inspect_api()
c = Client(
args.base_url,
args.ca_file,
timeout=30,
)
alice, bob, _, chat = c.bootstrap()
chat_id = str(uuid.UUID(chat["id"]))
stable, content = c.attachment(alice, chat)
missing, missing_bytes = c.attachment(alice, chat)
message = c.message(
alice,
chat,
"Reconciler experiment",
attachments=[
stable["id"],
missing["id"],
],
)["json"]
missing_id = str(uuid.UUID(missing["id"]))
key = sql(
"SELECT object_key FROM attachments "
"WHERE id='" + missing_id + "'::uuid;"
)
check(
key.startswith("originals/" + chat_id + "/"),
"Unexpected original key",
)
prefix = "originals/" + chat_id + "/"
old_key = (
prefix + "orphan-old-" + uuid.uuid4().hex
)
fresh_key = None
fresh_started = None
# Запоминаем все созданные orphan-ключи,
# чтобы попытаться убрать их при исключении.
synthetic_keys = {old_key}
backups = (
ROOT / "backups" / ("reconcile-" + chat_id)
)
directory = (
ROOT / "results" / ("reconcile-" + chat_id)
)
backups.mkdir(parents=True, exist_ok=False)
directory.mkdir(parents=True, exist_ok=False)
metadata_before = snapshot(chat_id)
needs_restore = False
completed = False
def verify_metadata():
check(
snapshot(chat_id) == metadata_before,
"Attachment metadata changed",
)
page = c.call(
"GET",
"/api/v1/chats/"
+ chat_id
+ "/messages?limit=100",
actor=bob,
schema="MessagePage",
)["json"]
found = next(
(
item
for item in page["items"]
if item["id"] == message["id"]
),
None,
)
check(
found == message,
"Message or attachment references changed",
)
def prepare_fresh():
nonlocal fresh_key, fresh_started
# Предыдущий свежий объект больше не нужен.
# Его сохранность проверена предыдущим этапом.
if fresh_key is not None:
previous_name = fresh_key.rsplit("/", 1)[-1]
if previous_name in object_names(chat_id):
mc("rm", target(fresh_key))
fresh_key = (
prefix + "orphan-fresh-" + uuid.uuid4().hex
)
synthetic_keys.add(fresh_key)
# Начинаем отсчёт до PUT: это консервативная
# верхняя оценка возраста объекта.
fresh_started = time.monotonic()
mc(
"pipe",
target(fresh_key),
data=content,
)
def verify_fresh():
check(
fresh_key is not None
and fresh_started is not None,
"Fresh test object was not created",
)
names = object_names(chat_id)
elapsed = time.monotonic() - fresh_started
check(
elapsed < 55,
"A single reconciliation stage exceeded "
"the fresh-object timing budget; "
"repeat on an idle stand",
)
check(
fresh_key.rsplit("/", 1)[-1] in names,
"Fresh orphan was removed",
)
def download(attachment, expected):
result = c.call(
"GET",
"/api/v1/attachments/"
+ attachment["id"]
+ "/content",
actor=bob,
)
check(
result["bytes"] == expected,
"Original bytes changed",
)
try:
mc(
"pipe",
target(old_key),
data=content,
)
print(
"Old orphan created. Waiting 65 seconds "
"for the real grace period...",
flush=True,
)
time.sleep(65)
prepare_fresh()
# Байты пока доступны только тесту.
# Reconciler ещё не получает резервную копию.
needs_restore = True
mc("rm", target(key))
before = object_names(chat_id)
dry = reconcile(
chat_id,
backups,
directory,
"01-dry-run",
)
check(
count(dry, "would_delete_orphans") == 1,
"Old orphan not detected",
)
check(
count(dry, "unresolved_missing") == 1,
"Missing original not detected",
)
check(
object_names(chat_id) == before,
"Dry-run changed S3 objects",
)
verify_metadata()
verify_fresh()
download(stable, content)
print(
"PASS: dry-run found both faults "
"and changed no test data",
flush=True,
)
# Для следующего этапа создаём новый свежий объект.
prepare_fresh()
applied = reconcile(
chat_id,
backups,
directory,
"02-apply-no-backup",
apply=True,
)
check(
count(applied, "deleted_orphans") == 1,
"Old orphan not deleted",
)
check(
count(applied, "unresolved_missing") == 1,
"Missing original incorrectly repaired",
)
names = object_names(chat_id)
check(
old_key.rsplit("/", 1)[-1] not in names,
"Old orphan still exists",
)
check(
key.rsplit("/", 1)[-1] not in names,
"Missing original unexpectedly exists",
)
verify_metadata()
verify_fresh()
c.call(
"GET",
"/api/v1/attachments/"
+ missing_id
+ "/content",
expected=503,
actor=bob,
)
print(
"PASS: old orphan deleted; missing original "
"reported without metadata loss",
flush=True,
)
# Предоставляем reconciler достоверную копию.
backup_file = (
backups / (missing["sha256"] + ".bin")
)
backup_file.write_bytes(missing_bytes)
prepare_fresh()
before = object_names(chat_id)
planned = reconcile(
chat_id,
backups,
directory,
"03-dry-run-with-backup",
)
check(
count(planned, "would_restore") == 1,
"Valid backup not recognized",
)
check(
object_names(chat_id) == before,
"Dry-run restored an object",
)
verify_metadata()
verify_fresh()
print(
"PASS: dry-run plans restoration "
"without changing objects",
flush=True,
)
prepare_fresh()
restored = reconcile(
chat_id,
backups,
directory,
"04-restore",
apply=True,
)
check(
count(restored, "restored") == 1,
"Original not restored",
)
download(missing, missing_bytes)
needs_restore = False
download(stable, content)
verify_metadata()
verify_fresh()
print(
"PASS: original restored byte-for-byte; "
"used attachment and message preserved",
flush=True,
)
prepare_fresh()
repeated = reconcile(
chat_id,
backups,
directory,
"05-repeat",
apply=True,
)
for name in (
"deleted_orphans",
"restored",
"unresolved_missing",
"missing_objects",
):
check(
count(repeated, name) == 0,
"Repeated run is not clean: " + name,
)
verify_metadata()
verify_fresh()
download(stable, content)
download(missing, missing_bytes)
completed = True
print(
"PASS: repeated apply makes "
"no additional changes",
flush=True,
)
print(
"Reports: " + str(directory),
flush=True,
)
finally:
# При исключении пытаемся восстановить оригинал,
# намеренно удалённый этим тестом.
if needs_restore:
try:
mc(
"pipe",
target(key),
data=missing_bytes,
)
print(
"Test original restored "
"during final cleanup",
flush=True,
)
except Exception:
print(
"WARNING: restore test original manually "
"from backup; key=" + key,
flush=True,
)
(
backups / (missing["sha256"] + ".bin")
).write_bytes(missing_bytes)
cleanup_ok = True
# Удаляем только существующие синтетические объекты.
# Уже удалённый reconciler старый orphan — не ошибка.
for orphan in sorted(synthetic_keys):
try:
name = orphan.rsplit("/", 1)[-1]
if name in object_names(chat_id):
mc("rm", target(orphan))
except Exception:
cleanup_ok = False
print(
"WARNING: cannot remove synthetic orphan: "
+ orphan,
flush=True,
)
if completed:
check(
cleanup_ok,
"Scenario passed but final "
"test-object cleanup failed",
)
print("PASS: reconciler experiment completed")
if __name__ == "__main__":
run(main)
+561
View File
@@ -0,0 +1,561 @@
import csv
import hashlib
import json
import math
from pathlib import Path
import random
import statistics
import subprocess
import sys
import time
import uuid
from client import Client, arguments, check, run
import reconcile_check as helpers
ROOT = Path(__file__).resolve().parents[1]
def options(parser):
parser.add_argument(
"--sizes",
type=int,
nargs="+",
default=[10, 100, 1000],
)
parser.add_argument(
"--repeats",
type=int,
default=3,
)
parser.add_argument(
"--max-seconds",
type=int,
default=600,
)
def digest_metadata(chat_id):
snapshot = helpers.snapshot(chat_id)
return hashlib.sha256(
snapshot.encode()
).hexdigest()
def environment_info():
ids = helpers.command([
"docker",
"compose",
"ps",
"-q",
"api",
"postgres",
"minio",
]).decode().split()
containers = json.loads(
helpers.command([
"docker",
"inspect",
*ids,
])
)
cpu_count = int(helpers.command([
"docker",
"info",
"--format",
"{{.NCPU}}",
]))
memory_bytes = int(helpers.command([
"docker",
"info",
"--format",
"{{.MemTotal}}",
]))
worker_image = helpers.command([
"docker",
"image",
"inspect",
"--format",
"{{.Id}}",
"mtusi4-reconciler:local",
]).decode().strip()
# Сохраняем только параметры окружения,
# необходимые для воспроизводимости.
# Credentials и полный docker inspect не записываем.
services = []
for container in containers:
services.append({
"service": container["Config"]["Labels"].get(
"com.docker.compose.service"
),
"image_id": container["Image"],
"memory_limit_bytes": (
container["HostConfig"]["Memory"]
),
"nano_cpus": (
container["HostConfig"]["NanoCpus"]
),
})
return {
"docker_cpus": cpu_count,
"docker_memory_bytes": memory_bytes,
"worker_image_id": worker_image,
"worker_memory_limit_bytes": 256 * 1024 * 1024,
"worker_cpu_limit": 1,
"services": services,
}
def main():
args = arguments(
"Measure reconciler traversal cost",
options,
)
sizes = sorted(set(args.sizes))
check(
all(1 <= size <= 5000 for size in sizes),
"Sizes must be between 1 and 5000",
)
check(
1 <= args.repeats <= 10,
"Repeats must be between 1 and 10",
)
check(
args.max_seconds >= 1,
"max-seconds must be positive",
)
# Проверяет две API-реплики и отключённую очистку.
helpers.BUCKET = helpers.inspect_api()
fixture = (
ROOT / "tests/fixtures/sample.png"
).read_bytes()
fixture_sha = hashlib.sha256(fixture).hexdigest()
directory = (
ROOT
/ "results"
/ ("reconcile-cost-" + uuid.uuid4().hex)
)
backups = directory / "empty-backups"
backups.mkdir(parents=True, exist_ok=False)
report = {
"passed": False,
"hypothesis": (
"Traversal calls grow approximately linearly "
"with attachment count; wall time also "
"contains fixed overhead."
),
"sizes": sizes,
"repeats": args.repeats,
"grace_seconds": 60,
"fixture_size_bytes": len(fixture),
"fixture_sha256": fixture_sha,
"environment": environment_info(),
"groups": [],
"samples": [],
"summary": [],
"method": (
"Separate chats; API uploads; real grace wait; "
"one warm-up per size; shuffled measured rounds; "
"dry-run only."
),
}
groups = []
client = Client(
args.base_url,
args.ca_file,
timeout=30,
)
def save():
(directory / "summary.json").write_text(
json.dumps(
report,
ensure_ascii=False,
indent=2,
) + "\n",
encoding="utf-8",
)
fields = [
"attachments",
"trial",
"worker_seconds",
"launcher_seconds",
"sql_calls",
"s3_calls",
"s3_list_calls",
"s3_head_calls",
]
with (directory / "samples.csv").open(
"w",
encoding="utf-8",
newline="",
) as stream:
writer = csv.DictWriter(
stream,
fieldnames=fields,
)
writer.writeheader()
writer.writerows(report["samples"])
def scan(group, label):
path = directory / (
f'n{group["n"]}-{label}.json'
)
started = time.monotonic()
result = subprocess.run(
[
sys.executable,
"-X",
"utf8",
str(ROOT / "scripts/reconcile.py"),
"--dry-run",
"--chat-id",
group["chat_id"],
"--grace-seconds",
"60",
"--max-seconds",
str(args.max_seconds),
"--backup-dir",
str(backups),
"--output",
str(path),
],
cwd=ROOT,
capture_output=True,
text=True,
timeout=args.max_seconds + 180,
)
wall = time.monotonic() - started
check(
result.returncode == 0,
"Scan failed: " + str(path),
)
data = json.loads(
path.read_text(encoding="utf-8")
)
check(
data["status"] == "completed"
and data["mode"] == "dry-run",
"Unexpected scan status",
)
check(
data.get("chat_id") == group["chat_id"],
"Wrong scan scope",
)
counts = data["counts"]
# Каждый объект и каждая запись должны пройти
# полноценную проверку после grace period.
for field in (
"objects_scanned",
"referenced_objects",
"metadata_scanned",
"present_metadata",
):
check(
counts.get(field, 0) == group["n"],
"Incomplete scan: " + field,
)
for field in (
"fresh_objects",
"fresh_metadata",
"busy",
"missing_objects",
"unresolved_missing",
"size_mismatches",
"would_delete_orphans",
"deleted_orphans",
"restored",
):
check(
counts.get(field, 0) == 0,
"Unexpected outcome: " + field,
)
calls = data["s3_operation_calls"]
check(
calls.get("put_object", 0) == 0
and calls.get("delete_object", 0) == 0,
"Dry-run attempted mutations",
)
return {
"attachments": group["n"],
"trial": label,
"worker_seconds": data["elapsed_seconds"],
"launcher_seconds": round(wall, 3),
"sql_calls": data["sql_statement_calls"],
"s3_calls": sum(calls.values()),
"s3_list_calls": calls.get(
"list_objects_v2", 0
),
"s3_head_calls": calls.get(
"head_object", 0
),
}
save()
try:
setup_started = time.monotonic()
# Каждый размер получает отдельный чат.
for size in sizes:
alice, bob, _, chat = client.bootstrap()
chat_id = str(uuid.UUID(chat["id"]))
entry = {
"attachments": size,
"chat_id": chat_id,
"uploaded": 0,
}
report["groups"].append(entry)
save()
sample_attachments = []
# Последовательные загрузки.
# Автоматических повторов POST нет.
for index in range(size):
attachment, content = client.attachment(
alice,
chat,
)
check(
content == fixture,
"Fixture changed during experiment",
)
# Для контрольного скачивания сохраняем
# первое и последнее вложение.
if index in {0, size - 1}:
sample_attachments.append(attachment)
entry["uploaded"] = index + 1
if (
(index + 1) % 100 == 0
or index + 1 == size
):
print(
f"Preparing N={size}: "
f"{index + 1}/{size}",
flush=True,
)
save()
message = client.message(
alice,
chat,
"Reconciler cost experiment",
attachments=[
attachment["id"]
for attachment in sample_attachments
],
)["json"]
groups.append({
"n": size,
"chat_id": chat_id,
"bob": bob,
"samples": sample_attachments,
"message": message,
"metadata_digest": digest_metadata(chat_id),
})
report["setup_seconds"] = round(
time.monotonic() - setup_started,
3,
)
save()
print(
"All uploads completed. Waiting 65 seconds "
"for the real grace period...",
flush=True,
)
time.sleep(65)
# Прогрев сохраняется отдельными JSON,
# но не включается в итоговую статистику.
for group in groups:
scan(group, "warmup")
print(
f'Warm-up N={group["n"]}: complete',
flush=True,
)
# В каждом раунде меняем порядок объёмов.
# Фиксированный seed делает порядок повторяемым.
rng = random.Random(1729)
for trial in range(1, args.repeats + 1):
order = list(groups)
rng.shuffle(order)
for group in order:
sample = scan(
group,
f"run-{trial}",
)
report["samples"].append(sample)
save()
print(
f'N={group["n"]}, run={trial}: '
f'worker={sample["worker_seconds"]}s, '
f'launcher={sample["launcher_seconds"]}s, '
f'SQL={sample["sql_calls"]}, '
f'S3={sample["s3_calls"]}',
flush=True,
)
# Контроль после измерений.
for group in groups:
check(
digest_metadata(group["chat_id"])
== group["metadata_digest"],
"Metadata changed during benchmark",
)
for attachment in group["samples"]:
downloaded = client.call(
"GET",
"/api/v1/attachments/"
+ attachment["id"]
+ "/content",
actor=group["bob"],
)
check(
downloaded["bytes"] == fixture,
"Sample original changed",
)
page = client.call(
"GET",
"/api/v1/chats/"
+ group["chat_id"]
+ "/messages?limit=100",
actor=group["bob"],
schema="MessagePage",
)["json"]
check(
group["message"] in page["items"],
"Sample message changed",
)
for size in sizes:
samples = [
sample
for sample in report["samples"]
if sample["attachments"] == size
]
worker_times = [
sample["worker_seconds"]
for sample in samples
]
launcher_times = [
sample["launcher_seconds"]
for sample in samples
]
report["summary"].append({
"attachments": size,
"worker_seconds_min": min(worker_times),
"worker_seconds_median": round(
statistics.median(worker_times),
3,
),
"worker_seconds_max": max(worker_times),
"launcher_seconds_median": round(
statistics.median(launcher_times),
3,
),
"sql_calls_observed": sorted({
sample["sql_calls"]
for sample in samples
}),
"s3_calls_observed": sorted({
sample["s3_calls"]
for sample in samples
}),
"sql_calls_expected_for_current_algorithm": (
5 * size
+ math.ceil(size / 500)
+ 2
),
"s3_calls_expected_for_current_algorithm": (
size
+ math.ceil(size / 500)
+ 1
),
})
report["passed"] = True
finally:
save()
print(json.dumps(
report["summary"],
ensure_ascii=False,
indent=2,
))
print("Reports: " + str(directory))
print(
"PASS: reconciler traversal cost "
"experiment completed"
)
if __name__ == "__main__":
run(main)
+492
View File
@@ -0,0 +1,492 @@
import contextlib
import importlib.util
import io
import json
from pathlib import Path
import sys
import time
import uuid
from client import Client, Failure, arguments, check, run
import reconcile_check as helpers
ROOT = Path(__file__).resolve().parents[1]
def load_launcher():
spec = importlib.util.spec_from_file_location(
"reconciler_launcher",
ROOT / "scripts/reconcile.py",
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def state(container_id):
raw = helpers.command([
"docker",
"inspect",
"--format",
"{{json .State}}",
container_id,
])
return json.loads(raw)
def wait_container(container_id, seconds=90):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
current = state(container_id)
healthy = (
current.get("Running")
and current.get("Health", {}).get("Status")
== "healthy"
)
if healthy:
return
time.sleep(2)
raise Failure(
"Container did not become healthy: "
+ container_id[:12]
)
def wait_api(client, seconds=90):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
try:
client.call(
"GET",
"/health/ready",
schema="Health",
)
return
except Failure:
time.sleep(2)
raise Failure(
"API did not recover after dependency restart"
)
def main():
args = arguments(
"Reconciler failure checks: "
"MinIO and PostgreSQL outage"
)
helpers.BUCKET = helpers.inspect_api()
launcher = load_launcher()
saved_services = {
name: launcher.service(name)
for name in ("api", "postgres", "minio")
}
launcher.service = lambda name: saved_services[name]
c = Client(
args.base_url,
args.ca_file,
timeout=10,
)
alice, bob, _, chat = c.bootstrap()
chat_id = str(uuid.UUID(chat["id"]))
attachment, content = c.attachment(alice, chat)
message = c.message(
alice,
chat,
"Reconciler dependency failure",
attachments=[attachment["id"]],
)["json"]
original_name = str(uuid.UUID(attachment["id"]))
orphan_key = (
"originals/"
+ chat_id
+ "/orphan-failure-"
+ uuid.uuid4().hex
)
orphan_name = orphan_key.rsplit("/", 1)[-1]
metadata_before = helpers.snapshot(chat_id)
directory = (
ROOT
/ "results"
/ ("reconcile-failure-" + chat_id)
)
backups = (
ROOT
/ "backups"
/ ("reconcile-failure-" + chat_id)
)
directory.mkdir(parents=True, exist_ok=False)
backups.mkdir(parents=True, exist_ok=False)
stopped = set()
summary = []
completed = False
def worker(label, apply, expected_code):
path = directory / (label + ".json")
previous_argv = sys.argv
try:
sys.argv = [
str(ROOT / "scripts/reconcile.py"),
"--apply" if apply else "--dry-run",
"--chat-id",
chat_id,
"--grace-seconds",
"60",
"--max-seconds",
"120",
"--backup-dir",
str(backups),
"--output",
str(path),
]
with contextlib.redirect_stdout(io.StringIO()):
code = launcher.main()
finally:
sys.argv = previous_argv
check(
code == expected_code,
"Unexpected return code at " + label,
)
report = json.loads(
path.read_text(encoding="utf-8")
)
summary.append({
"stage": label,
"status": report["status"],
"elapsed_seconds": report["elapsed_seconds"],
"error_type": report["error_type"],
})
print(
label
+ ": status="
+ report["status"]
+ ", error_type="
+ str(report["error_type"]),
flush=True,
)
return report
def verify_data(expect_orphan):
check(
helpers.snapshot(chat_id) == metadata_before,
"Attachment metadata changed",
)
names = helpers.object_names(chat_id)
expected = {original_name}
if expect_orphan:
expected.add(orphan_name)
check(
names == expected,
"Unexpected S3 object set",
)
result = c.call(
"GET",
"/api/v1/attachments/"
+ attachment["id"]
+ "/content",
actor=bob,
)
check(
result["bytes"] == content,
"Confirmed original changed",
)
page = c.call(
"GET",
"/api/v1/chats/"
+ chat_id
+ "/messages?limit=100",
actor=bob,
schema="MessagePage",
)["json"]
check(
message in page["items"],
"Message or attachment references changed",
)
def restore(name):
container_id = saved_services[name]["Id"]
helpers.command(
[
"docker",
"start",
container_id,
],
timeout=60,
)
wait_container(container_id)
stopped.discard(name)
wait_api(c)
try:
helpers.mc(
"pipe",
helpers.target(orphan_key),
data=content,
)
print(
"Waiting 65 seconds "
"for the real orphan grace period...",
flush=True,
)
time.sleep(65)
baseline = worker(
"01-baseline",
apply=False,
expected_code=0,
)
check(
baseline["counts"].get(
"would_delete_orphans", 0
) == 1,
"No deletion candidate in baseline",
)
verify_data(expect_orphan=True)
for number, name in enumerate(
("minio", "postgres"),
start=2,
):
container_id = saved_services[name]["Id"]
print(
"Stopping "
+ name
+ " for the experiment",
flush=True,
)
stopped.add(name)
helpers.command(
[
"docker",
"stop",
"-t",
"10",
container_id,
],
timeout=60,
)
check(
not state(container_id).get("Running"),
"Dependency did not stop",
)
try:
report = worker(
f"{number:02d}-{name}-unavailable",
apply=True,
expected_code=1,
)
check(
report["status"] == "failed",
"Outage was not reported as failed",
)
check(
report["error_type"] is not None,
"Missing error type",
)
for operation in (
"delete_object",
"put_object",
):
check(
report["s3_operation_calls"].get(
operation, 0
) == 0,
"Mutation attempted "
"during dependency outage",
)
for counter in (
"missing_objects",
"unresolved_missing",
"deleted_orphans",
"restored",
):
check(
report["counts"].get(
counter, 0
) == 0,
"Outage was confused "
"with a data inconsistency",
)
finally:
restore(name)
verify_data(expect_orphan=True)
print(
"PASS: "
+ name
+ " outage aborted reconciliation; "
"data and session survived",
flush=True,
)
repaired = worker(
"04-recovery-apply",
apply=True,
expected_code=0,
)
check(
repaired["counts"].get(
"deleted_orphans", 0
) == 1,
"Orphan was not removed after recovery",
)
verify_data(expect_orphan=False)
repeated = worker(
"05-recovery-repeat",
apply=True,
expected_code=0,
)
for counter in (
"deleted_orphans",
"restored",
"missing_objects",
"unresolved_missing",
):
check(
repeated["counts"].get(
counter, 0
) == 0,
"Repeated run is not clean",
)
verify_data(expect_orphan=False)
completed = True
finally:
recovery_errors = []
for name in tuple(stopped):
try:
restore(name)
except Exception:
recovery_errors.append(name)
print(
"WARNING: could not restore service: "
+ name,
flush=True,
)
if not recovery_errors:
try:
if orphan_name in helpers.object_names(chat_id):
helpers.mc(
"rm",
helpers.target(orphan_key),
)
except Exception:
recovery_errors.append(
"test orphan cleanup"
)
print(
"WARNING: cannot remove synthetic orphan: "
+ orphan_key,
flush=True,
)
check(
not recovery_errors,
"Recovery incomplete: "
+ ", ".join(recovery_errors),
)
if completed:
report = {
"passed": True,
"chat_id": chat_id,
"stages": summary,
}
(directory / "summary.json").write_text(
json.dumps(
report,
ensure_ascii=False,
indent=2,
) + "\n",
encoding="utf-8",
)
print(
"PASS: reconciliation resumes after recovery "
"and repeated apply changes nothing"
)
print("Reports: " + str(directory))
print(
"PASS: reconciler dependency failure "
"scenario completed"
)
if __name__ == "__main__":
run(main)
+273
View File
@@ -0,0 +1,273 @@
#!/usr/bin/env python3
import hashlib
import re
import socket
import subprocess
import time
from pathlib import Path
from urllib.parse import urlsplit
from client import Client, arguments, check, run
FIXTURE = (
Path(__file__).resolve().parent
/ "fixtures"
/ "sample.png"
)
def command_output(command):
result = subprocess.run(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
timeout=10,
)
check(
result.returncode == 0,
"Docker command failed; check that API containers are running",
)
return result.stdout.strip()
def api_containers():
output = command_output([
"docker", "compose", "ps", "-q", "api",
])
containers = output.splitlines()
check(
len(containers) == 2,
"This test requires exactly two running API replicas",
)
for container in containers:
check(
re.fullmatch(r"[a-f0-9]{12,64}", container) is not None,
"Unexpected container ID",
)
return containers
def temporary_files(containers):
result = set()
for container in containers:
output = command_output([
"docker", "exec", container,
"sh", "-ec",
'''
directory="${UPLOAD_TEMP_DIR:-/tmp/messenger-uploads}"
if [ -d "$directory" ]; then
find "$directory" \
-maxdepth 1 \
-type f \
-name 'upload-*.tmp' \
-print
fi
''',
])
for filename in output.splitlines():
if filename:
result.add((container, filename))
return result
def wait_for_empty_directory(containers, seconds=25):
deadline = time.monotonic() + seconds
while True:
remaining = temporary_files(containers)
if not remaining:
return
check(
time.monotonic() < deadline,
"Temporary upload files were not removed",
)
time.sleep(0.5)
def main():
args = arguments("Interrupted upload cleanup check")
client = Client(
args.base_url,
args.ca_file,
timeout=30,
)
address = urlsplit(client.base)
check(
address.scheme == "http",
"This local test expects an HTTP endpoint",
)
containers = api_containers()
check(
not temporary_files(containers),
"Temporary directory is not empty. "
"Finish other uploads and inspect leftover files first",
)
alice, bob, eve, chat = client.bootstrap()
original = FIXTURE.read_bytes()
check(
len(original) > 32768,
"Fixture must be larger than the partial upload",
)
path = (
address.path.rstrip("/")
+ "/api/v1/chats/"
+ chat["id"]
+ "/attachments?filename=interrupted.png"
)
headers = (
f"POST {path} HTTP/1.1\r\n"
f"Host: {address.netloc}\r\n"
f"Authorization: Bearer {alice['token']}\r\n"
"Content-Type: image/png\r\n"
f"Content-Length: {len(original)}\r\n"
"Connection: close\r\n"
"\r\n"
).encode("ascii")
connection = socket.create_connection(
(address.hostname, address.port or 80),
timeout=10,
)
observed = False
try:
connection.sendall(headers)
connection.sendall(original[:32768])
print(
"Partial body sent; waiting for a temporary file",
flush=True,
)
deadline = time.monotonic() + 5
while time.monotonic() < deadline:
if temporary_files(containers):
observed = True
break
time.sleep(0.2)
check(
observed,
"Temporary file was not observed. "
"Check API logs and proxy timeouts",
)
print(
"PASS: API started receiving the incomplete upload",
flush=True,
)
finally:
# Закрываем соединение до передачи остальных байтов.
try:
connection.shutdown(socket.SHUT_RDWR)
except OSError:
pass
connection.close()
print("Client connection closed before upload completion", flush=True)
wait_for_empty_directory(containers)
print(
"PASS: temporary file removed after interrupted upload",
flush=True,
)
# Проверяем, что разрешение UploadLimiter не осталось занятым.
# Последовательными запросами через LB наблюдаем обе реплики.
instances = set()
last_attachment = None
for _ in range(20):
response = client.call(
"POST",
"/api/v1/chats/" + chat["id"]
+ "/attachments?filename=after-abort.png",
201,
actor=alice,
raw=original,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
attachment = response["json"]
check(
attachment["status"] == "ready",
"Upload did not become ready",
)
check(
attachment["sha256"]
== hashlib.sha256(original).hexdigest(),
"Uploaded checksum mismatch",
)
instances.add(response["headers"]["X-Instance-Id"])
last_attachment = attachment
if len(instances) >= 2:
break
check(
len(instances) == 2,
"Successful uploads were not observed on both API replicas",
)
print(
"PASS: both API replicas accept uploads after cancellation",
flush=True,
)
downloaded = client.call(
"GET",
"/api/v1/attachments/"
+ last_attachment["id"]
+ "/content",
actor=bob,
)
check(
downloaded["bytes"] == original,
"Downloaded file differs from the original",
)
wait_for_empty_directory(containers)
print("PASS: subsequent upload is intact", flush=True)
print("PASS: interrupted upload scenario completed", flush=True)
if __name__ == "__main__":
run(main)
+249
View File
@@ -0,0 +1,249 @@
#!/usr/bin/env python3
import socket
import subprocess
import time
from urllib.parse import urlsplit
from client import Client, arguments, check, run
from upload_abort_check import (
FIXTURE,
api_containers,
command_output,
temporary_files,
)
def container_running(container):
value = command_output([
"docker", "inspect",
"--format", "{{.State.Running}}",
container,
])
return value == "true"
def container_healthy(container):
value = command_output([
"docker", "inspect",
"--format", "{{.State.Health.Status}}",
container,
])
return value == "healthy"
def main():
args = arguments("API crash during an incomplete upload")
client = Client(
args.base_url,
args.ca_file,
timeout=30,
)
address = urlsplit(client.base)
check(
address.scheme == "http",
"This local test expects HTTP",
)
containers = api_containers()
check(
not temporary_files(containers),
"Temporary directories must be empty before the test",
)
alice, bob, eve, chat = client.bootstrap()
good_attachment, original = client.attachment(alice, chat)
content = FIXTURE.read_bytes()
check(
len(content) > 32768,
"Fixture is too small",
)
path = (
address.path.rstrip("/")
+ "/api/v1/chats/"
+ chat["id"]
+ "/attachments?filename=crash-test.png"
)
headers = (
f"POST {path} HTTP/1.1\r\n"
f"Host: {address.netloc}\r\n"
f"Authorization: Bearer {alice['token']}\r\n"
"Content-Type: image/png\r\n"
f"Content-Length: {len(content)}\r\n"
"Connection: close\r\n"
"\r\n"
).encode("ascii")
connection = None
target = None
temporary_path = None
try:
connection = socket.create_connection(
(address.hostname, address.port or 80),
timeout=10,
)
connection.sendall(headers)
connection.sendall(content[:32768])
print("Partial upload started", flush=True)
deadline = time.monotonic() + 5
while True:
found = temporary_files(containers)
if found:
check(
len(found) == 1,
"Expected exactly one active temporary file",
)
target, temporary_path = next(iter(found))
break
check(
time.monotonic() < deadline,
"Temporary file was not observed",
)
time.sleep(0.2)
command_output([
"docker", "exec", target,
"touch", "-t", "200001010000",
temporary_path,
])
print(
"Temporary file found; test timestamp set to the past",
flush=True,
)
command_output([
"docker", "kill",
"--signal", "KILL",
target,
])
check(
not container_running(target),
"Target API container did not stop",
)
connection.close()
connection = None
copied = subprocess.run(
[
"docker", "cp",
target + ":" + temporary_path,
"-",
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=10,
)
check(
copied.returncode == 0,
"Temporary file did not survive the process crash",
)
print(
"PASS: temporary file remains after SIGKILL",
flush=True,
)
command_output([
"docker", "start", target,
])
ready_deadline = time.monotonic() + 120
while not container_healthy(target):
check(
container_running(target),
"API exited after restart",
)
check(
time.monotonic() < ready_deadline,
"API did not become healthy after restart",
)
time.sleep(2)
print(
"PASS: the same API container recovered",
flush=True,
)
print(
"Waiting for temporary-file cleanup...",
flush=True,
)
cleanup_deadline = time.monotonic() + 120
while (target, temporary_path) in temporary_files([target]):
check(
time.monotonic() < cleanup_deadline,
"Old temporary file was not removed",
)
time.sleep(2)
print(
"PASS: stale temporary file removed after API restart",
flush=True,
)
downloaded = client.call(
"GET",
"/api/v1/attachments/"
+ good_attachment["id"]
+ "/content",
actor=bob,
)
check(
downloaded["bytes"] == original,
"Confirmed attachment was lost or changed",
)
print(
"PASS: existing session and confirmed attachment survived",
flush=True,
)
client.attachment(alice, chat)
print("PASS: subsequent upload succeeds", flush=True)
print("PASS: API crash scenario completed", flush=True)
finally:
if connection is not None:
connection.close()
if target is not None and not container_running(target):
command_output([
"docker", "start", target,
])
if __name__ == "__main__":
run(main)
+278
View File
@@ -0,0 +1,278 @@
#!/usr/bin/env python3
import json
import re
import subprocess
import time
import uuid
from urllib.parse import quote
from client import Client, arguments, check, run
def execute(command, input_text=None):
result = subprocess.run(
command,
input=input_text,
capture_output=True,
text=True,
timeout=30,
)
check(
result.returncode == 0,
"Infrastructure command failed. "
"Check PostgreSQL, MinIO and the configured bucket",
)
return result.stdout.strip()
def sql(query):
return execute(
[
"docker", "compose", "exec", "-T",
"postgres", "sh", "-ec",
'''
exec psql -X \
-v ON_ERROR_STOP=1 \
-U "$POSTGRES_USER" \
-d "$POSTGRES_DB" \
-At
''',
],
input_text=query,
)
def minio(script, *arguments):
return execute([
"docker", "compose", "exec", "-T",
"minio", "sh", "-ec",
script,
"failure-check",
*arguments,
])
def configure_minio():
minio(
'''
mc alias set failurecheck http://localhost:9000 \
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
'''
)
def list_objects(bucket, prefix):
output = minio(
'mc ls --json "failurecheck/$1/$2"',
bucket,
prefix,
)
objects = set()
for line in output.splitlines():
if not line.strip():
continue
item = json.loads(line)
check(
item.get("status") != "error",
"MinIO failed to list test objects",
)
if item.get("key"):
objects.add(item["key"])
return objects
def wait_until(deadline):
remaining = deadline - time.monotonic()
if remaining > 0:
time.sleep(remaining)
def main():
def configure(parser):
parser.add_argument(
"--bucket",
default="messenger-attachments",
)
args = arguments(
"Failure between S3 upload and metadata INSERT",
configure,
)
check(
re.fullmatch(
r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]",
args.bucket,
) is not None,
"Invalid bucket name",
)
client = Client(
args.base_url,
args.ca_file,
timeout=40,
)
alice, bob, eve, chat = client.bootstrap()
chat_id = str(uuid.UUID(chat["id"]))
configure_minio()
good_attachment, original = client.attachment(alice, chat)
prefix = f"originals/{chat_id}/"
before = list_objects(args.bucket, prefix)
check(
len(before) == 1,
"Expected one control object in the new test chat",
)
marker = uuid.uuid4().hex
filename = f"db-failure-{marker}.png"
function_name = f"test_attachment_failure_{marker}"
trigger_name = f"test_attachment_failure_{marker}"
cleanup_sql = f"""
DROP TRIGGER IF EXISTS {trigger_name}
ON public.attachments;
DROP FUNCTION IF EXISTS public.{function_name}();
"""
try:
sql(f"""
CREATE FUNCTION public.{function_name}()
RETURNS trigger
LANGUAGE plpgsql
AS $$
BEGIN
RAISE EXCEPTION
'Synthetic attachment metadata failure';
END;
$$;
CREATE TRIGGER {trigger_name}
BEFORE INSERT ON public.attachments
FOR EACH ROW
WHEN (NEW.filename = '{filename}')
EXECUTE FUNCTION public.{function_name}();
""")
print(
"Installed a DB failure for one synthetic filename",
flush=True,
)
response = client.call(
"POST",
"/api/v1/chats/" + chat_id
+ "/attachments?filename="
+ quote(filename, safe=""),
503,
actor=alice,
raw=original,
headers={"Content-Type": "image/png"},
)
check(
response["json"]["error"]["code"] == "unavailable",
"Expected unavailable error code",
)
print("PASS: failed metadata INSERT returns 503", flush=True)
sql(cleanup_sql)
count = sql(f"""
SELECT count(*)
FROM public.attachments
WHERE chat_id = '{chat_id}'::uuid
AND filename = '{filename}';
""")
check(
count == "0",
"Metadata was unexpectedly committed",
)
after = list_objects(args.bucket, prefix)
orphan_objects = after - before
check(
len(orphan_objects) == 1,
"Expected exactly one S3 object without metadata",
)
orphan = next(iter(orphan_objects))
observed_at = time.monotonic()
print(
"PASS: S3 object exists, but metadata was rolled back",
flush=True,
)
wait_until(observed_at + 25)
check(
orphan in list_objects(args.bucket, prefix),
"Orphan was deleted before the grace period",
)
print("PASS: grace period protects the recent object", flush=True)
print("Waiting for scheduled cleanup...", flush=True)
deadline = observed_at + 100
while orphan in list_objects(args.bucket, prefix):
check(
time.monotonic() < deadline,
"Orphan was not deleted. Check cleanup settings and logs",
)
time.sleep(2)
print(
"PASS: orphan created by a real failed INSERT was deleted",
flush=True,
)
downloaded = client.call(
"GET",
"/api/v1/attachments/"
+ good_attachment["id"]
+ "/content",
actor=bob,
)
check(
downloaded["bytes"] == original,
"Control attachment was lost or modified",
)
print("PASS: control attachment is intact", flush=True)
client.attachment(alice, chat)
print("PASS: normal uploads work after the failure", flush=True)
print("PASS: S3/DB failure scenario completed", flush=True)
finally:
sql(cleanup_sql)
if __name__ == "__main__":
run(main)
+495
View File
@@ -0,0 +1,495 @@
#!/usr/bin/env python3
import hashlib
import json
import subprocess
import threading
import time
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from attachments_negative import make_png
from client import Client, arguments, check, run
def docker_output(command):
result = subprocess.run(
command,
capture_output=True,
text=True,
timeout=10,
)
check(
result.returncode == 0,
"Docker command failed while measuring memory",
)
return result.stdout.strip()
def get_api_containers():
output = docker_output([
"docker", "compose", "ps", "-q", "api",
])
containers = output.splitlines()
check(
len(containers) == 2,
"Expected exactly two running API replicas",
)
return containers
def memory_snapshot(container):
# Docker Desktop с cgroup v2.
# memory.current включает память всего контейнера,
# а не только Java heap.
output = docker_output([
"docker", "exec", container,
"sh", "-ec",
'''
cat /sys/fs/cgroup/memory.current
if [ -r /sys/fs/cgroup/memory.peak ]; then
cat /sys/fs/cgroup/memory.peak
else
echo unavailable
fi
cat /sys/fs/cgroup/memory.events
''',
])
lines = output.splitlines()
check(
len(lines) >= 2,
"Cannot read cgroup v2 memory statistics",
)
events = {}
for line in lines[2:]:
key, value = line.split()
events[key] = int(value)
return {
"current_bytes": int(lines[0]),
"lifetime_peak_bytes": (
int(lines[1])
if lines[1].isdigit()
else None
),
"oom": events.get("oom", 0),
"oom_kill": events.get("oom_kill", 0),
}
def mib(value):
if value is None:
return None
return round(value / 1024 / 1024, 2)
def percentile(values, fraction):
if not values:
return None
ordered = sorted(values)
index = int((len(ordered) - 1) * fraction)
return round(ordered[index], 2)
def main():
def configure(parser):
parser.add_argument("--concurrency", type=int, default=10)
parser.add_argument("--duration", type=int, default=30)
parser.add_argument(
"--output",
default="results/upload-memory.json",
)
args = arguments(
"Concurrent image uploads and API memory measurement",
configure,
)
check(
args.concurrency >= 10,
"Use at least 10 concurrent clients",
)
check(
args.duration >= 10,
"Duration must be at least 10 seconds",
)
controller = Client(
args.base_url,
args.ca_file,
timeout=40,
)
alice, bob, eve, chat = controller.bootstrap()
containers = get_api_containers()
print("Generating PNG 6000x6000...", flush=True)
image = make_png(6000, 6000)
checksum = hashlib.sha256(image).hexdigest()
check(
len(image) <= 10 * 1024 * 1024,
"Generated image exceeds upload byte limit",
)
print(
f"Image: {len(image)} bytes, 36000000 pixels",
flush=True,
)
path = (
"/api/v1/chats/"
+ chat["id"]
+ "/attachments?filename=memory-test.png"
)
clients = [
Client(args.base_url, args.ca_file, timeout=40)
for _ in range(args.concurrency)
]
baseline = {
container: memory_snapshot(container)
for container in containers
}
samples = {container: [] for container in containers}
monitor_errors = []
stop_monitor = threading.Event()
started = time.monotonic()
def monitor():
while not stop_monitor.is_set():
for container in containers:
try:
snapshot = memory_snapshot(container)
samples[container].append({
"elapsed_seconds": round(
time.monotonic() - started,
3,
),
**snapshot,
})
except Exception as error:
monitor_errors.append(
type(error).__name__
)
stop_monitor.set()
return
stop_monitor.wait(0.5)
barrier = threading.Barrier(args.concurrency + 1)
def worker(index):
client = clients[index]
counters = Counter()
latencies = []
errors = []
instances = set()
successful_instances = set()
first_success = None
barrier.wait()
while time.monotonic() - started < args.duration:
request_started = time.monotonic()
try:
response = client.call(
"POST",
path,
expected=[201, 429],
actor=alice,
raw=image,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
instance = response["headers"]["X-Instance-Id"]
instances.add(instance)
if response["status"] == 429:
check(
response["json"]["error"]["code"]
== "rate_limited",
"Unexpected error code for 429",
)
counters["rate_limited"] += 1
else:
attachment = response["json"]
check(
attachment["status"] == "ready",
"Successful upload is not ready",
)
check(
attachment["sha256"] == checksum,
"Uploaded SHA-256 mismatch",
)
check(
attachment["size_bytes"] == len(image),
"Uploaded size mismatch",
)
counters["successful"] += 1
successful_instances.add(instance)
latencies.append(
(time.monotonic() - request_started) * 1000
)
if first_success is None:
first_success = attachment
except Exception as error:
counters["unexpected_errors"] += 1
if len(errors) < 3:
errors.append(str(error))
time.sleep(0.1)
return {
"counters": dict(counters),
"latencies": latencies,
"errors": errors,
"instances": instances,
"successful_instances": successful_instances,
"first_success": first_success,
}
monitor_thread = threading.Thread(
target=monitor,
daemon=True,
)
with ThreadPoolExecutor(
max_workers=args.concurrency
) as executor:
futures = [
executor.submit(worker, index)
for index in range(args.concurrency)
]
started = time.monotonic()
monitor_thread.start()
print(
f"Starting {args.concurrency} clients "
f"for {args.duration} seconds...",
flush=True,
)
barrier.wait()
worker_results = [
future.result()
for future in futures
]
elapsed = time.monotonic() - started
stop_monitor.set()
monitor_thread.join(timeout=25)
check(
not monitor_thread.is_alive(),
"Memory monitor did not stop",
)
final = {
container: memory_snapshot(container)
for container in containers
}
totals = Counter()
latencies = []
errors = []
instances = set()
successful_instances = set()
for result in worker_results:
totals.update(result["counters"])
latencies.extend(result["latencies"])
instances.update(result["instances"])
successful_instances.update(
result["successful_instances"]
)
errors.extend(result["errors"])
download_errors = []
for result in worker_results:
attachment = result["first_success"]
if attachment is None:
continue
try:
response = controller.call(
"GET",
"/api/v1/attachments/"
+ attachment["id"]
+ "/content",
actor=bob,
)
check(
response["bytes"] == image,
"Downloaded image differs from the original",
)
except Exception as error:
download_errors.append(str(error))
memory_report = {}
for container in containers:
measurements = [
baseline[container]["current_bytes"],
final[container]["current_bytes"],
]
measurements.extend(
sample["current_bytes"]
for sample in samples[container]
)
memory_report[container[:12]] = {
"baseline_mib": mib(
baseline[container]["current_bytes"]
),
"observed_peak_mib": mib(max(measurements)),
"final_mib": mib(
final[container]["current_bytes"]
),
"lifetime_peak_before_mib": mib(
baseline[container]["lifetime_peak_bytes"]
),
"lifetime_peak_after_mib": mib(
final[container]["lifetime_peak_bytes"]
),
"oom_delta": (
final[container]["oom"]
- baseline[container]["oom"]
),
"oom_kill_delta": (
final[container]["oom_kill"]
- baseline[container]["oom_kill"]
),
"sample_count": len(samples[container]),
}
success = totals["successful"]
limited = totals["rate_limited"]
unexpected = totals["unexpected_errors"]
passed = (
success > 0
and unexpected == 0
and not monitor_errors
and not download_errors
and len(successful_instances) >= 2
and all(
item["sample_count"] > 0
and item["oom_delta"] == 0
and item["oom_kill_delta"] == 0
for item in memory_report.values()
)
)
report = {
"passed": passed,
"workload": "closed-loop image uploads; no automatic retries",
"concurrency": args.concurrency,
"requested_duration_seconds": args.duration,
"elapsed_seconds": round(elapsed, 3),
"image": {
"width": 6000,
"height": 6000,
"pixels": 36_000_000,
"size_bytes": len(image),
"sha256": checksum,
},
"successful_uploads": success,
"rate_limited_requests": limited,
"unexpected_errors": unexpected,
"successful_rps": round(success / elapsed, 3),
"observed_instances": sorted(instances),
"successful_instances": sorted(successful_instances),
"successful_latency_ms": {
"p50": percentile(latencies, 0.50),
"p95": percentile(latencies, 0.95),
"p99": percentile(latencies, 0.99),
},
"memory": memory_report,
"error_examples": errors[:10],
"monitor_errors": monitor_errors,
"download_errors": download_errors,
"notes": [
"429 responses are admission-control rejections, not successful uploads.",
"Observed memory peak is sampled and may miss short spikes.",
"Lifetime cgroup peak includes earlier container activity and startup.",
"Container memory includes more than Java heap.",
"Concurrency means concurrent clients, not 10 simultaneous decoders.",
],
}
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
saved_report = {
**report,
"memory_samples": samples,
}
output.write_text(
json.dumps(saved_report, ensure_ascii=False, indent=2),
encoding="utf-8",
)
print(
json.dumps(report, ensure_ascii=False, indent=2),
flush=True,
)
print("Report saved to:", output, flush=True)
check(
passed,
"Experiment detected errors; inspect the saved report",
)
print("PASS: concurrent upload experiment completed", flush=True)
if __name__ == "__main__":
run(main)