forked from mtusi2026/mtusi2026lab4
all for lab4
This commit is contained in:
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import struct
|
||||
import uuid
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
|
||||
FIXTURE = (
|
||||
Path(__file__).resolve().parent
|
||||
/ "fixtures"
|
||||
/ "sample.png"
|
||||
)
|
||||
|
||||
|
||||
def make_png(width, height, noisy=False):
|
||||
"""Отдельное изображение для проверки лимита байтов или пикселей."""
|
||||
|
||||
def chunk(kind, data):
|
||||
return (
|
||||
struct.pack(">I", len(data))
|
||||
+ kind
|
||||
+ data
|
||||
+ struct.pack(
|
||||
">I",
|
||||
zlib.crc32(kind + data) & 0xFFFFFFFF,
|
||||
)
|
||||
)
|
||||
|
||||
compressor = zlib.compressobj(level=1)
|
||||
compressed = bytearray()
|
||||
|
||||
for _ in range(height):
|
||||
row = os.urandom(width) if noisy else bytes(width)
|
||||
compressed.extend(compressor.compress(b"\x00" + row))
|
||||
|
||||
compressed.extend(compressor.flush())
|
||||
|
||||
header = struct.pack(
|
||||
">IIBBBBB",
|
||||
width, height,
|
||||
8, 0, 0, 0, 0,
|
||||
)
|
||||
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", header)
|
||||
+ chunk(b"IDAT", bytes(compressed))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Attachment negative tests")
|
||||
c = Client(args.base_url, args.ca_file, timeout=30)
|
||||
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
chat_path = "/api/v1/chats/" + chat["id"]
|
||||
|
||||
original = FIXTURE.read_bytes()
|
||||
|
||||
def expect_error(label, method, path, status, code, **kwargs):
|
||||
response = c.call(method, path, status, **kwargs)
|
||||
|
||||
check(
|
||||
response["json"]["error"]["code"] == code,
|
||||
label + ": incorrect error code",
|
||||
)
|
||||
|
||||
print("PASS:", label, flush=True)
|
||||
|
||||
def reject_upload(
|
||||
label,
|
||||
content,
|
||||
status,
|
||||
code,
|
||||
mime="image/png",
|
||||
filename="sample.png",
|
||||
actor=None,
|
||||
):
|
||||
expect_error(
|
||||
label,
|
||||
"POST",
|
||||
chat_path + "/attachments?filename="
|
||||
+ quote(filename, safe=""),
|
||||
status,
|
||||
code,
|
||||
actor=alice if actor is None else actor,
|
||||
raw=content,
|
||||
headers={"Content-Type": mime},
|
||||
)
|
||||
|
||||
def reject_message(label, target_chat, ids, status, code):
|
||||
expect_error(
|
||||
label,
|
||||
"POST",
|
||||
"/api/v1/chats/" + target_chat["id"] + "/messages",
|
||||
status,
|
||||
code,
|
||||
actor=alice,
|
||||
body={"text": "", "attachment_ids": ids},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
|
||||
# 1. Контрольная загрузка исходного sample.png.
|
||||
attachment, uploaded = c.attachment(alice, chat)
|
||||
|
||||
check(uploaded == original, "Unexpected fixture content")
|
||||
|
||||
downloaded = c.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=bob,
|
||||
)
|
||||
|
||||
check(downloaded["bytes"] == original, "Original bytes changed")
|
||||
check(
|
||||
attachment["sha256"] == hashlib.sha256(original).hexdigest(),
|
||||
"SHA-256 mismatch",
|
||||
)
|
||||
|
||||
print("PASS: original PNG upload and download", flush=True)
|
||||
|
||||
# 2. Настоящий PNG больше 10 MiB, но меньше 40 млн пикселей.
|
||||
large = make_png(4096, 2700, noisy=True)
|
||||
check(len(large) > MAX_BYTES, "Fixture must exceed 10 MiB")
|
||||
|
||||
reject_upload(
|
||||
"file larger than 10 MiB",
|
||||
large,
|
||||
413,
|
||||
"payload_too_large",
|
||||
)
|
||||
del large
|
||||
|
||||
# 3. Небольшой по байтам PNG с 40 960 000 пикселями.
|
||||
many_pixels = make_png(6400, 6400)
|
||||
check(len(many_pixels) < MAX_BYTES, "Fixture exceeds byte limit")
|
||||
|
||||
reject_upload(
|
||||
"image larger than 40 million pixels",
|
||||
many_pixels,
|
||||
413,
|
||||
"payload_too_large",
|
||||
)
|
||||
del many_pixels
|
||||
|
||||
# 4. Повреждение создаётся в памяти, sample.png не меняется.
|
||||
reject_upload(
|
||||
"truncated PNG",
|
||||
original[:len(original) // 2],
|
||||
415,
|
||||
"unsupported_media_type",
|
||||
)
|
||||
|
||||
reject_upload(
|
||||
"text disguised as PNG",
|
||||
b"This is not an image",
|
||||
415,
|
||||
"unsupported_media_type",
|
||||
)
|
||||
|
||||
# 5. Реальный PNG объявлен как JPEG.
|
||||
reject_upload(
|
||||
"MIME spoofing",
|
||||
original,
|
||||
415,
|
||||
"unsupported_media_type",
|
||||
mime="image/jpeg",
|
||||
filename="fake.jpg",
|
||||
)
|
||||
|
||||
reject_upload(
|
||||
"empty file",
|
||||
b"",
|
||||
415,
|
||||
"unsupported_media_type",
|
||||
)
|
||||
|
||||
# 6. Опасные и недопустимые имена файлов.
|
||||
for filename in (
|
||||
"../sample.png",
|
||||
"folder\\sample.png",
|
||||
"bad\x00name.png",
|
||||
"bad\nname.png",
|
||||
"",
|
||||
"x" * 256,
|
||||
):
|
||||
reject_upload(
|
||||
"invalid filename " + repr(filename),
|
||||
original,
|
||||
400,
|
||||
"invalid_request",
|
||||
filename=filename,
|
||||
)
|
||||
|
||||
# 7. Посторонний не может читать метаданные и оригинал.
|
||||
for suffix in ("", "/content"):
|
||||
expect_error(
|
||||
"outsider GET " + (suffix or "metadata"),
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + suffix,
|
||||
404,
|
||||
"not_found",
|
||||
actor=eve,
|
||||
)
|
||||
|
||||
reject_upload(
|
||||
"outsider upload",
|
||||
original,
|
||||
404,
|
||||
"not_found",
|
||||
actor=eve,
|
||||
)
|
||||
|
||||
# 8. Участник чата может прикрепить файл другого участника.
|
||||
message = c.message(
|
||||
bob,
|
||||
chat,
|
||||
"",
|
||||
attachments=[attachment["id"]],
|
||||
)["json"]
|
||||
|
||||
check(
|
||||
message["attachment_ids"] == [attachment["id"]],
|
||||
"Chat member could not attach the file",
|
||||
)
|
||||
print("PASS: chat member can attach the file", flush=True)
|
||||
|
||||
# 9. Alice состоит в обоих чатах, но перенос вложения запрещён.
|
||||
second_chat = c.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={
|
||||
"title": "Second test chat",
|
||||
"member_ids": [alice["id"], bob["id"]],
|
||||
},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
|
||||
reject_message(
|
||||
"attachment from another chat",
|
||||
second_chat,
|
||||
[attachment["id"]],
|
||||
404,
|
||||
"not_found",
|
||||
)
|
||||
|
||||
reject_message(
|
||||
"unknown attachment",
|
||||
chat,
|
||||
[str(uuid.uuid4())],
|
||||
404,
|
||||
"not_found",
|
||||
)
|
||||
|
||||
reject_message(
|
||||
"duplicate attachment IDs",
|
||||
chat,
|
||||
[attachment["id"], attachment["id"]],
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
|
||||
# 10. Идемпотентность учитывает вложения.
|
||||
key = uuid.uuid4().hex
|
||||
|
||||
first = c.message(
|
||||
alice, chat, "Photo",
|
||||
key=key,
|
||||
attachments=[attachment["id"]],
|
||||
)["json"]
|
||||
|
||||
replay = c.message(
|
||||
alice, chat, "Photo",
|
||||
key=key,
|
||||
attachments=[attachment["id"]],
|
||||
)["json"]
|
||||
|
||||
check(first == replay, "Idempotent replay differs")
|
||||
|
||||
expect_error(
|
||||
"same key with different attachments",
|
||||
"POST",
|
||||
chat_path + "/messages",
|
||||
409,
|
||||
"conflict",
|
||||
actor=alice,
|
||||
body={"text": "Photo", "attachment_ids": []},
|
||||
headers={"Idempotency-Key": key},
|
||||
)
|
||||
|
||||
text_key = uuid.uuid4().hex
|
||||
|
||||
first = c.message(
|
||||
alice, chat, "Text only", key=text_key,
|
||||
)["json"]
|
||||
|
||||
replay = c.message(
|
||||
alice, chat, "Text only",
|
||||
key=text_key,
|
||||
attachments=[],
|
||||
)["json"]
|
||||
|
||||
check(first == replay, "Missing attachment_ids differs from []")
|
||||
print("PASS: missing attachment_ids equals []", flush=True)
|
||||
|
||||
print("PASS: attachment negative tests completed", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,240 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from client import Client
|
||||
|
||||
|
||||
def check(condition, message):
|
||||
if not condition:
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def docker_script(script, *arguments, input_bytes=None):
|
||||
command = [
|
||||
"docker", "compose", "exec", "-T",
|
||||
"minio",
|
||||
"sh", "-ec", script,
|
||||
"cleanup-check",
|
||||
*arguments,
|
||||
]
|
||||
|
||||
result = subprocess.run(
|
||||
command,
|
||||
input=input_bytes,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(
|
||||
"Команда MinIO завершилась ошибкой. "
|
||||
"Проверь доступность minio и название bucket."
|
||||
)
|
||||
|
||||
return result.stdout.decode("utf-8")
|
||||
|
||||
|
||||
def configure_minio():
|
||||
docker_script(
|
||||
'''
|
||||
mc alias set cleanupcheck http://localhost:9000 \
|
||||
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
|
||||
'''
|
||||
)
|
||||
|
||||
|
||||
def put_orphan(bucket, key):
|
||||
docker_script(
|
||||
'mc pipe "cleanupcheck/$1/$2" >/dev/null',
|
||||
bucket,
|
||||
key,
|
||||
input_bytes=b"synthetic cleanup test object\n",
|
||||
)
|
||||
|
||||
|
||||
def list_test_objects(bucket, prefix):
|
||||
output = docker_script(
|
||||
'mc ls --json "cleanupcheck/$1/$2"',
|
||||
bucket,
|
||||
prefix,
|
||||
)
|
||||
|
||||
names = set()
|
||||
|
||||
for line in output.splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
item = json.loads(line)
|
||||
|
||||
check(
|
||||
item.get("status") != "error",
|
||||
"MinIO вернул ошибку при получении списка объектов",
|
||||
)
|
||||
|
||||
key = item.get("key")
|
||||
|
||||
if key:
|
||||
names.add(key.rstrip("/").split("/")[-1])
|
||||
|
||||
return names
|
||||
|
||||
|
||||
def remove_test_object(bucket, key):
|
||||
docker_script(
|
||||
'mc rm --force "cleanupcheck/$1/$2" >/dev/null',
|
||||
bucket,
|
||||
key,
|
||||
)
|
||||
|
||||
|
||||
def check_download(client, actor, attachment, original):
|
||||
result = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=actor,
|
||||
)
|
||||
|
||||
check(
|
||||
result["bytes"] == original,
|
||||
"Корректное вложение потеряно или изменилось",
|
||||
)
|
||||
|
||||
check(
|
||||
hashlib.sha256(result["bytes"]).hexdigest()
|
||||
== attachment["sha256"],
|
||||
"SHA-256 корректного вложения не совпадает",
|
||||
)
|
||||
|
||||
|
||||
def wait_until(deadline):
|
||||
remaining = deadline - time.monotonic()
|
||||
|
||||
if remaining > 0:
|
||||
time.sleep(remaining)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--base-url",
|
||||
default="http://localhost:8080",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bucket",
|
||||
default="messenger-attachments",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
check(
|
||||
re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", args.bucket),
|
||||
"Некорректное название bucket",
|
||||
)
|
||||
|
||||
client = Client(args.base_url, None)
|
||||
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
|
||||
# Создаётся через API: в S3 и PostgreSQL.
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
|
||||
check_download(client, bob, attachment, original)
|
||||
|
||||
configure_minio()
|
||||
|
||||
prefix = "originals/cleanup-test-" + uuid.uuid4().hex + "/"
|
||||
old_key = prefix + "old.bin"
|
||||
fresh_key = prefix + "fresh.bin"
|
||||
|
||||
try:
|
||||
put_orphan(args.bucket, old_key)
|
||||
old_created = time.monotonic()
|
||||
|
||||
names = list_test_objects(args.bucket, prefix)
|
||||
|
||||
check(
|
||||
"old.bin" in names,
|
||||
"Тестовый объект не появился в S3",
|
||||
)
|
||||
|
||||
print("Created API attachment and an untracked S3 object", flush=True)
|
||||
|
||||
wait_until(old_created + 25)
|
||||
|
||||
names = list_test_objects(args.bucket, prefix)
|
||||
|
||||
check(
|
||||
"old.bin" in names,
|
||||
"Объект удалён раньше grace period",
|
||||
)
|
||||
|
||||
print("PASS: object younger than grace period is preserved", flush=True)
|
||||
|
||||
wait_until(old_created + 65)
|
||||
|
||||
put_orphan(args.bucket, fresh_key)
|
||||
fresh_created = time.monotonic()
|
||||
|
||||
print("Waiting for cleanup of the old object...", flush=True)
|
||||
|
||||
deadline = fresh_created + 30
|
||||
|
||||
while True:
|
||||
names = list_test_objects(args.bucket, prefix)
|
||||
|
||||
check(
|
||||
"fresh.bin" in names,
|
||||
"Свежий объект удалён раньше grace period",
|
||||
)
|
||||
|
||||
if "old.bin" not in names:
|
||||
break
|
||||
|
||||
check(
|
||||
time.monotonic() < deadline,
|
||||
"Старый объект не удалён. Проверь настройки и логи очистки",
|
||||
)
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
print("PASS: old object without metadata is deleted", flush=True)
|
||||
print("PASS: fresh object without metadata is preserved", flush=True)
|
||||
|
||||
client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"],
|
||||
actor=bob,
|
||||
schema="Attachment",
|
||||
)
|
||||
|
||||
check_download(client, bob, attachment, original)
|
||||
|
||||
print("PASS: attachment with metadata and its bytes are preserved", flush=True)
|
||||
print("PASS: cleanup scenario completed", flush=True)
|
||||
|
||||
finally:
|
||||
|
||||
for key in (old_key, fresh_key):
|
||||
try:
|
||||
remove_test_object(args.bucket, key)
|
||||
except Exception as error:
|
||||
print(
|
||||
f"WARNING: final cleanup failed for {key}: {error}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception as error:
|
||||
print("FAIL:", error, flush=True)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,686 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BUCKET = None
|
||||
|
||||
|
||||
def command(args, data=None, timeout=30):
|
||||
result = subprocess.run(
|
||||
args,
|
||||
cwd=ROOT,
|
||||
input=data,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
check(
|
||||
result.returncode == 0,
|
||||
"Command failed: " + " ".join(args[:3]),
|
||||
)
|
||||
|
||||
return result.stdout
|
||||
|
||||
|
||||
def sql(statement):
|
||||
result = command(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"sh",
|
||||
"-ec",
|
||||
(
|
||||
'exec psql -X -A -t -v ON_ERROR_STOP=1 '
|
||||
'-U "$POSTGRES_USER" '
|
||||
'-d "${POSTGRES_DB:-$POSTGRES_USER}"'
|
||||
),
|
||||
],
|
||||
statement.encode(),
|
||||
)
|
||||
|
||||
return result.decode().strip()
|
||||
|
||||
|
||||
def mc(*args, data=None):
|
||||
return command(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"minio",
|
||||
"sh",
|
||||
"-ec",
|
||||
(
|
||||
"mc alias set reconcile_test "
|
||||
"http://localhost:9000 "
|
||||
'"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" '
|
||||
'>/dev/null; exec mc "$@"'
|
||||
),
|
||||
"sh",
|
||||
*args,
|
||||
],
|
||||
data,
|
||||
)
|
||||
|
||||
|
||||
def target(key):
|
||||
return "reconcile_test/" + BUCKET + "/" + key
|
||||
|
||||
|
||||
def object_names(chat_id):
|
||||
raw = mc(
|
||||
"ls",
|
||||
"--recursive",
|
||||
"--json",
|
||||
target("originals/" + chat_id + "/"),
|
||||
)
|
||||
|
||||
names = set()
|
||||
|
||||
for line in raw.decode().splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
item = json.loads(line)
|
||||
|
||||
check(
|
||||
item.get("status") == "success",
|
||||
"Cannot list test S3 prefix",
|
||||
)
|
||||
|
||||
if item.get("type") == "file":
|
||||
names.add(
|
||||
item["key"].rstrip("/").rsplit("/", 1)[-1]
|
||||
)
|
||||
|
||||
return names
|
||||
|
||||
|
||||
def inspect_api():
|
||||
ids = command([
|
||||
"docker",
|
||||
"compose",
|
||||
"ps",
|
||||
"-q",
|
||||
"api",
|
||||
]).decode().split()
|
||||
|
||||
check(
|
||||
len(ids) == 2,
|
||||
"Start exactly two API replicas",
|
||||
)
|
||||
|
||||
containers = json.loads(command([
|
||||
"docker",
|
||||
"inspect",
|
||||
*ids,
|
||||
]))
|
||||
|
||||
buckets = set()
|
||||
|
||||
for container in containers:
|
||||
env = dict(
|
||||
value.split("=", 1)
|
||||
for value in container["Config"]["Env"]
|
||||
if "=" in value
|
||||
)
|
||||
|
||||
check(
|
||||
env.get("APP_CLEANUP_ENABLED", "").lower()
|
||||
== "false",
|
||||
"Disable AttachmentCleanupJob "
|
||||
"on both replicas first",
|
||||
)
|
||||
|
||||
check(
|
||||
bool(env.get("S3_BUCKET")),
|
||||
"S3_BUCKET is missing",
|
||||
)
|
||||
|
||||
buckets.add(env["S3_BUCKET"])
|
||||
|
||||
check(
|
||||
len(buckets) == 1,
|
||||
"API replicas use different buckets",
|
||||
)
|
||||
|
||||
return buckets.pop()
|
||||
|
||||
|
||||
def snapshot(chat_id):
|
||||
# chat_id предварительно проверен через uuid.UUID.
|
||||
return sql(
|
||||
"SELECT coalesce("
|
||||
"json_agg(a ORDER BY id), '[]'::json"
|
||||
") FROM attachments a "
|
||||
"WHERE chat_id='" + chat_id + "'::uuid;"
|
||||
)
|
||||
|
||||
|
||||
def reconcile(
|
||||
chat_id,
|
||||
backups,
|
||||
directory,
|
||||
label,
|
||||
apply=False,
|
||||
):
|
||||
path = directory / (label + ".json")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-X",
|
||||
"utf8",
|
||||
str(ROOT / "scripts/reconcile.py"),
|
||||
"--apply" if apply else "--dry-run",
|
||||
"--chat-id",
|
||||
chat_id,
|
||||
"--grace-seconds",
|
||||
"60",
|
||||
"--max-seconds",
|
||||
"120",
|
||||
"--backup-dir",
|
||||
str(backups),
|
||||
"--output",
|
||||
str(path),
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=180,
|
||||
)
|
||||
|
||||
check(
|
||||
result.returncode in (0, 2),
|
||||
"Reconciler failed at "
|
||||
+ label
|
||||
+ "; inspect "
|
||||
+ str(path),
|
||||
)
|
||||
|
||||
check(
|
||||
path.exists(),
|
||||
"Reconciler did not save report",
|
||||
)
|
||||
|
||||
report = json.loads(
|
||||
path.read_text(encoding="utf-8")
|
||||
)
|
||||
|
||||
check(
|
||||
report["status"] in (
|
||||
"completed",
|
||||
"needs_attention",
|
||||
),
|
||||
"Scan failed",
|
||||
)
|
||||
|
||||
check(
|
||||
report["counts"].get("busy", 0) == 0,
|
||||
"Object lock was busy; repeat experiment",
|
||||
)
|
||||
|
||||
check(
|
||||
report["counts"].get("size_mismatches", 0) == 0,
|
||||
"Unexpected size mismatch",
|
||||
)
|
||||
|
||||
print(
|
||||
label + ": " + json.dumps(report["counts"]),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def count(report, name):
|
||||
return report["counts"].get(name, 0)
|
||||
|
||||
|
||||
def main():
|
||||
global BUCKET
|
||||
|
||||
args = arguments(
|
||||
"Lab4 reconciler: dry-run, repair "
|
||||
"and repeated execution"
|
||||
)
|
||||
|
||||
BUCKET = inspect_api()
|
||||
|
||||
c = Client(
|
||||
args.base_url,
|
||||
args.ca_file,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
chat_id = str(uuid.UUID(chat["id"]))
|
||||
|
||||
stable, content = c.attachment(alice, chat)
|
||||
missing, missing_bytes = c.attachment(alice, chat)
|
||||
|
||||
message = c.message(
|
||||
alice,
|
||||
chat,
|
||||
"Reconciler experiment",
|
||||
attachments=[
|
||||
stable["id"],
|
||||
missing["id"],
|
||||
],
|
||||
)["json"]
|
||||
|
||||
missing_id = str(uuid.UUID(missing["id"]))
|
||||
|
||||
key = sql(
|
||||
"SELECT object_key FROM attachments "
|
||||
"WHERE id='" + missing_id + "'::uuid;"
|
||||
)
|
||||
|
||||
check(
|
||||
key.startswith("originals/" + chat_id + "/"),
|
||||
"Unexpected original key",
|
||||
)
|
||||
|
||||
prefix = "originals/" + chat_id + "/"
|
||||
|
||||
old_key = (
|
||||
prefix + "orphan-old-" + uuid.uuid4().hex
|
||||
)
|
||||
|
||||
fresh_key = None
|
||||
fresh_started = None
|
||||
|
||||
# Запоминаем все созданные orphan-ключи,
|
||||
# чтобы попытаться убрать их при исключении.
|
||||
synthetic_keys = {old_key}
|
||||
|
||||
backups = (
|
||||
ROOT / "backups" / ("reconcile-" + chat_id)
|
||||
)
|
||||
|
||||
directory = (
|
||||
ROOT / "results" / ("reconcile-" + chat_id)
|
||||
)
|
||||
|
||||
backups.mkdir(parents=True, exist_ok=False)
|
||||
directory.mkdir(parents=True, exist_ok=False)
|
||||
|
||||
metadata_before = snapshot(chat_id)
|
||||
|
||||
needs_restore = False
|
||||
completed = False
|
||||
|
||||
def verify_metadata():
|
||||
check(
|
||||
snapshot(chat_id) == metadata_before,
|
||||
"Attachment metadata changed",
|
||||
)
|
||||
|
||||
page = c.call(
|
||||
"GET",
|
||||
"/api/v1/chats/"
|
||||
+ chat_id
|
||||
+ "/messages?limit=100",
|
||||
actor=bob,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
|
||||
found = next(
|
||||
(
|
||||
item
|
||||
for item in page["items"]
|
||||
if item["id"] == message["id"]
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
check(
|
||||
found == message,
|
||||
"Message or attachment references changed",
|
||||
)
|
||||
|
||||
def prepare_fresh():
|
||||
nonlocal fresh_key, fresh_started
|
||||
|
||||
# Предыдущий свежий объект больше не нужен.
|
||||
# Его сохранность проверена предыдущим этапом.
|
||||
if fresh_key is not None:
|
||||
previous_name = fresh_key.rsplit("/", 1)[-1]
|
||||
|
||||
if previous_name in object_names(chat_id):
|
||||
mc("rm", target(fresh_key))
|
||||
|
||||
fresh_key = (
|
||||
prefix + "orphan-fresh-" + uuid.uuid4().hex
|
||||
)
|
||||
|
||||
synthetic_keys.add(fresh_key)
|
||||
|
||||
# Начинаем отсчёт до PUT: это консервативная
|
||||
# верхняя оценка возраста объекта.
|
||||
fresh_started = time.monotonic()
|
||||
|
||||
mc(
|
||||
"pipe",
|
||||
target(fresh_key),
|
||||
data=content,
|
||||
)
|
||||
|
||||
def verify_fresh():
|
||||
check(
|
||||
fresh_key is not None
|
||||
and fresh_started is not None,
|
||||
"Fresh test object was not created",
|
||||
)
|
||||
|
||||
names = object_names(chat_id)
|
||||
elapsed = time.monotonic() - fresh_started
|
||||
|
||||
check(
|
||||
elapsed < 55,
|
||||
"A single reconciliation stage exceeded "
|
||||
"the fresh-object timing budget; "
|
||||
"repeat on an idle stand",
|
||||
)
|
||||
|
||||
check(
|
||||
fresh_key.rsplit("/", 1)[-1] in names,
|
||||
"Fresh orphan was removed",
|
||||
)
|
||||
|
||||
def download(attachment, expected):
|
||||
result = c.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/"
|
||||
+ attachment["id"]
|
||||
+ "/content",
|
||||
actor=bob,
|
||||
)
|
||||
|
||||
check(
|
||||
result["bytes"] == expected,
|
||||
"Original bytes changed",
|
||||
)
|
||||
|
||||
try:
|
||||
mc(
|
||||
"pipe",
|
||||
target(old_key),
|
||||
data=content,
|
||||
)
|
||||
|
||||
print(
|
||||
"Old orphan created. Waiting 65 seconds "
|
||||
"for the real grace period...",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
time.sleep(65)
|
||||
|
||||
prepare_fresh()
|
||||
|
||||
# Байты пока доступны только тесту.
|
||||
# Reconciler ещё не получает резервную копию.
|
||||
needs_restore = True
|
||||
mc("rm", target(key))
|
||||
|
||||
before = object_names(chat_id)
|
||||
|
||||
dry = reconcile(
|
||||
chat_id,
|
||||
backups,
|
||||
directory,
|
||||
"01-dry-run",
|
||||
)
|
||||
|
||||
check(
|
||||
count(dry, "would_delete_orphans") == 1,
|
||||
"Old orphan not detected",
|
||||
)
|
||||
|
||||
check(
|
||||
count(dry, "unresolved_missing") == 1,
|
||||
"Missing original not detected",
|
||||
)
|
||||
|
||||
check(
|
||||
object_names(chat_id) == before,
|
||||
"Dry-run changed S3 objects",
|
||||
)
|
||||
|
||||
verify_metadata()
|
||||
verify_fresh()
|
||||
download(stable, content)
|
||||
|
||||
print(
|
||||
"PASS: dry-run found both faults "
|
||||
"and changed no test data",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Для следующего этапа создаём новый свежий объект.
|
||||
prepare_fresh()
|
||||
|
||||
applied = reconcile(
|
||||
chat_id,
|
||||
backups,
|
||||
directory,
|
||||
"02-apply-no-backup",
|
||||
apply=True,
|
||||
)
|
||||
|
||||
check(
|
||||
count(applied, "deleted_orphans") == 1,
|
||||
"Old orphan not deleted",
|
||||
)
|
||||
|
||||
check(
|
||||
count(applied, "unresolved_missing") == 1,
|
||||
"Missing original incorrectly repaired",
|
||||
)
|
||||
|
||||
names = object_names(chat_id)
|
||||
|
||||
check(
|
||||
old_key.rsplit("/", 1)[-1] not in names,
|
||||
"Old orphan still exists",
|
||||
)
|
||||
|
||||
check(
|
||||
key.rsplit("/", 1)[-1] not in names,
|
||||
"Missing original unexpectedly exists",
|
||||
)
|
||||
|
||||
verify_metadata()
|
||||
verify_fresh()
|
||||
|
||||
c.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/"
|
||||
+ missing_id
|
||||
+ "/content",
|
||||
expected=503,
|
||||
actor=bob,
|
||||
)
|
||||
|
||||
print(
|
||||
"PASS: old orphan deleted; missing original "
|
||||
"reported without metadata loss",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Предоставляем reconciler достоверную копию.
|
||||
backup_file = (
|
||||
backups / (missing["sha256"] + ".bin")
|
||||
)
|
||||
|
||||
backup_file.write_bytes(missing_bytes)
|
||||
|
||||
prepare_fresh()
|
||||
before = object_names(chat_id)
|
||||
|
||||
planned = reconcile(
|
||||
chat_id,
|
||||
backups,
|
||||
directory,
|
||||
"03-dry-run-with-backup",
|
||||
)
|
||||
|
||||
check(
|
||||
count(planned, "would_restore") == 1,
|
||||
"Valid backup not recognized",
|
||||
)
|
||||
|
||||
check(
|
||||
object_names(chat_id) == before,
|
||||
"Dry-run restored an object",
|
||||
)
|
||||
|
||||
verify_metadata()
|
||||
verify_fresh()
|
||||
|
||||
print(
|
||||
"PASS: dry-run plans restoration "
|
||||
"without changing objects",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
prepare_fresh()
|
||||
|
||||
restored = reconcile(
|
||||
chat_id,
|
||||
backups,
|
||||
directory,
|
||||
"04-restore",
|
||||
apply=True,
|
||||
)
|
||||
|
||||
check(
|
||||
count(restored, "restored") == 1,
|
||||
"Original not restored",
|
||||
)
|
||||
|
||||
download(missing, missing_bytes)
|
||||
needs_restore = False
|
||||
|
||||
download(stable, content)
|
||||
verify_metadata()
|
||||
verify_fresh()
|
||||
|
||||
print(
|
||||
"PASS: original restored byte-for-byte; "
|
||||
"used attachment and message preserved",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
prepare_fresh()
|
||||
|
||||
repeated = reconcile(
|
||||
chat_id,
|
||||
backups,
|
||||
directory,
|
||||
"05-repeat",
|
||||
apply=True,
|
||||
)
|
||||
|
||||
for name in (
|
||||
"deleted_orphans",
|
||||
"restored",
|
||||
"unresolved_missing",
|
||||
"missing_objects",
|
||||
):
|
||||
check(
|
||||
count(repeated, name) == 0,
|
||||
"Repeated run is not clean: " + name,
|
||||
)
|
||||
|
||||
verify_metadata()
|
||||
verify_fresh()
|
||||
|
||||
download(stable, content)
|
||||
download(missing, missing_bytes)
|
||||
|
||||
completed = True
|
||||
|
||||
print(
|
||||
"PASS: repeated apply makes "
|
||||
"no additional changes",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
print(
|
||||
"Reports: " + str(directory),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
finally:
|
||||
# При исключении пытаемся восстановить оригинал,
|
||||
# намеренно удалённый этим тестом.
|
||||
if needs_restore:
|
||||
try:
|
||||
mc(
|
||||
"pipe",
|
||||
target(key),
|
||||
data=missing_bytes,
|
||||
)
|
||||
|
||||
print(
|
||||
"Test original restored "
|
||||
"during final cleanup",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
except Exception:
|
||||
print(
|
||||
"WARNING: restore test original manually "
|
||||
"from backup; key=" + key,
|
||||
flush=True,
|
||||
)
|
||||
|
||||
(
|
||||
backups / (missing["sha256"] + ".bin")
|
||||
).write_bytes(missing_bytes)
|
||||
|
||||
cleanup_ok = True
|
||||
|
||||
# Удаляем только существующие синтетические объекты.
|
||||
# Уже удалённый reconciler старый orphan — не ошибка.
|
||||
for orphan in sorted(synthetic_keys):
|
||||
try:
|
||||
name = orphan.rsplit("/", 1)[-1]
|
||||
|
||||
if name in object_names(chat_id):
|
||||
mc("rm", target(orphan))
|
||||
|
||||
except Exception:
|
||||
cleanup_ok = False
|
||||
|
||||
print(
|
||||
"WARNING: cannot remove synthetic orphan: "
|
||||
+ orphan,
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if completed:
|
||||
check(
|
||||
cleanup_ok,
|
||||
"Scenario passed but final "
|
||||
"test-object cleanup failed",
|
||||
)
|
||||
|
||||
print("PASS: reconciler experiment completed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,561 @@
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import random
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
import reconcile_check as helpers
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def options(parser):
|
||||
parser.add_argument(
|
||||
"--sizes",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[10, 100, 1000],
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--repeats",
|
||||
type=int,
|
||||
default=3,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--max-seconds",
|
||||
type=int,
|
||||
default=600,
|
||||
)
|
||||
|
||||
|
||||
def digest_metadata(chat_id):
|
||||
snapshot = helpers.snapshot(chat_id)
|
||||
|
||||
return hashlib.sha256(
|
||||
snapshot.encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def environment_info():
|
||||
ids = helpers.command([
|
||||
"docker",
|
||||
"compose",
|
||||
"ps",
|
||||
"-q",
|
||||
"api",
|
||||
"postgres",
|
||||
"minio",
|
||||
]).decode().split()
|
||||
|
||||
containers = json.loads(
|
||||
helpers.command([
|
||||
"docker",
|
||||
"inspect",
|
||||
*ids,
|
||||
])
|
||||
)
|
||||
|
||||
cpu_count = int(helpers.command([
|
||||
"docker",
|
||||
"info",
|
||||
"--format",
|
||||
"{{.NCPU}}",
|
||||
]))
|
||||
|
||||
memory_bytes = int(helpers.command([
|
||||
"docker",
|
||||
"info",
|
||||
"--format",
|
||||
"{{.MemTotal}}",
|
||||
]))
|
||||
|
||||
worker_image = helpers.command([
|
||||
"docker",
|
||||
"image",
|
||||
"inspect",
|
||||
"--format",
|
||||
"{{.Id}}",
|
||||
"mtusi4-reconciler:local",
|
||||
]).decode().strip()
|
||||
|
||||
# Сохраняем только параметры окружения,
|
||||
# необходимые для воспроизводимости.
|
||||
# Credentials и полный docker inspect не записываем.
|
||||
services = []
|
||||
|
||||
for container in containers:
|
||||
services.append({
|
||||
"service": container["Config"]["Labels"].get(
|
||||
"com.docker.compose.service"
|
||||
),
|
||||
"image_id": container["Image"],
|
||||
"memory_limit_bytes": (
|
||||
container["HostConfig"]["Memory"]
|
||||
),
|
||||
"nano_cpus": (
|
||||
container["HostConfig"]["NanoCpus"]
|
||||
),
|
||||
})
|
||||
|
||||
return {
|
||||
"docker_cpus": cpu_count,
|
||||
"docker_memory_bytes": memory_bytes,
|
||||
"worker_image_id": worker_image,
|
||||
"worker_memory_limit_bytes": 256 * 1024 * 1024,
|
||||
"worker_cpu_limit": 1,
|
||||
"services": services,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(
|
||||
"Measure reconciler traversal cost",
|
||||
options,
|
||||
)
|
||||
|
||||
sizes = sorted(set(args.sizes))
|
||||
|
||||
check(
|
||||
all(1 <= size <= 5000 for size in sizes),
|
||||
"Sizes must be between 1 and 5000",
|
||||
)
|
||||
|
||||
check(
|
||||
1 <= args.repeats <= 10,
|
||||
"Repeats must be between 1 and 10",
|
||||
)
|
||||
|
||||
check(
|
||||
args.max_seconds >= 1,
|
||||
"max-seconds must be positive",
|
||||
)
|
||||
|
||||
# Проверяет две API-реплики и отключённую очистку.
|
||||
helpers.BUCKET = helpers.inspect_api()
|
||||
|
||||
fixture = (
|
||||
ROOT / "tests/fixtures/sample.png"
|
||||
).read_bytes()
|
||||
|
||||
fixture_sha = hashlib.sha256(fixture).hexdigest()
|
||||
|
||||
directory = (
|
||||
ROOT
|
||||
/ "results"
|
||||
/ ("reconcile-cost-" + uuid.uuid4().hex)
|
||||
)
|
||||
|
||||
backups = directory / "empty-backups"
|
||||
backups.mkdir(parents=True, exist_ok=False)
|
||||
|
||||
report = {
|
||||
"passed": False,
|
||||
"hypothesis": (
|
||||
"Traversal calls grow approximately linearly "
|
||||
"with attachment count; wall time also "
|
||||
"contains fixed overhead."
|
||||
),
|
||||
"sizes": sizes,
|
||||
"repeats": args.repeats,
|
||||
"grace_seconds": 60,
|
||||
"fixture_size_bytes": len(fixture),
|
||||
"fixture_sha256": fixture_sha,
|
||||
"environment": environment_info(),
|
||||
"groups": [],
|
||||
"samples": [],
|
||||
"summary": [],
|
||||
"method": (
|
||||
"Separate chats; API uploads; real grace wait; "
|
||||
"one warm-up per size; shuffled measured rounds; "
|
||||
"dry-run only."
|
||||
),
|
||||
}
|
||||
|
||||
groups = []
|
||||
|
||||
client = Client(
|
||||
args.base_url,
|
||||
args.ca_file,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
def save():
|
||||
(directory / "summary.json").write_text(
|
||||
json.dumps(
|
||||
report,
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
fields = [
|
||||
"attachments",
|
||||
"trial",
|
||||
"worker_seconds",
|
||||
"launcher_seconds",
|
||||
"sql_calls",
|
||||
"s3_calls",
|
||||
"s3_list_calls",
|
||||
"s3_head_calls",
|
||||
]
|
||||
|
||||
with (directory / "samples.csv").open(
|
||||
"w",
|
||||
encoding="utf-8",
|
||||
newline="",
|
||||
) as stream:
|
||||
writer = csv.DictWriter(
|
||||
stream,
|
||||
fieldnames=fields,
|
||||
)
|
||||
|
||||
writer.writeheader()
|
||||
writer.writerows(report["samples"])
|
||||
|
||||
def scan(group, label):
|
||||
path = directory / (
|
||||
f'n{group["n"]}-{label}.json'
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-X",
|
||||
"utf8",
|
||||
str(ROOT / "scripts/reconcile.py"),
|
||||
"--dry-run",
|
||||
"--chat-id",
|
||||
group["chat_id"],
|
||||
"--grace-seconds",
|
||||
"60",
|
||||
"--max-seconds",
|
||||
str(args.max_seconds),
|
||||
"--backup-dir",
|
||||
str(backups),
|
||||
"--output",
|
||||
str(path),
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=args.max_seconds + 180,
|
||||
)
|
||||
|
||||
wall = time.monotonic() - started
|
||||
|
||||
check(
|
||||
result.returncode == 0,
|
||||
"Scan failed: " + str(path),
|
||||
)
|
||||
|
||||
data = json.loads(
|
||||
path.read_text(encoding="utf-8")
|
||||
)
|
||||
|
||||
check(
|
||||
data["status"] == "completed"
|
||||
and data["mode"] == "dry-run",
|
||||
"Unexpected scan status",
|
||||
)
|
||||
|
||||
check(
|
||||
data.get("chat_id") == group["chat_id"],
|
||||
"Wrong scan scope",
|
||||
)
|
||||
|
||||
counts = data["counts"]
|
||||
|
||||
# Каждый объект и каждая запись должны пройти
|
||||
# полноценную проверку после grace period.
|
||||
for field in (
|
||||
"objects_scanned",
|
||||
"referenced_objects",
|
||||
"metadata_scanned",
|
||||
"present_metadata",
|
||||
):
|
||||
check(
|
||||
counts.get(field, 0) == group["n"],
|
||||
"Incomplete scan: " + field,
|
||||
)
|
||||
|
||||
for field in (
|
||||
"fresh_objects",
|
||||
"fresh_metadata",
|
||||
"busy",
|
||||
"missing_objects",
|
||||
"unresolved_missing",
|
||||
"size_mismatches",
|
||||
"would_delete_orphans",
|
||||
"deleted_orphans",
|
||||
"restored",
|
||||
):
|
||||
check(
|
||||
counts.get(field, 0) == 0,
|
||||
"Unexpected outcome: " + field,
|
||||
)
|
||||
|
||||
calls = data["s3_operation_calls"]
|
||||
|
||||
check(
|
||||
calls.get("put_object", 0) == 0
|
||||
and calls.get("delete_object", 0) == 0,
|
||||
"Dry-run attempted mutations",
|
||||
)
|
||||
|
||||
return {
|
||||
"attachments": group["n"],
|
||||
"trial": label,
|
||||
"worker_seconds": data["elapsed_seconds"],
|
||||
"launcher_seconds": round(wall, 3),
|
||||
"sql_calls": data["sql_statement_calls"],
|
||||
"s3_calls": sum(calls.values()),
|
||||
"s3_list_calls": calls.get(
|
||||
"list_objects_v2", 0
|
||||
),
|
||||
"s3_head_calls": calls.get(
|
||||
"head_object", 0
|
||||
),
|
||||
}
|
||||
|
||||
save()
|
||||
|
||||
try:
|
||||
setup_started = time.monotonic()
|
||||
|
||||
# Каждый размер получает отдельный чат.
|
||||
for size in sizes:
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
chat_id = str(uuid.UUID(chat["id"]))
|
||||
|
||||
entry = {
|
||||
"attachments": size,
|
||||
"chat_id": chat_id,
|
||||
"uploaded": 0,
|
||||
}
|
||||
|
||||
report["groups"].append(entry)
|
||||
save()
|
||||
|
||||
sample_attachments = []
|
||||
|
||||
# Последовательные загрузки.
|
||||
# Автоматических повторов POST нет.
|
||||
for index in range(size):
|
||||
attachment, content = client.attachment(
|
||||
alice,
|
||||
chat,
|
||||
)
|
||||
|
||||
check(
|
||||
content == fixture,
|
||||
"Fixture changed during experiment",
|
||||
)
|
||||
|
||||
# Для контрольного скачивания сохраняем
|
||||
# первое и последнее вложение.
|
||||
if index in {0, size - 1}:
|
||||
sample_attachments.append(attachment)
|
||||
|
||||
entry["uploaded"] = index + 1
|
||||
|
||||
if (
|
||||
(index + 1) % 100 == 0
|
||||
or index + 1 == size
|
||||
):
|
||||
print(
|
||||
f"Preparing N={size}: "
|
||||
f"{index + 1}/{size}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
save()
|
||||
|
||||
message = client.message(
|
||||
alice,
|
||||
chat,
|
||||
"Reconciler cost experiment",
|
||||
attachments=[
|
||||
attachment["id"]
|
||||
for attachment in sample_attachments
|
||||
],
|
||||
)["json"]
|
||||
|
||||
groups.append({
|
||||
"n": size,
|
||||
"chat_id": chat_id,
|
||||
"bob": bob,
|
||||
"samples": sample_attachments,
|
||||
"message": message,
|
||||
"metadata_digest": digest_metadata(chat_id),
|
||||
})
|
||||
|
||||
report["setup_seconds"] = round(
|
||||
time.monotonic() - setup_started,
|
||||
3,
|
||||
)
|
||||
|
||||
save()
|
||||
|
||||
print(
|
||||
"All uploads completed. Waiting 65 seconds "
|
||||
"for the real grace period...",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
time.sleep(65)
|
||||
|
||||
# Прогрев сохраняется отдельными JSON,
|
||||
# но не включается в итоговую статистику.
|
||||
for group in groups:
|
||||
scan(group, "warmup")
|
||||
|
||||
print(
|
||||
f'Warm-up N={group["n"]}: complete',
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# В каждом раунде меняем порядок объёмов.
|
||||
# Фиксированный seed делает порядок повторяемым.
|
||||
rng = random.Random(1729)
|
||||
|
||||
for trial in range(1, args.repeats + 1):
|
||||
order = list(groups)
|
||||
rng.shuffle(order)
|
||||
|
||||
for group in order:
|
||||
sample = scan(
|
||||
group,
|
||||
f"run-{trial}",
|
||||
)
|
||||
|
||||
report["samples"].append(sample)
|
||||
save()
|
||||
|
||||
print(
|
||||
f'N={group["n"]}, run={trial}: '
|
||||
f'worker={sample["worker_seconds"]}s, '
|
||||
f'launcher={sample["launcher_seconds"]}s, '
|
||||
f'SQL={sample["sql_calls"]}, '
|
||||
f'S3={sample["s3_calls"]}',
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Контроль после измерений.
|
||||
for group in groups:
|
||||
check(
|
||||
digest_metadata(group["chat_id"])
|
||||
== group["metadata_digest"],
|
||||
"Metadata changed during benchmark",
|
||||
)
|
||||
|
||||
for attachment in group["samples"]:
|
||||
downloaded = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/"
|
||||
+ attachment["id"]
|
||||
+ "/content",
|
||||
actor=group["bob"],
|
||||
)
|
||||
|
||||
check(
|
||||
downloaded["bytes"] == fixture,
|
||||
"Sample original changed",
|
||||
)
|
||||
|
||||
page = client.call(
|
||||
"GET",
|
||||
"/api/v1/chats/"
|
||||
+ group["chat_id"]
|
||||
+ "/messages?limit=100",
|
||||
actor=group["bob"],
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
|
||||
check(
|
||||
group["message"] in page["items"],
|
||||
"Sample message changed",
|
||||
)
|
||||
|
||||
for size in sizes:
|
||||
samples = [
|
||||
sample
|
||||
for sample in report["samples"]
|
||||
if sample["attachments"] == size
|
||||
]
|
||||
|
||||
worker_times = [
|
||||
sample["worker_seconds"]
|
||||
for sample in samples
|
||||
]
|
||||
|
||||
launcher_times = [
|
||||
sample["launcher_seconds"]
|
||||
for sample in samples
|
||||
]
|
||||
|
||||
report["summary"].append({
|
||||
"attachments": size,
|
||||
"worker_seconds_min": min(worker_times),
|
||||
"worker_seconds_median": round(
|
||||
statistics.median(worker_times),
|
||||
3,
|
||||
),
|
||||
"worker_seconds_max": max(worker_times),
|
||||
"launcher_seconds_median": round(
|
||||
statistics.median(launcher_times),
|
||||
3,
|
||||
),
|
||||
"sql_calls_observed": sorted({
|
||||
sample["sql_calls"]
|
||||
for sample in samples
|
||||
}),
|
||||
"s3_calls_observed": sorted({
|
||||
sample["s3_calls"]
|
||||
for sample in samples
|
||||
}),
|
||||
"sql_calls_expected_for_current_algorithm": (
|
||||
5 * size
|
||||
+ math.ceil(size / 500)
|
||||
+ 2
|
||||
),
|
||||
"s3_calls_expected_for_current_algorithm": (
|
||||
size
|
||||
+ math.ceil(size / 500)
|
||||
+ 1
|
||||
),
|
||||
})
|
||||
|
||||
report["passed"] = True
|
||||
|
||||
finally:
|
||||
save()
|
||||
|
||||
print(json.dumps(
|
||||
report["summary"],
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
))
|
||||
|
||||
print("Reports: " + str(directory))
|
||||
|
||||
print(
|
||||
"PASS: reconciler traversal cost "
|
||||
"experiment completed"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,492 @@
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from client import Client, Failure, arguments, check, run
|
||||
import reconcile_check as helpers
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def load_launcher():
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"reconciler_launcher",
|
||||
ROOT / "scripts/reconcile.py",
|
||||
)
|
||||
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
return module
|
||||
|
||||
|
||||
def state(container_id):
|
||||
raw = helpers.command([
|
||||
"docker",
|
||||
"inspect",
|
||||
"--format",
|
||||
"{{json .State}}",
|
||||
container_id,
|
||||
])
|
||||
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def wait_container(container_id, seconds=90):
|
||||
deadline = time.monotonic() + seconds
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
current = state(container_id)
|
||||
|
||||
healthy = (
|
||||
current.get("Running")
|
||||
and current.get("Health", {}).get("Status")
|
||||
== "healthy"
|
||||
)
|
||||
|
||||
if healthy:
|
||||
return
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
raise Failure(
|
||||
"Container did not become healthy: "
|
||||
+ container_id[:12]
|
||||
)
|
||||
|
||||
|
||||
def wait_api(client, seconds=90):
|
||||
deadline = time.monotonic() + seconds
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
client.call(
|
||||
"GET",
|
||||
"/health/ready",
|
||||
schema="Health",
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
except Failure:
|
||||
time.sleep(2)
|
||||
|
||||
raise Failure(
|
||||
"API did not recover after dependency restart"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(
|
||||
"Reconciler failure checks: "
|
||||
"MinIO and PostgreSQL outage"
|
||||
)
|
||||
|
||||
helpers.BUCKET = helpers.inspect_api()
|
||||
|
||||
launcher = load_launcher()
|
||||
|
||||
saved_services = {
|
||||
name: launcher.service(name)
|
||||
for name in ("api", "postgres", "minio")
|
||||
}
|
||||
|
||||
launcher.service = lambda name: saved_services[name]
|
||||
|
||||
c = Client(
|
||||
args.base_url,
|
||||
args.ca_file,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
chat_id = str(uuid.UUID(chat["id"]))
|
||||
|
||||
attachment, content = c.attachment(alice, chat)
|
||||
|
||||
message = c.message(
|
||||
alice,
|
||||
chat,
|
||||
"Reconciler dependency failure",
|
||||
attachments=[attachment["id"]],
|
||||
)["json"]
|
||||
|
||||
original_name = str(uuid.UUID(attachment["id"]))
|
||||
|
||||
orphan_key = (
|
||||
"originals/"
|
||||
+ chat_id
|
||||
+ "/orphan-failure-"
|
||||
+ uuid.uuid4().hex
|
||||
)
|
||||
|
||||
orphan_name = orphan_key.rsplit("/", 1)[-1]
|
||||
|
||||
metadata_before = helpers.snapshot(chat_id)
|
||||
|
||||
directory = (
|
||||
ROOT
|
||||
/ "results"
|
||||
/ ("reconcile-failure-" + chat_id)
|
||||
)
|
||||
|
||||
backups = (
|
||||
ROOT
|
||||
/ "backups"
|
||||
/ ("reconcile-failure-" + chat_id)
|
||||
)
|
||||
|
||||
directory.mkdir(parents=True, exist_ok=False)
|
||||
backups.mkdir(parents=True, exist_ok=False)
|
||||
|
||||
stopped = set()
|
||||
summary = []
|
||||
completed = False
|
||||
|
||||
def worker(label, apply, expected_code):
|
||||
path = directory / (label + ".json")
|
||||
previous_argv = sys.argv
|
||||
|
||||
try:
|
||||
sys.argv = [
|
||||
str(ROOT / "scripts/reconcile.py"),
|
||||
"--apply" if apply else "--dry-run",
|
||||
"--chat-id",
|
||||
chat_id,
|
||||
"--grace-seconds",
|
||||
"60",
|
||||
"--max-seconds",
|
||||
"120",
|
||||
"--backup-dir",
|
||||
str(backups),
|
||||
"--output",
|
||||
str(path),
|
||||
]
|
||||
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
code = launcher.main()
|
||||
|
||||
finally:
|
||||
sys.argv = previous_argv
|
||||
|
||||
check(
|
||||
code == expected_code,
|
||||
"Unexpected return code at " + label,
|
||||
)
|
||||
|
||||
report = json.loads(
|
||||
path.read_text(encoding="utf-8")
|
||||
)
|
||||
|
||||
summary.append({
|
||||
"stage": label,
|
||||
"status": report["status"],
|
||||
"elapsed_seconds": report["elapsed_seconds"],
|
||||
"error_type": report["error_type"],
|
||||
})
|
||||
|
||||
print(
|
||||
label
|
||||
+ ": status="
|
||||
+ report["status"]
|
||||
+ ", error_type="
|
||||
+ str(report["error_type"]),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
return report
|
||||
|
||||
def verify_data(expect_orphan):
|
||||
check(
|
||||
helpers.snapshot(chat_id) == metadata_before,
|
||||
"Attachment metadata changed",
|
||||
)
|
||||
|
||||
names = helpers.object_names(chat_id)
|
||||
|
||||
expected = {original_name}
|
||||
|
||||
if expect_orphan:
|
||||
expected.add(orphan_name)
|
||||
|
||||
check(
|
||||
names == expected,
|
||||
"Unexpected S3 object set",
|
||||
)
|
||||
|
||||
result = c.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/"
|
||||
+ attachment["id"]
|
||||
+ "/content",
|
||||
actor=bob,
|
||||
)
|
||||
|
||||
check(
|
||||
result["bytes"] == content,
|
||||
"Confirmed original changed",
|
||||
)
|
||||
|
||||
page = c.call(
|
||||
"GET",
|
||||
"/api/v1/chats/"
|
||||
+ chat_id
|
||||
+ "/messages?limit=100",
|
||||
actor=bob,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
|
||||
check(
|
||||
message in page["items"],
|
||||
"Message or attachment references changed",
|
||||
)
|
||||
|
||||
def restore(name):
|
||||
container_id = saved_services[name]["Id"]
|
||||
|
||||
helpers.command(
|
||||
[
|
||||
"docker",
|
||||
"start",
|
||||
container_id,
|
||||
],
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
wait_container(container_id)
|
||||
stopped.discard(name)
|
||||
|
||||
wait_api(c)
|
||||
|
||||
try:
|
||||
helpers.mc(
|
||||
"pipe",
|
||||
helpers.target(orphan_key),
|
||||
data=content,
|
||||
)
|
||||
|
||||
print(
|
||||
"Waiting 65 seconds "
|
||||
"for the real orphan grace period...",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
time.sleep(65)
|
||||
|
||||
baseline = worker(
|
||||
"01-baseline",
|
||||
apply=False,
|
||||
expected_code=0,
|
||||
)
|
||||
|
||||
check(
|
||||
baseline["counts"].get(
|
||||
"would_delete_orphans", 0
|
||||
) == 1,
|
||||
"No deletion candidate in baseline",
|
||||
)
|
||||
|
||||
verify_data(expect_orphan=True)
|
||||
|
||||
for number, name in enumerate(
|
||||
("minio", "postgres"),
|
||||
start=2,
|
||||
):
|
||||
container_id = saved_services[name]["Id"]
|
||||
|
||||
print(
|
||||
"Stopping "
|
||||
+ name
|
||||
+ " for the experiment",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
stopped.add(name)
|
||||
|
||||
helpers.command(
|
||||
[
|
||||
"docker",
|
||||
"stop",
|
||||
"-t",
|
||||
"10",
|
||||
container_id,
|
||||
],
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
check(
|
||||
not state(container_id).get("Running"),
|
||||
"Dependency did not stop",
|
||||
)
|
||||
|
||||
try:
|
||||
report = worker(
|
||||
f"{number:02d}-{name}-unavailable",
|
||||
apply=True,
|
||||
expected_code=1,
|
||||
)
|
||||
|
||||
check(
|
||||
report["status"] == "failed",
|
||||
"Outage was not reported as failed",
|
||||
)
|
||||
|
||||
check(
|
||||
report["error_type"] is not None,
|
||||
"Missing error type",
|
||||
)
|
||||
|
||||
for operation in (
|
||||
"delete_object",
|
||||
"put_object",
|
||||
):
|
||||
check(
|
||||
report["s3_operation_calls"].get(
|
||||
operation, 0
|
||||
) == 0,
|
||||
"Mutation attempted "
|
||||
"during dependency outage",
|
||||
)
|
||||
|
||||
for counter in (
|
||||
"missing_objects",
|
||||
"unresolved_missing",
|
||||
"deleted_orphans",
|
||||
"restored",
|
||||
):
|
||||
check(
|
||||
report["counts"].get(
|
||||
counter, 0
|
||||
) == 0,
|
||||
"Outage was confused "
|
||||
"with a data inconsistency",
|
||||
)
|
||||
|
||||
finally:
|
||||
restore(name)
|
||||
|
||||
verify_data(expect_orphan=True)
|
||||
|
||||
print(
|
||||
"PASS: "
|
||||
+ name
|
||||
+ " outage aborted reconciliation; "
|
||||
"data and session survived",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
repaired = worker(
|
||||
"04-recovery-apply",
|
||||
apply=True,
|
||||
expected_code=0,
|
||||
)
|
||||
|
||||
check(
|
||||
repaired["counts"].get(
|
||||
"deleted_orphans", 0
|
||||
) == 1,
|
||||
"Orphan was not removed after recovery",
|
||||
)
|
||||
|
||||
verify_data(expect_orphan=False)
|
||||
|
||||
repeated = worker(
|
||||
"05-recovery-repeat",
|
||||
apply=True,
|
||||
expected_code=0,
|
||||
)
|
||||
|
||||
for counter in (
|
||||
"deleted_orphans",
|
||||
"restored",
|
||||
"missing_objects",
|
||||
"unresolved_missing",
|
||||
):
|
||||
check(
|
||||
repeated["counts"].get(
|
||||
counter, 0
|
||||
) == 0,
|
||||
"Repeated run is not clean",
|
||||
)
|
||||
|
||||
verify_data(expect_orphan=False)
|
||||
completed = True
|
||||
|
||||
finally:
|
||||
recovery_errors = []
|
||||
|
||||
for name in tuple(stopped):
|
||||
try:
|
||||
restore(name)
|
||||
|
||||
except Exception:
|
||||
recovery_errors.append(name)
|
||||
|
||||
print(
|
||||
"WARNING: could not restore service: "
|
||||
+ name,
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if not recovery_errors:
|
||||
try:
|
||||
if orphan_name in helpers.object_names(chat_id):
|
||||
helpers.mc(
|
||||
"rm",
|
||||
helpers.target(orphan_key),
|
||||
)
|
||||
|
||||
except Exception:
|
||||
recovery_errors.append(
|
||||
"test orphan cleanup"
|
||||
)
|
||||
|
||||
print(
|
||||
"WARNING: cannot remove synthetic orphan: "
|
||||
+ orphan_key,
|
||||
flush=True,
|
||||
)
|
||||
|
||||
check(
|
||||
not recovery_errors,
|
||||
"Recovery incomplete: "
|
||||
+ ", ".join(recovery_errors),
|
||||
)
|
||||
|
||||
if completed:
|
||||
report = {
|
||||
"passed": True,
|
||||
"chat_id": chat_id,
|
||||
"stages": summary,
|
||||
}
|
||||
|
||||
(directory / "summary.json").write_text(
|
||||
json.dumps(
|
||||
report,
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
print(
|
||||
"PASS: reconciliation resumes after recovery "
|
||||
"and repeated apply changes nothing"
|
||||
)
|
||||
|
||||
print("Reports: " + str(directory))
|
||||
|
||||
print(
|
||||
"PASS: reconciler dependency failure "
|
||||
"scenario completed"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
FIXTURE = (
|
||||
Path(__file__).resolve().parent
|
||||
/ "fixtures"
|
||||
/ "sample.png"
|
||||
)
|
||||
|
||||
|
||||
def command_output(command):
|
||||
result = subprocess.run(
|
||||
command,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
check(
|
||||
result.returncode == 0,
|
||||
"Docker command failed; check that API containers are running",
|
||||
)
|
||||
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def api_containers():
|
||||
output = command_output([
|
||||
"docker", "compose", "ps", "-q", "api",
|
||||
])
|
||||
|
||||
containers = output.splitlines()
|
||||
|
||||
check(
|
||||
len(containers) == 2,
|
||||
"This test requires exactly two running API replicas",
|
||||
)
|
||||
|
||||
for container in containers:
|
||||
check(
|
||||
re.fullmatch(r"[a-f0-9]{12,64}", container) is not None,
|
||||
"Unexpected container ID",
|
||||
)
|
||||
|
||||
return containers
|
||||
|
||||
|
||||
def temporary_files(containers):
|
||||
result = set()
|
||||
|
||||
for container in containers:
|
||||
output = command_output([
|
||||
"docker", "exec", container,
|
||||
"sh", "-ec",
|
||||
'''
|
||||
directory="${UPLOAD_TEMP_DIR:-/tmp/messenger-uploads}"
|
||||
|
||||
if [ -d "$directory" ]; then
|
||||
find "$directory" \
|
||||
-maxdepth 1 \
|
||||
-type f \
|
||||
-name 'upload-*.tmp' \
|
||||
-print
|
||||
fi
|
||||
''',
|
||||
])
|
||||
|
||||
for filename in output.splitlines():
|
||||
if filename:
|
||||
result.add((container, filename))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def wait_for_empty_directory(containers, seconds=25):
|
||||
deadline = time.monotonic() + seconds
|
||||
|
||||
while True:
|
||||
remaining = temporary_files(containers)
|
||||
|
||||
if not remaining:
|
||||
return
|
||||
|
||||
check(
|
||||
time.monotonic() < deadline,
|
||||
"Temporary upload files were not removed",
|
||||
)
|
||||
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Interrupted upload cleanup check")
|
||||
|
||||
client = Client(
|
||||
args.base_url,
|
||||
args.ca_file,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
address = urlsplit(client.base)
|
||||
|
||||
check(
|
||||
address.scheme == "http",
|
||||
"This local test expects an HTTP endpoint",
|
||||
)
|
||||
|
||||
containers = api_containers()
|
||||
|
||||
check(
|
||||
not temporary_files(containers),
|
||||
"Temporary directory is not empty. "
|
||||
"Finish other uploads and inspect leftover files first",
|
||||
)
|
||||
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
|
||||
original = FIXTURE.read_bytes()
|
||||
|
||||
check(
|
||||
len(original) > 32768,
|
||||
"Fixture must be larger than the partial upload",
|
||||
)
|
||||
|
||||
path = (
|
||||
address.path.rstrip("/")
|
||||
+ "/api/v1/chats/"
|
||||
+ chat["id"]
|
||||
+ "/attachments?filename=interrupted.png"
|
||||
)
|
||||
|
||||
headers = (
|
||||
f"POST {path} HTTP/1.1\r\n"
|
||||
f"Host: {address.netloc}\r\n"
|
||||
f"Authorization: Bearer {alice['token']}\r\n"
|
||||
"Content-Type: image/png\r\n"
|
||||
f"Content-Length: {len(original)}\r\n"
|
||||
"Connection: close\r\n"
|
||||
"\r\n"
|
||||
).encode("ascii")
|
||||
|
||||
connection = socket.create_connection(
|
||||
(address.hostname, address.port or 80),
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
observed = False
|
||||
|
||||
try:
|
||||
connection.sendall(headers)
|
||||
connection.sendall(original[:32768])
|
||||
|
||||
print(
|
||||
"Partial body sent; waiting for a temporary file",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
deadline = time.monotonic() + 5
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
if temporary_files(containers):
|
||||
observed = True
|
||||
break
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
check(
|
||||
observed,
|
||||
"Temporary file was not observed. "
|
||||
"Check API logs and proxy timeouts",
|
||||
)
|
||||
|
||||
print(
|
||||
"PASS: API started receiving the incomplete upload",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
finally:
|
||||
# Закрываем соединение до передачи остальных байтов.
|
||||
try:
|
||||
connection.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
connection.close()
|
||||
|
||||
print("Client connection closed before upload completion", flush=True)
|
||||
|
||||
wait_for_empty_directory(containers)
|
||||
|
||||
print(
|
||||
"PASS: temporary file removed after interrupted upload",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Проверяем, что разрешение UploadLimiter не осталось занятым.
|
||||
# Последовательными запросами через LB наблюдаем обе реплики.
|
||||
instances = set()
|
||||
last_attachment = None
|
||||
|
||||
for _ in range(20):
|
||||
response = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"]
|
||||
+ "/attachments?filename=after-abort.png",
|
||||
201,
|
||||
actor=alice,
|
||||
raw=original,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
|
||||
attachment = response["json"]
|
||||
|
||||
check(
|
||||
attachment["status"] == "ready",
|
||||
"Upload did not become ready",
|
||||
)
|
||||
|
||||
check(
|
||||
attachment["sha256"]
|
||||
== hashlib.sha256(original).hexdigest(),
|
||||
"Uploaded checksum mismatch",
|
||||
)
|
||||
|
||||
instances.add(response["headers"]["X-Instance-Id"])
|
||||
last_attachment = attachment
|
||||
|
||||
if len(instances) >= 2:
|
||||
break
|
||||
|
||||
check(
|
||||
len(instances) == 2,
|
||||
"Successful uploads were not observed on both API replicas",
|
||||
)
|
||||
|
||||
print(
|
||||
"PASS: both API replicas accept uploads after cancellation",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
downloaded = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/"
|
||||
+ last_attachment["id"]
|
||||
+ "/content",
|
||||
actor=bob,
|
||||
)
|
||||
|
||||
check(
|
||||
downloaded["bytes"] == original,
|
||||
"Downloaded file differs from the original",
|
||||
)
|
||||
|
||||
wait_for_empty_directory(containers)
|
||||
|
||||
print("PASS: subsequent upload is intact", flush=True)
|
||||
print("PASS: interrupted upload scenario completed", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
from upload_abort_check import (
|
||||
FIXTURE,
|
||||
api_containers,
|
||||
command_output,
|
||||
temporary_files,
|
||||
)
|
||||
|
||||
|
||||
def container_running(container):
|
||||
value = command_output([
|
||||
"docker", "inspect",
|
||||
"--format", "{{.State.Running}}",
|
||||
container,
|
||||
])
|
||||
|
||||
return value == "true"
|
||||
|
||||
|
||||
def container_healthy(container):
|
||||
value = command_output([
|
||||
"docker", "inspect",
|
||||
"--format", "{{.State.Health.Status}}",
|
||||
container,
|
||||
])
|
||||
|
||||
return value == "healthy"
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("API crash during an incomplete upload")
|
||||
|
||||
client = Client(
|
||||
args.base_url,
|
||||
args.ca_file,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
address = urlsplit(client.base)
|
||||
|
||||
check(
|
||||
address.scheme == "http",
|
||||
"This local test expects HTTP",
|
||||
)
|
||||
|
||||
containers = api_containers()
|
||||
|
||||
check(
|
||||
not temporary_files(containers),
|
||||
"Temporary directories must be empty before the test",
|
||||
)
|
||||
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
|
||||
good_attachment, original = client.attachment(alice, chat)
|
||||
|
||||
content = FIXTURE.read_bytes()
|
||||
|
||||
check(
|
||||
len(content) > 32768,
|
||||
"Fixture is too small",
|
||||
)
|
||||
|
||||
path = (
|
||||
address.path.rstrip("/")
|
||||
+ "/api/v1/chats/"
|
||||
+ chat["id"]
|
||||
+ "/attachments?filename=crash-test.png"
|
||||
)
|
||||
|
||||
headers = (
|
||||
f"POST {path} HTTP/1.1\r\n"
|
||||
f"Host: {address.netloc}\r\n"
|
||||
f"Authorization: Bearer {alice['token']}\r\n"
|
||||
"Content-Type: image/png\r\n"
|
||||
f"Content-Length: {len(content)}\r\n"
|
||||
"Connection: close\r\n"
|
||||
"\r\n"
|
||||
).encode("ascii")
|
||||
|
||||
connection = None
|
||||
target = None
|
||||
temporary_path = None
|
||||
|
||||
try:
|
||||
connection = socket.create_connection(
|
||||
(address.hostname, address.port or 80),
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
connection.sendall(headers)
|
||||
connection.sendall(content[:32768])
|
||||
|
||||
print("Partial upload started", flush=True)
|
||||
|
||||
deadline = time.monotonic() + 5
|
||||
|
||||
while True:
|
||||
found = temporary_files(containers)
|
||||
|
||||
if found:
|
||||
check(
|
||||
len(found) == 1,
|
||||
"Expected exactly one active temporary file",
|
||||
)
|
||||
|
||||
target, temporary_path = next(iter(found))
|
||||
break
|
||||
|
||||
check(
|
||||
time.monotonic() < deadline,
|
||||
"Temporary file was not observed",
|
||||
)
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
command_output([
|
||||
"docker", "exec", target,
|
||||
"touch", "-t", "200001010000",
|
||||
temporary_path,
|
||||
])
|
||||
|
||||
print(
|
||||
"Temporary file found; test timestamp set to the past",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
command_output([
|
||||
"docker", "kill",
|
||||
"--signal", "KILL",
|
||||
target,
|
||||
])
|
||||
|
||||
check(
|
||||
not container_running(target),
|
||||
"Target API container did not stop",
|
||||
)
|
||||
|
||||
connection.close()
|
||||
connection = None
|
||||
|
||||
copied = subprocess.run(
|
||||
[
|
||||
"docker", "cp",
|
||||
target + ":" + temporary_path,
|
||||
"-",
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
check(
|
||||
copied.returncode == 0,
|
||||
"Temporary file did not survive the process crash",
|
||||
)
|
||||
|
||||
print(
|
||||
"PASS: temporary file remains after SIGKILL",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
command_output([
|
||||
"docker", "start", target,
|
||||
])
|
||||
|
||||
ready_deadline = time.monotonic() + 120
|
||||
|
||||
while not container_healthy(target):
|
||||
check(
|
||||
container_running(target),
|
||||
"API exited after restart",
|
||||
)
|
||||
|
||||
check(
|
||||
time.monotonic() < ready_deadline,
|
||||
"API did not become healthy after restart",
|
||||
)
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
print(
|
||||
"PASS: the same API container recovered",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
print(
|
||||
"Waiting for temporary-file cleanup...",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
cleanup_deadline = time.monotonic() + 120
|
||||
|
||||
while (target, temporary_path) in temporary_files([target]):
|
||||
check(
|
||||
time.monotonic() < cleanup_deadline,
|
||||
"Old temporary file was not removed",
|
||||
)
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
print(
|
||||
"PASS: stale temporary file removed after API restart",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
downloaded = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/"
|
||||
+ good_attachment["id"]
|
||||
+ "/content",
|
||||
actor=bob,
|
||||
)
|
||||
|
||||
check(
|
||||
downloaded["bytes"] == original,
|
||||
"Confirmed attachment was lost or changed",
|
||||
)
|
||||
|
||||
print(
|
||||
"PASS: existing session and confirmed attachment survived",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
client.attachment(alice, chat)
|
||||
|
||||
print("PASS: subsequent upload succeeds", flush=True)
|
||||
print("PASS: API crash scenario completed", flush=True)
|
||||
|
||||
finally:
|
||||
if connection is not None:
|
||||
connection.close()
|
||||
|
||||
if target is not None and not container_running(target):
|
||||
command_output([
|
||||
"docker", "start", target,
|
||||
])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,278 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def execute(command, input_text=None):
|
||||
result = subprocess.run(
|
||||
command,
|
||||
input=input_text,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
check(
|
||||
result.returncode == 0,
|
||||
"Infrastructure command failed. "
|
||||
"Check PostgreSQL, MinIO and the configured bucket",
|
||||
)
|
||||
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def sql(query):
|
||||
return execute(
|
||||
[
|
||||
"docker", "compose", "exec", "-T",
|
||||
"postgres", "sh", "-ec",
|
||||
'''
|
||||
exec psql -X \
|
||||
-v ON_ERROR_STOP=1 \
|
||||
-U "$POSTGRES_USER" \
|
||||
-d "$POSTGRES_DB" \
|
||||
-At
|
||||
''',
|
||||
],
|
||||
input_text=query,
|
||||
)
|
||||
|
||||
|
||||
def minio(script, *arguments):
|
||||
return execute([
|
||||
"docker", "compose", "exec", "-T",
|
||||
"minio", "sh", "-ec",
|
||||
script,
|
||||
"failure-check",
|
||||
*arguments,
|
||||
])
|
||||
|
||||
|
||||
def configure_minio():
|
||||
minio(
|
||||
'''
|
||||
mc alias set failurecheck http://localhost:9000 \
|
||||
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
|
||||
'''
|
||||
)
|
||||
|
||||
|
||||
def list_objects(bucket, prefix):
|
||||
output = minio(
|
||||
'mc ls --json "failurecheck/$1/$2"',
|
||||
bucket,
|
||||
prefix,
|
||||
)
|
||||
|
||||
objects = set()
|
||||
|
||||
for line in output.splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
item = json.loads(line)
|
||||
|
||||
check(
|
||||
item.get("status") != "error",
|
||||
"MinIO failed to list test objects",
|
||||
)
|
||||
|
||||
if item.get("key"):
|
||||
objects.add(item["key"])
|
||||
|
||||
return objects
|
||||
|
||||
|
||||
def wait_until(deadline):
|
||||
remaining = deadline - time.monotonic()
|
||||
|
||||
if remaining > 0:
|
||||
time.sleep(remaining)
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument(
|
||||
"--bucket",
|
||||
default="messenger-attachments",
|
||||
)
|
||||
|
||||
args = arguments(
|
||||
"Failure between S3 upload and metadata INSERT",
|
||||
configure,
|
||||
)
|
||||
|
||||
check(
|
||||
re.fullmatch(
|
||||
r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]",
|
||||
args.bucket,
|
||||
) is not None,
|
||||
"Invalid bucket name",
|
||||
)
|
||||
|
||||
client = Client(
|
||||
args.base_url,
|
||||
args.ca_file,
|
||||
timeout=40,
|
||||
)
|
||||
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
|
||||
chat_id = str(uuid.UUID(chat["id"]))
|
||||
|
||||
configure_minio()
|
||||
|
||||
good_attachment, original = client.attachment(alice, chat)
|
||||
|
||||
prefix = f"originals/{chat_id}/"
|
||||
|
||||
before = list_objects(args.bucket, prefix)
|
||||
|
||||
check(
|
||||
len(before) == 1,
|
||||
"Expected one control object in the new test chat",
|
||||
)
|
||||
|
||||
marker = uuid.uuid4().hex
|
||||
filename = f"db-failure-{marker}.png"
|
||||
|
||||
function_name = f"test_attachment_failure_{marker}"
|
||||
trigger_name = f"test_attachment_failure_{marker}"
|
||||
|
||||
cleanup_sql = f"""
|
||||
DROP TRIGGER IF EXISTS {trigger_name}
|
||||
ON public.attachments;
|
||||
|
||||
DROP FUNCTION IF EXISTS public.{function_name}();
|
||||
"""
|
||||
|
||||
try:
|
||||
sql(f"""
|
||||
CREATE FUNCTION public.{function_name}()
|
||||
RETURNS trigger
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
RAISE EXCEPTION
|
||||
'Synthetic attachment metadata failure';
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE TRIGGER {trigger_name}
|
||||
BEFORE INSERT ON public.attachments
|
||||
FOR EACH ROW
|
||||
WHEN (NEW.filename = '{filename}')
|
||||
EXECUTE FUNCTION public.{function_name}();
|
||||
""")
|
||||
|
||||
print(
|
||||
"Installed a DB failure for one synthetic filename",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
response = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat_id
|
||||
+ "/attachments?filename="
|
||||
+ quote(filename, safe=""),
|
||||
503,
|
||||
actor=alice,
|
||||
raw=original,
|
||||
headers={"Content-Type": "image/png"},
|
||||
)
|
||||
|
||||
check(
|
||||
response["json"]["error"]["code"] == "unavailable",
|
||||
"Expected unavailable error code",
|
||||
)
|
||||
|
||||
print("PASS: failed metadata INSERT returns 503", flush=True)
|
||||
|
||||
sql(cleanup_sql)
|
||||
|
||||
count = sql(f"""
|
||||
SELECT count(*)
|
||||
FROM public.attachments
|
||||
WHERE chat_id = '{chat_id}'::uuid
|
||||
AND filename = '{filename}';
|
||||
""")
|
||||
|
||||
check(
|
||||
count == "0",
|
||||
"Metadata was unexpectedly committed",
|
||||
)
|
||||
|
||||
after = list_objects(args.bucket, prefix)
|
||||
orphan_objects = after - before
|
||||
|
||||
check(
|
||||
len(orphan_objects) == 1,
|
||||
"Expected exactly one S3 object without metadata",
|
||||
)
|
||||
|
||||
orphan = next(iter(orphan_objects))
|
||||
observed_at = time.monotonic()
|
||||
|
||||
print(
|
||||
"PASS: S3 object exists, but metadata was rolled back",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
wait_until(observed_at + 25)
|
||||
|
||||
check(
|
||||
orphan in list_objects(args.bucket, prefix),
|
||||
"Orphan was deleted before the grace period",
|
||||
)
|
||||
|
||||
print("PASS: grace period protects the recent object", flush=True)
|
||||
print("Waiting for scheduled cleanup...", flush=True)
|
||||
|
||||
deadline = observed_at + 100
|
||||
|
||||
while orphan in list_objects(args.bucket, prefix):
|
||||
check(
|
||||
time.monotonic() < deadline,
|
||||
"Orphan was not deleted. Check cleanup settings and logs",
|
||||
)
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
print(
|
||||
"PASS: orphan created by a real failed INSERT was deleted",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
downloaded = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/"
|
||||
+ good_attachment["id"]
|
||||
+ "/content",
|
||||
actor=bob,
|
||||
)
|
||||
|
||||
check(
|
||||
downloaded["bytes"] == original,
|
||||
"Control attachment was lost or modified",
|
||||
)
|
||||
|
||||
print("PASS: control attachment is intact", flush=True)
|
||||
|
||||
client.attachment(alice, chat)
|
||||
|
||||
print("PASS: normal uploads work after the failure", flush=True)
|
||||
print("PASS: S3/DB failure scenario completed", flush=True)
|
||||
|
||||
finally:
|
||||
sql(cleanup_sql)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,495 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from attachments_negative import make_png
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def docker_output(command):
|
||||
result = subprocess.run(
|
||||
command,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
check(
|
||||
result.returncode == 0,
|
||||
"Docker command failed while measuring memory",
|
||||
)
|
||||
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def get_api_containers():
|
||||
output = docker_output([
|
||||
"docker", "compose", "ps", "-q", "api",
|
||||
])
|
||||
|
||||
containers = output.splitlines()
|
||||
|
||||
check(
|
||||
len(containers) == 2,
|
||||
"Expected exactly two running API replicas",
|
||||
)
|
||||
|
||||
return containers
|
||||
|
||||
|
||||
def memory_snapshot(container):
|
||||
# Docker Desktop с cgroup v2.
|
||||
# memory.current включает память всего контейнера,
|
||||
# а не только Java heap.
|
||||
output = docker_output([
|
||||
"docker", "exec", container,
|
||||
"sh", "-ec",
|
||||
'''
|
||||
cat /sys/fs/cgroup/memory.current
|
||||
|
||||
if [ -r /sys/fs/cgroup/memory.peak ]; then
|
||||
cat /sys/fs/cgroup/memory.peak
|
||||
else
|
||||
echo unavailable
|
||||
fi
|
||||
|
||||
cat /sys/fs/cgroup/memory.events
|
||||
''',
|
||||
])
|
||||
|
||||
lines = output.splitlines()
|
||||
|
||||
check(
|
||||
len(lines) >= 2,
|
||||
"Cannot read cgroup v2 memory statistics",
|
||||
)
|
||||
|
||||
events = {}
|
||||
|
||||
for line in lines[2:]:
|
||||
key, value = line.split()
|
||||
events[key] = int(value)
|
||||
|
||||
return {
|
||||
"current_bytes": int(lines[0]),
|
||||
"lifetime_peak_bytes": (
|
||||
int(lines[1])
|
||||
if lines[1].isdigit()
|
||||
else None
|
||||
),
|
||||
"oom": events.get("oom", 0),
|
||||
"oom_kill": events.get("oom_kill", 0),
|
||||
}
|
||||
|
||||
|
||||
def mib(value):
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
return round(value / 1024 / 1024, 2)
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
if not values:
|
||||
return None
|
||||
|
||||
ordered = sorted(values)
|
||||
index = int((len(ordered) - 1) * fraction)
|
||||
|
||||
return round(ordered[index], 2)
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument("--concurrency", type=int, default=10)
|
||||
parser.add_argument("--duration", type=int, default=30)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default="results/upload-memory.json",
|
||||
)
|
||||
|
||||
args = arguments(
|
||||
"Concurrent image uploads and API memory measurement",
|
||||
configure,
|
||||
)
|
||||
|
||||
check(
|
||||
args.concurrency >= 10,
|
||||
"Use at least 10 concurrent clients",
|
||||
)
|
||||
|
||||
check(
|
||||
args.duration >= 10,
|
||||
"Duration must be at least 10 seconds",
|
||||
)
|
||||
|
||||
controller = Client(
|
||||
args.base_url,
|
||||
args.ca_file,
|
||||
timeout=40,
|
||||
)
|
||||
|
||||
alice, bob, eve, chat = controller.bootstrap()
|
||||
containers = get_api_containers()
|
||||
|
||||
print("Generating PNG 6000x6000...", flush=True)
|
||||
|
||||
image = make_png(6000, 6000)
|
||||
checksum = hashlib.sha256(image).hexdigest()
|
||||
|
||||
check(
|
||||
len(image) <= 10 * 1024 * 1024,
|
||||
"Generated image exceeds upload byte limit",
|
||||
)
|
||||
|
||||
print(
|
||||
f"Image: {len(image)} bytes, 36000000 pixels",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
path = (
|
||||
"/api/v1/chats/"
|
||||
+ chat["id"]
|
||||
+ "/attachments?filename=memory-test.png"
|
||||
)
|
||||
|
||||
clients = [
|
||||
Client(args.base_url, args.ca_file, timeout=40)
|
||||
for _ in range(args.concurrency)
|
||||
]
|
||||
|
||||
baseline = {
|
||||
container: memory_snapshot(container)
|
||||
for container in containers
|
||||
}
|
||||
|
||||
samples = {container: [] for container in containers}
|
||||
monitor_errors = []
|
||||
stop_monitor = threading.Event()
|
||||
|
||||
started = time.monotonic()
|
||||
|
||||
def monitor():
|
||||
while not stop_monitor.is_set():
|
||||
for container in containers:
|
||||
try:
|
||||
snapshot = memory_snapshot(container)
|
||||
|
||||
samples[container].append({
|
||||
"elapsed_seconds": round(
|
||||
time.monotonic() - started,
|
||||
3,
|
||||
),
|
||||
**snapshot,
|
||||
})
|
||||
|
||||
except Exception as error:
|
||||
monitor_errors.append(
|
||||
type(error).__name__
|
||||
)
|
||||
stop_monitor.set()
|
||||
return
|
||||
|
||||
stop_monitor.wait(0.5)
|
||||
|
||||
barrier = threading.Barrier(args.concurrency + 1)
|
||||
|
||||
def worker(index):
|
||||
client = clients[index]
|
||||
|
||||
counters = Counter()
|
||||
latencies = []
|
||||
errors = []
|
||||
instances = set()
|
||||
successful_instances = set()
|
||||
first_success = None
|
||||
|
||||
barrier.wait()
|
||||
|
||||
while time.monotonic() - started < args.duration:
|
||||
request_started = time.monotonic()
|
||||
|
||||
try:
|
||||
response = client.call(
|
||||
"POST",
|
||||
path,
|
||||
expected=[201, 429],
|
||||
actor=alice,
|
||||
raw=image,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
|
||||
instance = response["headers"]["X-Instance-Id"]
|
||||
instances.add(instance)
|
||||
|
||||
if response["status"] == 429:
|
||||
check(
|
||||
response["json"]["error"]["code"]
|
||||
== "rate_limited",
|
||||
"Unexpected error code for 429",
|
||||
)
|
||||
|
||||
counters["rate_limited"] += 1
|
||||
|
||||
else:
|
||||
attachment = response["json"]
|
||||
|
||||
check(
|
||||
attachment["status"] == "ready",
|
||||
"Successful upload is not ready",
|
||||
)
|
||||
|
||||
check(
|
||||
attachment["sha256"] == checksum,
|
||||
"Uploaded SHA-256 mismatch",
|
||||
)
|
||||
|
||||
check(
|
||||
attachment["size_bytes"] == len(image),
|
||||
"Uploaded size mismatch",
|
||||
)
|
||||
|
||||
counters["successful"] += 1
|
||||
successful_instances.add(instance)
|
||||
|
||||
latencies.append(
|
||||
(time.monotonic() - request_started) * 1000
|
||||
)
|
||||
|
||||
if first_success is None:
|
||||
first_success = attachment
|
||||
|
||||
except Exception as error:
|
||||
counters["unexpected_errors"] += 1
|
||||
|
||||
if len(errors) < 3:
|
||||
errors.append(str(error))
|
||||
|
||||
time.sleep(0.1)
|
||||
|
||||
return {
|
||||
"counters": dict(counters),
|
||||
"latencies": latencies,
|
||||
"errors": errors,
|
||||
"instances": instances,
|
||||
"successful_instances": successful_instances,
|
||||
"first_success": first_success,
|
||||
}
|
||||
|
||||
monitor_thread = threading.Thread(
|
||||
target=monitor,
|
||||
daemon=True,
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=args.concurrency
|
||||
) as executor:
|
||||
|
||||
futures = [
|
||||
executor.submit(worker, index)
|
||||
for index in range(args.concurrency)
|
||||
]
|
||||
|
||||
started = time.monotonic()
|
||||
monitor_thread.start()
|
||||
|
||||
print(
|
||||
f"Starting {args.concurrency} clients "
|
||||
f"for {args.duration} seconds...",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
barrier.wait()
|
||||
|
||||
worker_results = [
|
||||
future.result()
|
||||
for future in futures
|
||||
]
|
||||
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
stop_monitor.set()
|
||||
monitor_thread.join(timeout=25)
|
||||
|
||||
check(
|
||||
not monitor_thread.is_alive(),
|
||||
"Memory monitor did not stop",
|
||||
)
|
||||
|
||||
final = {
|
||||
container: memory_snapshot(container)
|
||||
for container in containers
|
||||
}
|
||||
|
||||
totals = Counter()
|
||||
latencies = []
|
||||
errors = []
|
||||
instances = set()
|
||||
successful_instances = set()
|
||||
|
||||
for result in worker_results:
|
||||
totals.update(result["counters"])
|
||||
latencies.extend(result["latencies"])
|
||||
instances.update(result["instances"])
|
||||
successful_instances.update(
|
||||
result["successful_instances"]
|
||||
)
|
||||
|
||||
errors.extend(result["errors"])
|
||||
|
||||
download_errors = []
|
||||
|
||||
for result in worker_results:
|
||||
attachment = result["first_success"]
|
||||
|
||||
if attachment is None:
|
||||
continue
|
||||
|
||||
try:
|
||||
response = controller.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/"
|
||||
+ attachment["id"]
|
||||
+ "/content",
|
||||
actor=bob,
|
||||
)
|
||||
|
||||
check(
|
||||
response["bytes"] == image,
|
||||
"Downloaded image differs from the original",
|
||||
)
|
||||
|
||||
except Exception as error:
|
||||
download_errors.append(str(error))
|
||||
|
||||
memory_report = {}
|
||||
|
||||
for container in containers:
|
||||
measurements = [
|
||||
baseline[container]["current_bytes"],
|
||||
final[container]["current_bytes"],
|
||||
]
|
||||
|
||||
measurements.extend(
|
||||
sample["current_bytes"]
|
||||
for sample in samples[container]
|
||||
)
|
||||
|
||||
memory_report[container[:12]] = {
|
||||
"baseline_mib": mib(
|
||||
baseline[container]["current_bytes"]
|
||||
),
|
||||
"observed_peak_mib": mib(max(measurements)),
|
||||
"final_mib": mib(
|
||||
final[container]["current_bytes"]
|
||||
),
|
||||
"lifetime_peak_before_mib": mib(
|
||||
baseline[container]["lifetime_peak_bytes"]
|
||||
),
|
||||
"lifetime_peak_after_mib": mib(
|
||||
final[container]["lifetime_peak_bytes"]
|
||||
),
|
||||
"oom_delta": (
|
||||
final[container]["oom"]
|
||||
- baseline[container]["oom"]
|
||||
),
|
||||
"oom_kill_delta": (
|
||||
final[container]["oom_kill"]
|
||||
- baseline[container]["oom_kill"]
|
||||
),
|
||||
"sample_count": len(samples[container]),
|
||||
}
|
||||
|
||||
success = totals["successful"]
|
||||
limited = totals["rate_limited"]
|
||||
unexpected = totals["unexpected_errors"]
|
||||
|
||||
passed = (
|
||||
success > 0
|
||||
and unexpected == 0
|
||||
and not monitor_errors
|
||||
and not download_errors
|
||||
and len(successful_instances) >= 2
|
||||
and all(
|
||||
item["sample_count"] > 0
|
||||
and item["oom_delta"] == 0
|
||||
and item["oom_kill_delta"] == 0
|
||||
for item in memory_report.values()
|
||||
)
|
||||
)
|
||||
|
||||
report = {
|
||||
"passed": passed,
|
||||
"workload": "closed-loop image uploads; no automatic retries",
|
||||
"concurrency": args.concurrency,
|
||||
"requested_duration_seconds": args.duration,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"image": {
|
||||
"width": 6000,
|
||||
"height": 6000,
|
||||
"pixels": 36_000_000,
|
||||
"size_bytes": len(image),
|
||||
"sha256": checksum,
|
||||
},
|
||||
"successful_uploads": success,
|
||||
"rate_limited_requests": limited,
|
||||
"unexpected_errors": unexpected,
|
||||
"successful_rps": round(success / elapsed, 3),
|
||||
"observed_instances": sorted(instances),
|
||||
"successful_instances": sorted(successful_instances),
|
||||
"successful_latency_ms": {
|
||||
"p50": percentile(latencies, 0.50),
|
||||
"p95": percentile(latencies, 0.95),
|
||||
"p99": percentile(latencies, 0.99),
|
||||
},
|
||||
"memory": memory_report,
|
||||
"error_examples": errors[:10],
|
||||
"monitor_errors": monitor_errors,
|
||||
"download_errors": download_errors,
|
||||
"notes": [
|
||||
"429 responses are admission-control rejections, not successful uploads.",
|
||||
"Observed memory peak is sampled and may miss short spikes.",
|
||||
"Lifetime cgroup peak includes earlier container activity and startup.",
|
||||
"Container memory includes more than Java heap.",
|
||||
"Concurrency means concurrent clients, not 10 simultaneous decoders.",
|
||||
],
|
||||
}
|
||||
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
saved_report = {
|
||||
**report,
|
||||
"memory_samples": samples,
|
||||
}
|
||||
|
||||
output.write_text(
|
||||
json.dumps(saved_report, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
print(
|
||||
json.dumps(report, ensure_ascii=False, indent=2),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
print("Report saved to:", output, flush=True)
|
||||
|
||||
check(
|
||||
passed,
|
||||
"Experiment detected errors; inspect the saved report",
|
||||
)
|
||||
|
||||
print("PASS: concurrent upload experiment completed", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Reference in New Issue
Block a user