|
|
|
@@ -0,0 +1,345 @@
|
|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
"""Сценарии уровня 4. Стенд уже запущен. Секреты и токены не печатаются."""
|
|
|
|
|
import base64
|
|
|
|
|
import hashlib
|
|
|
|
|
import json
|
|
|
|
|
import struct
|
|
|
|
|
import subprocess
|
|
|
|
|
import sys
|
|
|
|
|
import time
|
|
|
|
|
import uuid
|
|
|
|
|
import zlib
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
|
|
|
from client import Client, Failure, check # noqa: E402
|
|
|
|
|
from image_probe import jpeg_size # noqa: E402
|
|
|
|
|
|
|
|
|
|
BASE = "http://127.0.0.1:8080"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def compose(*args, check_code=True):
|
|
|
|
|
proc = subprocess.run(
|
|
|
|
|
["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", *args],
|
|
|
|
|
cwd=ROOT,
|
|
|
|
|
text=True,
|
|
|
|
|
stdout=subprocess.PIPE,
|
|
|
|
|
stderr=subprocess.STDOUT,
|
|
|
|
|
)
|
|
|
|
|
if check_code and proc.returncode != 0:
|
|
|
|
|
raise Failure(proc.stdout[-2000:])
|
|
|
|
|
return proc
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def psql(sql):
|
|
|
|
|
proc = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-v", "ON_ERROR_STOP=1", "-At", "-c", sql)
|
|
|
|
|
return proc.stdout.strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def queues():
|
|
|
|
|
proc = compose("exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", "name", "messages")
|
|
|
|
|
return proc.stdout
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def queue_depth(name):
|
|
|
|
|
text = queues()
|
|
|
|
|
for line in text.splitlines():
|
|
|
|
|
parts = line.split()
|
|
|
|
|
if len(parts) >= 2 and parts[0] == name and parts[-1].isdigit():
|
|
|
|
|
return int(parts[-1])
|
|
|
|
|
raise Failure("queue %s missing:\n%s" % (name, text))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def wait_depth(name, minimum, seconds=20):
|
|
|
|
|
deadline = time.monotonic() + seconds
|
|
|
|
|
last = None
|
|
|
|
|
while time.monotonic() < deadline:
|
|
|
|
|
try:
|
|
|
|
|
last = queue_depth(name)
|
|
|
|
|
except Failure:
|
|
|
|
|
last = None
|
|
|
|
|
if last is not None and last >= minimum:
|
|
|
|
|
return last
|
|
|
|
|
time.sleep(0.5)
|
|
|
|
|
raise Failure("queue %s depth %s, want >= %s" % (name, last, minimum))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def wait_status(client, actor, att, wanted, seconds=60):
|
|
|
|
|
deadline = time.monotonic() + seconds
|
|
|
|
|
last = att
|
|
|
|
|
while time.monotonic() < deadline:
|
|
|
|
|
last = client.call("GET", "/api/v1/attachments/" + att["id"], actor=actor, schema="Attachment")["json"]
|
|
|
|
|
if last["status"] == wanted:
|
|
|
|
|
return last
|
|
|
|
|
if wanted != "failed" and last["status"] == "failed":
|
|
|
|
|
raise Failure("attachment failed: " + str(last.get("error_code")))
|
|
|
|
|
time.sleep(0.4)
|
|
|
|
|
raise Failure("status %s did not become %s" % (last.get("status"), wanted))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def wait_postgres_ready(att_id, seconds=70):
|
|
|
|
|
deadline = time.monotonic() + seconds
|
|
|
|
|
last = ""
|
|
|
|
|
while time.monotonic() < deadline:
|
|
|
|
|
last = psql("SELECT status FROM attachments WHERE id = '%s'" % att_id)
|
|
|
|
|
if last == "ready":
|
|
|
|
|
return
|
|
|
|
|
time.sleep(0.5)
|
|
|
|
|
raise Failure("postgres status stayed " + last)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def register(client):
|
|
|
|
|
username = "u_g4_" + uuid.uuid4().hex[:12]
|
|
|
|
|
password = "T3st-" + uuid.uuid4().hex
|
|
|
|
|
user = client.call(
|
|
|
|
|
"POST", "/api/v1/users", 201,
|
|
|
|
|
body={"username": username, "display_name": "Проверка 4", "password": password},
|
|
|
|
|
schema="User",
|
|
|
|
|
)["json"]
|
|
|
|
|
raw = base64.b64encode((username + ":" + password).encode()).decode()
|
|
|
|
|
session = client.call(
|
|
|
|
|
"POST", "/api/v1/auth/sessions", 201,
|
|
|
|
|
headers={"Authorization": "Basic " + raw},
|
|
|
|
|
schema="Session",
|
|
|
|
|
)["json"]
|
|
|
|
|
return {"id": user["id"], "token": session["token"], "user": user}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def upload(client, actor, chat_id, filename, content, content_type="image/png"):
|
|
|
|
|
return client.call(
|
|
|
|
|
"POST",
|
|
|
|
|
"/api/v1/chats/%s/attachments?filename=%s" % (chat_id, filename),
|
|
|
|
|
202,
|
|
|
|
|
actor=actor,
|
|
|
|
|
raw=content,
|
|
|
|
|
headers={"Content-Type": content_type},
|
|
|
|
|
schema="Attachment",
|
|
|
|
|
)["json"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def bad_png():
|
|
|
|
|
ihdr = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)
|
|
|
|
|
chunk = b"IHDR" + ihdr
|
|
|
|
|
crc = zlib.crc32(chunk) & 0xFFFFFFFF
|
|
|
|
|
return b"\x89PNG\r\n\x1a\n" + struct.pack(">I", len(ihdr)) + chunk + struct.pack(">I", crc)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def webp_size(data):
|
|
|
|
|
check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP")
|
|
|
|
|
index = 12
|
|
|
|
|
while index + 8 <= len(data):
|
|
|
|
|
kind = data[index:index + 4]
|
|
|
|
|
size = int.from_bytes(data[index + 4:index + 8], "little")
|
|
|
|
|
chunk = data[index + 8:index + 8 + size]
|
|
|
|
|
if kind == b"VP8X" and len(chunk) >= 10:
|
|
|
|
|
return (1 + int.from_bytes(chunk[4:7], "little"), 1 + int.from_bytes(chunk[7:10], "little"))
|
|
|
|
|
if kind == b"VP8 " and len(chunk) >= 10:
|
|
|
|
|
return (int.from_bytes(chunk[6:8], "little") & 0x3FFF, int.from_bytes(chunk[8:10], "little") & 0x3FFF)
|
|
|
|
|
if kind == b"VP8L" and len(chunk) >= 5:
|
|
|
|
|
bits = int.from_bytes(chunk[1:5], "little")
|
|
|
|
|
return ((bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1)
|
|
|
|
|
index += 8 + size + (size & 1)
|
|
|
|
|
raise Failure("WebP dimensions not found")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def variant_count(att_id):
|
|
|
|
|
return int(psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % att_id))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
|
|
|
|
sample = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
|
|
|
client = Client(BASE, None)
|
|
|
|
|
client.call("GET", "/health/ready", schema="Health")
|
|
|
|
|
alice = register(client)
|
|
|
|
|
bob_name = "u_g4b_" + uuid.uuid4().hex[:12]
|
|
|
|
|
bob_password = "T3st-" + uuid.uuid4().hex
|
|
|
|
|
bob_user = client.call(
|
|
|
|
|
"POST", "/api/v1/users", 201,
|
|
|
|
|
body={"username": bob_name, "display_name": "Боб", "password": bob_password},
|
|
|
|
|
schema="User",
|
|
|
|
|
)["json"]
|
|
|
|
|
chat = client.call(
|
|
|
|
|
"POST", "/api/v1/chats", 201, actor=alice,
|
|
|
|
|
body={"title": "Очередь", "member_ids": [alice["id"], bob_user["id"]]},
|
|
|
|
|
schema="Chat",
|
|
|
|
|
)["json"]
|
|
|
|
|
notes = []
|
|
|
|
|
|
|
|
|
|
att = upload(client, alice, chat["id"], "sample.png", sample)
|
|
|
|
|
check(att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, "upload snapshot")
|
|
|
|
|
ready = wait_status(client, alice, att, "ready")
|
|
|
|
|
names = [item["name"] for item in ready["variants"]]
|
|
|
|
|
check(names == ["thumbnail", "crop", "optimized"], "variant set " + ",".join(names))
|
|
|
|
|
thumb = next(item for item in ready["variants"] if item["name"] == "thumbnail")
|
|
|
|
|
crop = next(item for item in ready["variants"] if item["name"] == "crop")
|
|
|
|
|
optimized = next(item for item in ready["variants"] if item["name"] == "optimized")
|
|
|
|
|
check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail meta")
|
|
|
|
|
check((crop["width"], crop["height"]) == (128, 128) and crop["content_type"] == "image/jpeg", "crop meta")
|
|
|
|
|
check(optimized["content_type"] == "image/webp" and (optimized["width"], optimized["height"]) == (320, 200), "webp meta")
|
|
|
|
|
original = client.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=alice)
|
|
|
|
|
check(original["bytes"] == sample, "original bytes changed")
|
|
|
|
|
thumb_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=alice)
|
|
|
|
|
crop_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=crop" % att["id"], actor=alice)
|
|
|
|
|
webp_bytes = client.call("GET", "/api/v1/attachments/%s/content?variant=optimized" % att["id"], actor=alice)
|
|
|
|
|
check(jpeg_size(thumb_bytes["bytes"]) == (256, 160), "thumbnail jpeg")
|
|
|
|
|
check(jpeg_size(crop_bytes["bytes"]) == (128, 128), "crop jpeg")
|
|
|
|
|
check(webp_size(webp_bytes["bytes"]) == (320, 200), "webp size")
|
|
|
|
|
check(hashlib.sha256(thumb_bytes["bytes"]).hexdigest() == thumb["sha256"], "thumb sha")
|
|
|
|
|
replay = client.call("POST", "/api/v1/attachments/%s/replay" % att["id"], 200, actor=alice, schema="Attachment")["json"]
|
|
|
|
|
check(replay["status"] == "ready" and replay["variants"] == ready["variants"], "ready replay changed variants")
|
|
|
|
|
check(variant_count(att["id"]) == 3, "replay created another variant set")
|
|
|
|
|
check(psql("SELECT outcome FROM job_replays WHERE attachment_id = '%s' ORDER BY created_at DESC LIMIT 1" % att["id"]) == "ignored_ready", "replay audit")
|
|
|
|
|
notes.append("happy-path ready id=%s variants=3 replay=ignored_ready" % att["id"])
|
|
|
|
|
|
|
|
|
|
compose("stop", "worker")
|
|
|
|
|
held = upload(client, alice, chat["id"], "while-worker-down.png", sample)
|
|
|
|
|
client.call("GET", "/api/v1/attachments/%s/content" % held["id"], 409, actor=alice)
|
|
|
|
|
client.call(
|
|
|
|
|
"POST", "/api/v1/chats/%s/messages" % chat["id"], 201, actor=alice,
|
|
|
|
|
body={"text": "очередь не блокирует чат", "attachment_ids": [held["id"]]},
|
|
|
|
|
headers={"Idempotency-Key": uuid.uuid4().hex},
|
|
|
|
|
schema="Message",
|
|
|
|
|
)
|
|
|
|
|
before = wait_depth("attachment.process", 1)
|
|
|
|
|
compose("restart", "rabbitmq")
|
|
|
|
|
for _ in range(40):
|
|
|
|
|
probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False)
|
|
|
|
|
if probe.returncode == 0:
|
|
|
|
|
break
|
|
|
|
|
time.sleep(1)
|
|
|
|
|
else:
|
|
|
|
|
raise Failure("rabbitmq did not come back")
|
|
|
|
|
after = wait_depth("attachment.process", 1, seconds=30)
|
|
|
|
|
notes.append("broker restart kept %s then %s process messages" % (before, after))
|
|
|
|
|
compose("start", "worker")
|
|
|
|
|
held_ready = wait_status(client, alice, held, "ready")
|
|
|
|
|
check(variant_count(held["id"]) == 3, "restarted job duplicated variants")
|
|
|
|
|
notes.append("worker-down then broker restart kept queue; ready id=%s" % held_ready["id"])
|
|
|
|
|
|
|
|
|
|
compose("stop", "worker")
|
|
|
|
|
compose("stop", "rabbitmq")
|
|
|
|
|
gap = upload(client, alice, chat["id"], "outbox-gap.png", sample)
|
|
|
|
|
unpublished = psql(
|
|
|
|
|
"SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id "
|
|
|
|
|
"WHERE j.attachment_id = '%s' AND o.published_at IS NULL" % gap["id"]
|
|
|
|
|
)
|
|
|
|
|
check(unpublished == "1", "outbox was published while broker was down: " + unpublished)
|
|
|
|
|
compose("stop", "api")
|
|
|
|
|
compose("start", "rabbitmq")
|
|
|
|
|
for _ in range(40):
|
|
|
|
|
probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False)
|
|
|
|
|
if probe.returncode == 0:
|
|
|
|
|
break
|
|
|
|
|
time.sleep(1)
|
|
|
|
|
compose("start", "worker")
|
|
|
|
|
wait_postgres_ready(gap["id"])
|
|
|
|
|
ready_before_api = psql("SELECT status FROM attachments WHERE id = '%s'" % gap["id"])
|
|
|
|
|
check(ready_before_api == "ready", "worker did not recover unpublished outbox")
|
|
|
|
|
compose("start", "api")
|
|
|
|
|
for _ in range(40):
|
|
|
|
|
try:
|
|
|
|
|
client.call("GET", "/health/ready", schema="Health")
|
|
|
|
|
break
|
|
|
|
|
except Failure:
|
|
|
|
|
time.sleep(1)
|
|
|
|
|
else:
|
|
|
|
|
raise Failure("api did not return")
|
|
|
|
|
gap_body = client.call("GET", "/api/v1/attachments/%s" % gap["id"], actor=alice, schema="Attachment")["json"]
|
|
|
|
|
check(gap_body["status"] == "ready" and len(gap_body["variants"]) == 3, "gap attachment not ready via api")
|
|
|
|
|
notes.append("commit/publish gap recovered by worker without a new upload id=%s" % gap["id"])
|
|
|
|
|
|
|
|
|
|
broken = upload(client, alice, chat["id"], "broken.png", bad_png())
|
|
|
|
|
failed = wait_status(client, alice, broken, "failed")
|
|
|
|
|
check(failed["error_code"] == "invalid_image", "permanent code " + str(failed.get("error_code")))
|
|
|
|
|
check(variant_count(broken["id"]) == 0, "failed job stored variants")
|
|
|
|
|
client.call(
|
|
|
|
|
"POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice,
|
|
|
|
|
body={"text": "", "attachment_ids": [broken["id"]]},
|
|
|
|
|
headers={"Idempotency-Key": uuid.uuid4().hex},
|
|
|
|
|
)
|
|
|
|
|
again = client.call("POST", "/api/v1/attachments/%s/replay" % broken["id"], 202, actor=alice, schema="Attachment")["json"]
|
|
|
|
|
check(again["status"] == "queued", "replay of failed did not queue")
|
|
|
|
|
failed_again = wait_status(client, alice, broken, "failed")
|
|
|
|
|
check(failed_again["error_code"] == "invalid_image", "replay did not finish as permanent failure")
|
|
|
|
|
check(variant_count(broken["id"]) == 0, "replay of failed created variants")
|
|
|
|
|
dlq = queues()
|
|
|
|
|
check("attachment.failed" in dlq, "dlq missing")
|
|
|
|
|
notes.append("permanent invalid_image + replay id=%s dlq present" % broken["id"])
|
|
|
|
|
|
|
|
|
|
flaky = upload(client, alice, chat["id"], "__lab_fault_transient.png", sample)
|
|
|
|
|
flaky_ready = wait_status(client, alice, flaky, "ready", seconds=40)
|
|
|
|
|
failures = psql(
|
|
|
|
|
"SELECT count(*) FROM job_failures WHERE attachment_id = '%s' AND permanent = false AND error_code = 'unavailable'"
|
|
|
|
|
% flaky["id"]
|
|
|
|
|
)
|
|
|
|
|
check(int(failures) >= 2, "expected two transient failures, got " + failures)
|
|
|
|
|
check(variant_count(flaky["id"]) == 3, "transient retries duplicated variants")
|
|
|
|
|
attempt = psql("SELECT attempt FROM attachment_jobs WHERE attachment_id = '%s'" % flaky["id"])
|
|
|
|
|
notes.append("backoff retries=%s final_attempt=%s id=%s" % (failures, attempt, flaky_ready["id"]))
|
|
|
|
|
|
|
|
|
|
compose("stop", "worker")
|
|
|
|
|
crash = upload(client, alice, chat["id"], "crash-before-ack.png", sample)
|
|
|
|
|
wait_depth("attachment.process", 1)
|
|
|
|
|
crashed = subprocess.run(
|
|
|
|
|
["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "run", "--rm", "--no-deps", "--name", "lab5-crash-once", "-e", "CRASH_AFTER_S3=1", "worker"],
|
|
|
|
|
cwd=ROOT,
|
|
|
|
|
text=True,
|
|
|
|
|
stdout=subprocess.PIPE,
|
|
|
|
|
stderr=subprocess.STDOUT,
|
|
|
|
|
timeout=45,
|
|
|
|
|
)
|
|
|
|
|
check(crashed.returncode != 0, "crash hook did not stop the worker")
|
|
|
|
|
status_after_crash = psql("SELECT status FROM attachments WHERE id = '%s'" % crash["id"])
|
|
|
|
|
check(status_after_crash == "processing", "crash did not stop between claim and ready: " + status_after_crash)
|
|
|
|
|
compose("start", "worker")
|
|
|
|
|
crash_ready = wait_status(client, alice, crash, "ready")
|
|
|
|
|
check(variant_count(crash["id"]) == 3, "redelivery created a second variant set")
|
|
|
|
|
original_after = client.call("GET", "/api/v1/attachments/%s/content" % crash["id"], actor=alice)
|
|
|
|
|
check(original_after["bytes"] == sample, "redelivery changed the original")
|
|
|
|
|
staging_left = psql("SELECT staging_key IS NULL FROM attachments WHERE id = '%s'" % crash["id"])
|
|
|
|
|
check(staging_left == "t", "staging was not cleared after ready")
|
|
|
|
|
notes.append("crash after s3 before ack redelivery id=%s variants=3 staging cleared" % crash_ready["id"])
|
|
|
|
|
|
|
|
|
|
compose("stop", "worker")
|
|
|
|
|
compose("stop", "rabbitmq")
|
|
|
|
|
leased = upload(client, alice, chat["id"], "expired-lease.png", sample)
|
|
|
|
|
psql(
|
|
|
|
|
"UPDATE attachment_jobs SET status = 'processing', attempt = 1, "
|
|
|
|
|
"lease_token = gen_random_uuid(), lease_until = now() - interval '2 minutes', "
|
|
|
|
|
"updated_at = now() WHERE attachment_id = '%s'" % leased["id"]
|
|
|
|
|
)
|
|
|
|
|
psql("UPDATE attachments SET status = 'processing', updated_at = now() WHERE id = '%s'" % leased["id"])
|
|
|
|
|
psql(
|
|
|
|
|
"UPDATE outbox SET published_at = now() WHERE job_id = "
|
|
|
|
|
"(SELECT id FROM attachment_jobs WHERE attachment_id = '%s')" % leased["id"]
|
|
|
|
|
)
|
|
|
|
|
compose("start", "rabbitmq")
|
|
|
|
|
for _ in range(40):
|
|
|
|
|
probe = compose("exec", "-T", "rabbitmq", "rabbitmq-diagnostics", "-q", "ping", check_code=False)
|
|
|
|
|
if probe.returncode == 0:
|
|
|
|
|
break
|
|
|
|
|
time.sleep(1)
|
|
|
|
|
compose("start", "worker")
|
|
|
|
|
lease_ready = wait_status(client, alice, leased, "ready")
|
|
|
|
|
lease_rows = psql(
|
|
|
|
|
"SELECT count(*) FROM job_failures WHERE attachment_id = '%s' AND error_code = 'lease_expired'"
|
|
|
|
|
% leased["id"]
|
|
|
|
|
)
|
|
|
|
|
check(int(lease_rows) >= 1, "expired lease was not recovered")
|
|
|
|
|
check(variant_count(leased["id"]) == 3, "lease recovery duplicated variants")
|
|
|
|
|
notes.append("expired processing lease recovered id=%s" % lease_ready["id"])
|
|
|
|
|
|
|
|
|
|
text = "\n".join(notes)
|
|
|
|
|
print(text)
|
|
|
|
|
evidence = ROOT / "evidence" / "grade4.txt"
|
|
|
|
|
evidence.write_text(text + "\n", encoding="utf-8")
|
|
|
|
|
print("PASS: grade 4 fault scenarios")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
try:
|
|
|
|
|
main()
|
|
|
|
|
except (Failure, OSError, subprocess.SubprocessError) as exc:
|
|
|
|
|
print("FAIL:", exc)
|
|
|
|
|
raise SystemExit(1)
|