Files
mtusi2026lab4/student-tests/test_consistency.py
T
Alexey BodanyukandClaude Opus 5.5 a2d867bec6 lab4 трек 2: reconciler S3/PostgreSQL, тесты вложений, отказов и согласованности
- python -m messenger.reconcile: merge-join листинга S3 и attachments по object_key;
  dry-run read-only конструктивно (default_transaction_read_only + S3 без записи),
  --apply: orphan/stale intent удаляются, ready-строки только в карантин (обратимо),
  перепроверка под FOR UPDATE, предохранитель от массового удаления;
- attachment_ids хранится в строке сообщения (история остаётся одним index range scan),
  message_attachments — для FK и обратного поиска;
- семафор полной проверки изображений (IMAGE_VERIFY_CONCURRENCY);
- тесты: вложения (56), отказ/пауза S3 и деградированный режим LB, failover загрузки,
  приватность bucket, FAULT_POINTS после PUT/после COMMIT, janitor, reconciler;
- бенчмарки памяти загрузок и стоимости обхода reconciler.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 14:05:03 +03:00

112 lines
5.6 KiB
Python

"""Consistency of PostgreSQL and S3 around one upload (lab 4, level 4).
The two-store write is: intent row 'pending' -> S3 PUT -> row 'ready' -> 201. Failures between
the steps are injected with FAULT_POINTS (compose variable, empty in normal runs; the API
raises 503 at that exact point, as if the process or the network failed there):
* after_s3_put object written, row still pending, client got 503 -> the janitor removes
the object and the row after PENDING_GRACE_SECONDS (here 3 s);
* after_ready_commit row ready and object written, but the answer is lost (client got 503)
-> nothing may be deleted: the attachment stays intact and downloadable.
Plus: an upload interrupted by the client and a stalled body leave nothing behind."""
from __future__ import annotations
import socket
import time
from pathlib import Path
import pytest
import imagegen
from conftest import api_with_env, attachment_row, psql, s3_size
pytestmark = pytest.mark.stack
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
def rows_of_chat(chat_id: str) -> list[tuple[str, str, str]]:
out = psql(f"SELECT id || ' ' || status || ' ' || object_key FROM attachments WHERE chat_id = '{chat_id}'")
return [tuple(line.split()) for line in out.splitlines() if line]
def wait_for(condition, timeout: float, what: str) -> None:
deadline = time.monotonic() + timeout
while not condition():
assert time.monotonic() < deadline, what
time.sleep(0.5)
def raw_upload(token: str, chat_id: str, declared: int, body: bytes, *, wait_answer: float | None) -> bytes:
"""Hand-made HTTP request: announce `declared` bytes, send only `body`, then either close the
socket (client gone) or keep it open and read the answer after `wait_answer` seconds."""
sock = socket.create_connection(("127.0.0.1", 8080), timeout=30)
head = (f"POST /api/v1/chats/{chat_id}/attachments?filename=x.png HTTP/1.1\r\nHost: localhost\r\n"
f"Authorization: Bearer {token}\r\nContent-Type: image/png\r\nContent-Length: {declared}\r\n\r\n")
sock.sendall(head.encode() + body)
if wait_answer is None:
sock.close()
return b""
time.sleep(wait_answer)
answer = sock.recv(65536)
sock.close()
return answer
def test_interrupted_and_stalled_uploads_leave_nothing(api):
alice = api.user("alice")
chat = api.chat(alice)
data = imagegen.png_of_size(2_000_000)
raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=None) # client disconnects
with api_with_env(UPLOAD_READ_TIMEOUT_SECONDS="2"):
answer = raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=5) # client stalls
assert answer.startswith(b"HTTP/1.1 400") and b"invalid_request" in answer, answer[:200]
assert b"x-instance-id" in answer.lower()
time.sleep(1)
assert rows_of_chat(chat["id"]) == [] # nothing was written: the body is checked before any write
def test_crash_between_s3_put_and_commit_is_cleaned_by_janitor(api):
alice = api.user("alice")
chat = api.chat(alice)
with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"):
assert api.upload(alice, chat, SAMPLE, 503).json()["error"]["code"] == "unavailable"
[(attachment_id, status, key)] = rows_of_chat(chat["id"])
assert status == "pending" and s3_size(key) == len(SAMPLE) # the orphan-to-be exists
api.call("GET", f"/api/v1/attachments/{attachment_id}", 404, token=alice["token"]) # invisible
api.post_message(alice, chat, {"text": "", "attachment_ids": [attachment_id]}, 404) # not attachable
wait_for(lambda: rows_of_chat(chat["id"]) == [], 30, "janitor did not remove the stale intent")
assert s3_size(key) is None # object removed first
# Normal configuration again: an upload in the same chat works and is not touched.
att = api.upload(alice, chat, SAMPLE).json()
assert attachment_row(att["id"])["status"] == "ready"
def test_lost_answer_after_commit_never_deletes_the_object(api):
"""The reviewer found this in another PR: COMMIT succeeded, the answer was lost, the API
'compensated' by deleting the object -> ready row without bytes. Here nothing compensates
in the request path, and the janitor only touches 'pending' rows."""
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
with api_with_env(FAULT_POINTS="after_ready_commit", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"):
api.upload(alice, chat, SAMPLE, 503)
[(attachment_id, status, key)] = rows_of_chat(chat["id"])
assert status == "ready"
time.sleep(6) # several janitor rounds
assert rows_of_chat(chat["id"]) == [(attachment_id, "ready", key)] and s3_size(key) == len(SAMPLE)
got = api.call("GET", f"/api/v1/attachments/{attachment_id}/content", 200, token=bob["token"])
assert got.content == SAMPLE
def test_fresh_pending_upload_is_not_touched_by_janitor(api):
"""An intent younger than PENDING_GRACE_SECONDS is an upload in progress: never removed."""
alice = api.user("alice")
chat = api.chat(alice)
with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="600", JANITOR_INTERVAL_SECONDS="1"):
api.upload(alice, chat, SAMPLE, 503)
time.sleep(4)
[(_, status, key)] = rows_of_chat(chat["id"])
assert status == "pending" and s3_size(key) == len(SAMPLE)