forked from mtusi2026/mtusi2026lab5
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> PNG с фильтром строки 5/0xff, IDAT не zlib, JPEG из 27 байт, SOS с компонентом не из SOF: 415 при загрузке. 40 Мпикс проходит до ready в лимите памяти worker.
126 lines
6.7 KiB
Python
126 lines
6.7 KiB
Python
"""Consistency of PostgreSQL and S3 around one upload (lab 4 level 4, lab 5 failure windows of the API).
|
|
|
|
Lab 5 upload: intent row 'pending' -> staging PUT -> ONE transaction 'queued' + job + outbox -> 202.
|
|
Failures between the steps are injected with FAULT_POINTS of the api service (compose variable
|
|
API_FAULT_POINTS, empty in normal runs; the API raises 503 at that exact point, as if the process
|
|
or the network failed there):
|
|
|
|
* after_s3_put "after staging, before the job is written": staging object exists, row
|
|
still pending, NO job and NO outbox row; the client got 503 -> invisible,
|
|
the janitor removes the staging object and the row after PENDING_GRACE_SECONDS;
|
|
* after_enqueue_commit job committed, the answer is lost (client got 503) -> nothing may be
|
|
deleted: the worker processes the job and the attachment becomes ready.
|
|
|
|
Plus: an upload interrupted by the client and a stalled body leave nothing behind."""
|
|
from __future__ import annotations
|
|
|
|
import socket
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
import imagegen
|
|
from conftest import api_with_env, attachment_row, psql, s3_size
|
|
|
|
pytestmark = pytest.mark.stack
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
|
|
|
|
|
def rows_of_chat(chat_id: str) -> list[tuple[str, str, str]]:
|
|
out = psql(f"SELECT id || ' ' || status || ' ' || staging_key FROM attachments WHERE chat_id = '{chat_id}'")
|
|
return [tuple(line.split()) for line in out.splitlines() if line]
|
|
|
|
|
|
def jobs_of(attachment_id: str) -> tuple[int, int]:
|
|
"""(jobs rows, outbox rows) of an attachment."""
|
|
out = psql(f"SELECT (SELECT count(*) FROM jobs WHERE attachment_id = '{attachment_id}') || ' ' || "
|
|
f"(SELECT count(*) FROM outbox o JOIN jobs j ON j.id = o.job_id WHERE j.attachment_id = '{attachment_id}')")
|
|
jobs, outbox = out.split()
|
|
return int(jobs), int(outbox)
|
|
|
|
|
|
def wait_for(condition, timeout: float, what: str) -> None:
|
|
deadline = time.monotonic() + timeout
|
|
while not condition():
|
|
assert time.monotonic() < deadline, what
|
|
time.sleep(0.5)
|
|
|
|
|
|
def raw_upload(token: str, chat_id: str, declared: int, body: bytes, *, wait_answer: float | None) -> bytes:
|
|
"""Hand-made HTTP request: announce `declared` bytes, send only `body`, then either close the
|
|
socket (client gone) or keep it open and read the answer after `wait_answer` seconds."""
|
|
sock = socket.create_connection(("127.0.0.1", 8080), timeout=30)
|
|
head = (f"POST /api/v1/chats/{chat_id}/attachments?filename=x.png HTTP/1.1\r\nHost: localhost\r\n"
|
|
f"Authorization: Bearer {token}\r\nContent-Type: image/png\r\nContent-Length: {declared}\r\n\r\n")
|
|
sock.sendall(head.encode() + body)
|
|
if wait_answer is None:
|
|
sock.close()
|
|
return b""
|
|
time.sleep(wait_answer)
|
|
answer = sock.recv(65536)
|
|
sock.close()
|
|
return answer
|
|
|
|
|
|
def test_interrupted_and_stalled_uploads_leave_nothing(api):
|
|
alice = api.user("alice")
|
|
chat = api.chat(alice)
|
|
data = imagegen.png_of_size(2_000_000)
|
|
raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=None) # client disconnects
|
|
with api_with_env(UPLOAD_READ_TIMEOUT_SECONDS="2"):
|
|
answer = raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=5) # client stalls
|
|
assert answer.startswith(b"HTTP/1.1 400") and b"invalid_request" in answer, answer[:200]
|
|
assert b"x-instance-id" in answer.lower()
|
|
time.sleep(1)
|
|
assert rows_of_chat(chat["id"]) == [] # nothing was written: the body is checked before any write
|
|
|
|
|
|
def test_crash_after_staging_before_job_is_cleaned_by_janitor(api):
|
|
"""Failure window 1 of contracts/attachment-states.md: the original is in staging, the job is
|
|
not written. No 202 was sent, so nothing was promised: the janitor removes the staging copy
|
|
(object first) and then the intent row; no job, no message ever existed."""
|
|
alice = api.user("alice")
|
|
chat = api.chat(alice)
|
|
with api_with_env(API_FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"):
|
|
assert api.upload(alice, chat, SAMPLE, 503).json()["error"]["code"] == "unavailable"
|
|
[(attachment_id, status, key)] = rows_of_chat(chat["id"])
|
|
assert status == "pending" and s3_size(key, "staging") == len(SAMPLE) # the orphan-to-be exists
|
|
assert jobs_of(attachment_id) == (0, 0) # no job, no outbox row
|
|
api.call("GET", f"/api/v1/attachments/{attachment_id}", 404, token=alice["token"]) # invisible
|
|
api.post_message(alice, chat, {"text": "", "attachment_ids": [attachment_id]}, 404) # not attachable
|
|
wait_for(lambda: rows_of_chat(chat["id"]) == [], 30, "janitor did not remove the stale intent")
|
|
assert s3_size(key, "staging") is None # object removed first
|
|
# Normal configuration again: an upload in the same chat works and is not touched.
|
|
att = api.upload(alice, chat, SAMPLE).json()
|
|
assert api.wait_status(alice, att["id"], "ready")["status"] == "ready"
|
|
|
|
|
|
def test_lost_answer_after_job_commit_still_becomes_ready(api):
|
|
"""The job COMMIT succeeded, the answer was lost (503). Nothing compensates in the request path
|
|
(the janitor only touches 'pending' rows); the worker processes the job as usual."""
|
|
alice, bob = api.user("alice"), api.user("bob")
|
|
chat = api.chat(alice, bob)
|
|
with api_with_env(API_FAULT_POINTS="after_enqueue_commit", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"):
|
|
api.upload(alice, chat, SAMPLE, 503)
|
|
[(attachment_id, status, key)] = rows_of_chat(chat["id"])
|
|
assert status in ("queued", "processing", "ready") and jobs_of(attachment_id)[0] == 1
|
|
att = api.wait_status(bob, attachment_id, "ready")
|
|
time.sleep(4) # several janitor rounds
|
|
assert attachment_row(attachment_id)["status"] == "ready"
|
|
got = api.call("GET", f"/api/v1/attachments/{attachment_id}/content", 200, token=bob["token"])
|
|
assert got.content == SAMPLE and att["sha256"] == __import__("hashlib").sha256(SAMPLE).hexdigest()
|
|
|
|
|
|
def test_fresh_pending_upload_is_not_touched_by_janitor(api):
|
|
"""An intent younger than PENDING_GRACE_SECONDS is an upload in progress: never removed."""
|
|
alice = api.user("alice")
|
|
chat = api.chat(alice)
|
|
with api_with_env(API_FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="600", JANITOR_INTERVAL_SECONDS="1"):
|
|
api.upload(alice, chat, SAMPLE, 503)
|
|
time.sleep(4)
|
|
[(_, status, key)] = rows_of_chat(chat["id"])
|
|
assert status == "pending" and s3_size(key, "staging") == len(SAMPLE)
|