forked from mtusi2026/mtusi2026lab5
268 lines
10 KiB
Python
268 lines
10 KiB
Python
"""Retry/replay, fencing, cleanup, limits и транзакции реального PostgreSQL."""
|
|
|
|
import asyncio
|
|
from datetime import timedelta
|
|
import errno
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
from uuid import UUID, uuid4
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.core.jobs import JobSettings
|
|
from app.domain import attachment_jobs
|
|
from app.domain.attachment_jobs import (
|
|
claim_job,
|
|
process_attachment_job,
|
|
cleanup_staging,
|
|
build_result,
|
|
)
|
|
from app.domain.job_admin import replay_failed_job
|
|
from app.domain.job_retry import classify_failure, retry_delay
|
|
from app.errors.exceptions import JobOperationError, ProcessingError
|
|
from app.models.attachment import Attachment
|
|
from app.models.job import AttachmentJob
|
|
from app.models.job_replay import JobReplay
|
|
from app.schemas.job import JobMessage
|
|
from app.storage import jobs
|
|
from test_attachments import context, upload, finish_attachment
|
|
|
|
|
|
async def envelope(session, attachment_id):
|
|
job = await session.scalar(
|
|
select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))
|
|
)
|
|
return JobMessage(
|
|
job_id=job.id,
|
|
attachment_id=job.attachment_id,
|
|
pipeline_version=1,
|
|
attempt=job.attempt + 1,
|
|
created_at=job.created_at,
|
|
)
|
|
|
|
|
|
async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_engine):
|
|
storage, alice, _, _, chat = context
|
|
attachment = (await upload(client, alice, chat)).json()
|
|
settings = JobSettings(_env_file=None, job_max_attempts=3)
|
|
|
|
def fail(_):
|
|
raise TimeoutError
|
|
|
|
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
|
first = await envelope(session, attachment["id"])
|
|
for attempt in range(1, 4):
|
|
message = await envelope(session, attachment["id"])
|
|
assert await process_attachment_job(
|
|
session, storage, message, settings, before_processing=fail
|
|
)
|
|
job, item = await jobs.lock_job(session, first.job_id)
|
|
assert job.attempt == attempt
|
|
delay = (job.next_attempt_at - item.updated_at).total_seconds()
|
|
assert 5 * 2 ** (attempt - 1) / 2 <= delay <= 5 * 2 ** (attempt - 1)
|
|
assert item.status == ("failed" if attempt == 3 else "queued")
|
|
assert job.last_error_permanent is False
|
|
job.next_attempt_at = await jobs.database_time(session) - timedelta(
|
|
seconds=1
|
|
)
|
|
await session.commit()
|
|
assert item.error_code == "retry_exhausted"
|
|
request_id = uuid4()
|
|
result = await replay_failed_job(
|
|
session,
|
|
storage,
|
|
settings,
|
|
first.job_id,
|
|
request_id=request_id,
|
|
actor="test",
|
|
reason="dependency_restored",
|
|
)
|
|
assert result["attempt_limit"] == 6
|
|
# Old envelope cannot consume the fresh replay budget.
|
|
assert await process_attachment_job(session, storage, first, settings)
|
|
job, item = await jobs.lock_job(session, first.job_id)
|
|
assert job.attempt == 3 and item.status == "queued"
|
|
await session.commit()
|
|
message = await envelope(session, attachment["id"])
|
|
assert await process_attachment_job(session, storage, message, settings)
|
|
before = dict(storage.objects)
|
|
assert await process_attachment_job(session, storage, message, settings)
|
|
assert storage.objects == before
|
|
repeat = await replay_failed_job(
|
|
session,
|
|
storage,
|
|
settings,
|
|
first.job_id,
|
|
request_id=request_id,
|
|
actor="test",
|
|
reason="dependency_restored",
|
|
)
|
|
assert repeat["already_applied"]
|
|
with pytest.raises(JobOperationError, match="job_not_failed"):
|
|
await replay_failed_job(
|
|
session,
|
|
storage,
|
|
settings,
|
|
first.job_id,
|
|
request_id=uuid4(),
|
|
actor="test",
|
|
reason="manual_review",
|
|
)
|
|
await session.rollback()
|
|
assert len((await session.scalars(select(JobReplay))).all()) == 1
|
|
|
|
|
|
async def test_permanent_failure_and_missing_source_replay(client, context, db_engine):
|
|
storage, alice, _, _, chat = context
|
|
attachment = (await upload(client, alice, chat)).json()
|
|
|
|
def fail(_):
|
|
raise ProcessingError("image_limits")
|
|
|
|
settings = JobSettings(_env_file=None)
|
|
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
|
message = await envelope(session, attachment["id"])
|
|
await process_attachment_job(
|
|
session, storage, message, settings, before_processing=fail
|
|
)
|
|
job, item = await jobs.lock_job(session, message.job_id)
|
|
assert item.status == "failed" and job.attempt == 1 and job.last_error_permanent
|
|
job.staging_key = None
|
|
await session.commit()
|
|
with pytest.raises(JobOperationError, match="source_expired"):
|
|
await replay_failed_job(
|
|
session,
|
|
storage,
|
|
settings,
|
|
job.id,
|
|
request_id=uuid4(),
|
|
actor="test",
|
|
reason="manual_review",
|
|
)
|
|
|
|
|
|
async def test_stale_worker_cannot_write_and_cleanup_keeps_active(
|
|
client, context, db_engine
|
|
):
|
|
storage, alice, _, _, chat = context
|
|
attachment = (await upload(client, alice, chat)).json()
|
|
settings = JobSettings(_env_file=None)
|
|
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
|
message = await envelope(session, attachment["id"])
|
|
claim = await claim_job(session, message, settings)
|
|
job, item = await jobs.lock_job(session, message.job_id)
|
|
job.lease_token = uuid4()
|
|
job.finished_at = await jobs.database_time(session) - timedelta(days=10)
|
|
await session.commit()
|
|
assert await cleanup_staging(session, storage, settings, apply=True) == 0
|
|
# Rendering must not be allowed to PUT on a stale lease.
|
|
before = dict(storage.objects)
|
|
assert await build_result(session, storage, claim.work)
|
|
assert storage.objects == before
|
|
|
|
|
|
async def test_cleanup_ready_preserves_final_objects(client, context, db_engine):
|
|
storage, alice, _, _, chat = context
|
|
attachment = (await upload(client, alice, chat)).json()
|
|
await finish_attachment(client, storage, alice, attachment["id"])
|
|
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
|
message = await envelope(session, attachment["id"])
|
|
job, item = await jobs.lock_job(session, message.job_id)
|
|
staging = job.staging_key
|
|
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
|
|
await session.commit()
|
|
assert (
|
|
await cleanup_staging(
|
|
session, storage, JobSettings(_env_file=None), apply=False
|
|
)
|
|
== 1
|
|
)
|
|
assert staging in storage.objects
|
|
assert (
|
|
await cleanup_staging(
|
|
session, storage, JobSettings(_env_file=None), apply=True
|
|
)
|
|
== 1
|
|
)
|
|
assert staging not in storage.objects and len(storage.objects) == 4
|
|
assert (
|
|
await cleanup_staging(
|
|
session, storage, JobSettings(_env_file=None), apply=True
|
|
)
|
|
== 0
|
|
)
|
|
|
|
|
|
async def test_retry_classification_and_cap():
|
|
settings = JobSettings(_env_file=None)
|
|
for attempt in (1, 2, 6, 100):
|
|
cap = min(120, 5 * 2 ** min(30, attempt - 1))
|
|
assert all(cap / 2 <= retry_delay(settings, attempt) <= cap for _ in range(100))
|
|
assert classify_failure(TimeoutError()) == ("processing_timeout", False)
|
|
assert classify_failure(OSError(errno.EACCES, "denied"))[1]
|
|
assert not classify_failure(OSError(errno.ENOSPC, "full"))[1]
|
|
|
|
|
|
async def test_timeout_records_retry_and_clears_lease(
|
|
client, context, db_engine, monkeypatch
|
|
):
|
|
storage, alice, _, _, chat = context
|
|
attachment = (await upload(client, alice, chat)).json()
|
|
|
|
async def stalled(*args, **kwargs):
|
|
await asyncio.sleep(60)
|
|
|
|
monkeypatch.setattr(attachment_jobs, "build_result", stalled)
|
|
settings = JobSettings(_env_file=None).model_copy(
|
|
update={"job_timeout_seconds": 0.05}
|
|
)
|
|
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
|
message = await envelope(session, attachment["id"])
|
|
assert await process_attachment_job(session, storage, message, settings)
|
|
job, item = await jobs.lock_job(session, message.job_id)
|
|
assert item.status == "queued" and job.last_error_code == "processing_timeout"
|
|
assert job.lease_token is None and job.lease_until is None
|
|
|
|
|
|
async def test_failed_retention_and_source_checksum_protect_replay(
|
|
client, context, db_engine
|
|
):
|
|
storage, alice, _, _, chat = context
|
|
attachment = (await upload(client, alice, chat)).json()
|
|
|
|
def fail(_):
|
|
raise ProcessingError("invalid_image")
|
|
|
|
settings = JobSettings(_env_file=None)
|
|
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
|
message = await envelope(session, attachment["id"])
|
|
await process_attachment_job(
|
|
session, storage, message, settings, before_processing=fail
|
|
)
|
|
job, item = await jobs.lock_job(session, message.job_id)
|
|
key = job.staging_key
|
|
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
|
|
await session.commit()
|
|
assert await cleanup_staging(session, storage, settings, apply=True) == 0
|
|
data, metadata = storage.objects[key]
|
|
storage.objects[key] = (b"x" * len(data), metadata)
|
|
with pytest.raises(JobOperationError, match="source_mismatch"):
|
|
await replay_failed_job(
|
|
session,
|
|
storage,
|
|
settings,
|
|
message.job_id,
|
|
request_id=uuid4(),
|
|
actor="test",
|
|
reason="manual_review",
|
|
)
|
|
await session.rollback()
|
|
assert not (await session.scalars(select(JobReplay))).all()
|
|
job, item = await jobs.lock_job(session, message.job_id)
|
|
job.finished_at = await jobs.database_time(session) - timedelta(days=8)
|
|
await session.commit()
|
|
assert await cleanup_staging(session, storage, settings, apply=True) == 1
|
|
assert key not in storage.objects
|