Files
mtusi2026lab5/student-tests/test_jobs.py
T

268 lines
10 KiB
Python

"""Retry/replay, fencing, cleanup, limits и транзакции реального PostgreSQL."""
import asyncio
from datetime import timedelta
import errno
from pathlib import Path
from tempfile import TemporaryDirectory
from uuid import UUID, uuid4
import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.jobs import JobSettings
from app.domain import attachment_jobs
from app.domain.attachment_jobs import (
claim_job,
process_attachment_job,
cleanup_staging,
build_result,
)
from app.domain.job_admin import replay_failed_job
from app.domain.job_retry import classify_failure, retry_delay
from app.errors.exceptions import JobOperationError, ProcessingError
from app.models.attachment import Attachment
from app.models.job import AttachmentJob
from app.models.job_replay import JobReplay
from app.schemas.job import JobMessage
from app.storage import jobs
from test_attachments import context, upload, finish_attachment
async def envelope(session, attachment_id):
job = await session.scalar(
select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))
)
return JobMessage(
job_id=job.id,
attachment_id=job.attachment_id,
pipeline_version=1,
attempt=job.attempt + 1,
created_at=job.created_at,
)
async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_engine):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
settings = JobSettings(_env_file=None, job_max_attempts=3)
def fail(_):
raise TimeoutError
async with AsyncSession(db_engine, expire_on_commit=False) as session:
first = await envelope(session, attachment["id"])
for attempt in range(1, 4):
message = await envelope(session, attachment["id"])
assert await process_attachment_job(
session, storage, message, settings, before_processing=fail
)
job, item = await jobs.lock_job(session, first.job_id)
assert job.attempt == attempt
delay = (job.next_attempt_at - item.updated_at).total_seconds()
assert 5 * 2 ** (attempt - 1) / 2 <= delay <= 5 * 2 ** (attempt - 1)
assert item.status == ("failed" if attempt == 3 else "queued")
assert job.last_error_permanent is False
job.next_attempt_at = await jobs.database_time(session) - timedelta(
seconds=1
)
await session.commit()
assert item.error_code == "retry_exhausted"
request_id = uuid4()
result = await replay_failed_job(
session,
storage,
settings,
first.job_id,
request_id=request_id,
actor="test",
reason="dependency_restored",
)
assert result["attempt_limit"] == 6
# Old envelope cannot consume the fresh replay budget.
assert await process_attachment_job(session, storage, first, settings)
job, item = await jobs.lock_job(session, first.job_id)
assert job.attempt == 3 and item.status == "queued"
await session.commit()
message = await envelope(session, attachment["id"])
assert await process_attachment_job(session, storage, message, settings)
before = dict(storage.objects)
assert await process_attachment_job(session, storage, message, settings)
assert storage.objects == before
repeat = await replay_failed_job(
session,
storage,
settings,
first.job_id,
request_id=request_id,
actor="test",
reason="dependency_restored",
)
assert repeat["already_applied"]
with pytest.raises(JobOperationError, match="job_not_failed"):
await replay_failed_job(
session,
storage,
settings,
first.job_id,
request_id=uuid4(),
actor="test",
reason="manual_review",
)
await session.rollback()
assert len((await session.scalars(select(JobReplay))).all()) == 1
async def test_permanent_failure_and_missing_source_replay(client, context, db_engine):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
def fail(_):
raise ProcessingError("image_limits")
settings = JobSettings(_env_file=None)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment["id"])
await process_attachment_job(
session, storage, message, settings, before_processing=fail
)
job, item = await jobs.lock_job(session, message.job_id)
assert item.status == "failed" and job.attempt == 1 and job.last_error_permanent
job.staging_key = None
await session.commit()
with pytest.raises(JobOperationError, match="source_expired"):
await replay_failed_job(
session,
storage,
settings,
job.id,
request_id=uuid4(),
actor="test",
reason="manual_review",
)
async def test_stale_worker_cannot_write_and_cleanup_keeps_active(
client, context, db_engine
):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
settings = JobSettings(_env_file=None)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment["id"])
claim = await claim_job(session, message, settings)
job, item = await jobs.lock_job(session, message.job_id)
job.lease_token = uuid4()
job.finished_at = await jobs.database_time(session) - timedelta(days=10)
await session.commit()
assert await cleanup_staging(session, storage, settings, apply=True) == 0
# Rendering must not be allowed to PUT on a stale lease.
before = dict(storage.objects)
assert await build_result(session, storage, claim.work)
assert storage.objects == before
async def test_cleanup_ready_preserves_final_objects(client, context, db_engine):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
await finish_attachment(client, storage, alice, attachment["id"])
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment["id"])
job, item = await jobs.lock_job(session, message.job_id)
staging = job.staging_key
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
await session.commit()
assert (
await cleanup_staging(
session, storage, JobSettings(_env_file=None), apply=False
)
== 1
)
assert staging in storage.objects
assert (
await cleanup_staging(
session, storage, JobSettings(_env_file=None), apply=True
)
== 1
)
assert staging not in storage.objects and len(storage.objects) == 4
assert (
await cleanup_staging(
session, storage, JobSettings(_env_file=None), apply=True
)
== 0
)
async def test_retry_classification_and_cap():
settings = JobSettings(_env_file=None)
for attempt in (1, 2, 6, 100):
cap = min(120, 5 * 2 ** min(30, attempt - 1))
assert all(cap / 2 <= retry_delay(settings, attempt) <= cap for _ in range(100))
assert classify_failure(TimeoutError()) == ("processing_timeout", False)
assert classify_failure(OSError(errno.EACCES, "denied"))[1]
assert not classify_failure(OSError(errno.ENOSPC, "full"))[1]
async def test_timeout_records_retry_and_clears_lease(
client, context, db_engine, monkeypatch
):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
async def stalled(*args, **kwargs):
await asyncio.sleep(60)
monkeypatch.setattr(attachment_jobs, "build_result", stalled)
settings = JobSettings(_env_file=None).model_copy(
update={"job_timeout_seconds": 0.05}
)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment["id"])
assert await process_attachment_job(session, storage, message, settings)
job, item = await jobs.lock_job(session, message.job_id)
assert item.status == "queued" and job.last_error_code == "processing_timeout"
assert job.lease_token is None and job.lease_until is None
async def test_failed_retention_and_source_checksum_protect_replay(
client, context, db_engine
):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
def fail(_):
raise ProcessingError("invalid_image")
settings = JobSettings(_env_file=None)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment["id"])
await process_attachment_job(
session, storage, message, settings, before_processing=fail
)
job, item = await jobs.lock_job(session, message.job_id)
key = job.staging_key
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
await session.commit()
assert await cleanup_staging(session, storage, settings, apply=True) == 0
data, metadata = storage.objects[key]
storage.objects[key] = (b"x" * len(data), metadata)
with pytest.raises(JobOperationError, match="source_mismatch"):
await replay_failed_job(
session,
storage,
settings,
message.job_id,
request_id=uuid4(),
actor="test",
reason="manual_review",
)
await session.rollback()
assert not (await session.scalars(select(JobReplay))).all()
job, item = await jobs.lock_job(session, message.job_id)
job.finished_at = await jobs.database_time(session) - timedelta(days=8)
await session.commit()
assert await cleanup_staging(session, storage, settings, apply=True) == 1
assert key not in storage.objects