Files
mtusi2026lab5/app/domain/job_admin.py
T

146 lines
4.7 KiB
Python

"""Replay только failed с проверенным исходником и ограниченным новым бюджетом."""
import asyncio
import re
from pathlib import Path
from tempfile import TemporaryDirectory
from uuid import uuid4
from minio.error import S3Error
from app.domain.job_files import fetch_original
from app.errors.exceptions import JobOperationError, ProcessingError
from app.storage import job_admin, jobs
from app.storage.jobs import JobWork
REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review")
def snapshot(job, attachment):
return dict(
job_id=str(job.id),
attachment_id=str(attachment.id),
status=attachment.status,
attempt=job.attempt,
attempt_limit=job.attempt_limit,
cycle_start_attempt=job.cycle_start_attempt,
error_code=attachment.error_code,
last_error_code=job.last_error_code,
last_error_permanent=job.last_error_permanent,
staging_retained=job.staging_key is not None,
next_attempt_at=job.next_attempt_at,
lease_until=job.lease_until,
finished_at=job.finished_at,
)
async def list_failed(session, limit):
return [
snapshot(job, attachment)
for job, attachment in await job_admin.failed_jobs(session, limit)
]
async def show_job(session, job_id):
row = await jobs.lock_job(session, job_id)
if row is None:
raise JobOperationError("unknown_job")
result = snapshot(*row)
await session.rollback()
return result
async def history(session, job_id):
return [
dict(
request_id=str(row.id),
actor=row.actor,
reason=row.reason,
previous_attempt=row.previous_attempt,
attempt_limit=row.attempt_limit,
previous_error_code=row.previous_error_code,
created_at=row.created_at,
)
for row in await job_admin.replay_history(session, job_id)
]
def same_request(record, job_id, actor, reason):
if record.job_id != job_id or record.actor != actor or record.reason != reason:
raise JobOperationError("request_conflict")
return {
"request_id": str(record.id),
"job_id": str(job_id),
"already_applied": True,
}
async def replay_failed_job(
session, storage, settings, job_id, *, request_id, actor, reason
):
if (
re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None
or reason not in REPLAY_REASONS
):
raise JobOperationError("invalid_audit_fields")
row = await jobs.lock_job(session, job_id)
if row is None:
raise JobOperationError("unknown_job")
record = await job_admin.replay_record(session, request_id)
if record is not None:
result = same_request(record, job_id, actor, reason)
await session.rollback()
return result
job, attachment = row
if attachment.status != "failed":
raise JobOperationError("job_not_failed")
if job.staging_key is None:
raise JobOperationError("source_expired")
# Та же блокировка защищает исходник от cleanup до окончания проверки и COMMIT.
work = JobWork(
job.id,
attachment.id,
uuid4(),
job.pipeline_version,
job.staging_key,
attachment.content_type,
attachment.size_bytes,
attachment.sha256,
)
try:
async with asyncio.timeout(settings.job_timeout_seconds):
with TemporaryDirectory(prefix="replay-source-") as directory:
await fetch_original(storage, work, Path(directory) / "original")
except ProcessingError as error:
raise JobOperationError(error.code) from None
except S3Error as error:
if error.code in {"NoSuchKey", "NoSuchObject"}:
raise JobOperationError("source_missing") from None
raise
limit = job.attempt + settings.job_max_attempts
inserted = await job_admin.insert_replay(
session,
id=request_id,
job_id=job_id,
actor=actor,
reason=reason,
previous_attempt=job.attempt,
attempt_limit=limit,
previous_error_code=job.last_error_code,
)
if not inserted:
result = same_request(
await job_admin.replay_record(session, request_id), job_id, actor, reason
)
await session.rollback()
return result
job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit)
await session.commit()
return {
"request_id": str(request_id),
"job_id": str(job_id),
"already_applied": False,
"status": "queued",
"attempt_limit": limit,
}