forked from mtusi2026/mtusi2026lab5
146 lines
4.7 KiB
Python
146 lines
4.7 KiB
Python
"""Replay только failed с проверенным исходником и ограниченным новым бюджетом."""
|
|
|
|
import asyncio
|
|
import re
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
from uuid import uuid4
|
|
|
|
from minio.error import S3Error
|
|
|
|
from app.domain.job_files import fetch_original
|
|
from app.errors.exceptions import JobOperationError, ProcessingError
|
|
from app.storage import job_admin, jobs
|
|
from app.storage.jobs import JobWork
|
|
|
|
REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review")
|
|
|
|
|
|
def snapshot(job, attachment):
|
|
return dict(
|
|
job_id=str(job.id),
|
|
attachment_id=str(attachment.id),
|
|
status=attachment.status,
|
|
attempt=job.attempt,
|
|
attempt_limit=job.attempt_limit,
|
|
cycle_start_attempt=job.cycle_start_attempt,
|
|
error_code=attachment.error_code,
|
|
last_error_code=job.last_error_code,
|
|
last_error_permanent=job.last_error_permanent,
|
|
staging_retained=job.staging_key is not None,
|
|
next_attempt_at=job.next_attempt_at,
|
|
lease_until=job.lease_until,
|
|
finished_at=job.finished_at,
|
|
)
|
|
|
|
|
|
async def list_failed(session, limit):
|
|
return [
|
|
snapshot(job, attachment)
|
|
for job, attachment in await job_admin.failed_jobs(session, limit)
|
|
]
|
|
|
|
|
|
async def show_job(session, job_id):
|
|
row = await jobs.lock_job(session, job_id)
|
|
if row is None:
|
|
raise JobOperationError("unknown_job")
|
|
result = snapshot(*row)
|
|
await session.rollback()
|
|
return result
|
|
|
|
|
|
async def history(session, job_id):
|
|
return [
|
|
dict(
|
|
request_id=str(row.id),
|
|
actor=row.actor,
|
|
reason=row.reason,
|
|
previous_attempt=row.previous_attempt,
|
|
attempt_limit=row.attempt_limit,
|
|
previous_error_code=row.previous_error_code,
|
|
created_at=row.created_at,
|
|
)
|
|
for row in await job_admin.replay_history(session, job_id)
|
|
]
|
|
|
|
|
|
def same_request(record, job_id, actor, reason):
|
|
if record.job_id != job_id or record.actor != actor or record.reason != reason:
|
|
raise JobOperationError("request_conflict")
|
|
return {
|
|
"request_id": str(record.id),
|
|
"job_id": str(job_id),
|
|
"already_applied": True,
|
|
}
|
|
|
|
|
|
async def replay_failed_job(
|
|
session, storage, settings, job_id, *, request_id, actor, reason
|
|
):
|
|
if (
|
|
re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None
|
|
or reason not in REPLAY_REASONS
|
|
):
|
|
raise JobOperationError("invalid_audit_fields")
|
|
row = await jobs.lock_job(session, job_id)
|
|
if row is None:
|
|
raise JobOperationError("unknown_job")
|
|
record = await job_admin.replay_record(session, request_id)
|
|
if record is not None:
|
|
result = same_request(record, job_id, actor, reason)
|
|
await session.rollback()
|
|
return result
|
|
job, attachment = row
|
|
if attachment.status != "failed":
|
|
raise JobOperationError("job_not_failed")
|
|
if job.staging_key is None:
|
|
raise JobOperationError("source_expired")
|
|
# Та же блокировка защищает исходник от cleanup до окончания проверки и COMMIT.
|
|
work = JobWork(
|
|
job.id,
|
|
attachment.id,
|
|
uuid4(),
|
|
job.pipeline_version,
|
|
job.staging_key,
|
|
attachment.content_type,
|
|
attachment.size_bytes,
|
|
attachment.sha256,
|
|
)
|
|
try:
|
|
async with asyncio.timeout(settings.job_timeout_seconds):
|
|
with TemporaryDirectory(prefix="replay-source-") as directory:
|
|
await fetch_original(storage, work, Path(directory) / "original")
|
|
except ProcessingError as error:
|
|
raise JobOperationError(error.code) from None
|
|
except S3Error as error:
|
|
if error.code in {"NoSuchKey", "NoSuchObject"}:
|
|
raise JobOperationError("source_missing") from None
|
|
raise
|
|
limit = job.attempt + settings.job_max_attempts
|
|
inserted = await job_admin.insert_replay(
|
|
session,
|
|
id=request_id,
|
|
job_id=job_id,
|
|
actor=actor,
|
|
reason=reason,
|
|
previous_attempt=job.attempt,
|
|
attempt_limit=limit,
|
|
previous_error_code=job.last_error_code,
|
|
)
|
|
if not inserted:
|
|
result = same_request(
|
|
await job_admin.replay_record(session, request_id), job_id, actor, reason
|
|
)
|
|
await session.rollback()
|
|
return result
|
|
job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit)
|
|
await session.commit()
|
|
return {
|
|
"request_id": str(request_id),
|
|
"job_id": str(job_id),
|
|
"already_applied": False,
|
|
"status": "queued",
|
|
"attempt_limit": limit,
|
|
}
|