Fix cold worker claim result mapping with Core updates
This commit is contained in:
@@ -28,3 +28,5 @@ PostgreSQL — источник истины, RabbitMQ с FastStream/aio-pika
|
||||
## Найденный обязательный отказ
|
||||
|
||||
Первый track2 (run1.json) выявил queued задание без claims и два сообщения в DLQ: duplicate_in_flight не достиг ready за 90 s. Повтор именно этого job_id воспроизводит DBAPIError; другой дубль проходит. Сохраняются оба результата, SQLSTATE добавляется в безопасный журнал для точного установления причины. До исправления и трёх новых успешных серий change не завершён.
|
||||
|
||||
Регрессия воспроизводится в чистом процессе с графом импортов worker: UPDATE ORM до первичной настройки mappers возвращает UUID job_id как attempt; INSERT job_events затем падает с SQLSTATE 22000. Все UPDATE jobs используют SQLAlchemy Core table, без ORM synchronize_session и дополнительного RETURNING PK. Схема/данные не меняются. Регрессионный тест запускает два конкурентных claim в отдельных холодных процессах с разными hash seeds; исходный вариант падает, исправленный обязан пройти. В журнале остаётся только класс ошибки и SQLSTATE, без параметров.
|
||||
|
||||
@@ -39,9 +39,6 @@ async def handle_job(
|
||||
job_id=str(job.id),
|
||||
error=type(error).__name__,
|
||||
error_code=getattr(getattr(error, "orig", error), "sqlstate", None),
|
||||
reason=type(
|
||||
getattr(getattr(getattr(error, "orig", None), "__cause__", None), "__cause__", None)
|
||||
).__name__,
|
||||
)
|
||||
await asyncio.sleep(REQUEUE_DELAY_SECONDS)
|
||||
await message.nack(requeue=True)
|
||||
|
||||
@@ -2,7 +2,7 @@ import json
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import (
|
||||
@@ -16,6 +16,7 @@ from sqlalchemy import (
|
||||
Integer,
|
||||
RowMapping,
|
||||
Select,
|
||||
Table,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
delete,
|
||||
@@ -498,7 +499,7 @@ class PostgresJobRepository:
|
||||
)
|
||||
|
||||
def _owned(self, lease: Lease):
|
||||
return update(JobRow).where(
|
||||
return update(cast(Table, JobRow.__table__)).where(
|
||||
JobRow.job_id == lease.job.id,
|
||||
JobRow.status == "processing",
|
||||
JobRow.lease_token == lease.token,
|
||||
@@ -535,7 +536,7 @@ class PostgresJobRepository:
|
||||
async def claim(self, job: Job, token: UUID, lease: timedelta) -> Claim | None:
|
||||
row = (
|
||||
await self._session.execute(
|
||||
update(JobRow)
|
||||
update(cast(Table, JobRow.__table__))
|
||||
.where(
|
||||
JobRow.job_id == job.id,
|
||||
JobRow.attachment_id == job.attachment_id,
|
||||
@@ -634,7 +635,7 @@ class PostgresJobRepository:
|
||||
outcome = Outcome.SUPERSEDED if published is None else Outcome.DONE
|
||||
if outcome is Outcome.SUPERSEDED:
|
||||
await self._session.execute(
|
||||
update(JobRow)
|
||||
update(cast(Table, JobRow.__table__))
|
||||
.where(JobRow.job_id == job.id)
|
||||
.values(status="superseded", updated_at=func.now())
|
||||
)
|
||||
@@ -761,7 +762,7 @@ class PostgresJobRepository:
|
||||
)
|
||||
row = (
|
||||
await self._session.execute(
|
||||
update(JobRow)
|
||||
update(cast(Table, JobRow.__table__))
|
||||
.where(JobRow.job_id == job_id)
|
||||
.values(
|
||||
status="queued",
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
from httpx import AsyncClient
|
||||
from test_pipeline import accepted, job_of
|
||||
|
||||
from conftest import DATABASE
|
||||
|
||||
PROBE = """
|
||||
import asyncio, os
|
||||
from datetime import timedelta
|
||||
from uuid import uuid4
|
||||
from messenger.worker import create_worker
|
||||
from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork
|
||||
from messenger.attachments.entrypoints.schemas import JobMessage
|
||||
from messenger.shared.infrastructure.postgres import Database
|
||||
|
||||
async def main():
|
||||
db = Database(os.environ['TEST_DATABASE_URL'], max_size=2, connect_timeout=2,
|
||||
acquire_timeout=2, command_timeout=3)
|
||||
job = JobMessage.model_validate_json(os.environ['TEST_JOB']).job()
|
||||
async def claim():
|
||||
uow = PostgresAttachmentsUnitOfWork(db)
|
||||
async with uow.transaction():
|
||||
claimed = await uow.jobs.claim(job, uuid4(), timedelta(seconds=5))
|
||||
assert claimed is not None
|
||||
assert claimed.lease.job.attempt == 1
|
||||
assert claimed.lease.job.created_at == job.created_at
|
||||
try:
|
||||
await asyncio.gather(claim(), claim())
|
||||
finally:
|
||||
await db.close()
|
||||
asyncio.run(main())
|
||||
"""
|
||||
|
||||
|
||||
async def test_cold_worker_claim_returning_is_stable(client: AsyncClient) -> None:
|
||||
_, _, ident = await accepted(client)
|
||||
job = await job_of(ident)
|
||||
for seed in range(24):
|
||||
child = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
"-c",
|
||||
PROBE,
|
||||
env={
|
||||
**os.environ,
|
||||
"PYTHONHASHSEED": str(seed),
|
||||
"TEST_DATABASE_URL": DATABASE["url"],
|
||||
"TEST_JOB": json.dumps(job.payload()),
|
||||
},
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
try:
|
||||
_, errors = await asyncio.wait_for(child.communicate(), 20)
|
||||
finally:
|
||||
if child.returncode is None:
|
||||
child.kill()
|
||||
await child.wait()
|
||||
assert child.returncode == 0, f"seed={seed}: {errors.decode()}"
|
||||
Reference in New Issue
Block a user