formatted project by python-black
This commit is contained in:
@@ -8,7 +8,14 @@ from aio_pika.exceptions import AMQPException, ChannelInvalidStateError
|
||||
from sqlalchemy.exc import InterfaceError, OperationalError
|
||||
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
|
||||
|
||||
TRANSIENT_JOB_ERRORS = (AMQPException, ChannelInvalidStateError, InterfaceError, OperationalError, PoolTimeoutError, OSError)
|
||||
TRANSIENT_JOB_ERRORS = (
|
||||
AMQPException,
|
||||
ChannelInvalidStateError,
|
||||
InterfaceError,
|
||||
OperationalError,
|
||||
PoolTimeoutError,
|
||||
OSError,
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
+3
-1
@@ -21,7 +21,9 @@ async def cleanup_once(factory, storage, apply: bool, jobs: JobSettings) -> bool
|
||||
async with factory() as session:
|
||||
result = await reconcile_attachments(session, storage, dry_run=not apply)
|
||||
staging = await cleanup_staging(session, storage, jobs, apply=apply)
|
||||
print(json.dumps({**asdict(result), "terminal_staging": staging}), flush=True)
|
||||
print(
|
||||
json.dumps({**asdict(result), "terminal_staging": staging}), flush=True
|
||||
)
|
||||
except RECOVERABLE_STORAGE_ERRORS as error:
|
||||
log_recoverable_error("Reconciliation", error)
|
||||
completed = False
|
||||
|
||||
@@ -5,7 +5,9 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class ImageSettings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_prefix="IMAGE_", env_file=".env", extra="ignore")
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="IMAGE_", env_file=".env", extra="ignore"
|
||||
)
|
||||
|
||||
jpeg_quality: int = Field(default=85, ge=1, le=95)
|
||||
webp_quality: int = Field(default=80, ge=1, le=100)
|
||||
|
||||
+22
-5
@@ -9,7 +9,13 @@ from app.core.background import log_background_error
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.core.jobs import JobSettings
|
||||
from app.domain.job_admin import REPLAY_REASONS, history, list_failed, replay_failed_job, show_job
|
||||
from app.domain.job_admin import (
|
||||
REPLAY_REASONS,
|
||||
history,
|
||||
list_failed,
|
||||
replay_failed_job,
|
||||
show_job,
|
||||
)
|
||||
from app.errors.exceptions import JobOperationError
|
||||
from app.storage.s3 import create_s3_storage
|
||||
|
||||
@@ -29,8 +35,15 @@ async def execute(args):
|
||||
result = await history(session, args.job_id)
|
||||
else:
|
||||
storage = create_s3_storage(settings.s3)
|
||||
result = await replay_failed_job(session, storage, JobSettings(), args.job_id,
|
||||
request_id=args.request_id, actor=args.actor, reason=args.reason)
|
||||
result = await replay_failed_job(
|
||||
session,
|
||||
storage,
|
||||
JobSettings(),
|
||||
args.job_id,
|
||||
request_id=args.request_id,
|
||||
actor=args.actor,
|
||||
reason=args.reason,
|
||||
)
|
||||
print(json.dumps(result, default=str), flush=True)
|
||||
finally:
|
||||
if storage is not None:
|
||||
@@ -39,10 +52,14 @@ async def execute(args):
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Реестр failed и явный replay с аудитом")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Реестр failed и явный replay с аудитом"
|
||||
)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
listing = commands.add_parser("list-failed")
|
||||
listing.add_argument("--limit", type=int, choices=range(1, 501), default=50, metavar="1..500")
|
||||
listing.add_argument(
|
||||
"--limit", type=int, choices=range(1, 501), default=50, metavar="1..500"
|
||||
)
|
||||
for name in ("show", "history", "replay"):
|
||||
command = commands.add_parser(name)
|
||||
command.add_argument("job_id", type=UUID)
|
||||
|
||||
+42
-9
@@ -12,7 +12,12 @@ from uuid import UUID, uuid4
|
||||
from PIL import features
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
|
||||
from app.core.background import (
|
||||
TRANSIENT_JOB_ERRORS,
|
||||
log_background_error,
|
||||
shutdown_event,
|
||||
wait_or_stop,
|
||||
)
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.core.jobs import JobSettings
|
||||
@@ -34,7 +39,10 @@ class Faults:
|
||||
object_name: str = "optimized"
|
||||
|
||||
def kill(self, attachment_id: UUID, phase: str) -> None:
|
||||
print(json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), flush=True)
|
||||
print(
|
||||
json.dumps({"fault": phase, "attachment_id": str(attachment_id)}),
|
||||
flush=True,
|
||||
)
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
|
||||
def after_object(self, attachment_id: UUID, name: str) -> None:
|
||||
@@ -52,7 +60,14 @@ class Faults:
|
||||
raise ProcessingError("injected_permanent")
|
||||
|
||||
|
||||
async def consume_connected(factory, storage, broker: JobBroker, settings: JobSettings, stop: asyncio.Event, faults: Faults) -> None:
|
||||
async def consume_connected(
|
||||
factory,
|
||||
storage,
|
||||
broker: JobBroker,
|
||||
settings: JobSettings,
|
||||
stop: asyncio.Event,
|
||||
faults: Faults,
|
||||
) -> None:
|
||||
consumer = uuid4().hex
|
||||
while not stop.is_set():
|
||||
try:
|
||||
@@ -68,8 +83,12 @@ async def consume_connected(factory, storage, broker: JobBroker, settings: JobSe
|
||||
try:
|
||||
async with factory() as session:
|
||||
acknowledge = await process_attachment_job(
|
||||
session, storage, message, settings,
|
||||
after_object=faults.after_object, before_processing=faults.before_processing,
|
||||
session,
|
||||
storage,
|
||||
message,
|
||||
settings,
|
||||
after_object=faults.after_object,
|
||||
before_processing=faults.before_processing,
|
||||
)
|
||||
except ProcessingError:
|
||||
await broker.reject_invalid(delivery.id)
|
||||
@@ -107,7 +126,10 @@ async def run_worker(faults: Faults) -> None:
|
||||
storage = create_s3_storage(settings.s3)
|
||||
factory = create_session_factory(engine)
|
||||
async with asyncio.TaskGroup() as group:
|
||||
tasks = [group.create_task(consume(factory, storage, jobs, stop, faults)) for _ in range(jobs.worker_concurrency)]
|
||||
tasks = [
|
||||
group.create_task(consume(factory, storage, jobs, stop, faults))
|
||||
for _ in range(jobs.worker_concurrency)
|
||||
]
|
||||
await stop.wait()
|
||||
# Отмена убивает resize-процесс, дожидается SDK и не отправляет ack.
|
||||
for task in tasks:
|
||||
@@ -119,11 +141,22 @@ async def run_worker(faults: Faults) -> None:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Image worker с явными fault-injection hooks")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Image worker с явными fault-injection hooks"
|
||||
)
|
||||
modes = parser.add_mutually_exclusive_group()
|
||||
for flag in ("crash-after-object", "crash-before-ack", "fail-transient", "fail-permanent"):
|
||||
for flag in (
|
||||
"crash-after-object",
|
||||
"crash-before-ack",
|
||||
"fail-transient",
|
||||
"fail-permanent",
|
||||
):
|
||||
modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID")
|
||||
parser.add_argument("--object-name", choices=("original", "thumbnail", "crop", "optimized"), default="optimized")
|
||||
parser.add_argument(
|
||||
"--object-name",
|
||||
choices=("original", "thumbnail", "crop", "optimized"),
|
||||
default="optimized",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
faults = Faults(**vars(args))
|
||||
try:
|
||||
|
||||
+3
-1
@@ -29,7 +29,9 @@ class JobSettings(BaseSettings):
|
||||
@model_validator(mode="after")
|
||||
def validate_lease(self) -> Self:
|
||||
if self.job_lease_seconds < self.job_timeout_seconds + 30:
|
||||
raise ValueError("Lease должен превышать timeout задания минимум на 30 секунд")
|
||||
raise ValueError(
|
||||
"Lease должен превышать timeout задания минимум на 30 секунд"
|
||||
)
|
||||
if self.job_retry_max_seconds < self.job_retry_seconds:
|
||||
raise ValueError("Retry maximum must be at least the base delay")
|
||||
return self
|
||||
|
||||
@@ -8,7 +8,12 @@ import os
|
||||
import signal
|
||||
from uuid import UUID
|
||||
|
||||
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
|
||||
from app.core.background import (
|
||||
TRANSIENT_JOB_ERRORS,
|
||||
log_background_error,
|
||||
shutdown_event,
|
||||
wait_or_stop,
|
||||
)
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.core.jobs import JobSettings
|
||||
@@ -18,7 +23,10 @@ from app.storage.broker import JobBroker
|
||||
|
||||
def crash_before_publish(attachment_id: UUID) -> None:
|
||||
"""Реальный SIGKILL после чтения принятой job, до publisher confirm и отметки publish."""
|
||||
print(json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}), flush=True)
|
||||
print(
|
||||
json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}),
|
||||
flush=True,
|
||||
)
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
|
||||
|
||||
@@ -39,8 +47,14 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
|
||||
async with factory() as session:
|
||||
await recover_expired_jobs(session, jobs)
|
||||
published = await publish_next_job(
|
||||
session, broker, attachment_id=crash_attachment_id,
|
||||
before_publish=crash_before_publish if crash_attachment_id is not None else None,
|
||||
session,
|
||||
broker,
|
||||
attachment_id=crash_attachment_id,
|
||||
before_publish=(
|
||||
crash_before_publish
|
||||
if crash_attachment_id is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
except TRANSIENT_JOB_ERRORS as error:
|
||||
log_background_error("Publisher deferred", error)
|
||||
@@ -53,9 +67,13 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Публикация устойчивых jobs в RabbitMQ")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Публикация устойчивых jobs в RabbitMQ"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--crash-before-publish", type=UUID, metavar="ATTACHMENT_ID",
|
||||
"--crash-before-publish",
|
||||
type=UUID,
|
||||
metavar="ATTACHMENT_ID",
|
||||
help="аварийно завершить этот процесс перед публикацией только указанного вложения",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
+1
-2
@@ -4,6 +4,7 @@ from redis.backoff import NoBackoff
|
||||
|
||||
from app.core.config import Settings
|
||||
|
||||
|
||||
def create_redis_client(settings: Settings) -> Redis:
|
||||
password = (
|
||||
settings.redis_password.get_secret_value() if settings.redis_password else None
|
||||
@@ -21,5 +22,3 @@ def create_redis_client(settings: Settings) -> Redis:
|
||||
decode_responses=True,
|
||||
)
|
||||
return Redis.from_pool(pool)
|
||||
|
||||
|
||||
|
||||
+16
-4
@@ -6,6 +6,7 @@ from urllib.parse import urlsplit
|
||||
from pydantic import Field, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class S3Settings(BaseSettings):
|
||||
endpoint: str = "http://minio:9000"
|
||||
region: str = "us-east-1"
|
||||
@@ -24,12 +25,23 @@ class S3Settings(BaseSettings):
|
||||
cleanup_grace_seconds: int = Field(default=86_400, ge=300)
|
||||
cleanup_interval_seconds: int = Field(default=300, ge=1)
|
||||
|
||||
model_config = SettingsConfigDict(env_prefix="S3_", env_file=".env", extra="ignore", frozen=True)
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="S3_", env_file=".env", extra="ignore", frozen=True
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_endpoint(self):
|
||||
url = urlsplit(self.endpoint)
|
||||
if (url.scheme not in {"http", "https"} or not url.hostname or url.username
|
||||
or url.password or url.path not in {"", "/"} or url.query or url.fragment):
|
||||
raise ValueError("S3_ENDPOINT должен быть http(s) адресом без credentials/пути")
|
||||
if (
|
||||
url.scheme not in {"http", "https"}
|
||||
or not url.hostname
|
||||
or url.username
|
||||
or url.password
|
||||
or url.path not in {"", "/"}
|
||||
or url.query
|
||||
or url.fragment
|
||||
):
|
||||
raise ValueError(
|
||||
"S3_ENDPOINT должен быть http(s) адресом без credentials/пути"
|
||||
)
|
||||
return self
|
||||
|
||||
@@ -13,11 +13,15 @@ def main() -> int:
|
||||
settings = ImageSettings()
|
||||
memory = settings.memory_mib * 1024 * 1024
|
||||
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds))
|
||||
resource.setrlimit(
|
||||
resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds)
|
||||
)
|
||||
source, directory, metadata, content_type, maximum = sys.argv[1:]
|
||||
code = 0
|
||||
try:
|
||||
result = create_variants(Path(source), Path(directory), content_type, int(maximum), settings)
|
||||
result = create_variants(
|
||||
Path(source), Path(directory), content_type, int(maximum), settings
|
||||
)
|
||||
Path(metadata).write_text(result.model_dump_json())
|
||||
except IMAGE_WORKER_ERRORS as error:
|
||||
code = pipeline_worker_error_code(error)
|
||||
|
||||
@@ -29,13 +29,17 @@ class Claim:
|
||||
acknowledge: bool = False
|
||||
|
||||
|
||||
async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID:
|
||||
async def enqueue_attachment_job(
|
||||
session: AsyncSession, attachment_id: UUID, staging_key: str
|
||||
) -> UUID:
|
||||
"""Включить намерение обработки в транзакцию queued metadata; без commit."""
|
||||
return await jobs.create_job(session, attachment_id, staging_key)
|
||||
|
||||
|
||||
async def publish_next_job(
|
||||
session: AsyncSession, broker: JobBroker, *,
|
||||
session: AsyncSession,
|
||||
broker: JobBroker,
|
||||
*,
|
||||
attachment_id: UUID | None = None,
|
||||
before_publish: Callable[[UUID], None] | None = None,
|
||||
) -> bool:
|
||||
@@ -45,24 +49,33 @@ async def publish_next_job(
|
||||
if job is not None:
|
||||
if before_publish is not None:
|
||||
before_publish(job.attachment_id)
|
||||
await broker.publish_job(JobMessage(
|
||||
job_id=job.id, attachment_id=job.attachment_id,
|
||||
pipeline_version=job.pipeline_version, attempt=job.attempt + 1,
|
||||
created_at=job.created_at,
|
||||
))
|
||||
await broker.publish_job(
|
||||
JobMessage(
|
||||
job_id=job.id,
|
||||
attachment_id=job.attachment_id,
|
||||
pipeline_version=job.pipeline_version,
|
||||
attempt=job.attempt + 1,
|
||||
created_at=job.created_at,
|
||||
)
|
||||
)
|
||||
outbox.mark_published(job)
|
||||
await session.commit()
|
||||
return published
|
||||
|
||||
|
||||
async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSettings) -> Claim:
|
||||
async def claim_job(
|
||||
session: AsyncSession, message: JobMessage, settings: JobSettings
|
||||
) -> Claim:
|
||||
"""Дубликат не захватывает живую аренду и не откатывает terminal-состояние."""
|
||||
row = await jobs.lock_job(session, message.job_id)
|
||||
result = Claim()
|
||||
if row is None:
|
||||
raise ProcessingError("unknown_job")
|
||||
job, attachment = row
|
||||
if message.attachment_id != attachment.id or message.pipeline_version != job.pipeline_version:
|
||||
if (
|
||||
message.attachment_id != attachment.id
|
||||
or message.pipeline_version != job.pipeline_version
|
||||
):
|
||||
raise ProcessingError("invalid_job")
|
||||
now = await jobs.database_time(session)
|
||||
if job.attempt_limit is None:
|
||||
@@ -77,16 +90,32 @@ async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSet
|
||||
elif message.attempt > job.attempt + 1:
|
||||
raise ProcessingError("invalid_attempt")
|
||||
elif attachment.status == "queued" and job.next_attempt_at <= now:
|
||||
if job.attempt >= (job.attempt_limit or settings.job_max_attempts) or job.staging_key is None:
|
||||
jobs.record_failure(job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0)
|
||||
if (
|
||||
job.attempt >= (job.attempt_limit or settings.job_max_attempts)
|
||||
or job.staging_key is None
|
||||
):
|
||||
jobs.record_failure(
|
||||
job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0
|
||||
)
|
||||
result = Claim(acknowledge=True)
|
||||
else:
|
||||
result = Claim(work=jobs.start_job(job, attachment, uuid4(), now, settings.job_lease_seconds))
|
||||
result = Claim(
|
||||
work=jobs.start_job(
|
||||
job, attachment, uuid4(), now, settings.job_lease_seconds
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return result
|
||||
|
||||
|
||||
async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettings, code: str, *, permanent: bool) -> None:
|
||||
async def save_failure(
|
||||
session: AsyncSession,
|
||||
work: JobWork,
|
||||
settings: JobSettings,
|
||||
code: str,
|
||||
*,
|
||||
permanent: bool,
|
||||
) -> None:
|
||||
# Неоднозначный COMMIT проверяется повторным чтением: ready сохраняется.
|
||||
await session.rollback()
|
||||
row = await jobs.lock_job(session, work.job_id)
|
||||
@@ -95,10 +124,14 @@ async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettin
|
||||
if attachment.status == "processing" and job.lease_token == work.lease_token:
|
||||
exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
|
||||
jobs.record_failure(
|
||||
job, attachment, await jobs.database_time(session),
|
||||
job,
|
||||
attachment,
|
||||
await jobs.database_time(session),
|
||||
"retry_exhausted" if exhausted and not permanent else code,
|
||||
terminal=permanent or exhausted,
|
||||
retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
|
||||
retry_seconds=retry_delay(
|
||||
settings, job.attempt - job.cycle_start_attempt
|
||||
),
|
||||
)
|
||||
job.last_error_code = code
|
||||
job.last_error_permanent = permanent
|
||||
@@ -106,7 +139,9 @@ async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettin
|
||||
|
||||
|
||||
async def build_result(
|
||||
session: AsyncSession, storage: MinioStorage, work: JobWork,
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage,
|
||||
work: JobWork,
|
||||
after_object: Callable[[UUID, str], None] | None = None,
|
||||
) -> bool:
|
||||
"""Фиксированные ключи и блокировка job защищают PUT от устаревшей попытки."""
|
||||
@@ -129,26 +164,42 @@ async def build_result(
|
||||
# Блокировка остаётся до завершения PUT/COMMIT; reaper использует SKIP LOCKED.
|
||||
# При отмене S3 adapter дожидается SDK-потока до освобождения блокировки.
|
||||
await storage.put_file(
|
||||
work.original_key, source, size_bytes=work.size_bytes,
|
||||
content_type=work.content_type, sha256=work.sha256,
|
||||
work.original_key,
|
||||
source,
|
||||
size_bytes=work.size_bytes,
|
||||
content_type=work.content_type,
|
||||
sha256=work.sha256,
|
||||
)
|
||||
if after_object is not None:
|
||||
after_object(work.attachment_id, "original")
|
||||
for variant in result.variants:
|
||||
await storage.put_file(
|
||||
work.variant_key(variant.name), Path(directory) / VARIANT_FILENAMES[variant.name],
|
||||
size_bytes=variant.size_bytes, content_type=variant.content_type, sha256=variant.sha256,
|
||||
work.variant_key(variant.name),
|
||||
Path(directory) / VARIANT_FILENAMES[variant.name],
|
||||
size_bytes=variant.size_bytes,
|
||||
content_type=variant.content_type,
|
||||
sha256=variant.sha256,
|
||||
)
|
||||
if after_object is not None:
|
||||
after_object(work.attachment_id, variant.name)
|
||||
jobs.complete_job(job, attachment, work, [variant.model_dump() for variant in result.variants], await jobs.database_time(session))
|
||||
jobs.complete_job(
|
||||
job,
|
||||
attachment,
|
||||
work,
|
||||
[variant.model_dump() for variant in result.variants],
|
||||
await jobs.database_time(session),
|
||||
)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def process_attachment_job(
|
||||
session: AsyncSession, storage: MinioStorage, message: JobMessage, settings: JobSettings,
|
||||
*, after_object: Callable[[UUID, str], None] | None = None,
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage,
|
||||
message: JobMessage,
|
||||
settings: JobSettings,
|
||||
*,
|
||||
after_object: Callable[[UUID, str], None] | None = None,
|
||||
before_processing: Callable[[UUID], None] | None = None,
|
||||
) -> bool:
|
||||
claim = await claim_job(session, message, settings)
|
||||
@@ -160,7 +211,13 @@ async def process_attachment_job(
|
||||
if before_processing is not None:
|
||||
before_processing(work.attachment_id)
|
||||
acknowledge = await build_result(session, storage, work, after_object)
|
||||
except (ProcessingError, MinioException, StorageHTTPError, OSError, TimeoutError) as error:
|
||||
except (
|
||||
ProcessingError,
|
||||
MinioException,
|
||||
StorageHTTPError,
|
||||
OSError,
|
||||
TimeoutError,
|
||||
) as error:
|
||||
code, permanent = classify_failure(error)
|
||||
await save_failure(session, work, settings, code, permanent=permanent)
|
||||
acknowledge = True
|
||||
@@ -176,8 +233,12 @@ async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) ->
|
||||
job.attempt_limit = settings.job_max_attempts
|
||||
exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
|
||||
jobs.record_failure(
|
||||
job, attachment, now, "retry_exhausted" if exhausted else "worker_timeout",
|
||||
terminal=exhausted, retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
|
||||
job,
|
||||
attachment,
|
||||
now,
|
||||
"retry_exhausted" if exhausted else "worker_timeout",
|
||||
terminal=exhausted,
|
||||
retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
|
||||
)
|
||||
job.last_error_code = "worker_timeout"
|
||||
job.last_error_permanent = False
|
||||
@@ -185,8 +246,14 @@ async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) ->
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def cleanup_staging(session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool) -> int:
|
||||
rows = await jobs.terminal_staging(session, settings.staging_retention_seconds, settings.failed_staging_retention_seconds)
|
||||
async def cleanup_staging(
|
||||
session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool
|
||||
) -> int:
|
||||
rows = await jobs.terminal_staging(
|
||||
session,
|
||||
settings.staging_retention_seconds,
|
||||
settings.failed_staging_retention_seconds,
|
||||
)
|
||||
if apply:
|
||||
for job, attachment in rows:
|
||||
if job.staging_key != attachment.object_key:
|
||||
|
||||
@@ -276,12 +276,20 @@ async def download_attachment(
|
||||
key = record.object_key
|
||||
info = DownloadInfo(result.filename, result.content_type, result.size_bytes)
|
||||
else:
|
||||
selected = next((item for item in result.variants if item.name == variant), None)
|
||||
keys = {"thumbnail": record.thumbnail_key, "crop": record.crop_key, "optimized": record.optimized_key}
|
||||
selected = next(
|
||||
(item for item in result.variants if item.name == variant), None
|
||||
)
|
||||
keys = {
|
||||
"thumbnail": record.thumbnail_key,
|
||||
"crop": record.crop_key,
|
||||
"optimized": record.optimized_key,
|
||||
}
|
||||
key = keys.get(variant)
|
||||
if selected is None or key is None:
|
||||
raise not_found()
|
||||
info = DownloadInfo(VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes)
|
||||
info = DownloadInfo(
|
||||
VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes
|
||||
)
|
||||
await session.rollback()
|
||||
stored = await storage.stat_object(key)
|
||||
if stored.size_bytes != info.size_bytes or stored.content_type != info.content_type:
|
||||
|
||||
@@ -14,12 +14,15 @@ from app.storage.s3 import MinioStorage
|
||||
|
||||
|
||||
def liveness() -> Health:
|
||||
return Health(status='ok')
|
||||
return Health(status="ok")
|
||||
|
||||
|
||||
async def readiness(
|
||||
engine: AsyncEngine, redis: Redis, storage: MinioStorage | None,
|
||||
lifecycle: Lifecycle, timeout_seconds: float,
|
||||
engine: AsyncEngine,
|
||||
redis: Redis,
|
||||
storage: MinioStorage | None,
|
||||
lifecycle: Lifecycle,
|
||||
timeout_seconds: float,
|
||||
) -> Health:
|
||||
if lifecycle.draining:
|
||||
raise unavailable()
|
||||
@@ -30,4 +33,4 @@ async def readiness(
|
||||
dependencies = await asyncio.gather(*checks, return_exceptions=True)
|
||||
if not all(result is True for result in dependencies):
|
||||
raise unavailable()
|
||||
return Health(status='ok')
|
||||
return Health(status="ok")
|
||||
|
||||
@@ -57,16 +57,22 @@ async def send_message_idempotent(
|
||||
raise conflict("Idempotency-Key уже использован для другого сообщения")
|
||||
message = await message_storage.get_message(session, previous.message_id)
|
||||
else:
|
||||
await validate_message_attachments(session, sender, chat_id, new_message.attachment_ids)
|
||||
await validate_message_attachments(
|
||||
session, sender, chat_id, new_message.attachment_ids
|
||||
)
|
||||
if not await message_storage.reserve_message_slot(session, max_messages):
|
||||
raise unavailable("Достигнут лимит количества сообщений")
|
||||
message = await message_storage.create_message(
|
||||
session, chat_id=chat_id, sender_id=sender.id, text=new_message.text
|
||||
)
|
||||
await attachment_storage.bind_attachments(session, message.id, new_message.attachment_ids)
|
||||
await attachment_storage.bind_attachments(
|
||||
session, message.id, new_message.attachment_ids
|
||||
)
|
||||
await key_storage.save_result(session, scope, fingerprint, message.id)
|
||||
|
||||
links = await attachment_storage.message_attachment_ids(session, [message.id])
|
||||
result = Message.model_validate(message).model_copy(update={"attachment_ids": links[message.id]})
|
||||
result = Message.model_validate(message).model_copy(
|
||||
update={"attachment_ids": links[message.id]}
|
||||
)
|
||||
await session.commit()
|
||||
return result
|
||||
|
||||
@@ -15,18 +15,27 @@ def fitted_size(size: tuple[int, int], maximum: int) -> tuple[int, int]:
|
||||
longest = max(size)
|
||||
result = size
|
||||
if longest > maximum:
|
||||
result = (max(1, size[0] * maximum // longest), max(1, size[1] * maximum // longest))
|
||||
result = (
|
||||
max(1, size[0] * maximum // longest),
|
||||
max(1, size[1] * maximum // longest),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def describe_variant(name: str, content_type: str, path: Path, size: tuple[int, int]) -> Variant:
|
||||
def describe_variant(
|
||||
name: str, content_type: str, path: Path, size: tuple[int, int]
|
||||
) -> Variant:
|
||||
checksum = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(65536), b""):
|
||||
checksum.update(chunk)
|
||||
return Variant(
|
||||
name=name, content_type=content_type, size_bytes=path.stat().st_size,
|
||||
width=size[0], height=size[1], sha256=checksum.hexdigest(),
|
||||
name=name,
|
||||
content_type=content_type,
|
||||
size_bytes=path.stat().st_size,
|
||||
width=size[0],
|
||||
height=size[1],
|
||||
sha256=checksum.hexdigest(),
|
||||
)
|
||||
|
||||
|
||||
@@ -47,32 +56,49 @@ def create_thumbnail(image: Image.Image, destination: Path, quality: int) -> Var
|
||||
|
||||
def create_crop(image: Image.Image, destination: Path, quality: int) -> Variant:
|
||||
"""Центрированный JPEG 128×128; маленький исходник допускает увеличение."""
|
||||
with ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) as crop:
|
||||
with ImageOps.fit(
|
||||
image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)
|
||||
) as crop:
|
||||
save_jpeg(crop, destination, quality)
|
||||
return describe_variant("crop", "image/jpeg", destination, (128, 128))
|
||||
|
||||
|
||||
def create_optimized(image: Image.Image, destination: Path, settings: ImageSettings) -> Variant:
|
||||
def create_optimized(
|
||||
image: Image.Image, destination: Path, settings: ImageSettings
|
||||
) -> Variant:
|
||||
"""WebP ≤1280×1280 без увеличения; alpha сохраняется, метаданные удаляются."""
|
||||
size = fitted_size(image.size, 1280)
|
||||
with image.resize(size, Image.Resampling.LANCZOS) as resized:
|
||||
with Image.new("RGBA", size) as clean:
|
||||
clean.paste(resized)
|
||||
clean.save(
|
||||
destination, format="WEBP", lossless=False,
|
||||
quality=settings.webp_quality, method=settings.webp_method,
|
||||
exif=b"", icc_profile=b"", xmp=b"",
|
||||
destination,
|
||||
format="WEBP",
|
||||
lossless=False,
|
||||
quality=settings.webp_quality,
|
||||
method=settings.webp_method,
|
||||
exif=b"",
|
||||
icc_profile=b"",
|
||||
xmp=b"",
|
||||
)
|
||||
return describe_variant("optimized", "image/webp", destination, size)
|
||||
|
||||
|
||||
def create_variants(source: Path, directory: Path, content_type: str, max_pixels: int, settings: ImageSettings) -> PipelineResult:
|
||||
def create_variants(
|
||||
source: Path,
|
||||
directory: Path,
|
||||
content_type: str,
|
||||
max_pixels: int,
|
||||
settings: ImageSettings,
|
||||
) -> PipelineResult:
|
||||
decode_image(source, content_type, max_pixels)
|
||||
with Image.open(source, formats=["PNG", "JPEG"]) as image:
|
||||
ImageOps.exif_transpose(image, in_place=True)
|
||||
with image.convert("RGBA") as pixels:
|
||||
variants = [
|
||||
create_thumbnail(pixels, directory / "thumbnail.jpg", settings.jpeg_quality),
|
||||
create_thumbnail(
|
||||
pixels, directory / "thumbnail.jpg", settings.jpeg_quality
|
||||
),
|
||||
create_crop(pixels, directory / "crop.jpg", settings.jpeg_quality),
|
||||
create_optimized(pixels, directory / "optimized.webp", settings),
|
||||
]
|
||||
|
||||
+71
-19
@@ -17,15 +17,28 @@ REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review")
|
||||
|
||||
|
||||
def snapshot(job, attachment):
|
||||
return dict(job_id=str(job.id), attachment_id=str(attachment.id), status=attachment.status,
|
||||
attempt=job.attempt, attempt_limit=job.attempt_limit, cycle_start_attempt=job.cycle_start_attempt,
|
||||
error_code=attachment.error_code, last_error_code=job.last_error_code,
|
||||
last_error_permanent=job.last_error_permanent, staging_retained=job.staging_key is not None,
|
||||
next_attempt_at=job.next_attempt_at, lease_until=job.lease_until, finished_at=job.finished_at)
|
||||
return dict(
|
||||
job_id=str(job.id),
|
||||
attachment_id=str(attachment.id),
|
||||
status=attachment.status,
|
||||
attempt=job.attempt,
|
||||
attempt_limit=job.attempt_limit,
|
||||
cycle_start_attempt=job.cycle_start_attempt,
|
||||
error_code=attachment.error_code,
|
||||
last_error_code=job.last_error_code,
|
||||
last_error_permanent=job.last_error_permanent,
|
||||
staging_retained=job.staging_key is not None,
|
||||
next_attempt_at=job.next_attempt_at,
|
||||
lease_until=job.lease_until,
|
||||
finished_at=job.finished_at,
|
||||
)
|
||||
|
||||
|
||||
async def list_failed(session, limit):
|
||||
return [snapshot(job, attachment) for job, attachment in await job_admin.failed_jobs(session, limit)]
|
||||
return [
|
||||
snapshot(job, attachment)
|
||||
for job, attachment in await job_admin.failed_jobs(session, limit)
|
||||
]
|
||||
|
||||
|
||||
async def show_job(session, job_id):
|
||||
@@ -38,20 +51,37 @@ async def show_job(session, job_id):
|
||||
|
||||
|
||||
async def history(session, job_id):
|
||||
return [dict(request_id=str(row.id), actor=row.actor, reason=row.reason,
|
||||
previous_attempt=row.previous_attempt, attempt_limit=row.attempt_limit,
|
||||
previous_error_code=row.previous_error_code, created_at=row.created_at)
|
||||
for row in await job_admin.replay_history(session, job_id)]
|
||||
return [
|
||||
dict(
|
||||
request_id=str(row.id),
|
||||
actor=row.actor,
|
||||
reason=row.reason,
|
||||
previous_attempt=row.previous_attempt,
|
||||
attempt_limit=row.attempt_limit,
|
||||
previous_error_code=row.previous_error_code,
|
||||
created_at=row.created_at,
|
||||
)
|
||||
for row in await job_admin.replay_history(session, job_id)
|
||||
]
|
||||
|
||||
|
||||
def same_request(record, job_id, actor, reason):
|
||||
if record.job_id != job_id or record.actor != actor or record.reason != reason:
|
||||
raise JobOperationError("request_conflict")
|
||||
return {"request_id": str(record.id), "job_id": str(job_id), "already_applied": True}
|
||||
return {
|
||||
"request_id": str(record.id),
|
||||
"job_id": str(job_id),
|
||||
"already_applied": True,
|
||||
}
|
||||
|
||||
|
||||
async def replay_failed_job(session, storage, settings, job_id, *, request_id, actor, reason):
|
||||
if re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None or reason not in REPLAY_REASONS:
|
||||
async def replay_failed_job(
|
||||
session, storage, settings, job_id, *, request_id, actor, reason
|
||||
):
|
||||
if (
|
||||
re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None
|
||||
or reason not in REPLAY_REASONS
|
||||
):
|
||||
raise JobOperationError("invalid_audit_fields")
|
||||
row = await jobs.lock_job(session, job_id)
|
||||
if row is None:
|
||||
@@ -67,8 +97,16 @@ async def replay_failed_job(session, storage, settings, job_id, *, request_id, a
|
||||
if job.staging_key is None:
|
||||
raise JobOperationError("source_expired")
|
||||
# Та же блокировка защищает исходник от cleanup до окончания проверки и COMMIT.
|
||||
work = JobWork(job.id, attachment.id, uuid4(), job.pipeline_version, job.staging_key,
|
||||
attachment.content_type, attachment.size_bytes, attachment.sha256)
|
||||
work = JobWork(
|
||||
job.id,
|
||||
attachment.id,
|
||||
uuid4(),
|
||||
job.pipeline_version,
|
||||
job.staging_key,
|
||||
attachment.content_type,
|
||||
attachment.size_bytes,
|
||||
attachment.sha256,
|
||||
)
|
||||
try:
|
||||
async with asyncio.timeout(settings.job_timeout_seconds):
|
||||
with TemporaryDirectory(prefix="replay-source-") as directory:
|
||||
@@ -81,13 +119,27 @@ async def replay_failed_job(session, storage, settings, job_id, *, request_id, a
|
||||
raise
|
||||
limit = job.attempt + settings.job_max_attempts
|
||||
inserted = await job_admin.insert_replay(
|
||||
session, id=request_id, job_id=job_id, actor=actor, reason=reason,
|
||||
previous_attempt=job.attempt, attempt_limit=limit, previous_error_code=job.last_error_code,
|
||||
session,
|
||||
id=request_id,
|
||||
job_id=job_id,
|
||||
actor=actor,
|
||||
reason=reason,
|
||||
previous_attempt=job.attempt,
|
||||
attempt_limit=limit,
|
||||
previous_error_code=job.last_error_code,
|
||||
)
|
||||
if not inserted:
|
||||
result = same_request(await job_admin.replay_record(session, request_id), job_id, actor, reason)
|
||||
result = same_request(
|
||||
await job_admin.replay_record(session, request_id), job_id, actor, reason
|
||||
)
|
||||
await session.rollback()
|
||||
return result
|
||||
job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit)
|
||||
await session.commit()
|
||||
return {"request_id": str(request_id), "job_id": str(job_id), "already_applied": False, "status": "queued", "attempt_limit": limit}
|
||||
return {
|
||||
"request_id": str(request_id),
|
||||
"job_id": str(job_id),
|
||||
"already_applied": False,
|
||||
"status": "queued",
|
||||
"attempt_limit": limit,
|
||||
}
|
||||
|
||||
+16
-5
@@ -14,7 +14,9 @@ from app.storage.jobs import JobWork
|
||||
from app.storage.s3 import MinioStorage
|
||||
|
||||
|
||||
async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path) -> None:
|
||||
async def fetch_original(
|
||||
storage: MinioStorage, work: JobWork, destination: Path
|
||||
) -> None:
|
||||
if work.size_bytes > storage.settings.max_upload_bytes:
|
||||
raise ProcessingError("image_limits")
|
||||
checksum = hashlib.sha256()
|
||||
@@ -34,12 +36,21 @@ async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path
|
||||
raise ProcessingError("source_mismatch")
|
||||
|
||||
|
||||
async def render_variants(storage: MinioStorage, work: JobWork, source: Path, directory: Path) -> PipelineResult:
|
||||
async def render_variants(
|
||||
storage: MinioStorage, work: JobWork, source: Path, directory: Path
|
||||
) -> PipelineResult:
|
||||
metadata = directory / "variants.json"
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "-m", "app.core.thumbnail_worker", str(source), str(directory),
|
||||
str(metadata), work.content_type, str(storage.settings.max_image_pixels),
|
||||
stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
|
||||
sys.executable,
|
||||
"-m",
|
||||
"app.core.thumbnail_worker",
|
||||
str(source),
|
||||
str(directory),
|
||||
str(metadata),
|
||||
work.content_type,
|
||||
str(storage.settings.max_image_pixels),
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
code = await process.wait()
|
||||
|
||||
+23
-4
@@ -10,7 +10,10 @@ from app.errors.exceptions import ProcessingError
|
||||
|
||||
|
||||
def retry_delay(settings: JobSettings, cycle_attempt: int) -> float:
|
||||
ceiling = min(settings.job_retry_max_seconds, settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)))
|
||||
ceiling = min(
|
||||
settings.job_retry_max_seconds,
|
||||
settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)),
|
||||
)
|
||||
return random.uniform(ceiling / 2, ceiling)
|
||||
|
||||
|
||||
@@ -20,13 +23,29 @@ def classify_failure(error: Exception) -> tuple[str, bool]:
|
||||
if isinstance(error, S3Error):
|
||||
if error.code in {"NoSuchKey", "NoSuchObject"}:
|
||||
return "source_missing", True
|
||||
if error.code in {"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "NoSuchBucket"}:
|
||||
if error.code in {
|
||||
"AccessDenied",
|
||||
"InvalidAccessKeyId",
|
||||
"SignatureDoesNotMatch",
|
||||
"NoSuchBucket",
|
||||
}:
|
||||
return "storage_configuration", True
|
||||
if error.code in {"InternalError", "SlowDown", "ServiceUnavailable", "RequestTimeout", "Throttling"}:
|
||||
if error.code in {
|
||||
"InternalError",
|
||||
"SlowDown",
|
||||
"ServiceUnavailable",
|
||||
"RequestTimeout",
|
||||
"Throttling",
|
||||
}:
|
||||
return "storage_unavailable", False
|
||||
return "storage_rejected", True
|
||||
if isinstance(error, TimeoutError):
|
||||
return "processing_timeout", False
|
||||
if isinstance(error, OSError) and error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}:
|
||||
if isinstance(error, OSError) and error.errno in {
|
||||
errno.EACCES,
|
||||
errno.EPERM,
|
||||
errno.EROFS,
|
||||
errno.ENOENT,
|
||||
}:
|
||||
return "processing_configuration", True
|
||||
return "processing_unavailable", False
|
||||
|
||||
@@ -32,10 +32,15 @@ async def list_messages(
|
||||
|
||||
rows = await messages_storage.list_messages(session, chat_id, after, limit + 1)
|
||||
page_rows, has_next = split_page(rows, limit)
|
||||
links = await attachments_storage.message_attachment_ids(session, [message.id for message in page_rows])
|
||||
items = [Message.model_validate(message).model_copy(
|
||||
update={"attachment_ids": links[message.id]},
|
||||
) for message in page_rows]
|
||||
links = await attachments_storage.message_attachment_ids(
|
||||
session, [message.id for message in page_rows]
|
||||
)
|
||||
items = [
|
||||
Message.model_validate(message).model_copy(
|
||||
update={"attachment_ids": links[message.id]},
|
||||
)
|
||||
for message in page_rows
|
||||
]
|
||||
page = MessagePage(
|
||||
items=items,
|
||||
next_cursor=build_next_cursor(KIND_MESSAGES, chat_id, items, has_next),
|
||||
|
||||
@@ -12,41 +12,63 @@ from app.domain import attachments
|
||||
from app.endpoints.dependencies import CurrentUser, DbSession, S3Storage, UploadCapacity
|
||||
from app.schemas.attachment import Attachment
|
||||
|
||||
router = APIRouter(prefix='/api/v1', tags=['attachments'])
|
||||
router = APIRouter(prefix="/api/v1", tags=["attachments"])
|
||||
|
||||
|
||||
@router.post('/chats/{chat_id}/attachments', status_code=202)
|
||||
@router.post("/chats/{chat_id}/attachments", status_code=202)
|
||||
async def upload(
|
||||
request: Request, chat_id: UUID, user: CurrentUser, session: DbSession,
|
||||
storage: S3Storage, capacity: UploadCapacity,
|
||||
request: Request,
|
||||
chat_id: UUID,
|
||||
user: CurrentUser,
|
||||
session: DbSession,
|
||||
storage: S3Storage,
|
||||
capacity: UploadCapacity,
|
||||
filename: Annotated[str, Query(min_length=1, max_length=255)],
|
||||
) -> Attachment:
|
||||
return await attachments.receive_attachment(
|
||||
session, storage, user, chat_id, capacity=capacity, filename=filename,
|
||||
content_type=request.headers.get('content-type', '').lower(),
|
||||
declared_length=request.headers.get('content-length'), chunks=request.stream(),
|
||||
session,
|
||||
storage,
|
||||
user,
|
||||
chat_id,
|
||||
capacity=capacity,
|
||||
filename=filename,
|
||||
content_type=request.headers.get("content-type", "").lower(),
|
||||
declared_length=request.headers.get("content-length"),
|
||||
chunks=request.stream(),
|
||||
)
|
||||
|
||||
|
||||
@router.get('/attachments/{attachment_id}')
|
||||
async def metadata(attachment_id: UUID, user: CurrentUser, session: DbSession) -> Attachment:
|
||||
@router.get("/attachments/{attachment_id}")
|
||||
async def metadata(
|
||||
attachment_id: UUID, user: CurrentUser, session: DbSession
|
||||
) -> Attachment:
|
||||
return await attachments.get_attachment(session, user, attachment_id)
|
||||
|
||||
|
||||
@router.get('/attachments/{attachment_id}/content')
|
||||
@router.get("/attachments/{attachment_id}/content")
|
||||
async def content(
|
||||
attachment_id: UUID, user: CurrentUser, session: DbSession, storage: S3Storage,
|
||||
variant: Annotated[str, Query()] = 'original',
|
||||
attachment_id: UUID,
|
||||
user: CurrentUser,
|
||||
session: DbSession,
|
||||
storage: S3Storage,
|
||||
variant: Annotated[str, Query()] = "original",
|
||||
) -> StreamingResponse:
|
||||
record, stream = await attachments.download_attachment(
|
||||
session, storage, user, attachment_id, variant=variant,
|
||||
session,
|
||||
storage,
|
||||
user,
|
||||
attachment_id,
|
||||
variant=variant,
|
||||
)
|
||||
return StreamingResponse(
|
||||
stream, media_type=record.content_type,
|
||||
stream,
|
||||
media_type=record.content_type,
|
||||
headers={
|
||||
'Content-Length': str(record.size_bytes),
|
||||
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(record.filename, safe=''),
|
||||
'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'private, no-store',
|
||||
"Content-Length": str(record.size_bytes),
|
||||
"Content-Disposition": "attachment; filename*=UTF-8''"
|
||||
+ quote(record.filename, safe=""),
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Cache-Control": "private, no-store",
|
||||
},
|
||||
background=BackgroundTask(stream.aclose),
|
||||
)
|
||||
|
||||
@@ -3,18 +3,21 @@ from fastapi import APIRouter, Request
|
||||
from app.domain import health
|
||||
from app.schemas.base import Health
|
||||
|
||||
router = APIRouter(prefix='/health', tags=['health'])
|
||||
router = APIRouter(prefix="/health", tags=["health"])
|
||||
|
||||
|
||||
@router.get('/live')
|
||||
@router.get("/live")
|
||||
async def liveness() -> Health:
|
||||
return health.liveness()
|
||||
|
||||
|
||||
@router.get('/ready')
|
||||
@router.get("/ready")
|
||||
async def readiness(request: Request) -> Health:
|
||||
state = request.app.state
|
||||
return await health.readiness(
|
||||
state.engine, state.redis, state.s3, state.lifecycle,
|
||||
state.engine,
|
||||
state.redis,
|
||||
state.s3,
|
||||
state.lifecycle,
|
||||
state.settings.readiness_timeout_seconds,
|
||||
)
|
||||
|
||||
@@ -21,9 +21,13 @@ async def send_new_message(
|
||||
idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None,
|
||||
) -> Message:
|
||||
return await send_message_idempotent(
|
||||
session, sender, chat_id, new_message,
|
||||
session,
|
||||
sender,
|
||||
chat_id,
|
||||
new_message,
|
||||
validate_idempotency_key(idempotency_key),
|
||||
settings.max_messages, settings.idempotency_ttl_seconds,
|
||||
settings.max_messages,
|
||||
settings.idempotency_ttl_seconds,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -90,14 +90,23 @@ def internal_error() -> ApiError:
|
||||
|
||||
|
||||
IMAGE_WORKER_ERRORS = (
|
||||
ApiError, Image.DecompressionBombError, Image.DecompressionBombWarning,
|
||||
MemoryError, UnidentifiedImageError, OSError, ValueError, SyntaxError,
|
||||
ApiError,
|
||||
Image.DecompressionBombError,
|
||||
Image.DecompressionBombWarning,
|
||||
MemoryError,
|
||||
UnidentifiedImageError,
|
||||
OSError,
|
||||
ValueError,
|
||||
SyntaxError,
|
||||
)
|
||||
|
||||
|
||||
def image_worker_error_code(error: Exception) -> int:
|
||||
code = 2
|
||||
resource_error = isinstance(error, (Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError))
|
||||
resource_error = isinstance(
|
||||
error,
|
||||
(Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError),
|
||||
)
|
||||
declared_limit = isinstance(error, ApiError) and error.status_code == 413
|
||||
if resource_error or declared_limit:
|
||||
code = 3
|
||||
|
||||
+17
-4
@@ -18,14 +18,25 @@ from urllib3.exceptions import HTTPError as StorageHTTPError
|
||||
|
||||
from app.core.instance import INSTANCE_ID_HEADER
|
||||
from app.core.request_id import REQUEST_ID_HEADER
|
||||
from app.errors.exceptions import ApiError, internal_error, invalid_request, not_found, unavailable
|
||||
from app.errors.exceptions import (
|
||||
ApiError,
|
||||
internal_error,
|
||||
invalid_request,
|
||||
not_found,
|
||||
unavailable,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TRANSIENT_SQLSTATE_CLASSES = ("08", "40", "53", "57")
|
||||
TRANSIENT_SQLSTATES = frozenset({"55P03"})
|
||||
RECOVERABLE_STORAGE_ERRORS = (
|
||||
ApiError, SQLAlchemyError, MinioException, StorageHTTPError, OSError, TimeoutError,
|
||||
ApiError,
|
||||
SQLAlchemyError,
|
||||
MinioException,
|
||||
StorageHTTPError,
|
||||
OSError,
|
||||
TimeoutError,
|
||||
)
|
||||
|
||||
|
||||
@@ -40,12 +51,14 @@ async def operation_timeout(seconds: float, error: ApiError):
|
||||
|
||||
|
||||
async def client_disconnect_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||
return await api_error_handler(request, invalid_request("Загрузка прервана клиентом"))
|
||||
return await api_error_handler(
|
||||
request, invalid_request("Загрузка прервана клиентом")
|
||||
)
|
||||
|
||||
|
||||
def log_recoverable_error(operation: str, error: Exception) -> None:
|
||||
"""Согласованный безопасный лог восстановления без содержимого SDK-ошибки."""
|
||||
logger.warning('%s deferred: %s', operation, type(error).__name__)
|
||||
logger.warning("%s deferred: %s", operation, type(error).__name__)
|
||||
|
||||
|
||||
def _get_request_id(request: Request) -> str:
|
||||
|
||||
+3
-1
@@ -52,7 +52,9 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
application.state.session_factory = create_session_factory(engine)
|
||||
application.state.redis = redis
|
||||
application.state.s3 = s3
|
||||
application.state.upload_capacity = create_request_capacity(settings.s3.max_concurrent_uploads)
|
||||
application.state.upload_capacity = create_request_capacity(
|
||||
settings.s3.max_concurrent_uploads
|
||||
)
|
||||
application.state.request_capacity = create_request_capacity(settings.max_in_flight)
|
||||
application.state.instance_id = resolve_instance_id(settings.instance_id)
|
||||
application.state.lifecycle = Lifecycle()
|
||||
|
||||
+14
-1
@@ -9,4 +9,17 @@ from app.models.idempotency_key import IdempotencyKey
|
||||
from app.models.message import Message
|
||||
from app.models.user import User
|
||||
|
||||
__all__ = ["JobReplay", "AttachmentJob", "Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"]
|
||||
__all__ = [
|
||||
"JobReplay",
|
||||
"AttachmentJob",
|
||||
"Base",
|
||||
"Attachment",
|
||||
"MessageAttachment",
|
||||
"UploadIntent",
|
||||
"Chat",
|
||||
"ChatMember",
|
||||
"Counter",
|
||||
"IdempotencyKey",
|
||||
"Message",
|
||||
"User",
|
||||
]
|
||||
|
||||
@@ -18,8 +18,13 @@ class Attachment(Base):
|
||||
"content_type IN ('image/png', 'image/jpeg')", name="content_type"
|
||||
),
|
||||
CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"),
|
||||
CheckConstraint("status IN ('queued', 'processing', 'ready', 'failed')", name="status"),
|
||||
CheckConstraint("(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", name="error_code"),
|
||||
CheckConstraint(
|
||||
"status IN ('queued', 'processing', 'ready', 'failed')", name="status"
|
||||
),
|
||||
CheckConstraint(
|
||||
"(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)",
|
||||
name="error_code",
|
||||
),
|
||||
Index("ix_attachments_chat_id", "chat_id"),
|
||||
)
|
||||
|
||||
@@ -36,7 +41,9 @@ class Attachment(Base):
|
||||
thumbnail_key: Mapped[str | None] = mapped_column(String(128))
|
||||
crop_key: Mapped[str | None] = mapped_column(String(128))
|
||||
optimized_key: Mapped[str | None] = mapped_column(String(128))
|
||||
variants: Mapped[list[dict]] = mapped_column(JSONB, default=list, server_default=text("'[]'::jsonb"))
|
||||
variants: Mapped[list[dict]] = mapped_column(
|
||||
JSONB, default=list, server_default=text("'[]'::jsonb")
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
|
||||
|
||||
+5
-2
@@ -20,7 +20,8 @@ class AttachmentJob(Base):
|
||||
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4)
|
||||
attachment_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey("attachments.id"), unique=True,
|
||||
ForeignKey("attachments.id"),
|
||||
unique=True,
|
||||
)
|
||||
staging_key: Mapped[str | None] = mapped_column(String(128))
|
||||
pipeline_version: Mapped[int] = mapped_column(server_default="1")
|
||||
@@ -29,7 +30,9 @@ class AttachmentJob(Base):
|
||||
attempt_limit: Mapped[int | None]
|
||||
last_error_permanent: Mapped[bool | None]
|
||||
published_attempt: Mapped[int] = mapped_column(server_default="0")
|
||||
next_attempt_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
next_attempt_at: Mapped[datetime] = mapped_column(
|
||||
server_default=func.clock_timestamp()
|
||||
)
|
||||
lease_token: Mapped[UUID | None]
|
||||
lease_until: Mapped[datetime | None]
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
@@ -35,7 +35,9 @@ class Attachment(Base):
|
||||
|
||||
|
||||
VARIANT_FILENAMES = {
|
||||
"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp",
|
||||
"thumbnail": "thumbnail.jpg",
|
||||
"crop": "crop.jpg",
|
||||
"optimized": "optimized.webp",
|
||||
}
|
||||
|
||||
|
||||
|
||||
+3
-1
@@ -16,5 +16,7 @@ class JobMessage(BaseModel):
|
||||
pipeline_version: int = Field(default=1, ge=1)
|
||||
attempt: int = Field(ge=1)
|
||||
created_at: AwareDatetime
|
||||
traceparent: str | None = Field(default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
|
||||
traceparent: str | None = Field(
|
||||
default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$"
|
||||
)
|
||||
tracestate: str | None = Field(default=None, max_length=512)
|
||||
|
||||
+23
-7
@@ -28,26 +28,42 @@ class JobBroker:
|
||||
logging.getLogger("aiormq").setLevel(logging.CRITICAL)
|
||||
|
||||
async def ensure_ready(self) -> None:
|
||||
if self.connection is None or self.connection.is_closed or self.channel is None or self.channel.is_closed:
|
||||
if (
|
||||
self.connection is None
|
||||
or self.connection.is_closed
|
||||
or self.channel is None
|
||||
or self.channel.is_closed
|
||||
):
|
||||
await self.aclose()
|
||||
self.connection = await aio_pika.connect(
|
||||
host=self.settings.broker_host, port=self.settings.broker_port,
|
||||
host=self.settings.broker_host,
|
||||
port=self.settings.broker_port,
|
||||
login=self.settings.broker_user,
|
||||
password=self.settings.broker_password_file.read_text().strip(),
|
||||
virtualhost=self.settings.broker_vhost, timeout=5, heartbeat=30,
|
||||
virtualhost=self.settings.broker_vhost,
|
||||
timeout=5,
|
||||
heartbeat=30,
|
||||
)
|
||||
self.channel = await self.connection.channel(
|
||||
publisher_confirms=True, on_return_raises=True
|
||||
)
|
||||
self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True)
|
||||
await self.channel.set_qos(prefetch_count=1)
|
||||
self.queue = await self.channel.declare_queue(self.settings.broker_queue, durable=True)
|
||||
self.queue = await self.channel.declare_queue(
|
||||
self.settings.broker_queue, durable=True
|
||||
)
|
||||
|
||||
async def publish_job(self, message: JobMessage) -> None:
|
||||
await self.ensure_ready()
|
||||
await self.channel.default_exchange.publish(
|
||||
aio_pika.Message(
|
||||
body=message.model_dump_json(exclude_none=True).encode(),
|
||||
content_type="application/json", delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
content_type="application/json",
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
message_id=f"{message.job_id}:{message.attempt}",
|
||||
), routing_key=self.settings.broker_queue, mandatory=True, timeout=10,
|
||||
),
|
||||
routing_key=self.settings.broker_queue,
|
||||
mandatory=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
async def receive(self, consumer: str) -> Delivery:
|
||||
|
||||
@@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
|
||||
async def check_database(engine: AsyncEngine) -> bool:
|
||||
async with engine.connect() as connection:
|
||||
await connection.execute(text('SELECT 1'))
|
||||
await connection.execute(text("SELECT 1"))
|
||||
return True
|
||||
|
||||
|
||||
|
||||
@@ -33,7 +33,8 @@ async def get_active_key(
|
||||
IdempotencyKey.method,
|
||||
IdempotencyKey.path,
|
||||
IdempotencyKey.key,
|
||||
) == tuple_(*scope),
|
||||
)
|
||||
== tuple_(*scope),
|
||||
IdempotencyKey.created_at
|
||||
> func.clock_timestamp() - timedelta(seconds=ttl_seconds),
|
||||
)
|
||||
@@ -45,14 +46,20 @@ async def save_result(
|
||||
) -> None:
|
||||
"""Сохраняет новый результат или заменяет истёкший."""
|
||||
stmt = insert(IdempotencyKey).values(
|
||||
user_id=scope[0], method=scope[1], path=scope[2], key=scope[3],
|
||||
request_hash=request_hash, message_id=message_id,
|
||||
user_id=scope[0],
|
||||
method=scope[1],
|
||||
path=scope[2],
|
||||
key=scope[3],
|
||||
request_hash=request_hash,
|
||||
message_id=message_id,
|
||||
)
|
||||
await session.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
index_elements=[
|
||||
IdempotencyKey.user_id, IdempotencyKey.method,
|
||||
IdempotencyKey.path, IdempotencyKey.key,
|
||||
IdempotencyKey.user_id,
|
||||
IdempotencyKey.method,
|
||||
IdempotencyKey.path,
|
||||
IdempotencyKey.key,
|
||||
],
|
||||
set_={
|
||||
"request_hash": request_hash,
|
||||
|
||||
@@ -13,7 +13,8 @@ async def failed_jobs(session, limit: int):
|
||||
select(AttachmentJob, Attachment)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(Attachment.status == "failed")
|
||||
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id).limit(limit)
|
||||
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id)
|
||||
.limit(limit)
|
||||
)
|
||||
return result.all()
|
||||
|
||||
@@ -23,12 +24,24 @@ async def replay_record(session, request_id):
|
||||
|
||||
|
||||
async def insert_replay(session, **values) -> bool:
|
||||
result = await session.scalar(insert(JobReplay).values(**values).on_conflict_do_nothing(index_elements=[JobReplay.id]).returning(JobReplay.id))
|
||||
result = await session.scalar(
|
||||
insert(JobReplay)
|
||||
.values(**values)
|
||||
.on_conflict_do_nothing(index_elements=[JobReplay.id])
|
||||
.returning(JobReplay.id)
|
||||
)
|
||||
return result is not None
|
||||
|
||||
|
||||
async def replay_history(session, job_id):
|
||||
return (await session.scalars(select(JobReplay).where(JobReplay.job_id == job_id).order_by(JobReplay.created_at).limit(100))).all()
|
||||
return (
|
||||
await session.scalars(
|
||||
select(JobReplay)
|
||||
.where(JobReplay.job_id == job_id)
|
||||
.order_by(JobReplay.created_at)
|
||||
.limit(100)
|
||||
)
|
||||
).all()
|
||||
|
||||
|
||||
def reset_failed(job, attachment, now, attempt_limit):
|
||||
|
||||
+43
-10
@@ -32,7 +32,9 @@ class JobWork:
|
||||
return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}"
|
||||
|
||||
|
||||
async def create_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID:
|
||||
async def create_job(
|
||||
session: AsyncSession, attachment_id: UUID, staging_key: str
|
||||
) -> UUID:
|
||||
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key)
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
@@ -58,7 +60,10 @@ async def expired_jobs(session: AsyncSession, limit: int = 20):
|
||||
result = await session.execute(
|
||||
select(AttachmentJob, Attachment)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(Attachment.status == "processing", AttachmentJob.lease_until < func.clock_timestamp())
|
||||
.where(
|
||||
Attachment.status == "processing",
|
||||
AttachmentJob.lease_until < func.clock_timestamp(),
|
||||
)
|
||||
.order_by(AttachmentJob.lease_until)
|
||||
.limit(limit)
|
||||
.with_for_update(of=AttachmentJob, skip_locked=True)
|
||||
@@ -66,19 +71,35 @@ async def expired_jobs(session: AsyncSession, limit: int = 20):
|
||||
return result.all()
|
||||
|
||||
|
||||
def start_job(job, attachment, token: UUID, now: datetime, lease_seconds: int) -> JobWork:
|
||||
def start_job(
|
||||
job, attachment, token: UUID, now: datetime, lease_seconds: int
|
||||
) -> JobWork:
|
||||
job.attempt += 1
|
||||
job.lease_token = token
|
||||
job.lease_until = now + timedelta(seconds=lease_seconds)
|
||||
attachment.status = "processing"
|
||||
attachment.updated_at = now
|
||||
return JobWork(
|
||||
job.id, attachment.id, token, job.pipeline_version, job.staging_key,
|
||||
attachment.content_type, attachment.size_bytes, attachment.sha256,
|
||||
job.id,
|
||||
attachment.id,
|
||||
token,
|
||||
job.pipeline_version,
|
||||
job.staging_key,
|
||||
attachment.content_type,
|
||||
attachment.size_bytes,
|
||||
attachment.sha256,
|
||||
)
|
||||
|
||||
|
||||
def record_failure(job, attachment, now: datetime, error_code: str, *, terminal: bool, retry_seconds: float) -> None:
|
||||
def record_failure(
|
||||
job,
|
||||
attachment,
|
||||
now: datetime,
|
||||
error_code: str,
|
||||
*,
|
||||
terminal: bool,
|
||||
retry_seconds: float,
|
||||
) -> None:
|
||||
job.lease_token = None
|
||||
job.lease_until = None
|
||||
job.last_error_code = error_code
|
||||
@@ -89,7 +110,9 @@ def record_failure(job, attachment, now: datetime, error_code: str, *, terminal:
|
||||
attachment.updated_at = now
|
||||
|
||||
|
||||
def complete_job(job, attachment, work: JobWork, variants: list[dict], now: datetime) -> None:
|
||||
def complete_job(
|
||||
job, attachment, work: JobWork, variants: list[dict], now: datetime
|
||||
) -> None:
|
||||
attachment.object_key = work.original_key
|
||||
attachment.thumbnail_key = work.variant_key("thumbnail")
|
||||
attachment.crop_key = work.variant_key("crop")
|
||||
@@ -105,15 +128,25 @@ def complete_job(job, attachment, work: JobWork, variants: list[dict], now: date
|
||||
job.finished_at = now
|
||||
|
||||
|
||||
async def terminal_staging(session: AsyncSession, retention: int, failed_retention: int, limit: int = 20):
|
||||
async def terminal_staging(
|
||||
session: AsyncSession, retention: int, failed_retention: int, limit: int = 20
|
||||
):
|
||||
result = await session.execute(
|
||||
select(AttachmentJob, Attachment)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(
|
||||
AttachmentJob.staging_key.is_not(None),
|
||||
or_(
|
||||
and_(Attachment.status == "ready", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=retention)),
|
||||
and_(Attachment.status == "failed", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=failed_retention)),
|
||||
and_(
|
||||
Attachment.status == "ready",
|
||||
AttachmentJob.finished_at
|
||||
< func.clock_timestamp() - timedelta(seconds=retention),
|
||||
),
|
||||
and_(
|
||||
Attachment.status == "failed",
|
||||
AttachmentJob.finished_at
|
||||
< func.clock_timestamp() - timedelta(seconds=failed_retention),
|
||||
),
|
||||
),
|
||||
)
|
||||
.order_by(AttachmentJob.finished_at)
|
||||
|
||||
@@ -9,7 +9,9 @@ from app.models.attachment import Attachment
|
||||
from app.models.job import AttachmentJob
|
||||
|
||||
|
||||
async def next_unpublished_job(session: AsyncSession, *, attachment_id: UUID | None = None) -> AttachmentJob | None:
|
||||
async def next_unpublished_job(
|
||||
session: AsyncSession, *, attachment_id: UUID | None = None
|
||||
) -> AttachmentJob | None:
|
||||
statement = (
|
||||
select(AttachmentJob)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
|
||||
@@ -81,7 +81,10 @@ def main():
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": [f"arn:aws:s3:::{bucket}/{prefix}/*" for prefix in ("originals", "staging", "variants")],
|
||||
"Resource": [
|
||||
f"arn:aws:s3:::{bucket}/{prefix}/*"
|
||||
for prefix in ("originals", "staging", "variants")
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
@@ -20,21 +20,26 @@ def upgrade() -> None:
|
||||
sa.Column("request_hash", sa.String(64), nullable=False),
|
||||
sa.Column("message_id", sa.UUID(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("clock_timestamp()"), nullable=False,
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("clock_timestamp()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["user_id"], ["users.id"], name=op.f("fk_idempotency_keys_user_id_users")
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["message_id"], ["messages.id"],
|
||||
["message_id"],
|
||||
["messages.id"],
|
||||
name=op.f("fk_idempotency_keys_message_id_messages"),
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"user_id", "method", "path", "key", name=op.f("pk_idempotency_keys")
|
||||
),
|
||||
)
|
||||
op.create_index("ix_idempotency_keys_created_at", "idempotency_keys", ["created_at"])
|
||||
op.create_index(
|
||||
"ix_idempotency_keys_created_at", "idempotency_keys", ["created_at"]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
|
||||
@@ -21,14 +21,35 @@ def upgrade() -> None:
|
||||
sa.Column("size_bytes", sa.Integer(), nullable=False),
|
||||
sa.Column("sha256", sa.String(64), nullable=False),
|
||||
sa.Column("status", sa.String(16), server_default="ready", nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
|
||||
sa.CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name=op.f("ck_attachments_size_bytes")),
|
||||
sa.CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name=op.f("ck_attachments_content_type")),
|
||||
sa.CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name=op.f("ck_attachments_sha256")),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("clock_timestamp()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("clock_timestamp()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"size_bytes BETWEEN 1 AND 10485760", name=op.f("ck_attachments_size_bytes")
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"content_type IN ('image/png', 'image/jpeg')",
|
||||
name=op.f("ck_attachments_content_type"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"sha256 ~ '^[a-f0-9]{64}$'", name=op.f("ck_attachments_sha256")
|
||||
),
|
||||
sa.CheckConstraint("status = 'ready'", name=op.f("ck_attachments_status")),
|
||||
sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], name=op.f("fk_attachments_chat_id_chats")),
|
||||
sa.ForeignKeyConstraint(["owner_id"], ["users.id"], name=op.f("fk_attachments_owner_id_users")),
|
||||
sa.ForeignKeyConstraint(
|
||||
["chat_id"], ["chats.id"], name=op.f("fk_attachments_chat_id_chats")
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["owner_id"], ["users.id"], name=op.f("fk_attachments_owner_id_users")
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_attachments")),
|
||||
sa.UniqueConstraint("object_key", name=op.f("uq_attachments_object_key")),
|
||||
)
|
||||
@@ -38,15 +59,31 @@ def upgrade() -> None:
|
||||
sa.Column("message_id", sa.UUID(), nullable=False),
|
||||
sa.Column("attachment_id", sa.UUID(), nullable=False),
|
||||
sa.Column("position", sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE", name=op.f("fk_message_attachments_message_id_messages")),
|
||||
sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_message_attachments_attachment_id_attachments")),
|
||||
sa.PrimaryKeyConstraint("message_id", "attachment_id", name=op.f("pk_message_attachments")),
|
||||
sa.ForeignKeyConstraint(
|
||||
["message_id"],
|
||||
["messages.id"],
|
||||
ondelete="CASCADE",
|
||||
name=op.f("fk_message_attachments_message_id_messages"),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["attachment_id"],
|
||||
["attachments.id"],
|
||||
name=op.f("fk_message_attachments_attachment_id_attachments"),
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"message_id", "attachment_id", name=op.f("pk_message_attachments")
|
||||
),
|
||||
)
|
||||
op.create_table(
|
||||
"upload_intents",
|
||||
sa.Column("id", sa.UUID(), nullable=False),
|
||||
sa.Column("object_key", sa.String(128), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("clock_timestamp()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_upload_intents")),
|
||||
sa.UniqueConstraint("object_key", name=op.f("uq_upload_intents_object_key")),
|
||||
)
|
||||
|
||||
@@ -5,29 +5,30 @@ Revises: db06fd3126ff
|
||||
Create Date: 2026-10-26 22:01:01.910396
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = 'abd273076068'
|
||||
down_revision: Union[str, None] = 'db06fd3126ff'
|
||||
revision: str = "abd273076068"
|
||||
down_revision: Union[str, None] = "db06fd3126ff"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('counters',
|
||||
sa.Column('name', sa.String(length=64), nullable=False),
|
||||
sa.Column('value', sa.BigInteger(), nullable=False),
|
||||
sa.PrimaryKeyConstraint('name', name=op.f('pk_counters'))
|
||||
op.create_table(
|
||||
"counters",
|
||||
sa.Column("name", sa.String(length=64), nullable=False),
|
||||
sa.Column("value", sa.BigInteger(), nullable=False),
|
||||
sa.PrimaryKeyConstraint("name", name=op.f("pk_counters")),
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('counters')
|
||||
op.drop_table("counters")
|
||||
# ### end Alembic commands ###
|
||||
|
||||
@@ -12,12 +12,30 @@ depends_on = None
|
||||
|
||||
def upgrade() -> None:
|
||||
op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check")
|
||||
op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status IN ('queued', 'processing', 'ready', 'failed')")
|
||||
op.create_check_constraint(
|
||||
op.f("ck_attachments_status"),
|
||||
"attachments",
|
||||
"status IN ('queued', 'processing', 'ready', 'failed')",
|
||||
)
|
||||
op.alter_column("attachments", "status", server_default="queued")
|
||||
op.add_column("attachments", sa.Column("error_code", sa.String(64), nullable=True))
|
||||
op.add_column("attachments", sa.Column("thumbnail_key", sa.String(128), nullable=True))
|
||||
op.add_column("attachments", sa.Column("variants", postgresql.JSONB(), nullable=False, server_default=sa.text("'[]'::jsonb")))
|
||||
op.create_check_constraint(op.f("ck_attachments_error_code"), "attachments", "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)")
|
||||
op.add_column(
|
||||
"attachments", sa.Column("thumbnail_key", sa.String(128), nullable=True)
|
||||
)
|
||||
op.add_column(
|
||||
"attachments",
|
||||
sa.Column(
|
||||
"variants",
|
||||
postgresql.JSONB(),
|
||||
nullable=False,
|
||||
server_default=sa.text("'[]'::jsonb"),
|
||||
),
|
||||
)
|
||||
op.create_check_constraint(
|
||||
op.f("ck_attachments_error_code"),
|
||||
"attachments",
|
||||
"(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)",
|
||||
)
|
||||
op.create_table(
|
||||
"attachment_jobs",
|
||||
sa.Column("id", sa.UUID(), nullable=False),
|
||||
@@ -25,31 +43,62 @@ def upgrade() -> None:
|
||||
sa.Column("staging_key", sa.String(128), nullable=True),
|
||||
sa.Column("pipeline_version", sa.Integer(), server_default="1", nullable=False),
|
||||
sa.Column("attempt", sa.Integer(), server_default="0", nullable=False),
|
||||
sa.Column("published_attempt", sa.Integer(), server_default="0", nullable=False),
|
||||
sa.Column("next_attempt_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
|
||||
sa.Column(
|
||||
"published_attempt", sa.Integer(), server_default="0", nullable=False
|
||||
),
|
||||
sa.Column(
|
||||
"next_attempt_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("clock_timestamp()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("lease_token", sa.UUID(), nullable=True),
|
||||
sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_error_code", sa.String(64), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("clock_timestamp()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_attachment_jobs")),
|
||||
sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_attachment_jobs_attachment_id_attachments")),
|
||||
sa.UniqueConstraint("attachment_id", name=op.f("uq_attachment_jobs_attachment_id")),
|
||||
sa.CheckConstraint("pipeline_version = 1", name=op.f("ck_attachment_jobs_pipeline_version")),
|
||||
sa.CheckConstraint("attempt >= 0 AND published_attempt >= 0", name=op.f("ck_attachment_jobs_attempt")),
|
||||
sa.ForeignKeyConstraint(
|
||||
["attachment_id"],
|
||||
["attachments.id"],
|
||||
name=op.f("fk_attachment_jobs_attachment_id_attachments"),
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"attachment_id", name=op.f("uq_attachment_jobs_attachment_id")
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"pipeline_version = 1", name=op.f("ck_attachment_jobs_pipeline_version")
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"attempt >= 0 AND published_attempt >= 0",
|
||||
name=op.f("ck_attachment_jobs_attempt"),
|
||||
),
|
||||
)
|
||||
op.create_index("ix_attachment_jobs_due", "attachment_jobs", ["next_attempt_at"])
|
||||
op.create_index("ix_attachment_jobs_lease", "attachment_jobs", ["lease_until"])
|
||||
# Прежние оригиналы остаются источником; worker создаст отсутствующий thumbnail.
|
||||
op.execute("INSERT INTO attachment_jobs (id, attachment_id, staging_key) SELECT id, id, object_key FROM attachments")
|
||||
op.execute("UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()")
|
||||
op.execute(
|
||||
"INSERT INTO attachment_jobs (id, attachment_id, staging_key) SELECT id, id, object_key FROM attachments"
|
||||
)
|
||||
op.execute(
|
||||
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
connection = op.get_bind()
|
||||
unfinished = connection.scalar(sa.text("SELECT count(*) FROM attachments WHERE status <> 'ready'"))
|
||||
unfinished = connection.scalar(
|
||||
sa.text("SELECT count(*) FROM attachments WHERE status <> 'ready'")
|
||||
)
|
||||
if unfinished:
|
||||
raise RuntimeError("Перед downgrade нужно завершить или вручную разобрать все queued/processing/failed вложения")
|
||||
raise RuntimeError(
|
||||
"Перед downgrade нужно завершить или вручную разобрать все queued/processing/failed вложения"
|
||||
)
|
||||
op.drop_index("ix_attachment_jobs_lease", table_name="attachment_jobs")
|
||||
op.drop_index("ix_attachment_jobs_due", table_name="attachment_jobs")
|
||||
op.drop_table("attachment_jobs")
|
||||
@@ -58,5 +107,7 @@ def downgrade() -> None:
|
||||
op.drop_column("attachments", "thumbnail_key")
|
||||
op.drop_column("attachments", "error_code")
|
||||
op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check")
|
||||
op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status = 'ready'")
|
||||
op.create_check_constraint(
|
||||
op.f("ck_attachments_status"), "attachments", "status = 'ready'"
|
||||
)
|
||||
op.alter_column("attachments", "status", server_default="ready")
|
||||
|
||||
@@ -11,7 +11,9 @@ depends_on = None
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("attachments", sa.Column("crop_key", sa.String(128), nullable=True))
|
||||
op.add_column("attachments", sa.Column("optimized_key", sa.String(128), nullable=True))
|
||||
op.add_column(
|
||||
"attachments", sa.Column("optimized_key", sa.String(128), nullable=True)
|
||||
)
|
||||
# Старый staging сменяется надёжным ready-оригиналом, но не теряется из журнала cleanup.
|
||||
op.execute("""
|
||||
INSERT INTO upload_intents (id, object_key)
|
||||
@@ -37,7 +39,11 @@ def upgrade() -> None:
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
active = op.get_bind().scalar(sa.text("SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')"))
|
||||
active = op.get_bind().scalar(
|
||||
sa.text(
|
||||
"SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')"
|
||||
)
|
||||
)
|
||||
if active:
|
||||
raise RuntimeError("Перед downgrade необходимо завершить активные задания")
|
||||
op.execute("""
|
||||
|
||||
@@ -10,19 +10,36 @@ depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("attachment_jobs", sa.Column("cycle_start_attempt", sa.Integer(), nullable=False, server_default="0"))
|
||||
op.add_column("attachment_jobs", sa.Column("attempt_limit", sa.Integer(), nullable=True))
|
||||
op.add_column("attachment_jobs", sa.Column("last_error_permanent", sa.Boolean(), nullable=True))
|
||||
op.add_column(
|
||||
"attachment_jobs",
|
||||
sa.Column(
|
||||
"cycle_start_attempt", sa.Integer(), nullable=False, server_default="0"
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"attachment_jobs", sa.Column("attempt_limit", sa.Integer(), nullable=True)
|
||||
)
|
||||
op.add_column(
|
||||
"attachment_jobs",
|
||||
sa.Column("last_error_permanent", sa.Boolean(), nullable=True),
|
||||
)
|
||||
op.create_table(
|
||||
"job_replays",
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.Column("job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False),
|
||||
sa.Column(
|
||||
"job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False
|
||||
),
|
||||
sa.Column("actor", sa.String(64), nullable=False),
|
||||
sa.Column("reason", sa.String(32), nullable=False),
|
||||
sa.Column("previous_attempt", sa.Integer(), nullable=False),
|
||||
sa.Column("attempt_limit", sa.Integer(), nullable=False),
|
||||
sa.Column("previous_error_code", sa.String(64), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.clock_timestamp()),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.clock_timestamp(),
|
||||
),
|
||||
)
|
||||
op.create_index("ix_job_replays_job_id", "job_replays", ["job_id"])
|
||||
|
||||
|
||||
+56
-32
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline structural checks; full schema validation is a separate target."""
|
||||
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
@@ -7,48 +8,71 @@ import re
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
lab = json.loads((ROOT/'lab.json').read_text())['number']
|
||||
spec = json.loads((ROOT/'contracts/openapi.json').read_text())
|
||||
assert spec['openapi']=='3.1.0'
|
||||
assert spec['info']['version']=='1.%d.0'%lab
|
||||
lab = json.loads((ROOT / "lab.json").read_text())["number"]
|
||||
spec = json.loads((ROOT / "contracts/openapi.json").read_text())
|
||||
assert spec["openapi"] == "3.1.0"
|
||||
assert spec["info"]["version"] == "1.%d.0" % lab
|
||||
operation_ids = set()
|
||||
|
||||
def walk(value):
|
||||
if isinstance(value,dict):
|
||||
if '$ref' in value:
|
||||
pointer = value['$ref']
|
||||
assert pointer.startswith('#/'), 'unexpected external reference'
|
||||
if isinstance(value, dict):
|
||||
if "$ref" in value:
|
||||
pointer = value["$ref"]
|
||||
assert pointer.startswith("#/"), "unexpected external reference"
|
||||
target = spec
|
||||
for part in pointer[2:].split('/'):
|
||||
target = target[part.replace('~1','/').replace('~0','~')]
|
||||
for part in pointer[2:].split("/"):
|
||||
target = target[part.replace("~1", "/").replace("~0", "~")]
|
||||
for v in value.values():
|
||||
walk(v)
|
||||
elif isinstance(value,list):
|
||||
elif isinstance(value, list):
|
||||
for v in value:
|
||||
walk(v)
|
||||
|
||||
walk(spec)
|
||||
for route,methods in spec['paths'].items():
|
||||
placeholders = set(re.findall(r'\{([^}]+)\}',route))
|
||||
for method,op in methods.items():
|
||||
assert op['operationId'] not in operation_ids, 'duplicate operationId'
|
||||
operation_ids.add(op['operationId'])
|
||||
params = {p['name'] for p in op.get('parameters',[]) if p['in']=='path' and p.get('required')}
|
||||
assert placeholders==params, 'path parameter mismatch: '+route
|
||||
for response in op['responses'].values():
|
||||
assert 'X-Request-Id' in response['headers']
|
||||
if lab>=3:
|
||||
assert 'X-Instance-Id' in response['headers']
|
||||
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
|
||||
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
|
||||
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
|
||||
ast.parse(file.read_text(),filename=str(file))
|
||||
for file in ROOT.rglob('*.json'):
|
||||
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
|
||||
for route, methods in spec["paths"].items():
|
||||
placeholders = set(re.findall(r"\{([^}]+)\}", route))
|
||||
for method, op in methods.items():
|
||||
assert op["operationId"] not in operation_ids, "duplicate operationId"
|
||||
operation_ids.add(op["operationId"])
|
||||
params = {
|
||||
p["name"]
|
||||
for p in op.get("parameters", [])
|
||||
if p["in"] == "path" and p.get("required")
|
||||
}
|
||||
assert placeholders == params, "path parameter mismatch: " + route
|
||||
for response in op["responses"].values():
|
||||
assert "X-Request-Id" in response["headers"]
|
||||
if lab >= 3:
|
||||
assert "X-Instance-Id" in response["headers"]
|
||||
assert ("/api/v1/auth/sessions" in spec["paths"]) == (lab >= 2)
|
||||
assert ("/api/v1/attachments/{attachment_id}" in spec["paths"]) == (lab >= 4)
|
||||
for file in list((ROOT / "tests").glob("*.py")) + list(
|
||||
(ROOT / "scripts").glob("*.py")
|
||||
):
|
||||
ast.parse(file.read_text(), filename=str(file))
|
||||
for file in ROOT.rglob("*.json"):
|
||||
if any(
|
||||
part in {".git", ".venv", "node_modules", "vendor", "output"}
|
||||
for part in file.parts
|
||||
):
|
||||
continue
|
||||
json.loads(file.read_text())
|
||||
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
|
||||
assert (ROOT/required).is_file(), 'missing '+required
|
||||
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
|
||||
for required in (
|
||||
"README.md",
|
||||
"COURSE.md",
|
||||
"CONTRIBUTING.md",
|
||||
"REPORT.md",
|
||||
"contracts/README.md",
|
||||
"tests/smoke.py",
|
||||
"tests/README.md",
|
||||
):
|
||||
assert (ROOT / required).is_file(), "missing " + required
|
||||
print(
|
||||
"PASS: offline skeleton structure, local refs and Python syntax; application NOT tested."
|
||||
)
|
||||
|
||||
if __name__=='__main__':
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
+340
-122
@@ -16,7 +16,7 @@ from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compos
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, check
|
||||
|
||||
PROBE = '''
|
||||
PROBE = """
|
||||
import asyncio, hashlib, json
|
||||
from uuid import UUID
|
||||
from sqlalchemy import select
|
||||
@@ -43,169 +43,387 @@ async def main():
|
||||
result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
|
||||
finally: await storage.aclose();await engine.dispose()
|
||||
asyncio.run(main())
|
||||
'''
|
||||
"""
|
||||
|
||||
|
||||
def probe(attachment_id):
|
||||
code = 'ATTACHMENT_ID = ' + repr(str(UUID(attachment_id))) + '\n' + PROBE
|
||||
result = subprocess.run(COMPOSE + ['exec', '-T', 'api', 'python', '-'], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30)
|
||||
check(result.returncode == 0, 'Probe failed; diagnostic output hidden')
|
||||
code = "ATTACHMENT_ID = " + repr(str(UUID(attachment_id))) + "\n" + PROBE
|
||||
result = subprocess.run(
|
||||
COMPOSE + ["exec", "-T", "api", "python", "-"],
|
||||
input=code,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
cwd=ROOT,
|
||||
timeout=30,
|
||||
)
|
||||
check(result.returncode == 0, "Probe failed; diagnostic output hidden")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def admin(*arguments, expected=0):
|
||||
result = compose('run', '--rm', '--no-deps', 'worker', 'python', '-m', 'app.core.job_admin', *arguments)
|
||||
check(result.returncode == expected, 'Unexpected operator command exit status')
|
||||
result = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"app.core.job_admin",
|
||||
*arguments
|
||||
)
|
||||
check(result.returncode == expected, "Unexpected operator command exit status")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def verify_result(client, actor, attachment, original):
|
||||
ready = client.wait_ready(actor, attachment, wait_seconds=180)
|
||||
check(ready['id'] == attachment['id'], 'Attachment identity changed')
|
||||
path = '/api/v1/attachments/' + ready['id'] + '/content'
|
||||
check(client.call('GET', path, actor=actor)['bytes'] == original, 'Original changed')
|
||||
check({v['name'] for v in ready['variants']} == {'thumbnail','crop','optimized'}, 'Incomplete pipeline')
|
||||
for variant in ready['variants']:
|
||||
data = client.call('GET', path+'?variant='+variant['name'], actor=actor)['bytes']
|
||||
check(hashlib.sha256(data).hexdigest() == variant['sha256'], 'Checksum mismatch')
|
||||
check(ready["id"] == attachment["id"], "Attachment identity changed")
|
||||
path = "/api/v1/attachments/" + ready["id"] + "/content"
|
||||
check(
|
||||
client.call("GET", path, actor=actor)["bytes"] == original, "Original changed"
|
||||
)
|
||||
check(
|
||||
{v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"},
|
||||
"Incomplete pipeline",
|
||||
)
|
||||
for variant in ready["variants"]:
|
||||
data = client.call("GET", path + "?variant=" + variant["name"], actor=actor)[
|
||||
"bytes"
|
||||
]
|
||||
check(
|
||||
hashlib.sha256(data).hexdigest() == variant["sha256"], "Checksum mismatch"
|
||||
)
|
||||
with Image.open(BytesIO(data)) as image:
|
||||
check(image.size == (variant['width'],variant['height']), 'Metadata dimensions mismatch')
|
||||
check(image.format == ('WEBP' if variant['name']=='optimized' else 'JPEG'), 'Wrong format')
|
||||
check(not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp'), 'Metadata leaked')
|
||||
if variant['name']=='crop':check(image.size==(128,128),'Wrong crop size')
|
||||
snapshot=probe(ready['id'])
|
||||
check(len(snapshot['objects']) == 4, 'Expected exactly one original and three variants')
|
||||
check(
|
||||
image.size == (variant["width"], variant["height"]),
|
||||
"Metadata dimensions mismatch",
|
||||
)
|
||||
check(
|
||||
image.format == ("WEBP" if variant["name"] == "optimized" else "JPEG"),
|
||||
"Wrong format",
|
||||
)
|
||||
check(
|
||||
not image.getexif()
|
||||
and not image.info.get("icc_profile")
|
||||
and not image.info.get("xmp"),
|
||||
"Metadata leaked",
|
||||
)
|
||||
if variant["name"] == "crop":
|
||||
check(image.size == (128, 128), "Wrong crop size")
|
||||
snapshot = probe(ready["id"])
|
||||
check(
|
||||
len(snapshot["objects"]) == 4,
|
||||
"Expected exactly one original and three variants",
|
||||
)
|
||||
return snapshot
|
||||
|
||||
|
||||
def crash(client, actor, chat, mode):
|
||||
name='messenger-worker-fault-'+uuid4().hex
|
||||
name = "messenger-worker-fault-" + uuid4().hex
|
||||
try:
|
||||
require_compose('stop','worker')
|
||||
attachment,original=client.attachment(actor,chat)
|
||||
result=compose('run','--rm','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
|
||||
phase='after_object:optimized' if mode=='crash-after-object' else 'before_ack'
|
||||
marker=json.dumps({'fault':phase,'attachment_id':attachment['id']})
|
||||
check(result.returncode==137 and marker in result.stdout,'Targeted crash not observed')
|
||||
before=probe(attachment['id'])
|
||||
check(before['status']==('processing' if mode=='crash-after-object' else 'ready'),'Unexpected crash state')
|
||||
check(len(before['objects'])==4,'Crash must occur after all four PUTs')
|
||||
require_compose("stop", "worker")
|
||||
attachment, original = client.attachment(actor, chat)
|
||||
result = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"--name",
|
||||
name,
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"app.core.job_worker",
|
||||
"--" + mode,
|
||||
attachment["id"],
|
||||
)
|
||||
phase = (
|
||||
"after_object:optimized" if mode == "crash-after-object" else "before_ack"
|
||||
)
|
||||
marker = json.dumps({"fault": phase, "attachment_id": attachment["id"]})
|
||||
check(
|
||||
result.returncode == 137 and marker in result.stdout,
|
||||
"Targeted crash not observed",
|
||||
)
|
||||
before = probe(attachment["id"])
|
||||
check(
|
||||
before["status"]
|
||||
== ("processing" if mode == "crash-after-object" else "ready"),
|
||||
"Unexpected crash state",
|
||||
)
|
||||
check(len(before["objects"]) == 4, "Crash must occur after all four PUTs")
|
||||
finally:
|
||||
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
|
||||
require_compose('start','worker')
|
||||
after=verify_result(client,actor,attachment,original)
|
||||
if mode=='crash-before-ack':
|
||||
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
|
||||
require_compose("start", "worker")
|
||||
after = verify_result(client, actor, attachment, original)
|
||||
if mode == "crash-before-ack":
|
||||
# Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
|
||||
wait_queue_empty()
|
||||
after=probe(attachment['id'])
|
||||
check(after==before,'Ready duplicate rewrote result')
|
||||
after = probe(attachment["id"])
|
||||
check(after == before, "Ready duplicate rewrote result")
|
||||
else:
|
||||
check(after['attempt']==before['attempt']+1,'Lease did not recover the interrupted attempt')
|
||||
check([(x['name'],x['sha256']) for x in before['objects']]==[(x['name'],x['sha256']) for x in after['objects']],'Replay changed object content')
|
||||
return dict(scenario=mode,attachment_id=attachment['id'],exit_code=137,before=before,after=after)
|
||||
check(
|
||||
after["attempt"] == before["attempt"] + 1,
|
||||
"Lease did not recover the interrupted attempt",
|
||||
)
|
||||
check(
|
||||
[(x["name"], x["sha256"]) for x in before["objects"]]
|
||||
== [(x["name"], x["sha256"]) for x in after["objects"]],
|
||||
"Replay changed object content",
|
||||
)
|
||||
return dict(
|
||||
scenario=mode,
|
||||
attachment_id=attachment["id"],
|
||||
exit_code=137,
|
||||
before=before,
|
||||
after=after,
|
||||
)
|
||||
|
||||
|
||||
def retry_replay(client,actor,chat,permanent):
|
||||
name='messenger-retry-fault-'+uuid4().hex
|
||||
samples=[]
|
||||
configuration=ComposeProject().config['services']['worker']['environment']
|
||||
retry_base=float(configuration['JOB_RETRY_SECONDS'])
|
||||
retry_max=float(configuration['JOB_RETRY_MAX_SECONDS'])
|
||||
def retry_replay(client, actor, chat, permanent):
|
||||
name = "messenger-retry-fault-" + uuid4().hex
|
||||
samples = []
|
||||
configuration = ComposeProject().config["services"]["worker"]["environment"]
|
||||
retry_base = float(configuration["JOB_RETRY_SECONDS"])
|
||||
retry_max = float(configuration["JOB_RETRY_MAX_SECONDS"])
|
||||
try:
|
||||
require_compose('stop','worker')
|
||||
attachment,original=client.attachment(actor,chat)
|
||||
mode='fail-permanent' if permanent else 'fail-transient'
|
||||
require_compose('run','-d','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
|
||||
deadline=time.monotonic()+180
|
||||
while time.monotonic()<deadline:
|
||||
sample=probe(attachment['id'])
|
||||
if sample['attempt'] and sample['status'] in {'queued', 'failed'} and (not samples or samples[-1]['attempt']!=sample['attempt']):samples.append(sample)
|
||||
if sample['status']=='failed':break
|
||||
time.sleep(.5)
|
||||
check(sample['status']=='failed','Retry never reached failed')
|
||||
check(sample['attempt']==(1 if permanent else sample['limit']),'Incorrect attempt budget')
|
||||
check(sample['permanent']==permanent,'Incorrect error classification')
|
||||
require_compose("stop", "worker")
|
||||
attachment, original = client.attachment(actor, chat)
|
||||
mode = "fail-permanent" if permanent else "fail-transient"
|
||||
require_compose(
|
||||
"run",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--name",
|
||||
name,
|
||||
"worker",
|
||||
"python",
|
||||
"-m",
|
||||
"app.core.job_worker",
|
||||
"--" + mode,
|
||||
attachment["id"],
|
||||
)
|
||||
deadline = time.monotonic() + 180
|
||||
while time.monotonic() < deadline:
|
||||
sample = probe(attachment["id"])
|
||||
if (
|
||||
sample["attempt"]
|
||||
and sample["status"] in {"queued", "failed"}
|
||||
and (not samples or samples[-1]["attempt"] != sample["attempt"])
|
||||
):
|
||||
samples.append(sample)
|
||||
if sample["status"] == "failed":
|
||||
break
|
||||
time.sleep(0.5)
|
||||
check(sample["status"] == "failed", "Retry never reached failed")
|
||||
check(
|
||||
sample["attempt"] == (1 if permanent else sample["limit"]),
|
||||
"Incorrect attempt budget",
|
||||
)
|
||||
check(sample["permanent"] == permanent, "Incorrect error classification")
|
||||
if not permanent:
|
||||
check(len(samples)==sample['limit'],'Missed retry samples')
|
||||
check(len(samples) == sample["limit"], "Missed retry samples")
|
||||
for entry in samples:
|
||||
delay=(datetime.fromisoformat(entry['next_attempt_at'])-datetime.fromisoformat(entry['updated_at'])).total_seconds()
|
||||
ceiling=min(retry_max,retry_base*2**(entry['attempt']-1))
|
||||
check(ceiling/2<=delay<=ceiling,'Retry delay outside configured jitter bounds')
|
||||
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
|
||||
job_id=sample['job_id'];request_id=str(uuid4())
|
||||
listing=admin('list-failed')
|
||||
check(any(row['job_id']==job_id for row in listing),'Job missing from failed registry')
|
||||
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30,check=True)
|
||||
first=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
|
||||
check(first['status']=='queued','Replay did not queue job')
|
||||
delay = (
|
||||
datetime.fromisoformat(entry["next_attempt_at"])
|
||||
- datetime.fromisoformat(entry["updated_at"])
|
||||
).total_seconds()
|
||||
ceiling = min(retry_max, retry_base * 2 ** (entry["attempt"] - 1))
|
||||
check(
|
||||
ceiling / 2 <= delay <= ceiling,
|
||||
"Retry delay outside configured jitter bounds",
|
||||
)
|
||||
client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=actor,
|
||||
expected=409,
|
||||
)
|
||||
job_id = sample["job_id"]
|
||||
request_id = str(uuid4())
|
||||
listing = admin("list-failed")
|
||||
check(
|
||||
any(row["job_id"] == job_id for row in listing),
|
||||
"Job missing from failed registry",
|
||||
)
|
||||
subprocess.run(
|
||||
["docker", "rm", "-f", name], capture_output=True, timeout=30, check=True
|
||||
)
|
||||
first = admin(
|
||||
"replay",
|
||||
job_id,
|
||||
"--request-id",
|
||||
request_id,
|
||||
"--actor",
|
||||
"verification",
|
||||
"--reason",
|
||||
"dependency_restored",
|
||||
)
|
||||
check(first["status"] == "queued", "Replay did not queue job")
|
||||
finally:
|
||||
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
|
||||
require_compose('start','worker')
|
||||
after=verify_result(client,actor,attachment,original)
|
||||
repeat=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
|
||||
check(repeat['already_applied'],'Replay request id not idempotent')
|
||||
refused=admin('replay',job_id,'--request-id',str(uuid4()),'--actor','verification','--reason','manual_review',expected=2)
|
||||
check(refused['error_code']=='job_not_failed','Successful job was replayed')
|
||||
check(probe(attachment['id'])==after,'Operator replay altered ready job')
|
||||
check(len(admin('history',job_id))==1,'Incorrect audit count')
|
||||
return dict(scenario=mode,attachment_id=attachment['id'],samples=samples,after=after,replay_idempotent=True)
|
||||
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
|
||||
require_compose("start", "worker")
|
||||
after = verify_result(client, actor, attachment, original)
|
||||
repeat = admin(
|
||||
"replay",
|
||||
job_id,
|
||||
"--request-id",
|
||||
request_id,
|
||||
"--actor",
|
||||
"verification",
|
||||
"--reason",
|
||||
"dependency_restored",
|
||||
)
|
||||
check(repeat["already_applied"], "Replay request id not idempotent")
|
||||
refused = admin(
|
||||
"replay",
|
||||
job_id,
|
||||
"--request-id",
|
||||
str(uuid4()),
|
||||
"--actor",
|
||||
"verification",
|
||||
"--reason",
|
||||
"manual_review",
|
||||
expected=2,
|
||||
)
|
||||
check(refused["error_code"] == "job_not_failed", "Successful job was replayed")
|
||||
check(probe(attachment["id"]) == after, "Operator replay altered ready job")
|
||||
check(len(admin("history", job_id)) == 1, "Incorrect audit count")
|
||||
return dict(
|
||||
scenario=mode,
|
||||
attachment_id=attachment["id"],
|
||||
samples=samples,
|
||||
after=after,
|
||||
replay_idempotent=True,
|
||||
)
|
||||
|
||||
|
||||
def queue_state():
|
||||
output=require_compose('exec','-T','broker','rabbitmqctl','list_queues','--formatter=json','name','messages_ready','messages_unacknowledged','consumers','durable')
|
||||
return next(row for row in json.loads(output) if row['name']=='attachment-jobs')
|
||||
output = require_compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"broker",
|
||||
"rabbitmqctl",
|
||||
"list_queues",
|
||||
"--formatter=json",
|
||||
"name",
|
||||
"messages_ready",
|
||||
"messages_unacknowledged",
|
||||
"consumers",
|
||||
"durable",
|
||||
)
|
||||
return next(row for row in json.loads(output) if row["name"] == "attachment-jobs")
|
||||
|
||||
|
||||
def wait_queue_empty():
|
||||
deadline=time.monotonic()+20
|
||||
while time.monotonic()<deadline:
|
||||
state=queue_state()
|
||||
if state['messages_ready']==0 and state['messages_unacknowledged']==0:
|
||||
deadline = time.monotonic() + 20
|
||||
while time.monotonic() < deadline:
|
||||
state = queue_state()
|
||||
if state["messages_ready"] == 0 and state["messages_unacknowledged"] == 0:
|
||||
return state
|
||||
time.sleep(.5)
|
||||
check(False,'RabbitMQ did not acknowledge all completed deliveries')
|
||||
time.sleep(0.5)
|
||||
check(False, "RabbitMQ did not acknowledge all completed deliveries")
|
||||
|
||||
|
||||
def persistence(client,actor,chat):
|
||||
def persistence(client, actor, chat):
|
||||
try:
|
||||
require_compose('stop','worker')
|
||||
attachment,original=client.attachment(actor,chat)
|
||||
client.message(actor,chat,'',attachments=[attachment['id']])
|
||||
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
|
||||
require_compose("stop", "worker")
|
||||
attachment, original = client.attachment(actor, chat)
|
||||
client.message(actor, chat, "", attachments=[attachment["id"]])
|
||||
client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=actor,
|
||||
expected=409,
|
||||
)
|
||||
time.sleep(2)
|
||||
queued_before=queue_state()
|
||||
check(queued_before['messages_ready']>=1 and queued_before['durable'], 'No durable queued message before restart')
|
||||
result=subprocess.run([str(ROOT/'scripts/restart.sh')],cwd=ROOT,capture_output=True,timeout=180)
|
||||
check(result.returncode==0,'Restart hook failed')
|
||||
check('worker' not in require_compose('ps','--status','running','--services').splitlines(),'Restart started deliberately stopped worker')
|
||||
queued_after=queue_state()
|
||||
check(queued_after['messages_ready']>=1 and queued_after['durable'], 'Broker lost queued delivery across restart')
|
||||
client.message(actor,chat,'session and chat survived broker/API restart')
|
||||
finally:require_compose('start','worker')
|
||||
after=verify_result(client,actor,attachment,original)
|
||||
return dict(scenario='worker_stopped_restart_all',attachment_id=attachment['id'],queue_before=queued_before,queue_after=queued_after,after=after)
|
||||
queued_before = queue_state()
|
||||
check(
|
||||
queued_before["messages_ready"] >= 1 and queued_before["durable"],
|
||||
"No durable queued message before restart",
|
||||
)
|
||||
result = subprocess.run(
|
||||
[str(ROOT / "scripts/restart.sh")],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
timeout=180,
|
||||
)
|
||||
check(result.returncode == 0, "Restart hook failed")
|
||||
check(
|
||||
"worker"
|
||||
not in require_compose(
|
||||
"ps", "--status", "running", "--services"
|
||||
).splitlines(),
|
||||
"Restart started deliberately stopped worker",
|
||||
)
|
||||
queued_after = queue_state()
|
||||
check(
|
||||
queued_after["messages_ready"] >= 1 and queued_after["durable"],
|
||||
"Broker lost queued delivery across restart",
|
||||
)
|
||||
client.message(actor, chat, "session and chat survived broker/API restart")
|
||||
finally:
|
||||
require_compose("start", "worker")
|
||||
after = verify_result(client, actor, attachment, original)
|
||||
return dict(
|
||||
scenario="worker_stopped_restart_all",
|
||||
attachment_id=attachment["id"],
|
||||
queue_before=queued_before,
|
||||
queue_after=queued_after,
|
||||
after=after,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url',required=True)
|
||||
parser.add_argument('--ca-file')
|
||||
parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all')
|
||||
parser.add_argument('--output',default='evidence/jobs.json')
|
||||
args=parser.parse_args()
|
||||
client=Client(args.base_url,args.ca_file,timeout=40)
|
||||
aliases={r.instance_id for r in ComposeProject().replicas() if r.running}
|
||||
check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL')
|
||||
actor,_,_,chat=client.bootstrap()
|
||||
results=[]
|
||||
operations={'persistence':persistence,'crash-after-object':lambda *x:crash(*x,'crash-after-object'),'crash-before-ack':lambda *x:crash(*x,'crash-before-ack'),'fail-transient':lambda *x:retry_replay(*x,False),'fail-permanent':lambda *x:retry_replay(*x,True)}
|
||||
selected=operations.values() if args.scenario=='all' else [operations[args.scenario]]
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--ca-file")
|
||||
parser.add_argument(
|
||||
"--scenario",
|
||||
choices=[
|
||||
"all",
|
||||
"persistence",
|
||||
"crash-after-object",
|
||||
"crash-before-ack",
|
||||
"fail-transient",
|
||||
"fail-permanent",
|
||||
],
|
||||
default="all",
|
||||
)
|
||||
parser.add_argument("--output", default="evidence/jobs.json")
|
||||
args = parser.parse_args()
|
||||
client = Client(args.base_url, args.ca_file, timeout=40)
|
||||
aliases = {r.instance_id for r in ComposeProject().replicas() if r.running}
|
||||
check(
|
||||
client.call("GET", "/health/live", schema="Health")["headers"].get(
|
||||
"X-Instance-Id"
|
||||
)
|
||||
in aliases,
|
||||
"Wrong Compose project for BASE_URL",
|
||||
)
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
results = []
|
||||
operations = {
|
||||
"persistence": persistence,
|
||||
"crash-after-object": lambda *x: crash(*x, "crash-after-object"),
|
||||
"crash-before-ack": lambda *x: crash(*x, "crash-before-ack"),
|
||||
"fail-transient": lambda *x: retry_replay(*x, False),
|
||||
"fail-permanent": lambda *x: retry_replay(*x, True),
|
||||
}
|
||||
selected = (
|
||||
operations.values() if args.scenario == "all" else [operations[args.scenario]]
|
||||
)
|
||||
for operation in selected:
|
||||
result=operation(client,actor,chat);results.append(result)
|
||||
Path(args.output).parent.mkdir(parents=True,exist_ok=True)
|
||||
Path(args.output).write_text(json.dumps(dict(status='in_progress',results=results),indent=2)+'\n')
|
||||
print('PASS: '+result['scenario'],flush=True)
|
||||
Path(args.output).write_text(json.dumps(dict(status='passed',results=results),indent=2)+'\n')
|
||||
result = operation(client, actor, chat)
|
||||
results.append(result)
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.output).write_text(
|
||||
json.dumps(dict(status="in_progress", results=results), indent=2) + "\n"
|
||||
)
|
||||
print("PASS: " + result["scenario"], flush=True)
|
||||
Path(args.output).write_text(
|
||||
json.dumps(dict(status="passed", results=results), indent=2) + "\n"
|
||||
)
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -19,7 +19,13 @@ COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(COMPOSE + list(arguments), cwd=ROOT, text=True, capture_output=True, timeout=timeout)
|
||||
return subprocess.run(
|
||||
COMPOSE + list(arguments),
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def require_compose(*arguments: str) -> str:
|
||||
@@ -29,54 +35,112 @@ def require_compose(*arguments: str) -> str:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Отказ publisher после COMMIT перед RabbitMQ publish")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Отказ publisher после COMMIT перед RabbitMQ publish"
|
||||
)
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--ca-file")
|
||||
args = parser.parse_args()
|
||||
running = set(require_compose("ps", "--status", "running", "--services").splitlines())
|
||||
check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the application stack before this scenario")
|
||||
running = set(
|
||||
require_compose("ps", "--status", "running", "--services").splitlines()
|
||||
)
|
||||
check(
|
||||
{"api", "postgres", "minio", "broker", "publisher", "worker"} <= running,
|
||||
"Start the application stack before this scenario",
|
||||
)
|
||||
client = Client(args.base_url, args.ca_file, timeout=40)
|
||||
project = ComposeProject()
|
||||
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
|
||||
live = client.call("GET", "/health/live", schema="Health")
|
||||
check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project")
|
||||
check(
|
||||
live["headers"].get("X-Instance-Id") in aliases,
|
||||
"BASE_URL does not belong to the selected Compose project",
|
||||
)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
attachment = None
|
||||
fault_container = "messenger-outbox-fault-" + uuid4().hex
|
||||
try:
|
||||
require_compose("stop", "publisher")
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
|
||||
client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected=409, actor=alice)
|
||||
killed = compose(
|
||||
"run", "--rm", "--no-deps", "--name", fault_container, "publisher", "python", "-m",
|
||||
"app.core.outbox_publisher", "--crash-before-publish", attachment["id"],
|
||||
message = client.message(alice, chat, "", attachments=[attachment["id"]])[
|
||||
"json"
|
||||
]
|
||||
client.call(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment['id']}/content",
|
||||
expected=409,
|
||||
actor=alice,
|
||||
)
|
||||
marker = json.dumps({"fault": "before_publish", "attachment_id": attachment["id"]})
|
||||
check(killed.returncode == 137 and marker in killed.stdout, "Expected targeted SIGKILL before publish was not observed")
|
||||
pending = client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=alice, schema="Attachment")["json"]
|
||||
killed = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"--name",
|
||||
fault_container,
|
||||
"publisher",
|
||||
"python",
|
||||
"-m",
|
||||
"app.core.outbox_publisher",
|
||||
"--crash-before-publish",
|
||||
attachment["id"],
|
||||
)
|
||||
marker = json.dumps(
|
||||
{"fault": "before_publish", "attachment_id": attachment["id"]}
|
||||
)
|
||||
check(
|
||||
killed.returncode == 137 and marker in killed.stdout,
|
||||
"Expected targeted SIGKILL before publish was not observed",
|
||||
)
|
||||
pending = client.call(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment['id']}",
|
||||
actor=alice,
|
||||
schema="Attachment",
|
||||
)["json"]
|
||||
check(pending["status"] == "queued", "Crash must leave accepted job queued")
|
||||
finally:
|
||||
# Если docker compose run оборвался по timeout, одноразовый контейнер тоже удаляется.
|
||||
try:
|
||||
subprocess.run(["docker", "rm", "-f", fault_container], capture_output=True, timeout=30)
|
||||
subprocess.run(
|
||||
["docker", "rm", "-f", fault_container], capture_output=True, timeout=30
|
||||
)
|
||||
finally:
|
||||
require_compose("start", "publisher")
|
||||
ready = client.wait_ready(alice, attachment, wait_seconds=60)
|
||||
check(ready["id"] == attachment["id"], "Recovered a different attachment")
|
||||
check({variant["name"] for variant in ready["variants"]} == {"thumbnail", "crop", "optimized"}, "Missing required variants")
|
||||
check(
|
||||
{variant["name"] for variant in ready["variants"]}
|
||||
== {"thumbnail", "crop", "optimized"},
|
||||
"Missing required variants",
|
||||
)
|
||||
path = f"/api/v1/attachments/{attachment['id']}/content"
|
||||
check(client.call("GET", path, actor=alice)["bytes"] == original, "Original changed after recovery")
|
||||
check(
|
||||
client.call("GET", path, actor=alice)["bytes"] == original,
|
||||
"Original changed after recovery",
|
||||
)
|
||||
for variant in ready["variants"]:
|
||||
downloaded = client.call("GET", path + "?variant=" + variant["name"], actor=alice)["bytes"]
|
||||
check(len(downloaded) == variant["size_bytes"] and hashlib.sha256(downloaded).hexdigest() == variant["sha256"], "Recovered variant checksum mismatch")
|
||||
print(json.dumps({
|
||||
"scenario": "commit_before_publish", "attachment_id": ready["id"],
|
||||
"message_id": message["id"], "publisher_exit_code": killed.returncode,
|
||||
"status_after_crash": pending["status"], "status_after_restart": ready["status"],
|
||||
"variants": [variant["name"] for variant in ready["variants"]],
|
||||
"reuploaded": False,
|
||||
}))
|
||||
downloaded = client.call(
|
||||
"GET", path + "?variant=" + variant["name"], actor=alice
|
||||
)["bytes"]
|
||||
check(
|
||||
len(downloaded) == variant["size_bytes"]
|
||||
and hashlib.sha256(downloaded).hexdigest() == variant["sha256"],
|
||||
"Recovered variant checksum mismatch",
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"scenario": "commit_before_publish",
|
||||
"attachment_id": ready["id"],
|
||||
"message_id": message["id"],
|
||||
"publisher_exit_code": killed.returncode,
|
||||
"status_after_crash": pending["status"],
|
||||
"status_after_restart": ready["status"],
|
||||
"variants": [variant["name"] for variant in ready["variants"]],
|
||||
"reuploaded": False,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+61
-30
@@ -7,7 +7,7 @@ import time
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, arguments, check, run
|
||||
|
||||
PROBE = r'''
|
||||
PROBE = r"""
|
||||
import asyncio, json, sys
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
@@ -62,60 +62,91 @@ async def main():
|
||||
await storage.aclose()
|
||||
await engine.dispose()
|
||||
asyncio.run(main())
|
||||
'''
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--with-outage', action='store_true', help='временно остановить MinIO выбранного проекта')
|
||||
args = arguments('MinIO private bucket and failure checks', configure)
|
||||
parser.add_argument(
|
||||
"--with-outage",
|
||||
action="store_true",
|
||||
help="временно остановить MinIO выбранного проекта",
|
||||
)
|
||||
|
||||
args = arguments("MinIO private bucket and failure checks", configure)
|
||||
project = ComposeProject()
|
||||
client = Client(args.base_url, args.ca_file, timeout=40)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
|
||||
live = client.call('GET', '/health/live', schema='Health')
|
||||
check(live['headers'].get('X-Instance-Id') in aliases, 'BASE_URL does not belong to the selected Compose project')
|
||||
live = client.call("GET", "/health/live", schema="Health")
|
||||
check(
|
||||
live["headers"].get("X-Instance-Id") in aliases,
|
||||
"BASE_URL does not belong to the selected Compose project",
|
||||
)
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
attachment = client.wait_ready(alice, attachment)
|
||||
# Повторный bootstrap не должен очистить bucket или сменить используемые credentials.
|
||||
for _ in range(2):
|
||||
project.run('run', '--rm', '--no-deps', 'minio-bootstrap', timeout=100)
|
||||
project.run("run", "--rm", "--no-deps", "minio-bootstrap", timeout=100)
|
||||
replicas = project.replicas()
|
||||
check(bool(replicas), 'API not running')
|
||||
result = subprocess.run(['docker', 'exec', '-i', replicas[0].id, 'python', '-c', PROBE],
|
||||
input=json.dumps({'attachment_id': attachment['id']}),
|
||||
text=True, capture_output=True, timeout=20)
|
||||
check(result.returncode == 0, 'private bucket probe failed (details hidden)')
|
||||
check(bool(replicas), "API not running")
|
||||
result = subprocess.run(
|
||||
["docker", "exec", "-i", replicas[0].id, "python", "-c", PROBE],
|
||||
input=json.dumps({"attachment_id": attachment["id"]}),
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=20,
|
||||
)
|
||||
check(result.returncode == 0, "private bucket probe failed (details hidden)")
|
||||
statuses = json.loads(result.stdout)
|
||||
check(statuses['GET'] == statuses['PUT'] == 403, 'anonymous S3 access allowed')
|
||||
check(statuses['admin_denied'] and statuses['other_buckets_denied'] and statuses['root_files_absent'], 'API has excessive access')
|
||||
check(statuses['console_unreachable'], 'console reachable from API network')
|
||||
check(statuses['temp_originals_absent'], 'API retained temporary originals')
|
||||
check(not project.config['services']['minio'].get('ports'), 'MinIO publishes host ports')
|
||||
env = project.config['services']['minio']['environment']
|
||||
check(env['MINIO_BROWSER'] == 'off', 'browser enabled')
|
||||
check('127.0.0.1:9001' in project.config['services']['minio']['command'], 'console is not bound to loopback')
|
||||
path = '/api/v1/attachments/' + attachment['id'] + '/content'
|
||||
check(client.call('GET', path, actor=alice)['bytes'] == original, 'bootstrap changed original')
|
||||
check(statuses["GET"] == statuses["PUT"] == 403, "anonymous S3 access allowed")
|
||||
check(
|
||||
statuses["admin_denied"]
|
||||
and statuses["other_buckets_denied"]
|
||||
and statuses["root_files_absent"],
|
||||
"API has excessive access",
|
||||
)
|
||||
check(statuses["console_unreachable"], "console reachable from API network")
|
||||
check(statuses["temp_originals_absent"], "API retained temporary originals")
|
||||
check(
|
||||
not project.config["services"]["minio"].get("ports"),
|
||||
"MinIO publishes host ports",
|
||||
)
|
||||
env = project.config["services"]["minio"]["environment"]
|
||||
check(env["MINIO_BROWSER"] == "off", "browser enabled")
|
||||
check(
|
||||
"127.0.0.1:9001" in project.config["services"]["minio"]["command"],
|
||||
"console is not bound to loopback",
|
||||
)
|
||||
path = "/api/v1/attachments/" + attachment["id"] + "/content"
|
||||
check(
|
||||
client.call("GET", path, actor=alice)["bytes"] == original,
|
||||
"bootstrap changed original",
|
||||
)
|
||||
if args.with_outage:
|
||||
try:
|
||||
project.run('stop', 'minio')
|
||||
client.call('GET', path, expected=503, actor=alice)
|
||||
project.run("stop", "minio")
|
||||
client.call("GET", path, expected=503, actor=alice)
|
||||
finally:
|
||||
project.run('start', 'minio')
|
||||
project.run("start", "minio")
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
client.call('GET', '/health/ready', schema='Health')
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except Exception:
|
||||
if time.monotonic() >= deadline:
|
||||
raise
|
||||
time.sleep(0.5)
|
||||
check(client.call('GET', path, actor=alice)['bytes'] == original, 'original lost after MinIO restart')
|
||||
print('PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console'
|
||||
+ (', MinIO outage/recovery' if args.with_outage else ''))
|
||||
check(
|
||||
client.call("GET", path, actor=alice)["bytes"] == original,
|
||||
"original lost after MinIO restart",
|
||||
)
|
||||
print(
|
||||
"PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console"
|
||||
+ (", MinIO outage/recovery" if args.with_outage else "")
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
|
||||
+17
-4
@@ -31,8 +31,12 @@ class ComposeProject:
|
||||
def docker(self, *args: str, timeout: int = 90, env=None) -> str:
|
||||
"""Выполняет Docker-команду с ограничением времени."""
|
||||
result = subprocess.run(
|
||||
["docker", *args], cwd=ROOT, text=True, capture_output=True,
|
||||
timeout=timeout, env={**os.environ, **(env or {})},
|
||||
["docker", *args],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
env={**os.environ, **(env or {})},
|
||||
)
|
||||
if result.returncode:
|
||||
raise RuntimeError(result.stderr.strip() or result.stdout.strip())
|
||||
@@ -53,8 +57,17 @@ class ComposeProject:
|
||||
'"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}}'
|
||||
)
|
||||
rows = self.docker("inspect", "--format", template, *ids).splitlines()
|
||||
configured = str(self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID") or "").strip()
|
||||
configured = str(
|
||||
self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID")
|
||||
or ""
|
||||
).strip()
|
||||
return [
|
||||
Replica(row["id"], row["image"], configured or row["hostname"], row["running"], row["health"])
|
||||
Replica(
|
||||
row["id"],
|
||||
row["image"],
|
||||
configured or row["hostname"],
|
||||
row["running"],
|
||||
row["health"],
|
||||
)
|
||||
for row in map(json.loads, rows)
|
||||
]
|
||||
|
||||
@@ -14,7 +14,15 @@ def main() -> None:
|
||||
destination.chmod(0o600)
|
||||
directory = root / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
for name in ("db_password", "redis_password", "broker_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"):
|
||||
for name in (
|
||||
"db_password",
|
||||
"redis_password",
|
||||
"broker_password",
|
||||
"minio_root_user",
|
||||
"minio_root_password",
|
||||
"s3_access_key_id",
|
||||
"s3_secret_access_key",
|
||||
):
|
||||
path = directory / name
|
||||
try:
|
||||
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444)
|
||||
|
||||
@@ -1,23 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Full OpenAPI and JSON Schema checks; install requirements-dev.txt first."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from openapi_spec_validator import validate
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
spec = json.loads((root/'contracts/openapi.json').read_text())
|
||||
spec = json.loads((root / "contracts/openapi.json").read_text())
|
||||
validate(spec)
|
||||
from referencing import Registry, Resource
|
||||
|
||||
# The base URI resolves local #/components references inside examples.
|
||||
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
|
||||
registry = Registry().with_resource('urn:messenger:openapi', Resource.from_contents(resource))
|
||||
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
|
||||
schema = {'$ref': 'urn:messenger:openapi#/components/schemas/' + name}
|
||||
resource = dict(spec, **{"$schema": "https://json-schema.org/draft/2020-12/schema"})
|
||||
registry = Registry().with_resource(
|
||||
"urn:messenger:openapi", Resource.from_contents(resource)
|
||||
)
|
||||
for name, example in json.loads((root / "contracts/examples.json").read_text()).items():
|
||||
schema = {"$ref": "urn:messenger:openapi#/components/schemas/" + name}
|
||||
Draft202012Validator(schema, registry=registry).validate(example)
|
||||
for path in (root/'contracts').glob('*.schema.json'):
|
||||
for path in (root / "contracts").glob("*.schema.json"):
|
||||
schema = json.loads(path.read_text())
|
||||
Draft202012Validator.check_schema(schema)
|
||||
for example in schema.get('examples',[]):
|
||||
for example in schema.get("examples", []):
|
||||
Draft202012Validator(schema).validate(example)
|
||||
print('PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.')
|
||||
print("PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.")
|
||||
|
||||
@@ -28,7 +28,7 @@ from app.storage import attachments as repository
|
||||
from app.storage.s3 import ObjectInfo
|
||||
from helpers import bearer, create_chat, create_user, idempotency_header
|
||||
|
||||
PNG = Path('tests/fixtures/sample.png').read_bytes()
|
||||
PNG = Path("tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
class MemoryStorage:
|
||||
@@ -55,12 +55,15 @@ class MemoryStorage:
|
||||
info = None
|
||||
if key in self.objects:
|
||||
data, metadata = self.objects[key]
|
||||
info = ObjectInfo(key, len(data), metadata['content_type'], datetime.now(timezone.utc))
|
||||
info = ObjectInfo(
|
||||
key, len(data), metadata["content_type"], datetime.now(timezone.utc)
|
||||
)
|
||||
return info
|
||||
|
||||
async def open_object(self, key):
|
||||
async def chunks():
|
||||
yield self.objects[key][0]
|
||||
|
||||
return chunks()
|
||||
|
||||
async def check_available(self):
|
||||
@@ -76,23 +79,40 @@ async def context(client, app):
|
||||
return storage, alice, bob, eve, chat
|
||||
|
||||
|
||||
async def upload(client, actor, chat, data=PNG, filename='фото.png', content_type='image/png'):
|
||||
async def upload(
|
||||
client, actor, chat, data=PNG, filename="фото.png", content_type="image/png"
|
||||
):
|
||||
return await client.post(
|
||||
f"/api/v1/chats/{chat['id']}/attachments", params={'filename': filename},
|
||||
headers={**bearer(actor), 'Content-Type': content_type}, content=data,
|
||||
f"/api/v1/chats/{chat['id']}/attachments",
|
||||
params={"filename": filename},
|
||||
headers={**bearer(actor), "Content-Type": content_type},
|
||||
content=data,
|
||||
)
|
||||
|
||||
|
||||
async def finish_attachment(client, storage, actor, attachment_id):
|
||||
engine = client._transport.app.state.engine
|
||||
async with AsyncSession(engine) as session:
|
||||
job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)))
|
||||
message = JobMessage(job_id=job.id, attachment_id=job.attachment_id,
|
||||
pipeline_version=job.pipeline_version, attempt=job.attempt + 1, created_at=job.created_at)
|
||||
job = await session.scalar(
|
||||
select(AttachmentJob).where(
|
||||
AttachmentJob.attachment_id == UUID(attachment_id)
|
||||
)
|
||||
)
|
||||
message = JobMessage(
|
||||
job_id=job.id,
|
||||
attachment_id=job.attachment_id,
|
||||
pipeline_version=job.pipeline_version,
|
||||
attempt=job.attempt + 1,
|
||||
created_at=job.created_at,
|
||||
)
|
||||
await session.rollback()
|
||||
assert await process_attachment_job(session, storage, message, JobSettings(_env_file=None))
|
||||
result = await client.get(f'/api/v1/attachments/{attachment_id}', headers=bearer(actor))
|
||||
assert result.json()['status'] == 'ready'
|
||||
assert await process_attachment_job(
|
||||
session, storage, message, JobSettings(_env_file=None)
|
||||
)
|
||||
result = await client.get(
|
||||
f"/api/v1/attachments/{attachment_id}", headers=bearer(actor)
|
||||
)
|
||||
assert result.json()["status"] == "ready"
|
||||
return result.json()
|
||||
|
||||
|
||||
@@ -101,37 +121,52 @@ async def test_upload_download_and_attachment_only_message(client, context):
|
||||
response = await upload(client, alice, chat)
|
||||
assert response.status_code == 202, response.text
|
||||
attachment = response.json()
|
||||
assert attachment['status'] == 'queued'
|
||||
before = await client.get(f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice))
|
||||
assert attachment["status"] == "queued"
|
||||
before = await client.get(
|
||||
f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice)
|
||||
)
|
||||
assert before.status_code == 409
|
||||
attachment = await finish_attachment(client, storage, alice, attachment['id'])
|
||||
assert attachment['sha256'] == hashlib.sha256(PNG).hexdigest()
|
||||
assert {v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'}
|
||||
assert attachment['error_code'] is None
|
||||
assert 'object_key' not in attachment
|
||||
id = attachment['id']
|
||||
attachment = await finish_attachment(client, storage, alice, attachment["id"])
|
||||
assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest()
|
||||
assert {v["name"] for v in attachment["variants"]} == {
|
||||
"thumbnail",
|
||||
"crop",
|
||||
"optimized",
|
||||
}
|
||||
assert attachment["error_code"] is None
|
||||
assert "object_key" not in attachment
|
||||
id = attachment["id"]
|
||||
for actor in (alice, bob):
|
||||
content = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(actor))
|
||||
content = await client.get(
|
||||
f"/api/v1/attachments/{id}/content", headers=bearer(actor)
|
||||
)
|
||||
assert content.status_code == 200 and content.content == PNG
|
||||
assert content.headers['content-type'] == 'image/png'
|
||||
assert content.headers["content-type"] == "image/png"
|
||||
key = idempotency_header()
|
||||
body = {'text': '', 'attachment_ids': [id]}
|
||||
body = {"text": "", "attachment_ids": [id]}
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
message = await client.post(path, json=body, headers={**bearer(bob), **key})
|
||||
assert message.status_code == 201, message.text
|
||||
repeat = await client.post(path, json=body, headers={**bearer(bob), **key})
|
||||
assert repeat.json() == message.json()
|
||||
history = await client.get(path, headers=bearer(alice))
|
||||
assert history.json()['items'][0]['attachment_ids'] == [id]
|
||||
for suffix in ('', '/content'):
|
||||
denied = await client.get(f'/api/v1/attachments/{id}{suffix}', headers=bearer(eve))
|
||||
assert history.json()["items"][0]["attachment_ids"] == [id]
|
||||
for suffix in ("", "/content"):
|
||||
denied = await client.get(
|
||||
f"/api/v1/attachments/{id}{suffix}", headers=bearer(eve)
|
||||
)
|
||||
assert denied.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'data,content_type',
|
||||
[(b'not an image', 'image/png'), (PNG[:30], 'image/png'), (PNG, 'image/jpeg'), (PNG, 'application/octet-stream')],
|
||||
ids=['not-image', 'truncated-png', 'mime-spoofing', 'unsupported-type'],
|
||||
"data,content_type",
|
||||
[
|
||||
(b"not an image", "image/png"),
|
||||
(PNG[:30], "image/png"),
|
||||
(PNG, "image/jpeg"),
|
||||
(PNG, "application/octet-stream"),
|
||||
],
|
||||
ids=["not-image", "truncated-png", "mime-spoofing", "unsupported-type"],
|
||||
)
|
||||
async def test_invalid_image_and_mime_are_415(client, context, data, content_type):
|
||||
storage, alice, _, _, chat = context
|
||||
@@ -143,17 +178,21 @@ async def test_invalid_image_and_mime_are_415(client, context, data, content_typ
|
||||
async def test_jpeg_is_stored_without_reencoding(client, context):
|
||||
storage, alice, _, _, chat = context
|
||||
output = BytesIO()
|
||||
Image.new('RGB', (32, 32), 'blue').save(output, format='JPEG')
|
||||
Image.new("RGB", (32, 32), "blue").save(output, format="JPEG")
|
||||
original = output.getvalue()
|
||||
response = await upload(client, alice, chat, original, content_type='image/jpeg')
|
||||
response = await upload(client, alice, chat, original, content_type="image/jpeg")
|
||||
assert response.status_code == 202, response.text
|
||||
id = response.json()['id']
|
||||
id = response.json()["id"]
|
||||
await finish_attachment(client, storage, alice, id)
|
||||
downloaded = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice))
|
||||
downloaded = await client.get(
|
||||
f"/api/v1/attachments/{id}/content", headers=bearer(alice)
|
||||
)
|
||||
assert downloaded.content == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize('filename', ['../a.png', 'a\\b.png', 'a\x00.png', 'a\n.png', 'x' * 256])
|
||||
@pytest.mark.parametrize(
|
||||
"filename", ["../a.png", "a\\b.png", "a\x00.png", "a\n.png", "x" * 256]
|
||||
)
|
||||
async def test_unsafe_filename_is_400(client, context, filename):
|
||||
storage, alice, _, _, chat = context
|
||||
response = await upload(client, alice, chat, filename=filename)
|
||||
@@ -163,9 +202,11 @@ async def test_unsafe_filename_is_400(client, context, filename):
|
||||
|
||||
async def test_oversize_chunked_upload_is_413(client, context):
|
||||
storage, alice, _, _, chat = context
|
||||
|
||||
async def chunks():
|
||||
for _ in range(11):
|
||||
yield b'x' * 1048576
|
||||
yield b"x" * 1048576
|
||||
|
||||
response = await upload(client, alice, chat, chunks())
|
||||
assert response.status_code == 413
|
||||
assert not storage.objects
|
||||
@@ -175,7 +216,12 @@ async def test_oversize_content_length_is_413(client, context):
|
||||
_, alice, _, _, chat = context
|
||||
response = await client.post(
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename=a.png",
|
||||
headers={**bearer(alice), 'Content-Type': 'image/png', 'Content-Length': '10485761'}, content=PNG,
|
||||
headers={
|
||||
**bearer(alice),
|
||||
"Content-Type": "image/png",
|
||||
"Content-Length": "10485761",
|
||||
},
|
||||
content=PNG,
|
||||
)
|
||||
assert response.status_code == 413
|
||||
|
||||
@@ -185,30 +231,47 @@ async def test_exact_byte_limit_accepts_valid_png(client, context):
|
||||
maximum = 10_485_760
|
||||
# Валидный ancillary chunk вместо данных после IEND.
|
||||
payload = bytes(maximum - len(PNG) - 12)
|
||||
kind = b'npAD'
|
||||
padding = struct.pack('!I', len(payload)) + kind + payload + struct.pack('!I', zlib.crc32(kind + payload))
|
||||
kind = b"npAD"
|
||||
padding = (
|
||||
struct.pack("!I", len(payload))
|
||||
+ kind
|
||||
+ payload
|
||||
+ struct.pack("!I", zlib.crc32(kind + payload))
|
||||
)
|
||||
image = PNG[:-12] + padding + PNG[-12:]
|
||||
response = await upload(client, alice, chat, image)
|
||||
assert response.status_code == 202, response.text
|
||||
assert response.json()['size_bytes'] == maximum
|
||||
assert response.json()['sha256'] == hashlib.sha256(image).hexdigest()
|
||||
assert response.json()["size_bytes"] == maximum
|
||||
assert response.json()["sha256"] == hashlib.sha256(image).hexdigest()
|
||||
|
||||
|
||||
async def test_exact_pixel_limit_accepts_valid_png(client, context):
|
||||
_, alice, _, _, chat = context
|
||||
output = BytesIO()
|
||||
with Image.new('RGB', (8000, 5000), 'blue') as image:
|
||||
image.save(output, format='PNG')
|
||||
with Image.new("RGB", (8000, 5000), "blue") as image:
|
||||
image.save(output, format="PNG")
|
||||
response = await upload(client, alice, chat, output.getvalue())
|
||||
assert response.status_code == 202, response.text
|
||||
|
||||
|
||||
async def test_pixel_limit_before_decoding(client, context):
|
||||
_, alice, _, _, chat = context
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack('!I', len(data)) + kind + data + struct.pack('!I', zlib.crc32(kind + data))
|
||||
header = struct.pack('!IIBBBBB', 8000, 5001, 8, 2, 0, 0, 0)
|
||||
image = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(b'\0')) + chunk(b'IEND', b'')
|
||||
return (
|
||||
struct.pack("!I", len(data))
|
||||
+ kind
|
||||
+ data
|
||||
+ struct.pack("!I", zlib.crc32(kind + data))
|
||||
)
|
||||
|
||||
header = struct.pack("!IIBBBBB", 8000, 5001, 8, 2, 0, 0, 0)
|
||||
image = (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", header)
|
||||
+ chunk(b"IDAT", zlib.compress(b"\0"))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
response = await upload(client, alice, chat, image)
|
||||
assert response.status_code == 413, response.text
|
||||
|
||||
@@ -217,10 +280,10 @@ async def test_unknown_and_other_chat_attachment_are_404(client, context):
|
||||
_, alice, bob, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
other_chat = await create_chat(client, alice, bob)
|
||||
for id in (attachment['id'], str(uuid4())):
|
||||
for id in (attachment["id"], str(uuid4())):
|
||||
response = await client.post(
|
||||
f"/api/v1/chats/{other_chat['id']}/messages",
|
||||
json={'text': '', 'attachment_ids': [id]},
|
||||
json={"text": "", "attachment_ids": [id]},
|
||||
headers={**bearer(alice), **idempotency_header()},
|
||||
)
|
||||
assert response.status_code == 404
|
||||
@@ -228,10 +291,12 @@ async def test_unknown_and_other_chat_attachment_are_404(client, context):
|
||||
|
||||
async def test_duplicate_ids_are_400(client, context):
|
||||
_, alice, _, _, chat = context
|
||||
id = (await upload(client, alice, chat)).json()['id']
|
||||
response = await client.post(f"/api/v1/chats/{chat['id']}/messages",
|
||||
json={'text': '', 'attachment_ids': [id, id]},
|
||||
headers={**bearer(alice), **idempotency_header()})
|
||||
id = (await upload(client, alice, chat)).json()["id"]
|
||||
response = await client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
json={"text": "", "attachment_ids": [id, id]},
|
||||
headers={**bearer(alice), **idempotency_header()},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@@ -239,20 +304,24 @@ async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context):
|
||||
_, alice, _, _, chat = context
|
||||
headers = {**bearer(alice), **idempotency_header()}
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
first = await client.post(path, json={'text': 'same'}, headers=headers)
|
||||
second = await client.post(path, json={'text': 'same', 'attachment_ids': []}, headers=headers)
|
||||
first = await client.post(path, json={"text": "same"}, headers=headers)
|
||||
second = await client.post(
|
||||
path, json={"text": "same", "attachment_ids": []}, headers=headers
|
||||
)
|
||||
assert first.status_code == second.status_code == 201
|
||||
assert first.json() == second.json()
|
||||
|
||||
|
||||
async def test_s3_outage_is_503(client, context):
|
||||
storage, alice, _, _, chat = context
|
||||
id = (await upload(client, alice, chat)).json()['id']
|
||||
id = (await upload(client, alice, chat)).json()["id"]
|
||||
await finish_attachment(client, storage, alice, id)
|
||||
storage.down = True
|
||||
response = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice))
|
||||
response = await client.get(
|
||||
f"/api/v1/attachments/{id}/content", headers=bearer(alice)
|
||||
)
|
||||
assert response.status_code == 503
|
||||
assert (await client.get('/health/ready')).status_code == 503
|
||||
assert (await client.get("/health/ready")).status_code == 503
|
||||
|
||||
|
||||
async def test_uncertain_put_is_compensated(client, context, db_engine):
|
||||
@@ -267,25 +336,31 @@ async def test_uncertain_put_is_compensated(client, context, db_engine):
|
||||
|
||||
async def test_failure_after_s3_write_compensates(client, context, monkeypatch):
|
||||
storage, alice, _, _, chat = context
|
||||
|
||||
async def fail(*args, **kwargs):
|
||||
raise OperationalError('injected', {}, None)
|
||||
monkeypatch.setattr(repository, 'create_attachment', fail)
|
||||
raise OperationalError("injected", {}, None)
|
||||
|
||||
monkeypatch.setattr(repository, "create_attachment", fail)
|
||||
response = await upload(client, alice, chat)
|
||||
assert response.status_code == 503
|
||||
assert not storage.objects
|
||||
|
||||
|
||||
async def test_ambiguous_successful_commit_keeps_original(client, context, db_engine, monkeypatch):
|
||||
async def test_ambiguous_successful_commit_keeps_original(
|
||||
client, context, db_engine, monkeypatch
|
||||
):
|
||||
storage, alice, _, _, chat = context
|
||||
original = AsyncSession.commit
|
||||
count = 0
|
||||
|
||||
async def uncertain(session):
|
||||
nonlocal count
|
||||
count += 1
|
||||
await original(session)
|
||||
if count == 2:
|
||||
raise OperationalError('lost commit acknowledgement', {}, None)
|
||||
monkeypatch.setattr(AsyncSession, 'commit', uncertain)
|
||||
raise OperationalError("lost commit acknowledgement", {}, None)
|
||||
|
||||
monkeypatch.setattr(AsyncSession, "commit", uncertain)
|
||||
response = await upload(client, alice, chat)
|
||||
assert response.status_code == 503
|
||||
assert len(storage.objects) == 1
|
||||
@@ -318,10 +393,16 @@ async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_
|
||||
storage, _, _, _, _ = context
|
||||
old_id, fresh_id = uuid4(), uuid4()
|
||||
async with AsyncSession(db_engine) as session:
|
||||
session.add_all([
|
||||
UploadIntent(id=old_id, object_key='originals/old', created_at=datetime.now(timezone.utc) - timedelta(days=2)),
|
||||
UploadIntent(id=fresh_id, object_key='originals/fresh'),
|
||||
])
|
||||
session.add_all(
|
||||
[
|
||||
UploadIntent(
|
||||
id=old_id,
|
||||
object_key="originals/old",
|
||||
created_at=datetime.now(timezone.utc) - timedelta(days=2),
|
||||
),
|
||||
UploadIntent(id=fresh_id, object_key="originals/fresh"),
|
||||
]
|
||||
)
|
||||
await session.commit()
|
||||
async with AsyncSession(db_engine) as uploader, AsyncSession(db_engine) as cleaner:
|
||||
await repository.lock_upload(uploader, old_id)
|
||||
@@ -335,9 +416,11 @@ async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_
|
||||
|
||||
async def test_client_disconnect_before_put_has_no_side_effects(client, context):
|
||||
storage, alice, _, _, chat = context
|
||||
|
||||
async def interrupted():
|
||||
yield PNG[:16]
|
||||
raise ClientDisconnect()
|
||||
|
||||
response = await upload(client, alice, chat, interrupted())
|
||||
assert response.status_code == 400
|
||||
assert not storage.objects
|
||||
@@ -345,10 +428,14 @@ async def test_client_disconnect_before_put_has_no_side_effects(client, context)
|
||||
|
||||
async def test_upload_deadline_cancels_validation(client, context, monkeypatch):
|
||||
storage, alice, _, _, chat = context
|
||||
storage.settings = storage.settings.model_copy(update={'upload_timeout_seconds': 0.02})
|
||||
storage.settings = storage.settings.model_copy(
|
||||
update={"upload_timeout_seconds": 0.02}
|
||||
)
|
||||
|
||||
async def stalled(*args):
|
||||
await asyncio.sleep(10)
|
||||
monkeypatch.setattr(domain, 'validate_image', stalled)
|
||||
|
||||
monkeypatch.setattr(domain, "validate_image", stalled)
|
||||
response = await upload(client, alice, chat)
|
||||
assert response.status_code == 400
|
||||
assert not storage.objects
|
||||
@@ -358,11 +445,13 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa
|
||||
storage, alice, _, _, chat = context
|
||||
entered = asyncio.Event()
|
||||
original = storage.put_file
|
||||
|
||||
async def interrupted(*args, **kwargs):
|
||||
await original(*args, **kwargs)
|
||||
entered.set()
|
||||
await asyncio.sleep(10)
|
||||
monkeypatch.setattr(storage, 'put_file', interrupted)
|
||||
|
||||
monkeypatch.setattr(storage, "put_file", interrupted)
|
||||
task = asyncio.create_task(upload(client, alice, chat))
|
||||
await asyncio.wait_for(entered.wait(), timeout=10)
|
||||
task.cancel()
|
||||
@@ -375,14 +464,19 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa
|
||||
|
||||
async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine):
|
||||
storage, alice, _, _, chat = context
|
||||
id = (await upload(client, alice, chat)).json()['id']
|
||||
id = (await upload(client, alice, chat)).json()["id"]
|
||||
await finish_attachment(client, storage, alice, id)
|
||||
async with AsyncSession(db_engine) as session:
|
||||
attachment = await session.get(Attachment, UUID(id))
|
||||
key = attachment.object_key
|
||||
# Искусственно оставляем старый журнал у уже подтверждённого объекта.
|
||||
session.add(UploadIntent(id=uuid4(), object_key=key,
|
||||
created_at=datetime.now(timezone.utc) - timedelta(days=2)))
|
||||
session.add(
|
||||
UploadIntent(
|
||||
id=uuid4(),
|
||||
object_key=key,
|
||||
created_at=datetime.now(timezone.utc) - timedelta(days=2),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
result = await domain.reconcile_attachments(session, storage, dry_run=False)
|
||||
assert result.examined == 1 and result.deleted == 0
|
||||
|
||||
@@ -8,7 +8,14 @@ from sqlalchemy import text
|
||||
|
||||
from app.main import create_app
|
||||
from conftest import close_app, make_client
|
||||
from helpers import PASSWORD, basic_header, bearer, create_user, idempotency_header, register
|
||||
from helpers import (
|
||||
PASSWORD,
|
||||
basic_header,
|
||||
bearer,
|
||||
create_user,
|
||||
idempotency_header,
|
||||
register,
|
||||
)
|
||||
|
||||
|
||||
async def test_parallel_registration_of_same_username_creates_exactly_one_user(
|
||||
|
||||
@@ -29,14 +29,12 @@ async def seed_messages(
|
||||
) -> None:
|
||||
async with engine.begin() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
text("""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), :chat_id, :sender_id, 'message ' || g,
|
||||
now() - make_interval(secs => :count - g)
|
||||
FROM generate_series(1, :count) AS g
|
||||
"""
|
||||
),
|
||||
"""),
|
||||
{"chat_id": chat_id, "sender_id": sender_id, "count": count},
|
||||
)
|
||||
await connection.execute(text("ANALYZE messages"))
|
||||
|
||||
@@ -16,10 +16,15 @@ from helpers import bearer, create_chat, create_user, idempotency_header, send_m
|
||||
@pytest.fixture
|
||||
async def replicas(request, make_settings):
|
||||
overrides = getattr(request, "param", {})
|
||||
apps = [create_app(make_settings(instance_id=name, **overrides)) for name in ("api-a", "api-b")]
|
||||
apps = [
|
||||
create_app(make_settings(instance_id=name, **overrides))
|
||||
for name in ("api-a", "api-b")
|
||||
]
|
||||
try:
|
||||
async with AsyncExitStack() as stack:
|
||||
clients = [await stack.enter_async_context(make_client(app)) for app in apps]
|
||||
clients = [
|
||||
await stack.enter_async_context(make_client(app)) for app in apps
|
||||
]
|
||||
yield clients
|
||||
finally:
|
||||
for app in apps:
|
||||
@@ -54,7 +59,11 @@ async def test_replay_and_conflict_across_replicas(replicas, db_engine):
|
||||
original = await send_message(first, user, chat, "Привет", "same-key")
|
||||
repeated = await second.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
headers={**bearer(user), **idempotency_header("same-key"), "Content-Type": "application/json"},
|
||||
headers={
|
||||
**bearer(user),
|
||||
**idempotency_header("same-key"),
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
content='{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442" }',
|
||||
)
|
||||
conflict = await send_message(second, user, chat, "другой текст", "same-key")
|
||||
@@ -63,23 +72,38 @@ async def test_replay_and_conflict_across_replicas(replicas, db_engine):
|
||||
assert original.headers["X-Instance-Id"] != repeated.headers["X-Instance-Id"]
|
||||
assert conflict.status_code == 409
|
||||
async with db_engine.connect() as connection:
|
||||
assert (await connection.execute(text("SELECT count(*) FROM messages"))).scalar_one() == 1
|
||||
assert (await connection.execute(text("SELECT value FROM counters WHERE name='messages'"))).scalar_one() == 1
|
||||
assert (
|
||||
await connection.execute(text("SELECT count(*) FROM messages"))
|
||||
).scalar_one() == 1
|
||||
assert (
|
||||
await connection.execute(
|
||||
text("SELECT value FROM counters WHERE name='messages'")
|
||||
)
|
||||
).scalar_one() == 1
|
||||
|
||||
|
||||
async def test_parallel_replays_create_one_message(replicas, db_engine):
|
||||
user = await create_user(replicas[0])
|
||||
chat = await create_chat(replicas[0], user)
|
||||
responses = await asyncio.gather(*(
|
||||
send_message(replicas[index % 2], user, chat, "parallel", "one-key")
|
||||
for index in range(24)
|
||||
))
|
||||
responses = await asyncio.gather(
|
||||
*(
|
||||
send_message(replicas[index % 2], user, chat, "parallel", "one-key")
|
||||
for index in range(24)
|
||||
)
|
||||
)
|
||||
assert all(response.status_code == 201 for response in responses)
|
||||
assert len({response.json()["id"] for response in responses}) == 1
|
||||
assert {response.headers["X-Instance-Id"] for response in responses} == {"api-a", "api-b"}
|
||||
assert {response.headers["X-Instance-Id"] for response in responses} == {
|
||||
"api-a",
|
||||
"api-b",
|
||||
}
|
||||
async with db_engine.connect() as connection:
|
||||
assert (await connection.execute(text("SELECT count(*) FROM messages"))).scalar_one() == 1
|
||||
assert (await connection.execute(text("SELECT count(*) FROM idempotency_keys"))).scalar_one() == 1
|
||||
assert (
|
||||
await connection.execute(text("SELECT count(*) FROM messages"))
|
||||
).scalar_one() == 1
|
||||
assert (
|
||||
await connection.execute(text("SELECT count(*) FROM idempotency_keys"))
|
||||
).scalar_one() == 1
|
||||
|
||||
|
||||
async def test_key_scope_separates_users_and_chats(client):
|
||||
@@ -101,9 +125,11 @@ async def test_expired_key_can_be_reused_without_removing_message(client, db_eng
|
||||
chat = await create_chat(client, user)
|
||||
first = await send_message(client, user, chat, "old", "expiring-key")
|
||||
async with db_engine.begin() as connection:
|
||||
await connection.execute(text(
|
||||
"UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'"
|
||||
))
|
||||
await connection.execute(
|
||||
text(
|
||||
"UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'"
|
||||
)
|
||||
)
|
||||
second = await send_message(client, user, chat, "new", "expiring-key")
|
||||
assert first.status_code == second.status_code == 201
|
||||
assert first.json()["id"] != second.json()["id"]
|
||||
@@ -127,7 +153,9 @@ async def test_failed_transaction_does_not_poison_key(client, db_engine, monkeyp
|
||||
assert failed.status_code == 503
|
||||
async with db_engine.connect() as connection:
|
||||
for table in ("messages", "idempotency_keys", "counters"):
|
||||
assert (await connection.execute(text(f"SELECT count(*) FROM {table}"))).scalar_one() == 0
|
||||
assert (
|
||||
await connection.execute(text(f"SELECT count(*) FROM {table}"))
|
||||
).scalar_one() == 0
|
||||
monkeypatch.setattr(key_storage, "save_result", original)
|
||||
recovered = await send_message(client, user, chat, "retry", "retry-key")
|
||||
assert recovered.status_code == 201
|
||||
@@ -148,9 +176,15 @@ async def test_replay_works_when_message_limit_is_full(replicas):
|
||||
async def test_replay_checks_session_and_membership(client, db_engine):
|
||||
user = await create_user(client)
|
||||
chat = await create_chat(client, user)
|
||||
assert (await send_message(client, user, chat, "private", "private-key")).status_code == 201
|
||||
assert (
|
||||
await send_message(client, user, chat, "private", "private-key")
|
||||
).status_code == 201
|
||||
async with db_engine.begin() as connection:
|
||||
await connection.execute(text("DELETE FROM chat_members"))
|
||||
assert (await send_message(client, user, chat, "private", "private-key")).status_code == 404
|
||||
assert (
|
||||
await send_message(client, user, chat, "private", "private-key")
|
||||
).status_code == 404
|
||||
await client.delete("/api/v1/auth/sessions/current", headers=bearer(user))
|
||||
assert (await send_message(client, user, chat, "private", "private-key")).status_code == 401
|
||||
assert (
|
||||
await send_message(client, user, chat, "private", "private-key")
|
||||
).status_code == 401
|
||||
|
||||
@@ -18,54 +18,96 @@ from app.storage.jobs import JobWork
|
||||
from test_attachments import MemoryStorage
|
||||
|
||||
|
||||
@pytest.mark.parametrize('size,transparent', [((1600,800),False),((24,12),True)])
|
||||
async def test_variant_geometry_metadata_and_alpha(tmp_path,size,transparent):
|
||||
source=tmp_path/'source.png'
|
||||
with Image.new('RGBA',size,(255,0,0,0 if transparent else 255)) as image:
|
||||
exif=Image.Exif();exif[270]='private description'
|
||||
image.save(source,format='PNG',exif=exif)
|
||||
result=create_variants(source,tmp_path,'image/png',40_000_000,ImageSettings(_env_file=None))
|
||||
variants={v.name:v for v in result.variants}
|
||||
assert (variants['crop'].width,variants['crop'].height)==(128,128)
|
||||
assert (variants['thumbnail'].width,variants['thumbnail'].height)==((256,128) if size[0]>256 else size)
|
||||
assert (variants['optimized'].width,variants['optimized'].height)==((1280,640) if size[0]>1280 else size)
|
||||
for name in ('thumbnail.jpg','crop.jpg','optimized.webp'):
|
||||
with Image.open(tmp_path/name) as image:
|
||||
assert not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp')
|
||||
@pytest.mark.parametrize("size,transparent", [((1600, 800), False), ((24, 12), True)])
|
||||
async def test_variant_geometry_metadata_and_alpha(tmp_path, size, transparent):
|
||||
source = tmp_path / "source.png"
|
||||
with Image.new("RGBA", size, (255, 0, 0, 0 if transparent else 255)) as image:
|
||||
exif = Image.Exif()
|
||||
exif[270] = "private description"
|
||||
image.save(source, format="PNG", exif=exif)
|
||||
result = create_variants(
|
||||
source, tmp_path, "image/png", 40_000_000, ImageSettings(_env_file=None)
|
||||
)
|
||||
variants = {v.name: v for v in result.variants}
|
||||
assert (variants["crop"].width, variants["crop"].height) == (128, 128)
|
||||
assert (variants["thumbnail"].width, variants["thumbnail"].height) == (
|
||||
(256, 128) if size[0] > 256 else size
|
||||
)
|
||||
assert (variants["optimized"].width, variants["optimized"].height) == (
|
||||
(1280, 640) if size[0] > 1280 else size
|
||||
)
|
||||
for name in ("thumbnail.jpg", "crop.jpg", "optimized.webp"):
|
||||
with Image.open(tmp_path / name) as image:
|
||||
assert (
|
||||
not image.getexif()
|
||||
and not image.info.get("icc_profile")
|
||||
and not image.info.get("xmp")
|
||||
)
|
||||
if transparent:
|
||||
if name.endswith('webp'):assert image.getpixel((0,0))[3]==0
|
||||
else:assert min(image.getpixel((0,0)))>=250
|
||||
if name.endswith("webp"):
|
||||
assert image.getpixel((0, 0))[3] == 0
|
||||
else:
|
||||
assert min(image.getpixel((0, 0))) >= 250
|
||||
|
||||
|
||||
async def test_exif_orientation_applied(tmp_path):
|
||||
source=tmp_path/'source.jpg'
|
||||
with Image.new('RGB',(400,200),'blue') as image:
|
||||
exif=Image.Exif();exif[274]=6
|
||||
image.save(source,exif=exif)
|
||||
result=create_variants(source,tmp_path,'image/jpeg',40_000_000,ImageSettings(_env_file=None))
|
||||
assert (result.variants[0].width,result.variants[0].height)==(128,256)
|
||||
source = tmp_path / "source.jpg"
|
||||
with Image.new("RGB", (400, 200), "blue") as image:
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6
|
||||
image.save(source, exif=exif)
|
||||
result = create_variants(
|
||||
source, tmp_path, "image/jpeg", 40_000_000, ImageSettings(_env_file=None)
|
||||
)
|
||||
assert (result.variants[0].width, result.variants[0].height) == (128, 256)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('failure', ['memory','cpu'])
|
||||
async def test_worker_os_limits(failure,tmp_path):
|
||||
body='return bytearray(2 * 1024 ** 3)' if failure=='memory' else 'while True: pass'
|
||||
code="from app.core import thumbnail_worker as w\ndef fail(*args):\n "+body+"\nw.create_variants=fail\nraise SystemExit(w.main())"
|
||||
result=await asyncio.to_thread(subprocess.run,[sys.executable,'-c',code,'source',str(tmp_path),str(tmp_path/'metadata'),'image/png','40000000'],
|
||||
env={**os.environ,'IMAGE_MEMORY_MIB':'256','IMAGE_CPU_SECONDS':'1'},capture_output=True,timeout=15)
|
||||
assert result.returncode==(3 if failure=='memory' else -9)
|
||||
assert not (tmp_path/'metadata').exists()
|
||||
@pytest.mark.parametrize("failure", ["memory", "cpu"])
|
||||
async def test_worker_os_limits(failure, tmp_path):
|
||||
body = (
|
||||
"return bytearray(2 * 1024 ** 3)" if failure == "memory" else "while True: pass"
|
||||
)
|
||||
code = (
|
||||
"from app.core import thumbnail_worker as w\ndef fail(*args):\n "
|
||||
+ body
|
||||
+ "\nw.create_variants=fail\nraise SystemExit(w.main())"
|
||||
)
|
||||
result = await asyncio.to_thread(
|
||||
subprocess.run,
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
code,
|
||||
"source",
|
||||
str(tmp_path),
|
||||
str(tmp_path / "metadata"),
|
||||
"image/png",
|
||||
"40000000",
|
||||
],
|
||||
env={**os.environ, "IMAGE_MEMORY_MIB": "256", "IMAGE_CPU_SECONDS": "1"},
|
||||
capture_output=True,
|
||||
timeout=15,
|
||||
)
|
||||
assert result.returncode == (3 if failure == "memory" else -9)
|
||||
assert not (tmp_path / "metadata").exists()
|
||||
|
||||
|
||||
async def test_render_timeout_kills_child(monkeypatch,tmp_path):
|
||||
create=asyncio.create_subprocess_exec
|
||||
children=[]
|
||||
async def slow_child(*args,**kwargs):
|
||||
child=await create(sys.executable,'-c','import time;time.sleep(60)',**kwargs)
|
||||
async def test_render_timeout_kills_child(monkeypatch, tmp_path):
|
||||
create = asyncio.create_subprocess_exec
|
||||
children = []
|
||||
|
||||
async def slow_child(*args, **kwargs):
|
||||
child = await create(
|
||||
sys.executable, "-c", "import time;time.sleep(60)", **kwargs
|
||||
)
|
||||
children.append(child)
|
||||
return child
|
||||
monkeypatch.setattr(asyncio,'create_subprocess_exec',slow_child)
|
||||
work=JobWork(uuid4(),uuid4(),uuid4(),1,'staging/test','image/png',1,'0'*64)
|
||||
|
||||
monkeypatch.setattr(asyncio, "create_subprocess_exec", slow_child)
|
||||
work = JobWork(
|
||||
uuid4(), uuid4(), uuid4(), 1, "staging/test", "image/png", 1, "0" * 64
|
||||
)
|
||||
with pytest.raises(TimeoutError):
|
||||
async with asyncio.timeout(.2):
|
||||
await render_variants(MemoryStorage(),work,tmp_path/'source',tmp_path)
|
||||
assert len(children)==1 and children[0].returncode is not None
|
||||
async with asyncio.timeout(0.2):
|
||||
await render_variants(MemoryStorage(), work, tmp_path / "source", tmp_path)
|
||||
assert len(children) == 1 and children[0].returncode is not None
|
||||
|
||||
@@ -92,7 +92,9 @@ async def test_instance_header_is_present_on_success_and_errors(
|
||||
assert response.headers["X-Instance-Id"] == "api-test"
|
||||
assert response.headers["X-Request-Id"]
|
||||
if status >= 400:
|
||||
assert response.json()["error"]["request_id"] == response.headers["X-Request-Id"]
|
||||
assert (
|
||||
response.json()["error"]["request_id"] == response.headers["X-Request-Id"]
|
||||
)
|
||||
|
||||
|
||||
async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_client):
|
||||
@@ -100,7 +102,9 @@ async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_cli
|
||||
second = await instance_client.get("/__test__/request-id")
|
||||
|
||||
assert first.status_code == second.status_code == 200
|
||||
assert first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test"
|
||||
assert (
|
||||
first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test"
|
||||
)
|
||||
assert first.headers["X-Request-Id"] != second.headers["X-Request-Id"]
|
||||
for response in (first, second):
|
||||
assert response.json()["request_id"] == response.headers["X-Request-Id"]
|
||||
|
||||
+137
-48
@@ -13,7 +13,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.jobs import JobSettings
|
||||
from app.domain import attachment_jobs
|
||||
from app.domain.attachment_jobs import claim_job, process_attachment_job, cleanup_staging, build_result
|
||||
from app.domain.attachment_jobs import (
|
||||
claim_job,
|
||||
process_attachment_job,
|
||||
cleanup_staging,
|
||||
build_result,
|
||||
)
|
||||
from app.domain.job_admin import replay_failed_job
|
||||
from app.domain.job_retry import classify_failure, retry_delay
|
||||
from app.errors.exceptions import JobOperationError, ProcessingError
|
||||
@@ -26,50 +31,85 @@ from test_attachments import context, upload, finish_attachment
|
||||
|
||||
|
||||
async def envelope(session, attachment_id):
|
||||
job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)))
|
||||
return JobMessage(job_id=job.id, attachment_id=job.attachment_id, pipeline_version=1,
|
||||
attempt=job.attempt + 1, created_at=job.created_at)
|
||||
job = await session.scalar(
|
||||
select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))
|
||||
)
|
||||
return JobMessage(
|
||||
job_id=job.id,
|
||||
attachment_id=job.attachment_id,
|
||||
pipeline_version=1,
|
||||
attempt=job.attempt + 1,
|
||||
created_at=job.created_at,
|
||||
)
|
||||
|
||||
|
||||
async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_engine):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
settings = JobSettings(_env_file=None, job_max_attempts=3)
|
||||
|
||||
def fail(_):
|
||||
raise TimeoutError
|
||||
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
first = await envelope(session, attachment['id'])
|
||||
first = await envelope(session, attachment["id"])
|
||||
for attempt in range(1, 4):
|
||||
message = await envelope(session, attachment['id'])
|
||||
assert await process_attachment_job(session, storage, message, settings, before_processing=fail)
|
||||
message = await envelope(session, attachment["id"])
|
||||
assert await process_attachment_job(
|
||||
session, storage, message, settings, before_processing=fail
|
||||
)
|
||||
job, item = await jobs.lock_job(session, first.job_id)
|
||||
assert job.attempt == attempt
|
||||
delay = (job.next_attempt_at - item.updated_at).total_seconds()
|
||||
assert 5 * 2 ** (attempt - 1) / 2 <= delay <= 5 * 2 ** (attempt - 1)
|
||||
assert item.status == ('failed' if attempt == 3 else 'queued')
|
||||
assert item.status == ("failed" if attempt == 3 else "queued")
|
||||
assert job.last_error_permanent is False
|
||||
job.next_attempt_at = await jobs.database_time(session) - timedelta(seconds=1)
|
||||
job.next_attempt_at = await jobs.database_time(session) - timedelta(
|
||||
seconds=1
|
||||
)
|
||||
await session.commit()
|
||||
assert item.error_code == 'retry_exhausted'
|
||||
assert item.error_code == "retry_exhausted"
|
||||
request_id = uuid4()
|
||||
result = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id,
|
||||
actor='test', reason='dependency_restored')
|
||||
assert result['attempt_limit'] == 6
|
||||
result = await replay_failed_job(
|
||||
session,
|
||||
storage,
|
||||
settings,
|
||||
first.job_id,
|
||||
request_id=request_id,
|
||||
actor="test",
|
||||
reason="dependency_restored",
|
||||
)
|
||||
assert result["attempt_limit"] == 6
|
||||
# Old envelope cannot consume the fresh replay budget.
|
||||
assert await process_attachment_job(session, storage, first, settings)
|
||||
job, item = await jobs.lock_job(session, first.job_id)
|
||||
assert job.attempt == 3 and item.status == 'queued'
|
||||
assert job.attempt == 3 and item.status == "queued"
|
||||
await session.commit()
|
||||
message = await envelope(session, attachment['id'])
|
||||
message = await envelope(session, attachment["id"])
|
||||
assert await process_attachment_job(session, storage, message, settings)
|
||||
before = dict(storage.objects)
|
||||
assert await process_attachment_job(session, storage, message, settings)
|
||||
assert storage.objects == before
|
||||
repeat = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id,
|
||||
actor='test', reason='dependency_restored')
|
||||
assert repeat['already_applied']
|
||||
with pytest.raises(JobOperationError, match='job_not_failed'):
|
||||
await replay_failed_job(session, storage, settings, first.job_id, request_id=uuid4(), actor='test', reason='manual_review')
|
||||
repeat = await replay_failed_job(
|
||||
session,
|
||||
storage,
|
||||
settings,
|
||||
first.job_id,
|
||||
request_id=request_id,
|
||||
actor="test",
|
||||
reason="dependency_restored",
|
||||
)
|
||||
assert repeat["already_applied"]
|
||||
with pytest.raises(JobOperationError, match="job_not_failed"):
|
||||
await replay_failed_job(
|
||||
session,
|
||||
storage,
|
||||
settings,
|
||||
first.job_id,
|
||||
request_id=uuid4(),
|
||||
actor="test",
|
||||
reason="manual_review",
|
||||
)
|
||||
await session.rollback()
|
||||
assert len((await session.scalars(select(JobReplay))).all()) == 1
|
||||
|
||||
@@ -77,26 +117,40 @@ async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_eng
|
||||
async def test_permanent_failure_and_missing_source_replay(client, context, db_engine):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
|
||||
def fail(_):
|
||||
raise ProcessingError('image_limits')
|
||||
raise ProcessingError("image_limits")
|
||||
|
||||
settings = JobSettings(_env_file=None)
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
await process_attachment_job(session, storage, message, settings, before_processing=fail)
|
||||
message = await envelope(session, attachment["id"])
|
||||
await process_attachment_job(
|
||||
session, storage, message, settings, before_processing=fail
|
||||
)
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
assert item.status == 'failed' and job.attempt == 1 and job.last_error_permanent
|
||||
assert item.status == "failed" and job.attempt == 1 and job.last_error_permanent
|
||||
job.staging_key = None
|
||||
await session.commit()
|
||||
with pytest.raises(JobOperationError, match='source_expired'):
|
||||
await replay_failed_job(session, storage, settings, job.id, request_id=uuid4(), actor='test', reason='manual_review')
|
||||
with pytest.raises(JobOperationError, match="source_expired"):
|
||||
await replay_failed_job(
|
||||
session,
|
||||
storage,
|
||||
settings,
|
||||
job.id,
|
||||
request_id=uuid4(),
|
||||
actor="test",
|
||||
reason="manual_review",
|
||||
)
|
||||
|
||||
|
||||
async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, context, db_engine):
|
||||
async def test_stale_worker_cannot_write_and_cleanup_keeps_active(
|
||||
client, context, db_engine
|
||||
):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
settings = JobSettings(_env_file=None)
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
message = await envelope(session, attachment["id"])
|
||||
claim = await claim_job(session, message, settings)
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
job.lease_token = uuid4()
|
||||
@@ -112,18 +166,33 @@ async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, contex
|
||||
async def test_cleanup_ready_preserves_final_objects(client, context, db_engine):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
await finish_attachment(client, storage, alice, attachment['id'])
|
||||
await finish_attachment(client, storage, alice, attachment["id"])
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
message = await envelope(session, attachment["id"])
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
staging = job.staging_key
|
||||
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
|
||||
await session.commit()
|
||||
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=False) == 1
|
||||
assert (
|
||||
await cleanup_staging(
|
||||
session, storage, JobSettings(_env_file=None), apply=False
|
||||
)
|
||||
== 1
|
||||
)
|
||||
assert staging in storage.objects
|
||||
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 1
|
||||
assert (
|
||||
await cleanup_staging(
|
||||
session, storage, JobSettings(_env_file=None), apply=True
|
||||
)
|
||||
== 1
|
||||
)
|
||||
assert staging not in storage.objects and len(storage.objects) == 4
|
||||
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 0
|
||||
assert (
|
||||
await cleanup_staging(
|
||||
session, storage, JobSettings(_env_file=None), apply=True
|
||||
)
|
||||
== 0
|
||||
)
|
||||
|
||||
|
||||
async def test_retry_classification_and_cap():
|
||||
@@ -131,44 +200,64 @@ async def test_retry_classification_and_cap():
|
||||
for attempt in (1, 2, 6, 100):
|
||||
cap = min(120, 5 * 2 ** min(30, attempt - 1))
|
||||
assert all(cap / 2 <= retry_delay(settings, attempt) <= cap for _ in range(100))
|
||||
assert classify_failure(TimeoutError()) == ('processing_timeout', False)
|
||||
assert classify_failure(OSError(errno.EACCES, 'denied'))[1]
|
||||
assert not classify_failure(OSError(errno.ENOSPC, 'full'))[1]
|
||||
assert classify_failure(TimeoutError()) == ("processing_timeout", False)
|
||||
assert classify_failure(OSError(errno.EACCES, "denied"))[1]
|
||||
assert not classify_failure(OSError(errno.ENOSPC, "full"))[1]
|
||||
|
||||
|
||||
async def test_timeout_records_retry_and_clears_lease(client, context, db_engine, monkeypatch):
|
||||
async def test_timeout_records_retry_and_clears_lease(
|
||||
client, context, db_engine, monkeypatch
|
||||
):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
|
||||
async def stalled(*args, **kwargs):
|
||||
await asyncio.sleep(60)
|
||||
monkeypatch.setattr(attachment_jobs, 'build_result', stalled)
|
||||
settings = JobSettings(_env_file=None).model_copy(update={'job_timeout_seconds': .05})
|
||||
|
||||
monkeypatch.setattr(attachment_jobs, "build_result", stalled)
|
||||
settings = JobSettings(_env_file=None).model_copy(
|
||||
update={"job_timeout_seconds": 0.05}
|
||||
)
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
message = await envelope(session, attachment["id"])
|
||||
assert await process_attachment_job(session, storage, message, settings)
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
assert item.status == 'queued' and job.last_error_code == 'processing_timeout'
|
||||
assert item.status == "queued" and job.last_error_code == "processing_timeout"
|
||||
assert job.lease_token is None and job.lease_until is None
|
||||
|
||||
|
||||
async def test_failed_retention_and_source_checksum_protect_replay(client, context, db_engine):
|
||||
async def test_failed_retention_and_source_checksum_protect_replay(
|
||||
client, context, db_engine
|
||||
):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
|
||||
def fail(_):
|
||||
raise ProcessingError('invalid_image')
|
||||
raise ProcessingError("invalid_image")
|
||||
|
||||
settings = JobSettings(_env_file=None)
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
await process_attachment_job(session, storage, message, settings, before_processing=fail)
|
||||
message = await envelope(session, attachment["id"])
|
||||
await process_attachment_job(
|
||||
session, storage, message, settings, before_processing=fail
|
||||
)
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
key = job.staging_key
|
||||
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
|
||||
await session.commit()
|
||||
assert await cleanup_staging(session, storage, settings, apply=True) == 0
|
||||
data, metadata = storage.objects[key]
|
||||
storage.objects[key] = (b'x' * len(data), metadata)
|
||||
with pytest.raises(JobOperationError, match='source_mismatch'):
|
||||
await replay_failed_job(session, storage, settings, message.job_id, request_id=uuid4(), actor='test', reason='manual_review')
|
||||
storage.objects[key] = (b"x" * len(data), metadata)
|
||||
with pytest.raises(JobOperationError, match="source_mismatch"):
|
||||
await replay_failed_job(
|
||||
session,
|
||||
storage,
|
||||
settings,
|
||||
message.job_id,
|
||||
request_id=uuid4(),
|
||||
actor="test",
|
||||
reason="manual_review",
|
||||
)
|
||||
await session.rollback()
|
||||
assert not (await session.scalars(select(JobReplay))).all()
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
|
||||
@@ -20,9 +20,9 @@ async def test_draining_keeps_liveness_but_rejects_new_work(make_settings):
|
||||
app.state.lifecycle.start_draining()
|
||||
try:
|
||||
async with make_client(app) as client:
|
||||
assert (await client.get('/health/live')).status_code == 200
|
||||
assert (await client.get('/health/ready')).status_code == 503
|
||||
assert (await client.get('/api/v1/users/me')).status_code == 503
|
||||
assert (await client.get("/health/live")).status_code == 200
|
||||
assert (await client.get("/health/ready")).status_code == 503
|
||||
assert (await client.get("/api/v1/users/me")).status_code == 503
|
||||
finally:
|
||||
await close_app(app)
|
||||
|
||||
@@ -36,13 +36,13 @@ async def test_readiness_timeout_cancels_dependency_checks(make_settings, monkey
|
||||
finally:
|
||||
cancelled.append(True)
|
||||
|
||||
monkeypatch.setattr(health, 'check_database', stalled)
|
||||
monkeypatch.setattr(health, 'check_redis', stalled)
|
||||
monkeypatch.setattr(health, "check_database", stalled)
|
||||
monkeypatch.setattr(health, "check_redis", stalled)
|
||||
app = create_app(make_settings(readiness_timeout_seconds=0.03))
|
||||
try:
|
||||
async with make_client(app) as client:
|
||||
started = time.monotonic()
|
||||
response = await client.get('/health/ready')
|
||||
response = await client.get("/health/ready")
|
||||
assert response.status_code == 503
|
||||
assert time.monotonic() - started < 1
|
||||
assert len(cancelled) == 2
|
||||
|
||||
@@ -6,7 +6,13 @@ import httpx
|
||||
|
||||
from app.main import create_app
|
||||
from conftest import close_app, make_client
|
||||
from helpers import assert_request_id, bearer, create_chat, create_user, idempotency_header
|
||||
from helpers import (
|
||||
assert_request_id,
|
||||
bearer,
|
||||
create_chat,
|
||||
create_user,
|
||||
idempotency_header,
|
||||
)
|
||||
|
||||
|
||||
def _assert_error(response: httpx.Response, status: int, code: str) -> None:
|
||||
@@ -25,12 +31,16 @@ async def test_message_limit_rejects_without_deleting_confirmed_messages(make_se
|
||||
|
||||
for index in range(2):
|
||||
ok = await client.post(
|
||||
url, headers={**bearer(user), **idempotency_header()}, json={"text": f"confirmed-{index}"}
|
||||
url,
|
||||
headers={**bearer(user), **idempotency_header()},
|
||||
json={"text": f"confirmed-{index}"},
|
||||
)
|
||||
assert ok.status_code == 201
|
||||
|
||||
rejected = await client.post(
|
||||
url, headers={**bearer(user), **idempotency_header()}, json={"text": "rejected"}
|
||||
url,
|
||||
headers={**bearer(user), **idempotency_header()},
|
||||
json={"text": "rejected"},
|
||||
)
|
||||
_assert_error(rejected, 503, "unavailable")
|
||||
|
||||
@@ -65,12 +75,18 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat
|
||||
user = await create_user(client)
|
||||
chat = await create_chat(client, user)
|
||||
|
||||
monkeypatch.setattr(message_endpoint, "send_message_idempotent", blocking_send_message)
|
||||
monkeypatch.setattr(
|
||||
message_endpoint, "send_message_idempotent", blocking_send_message
|
||||
)
|
||||
url = f"/api/v1/chats/{chat['id']}/messages"
|
||||
|
||||
held = [
|
||||
asyncio.create_task(
|
||||
client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": f"held-{i}"})
|
||||
client.post(
|
||||
url,
|
||||
headers={**bearer(user), **idempotency_header()},
|
||||
json={"text": f"held-{i}"},
|
||||
)
|
||||
)
|
||||
for i in range(2)
|
||||
]
|
||||
@@ -79,7 +95,11 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat
|
||||
|
||||
rejected = await asyncio.gather(
|
||||
*(
|
||||
client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": f"rejected-{i}"})
|
||||
client.post(
|
||||
url,
|
||||
headers={**bearer(user), **idempotency_header()},
|
||||
json={"text": f"rejected-{i}"},
|
||||
)
|
||||
for i in range(4)
|
||||
)
|
||||
)
|
||||
@@ -92,7 +112,11 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat
|
||||
release.set()
|
||||
assert all(r.status_code == 201 for r in await asyncio.gather(*held))
|
||||
|
||||
recovered = await client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": "after"})
|
||||
recovered = await client.post(
|
||||
url,
|
||||
headers={**bearer(user), **idempotency_header()},
|
||||
json={"text": "after"},
|
||||
)
|
||||
assert recovered.status_code == 201
|
||||
assert_request_id(recovered)
|
||||
await close_app(app)
|
||||
|
||||
@@ -8,7 +8,19 @@ from sqlalchemy import inspect, text
|
||||
from app.models import Base
|
||||
from conftest import reset_schema, upgrade_head
|
||||
|
||||
EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents", "attachment_jobs", "job_replays"}
|
||||
EXPECTED_TABLES = {
|
||||
"users",
|
||||
"chats",
|
||||
"chat_members",
|
||||
"messages",
|
||||
"counters",
|
||||
"idempotency_keys",
|
||||
"attachments",
|
||||
"message_attachments",
|
||||
"upload_intents",
|
||||
"attachment_jobs",
|
||||
"job_replays",
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -62,6 +74,6 @@ async def test_models_and_migrations_are_in_sync(migrated_engine):
|
||||
async with migrated_engine.connect() as connection:
|
||||
diff = await connection.run_sync(compare)
|
||||
|
||||
assert diff == [], (
|
||||
f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}"
|
||||
)
|
||||
assert (
|
||||
diff == []
|
||||
), f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}"
|
||||
|
||||
@@ -10,10 +10,12 @@ from app.storage.s3 import ObjectStream, run_sync
|
||||
|
||||
async def test_sdk_thread_finishes_before_cancellation_returns():
|
||||
entered, release, finished = Event(), Event(), Event()
|
||||
|
||||
def operation():
|
||||
entered.set()
|
||||
release.wait(timeout=3)
|
||||
finished.set()
|
||||
|
||||
task = asyncio.create_task(run_sync(operation))
|
||||
await asyncio.to_thread(entered.wait, 2)
|
||||
task.cancel()
|
||||
@@ -31,10 +33,12 @@ async def test_response_discarded_when_get_is_cancelled():
|
||||
entered, release = Event(), Event()
|
||||
discarded = []
|
||||
response = object()
|
||||
|
||||
def operation():
|
||||
entered.set()
|
||||
release.wait(timeout=3)
|
||||
return response
|
||||
|
||||
task = asyncio.create_task(run_sync(operation, discard_result=discarded.append))
|
||||
await asyncio.to_thread(entered.wait, 2)
|
||||
task.cancel()
|
||||
@@ -48,9 +52,12 @@ async def test_response_discarded_when_get_is_cancelled():
|
||||
|
||||
async def test_stream_closes_even_before_first_iteration():
|
||||
calls = []
|
||||
response = SimpleNamespace(close=lambda: calls.append('close'), release_conn=lambda: calls.append('release'))
|
||||
response = SimpleNamespace(
|
||||
close=lambda: calls.append("close"),
|
||||
release_conn=lambda: calls.append("release"),
|
||||
)
|
||||
storage = SimpleNamespace(settings=S3Settings(_env_file=None))
|
||||
stream = ObjectStream(storage, response)
|
||||
await stream.aclose()
|
||||
await stream.aclose()
|
||||
assert calls == ['close', 'release']
|
||||
assert calls == ["close", "release"]
|
||||
|
||||
+252
-120
@@ -1,4 +1,5 @@
|
||||
"""HTTP helpers for public black-box tests; Python 3.9+, standard library only."""
|
||||
|
||||
import base64
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
@@ -13,205 +14,336 @@ import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LAB = json.loads((ROOT / 'lab.json').read_text())['number']
|
||||
SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text())
|
||||
LAB = json.loads((ROOT / "lab.json").read_text())["number"]
|
||||
SPEC = json.loads((ROOT / "contracts/openapi.json").read_text())
|
||||
|
||||
|
||||
class Failure(AssertionError):
|
||||
pass
|
||||
|
||||
|
||||
def check(condition, message):
|
||||
if not condition:
|
||||
raise Failure(message)
|
||||
|
||||
def timestamp(value):
|
||||
check(isinstance(value, str) and value.endswith('Z'), 'timestamp must be UTC ending in Z')
|
||||
try:
|
||||
return dt.datetime.fromisoformat(value[:-1] + '+00:00')
|
||||
except ValueError as exc:
|
||||
raise Failure('invalid RFC3339 timestamp') from exc
|
||||
|
||||
def validate(value, schema, where='response'):
|
||||
def timestamp(value):
|
||||
check(
|
||||
isinstance(value, str) and value.endswith("Z"),
|
||||
"timestamp must be UTC ending in Z",
|
||||
)
|
||||
try:
|
||||
return dt.datetime.fromisoformat(value[:-1] + "+00:00")
|
||||
except ValueError as exc:
|
||||
raise Failure("invalid RFC3339 timestamp") from exc
|
||||
|
||||
|
||||
def validate(value, schema, where="response"):
|
||||
"""Validate the JSON Schema subset used by the supplied contract, not arbitrary OpenAPI."""
|
||||
if '$ref' in schema:
|
||||
if "$ref" in schema:
|
||||
target = SPEC
|
||||
for part in schema['$ref'].split('/')[1:]:
|
||||
for part in schema["$ref"].split("/")[1:]:
|
||||
target = target[part]
|
||||
return validate(value, target, where)
|
||||
kinds = schema.get('type', [])
|
||||
kinds = schema.get("type", [])
|
||||
if isinstance(kinds, str):
|
||||
kinds = [kinds]
|
||||
matches = {'null':value is None, 'boolean':isinstance(value,bool), 'integer':isinstance(value,int) and not isinstance(value,bool), 'number':isinstance(value,(int,float)) and not isinstance(value,bool), 'string':isinstance(value,str), 'array':isinstance(value,list), 'object':isinstance(value,dict)}
|
||||
matches = {
|
||||
"null": value is None,
|
||||
"boolean": isinstance(value, bool),
|
||||
"integer": isinstance(value, int) and not isinstance(value, bool),
|
||||
"number": isinstance(value, (int, float)) and not isinstance(value, bool),
|
||||
"string": isinstance(value, str),
|
||||
"array": isinstance(value, list),
|
||||
"object": isinstance(value, dict),
|
||||
}
|
||||
if kinds:
|
||||
check(any(matches.get(k,False) for k in kinds), where + ': incorrect type')
|
||||
if 'enum' in schema:
|
||||
check(value in schema['enum'], where + ': invalid enum')
|
||||
if 'const' in schema:
|
||||
check(value == schema['const'], where + ': invalid const')
|
||||
if isinstance(value,dict):
|
||||
props = schema.get('properties',{})
|
||||
check(all(k in value for k in schema.get('required',[])), where + ': missing required fields')
|
||||
if schema.get('additionalProperties') is False:
|
||||
check(not set(value).difference(props), where + ': unexpected fields')
|
||||
check(any(matches.get(k, False) for k in kinds), where + ": incorrect type")
|
||||
if "enum" in schema:
|
||||
check(value in schema["enum"], where + ": invalid enum")
|
||||
if "const" in schema:
|
||||
check(value == schema["const"], where + ": invalid const")
|
||||
if isinstance(value, dict):
|
||||
props = schema.get("properties", {})
|
||||
check(
|
||||
all(k in value for k in schema.get("required", [])),
|
||||
where + ": missing required fields",
|
||||
)
|
||||
if schema.get("additionalProperties") is False:
|
||||
check(not set(value).difference(props), where + ": unexpected fields")
|
||||
for key, sub in props.items():
|
||||
if key in value:
|
||||
validate(value[key], sub, where + '.' + key)
|
||||
if isinstance(value,list):
|
||||
check(len(value) >= schema.get('minItems',0), where + ': too few items')
|
||||
check(len(value) <= schema.get('maxItems',float('inf')), where + ': too many items')
|
||||
if schema.get('uniqueItems'):
|
||||
encoded = [json.dumps(x,sort_keys=True) for x in value]
|
||||
check(len(encoded)==len(set(encoded)), where + ': duplicate items')
|
||||
if 'items' in schema:
|
||||
validate(value[key], sub, where + "." + key)
|
||||
if isinstance(value, list):
|
||||
check(len(value) >= schema.get("minItems", 0), where + ": too few items")
|
||||
check(
|
||||
len(value) <= schema.get("maxItems", float("inf")),
|
||||
where + ": too many items",
|
||||
)
|
||||
if schema.get("uniqueItems"):
|
||||
encoded = [json.dumps(x, sort_keys=True) for x in value]
|
||||
check(len(encoded) == len(set(encoded)), where + ": duplicate items")
|
||||
if "items" in schema:
|
||||
for item in value:
|
||||
validate(item, schema['items'], where + '[]')
|
||||
if isinstance(value,str):
|
||||
check(len(value) >= schema.get('minLength',0), where + ': too short')
|
||||
check(len(value) <= schema.get('maxLength',float('inf')), where + ': too long')
|
||||
if 'pattern' in schema:
|
||||
check(re.search(schema['pattern'], value) is not None, where + ': pattern mismatch')
|
||||
if schema.get('format') == 'uuid':
|
||||
validate(item, schema["items"], where + "[]")
|
||||
if isinstance(value, str):
|
||||
check(len(value) >= schema.get("minLength", 0), where + ": too short")
|
||||
check(len(value) <= schema.get("maxLength", float("inf")), where + ": too long")
|
||||
if "pattern" in schema:
|
||||
check(
|
||||
re.search(schema["pattern"], value) is not None,
|
||||
where + ": pattern mismatch",
|
||||
)
|
||||
if schema.get("format") == "uuid":
|
||||
try:
|
||||
uuid.UUID(value)
|
||||
except ValueError as exc:
|
||||
raise Failure(where + ': invalid UUID') from exc
|
||||
if schema.get('format') == 'date-time':
|
||||
raise Failure(where + ": invalid UUID") from exc
|
||||
if schema.get("format") == "date-time":
|
||||
timestamp(value)
|
||||
if isinstance(value,(int,float)) and not isinstance(value,bool):
|
||||
check(value >= schema.get('minimum',-float('inf')), where + ': below minimum')
|
||||
check(value <= schema.get('maximum',float('inf')), where + ': above maximum')
|
||||
if 'anyOf' in schema:
|
||||
for alternative in schema['anyOf']:
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
check(value >= schema.get("minimum", -float("inf")), where + ": below minimum")
|
||||
check(value <= schema.get("maximum", float("inf")), where + ": above maximum")
|
||||
if "anyOf" in schema:
|
||||
for alternative in schema["anyOf"]:
|
||||
try:
|
||||
validate(value, alternative, where)
|
||||
break
|
||||
except Failure:
|
||||
pass
|
||||
else:
|
||||
raise Failure(where + ': no anyOf branch matches')
|
||||
raise Failure(where + ": no anyOf branch matches")
|
||||
|
||||
|
||||
def shape(value, name):
|
||||
validate(value, SPEC['components']['schemas'][name])
|
||||
validate(value, SPEC["components"]["schemas"][name])
|
||||
return value
|
||||
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
return None
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self, base_url=None, ca_file=None, timeout=10):
|
||||
self.base = (base_url or os.environ.get('BASE_URL') or ('https://localhost:8443' if LAB==7 else 'http://localhost:8080')).rstrip('/')
|
||||
check(self.base.startswith(('http://','https://')), 'BASE_URL must be HTTP(S)')
|
||||
check(LAB != 7 or self.base.startswith('https://'), 'lab7 requires HTTPS')
|
||||
self.base = (
|
||||
base_url
|
||||
or os.environ.get("BASE_URL")
|
||||
or ("https://localhost:8443" if LAB == 7 else "http://localhost:8080")
|
||||
).rstrip("/")
|
||||
check(self.base.startswith(("http://", "https://")), "BASE_URL must be HTTP(S)")
|
||||
check(LAB != 7 or self.base.startswith("https://"), "lab7 requires HTTPS")
|
||||
self.timeout = timeout
|
||||
context = ssl.create_default_context(cafile=ca_file or os.environ.get('CA_FILE') or None)
|
||||
self.opener = urllib.request.build_opener(NoRedirect(), urllib.request.HTTPSHandler(context=context))
|
||||
context = ssl.create_default_context(
|
||||
cafile=ca_file or os.environ.get("CA_FILE") or None
|
||||
)
|
||||
self.opener = urllib.request.build_opener(
|
||||
NoRedirect(), urllib.request.HTTPSHandler(context=context)
|
||||
)
|
||||
|
||||
def call(self, method, path, expected=200, body=None, actor=None, headers=None, raw=None, schema=None):
|
||||
h = {'Accept':'application/json'}
|
||||
def call(
|
||||
self,
|
||||
method,
|
||||
path,
|
||||
expected=200,
|
||||
body=None,
|
||||
actor=None,
|
||||
headers=None,
|
||||
raw=None,
|
||||
schema=None,
|
||||
):
|
||||
h = {"Accept": "application/json"}
|
||||
if actor:
|
||||
h['X-User-Id' if LAB==1 else 'Authorization'] = actor['id'] if LAB==1 else 'Bearer ' + actor['token']
|
||||
h["X-User-Id" if LAB == 1 else "Authorization"] = (
|
||||
actor["id"] if LAB == 1 else "Bearer " + actor["token"]
|
||||
)
|
||||
if body is not None:
|
||||
raw = json.dumps(body,ensure_ascii=False).encode('utf-8')
|
||||
h['Content-Type'] = 'application/json'
|
||||
raw = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||||
h["Content-Type"] = "application/json"
|
||||
h.update(headers or {})
|
||||
req = urllib.request.Request(self.base + path, data=raw, headers=h, method=method)
|
||||
req = urllib.request.Request(
|
||||
self.base + path, data=raw, headers=h, method=method
|
||||
)
|
||||
start = time.monotonic()
|
||||
try:
|
||||
response = self.opener.open(req, timeout=self.timeout)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
except (urllib.error.URLError,TimeoutError,OSError) as exc:
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
# Do not print URL query, headers, request body, or credentials.
|
||||
raise Failure(method + ' ' + path.split('?')[0] + ': connection/TLS failure (' + type(exc).__name__ + ')') from None
|
||||
raise Failure(
|
||||
method
|
||||
+ " "
|
||||
+ path.split("?")[0]
|
||||
+ ": connection/TLS failure ("
|
||||
+ type(exc).__name__
|
||||
+ ")"
|
||||
) from None
|
||||
with response:
|
||||
status = response.code
|
||||
rh = response.headers
|
||||
data = response.read(16*1024*1024 + 1)
|
||||
check(len(data) <= 16*1024*1024, 'response exceeds test safety limit')
|
||||
allowed = [expected] if isinstance(expected,int) else expected
|
||||
check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status))
|
||||
check(bool(rh.get('X-Request-Id')), 'missing X-Request-Id')
|
||||
if LAB>=3:
|
||||
check(bool(rh.get('X-Instance-Id')), 'missing X-Instance-Id')
|
||||
data = response.read(16 * 1024 * 1024 + 1)
|
||||
check(len(data) <= 16 * 1024 * 1024, "response exceeds test safety limit")
|
||||
allowed = [expected] if isinstance(expected, int) else expected
|
||||
check(
|
||||
status in allowed,
|
||||
method
|
||||
+ " "
|
||||
+ path.split("?")[0]
|
||||
+ ": expected "
|
||||
+ str(allowed)
|
||||
+ ", received "
|
||||
+ str(status),
|
||||
)
|
||||
check(bool(rh.get("X-Request-Id")), "missing X-Request-Id")
|
||||
if LAB >= 3:
|
||||
check(bool(rh.get("X-Instance-Id")), "missing X-Instance-Id")
|
||||
payload = None
|
||||
if schema or status>=400:
|
||||
check(rh.get_content_type()=='application/json', 'expected application/json')
|
||||
if schema or status >= 400:
|
||||
check(
|
||||
rh.get_content_type() == "application/json", "expected application/json"
|
||||
)
|
||||
try:
|
||||
payload = json.loads(data)
|
||||
except (ValueError,UnicodeError):
|
||||
raise Failure('invalid response JSON') from None
|
||||
shape(payload, 'Error' if status>=400 else schema)
|
||||
if status>=400:
|
||||
check(payload['error']['request_id']==rh.get('X-Request-Id'),'error request_id differs from header')
|
||||
if status==401 and LAB>=2:
|
||||
scheme = 'Basic' if path=='/api/v1/auth/sessions' else 'Bearer'
|
||||
check(rh.get('WWW-Authenticate','').lower().startswith(scheme.lower()), 'incorrect WWW-Authenticate challenge')
|
||||
if status==204:
|
||||
check(not data, '204 must not contain body')
|
||||
return {'status':status,'headers':rh,'json':payload,'bytes':data,'elapsed':time.monotonic()-start}
|
||||
except (ValueError, UnicodeError):
|
||||
raise Failure("invalid response JSON") from None
|
||||
shape(payload, "Error" if status >= 400 else schema)
|
||||
if status >= 400:
|
||||
check(
|
||||
payload["error"]["request_id"] == rh.get("X-Request-Id"),
|
||||
"error request_id differs from header",
|
||||
)
|
||||
if status == 401 and LAB >= 2:
|
||||
scheme = "Basic" if path == "/api/v1/auth/sessions" else "Bearer"
|
||||
check(
|
||||
rh.get("WWW-Authenticate", "").lower().startswith(scheme.lower()),
|
||||
"incorrect WWW-Authenticate challenge",
|
||||
)
|
||||
if status == 204:
|
||||
check(not data, "204 must not contain body")
|
||||
return {
|
||||
"status": status,
|
||||
"headers": rh,
|
||||
"json": payload,
|
||||
"bytes": data,
|
||||
"elapsed": time.monotonic() - start,
|
||||
}
|
||||
|
||||
def user(self, label):
|
||||
username = 'u_' + label + '_' + uuid.uuid4().hex[:12]
|
||||
password = 'T3st-' + uuid.uuid4().hex
|
||||
body = {'username':username,'display_name':'Студент ' + label}
|
||||
if LAB>=2:
|
||||
body['password'] = password
|
||||
user = self.call('POST','/api/v1/users',201,body=body,schema='User')['json']
|
||||
check(user['username']==username,'username changed')
|
||||
actor = {'id':user['id'],'user':user,'registration':body}
|
||||
if LAB>=2:
|
||||
credential = base64.b64encode((username+':'+password).encode()).decode()
|
||||
session = self.call('POST','/api/v1/auth/sessions',201,headers={'Authorization':'Basic '+credential},schema='Session')['json']
|
||||
check(session['user']==user,'session user mismatch')
|
||||
check(timestamp(session['expires_at']) > dt.datetime.now(dt.timezone.utc),'session already expired')
|
||||
actor['token'] = session['token']
|
||||
username = "u_" + label + "_" + uuid.uuid4().hex[:12]
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
body = {"username": username, "display_name": "Студент " + label}
|
||||
if LAB >= 2:
|
||||
body["password"] = password
|
||||
user = self.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"]
|
||||
check(user["username"] == username, "username changed")
|
||||
actor = {"id": user["id"], "user": user, "registration": body}
|
||||
if LAB >= 2:
|
||||
credential = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
session = self.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
201,
|
||||
headers={"Authorization": "Basic " + credential},
|
||||
schema="Session",
|
||||
)["json"]
|
||||
check(session["user"] == user, "session user mismatch")
|
||||
check(
|
||||
timestamp(session["expires_at"]) > dt.datetime.now(dt.timezone.utc),
|
||||
"session already expired",
|
||||
)
|
||||
actor["token"] = session["token"]
|
||||
return actor
|
||||
|
||||
def bootstrap(self):
|
||||
self.call('GET','/health/live',schema='Health')
|
||||
self.call('GET','/health/ready',schema='Health')
|
||||
alice, bob, eve = (self.user(label) for label in ('alice','bob','eve'))
|
||||
chat = self.call('POST','/api/v1/chats',201,actor=alice,body={'title':'Контрактный тест','member_ids':[alice['id'],bob['id']]},schema='Chat')['json']
|
||||
check(set(chat['member_ids'])=={alice['id'],bob['id']},'chat member mismatch')
|
||||
return alice,bob,eve,chat
|
||||
self.call("GET", "/health/live", schema="Health")
|
||||
self.call("GET", "/health/ready", schema="Health")
|
||||
alice, bob, eve = (self.user(label) for label in ("alice", "bob", "eve"))
|
||||
chat = self.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "Контрактный тест", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
check(
|
||||
set(chat["member_ids"]) == {alice["id"], bob["id"]}, "chat member mismatch"
|
||||
)
|
||||
return alice, bob, eve, chat
|
||||
|
||||
def message(self, actor, chat, text, key=None, attachments=None):
|
||||
body = {'text':text}
|
||||
body = {"text": text}
|
||||
if attachments is not None:
|
||||
body['attachment_ids'] = attachments
|
||||
headers = {'Idempotency-Key':key or uuid.uuid4().hex} if LAB>=3 else {}
|
||||
return self.call('POST','/api/v1/chats/'+chat['id']+'/messages',201,actor=actor,body=body,headers=headers,schema='Message')
|
||||
body["attachment_ids"] = attachments
|
||||
headers = {"Idempotency-Key": key or uuid.uuid4().hex} if LAB >= 3 else {}
|
||||
return self.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||
201,
|
||||
actor=actor,
|
||||
body=body,
|
||||
headers=headers,
|
||||
schema="Message",
|
||||
)
|
||||
|
||||
def attachment(self, actor, chat):
|
||||
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
att = self.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',201 if LAB==4 else 202,actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')['json']
|
||||
check(att['status']==('ready' if LAB==4 else 'queued'),'incorrect upload status')
|
||||
check(att['chat_id']==chat['id'] and att['owner_id']==actor['id'],'attachment ownership mismatch')
|
||||
check(att['sha256']==hashlib.sha256(content).hexdigest() and att['size_bytes']==len(content),'original checksum/size mismatch')
|
||||
return att,content
|
||||
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
att = self.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
|
||||
201 if LAB == 4 else 202,
|
||||
actor=actor,
|
||||
raw=content,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)["json"]
|
||||
check(
|
||||
att["status"] == ("ready" if LAB == 4 else "queued"),
|
||||
"incorrect upload status",
|
||||
)
|
||||
check(
|
||||
att["chat_id"] == chat["id"] and att["owner_id"] == actor["id"],
|
||||
"attachment ownership mismatch",
|
||||
)
|
||||
check(
|
||||
att["sha256"] == hashlib.sha256(content).hexdigest()
|
||||
and att["size_bytes"] == len(content),
|
||||
"original checksum/size mismatch",
|
||||
)
|
||||
return att, content
|
||||
|
||||
def wait_ready(self, actor, att, wait_seconds=60):
|
||||
deadline = time.monotonic()+wait_seconds
|
||||
while time.monotonic()<deadline:
|
||||
att = self.call('GET','/api/v1/attachments/'+att['id'],actor=actor,schema='Attachment')['json']
|
||||
check(att['status']!='failed','attachment reached failed')
|
||||
if att['status']=='ready':
|
||||
deadline = time.monotonic() + wait_seconds
|
||||
while time.monotonic() < deadline:
|
||||
att = self.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + att["id"],
|
||||
actor=actor,
|
||||
schema="Attachment",
|
||||
)["json"]
|
||||
check(att["status"] != "failed", "attachment reached failed")
|
||||
if att["status"] == "ready":
|
||||
return att
|
||||
time.sleep(0.3)
|
||||
raise Failure('attachment did not become ready within test deadline')
|
||||
raise Failure("attachment did not become ready within test deadline")
|
||||
|
||||
|
||||
def arguments(description, configure=None):
|
||||
import argparse
|
||||
|
||||
p = argparse.ArgumentParser(description=description)
|
||||
p.add_argument('--base-url', default=None)
|
||||
p.add_argument('--ca-file', default=None)
|
||||
p.add_argument("--base-url", default=None)
|
||||
p.add_argument("--ca-file", default=None)
|
||||
if configure:
|
||||
configure(p)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def run(main):
|
||||
try:
|
||||
main()
|
||||
except (Failure,ValueError,OSError) as exc:
|
||||
print('FAIL:', str(exc))
|
||||
except (Failure, ValueError, OSError) as exc:
|
||||
print("FAIL:", str(exc))
|
||||
raise SystemExit(1)
|
||||
|
||||
+18
-13
@@ -1,26 +1,31 @@
|
||||
"""Read JPEG SOF dimensions without external libraries; not a full image decoder."""
|
||||
|
||||
from client import Failure, check
|
||||
|
||||
|
||||
def jpeg_size(data):
|
||||
check(data[:2]==b'\xff\xd8','thumbnail is not JPEG')
|
||||
check(data[:2] == b"\xff\xd8", "thumbnail is not JPEG")
|
||||
i = 2
|
||||
sof = {0xc0,0xc1,0xc2,0xc3,0xc5,0xc6,0xc7,0xc9,0xca,0xcb,0xcd,0xce,0xcf}
|
||||
sof = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}
|
||||
while i < len(data):
|
||||
check(data[i]==0xff,'invalid JPEG marker')
|
||||
while i<len(data) and data[i]==0xff:
|
||||
check(data[i] == 0xFF, "invalid JPEG marker")
|
||||
while i < len(data) and data[i] == 0xFF:
|
||||
i += 1
|
||||
check(i<len(data),'truncated JPEG marker')
|
||||
check(i < len(data), "truncated JPEG marker")
|
||||
marker = data[i]
|
||||
i += 1
|
||||
if marker in {0xd9,0xda}:
|
||||
if marker in {0xD9, 0xDA}:
|
||||
break
|
||||
if marker in {0x01,0xd8} or 0xd0<=marker<=0xd7:
|
||||
if marker in {0x01, 0xD8} or 0xD0 <= marker <= 0xD7:
|
||||
continue
|
||||
check(i+2<=len(data),'truncated JPEG segment')
|
||||
length = int.from_bytes(data[i:i+2],'big')
|
||||
check(length>=2 and i+length<=len(data),'invalid JPEG segment length')
|
||||
check(i + 2 <= len(data), "truncated JPEG segment")
|
||||
length = int.from_bytes(data[i : i + 2], "big")
|
||||
check(length >= 2 and i + length <= len(data), "invalid JPEG segment length")
|
||||
if marker in sof:
|
||||
check(length>=8,'invalid JPEG SOF')
|
||||
return (int.from_bytes(data[i+5:i+7],'big'),int.from_bytes(data[i+3:i+5],'big'))
|
||||
check(length >= 8, "invalid JPEG SOF")
|
||||
return (
|
||||
int.from_bytes(data[i + 5 : i + 7], "big"),
|
||||
int.from_bytes(data[i + 3 : i + 5], "big"),
|
||||
)
|
||||
i += length
|
||||
raise Failure('JPEG dimensions not found')
|
||||
raise Failure("JPEG dimensions not found")
|
||||
|
||||
+58
-27
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Small closed-loop HTTP write benchmark; not an open-loop capacity proof."""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import math
|
||||
@@ -11,48 +12,78 @@ from client import Client, Failure, LAB, arguments, check, run
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument('--duration',type=float,default=30)
|
||||
p.add_argument('--concurrency',type=int,default=4)
|
||||
p.add_argument('--output',default=None,help='Optional JSON metrics, no session tokens')
|
||||
args = arguments('Synthetic POST message load, closed loop; creates users/messages',configure)
|
||||
check(1<=args.duration<=3600,'duration must be 1..3600 seconds')
|
||||
check(1<=args.concurrency<=128,'concurrency must be 1..128')
|
||||
c = Client(args.base_url,args.ca_file)
|
||||
alice,bob,eve,chat = c.bootstrap()
|
||||
p.add_argument("--duration", type=float, default=30)
|
||||
p.add_argument("--concurrency", type=int, default=4)
|
||||
p.add_argument(
|
||||
"--output", default=None, help="Optional JSON metrics, no session tokens"
|
||||
)
|
||||
|
||||
args = arguments(
|
||||
"Synthetic POST message load, closed loop; creates users/messages", configure
|
||||
)
|
||||
check(1 <= args.duration <= 3600, "duration must be 1..3600 seconds")
|
||||
check(1 <= args.concurrency <= 128, "concurrency must be 1..128")
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
barrier = threading.Barrier(args.concurrency)
|
||||
|
||||
def worker(worker_id):
|
||||
local = Client(args.base_url,args.ca_file,timeout=5)
|
||||
local = Client(args.base_url, args.ca_file, timeout=5)
|
||||
barrier.wait()
|
||||
deadline = time.monotonic()+args.duration
|
||||
latencies,errors = [],0
|
||||
deadline = time.monotonic() + args.duration
|
||||
latencies, errors = [], 0
|
||||
index = 0
|
||||
while time.monotonic()<deadline:
|
||||
while time.monotonic() < deadline:
|
||||
start = time.monotonic()
|
||||
try:
|
||||
local.message(alice,chat,'load-%s-%s' % (worker_id,index))
|
||||
latencies.append(time.monotonic()-start)
|
||||
local.message(alice, chat, "load-%s-%s" % (worker_id, index))
|
||||
latencies.append(time.monotonic() - start)
|
||||
except Failure:
|
||||
errors += 1
|
||||
index += 1
|
||||
return latencies,errors
|
||||
return latencies, errors
|
||||
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
results = list(pool.map(worker,range(args.concurrency)))
|
||||
elapsed = time.monotonic()-started
|
||||
latencies = sorted(x for values,_ in results for x in values)
|
||||
errors = sum(n for _,n in results)
|
||||
total = len(latencies)+errors
|
||||
results = list(pool.map(worker, range(args.concurrency)))
|
||||
elapsed = time.monotonic() - started
|
||||
latencies = sorted(x for values, _ in results for x in values)
|
||||
errors = sum(n for _, n in results)
|
||||
total = len(latencies) + errors
|
||||
|
||||
def percentile(p):
|
||||
return round(1000*latencies[max(0,math.ceil(p*len(latencies))-1)],3) if latencies else None
|
||||
report = {'lab':LAB,'workload':'closed-loop POST messages; no automatic retries','requested_duration_seconds':args.duration,'elapsed_seconds':round(elapsed,3),'concurrency':args.concurrency,'successful_requests':len(latencies),'failed_requests':errors,'error_ratio':round(errors/total,6) if total else None,'successful_rps':round(len(latencies)/elapsed,3),'successful_latency_ms':{'p50':percentile(.50),'p95':percentile(.95),'p99':percentile(.99)},'note':'Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.'}
|
||||
text = json.dumps(report,ensure_ascii=False,indent=2)
|
||||
return (
|
||||
round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 3)
|
||||
if latencies
|
||||
else None
|
||||
)
|
||||
|
||||
report = {
|
||||
"lab": LAB,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": args.duration,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"concurrency": args.concurrency,
|
||||
"successful_requests": len(latencies),
|
||||
"failed_requests": errors,
|
||||
"error_ratio": round(errors / total, 6) if total else None,
|
||||
"successful_rps": round(len(latencies) / elapsed, 3),
|
||||
"successful_latency_ms": {
|
||||
"p50": percentile(0.50),
|
||||
"p95": percentile(0.95),
|
||||
"p99": percentile(0.99),
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.",
|
||||
}
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
dest = Path(args.output)
|
||||
dest.parent.mkdir(parents=True,exist_ok=True)
|
||||
dest.write_text(text+'\n')
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text(text + "\n")
|
||||
print(text)
|
||||
check(bool(latencies),'no successful writes')
|
||||
check(bool(latencies), "no successful writes")
|
||||
# Errors are reported, not treated as a universal load-test threshold.
|
||||
|
||||
if __name__=='__main__':
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
|
||||
+142
-64
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit operator-authored hook; tokens remain in memory, volumes are never removed by this test."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
@@ -11,111 +12,187 @@ from client import Client, Failure, LAB, arguments, check, run
|
||||
|
||||
|
||||
def history(c, actor, chat):
|
||||
path = '/api/v1/chats/'+chat['id']+'/messages?limit=100'
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
|
||||
result = []
|
||||
seen = set()
|
||||
for _ in range(1000):
|
||||
page = c.call('GET',path,actor=actor,schema='MessagePage')['json']
|
||||
result.extend(page['items'])
|
||||
cursor = page['next_cursor']
|
||||
page = c.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
||||
result.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
return result
|
||||
check(cursor not in seen,'cursor cycle')
|
||||
check(cursor not in seen, "cursor cycle")
|
||||
seen.add(cursor)
|
||||
path = '/api/v1/chats/'+chat['id']+'/messages?limit=100&cursor='+quote(cursor,safe='')
|
||||
raise Failure('history pagination exceeded safety bound')
|
||||
path = (
|
||||
"/api/v1/chats/"
|
||||
+ chat["id"]
|
||||
+ "/messages?limit=100&cursor="
|
||||
+ quote(cursor, safe="")
|
||||
)
|
||||
raise Failure("history pagination exceeded safety bound")
|
||||
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument('--mode', choices=['persistence','failover'], required=True)
|
||||
p.add_argument('--action-script',required=True,help='Executable script authored by student; restarts stack or stops ONE API replica')
|
||||
args = arguments('Explicit restart/failover scenario; modifies only synthetic API data',configure)
|
||||
check(LAB>=2,'persistence requires lab2+')
|
||||
check(args.mode!='failover' or LAB>=3,'failover requires lab3+')
|
||||
p.add_argument("--mode", choices=["persistence", "failover"], required=True)
|
||||
p.add_argument(
|
||||
"--action-script",
|
||||
required=True,
|
||||
help="Executable script authored by student; restarts stack or stops ONE API replica",
|
||||
)
|
||||
|
||||
args = arguments(
|
||||
"Explicit restart/failover scenario; modifies only synthetic API data",
|
||||
configure,
|
||||
)
|
||||
check(LAB >= 2, "persistence requires lab2+")
|
||||
check(args.mode != "failover" or LAB >= 3, "failover requires lab3+")
|
||||
script = Path(args.action_script).expanduser().resolve()
|
||||
check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)')
|
||||
check('.example.' not in script.name,'copy and implement the example hook first')
|
||||
c = Client(args.base_url,args.ca_file,timeout=2)
|
||||
alice,bob,eve,chat = c.bootstrap()
|
||||
sent = c.message(alice,chat,'Сохранить при отказе')
|
||||
original = sent['json']
|
||||
target = sent['headers'].get('X-Instance-Id','')
|
||||
check(
|
||||
script.is_file() and os.access(script, os.X_OK),
|
||||
"action script must exist and be executable (chmod +x)",
|
||||
)
|
||||
check(".example." not in script.name, "copy and implement the example hook first")
|
||||
c = Client(args.base_url, args.ca_file, timeout=2)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
sent = c.message(alice, chat, "Сохранить при отказе")
|
||||
original = sent["json"]
|
||||
target = sent["headers"].get("X-Instance-Id", "")
|
||||
attachment = content = None
|
||||
if LAB>=4:
|
||||
attachment,content = c.attachment(alice,chat)
|
||||
if LAB>=5:
|
||||
attachment = c.wait_ready(alice,attachment)
|
||||
if LAB >= 4:
|
||||
attachment, content = c.attachment(alice, chat)
|
||||
if LAB >= 5:
|
||||
attachment = c.wait_ready(alice, attachment)
|
||||
env = dict(os.environ)
|
||||
env['TARGET_INSTANCE_ID'] = target
|
||||
env["TARGET_INSTANCE_ID"] = target
|
||||
observed = set()
|
||||
if args.mode=='failover':
|
||||
if args.mode == "failover":
|
||||
for _ in range(40):
|
||||
observed.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'])
|
||||
check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity')
|
||||
print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.')
|
||||
observed.add(
|
||||
c.call("GET", "/api/v1/users/me", actor=alice, schema="User")[
|
||||
"headers"
|
||||
]["X-Instance-Id"]
|
||||
)
|
||||
check(
|
||||
len(observed) >= 2,
|
||||
"less than two API instances observed before failure; verify LB/session affinity",
|
||||
)
|
||||
print(
|
||||
"Executing explicit "
|
||||
+ args.mode
|
||||
+ " hook; hook output suppressed to avoid leaking secrets."
|
||||
)
|
||||
# No shell interpolation and no Docker commands in this test.
|
||||
proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
proc = subprocess.Popen(
|
||||
[str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
|
||||
)
|
||||
started = time.monotonic()
|
||||
successes,failures,after_instances = [],0,[]
|
||||
successes, failures, after_instances = [], 0, []
|
||||
try:
|
||||
if args.mode=='persistence':
|
||||
if args.mode == "persistence":
|
||||
try:
|
||||
code = proc.wait(timeout=90)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise Failure('restart hook exceeded 90 seconds') from None
|
||||
check(code==0,'restart hook failed (inspect your script locally)')
|
||||
deadline = time.monotonic()+90
|
||||
raise Failure("restart hook exceeded 90 seconds") from None
|
||||
check(code == 0, "restart hook failed (inspect your script locally)")
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
c.call('GET','/health/ready',schema='Health')
|
||||
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
c.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic()<deadline,'stack/session did not recover within 90 seconds')
|
||||
check(
|
||||
time.monotonic() < deadline,
|
||||
"stack/session did not recover within 90 seconds",
|
||||
)
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
first_recovery = None
|
||||
key = uuid.uuid4().hex
|
||||
text = 'Запись во время отказа ' + key
|
||||
while time.monotonic()-started < 15:
|
||||
text = "Запись во время отказа " + key
|
||||
while time.monotonic() - started < 15:
|
||||
code = proc.poll()
|
||||
check(code is None or code==0,'stop-one hook failed (inspect your script locally)')
|
||||
check(code is not None or time.monotonic()-started<10,'stop-one hook must return within 10 seconds')
|
||||
check(
|
||||
code is None or code == 0,
|
||||
"stop-one hook failed (inspect your script locally)",
|
||||
)
|
||||
check(
|
||||
code is not None or time.monotonic() - started < 10,
|
||||
"stop-one hook must return within 10 seconds",
|
||||
)
|
||||
try:
|
||||
response = c.message(alice,chat,text,key=key)
|
||||
msg = response['json']
|
||||
if code==0:
|
||||
response = c.message(alice, chat, text, key=key)
|
||||
msg = response["json"]
|
||||
if code == 0:
|
||||
if first_recovery is None:
|
||||
first_recovery = time.monotonic()-started
|
||||
after_instances.append(response['headers']['X-Instance-Id'])
|
||||
first_recovery = time.monotonic() - started
|
||||
after_instances.append(response["headers"]["X-Instance-Id"])
|
||||
successes.append(msg)
|
||||
key = uuid.uuid4().hex
|
||||
text = 'Запись во время отказа ' + key
|
||||
text = "Запись во время отказа " + key
|
||||
except Failure:
|
||||
failures += 1
|
||||
# Keep SAME key/body after an uncertain POST result.
|
||||
time.sleep(0.1)
|
||||
check(proc.poll()==0,'stop-one hook did not finish successfully')
|
||||
check(first_recovery is not None and first_recovery<=10,'writes did not recover within 10 seconds from hook start')
|
||||
check(len(after_instances)>=5,'too few successful writes after stop')
|
||||
check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped')
|
||||
print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False))
|
||||
check(proc.poll() == 0, "stop-one hook did not finish successfully")
|
||||
check(
|
||||
first_recovery is not None and first_recovery <= 10,
|
||||
"writes did not recover within 10 seconds from hook start",
|
||||
)
|
||||
check(len(after_instances) >= 5, "too few successful writes after stop")
|
||||
check(
|
||||
target not in after_instances[-5:],
|
||||
"target replica still serves requests; hook must leave it stopped",
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"observed_instances_before": len(observed),
|
||||
"successful_writes": len(successes),
|
||||
"failed_attempts": failures,
|
||||
"first_recovery_seconds": round(first_recovery, 3),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
# Existing token, no re-login. Read all confirmations and reject duplicates.
|
||||
messages = history(c,bob,chat)
|
||||
ids = [m['id'] for m in messages]
|
||||
check(len(ids)==len(set(ids)),'duplicated message IDs in history')
|
||||
check(any(m==original for m in messages),'confirmed pre-failure message lost or altered')
|
||||
messages = history(c, bob, chat)
|
||||
ids = [m["id"] for m in messages]
|
||||
check(len(ids) == len(set(ids)), "duplicated message IDs in history")
|
||||
check(
|
||||
any(m == original for m in messages),
|
||||
"confirmed pre-failure message lost or altered",
|
||||
)
|
||||
for msg in successes:
|
||||
check(any(m==msg for m in messages),'confirmed in-flight message lost or altered')
|
||||
check(
|
||||
any(m == msg for m in messages),
|
||||
"confirmed in-flight message lost or altered",
|
||||
)
|
||||
# Bodies are unique in this scenario: an uncertain request may exist once, never twice.
|
||||
texts = [m['text'] for m in messages]
|
||||
check(len(texts)==len(set(texts)),'uncertain retry created duplicate message')
|
||||
texts = [m["text"] for m in messages]
|
||||
check(
|
||||
len(texts) == len(set(texts)), "uncertain retry created duplicate message"
|
||||
)
|
||||
if attachment:
|
||||
c.call('GET','/api/v1/attachments/'+attachment['id'],actor=bob,schema='Attachment')
|
||||
downloaded = c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob)
|
||||
check(downloaded['bytes']==content,'attachment lost or changed across failure')
|
||||
print('PASS: '+args.mode+' preserved confirmed data and existing session. Hook scope needs operator evidence.')
|
||||
c.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"],
|
||||
actor=bob,
|
||||
schema="Attachment",
|
||||
)
|
||||
downloaded = c.call(
|
||||
"GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob
|
||||
)
|
||||
check(
|
||||
downloaded["bytes"] == content,
|
||||
"attachment lost or changed across failure",
|
||||
)
|
||||
print(
|
||||
"PASS: "
|
||||
+ args.mode
|
||||
+ " preserved confirmed data and existing session. Hook scope needs operator evidence."
|
||||
)
|
||||
finally:
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
@@ -125,5 +202,6 @@ def main():
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
|
||||
if __name__=='__main__':
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
|
||||
+159
-69
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One complete public acceptance scenario, cumulative by lab number."""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import uuid
|
||||
@@ -7,76 +8,165 @@ from client import Client, LAB, arguments, check, run, timestamp
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Messenger public smoke; creates synthetic users and messages')
|
||||
c = Client(args.base_url,args.ca_file)
|
||||
alice,bob,eve,chat = c.bootstrap()
|
||||
path = '/api/v1/chats/'+chat['id']
|
||||
c.call('POST','/api/v1/users',409,body=alice['registration'])
|
||||
c.call('GET','/api/v1/users/me',401)
|
||||
c.call('GET',path,404,actor=eve)
|
||||
me = c.call('GET','/api/v1/users/me',actor=alice,schema='User')['json']
|
||||
check(me==alice['user'],'current user mismatch')
|
||||
listed = c.call('GET','/api/v1/chats?limit=100',actor=bob,schema='ChatPage')['json']
|
||||
check(any(x['id']==chat['id'] for x in listed['items']),'member cannot list chat')
|
||||
hidden = c.call('GET','/api/v1/chats?limit=100',actor=eve,schema='ChatPage')['json']
|
||||
check(all(x['id']!=chat['id'] for x in hidden['items']),'chat leaks to outsider')
|
||||
args = arguments("Messenger public smoke; creates synthetic users and messages")
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = "/api/v1/chats/" + chat["id"]
|
||||
c.call("POST", "/api/v1/users", 409, body=alice["registration"])
|
||||
c.call("GET", "/api/v1/users/me", 401)
|
||||
c.call("GET", path, 404, actor=eve)
|
||||
me = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"]
|
||||
check(me == alice["user"], "current user mismatch")
|
||||
listed = c.call("GET", "/api/v1/chats?limit=100", actor=bob, schema="ChatPage")[
|
||||
"json"
|
||||
]
|
||||
check(
|
||||
any(x["id"] == chat["id"] for x in listed["items"]), "member cannot list chat"
|
||||
)
|
||||
hidden = c.call("GET", "/api/v1/chats?limit=100", actor=eve, schema="ChatPage")[
|
||||
"json"
|
||||
]
|
||||
check(all(x["id"] != chat["id"] for x in hidden["items"]), "chat leaks to outsider")
|
||||
key = uuid.uuid4().hex
|
||||
first = c.message(alice,chat,'Привет, мир 👋',key=key)['json']
|
||||
second = c.message(bob,chat,'Сообщение Боба')['json']
|
||||
check(first['sender_id']==alice['id'] and second['sender_id']==bob['id'],'sender was not derived from actor')
|
||||
check(first['chat_id']==second['chat_id']==chat['id'],'wrong chat in message')
|
||||
check(first['text']=='Привет, мир 👋' and second['text']=='Сообщение Боба','message text changed')
|
||||
page1 = c.call('GET',path+'/messages?limit=1',actor=bob,schema='MessagePage')['json']
|
||||
check(len(page1['items'])==1 and page1['next_cursor'],'first page must have one item and cursor')
|
||||
first = c.message(alice, chat, "Привет, мир 👋", key=key)["json"]
|
||||
second = c.message(bob, chat, "Сообщение Боба")["json"]
|
||||
check(
|
||||
first["sender_id"] == alice["id"] and second["sender_id"] == bob["id"],
|
||||
"sender was not derived from actor",
|
||||
)
|
||||
check(first["chat_id"] == second["chat_id"] == chat["id"], "wrong chat in message")
|
||||
check(
|
||||
first["text"] == "Привет, мир 👋" and second["text"] == "Сообщение Боба",
|
||||
"message text changed",
|
||||
)
|
||||
page1 = c.call("GET", path + "/messages?limit=1", actor=bob, schema="MessagePage")[
|
||||
"json"
|
||||
]
|
||||
check(
|
||||
len(page1["items"]) == 1 and page1["next_cursor"],
|
||||
"first page must have one item and cursor",
|
||||
)
|
||||
from urllib.parse import quote
|
||||
page2 = c.call('GET',path+'/messages?limit=1&cursor='+quote(page1['next_cursor'],safe=''),actor=alice,schema='MessagePage')['json']
|
||||
check(len(page2['items'])==1 and page2['next_cursor'] is None,'second page must finish history')
|
||||
ordered = sorted([first,second],key=lambda x:(timestamp(x['created_at']),uuid.UUID(x['id']).int))
|
||||
check(page1['items']+page2['items']==ordered,'pagination lost, duplicated or reordered a message')
|
||||
h = {'Idempotency-Key':uuid.uuid4().hex} if LAB>=3 else {}
|
||||
c.call('POST',path+'/messages',400,actor=alice,body={'text':''},headers=h)
|
||||
c.call('POST',path+'/messages',404,actor=eve,body={'text':'чужое'},headers=h)
|
||||
c.call('GET',path+'/messages?limit=0',400,actor=alice)
|
||||
if LAB>=2:
|
||||
bad = base64.b64encode((alice['registration']['username']+':wrong-password').encode()).decode()
|
||||
c.call('POST','/api/v1/auth/sessions',401,headers={'Authorization':'Basic '+bad})
|
||||
good = base64.b64encode((alice['registration']['username']+':'+alice['registration']['password']).encode()).decode()
|
||||
c.call('GET','/api/v1/users/me',401,headers={'Authorization':'Basic '+good})
|
||||
c.call('GET','/api/v1/users/me',401,headers={'X-User-Id':alice['id']})
|
||||
me = c.call('GET','/api/v1/users/me',actor=alice,headers={'X-User-Id':eve['id']},schema='User')['json']
|
||||
check(me['id']==alice['id'],'X-User-Id overrode bearer identity')
|
||||
if LAB>=3:
|
||||
replay = c.message(alice,chat,first['text'],key=key)['json']
|
||||
check(replay==first,'idempotency replay differs')
|
||||
c.call('POST',path+'/messages',409,actor=alice,body={'text':'другой body'},headers={'Idempotency-Key':key})
|
||||
c.call('POST',path+'/messages',400,actor=alice,body={'text':'без ключа'})
|
||||
if LAB>=4:
|
||||
att,content = c.attachment(alice,chat)
|
||||
# A queued attachment must already be linkable to its own chat.
|
||||
attached = c.message(alice,chat,'',attachments=[att['id']])['json']
|
||||
check(attached['attachment_ids']==[att['id']],'message lost attachment')
|
||||
ready = c.wait_ready(bob,att) if LAB>=5 else att
|
||||
check(ready['error_code'] is None,'ready attachment has an error')
|
||||
download = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
|
||||
check(download['bytes']==content,'download differs from original')
|
||||
check(download['headers'].get_content_type()=='image/png','original content type mismatch')
|
||||
for suffix in ('','/content'):
|
||||
c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=eve)
|
||||
if LAB>=5:
|
||||
variants = [x for x in ready['variants'] if x['name']=='thumbnail']
|
||||
check(len(variants)==1,'expected exactly one thumbnail')
|
||||
v = variants[0]
|
||||
check((v['width'],v['height'])==(256,160),'thumbnail metadata has wrong dimensions')
|
||||
thumb = c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant=thumbnail',actor=bob)
|
||||
check(thumb['headers'].get_content_type()=='image/jpeg','thumbnail content type mismatch')
|
||||
check(len(thumb['bytes'])==v['size_bytes'] and hashlib.sha256(thumb['bytes']).hexdigest()==v['sha256'],'thumbnail checksum mismatch')
|
||||
from image_probe import jpeg_size
|
||||
check(jpeg_size(thumb['bytes'])==(256,160),'actual JPEG dimensions mismatch')
|
||||
if LAB>=2:
|
||||
c.call('DELETE','/api/v1/auth/sessions/current',204,actor=alice)
|
||||
c.call('GET','/api/v1/users/me',401,actor=alice)
|
||||
c.call('DELETE','/api/v1/auth/sessions/current',401,actor=alice)
|
||||
print('PASS: lab%d public API scenario; infrastructure and grading evidence remain separate.' % LAB)
|
||||
|
||||
if __name__=='__main__':
|
||||
page2 = c.call(
|
||||
"GET",
|
||||
path + "/messages?limit=1&cursor=" + quote(page1["next_cursor"], safe=""),
|
||||
actor=alice,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
check(
|
||||
len(page2["items"]) == 1 and page2["next_cursor"] is None,
|
||||
"second page must finish history",
|
||||
)
|
||||
ordered = sorted(
|
||||
[first, second],
|
||||
key=lambda x: (timestamp(x["created_at"]), uuid.UUID(x["id"]).int),
|
||||
)
|
||||
check(
|
||||
page1["items"] + page2["items"] == ordered,
|
||||
"pagination lost, duplicated or reordered a message",
|
||||
)
|
||||
h = {"Idempotency-Key": uuid.uuid4().hex} if LAB >= 3 else {}
|
||||
c.call("POST", path + "/messages", 400, actor=alice, body={"text": ""}, headers=h)
|
||||
c.call(
|
||||
"POST", path + "/messages", 404, actor=eve, body={"text": "чужое"}, headers=h
|
||||
)
|
||||
c.call("GET", path + "/messages?limit=0", 400, actor=alice)
|
||||
if LAB >= 2:
|
||||
bad = base64.b64encode(
|
||||
(alice["registration"]["username"] + ":wrong-password").encode()
|
||||
).decode()
|
||||
c.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
401,
|
||||
headers={"Authorization": "Basic " + bad},
|
||||
)
|
||||
good = base64.b64encode(
|
||||
(
|
||||
alice["registration"]["username"]
|
||||
+ ":"
|
||||
+ alice["registration"]["password"]
|
||||
).encode()
|
||||
).decode()
|
||||
c.call(
|
||||
"GET", "/api/v1/users/me", 401, headers={"Authorization": "Basic " + good}
|
||||
)
|
||||
c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]})
|
||||
me = c.call(
|
||||
"GET",
|
||||
"/api/v1/users/me",
|
||||
actor=alice,
|
||||
headers={"X-User-Id": eve["id"]},
|
||||
schema="User",
|
||||
)["json"]
|
||||
check(me["id"] == alice["id"], "X-User-Id overrode bearer identity")
|
||||
if LAB >= 3:
|
||||
replay = c.message(alice, chat, first["text"], key=key)["json"]
|
||||
check(replay == first, "idempotency replay differs")
|
||||
c.call(
|
||||
"POST",
|
||||
path + "/messages",
|
||||
409,
|
||||
actor=alice,
|
||||
body={"text": "другой body"},
|
||||
headers={"Idempotency-Key": key},
|
||||
)
|
||||
c.call("POST", path + "/messages", 400, actor=alice, body={"text": "без ключа"})
|
||||
if LAB >= 4:
|
||||
att, content = c.attachment(alice, chat)
|
||||
# A queued attachment must already be linkable to its own chat.
|
||||
attached = c.message(alice, chat, "", attachments=[att["id"]])["json"]
|
||||
check(attached["attachment_ids"] == [att["id"]], "message lost attachment")
|
||||
ready = c.wait_ready(bob, att) if LAB >= 5 else att
|
||||
check(ready["error_code"] is None, "ready attachment has an error")
|
||||
download = c.call(
|
||||
"GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob
|
||||
)
|
||||
check(download["bytes"] == content, "download differs from original")
|
||||
check(
|
||||
download["headers"].get_content_type() == "image/png",
|
||||
"original content type mismatch",
|
||||
)
|
||||
for suffix in ("", "/content"):
|
||||
c.call("GET", "/api/v1/attachments/" + att["id"] + suffix, 404, actor=eve)
|
||||
if LAB >= 5:
|
||||
variants = [x for x in ready["variants"] if x["name"] == "thumbnail"]
|
||||
check(len(variants) == 1, "expected exactly one thumbnail")
|
||||
v = variants[0]
|
||||
check(
|
||||
(v["width"], v["height"]) == (256, 160),
|
||||
"thumbnail metadata has wrong dimensions",
|
||||
)
|
||||
thumb = c.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail",
|
||||
actor=bob,
|
||||
)
|
||||
check(
|
||||
thumb["headers"].get_content_type() == "image/jpeg",
|
||||
"thumbnail content type mismatch",
|
||||
)
|
||||
check(
|
||||
len(thumb["bytes"]) == v["size_bytes"]
|
||||
and hashlib.sha256(thumb["bytes"]).hexdigest() == v["sha256"],
|
||||
"thumbnail checksum mismatch",
|
||||
)
|
||||
from image_probe import jpeg_size
|
||||
|
||||
check(
|
||||
jpeg_size(thumb["bytes"]) == (256, 160),
|
||||
"actual JPEG dimensions mismatch",
|
||||
)
|
||||
if LAB >= 2:
|
||||
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice)
|
||||
c.call("GET", "/api/v1/users/me", 401, actor=alice)
|
||||
c.call("DELETE", "/api/v1/auth/sessions/current", 401, actor=alice)
|
||||
print(
|
||||
"PASS: lab%d public API scenario; infrastructure and grading evidence remain separate."
|
||||
% LAB
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
|
||||
Reference in New Issue
Block a user