formatted project by python-black

This commit is contained in:
2026-11-10 09:52:42 +03:00
parent 7cd02da51d
commit 9899ee4676
66 changed files with 2517 additions and 944 deletions
+8 -1
View File
@@ -8,7 +8,14 @@ from aio_pika.exceptions import AMQPException, ChannelInvalidStateError
from sqlalchemy.exc import InterfaceError, OperationalError
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
TRANSIENT_JOB_ERRORS = (AMQPException, ChannelInvalidStateError, InterfaceError, OperationalError, PoolTimeoutError, OSError)
TRANSIENT_JOB_ERRORS = (
AMQPException,
ChannelInvalidStateError,
InterfaceError,
OperationalError,
PoolTimeoutError,
OSError,
)
logger = logging.getLogger(__name__)
+3 -1
View File
@@ -21,7 +21,9 @@ async def cleanup_once(factory, storage, apply: bool, jobs: JobSettings) -> bool
async with factory() as session:
result = await reconcile_attachments(session, storage, dry_run=not apply)
staging = await cleanup_staging(session, storage, jobs, apply=apply)
print(json.dumps({**asdict(result), "terminal_staging": staging}), flush=True)
print(
json.dumps({**asdict(result), "terminal_staging": staging}), flush=True
)
except RECOVERABLE_STORAGE_ERRORS as error:
log_recoverable_error("Reconciliation", error)
completed = False
+3 -1
View File
@@ -5,7 +5,9 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
class ImageSettings(BaseSettings):
model_config = SettingsConfigDict(env_prefix="IMAGE_", env_file=".env", extra="ignore")
model_config = SettingsConfigDict(
env_prefix="IMAGE_", env_file=".env", extra="ignore"
)
jpeg_quality: int = Field(default=85, ge=1, le=95)
webp_quality: int = Field(default=80, ge=1, le=100)
+22 -5
View File
@@ -9,7 +9,13 @@ from app.core.background import log_background_error
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
from app.domain.job_admin import REPLAY_REASONS, history, list_failed, replay_failed_job, show_job
from app.domain.job_admin import (
REPLAY_REASONS,
history,
list_failed,
replay_failed_job,
show_job,
)
from app.errors.exceptions import JobOperationError
from app.storage.s3 import create_s3_storage
@@ -29,8 +35,15 @@ async def execute(args):
result = await history(session, args.job_id)
else:
storage = create_s3_storage(settings.s3)
result = await replay_failed_job(session, storage, JobSettings(), args.job_id,
request_id=args.request_id, actor=args.actor, reason=args.reason)
result = await replay_failed_job(
session,
storage,
JobSettings(),
args.job_id,
request_id=args.request_id,
actor=args.actor,
reason=args.reason,
)
print(json.dumps(result, default=str), flush=True)
finally:
if storage is not None:
@@ -39,10 +52,14 @@ async def execute(args):
def main():
parser = argparse.ArgumentParser(description="Реестр failed и явный replay с аудитом")
parser = argparse.ArgumentParser(
description="Реестр failed и явный replay с аудитом"
)
commands = parser.add_subparsers(dest="command", required=True)
listing = commands.add_parser("list-failed")
listing.add_argument("--limit", type=int, choices=range(1, 501), default=50, metavar="1..500")
listing.add_argument(
"--limit", type=int, choices=range(1, 501), default=50, metavar="1..500"
)
for name in ("show", "history", "replay"):
command = commands.add_parser(name)
command.add_argument("job_id", type=UUID)
+42 -9
View File
@@ -12,7 +12,12 @@ from uuid import UUID, uuid4
from PIL import features
from pydantic import ValidationError
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
from app.core.background import (
TRANSIENT_JOB_ERRORS,
log_background_error,
shutdown_event,
wait_or_stop,
)
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
@@ -34,7 +39,10 @@ class Faults:
object_name: str = "optimized"
def kill(self, attachment_id: UUID, phase: str) -> None:
print(json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), flush=True)
print(
json.dumps({"fault": phase, "attachment_id": str(attachment_id)}),
flush=True,
)
os.kill(os.getpid(), signal.SIGKILL)
def after_object(self, attachment_id: UUID, name: str) -> None:
@@ -52,7 +60,14 @@ class Faults:
raise ProcessingError("injected_permanent")
async def consume_connected(factory, storage, broker: JobBroker, settings: JobSettings, stop: asyncio.Event, faults: Faults) -> None:
async def consume_connected(
factory,
storage,
broker: JobBroker,
settings: JobSettings,
stop: asyncio.Event,
faults: Faults,
) -> None:
consumer = uuid4().hex
while not stop.is_set():
try:
@@ -68,8 +83,12 @@ async def consume_connected(factory, storage, broker: JobBroker, settings: JobSe
try:
async with factory() as session:
acknowledge = await process_attachment_job(
session, storage, message, settings,
after_object=faults.after_object, before_processing=faults.before_processing,
session,
storage,
message,
settings,
after_object=faults.after_object,
before_processing=faults.before_processing,
)
except ProcessingError:
await broker.reject_invalid(delivery.id)
@@ -107,7 +126,10 @@ async def run_worker(faults: Faults) -> None:
storage = create_s3_storage(settings.s3)
factory = create_session_factory(engine)
async with asyncio.TaskGroup() as group:
tasks = [group.create_task(consume(factory, storage, jobs, stop, faults)) for _ in range(jobs.worker_concurrency)]
tasks = [
group.create_task(consume(factory, storage, jobs, stop, faults))
for _ in range(jobs.worker_concurrency)
]
await stop.wait()
# Отмена убивает resize-процесс, дожидается SDK и не отправляет ack.
for task in tasks:
@@ -119,11 +141,22 @@ async def run_worker(faults: Faults) -> None:
def main() -> None:
parser = argparse.ArgumentParser(description="Image worker с явными fault-injection hooks")
parser = argparse.ArgumentParser(
description="Image worker с явными fault-injection hooks"
)
modes = parser.add_mutually_exclusive_group()
for flag in ("crash-after-object", "crash-before-ack", "fail-transient", "fail-permanent"):
for flag in (
"crash-after-object",
"crash-before-ack",
"fail-transient",
"fail-permanent",
):
modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID")
parser.add_argument("--object-name", choices=("original", "thumbnail", "crop", "optimized"), default="optimized")
parser.add_argument(
"--object-name",
choices=("original", "thumbnail", "crop", "optimized"),
default="optimized",
)
args = parser.parse_args()
faults = Faults(**vars(args))
try:
+3 -1
View File
@@ -29,7 +29,9 @@ class JobSettings(BaseSettings):
@model_validator(mode="after")
def validate_lease(self) -> Self:
if self.job_lease_seconds < self.job_timeout_seconds + 30:
raise ValueError("Lease должен превышать timeout задания минимум на 30 секунд")
raise ValueError(
"Lease должен превышать timeout задания минимум на 30 секунд"
)
if self.job_retry_max_seconds < self.job_retry_seconds:
raise ValueError("Retry maximum must be at least the base delay")
return self
+24 -6
View File
@@ -8,7 +8,12 @@ import os
import signal
from uuid import UUID
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
from app.core.background import (
TRANSIENT_JOB_ERRORS,
log_background_error,
shutdown_event,
wait_or_stop,
)
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
@@ -18,7 +23,10 @@ from app.storage.broker import JobBroker
def crash_before_publish(attachment_id: UUID) -> None:
"""Реальный SIGKILL после чтения принятой job, до publisher confirm и отметки publish."""
print(json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}), flush=True)
print(
json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}),
flush=True,
)
os.kill(os.getpid(), signal.SIGKILL)
@@ -39,8 +47,14 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
async with factory() as session:
await recover_expired_jobs(session, jobs)
published = await publish_next_job(
session, broker, attachment_id=crash_attachment_id,
before_publish=crash_before_publish if crash_attachment_id is not None else None,
session,
broker,
attachment_id=crash_attachment_id,
before_publish=(
crash_before_publish
if crash_attachment_id is not None
else None
),
)
except TRANSIENT_JOB_ERRORS as error:
log_background_error("Publisher deferred", error)
@@ -53,9 +67,13 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
def main() -> None:
parser = argparse.ArgumentParser(description="Публикация устойчивых jobs в RabbitMQ")
parser = argparse.ArgumentParser(
description="Публикация устойчивых jobs в RabbitMQ"
)
parser.add_argument(
"--crash-before-publish", type=UUID, metavar="ATTACHMENT_ID",
"--crash-before-publish",
type=UUID,
metavar="ATTACHMENT_ID",
help="аварийно завершить этот процесс перед публикацией только указанного вложения",
)
args = parser.parse_args()
+1 -2
View File
@@ -4,6 +4,7 @@ from redis.backoff import NoBackoff
from app.core.config import Settings
def create_redis_client(settings: Settings) -> Redis:
password = (
settings.redis_password.get_secret_value() if settings.redis_password else None
@@ -21,5 +22,3 @@ def create_redis_client(settings: Settings) -> Redis:
decode_responses=True,
)
return Redis.from_pool(pool)
+16 -4
View File
@@ -6,6 +6,7 @@ from urllib.parse import urlsplit
from pydantic import Field, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
class S3Settings(BaseSettings):
endpoint: str = "http://minio:9000"
region: str = "us-east-1"
@@ -24,12 +25,23 @@ class S3Settings(BaseSettings):
cleanup_grace_seconds: int = Field(default=86_400, ge=300)
cleanup_interval_seconds: int = Field(default=300, ge=1)
model_config = SettingsConfigDict(env_prefix="S3_", env_file=".env", extra="ignore", frozen=True)
model_config = SettingsConfigDict(
env_prefix="S3_", env_file=".env", extra="ignore", frozen=True
)
@model_validator(mode="after")
def validate_endpoint(self):
url = urlsplit(self.endpoint)
if (url.scheme not in {"http", "https"} or not url.hostname or url.username
or url.password or url.path not in {"", "/"} or url.query or url.fragment):
raise ValueError("S3_ENDPOINT должен быть http(s) адресом без credentials/пути")
if (
url.scheme not in {"http", "https"}
or not url.hostname
or url.username
or url.password
or url.path not in {"", "/"}
or url.query
or url.fragment
):
raise ValueError(
"S3_ENDPOINT должен быть http(s) адресом без credentials/пути"
)
return self
+6 -2
View File
@@ -13,11 +13,15 @@ def main() -> int:
settings = ImageSettings()
memory = settings.memory_mib * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
resource.setrlimit(resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds))
resource.setrlimit(
resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds)
)
source, directory, metadata, content_type, maximum = sys.argv[1:]
code = 0
try:
result = create_variants(Path(source), Path(directory), content_type, int(maximum), settings)
result = create_variants(
Path(source), Path(directory), content_type, int(maximum), settings
)
Path(metadata).write_text(result.model_dump_json())
except IMAGE_WORKER_ERRORS as error:
code = pipeline_worker_error_code(error)
+95 -28
View File
@@ -29,13 +29,17 @@ class Claim:
acknowledge: bool = False
async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID:
async def enqueue_attachment_job(
session: AsyncSession, attachment_id: UUID, staging_key: str
) -> UUID:
"""Включить намерение обработки в транзакцию queued metadata; без commit."""
return await jobs.create_job(session, attachment_id, staging_key)
async def publish_next_job(
session: AsyncSession, broker: JobBroker, *,
session: AsyncSession,
broker: JobBroker,
*,
attachment_id: UUID | None = None,
before_publish: Callable[[UUID], None] | None = None,
) -> bool:
@@ -45,24 +49,33 @@ async def publish_next_job(
if job is not None:
if before_publish is not None:
before_publish(job.attachment_id)
await broker.publish_job(JobMessage(
job_id=job.id, attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version, attempt=job.attempt + 1,
created_at=job.created_at,
))
await broker.publish_job(
JobMessage(
job_id=job.id,
attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version,
attempt=job.attempt + 1,
created_at=job.created_at,
)
)
outbox.mark_published(job)
await session.commit()
return published
async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSettings) -> Claim:
async def claim_job(
session: AsyncSession, message: JobMessage, settings: JobSettings
) -> Claim:
"""Дубликат не захватывает живую аренду и не откатывает terminal-состояние."""
row = await jobs.lock_job(session, message.job_id)
result = Claim()
if row is None:
raise ProcessingError("unknown_job")
job, attachment = row
if message.attachment_id != attachment.id or message.pipeline_version != job.pipeline_version:
if (
message.attachment_id != attachment.id
or message.pipeline_version != job.pipeline_version
):
raise ProcessingError("invalid_job")
now = await jobs.database_time(session)
if job.attempt_limit is None:
@@ -77,16 +90,32 @@ async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSet
elif message.attempt > job.attempt + 1:
raise ProcessingError("invalid_attempt")
elif attachment.status == "queued" and job.next_attempt_at <= now:
if job.attempt >= (job.attempt_limit or settings.job_max_attempts) or job.staging_key is None:
jobs.record_failure(job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0)
if (
job.attempt >= (job.attempt_limit or settings.job_max_attempts)
or job.staging_key is None
):
jobs.record_failure(
job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0
)
result = Claim(acknowledge=True)
else:
result = Claim(work=jobs.start_job(job, attachment, uuid4(), now, settings.job_lease_seconds))
result = Claim(
work=jobs.start_job(
job, attachment, uuid4(), now, settings.job_lease_seconds
)
)
await session.commit()
return result
async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettings, code: str, *, permanent: bool) -> None:
async def save_failure(
session: AsyncSession,
work: JobWork,
settings: JobSettings,
code: str,
*,
permanent: bool,
) -> None:
# Неоднозначный COMMIT проверяется повторным чтением: ready сохраняется.
await session.rollback()
row = await jobs.lock_job(session, work.job_id)
@@ -95,10 +124,14 @@ async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettin
if attachment.status == "processing" and job.lease_token == work.lease_token:
exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
jobs.record_failure(
job, attachment, await jobs.database_time(session),
job,
attachment,
await jobs.database_time(session),
"retry_exhausted" if exhausted and not permanent else code,
terminal=permanent or exhausted,
retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
retry_seconds=retry_delay(
settings, job.attempt - job.cycle_start_attempt
),
)
job.last_error_code = code
job.last_error_permanent = permanent
@@ -106,7 +139,9 @@ async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettin
async def build_result(
session: AsyncSession, storage: MinioStorage, work: JobWork,
session: AsyncSession,
storage: MinioStorage,
work: JobWork,
after_object: Callable[[UUID, str], None] | None = None,
) -> bool:
"""Фиксированные ключи и блокировка job защищают PUT от устаревшей попытки."""
@@ -129,26 +164,42 @@ async def build_result(
# Блокировка остаётся до завершения PUT/COMMIT; reaper использует SKIP LOCKED.
# При отмене S3 adapter дожидается SDK-потока до освобождения блокировки.
await storage.put_file(
work.original_key, source, size_bytes=work.size_bytes,
content_type=work.content_type, sha256=work.sha256,
work.original_key,
source,
size_bytes=work.size_bytes,
content_type=work.content_type,
sha256=work.sha256,
)
if after_object is not None:
after_object(work.attachment_id, "original")
for variant in result.variants:
await storage.put_file(
work.variant_key(variant.name), Path(directory) / VARIANT_FILENAMES[variant.name],
size_bytes=variant.size_bytes, content_type=variant.content_type, sha256=variant.sha256,
work.variant_key(variant.name),
Path(directory) / VARIANT_FILENAMES[variant.name],
size_bytes=variant.size_bytes,
content_type=variant.content_type,
sha256=variant.sha256,
)
if after_object is not None:
after_object(work.attachment_id, variant.name)
jobs.complete_job(job, attachment, work, [variant.model_dump() for variant in result.variants], await jobs.database_time(session))
jobs.complete_job(
job,
attachment,
work,
[variant.model_dump() for variant in result.variants],
await jobs.database_time(session),
)
await session.commit()
return True
async def process_attachment_job(
session: AsyncSession, storage: MinioStorage, message: JobMessage, settings: JobSettings,
*, after_object: Callable[[UUID, str], None] | None = None,
session: AsyncSession,
storage: MinioStorage,
message: JobMessage,
settings: JobSettings,
*,
after_object: Callable[[UUID, str], None] | None = None,
before_processing: Callable[[UUID], None] | None = None,
) -> bool:
claim = await claim_job(session, message, settings)
@@ -160,7 +211,13 @@ async def process_attachment_job(
if before_processing is not None:
before_processing(work.attachment_id)
acknowledge = await build_result(session, storage, work, after_object)
except (ProcessingError, MinioException, StorageHTTPError, OSError, TimeoutError) as error:
except (
ProcessingError,
MinioException,
StorageHTTPError,
OSError,
TimeoutError,
) as error:
code, permanent = classify_failure(error)
await save_failure(session, work, settings, code, permanent=permanent)
acknowledge = True
@@ -176,8 +233,12 @@ async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) ->
job.attempt_limit = settings.job_max_attempts
exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
jobs.record_failure(
job, attachment, now, "retry_exhausted" if exhausted else "worker_timeout",
terminal=exhausted, retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
job,
attachment,
now,
"retry_exhausted" if exhausted else "worker_timeout",
terminal=exhausted,
retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
)
job.last_error_code = "worker_timeout"
job.last_error_permanent = False
@@ -185,8 +246,14 @@ async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) ->
return len(rows)
async def cleanup_staging(session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool) -> int:
rows = await jobs.terminal_staging(session, settings.staging_retention_seconds, settings.failed_staging_retention_seconds)
async def cleanup_staging(
session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool
) -> int:
rows = await jobs.terminal_staging(
session,
settings.staging_retention_seconds,
settings.failed_staging_retention_seconds,
)
if apply:
for job, attachment in rows:
if job.staging_key != attachment.object_key:
+11 -3
View File
@@ -276,12 +276,20 @@ async def download_attachment(
key = record.object_key
info = DownloadInfo(result.filename, result.content_type, result.size_bytes)
else:
selected = next((item for item in result.variants if item.name == variant), None)
keys = {"thumbnail": record.thumbnail_key, "crop": record.crop_key, "optimized": record.optimized_key}
selected = next(
(item for item in result.variants if item.name == variant), None
)
keys = {
"thumbnail": record.thumbnail_key,
"crop": record.crop_key,
"optimized": record.optimized_key,
}
key = keys.get(variant)
if selected is None or key is None:
raise not_found()
info = DownloadInfo(VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes)
info = DownloadInfo(
VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes
)
await session.rollback()
stored = await storage.stat_object(key)
if stored.size_bytes != info.size_bytes or stored.content_type != info.content_type:
+7 -4
View File
@@ -14,12 +14,15 @@ from app.storage.s3 import MinioStorage
def liveness() -> Health:
return Health(status='ok')
return Health(status="ok")
async def readiness(
engine: AsyncEngine, redis: Redis, storage: MinioStorage | None,
lifecycle: Lifecycle, timeout_seconds: float,
engine: AsyncEngine,
redis: Redis,
storage: MinioStorage | None,
lifecycle: Lifecycle,
timeout_seconds: float,
) -> Health:
if lifecycle.draining:
raise unavailable()
@@ -30,4 +33,4 @@ async def readiness(
dependencies = await asyncio.gather(*checks, return_exceptions=True)
if not all(result is True for result in dependencies):
raise unavailable()
return Health(status='ok')
return Health(status="ok")
+9 -3
View File
@@ -57,16 +57,22 @@ async def send_message_idempotent(
raise conflict("Idempotency-Key уже использован для другого сообщения")
message = await message_storage.get_message(session, previous.message_id)
else:
await validate_message_attachments(session, sender, chat_id, new_message.attachment_ids)
await validate_message_attachments(
session, sender, chat_id, new_message.attachment_ids
)
if not await message_storage.reserve_message_slot(session, max_messages):
raise unavailable("Достигнут лимит количества сообщений")
message = await message_storage.create_message(
session, chat_id=chat_id, sender_id=sender.id, text=new_message.text
)
await attachment_storage.bind_attachments(session, message.id, new_message.attachment_ids)
await attachment_storage.bind_attachments(
session, message.id, new_message.attachment_ids
)
await key_storage.save_result(session, scope, fingerprint, message.id)
links = await attachment_storage.message_attachment_ids(session, [message.id])
result = Message.model_validate(message).model_copy(update={"attachment_ids": links[message.id]})
result = Message.model_validate(message).model_copy(
update={"attachment_ids": links[message.id]}
)
await session.commit()
return result
+37 -11
View File
@@ -15,18 +15,27 @@ def fitted_size(size: tuple[int, int], maximum: int) -> tuple[int, int]:
longest = max(size)
result = size
if longest > maximum:
result = (max(1, size[0] * maximum // longest), max(1, size[1] * maximum // longest))
result = (
max(1, size[0] * maximum // longest),
max(1, size[1] * maximum // longest),
)
return result
def describe_variant(name: str, content_type: str, path: Path, size: tuple[int, int]) -> Variant:
def describe_variant(
name: str, content_type: str, path: Path, size: tuple[int, int]
) -> Variant:
checksum = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(65536), b""):
checksum.update(chunk)
return Variant(
name=name, content_type=content_type, size_bytes=path.stat().st_size,
width=size[0], height=size[1], sha256=checksum.hexdigest(),
name=name,
content_type=content_type,
size_bytes=path.stat().st_size,
width=size[0],
height=size[1],
sha256=checksum.hexdigest(),
)
@@ -47,32 +56,49 @@ def create_thumbnail(image: Image.Image, destination: Path, quality: int) -> Var
def create_crop(image: Image.Image, destination: Path, quality: int) -> Variant:
"""Центрированный JPEG 128×128; маленький исходник допускает увеличение."""
with ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) as crop:
with ImageOps.fit(
image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)
) as crop:
save_jpeg(crop, destination, quality)
return describe_variant("crop", "image/jpeg", destination, (128, 128))
def create_optimized(image: Image.Image, destination: Path, settings: ImageSettings) -> Variant:
def create_optimized(
image: Image.Image, destination: Path, settings: ImageSettings
) -> Variant:
"""WebP ≤1280×1280 без увеличения; alpha сохраняется, метаданные удаляются."""
size = fitted_size(image.size, 1280)
with image.resize(size, Image.Resampling.LANCZOS) as resized:
with Image.new("RGBA", size) as clean:
clean.paste(resized)
clean.save(
destination, format="WEBP", lossless=False,
quality=settings.webp_quality, method=settings.webp_method,
exif=b"", icc_profile=b"", xmp=b"",
destination,
format="WEBP",
lossless=False,
quality=settings.webp_quality,
method=settings.webp_method,
exif=b"",
icc_profile=b"",
xmp=b"",
)
return describe_variant("optimized", "image/webp", destination, size)
def create_variants(source: Path, directory: Path, content_type: str, max_pixels: int, settings: ImageSettings) -> PipelineResult:
def create_variants(
source: Path,
directory: Path,
content_type: str,
max_pixels: int,
settings: ImageSettings,
) -> PipelineResult:
decode_image(source, content_type, max_pixels)
with Image.open(source, formats=["PNG", "JPEG"]) as image:
ImageOps.exif_transpose(image, in_place=True)
with image.convert("RGBA") as pixels:
variants = [
create_thumbnail(pixels, directory / "thumbnail.jpg", settings.jpeg_quality),
create_thumbnail(
pixels, directory / "thumbnail.jpg", settings.jpeg_quality
),
create_crop(pixels, directory / "crop.jpg", settings.jpeg_quality),
create_optimized(pixels, directory / "optimized.webp", settings),
]
+71 -19
View File
@@ -17,15 +17,28 @@ REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review")
def snapshot(job, attachment):
return dict(job_id=str(job.id), attachment_id=str(attachment.id), status=attachment.status,
attempt=job.attempt, attempt_limit=job.attempt_limit, cycle_start_attempt=job.cycle_start_attempt,
error_code=attachment.error_code, last_error_code=job.last_error_code,
last_error_permanent=job.last_error_permanent, staging_retained=job.staging_key is not None,
next_attempt_at=job.next_attempt_at, lease_until=job.lease_until, finished_at=job.finished_at)
return dict(
job_id=str(job.id),
attachment_id=str(attachment.id),
status=attachment.status,
attempt=job.attempt,
attempt_limit=job.attempt_limit,
cycle_start_attempt=job.cycle_start_attempt,
error_code=attachment.error_code,
last_error_code=job.last_error_code,
last_error_permanent=job.last_error_permanent,
staging_retained=job.staging_key is not None,
next_attempt_at=job.next_attempt_at,
lease_until=job.lease_until,
finished_at=job.finished_at,
)
async def list_failed(session, limit):
return [snapshot(job, attachment) for job, attachment in await job_admin.failed_jobs(session, limit)]
return [
snapshot(job, attachment)
for job, attachment in await job_admin.failed_jobs(session, limit)
]
async def show_job(session, job_id):
@@ -38,20 +51,37 @@ async def show_job(session, job_id):
async def history(session, job_id):
return [dict(request_id=str(row.id), actor=row.actor, reason=row.reason,
previous_attempt=row.previous_attempt, attempt_limit=row.attempt_limit,
previous_error_code=row.previous_error_code, created_at=row.created_at)
for row in await job_admin.replay_history(session, job_id)]
return [
dict(
request_id=str(row.id),
actor=row.actor,
reason=row.reason,
previous_attempt=row.previous_attempt,
attempt_limit=row.attempt_limit,
previous_error_code=row.previous_error_code,
created_at=row.created_at,
)
for row in await job_admin.replay_history(session, job_id)
]
def same_request(record, job_id, actor, reason):
if record.job_id != job_id or record.actor != actor or record.reason != reason:
raise JobOperationError("request_conflict")
return {"request_id": str(record.id), "job_id": str(job_id), "already_applied": True}
return {
"request_id": str(record.id),
"job_id": str(job_id),
"already_applied": True,
}
async def replay_failed_job(session, storage, settings, job_id, *, request_id, actor, reason):
if re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None or reason not in REPLAY_REASONS:
async def replay_failed_job(
session, storage, settings, job_id, *, request_id, actor, reason
):
if (
re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None
or reason not in REPLAY_REASONS
):
raise JobOperationError("invalid_audit_fields")
row = await jobs.lock_job(session, job_id)
if row is None:
@@ -67,8 +97,16 @@ async def replay_failed_job(session, storage, settings, job_id, *, request_id, a
if job.staging_key is None:
raise JobOperationError("source_expired")
# Та же блокировка защищает исходник от cleanup до окончания проверки и COMMIT.
work = JobWork(job.id, attachment.id, uuid4(), job.pipeline_version, job.staging_key,
attachment.content_type, attachment.size_bytes, attachment.sha256)
work = JobWork(
job.id,
attachment.id,
uuid4(),
job.pipeline_version,
job.staging_key,
attachment.content_type,
attachment.size_bytes,
attachment.sha256,
)
try:
async with asyncio.timeout(settings.job_timeout_seconds):
with TemporaryDirectory(prefix="replay-source-") as directory:
@@ -81,13 +119,27 @@ async def replay_failed_job(session, storage, settings, job_id, *, request_id, a
raise
limit = job.attempt + settings.job_max_attempts
inserted = await job_admin.insert_replay(
session, id=request_id, job_id=job_id, actor=actor, reason=reason,
previous_attempt=job.attempt, attempt_limit=limit, previous_error_code=job.last_error_code,
session,
id=request_id,
job_id=job_id,
actor=actor,
reason=reason,
previous_attempt=job.attempt,
attempt_limit=limit,
previous_error_code=job.last_error_code,
)
if not inserted:
result = same_request(await job_admin.replay_record(session, request_id), job_id, actor, reason)
result = same_request(
await job_admin.replay_record(session, request_id), job_id, actor, reason
)
await session.rollback()
return result
job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit)
await session.commit()
return {"request_id": str(request_id), "job_id": str(job_id), "already_applied": False, "status": "queued", "attempt_limit": limit}
return {
"request_id": str(request_id),
"job_id": str(job_id),
"already_applied": False,
"status": "queued",
"attempt_limit": limit,
}
+16 -5
View File
@@ -14,7 +14,9 @@ from app.storage.jobs import JobWork
from app.storage.s3 import MinioStorage
async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path) -> None:
async def fetch_original(
storage: MinioStorage, work: JobWork, destination: Path
) -> None:
if work.size_bytes > storage.settings.max_upload_bytes:
raise ProcessingError("image_limits")
checksum = hashlib.sha256()
@@ -34,12 +36,21 @@ async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path
raise ProcessingError("source_mismatch")
async def render_variants(storage: MinioStorage, work: JobWork, source: Path, directory: Path) -> PipelineResult:
async def render_variants(
storage: MinioStorage, work: JobWork, source: Path, directory: Path
) -> PipelineResult:
metadata = directory / "variants.json"
process = await asyncio.create_subprocess_exec(
sys.executable, "-m", "app.core.thumbnail_worker", str(source), str(directory),
str(metadata), work.content_type, str(storage.settings.max_image_pixels),
stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
sys.executable,
"-m",
"app.core.thumbnail_worker",
str(source),
str(directory),
str(metadata),
work.content_type,
str(storage.settings.max_image_pixels),
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
try:
code = await process.wait()
+23 -4
View File
@@ -10,7 +10,10 @@ from app.errors.exceptions import ProcessingError
def retry_delay(settings: JobSettings, cycle_attempt: int) -> float:
ceiling = min(settings.job_retry_max_seconds, settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)))
ceiling = min(
settings.job_retry_max_seconds,
settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)),
)
return random.uniform(ceiling / 2, ceiling)
@@ -20,13 +23,29 @@ def classify_failure(error: Exception) -> tuple[str, bool]:
if isinstance(error, S3Error):
if error.code in {"NoSuchKey", "NoSuchObject"}:
return "source_missing", True
if error.code in {"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "NoSuchBucket"}:
if error.code in {
"AccessDenied",
"InvalidAccessKeyId",
"SignatureDoesNotMatch",
"NoSuchBucket",
}:
return "storage_configuration", True
if error.code in {"InternalError", "SlowDown", "ServiceUnavailable", "RequestTimeout", "Throttling"}:
if error.code in {
"InternalError",
"SlowDown",
"ServiceUnavailable",
"RequestTimeout",
"Throttling",
}:
return "storage_unavailable", False
return "storage_rejected", True
if isinstance(error, TimeoutError):
return "processing_timeout", False
if isinstance(error, OSError) and error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}:
if isinstance(error, OSError) and error.errno in {
errno.EACCES,
errno.EPERM,
errno.EROFS,
errno.ENOENT,
}:
return "processing_configuration", True
return "processing_unavailable", False
+9 -4
View File
@@ -32,10 +32,15 @@ async def list_messages(
rows = await messages_storage.list_messages(session, chat_id, after, limit + 1)
page_rows, has_next = split_page(rows, limit)
links = await attachments_storage.message_attachment_ids(session, [message.id for message in page_rows])
items = [Message.model_validate(message).model_copy(
update={"attachment_ids": links[message.id]},
) for message in page_rows]
links = await attachments_storage.message_attachment_ids(
session, [message.id for message in page_rows]
)
items = [
Message.model_validate(message).model_copy(
update={"attachment_ids": links[message.id]},
)
for message in page_rows
]
page = MessagePage(
items=items,
next_cursor=build_next_cursor(KIND_MESSAGES, chat_id, items, has_next),
+39 -17
View File
@@ -12,41 +12,63 @@ from app.domain import attachments
from app.endpoints.dependencies import CurrentUser, DbSession, S3Storage, UploadCapacity
from app.schemas.attachment import Attachment
router = APIRouter(prefix='/api/v1', tags=['attachments'])
router = APIRouter(prefix="/api/v1", tags=["attachments"])
@router.post('/chats/{chat_id}/attachments', status_code=202)
@router.post("/chats/{chat_id}/attachments", status_code=202)
async def upload(
request: Request, chat_id: UUID, user: CurrentUser, session: DbSession,
storage: S3Storage, capacity: UploadCapacity,
request: Request,
chat_id: UUID,
user: CurrentUser,
session: DbSession,
storage: S3Storage,
capacity: UploadCapacity,
filename: Annotated[str, Query(min_length=1, max_length=255)],
) -> Attachment:
return await attachments.receive_attachment(
session, storage, user, chat_id, capacity=capacity, filename=filename,
content_type=request.headers.get('content-type', '').lower(),
declared_length=request.headers.get('content-length'), chunks=request.stream(),
session,
storage,
user,
chat_id,
capacity=capacity,
filename=filename,
content_type=request.headers.get("content-type", "").lower(),
declared_length=request.headers.get("content-length"),
chunks=request.stream(),
)
@router.get('/attachments/{attachment_id}')
async def metadata(attachment_id: UUID, user: CurrentUser, session: DbSession) -> Attachment:
@router.get("/attachments/{attachment_id}")
async def metadata(
attachment_id: UUID, user: CurrentUser, session: DbSession
) -> Attachment:
return await attachments.get_attachment(session, user, attachment_id)
@router.get('/attachments/{attachment_id}/content')
@router.get("/attachments/{attachment_id}/content")
async def content(
attachment_id: UUID, user: CurrentUser, session: DbSession, storage: S3Storage,
variant: Annotated[str, Query()] = 'original',
attachment_id: UUID,
user: CurrentUser,
session: DbSession,
storage: S3Storage,
variant: Annotated[str, Query()] = "original",
) -> StreamingResponse:
record, stream = await attachments.download_attachment(
session, storage, user, attachment_id, variant=variant,
session,
storage,
user,
attachment_id,
variant=variant,
)
return StreamingResponse(
stream, media_type=record.content_type,
stream,
media_type=record.content_type,
headers={
'Content-Length': str(record.size_bytes),
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(record.filename, safe=''),
'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'private, no-store',
"Content-Length": str(record.size_bytes),
"Content-Disposition": "attachment; filename*=UTF-8''"
+ quote(record.filename, safe=""),
"X-Content-Type-Options": "nosniff",
"Cache-Control": "private, no-store",
},
background=BackgroundTask(stream.aclose),
)
+7 -4
View File
@@ -3,18 +3,21 @@ from fastapi import APIRouter, Request
from app.domain import health
from app.schemas.base import Health
router = APIRouter(prefix='/health', tags=['health'])
router = APIRouter(prefix="/health", tags=["health"])
@router.get('/live')
@router.get("/live")
async def liveness() -> Health:
return health.liveness()
@router.get('/ready')
@router.get("/ready")
async def readiness(request: Request) -> Health:
state = request.app.state
return await health.readiness(
state.engine, state.redis, state.s3, state.lifecycle,
state.engine,
state.redis,
state.s3,
state.lifecycle,
state.settings.readiness_timeout_seconds,
)
+6 -2
View File
@@ -21,9 +21,13 @@ async def send_new_message(
idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None,
) -> Message:
return await send_message_idempotent(
session, sender, chat_id, new_message,
session,
sender,
chat_id,
new_message,
validate_idempotency_key(idempotency_key),
settings.max_messages, settings.idempotency_ttl_seconds,
settings.max_messages,
settings.idempotency_ttl_seconds,
)
+12 -3
View File
@@ -90,14 +90,23 @@ def internal_error() -> ApiError:
IMAGE_WORKER_ERRORS = (
ApiError, Image.DecompressionBombError, Image.DecompressionBombWarning,
MemoryError, UnidentifiedImageError, OSError, ValueError, SyntaxError,
ApiError,
Image.DecompressionBombError,
Image.DecompressionBombWarning,
MemoryError,
UnidentifiedImageError,
OSError,
ValueError,
SyntaxError,
)
def image_worker_error_code(error: Exception) -> int:
code = 2
resource_error = isinstance(error, (Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError))
resource_error = isinstance(
error,
(Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError),
)
declared_limit = isinstance(error, ApiError) and error.status_code == 413
if resource_error or declared_limit:
code = 3
+17 -4
View File
@@ -18,14 +18,25 @@ from urllib3.exceptions import HTTPError as StorageHTTPError
from app.core.instance import INSTANCE_ID_HEADER
from app.core.request_id import REQUEST_ID_HEADER
from app.errors.exceptions import ApiError, internal_error, invalid_request, not_found, unavailable
from app.errors.exceptions import (
ApiError,
internal_error,
invalid_request,
not_found,
unavailable,
)
logger = logging.getLogger(__name__)
TRANSIENT_SQLSTATE_CLASSES = ("08", "40", "53", "57")
TRANSIENT_SQLSTATES = frozenset({"55P03"})
RECOVERABLE_STORAGE_ERRORS = (
ApiError, SQLAlchemyError, MinioException, StorageHTTPError, OSError, TimeoutError,
ApiError,
SQLAlchemyError,
MinioException,
StorageHTTPError,
OSError,
TimeoutError,
)
@@ -40,12 +51,14 @@ async def operation_timeout(seconds: float, error: ApiError):
async def client_disconnect_handler(request: Request, exc: Exception) -> JSONResponse:
return await api_error_handler(request, invalid_request("Загрузка прервана клиентом"))
return await api_error_handler(
request, invalid_request("Загрузка прервана клиентом")
)
def log_recoverable_error(operation: str, error: Exception) -> None:
"""Согласованный безопасный лог восстановления без содержимого SDK-ошибки."""
logger.warning('%s deferred: %s', operation, type(error).__name__)
logger.warning("%s deferred: %s", operation, type(error).__name__)
def _get_request_id(request: Request) -> str:
+3 -1
View File
@@ -52,7 +52,9 @@ def create_app(settings: Settings | None = None) -> FastAPI:
application.state.session_factory = create_session_factory(engine)
application.state.redis = redis
application.state.s3 = s3
application.state.upload_capacity = create_request_capacity(settings.s3.max_concurrent_uploads)
application.state.upload_capacity = create_request_capacity(
settings.s3.max_concurrent_uploads
)
application.state.request_capacity = create_request_capacity(settings.max_in_flight)
application.state.instance_id = resolve_instance_id(settings.instance_id)
application.state.lifecycle = Lifecycle()
+14 -1
View File
@@ -9,4 +9,17 @@ from app.models.idempotency_key import IdempotencyKey
from app.models.message import Message
from app.models.user import User
__all__ = ["JobReplay", "AttachmentJob", "Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"]
__all__ = [
"JobReplay",
"AttachmentJob",
"Base",
"Attachment",
"MessageAttachment",
"UploadIntent",
"Chat",
"ChatMember",
"Counter",
"IdempotencyKey",
"Message",
"User",
]
+10 -3
View File
@@ -18,8 +18,13 @@ class Attachment(Base):
"content_type IN ('image/png', 'image/jpeg')", name="content_type"
),
CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"),
CheckConstraint("status IN ('queued', 'processing', 'ready', 'failed')", name="status"),
CheckConstraint("(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", name="error_code"),
CheckConstraint(
"status IN ('queued', 'processing', 'ready', 'failed')", name="status"
),
CheckConstraint(
"(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)",
name="error_code",
),
Index("ix_attachments_chat_id", "chat_id"),
)
@@ -36,7 +41,9 @@ class Attachment(Base):
thumbnail_key: Mapped[str | None] = mapped_column(String(128))
crop_key: Mapped[str | None] = mapped_column(String(128))
optimized_key: Mapped[str | None] = mapped_column(String(128))
variants: Mapped[list[dict]] = mapped_column(JSONB, default=list, server_default=text("'[]'::jsonb"))
variants: Mapped[list[dict]] = mapped_column(
JSONB, default=list, server_default=text("'[]'::jsonb")
)
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+5 -2
View File
@@ -20,7 +20,8 @@ class AttachmentJob(Base):
id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4)
attachment_id: Mapped[UUID] = mapped_column(
ForeignKey("attachments.id"), unique=True,
ForeignKey("attachments.id"),
unique=True,
)
staging_key: Mapped[str | None] = mapped_column(String(128))
pipeline_version: Mapped[int] = mapped_column(server_default="1")
@@ -29,7 +30,9 @@ class AttachmentJob(Base):
attempt_limit: Mapped[int | None]
last_error_permanent: Mapped[bool | None]
published_attempt: Mapped[int] = mapped_column(server_default="0")
next_attempt_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
next_attempt_at: Mapped[datetime] = mapped_column(
server_default=func.clock_timestamp()
)
lease_token: Mapped[UUID | None]
lease_until: Mapped[datetime | None]
last_error_code: Mapped[str | None] = mapped_column(String(64))
+3 -1
View File
@@ -35,7 +35,9 @@ class Attachment(Base):
VARIANT_FILENAMES = {
"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp",
"thumbnail": "thumbnail.jpg",
"crop": "crop.jpg",
"optimized": "optimized.webp",
}
+3 -1
View File
@@ -16,5 +16,7 @@ class JobMessage(BaseModel):
pipeline_version: int = Field(default=1, ge=1)
attempt: int = Field(ge=1)
created_at: AwareDatetime
traceparent: str | None = Field(default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
traceparent: str | None = Field(
default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$"
)
tracestate: str | None = Field(default=None, max_length=512)
+23 -7
View File
@@ -28,26 +28,42 @@ class JobBroker:
logging.getLogger("aiormq").setLevel(logging.CRITICAL)
async def ensure_ready(self) -> None:
if self.connection is None or self.connection.is_closed or self.channel is None or self.channel.is_closed:
if (
self.connection is None
or self.connection.is_closed
or self.channel is None
or self.channel.is_closed
):
await self.aclose()
self.connection = await aio_pika.connect(
host=self.settings.broker_host, port=self.settings.broker_port,
host=self.settings.broker_host,
port=self.settings.broker_port,
login=self.settings.broker_user,
password=self.settings.broker_password_file.read_text().strip(),
virtualhost=self.settings.broker_vhost, timeout=5, heartbeat=30,
virtualhost=self.settings.broker_vhost,
timeout=5,
heartbeat=30,
)
self.channel = await self.connection.channel(
publisher_confirms=True, on_return_raises=True
)
self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True)
await self.channel.set_qos(prefetch_count=1)
self.queue = await self.channel.declare_queue(self.settings.broker_queue, durable=True)
self.queue = await self.channel.declare_queue(
self.settings.broker_queue, durable=True
)
async def publish_job(self, message: JobMessage) -> None:
await self.ensure_ready()
await self.channel.default_exchange.publish(
aio_pika.Message(
body=message.model_dump_json(exclude_none=True).encode(),
content_type="application/json", delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
content_type="application/json",
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
message_id=f"{message.job_id}:{message.attempt}",
), routing_key=self.settings.broker_queue, mandatory=True, timeout=10,
),
routing_key=self.settings.broker_queue,
mandatory=True,
timeout=10,
)
async def receive(self, consumer: str) -> Delivery:
+1 -1
View File
@@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine
async def check_database(engine: AsyncEngine) -> bool:
async with engine.connect() as connection:
await connection.execute(text('SELECT 1'))
await connection.execute(text("SELECT 1"))
return True
+12 -5
View File
@@ -33,7 +33,8 @@ async def get_active_key(
IdempotencyKey.method,
IdempotencyKey.path,
IdempotencyKey.key,
) == tuple_(*scope),
)
== tuple_(*scope),
IdempotencyKey.created_at
> func.clock_timestamp() - timedelta(seconds=ttl_seconds),
)
@@ -45,14 +46,20 @@ async def save_result(
) -> None:
"""Сохраняет новый результат или заменяет истёкший."""
stmt = insert(IdempotencyKey).values(
user_id=scope[0], method=scope[1], path=scope[2], key=scope[3],
request_hash=request_hash, message_id=message_id,
user_id=scope[0],
method=scope[1],
path=scope[2],
key=scope[3],
request_hash=request_hash,
message_id=message_id,
)
await session.execute(
stmt.on_conflict_do_update(
index_elements=[
IdempotencyKey.user_id, IdempotencyKey.method,
IdempotencyKey.path, IdempotencyKey.key,
IdempotencyKey.user_id,
IdempotencyKey.method,
IdempotencyKey.path,
IdempotencyKey.key,
],
set_={
"request_hash": request_hash,
+16 -3
View File
@@ -13,7 +13,8 @@ async def failed_jobs(session, limit: int):
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(Attachment.status == "failed")
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id).limit(limit)
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id)
.limit(limit)
)
return result.all()
@@ -23,12 +24,24 @@ async def replay_record(session, request_id):
async def insert_replay(session, **values) -> bool:
result = await session.scalar(insert(JobReplay).values(**values).on_conflict_do_nothing(index_elements=[JobReplay.id]).returning(JobReplay.id))
result = await session.scalar(
insert(JobReplay)
.values(**values)
.on_conflict_do_nothing(index_elements=[JobReplay.id])
.returning(JobReplay.id)
)
return result is not None
async def replay_history(session, job_id):
return (await session.scalars(select(JobReplay).where(JobReplay.job_id == job_id).order_by(JobReplay.created_at).limit(100))).all()
return (
await session.scalars(
select(JobReplay)
.where(JobReplay.job_id == job_id)
.order_by(JobReplay.created_at)
.limit(100)
)
).all()
def reset_failed(job, attachment, now, attempt_limit):
+43 -10
View File
@@ -32,7 +32,9 @@ class JobWork:
return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}"
async def create_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID:
async def create_job(
session: AsyncSession, attachment_id: UUID, staging_key: str
) -> UUID:
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key)
session.add(job)
await session.flush()
@@ -58,7 +60,10 @@ async def expired_jobs(session: AsyncSession, limit: int = 20):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(Attachment.status == "processing", AttachmentJob.lease_until < func.clock_timestamp())
.where(
Attachment.status == "processing",
AttachmentJob.lease_until < func.clock_timestamp(),
)
.order_by(AttachmentJob.lease_until)
.limit(limit)
.with_for_update(of=AttachmentJob, skip_locked=True)
@@ -66,19 +71,35 @@ async def expired_jobs(session: AsyncSession, limit: int = 20):
return result.all()
def start_job(job, attachment, token: UUID, now: datetime, lease_seconds: int) -> JobWork:
def start_job(
job, attachment, token: UUID, now: datetime, lease_seconds: int
) -> JobWork:
job.attempt += 1
job.lease_token = token
job.lease_until = now + timedelta(seconds=lease_seconds)
attachment.status = "processing"
attachment.updated_at = now
return JobWork(
job.id, attachment.id, token, job.pipeline_version, job.staging_key,
attachment.content_type, attachment.size_bytes, attachment.sha256,
job.id,
attachment.id,
token,
job.pipeline_version,
job.staging_key,
attachment.content_type,
attachment.size_bytes,
attachment.sha256,
)
def record_failure(job, attachment, now: datetime, error_code: str, *, terminal: bool, retry_seconds: float) -> None:
def record_failure(
job,
attachment,
now: datetime,
error_code: str,
*,
terminal: bool,
retry_seconds: float,
) -> None:
job.lease_token = None
job.lease_until = None
job.last_error_code = error_code
@@ -89,7 +110,9 @@ def record_failure(job, attachment, now: datetime, error_code: str, *, terminal:
attachment.updated_at = now
def complete_job(job, attachment, work: JobWork, variants: list[dict], now: datetime) -> None:
def complete_job(
job, attachment, work: JobWork, variants: list[dict], now: datetime
) -> None:
attachment.object_key = work.original_key
attachment.thumbnail_key = work.variant_key("thumbnail")
attachment.crop_key = work.variant_key("crop")
@@ -105,15 +128,25 @@ def complete_job(job, attachment, work: JobWork, variants: list[dict], now: date
job.finished_at = now
async def terminal_staging(session: AsyncSession, retention: int, failed_retention: int, limit: int = 20):
async def terminal_staging(
session: AsyncSession, retention: int, failed_retention: int, limit: int = 20
):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(
AttachmentJob.staging_key.is_not(None),
or_(
and_(Attachment.status == "ready", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=retention)),
and_(Attachment.status == "failed", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=failed_retention)),
and_(
Attachment.status == "ready",
AttachmentJob.finished_at
< func.clock_timestamp() - timedelta(seconds=retention),
),
and_(
Attachment.status == "failed",
AttachmentJob.finished_at
< func.clock_timestamp() - timedelta(seconds=failed_retention),
),
),
)
.order_by(AttachmentJob.finished_at)
+3 -1
View File
@@ -9,7 +9,9 @@ from app.models.attachment import Attachment
from app.models.job import AttachmentJob
async def next_unpublished_job(session: AsyncSession, *, attachment_id: UUID | None = None) -> AttachmentJob | None:
async def next_unpublished_job(
session: AsyncSession, *, attachment_id: UUID | None = None
) -> AttachmentJob | None:
statement = (
select(AttachmentJob)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
+4 -1
View File
@@ -81,7 +81,10 @@ def main():
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": [f"arn:aws:s3:::{bucket}/{prefix}/*" for prefix in ("originals", "staging", "variants")],
"Resource": [
f"arn:aws:s3:::{bucket}/{prefix}/*"
for prefix in ("originals", "staging", "variants")
],
},
],
}
@@ -20,21 +20,26 @@ def upgrade() -> None:
sa.Column("request_hash", sa.String(64), nullable=False),
sa.Column("message_id", sa.UUID(), nullable=False),
sa.Column(
"created_at", sa.DateTime(timezone=True),
server_default=sa.text("clock_timestamp()"), nullable=False,
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("clock_timestamp()"),
nullable=False,
),
sa.ForeignKeyConstraint(
["user_id"], ["users.id"], name=op.f("fk_idempotency_keys_user_id_users")
),
sa.ForeignKeyConstraint(
["message_id"], ["messages.id"],
["message_id"],
["messages.id"],
name=op.f("fk_idempotency_keys_message_id_messages"),
),
sa.PrimaryKeyConstraint(
"user_id", "method", "path", "key", name=op.f("pk_idempotency_keys")
),
)
op.create_index("ix_idempotency_keys_created_at", "idempotency_keys", ["created_at"])
op.create_index(
"ix_idempotency_keys_created_at", "idempotency_keys", ["created_at"]
)
def downgrade() -> None:
@@ -21,14 +21,35 @@ def upgrade() -> None:
sa.Column("size_bytes", sa.Integer(), nullable=False),
sa.Column("sha256", sa.String(64), nullable=False),
sa.Column("status", sa.String(16), server_default="ready", nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
sa.CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name=op.f("ck_attachments_size_bytes")),
sa.CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name=op.f("ck_attachments_content_type")),
sa.CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name=op.f("ck_attachments_sha256")),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("clock_timestamp()"),
nullable=False,
),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
server_default=sa.text("clock_timestamp()"),
nullable=False,
),
sa.CheckConstraint(
"size_bytes BETWEEN 1 AND 10485760", name=op.f("ck_attachments_size_bytes")
),
sa.CheckConstraint(
"content_type IN ('image/png', 'image/jpeg')",
name=op.f("ck_attachments_content_type"),
),
sa.CheckConstraint(
"sha256 ~ '^[a-f0-9]{64}$'", name=op.f("ck_attachments_sha256")
),
sa.CheckConstraint("status = 'ready'", name=op.f("ck_attachments_status")),
sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], name=op.f("fk_attachments_chat_id_chats")),
sa.ForeignKeyConstraint(["owner_id"], ["users.id"], name=op.f("fk_attachments_owner_id_users")),
sa.ForeignKeyConstraint(
["chat_id"], ["chats.id"], name=op.f("fk_attachments_chat_id_chats")
),
sa.ForeignKeyConstraint(
["owner_id"], ["users.id"], name=op.f("fk_attachments_owner_id_users")
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_attachments")),
sa.UniqueConstraint("object_key", name=op.f("uq_attachments_object_key")),
)
@@ -38,15 +59,31 @@ def upgrade() -> None:
sa.Column("message_id", sa.UUID(), nullable=False),
sa.Column("attachment_id", sa.UUID(), nullable=False),
sa.Column("position", sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE", name=op.f("fk_message_attachments_message_id_messages")),
sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_message_attachments_attachment_id_attachments")),
sa.PrimaryKeyConstraint("message_id", "attachment_id", name=op.f("pk_message_attachments")),
sa.ForeignKeyConstraint(
["message_id"],
["messages.id"],
ondelete="CASCADE",
name=op.f("fk_message_attachments_message_id_messages"),
),
sa.ForeignKeyConstraint(
["attachment_id"],
["attachments.id"],
name=op.f("fk_message_attachments_attachment_id_attachments"),
),
sa.PrimaryKeyConstraint(
"message_id", "attachment_id", name=op.f("pk_message_attachments")
),
)
op.create_table(
"upload_intents",
sa.Column("id", sa.UUID(), nullable=False),
sa.Column("object_key", sa.String(128), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("clock_timestamp()"),
nullable=False,
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_upload_intents")),
sa.UniqueConstraint("object_key", name=op.f("uq_upload_intents_object_key")),
)
@@ -5,29 +5,30 @@ Revises: db06fd3126ff
Create Date: 2026-10-26 22:01:01.910396
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = 'abd273076068'
down_revision: Union[str, None] = 'db06fd3126ff'
revision: str = "abd273076068"
down_revision: Union[str, None] = "db06fd3126ff"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('counters',
sa.Column('name', sa.String(length=64), nullable=False),
sa.Column('value', sa.BigInteger(), nullable=False),
sa.PrimaryKeyConstraint('name', name=op.f('pk_counters'))
op.create_table(
"counters",
sa.Column("name", sa.String(length=64), nullable=False),
sa.Column("value", sa.BigInteger(), nullable=False),
sa.PrimaryKeyConstraint("name", name=op.f("pk_counters")),
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('counters')
op.drop_table("counters")
# ### end Alembic commands ###
@@ -12,12 +12,30 @@ depends_on = None
def upgrade() -> None:
op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check")
op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status IN ('queued', 'processing', 'ready', 'failed')")
op.create_check_constraint(
op.f("ck_attachments_status"),
"attachments",
"status IN ('queued', 'processing', 'ready', 'failed')",
)
op.alter_column("attachments", "status", server_default="queued")
op.add_column("attachments", sa.Column("error_code", sa.String(64), nullable=True))
op.add_column("attachments", sa.Column("thumbnail_key", sa.String(128), nullable=True))
op.add_column("attachments", sa.Column("variants", postgresql.JSONB(), nullable=False, server_default=sa.text("'[]'::jsonb")))
op.create_check_constraint(op.f("ck_attachments_error_code"), "attachments", "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)")
op.add_column(
"attachments", sa.Column("thumbnail_key", sa.String(128), nullable=True)
)
op.add_column(
"attachments",
sa.Column(
"variants",
postgresql.JSONB(),
nullable=False,
server_default=sa.text("'[]'::jsonb"),
),
)
op.create_check_constraint(
op.f("ck_attachments_error_code"),
"attachments",
"(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)",
)
op.create_table(
"attachment_jobs",
sa.Column("id", sa.UUID(), nullable=False),
@@ -25,31 +43,62 @@ def upgrade() -> None:
sa.Column("staging_key", sa.String(128), nullable=True),
sa.Column("pipeline_version", sa.Integer(), server_default="1", nullable=False),
sa.Column("attempt", sa.Integer(), server_default="0", nullable=False),
sa.Column("published_attempt", sa.Integer(), server_default="0", nullable=False),
sa.Column("next_attempt_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
sa.Column(
"published_attempt", sa.Integer(), server_default="0", nullable=False
),
sa.Column(
"next_attempt_at",
sa.DateTime(timezone=True),
server_default=sa.text("clock_timestamp()"),
nullable=False,
),
sa.Column("lease_token", sa.UUID(), nullable=True),
sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error_code", sa.String(64), nullable=True),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("clock_timestamp()"),
nullable=False,
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_attachment_jobs")),
sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_attachment_jobs_attachment_id_attachments")),
sa.UniqueConstraint("attachment_id", name=op.f("uq_attachment_jobs_attachment_id")),
sa.CheckConstraint("pipeline_version = 1", name=op.f("ck_attachment_jobs_pipeline_version")),
sa.CheckConstraint("attempt >= 0 AND published_attempt >= 0", name=op.f("ck_attachment_jobs_attempt")),
sa.ForeignKeyConstraint(
["attachment_id"],
["attachments.id"],
name=op.f("fk_attachment_jobs_attachment_id_attachments"),
),
sa.UniqueConstraint(
"attachment_id", name=op.f("uq_attachment_jobs_attachment_id")
),
sa.CheckConstraint(
"pipeline_version = 1", name=op.f("ck_attachment_jobs_pipeline_version")
),
sa.CheckConstraint(
"attempt >= 0 AND published_attempt >= 0",
name=op.f("ck_attachment_jobs_attempt"),
),
)
op.create_index("ix_attachment_jobs_due", "attachment_jobs", ["next_attempt_at"])
op.create_index("ix_attachment_jobs_lease", "attachment_jobs", ["lease_until"])
# Прежние оригиналы остаются источником; worker создаст отсутствующий thumbnail.
op.execute("INSERT INTO attachment_jobs (id, attachment_id, staging_key) SELECT id, id, object_key FROM attachments")
op.execute("UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()")
op.execute(
"INSERT INTO attachment_jobs (id, attachment_id, staging_key) SELECT id, id, object_key FROM attachments"
)
op.execute(
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()"
)
def downgrade() -> None:
connection = op.get_bind()
unfinished = connection.scalar(sa.text("SELECT count(*) FROM attachments WHERE status <> 'ready'"))
unfinished = connection.scalar(
sa.text("SELECT count(*) FROM attachments WHERE status <> 'ready'")
)
if unfinished:
raise RuntimeError("Перед downgrade нужно завершить или вручную разобрать все queued/processing/failed вложения")
raise RuntimeError(
"Перед downgrade нужно завершить или вручную разобрать все queued/processing/failed вложения"
)
op.drop_index("ix_attachment_jobs_lease", table_name="attachment_jobs")
op.drop_index("ix_attachment_jobs_due", table_name="attachment_jobs")
op.drop_table("attachment_jobs")
@@ -58,5 +107,7 @@ def downgrade() -> None:
op.drop_column("attachments", "thumbnail_key")
op.drop_column("attachments", "error_code")
op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check")
op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status = 'ready'")
op.create_check_constraint(
op.f("ck_attachments_status"), "attachments", "status = 'ready'"
)
op.alter_column("attachments", "status", server_default="ready")
@@ -11,7 +11,9 @@ depends_on = None
def upgrade() -> None:
op.add_column("attachments", sa.Column("crop_key", sa.String(128), nullable=True))
op.add_column("attachments", sa.Column("optimized_key", sa.String(128), nullable=True))
op.add_column(
"attachments", sa.Column("optimized_key", sa.String(128), nullable=True)
)
# Старый staging сменяется надёжным ready-оригиналом, но не теряется из журнала cleanup.
op.execute("""
INSERT INTO upload_intents (id, object_key)
@@ -37,7 +39,11 @@ def upgrade() -> None:
def downgrade() -> None:
active = op.get_bind().scalar(sa.text("SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')"))
active = op.get_bind().scalar(
sa.text(
"SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')"
)
)
if active:
raise RuntimeError("Перед downgrade необходимо завершить активные задания")
op.execute("""
+22 -5
View File
@@ -10,19 +10,36 @@ depends_on = None
def upgrade() -> None:
op.add_column("attachment_jobs", sa.Column("cycle_start_attempt", sa.Integer(), nullable=False, server_default="0"))
op.add_column("attachment_jobs", sa.Column("attempt_limit", sa.Integer(), nullable=True))
op.add_column("attachment_jobs", sa.Column("last_error_permanent", sa.Boolean(), nullable=True))
op.add_column(
"attachment_jobs",
sa.Column(
"cycle_start_attempt", sa.Integer(), nullable=False, server_default="0"
),
)
op.add_column(
"attachment_jobs", sa.Column("attempt_limit", sa.Integer(), nullable=True)
)
op.add_column(
"attachment_jobs",
sa.Column("last_error_permanent", sa.Boolean(), nullable=True),
)
op.create_table(
"job_replays",
sa.Column("id", sa.Uuid(), primary_key=True),
sa.Column("job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False),
sa.Column(
"job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False
),
sa.Column("actor", sa.String(64), nullable=False),
sa.Column("reason", sa.String(32), nullable=False),
sa.Column("previous_attempt", sa.Integer(), nullable=False),
sa.Column("attempt_limit", sa.Integer(), nullable=False),
sa.Column("previous_error_code", sa.String(64), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.clock_timestamp()),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
nullable=False,
server_default=sa.func.clock_timestamp(),
),
)
op.create_index("ix_job_replays_job_id", "job_replays", ["job_id"])
+56 -32
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Offline structural checks; full schema validation is a separate target."""
import ast
import json
from pathlib import Path
@@ -7,48 +8,71 @@ import re
ROOT = Path(__file__).resolve().parents[1]
def main():
lab = json.loads((ROOT/'lab.json').read_text())['number']
spec = json.loads((ROOT/'contracts/openapi.json').read_text())
assert spec['openapi']=='3.1.0'
assert spec['info']['version']=='1.%d.0'%lab
lab = json.loads((ROOT / "lab.json").read_text())["number"]
spec = json.loads((ROOT / "contracts/openapi.json").read_text())
assert spec["openapi"] == "3.1.0"
assert spec["info"]["version"] == "1.%d.0" % lab
operation_ids = set()
def walk(value):
if isinstance(value,dict):
if '$ref' in value:
pointer = value['$ref']
assert pointer.startswith('#/'), 'unexpected external reference'
if isinstance(value, dict):
if "$ref" in value:
pointer = value["$ref"]
assert pointer.startswith("#/"), "unexpected external reference"
target = spec
for part in pointer[2:].split('/'):
target = target[part.replace('~1','/').replace('~0','~')]
for part in pointer[2:].split("/"):
target = target[part.replace("~1", "/").replace("~0", "~")]
for v in value.values():
walk(v)
elif isinstance(value,list):
elif isinstance(value, list):
for v in value:
walk(v)
walk(spec)
for route,methods in spec['paths'].items():
placeholders = set(re.findall(r'\{([^}]+)\}',route))
for method,op in methods.items():
assert op['operationId'] not in operation_ids, 'duplicate operationId'
operation_ids.add(op['operationId'])
params = {p['name'] for p in op.get('parameters',[]) if p['in']=='path' and p.get('required')}
assert placeholders==params, 'path parameter mismatch: '+route
for response in op['responses'].values():
assert 'X-Request-Id' in response['headers']
if lab>=3:
assert 'X-Instance-Id' in response['headers']
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
ast.parse(file.read_text(),filename=str(file))
for file in ROOT.rglob('*.json'):
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
for route, methods in spec["paths"].items():
placeholders = set(re.findall(r"\{([^}]+)\}", route))
for method, op in methods.items():
assert op["operationId"] not in operation_ids, "duplicate operationId"
operation_ids.add(op["operationId"])
params = {
p["name"]
for p in op.get("parameters", [])
if p["in"] == "path" and p.get("required")
}
assert placeholders == params, "path parameter mismatch: " + route
for response in op["responses"].values():
assert "X-Request-Id" in response["headers"]
if lab >= 3:
assert "X-Instance-Id" in response["headers"]
assert ("/api/v1/auth/sessions" in spec["paths"]) == (lab >= 2)
assert ("/api/v1/attachments/{attachment_id}" in spec["paths"]) == (lab >= 4)
for file in list((ROOT / "tests").glob("*.py")) + list(
(ROOT / "scripts").glob("*.py")
):
ast.parse(file.read_text(), filename=str(file))
for file in ROOT.rglob("*.json"):
if any(
part in {".git", ".venv", "node_modules", "vendor", "output"}
for part in file.parts
):
continue
json.loads(file.read_text())
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
assert (ROOT/required).is_file(), 'missing '+required
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
for required in (
"README.md",
"COURSE.md",
"CONTRIBUTING.md",
"REPORT.md",
"contracts/README.md",
"tests/smoke.py",
"tests/README.md",
):
assert (ROOT / required).is_file(), "missing " + required
print(
"PASS: offline skeleton structure, local refs and Python syntax; application NOT tested."
)
if __name__=='__main__':
if __name__ == "__main__":
main()
+340 -122
View File
@@ -16,7 +16,7 @@ from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compos
from scripts.deployment import ComposeProject
from tests.client import Client, check
PROBE = '''
PROBE = """
import asyncio, hashlib, json
from uuid import UUID
from sqlalchemy import select
@@ -43,169 +43,387 @@ async def main():
result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
finally: await storage.aclose();await engine.dispose()
asyncio.run(main())
'''
"""
def probe(attachment_id):
code = 'ATTACHMENT_ID = ' + repr(str(UUID(attachment_id))) + '\n' + PROBE
result = subprocess.run(COMPOSE + ['exec', '-T', 'api', 'python', '-'], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30)
check(result.returncode == 0, 'Probe failed; diagnostic output hidden')
code = "ATTACHMENT_ID = " + repr(str(UUID(attachment_id))) + "\n" + PROBE
result = subprocess.run(
COMPOSE + ["exec", "-T", "api", "python", "-"],
input=code,
text=True,
capture_output=True,
cwd=ROOT,
timeout=30,
)
check(result.returncode == 0, "Probe failed; diagnostic output hidden")
return json.loads(result.stdout)
def admin(*arguments, expected=0):
result = compose('run', '--rm', '--no-deps', 'worker', 'python', '-m', 'app.core.job_admin', *arguments)
check(result.returncode == expected, 'Unexpected operator command exit status')
result = compose(
"run",
"--rm",
"--no-deps",
"worker",
"python",
"-m",
"app.core.job_admin",
*arguments
)
check(result.returncode == expected, "Unexpected operator command exit status")
return json.loads(result.stdout)
def verify_result(client, actor, attachment, original):
ready = client.wait_ready(actor, attachment, wait_seconds=180)
check(ready['id'] == attachment['id'], 'Attachment identity changed')
path = '/api/v1/attachments/' + ready['id'] + '/content'
check(client.call('GET', path, actor=actor)['bytes'] == original, 'Original changed')
check({v['name'] for v in ready['variants']} == {'thumbnail','crop','optimized'}, 'Incomplete pipeline')
for variant in ready['variants']:
data = client.call('GET', path+'?variant='+variant['name'], actor=actor)['bytes']
check(hashlib.sha256(data).hexdigest() == variant['sha256'], 'Checksum mismatch')
check(ready["id"] == attachment["id"], "Attachment identity changed")
path = "/api/v1/attachments/" + ready["id"] + "/content"
check(
client.call("GET", path, actor=actor)["bytes"] == original, "Original changed"
)
check(
{v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"},
"Incomplete pipeline",
)
for variant in ready["variants"]:
data = client.call("GET", path + "?variant=" + variant["name"], actor=actor)[
"bytes"
]
check(
hashlib.sha256(data).hexdigest() == variant["sha256"], "Checksum mismatch"
)
with Image.open(BytesIO(data)) as image:
check(image.size == (variant['width'],variant['height']), 'Metadata dimensions mismatch')
check(image.format == ('WEBP' if variant['name']=='optimized' else 'JPEG'), 'Wrong format')
check(not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp'), 'Metadata leaked')
if variant['name']=='crop':check(image.size==(128,128),'Wrong crop size')
snapshot=probe(ready['id'])
check(len(snapshot['objects']) == 4, 'Expected exactly one original and three variants')
check(
image.size == (variant["width"], variant["height"]),
"Metadata dimensions mismatch",
)
check(
image.format == ("WEBP" if variant["name"] == "optimized" else "JPEG"),
"Wrong format",
)
check(
not image.getexif()
and not image.info.get("icc_profile")
and not image.info.get("xmp"),
"Metadata leaked",
)
if variant["name"] == "crop":
check(image.size == (128, 128), "Wrong crop size")
snapshot = probe(ready["id"])
check(
len(snapshot["objects"]) == 4,
"Expected exactly one original and three variants",
)
return snapshot
def crash(client, actor, chat, mode):
name='messenger-worker-fault-'+uuid4().hex
name = "messenger-worker-fault-" + uuid4().hex
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
result=compose('run','--rm','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
phase='after_object:optimized' if mode=='crash-after-object' else 'before_ack'
marker=json.dumps({'fault':phase,'attachment_id':attachment['id']})
check(result.returncode==137 and marker in result.stdout,'Targeted crash not observed')
before=probe(attachment['id'])
check(before['status']==('processing' if mode=='crash-after-object' else 'ready'),'Unexpected crash state')
check(len(before['objects'])==4,'Crash must occur after all four PUTs')
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
result = compose(
"run",
"--rm",
"--no-deps",
"--name",
name,
"worker",
"python",
"-m",
"app.core.job_worker",
"--" + mode,
attachment["id"],
)
phase = (
"after_object:optimized" if mode == "crash-after-object" else "before_ack"
)
marker = json.dumps({"fault": phase, "attachment_id": attachment["id"]})
check(
result.returncode == 137 and marker in result.stdout,
"Targeted crash not observed",
)
before = probe(attachment["id"])
check(
before["status"]
== ("processing" if mode == "crash-after-object" else "ready"),
"Unexpected crash state",
)
check(len(before["objects"]) == 4, "Crash must occur after all four PUTs")
finally:
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
if mode=='crash-before-ack':
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
if mode == "crash-before-ack":
# Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
wait_queue_empty()
after=probe(attachment['id'])
check(after==before,'Ready duplicate rewrote result')
after = probe(attachment["id"])
check(after == before, "Ready duplicate rewrote result")
else:
check(after['attempt']==before['attempt']+1,'Lease did not recover the interrupted attempt')
check([(x['name'],x['sha256']) for x in before['objects']]==[(x['name'],x['sha256']) for x in after['objects']],'Replay changed object content')
return dict(scenario=mode,attachment_id=attachment['id'],exit_code=137,before=before,after=after)
check(
after["attempt"] == before["attempt"] + 1,
"Lease did not recover the interrupted attempt",
)
check(
[(x["name"], x["sha256"]) for x in before["objects"]]
== [(x["name"], x["sha256"]) for x in after["objects"]],
"Replay changed object content",
)
return dict(
scenario=mode,
attachment_id=attachment["id"],
exit_code=137,
before=before,
after=after,
)
def retry_replay(client,actor,chat,permanent):
name='messenger-retry-fault-'+uuid4().hex
samples=[]
configuration=ComposeProject().config['services']['worker']['environment']
retry_base=float(configuration['JOB_RETRY_SECONDS'])
retry_max=float(configuration['JOB_RETRY_MAX_SECONDS'])
def retry_replay(client, actor, chat, permanent):
name = "messenger-retry-fault-" + uuid4().hex
samples = []
configuration = ComposeProject().config["services"]["worker"]["environment"]
retry_base = float(configuration["JOB_RETRY_SECONDS"])
retry_max = float(configuration["JOB_RETRY_MAX_SECONDS"])
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
mode='fail-permanent' if permanent else 'fail-transient'
require_compose('run','-d','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
deadline=time.monotonic()+180
while time.monotonic()<deadline:
sample=probe(attachment['id'])
if sample['attempt'] and sample['status'] in {'queued', 'failed'} and (not samples or samples[-1]['attempt']!=sample['attempt']):samples.append(sample)
if sample['status']=='failed':break
time.sleep(.5)
check(sample['status']=='failed','Retry never reached failed')
check(sample['attempt']==(1 if permanent else sample['limit']),'Incorrect attempt budget')
check(sample['permanent']==permanent,'Incorrect error classification')
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
mode = "fail-permanent" if permanent else "fail-transient"
require_compose(
"run",
"-d",
"--no-deps",
"--name",
name,
"worker",
"python",
"-m",
"app.core.job_worker",
"--" + mode,
attachment["id"],
)
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
sample = probe(attachment["id"])
if (
sample["attempt"]
and sample["status"] in {"queued", "failed"}
and (not samples or samples[-1]["attempt"] != sample["attempt"])
):
samples.append(sample)
if sample["status"] == "failed":
break
time.sleep(0.5)
check(sample["status"] == "failed", "Retry never reached failed")
check(
sample["attempt"] == (1 if permanent else sample["limit"]),
"Incorrect attempt budget",
)
check(sample["permanent"] == permanent, "Incorrect error classification")
if not permanent:
check(len(samples)==sample['limit'],'Missed retry samples')
check(len(samples) == sample["limit"], "Missed retry samples")
for entry in samples:
delay=(datetime.fromisoformat(entry['next_attempt_at'])-datetime.fromisoformat(entry['updated_at'])).total_seconds()
ceiling=min(retry_max,retry_base*2**(entry['attempt']-1))
check(ceiling/2<=delay<=ceiling,'Retry delay outside configured jitter bounds')
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
job_id=sample['job_id'];request_id=str(uuid4())
listing=admin('list-failed')
check(any(row['job_id']==job_id for row in listing),'Job missing from failed registry')
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30,check=True)
first=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
check(first['status']=='queued','Replay did not queue job')
delay = (
datetime.fromisoformat(entry["next_attempt_at"])
- datetime.fromisoformat(entry["updated_at"])
).total_seconds()
ceiling = min(retry_max, retry_base * 2 ** (entry["attempt"] - 1))
check(
ceiling / 2 <= delay <= ceiling,
"Retry delay outside configured jitter bounds",
)
client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=actor,
expected=409,
)
job_id = sample["job_id"]
request_id = str(uuid4())
listing = admin("list-failed")
check(
any(row["job_id"] == job_id for row in listing),
"Job missing from failed registry",
)
subprocess.run(
["docker", "rm", "-f", name], capture_output=True, timeout=30, check=True
)
first = admin(
"replay",
job_id,
"--request-id",
request_id,
"--actor",
"verification",
"--reason",
"dependency_restored",
)
check(first["status"] == "queued", "Replay did not queue job")
finally:
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
repeat=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
check(repeat['already_applied'],'Replay request id not idempotent')
refused=admin('replay',job_id,'--request-id',str(uuid4()),'--actor','verification','--reason','manual_review',expected=2)
check(refused['error_code']=='job_not_failed','Successful job was replayed')
check(probe(attachment['id'])==after,'Operator replay altered ready job')
check(len(admin('history',job_id))==1,'Incorrect audit count')
return dict(scenario=mode,attachment_id=attachment['id'],samples=samples,after=after,replay_idempotent=True)
subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
repeat = admin(
"replay",
job_id,
"--request-id",
request_id,
"--actor",
"verification",
"--reason",
"dependency_restored",
)
check(repeat["already_applied"], "Replay request id not idempotent")
refused = admin(
"replay",
job_id,
"--request-id",
str(uuid4()),
"--actor",
"verification",
"--reason",
"manual_review",
expected=2,
)
check(refused["error_code"] == "job_not_failed", "Successful job was replayed")
check(probe(attachment["id"]) == after, "Operator replay altered ready job")
check(len(admin("history", job_id)) == 1, "Incorrect audit count")
return dict(
scenario=mode,
attachment_id=attachment["id"],
samples=samples,
after=after,
replay_idempotent=True,
)
def queue_state():
output=require_compose('exec','-T','broker','rabbitmqctl','list_queues','--formatter=json','name','messages_ready','messages_unacknowledged','consumers','durable')
return next(row for row in json.loads(output) if row['name']=='attachment-jobs')
output = require_compose(
"exec",
"-T",
"broker",
"rabbitmqctl",
"list_queues",
"--formatter=json",
"name",
"messages_ready",
"messages_unacknowledged",
"consumers",
"durable",
)
return next(row for row in json.loads(output) if row["name"] == "attachment-jobs")
def wait_queue_empty():
deadline=time.monotonic()+20
while time.monotonic()<deadline:
state=queue_state()
if state['messages_ready']==0 and state['messages_unacknowledged']==0:
deadline = time.monotonic() + 20
while time.monotonic() < deadline:
state = queue_state()
if state["messages_ready"] == 0 and state["messages_unacknowledged"] == 0:
return state
time.sleep(.5)
check(False,'RabbitMQ did not acknowledge all completed deliveries')
time.sleep(0.5)
check(False, "RabbitMQ did not acknowledge all completed deliveries")
def persistence(client,actor,chat):
def persistence(client, actor, chat):
try:
require_compose('stop','worker')
attachment,original=client.attachment(actor,chat)
client.message(actor,chat,'',attachments=[attachment['id']])
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
client.message(actor, chat, "", attachments=[attachment["id"]])
client.call(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
actor=actor,
expected=409,
)
time.sleep(2)
queued_before=queue_state()
check(queued_before['messages_ready']>=1 and queued_before['durable'], 'No durable queued message before restart')
result=subprocess.run([str(ROOT/'scripts/restart.sh')],cwd=ROOT,capture_output=True,timeout=180)
check(result.returncode==0,'Restart hook failed')
check('worker' not in require_compose('ps','--status','running','--services').splitlines(),'Restart started deliberately stopped worker')
queued_after=queue_state()
check(queued_after['messages_ready']>=1 and queued_after['durable'], 'Broker lost queued delivery across restart')
client.message(actor,chat,'session and chat survived broker/API restart')
finally:require_compose('start','worker')
after=verify_result(client,actor,attachment,original)
return dict(scenario='worker_stopped_restart_all',attachment_id=attachment['id'],queue_before=queued_before,queue_after=queued_after,after=after)
queued_before = queue_state()
check(
queued_before["messages_ready"] >= 1 and queued_before["durable"],
"No durable queued message before restart",
)
result = subprocess.run(
[str(ROOT / "scripts/restart.sh")],
cwd=ROOT,
capture_output=True,
timeout=180,
)
check(result.returncode == 0, "Restart hook failed")
check(
"worker"
not in require_compose(
"ps", "--status", "running", "--services"
).splitlines(),
"Restart started deliberately stopped worker",
)
queued_after = queue_state()
check(
queued_after["messages_ready"] >= 1 and queued_after["durable"],
"Broker lost queued delivery across restart",
)
client.message(actor, chat, "session and chat survived broker/API restart")
finally:
require_compose("start", "worker")
after = verify_result(client, actor, attachment, original)
return dict(
scenario="worker_stopped_restart_all",
attachment_id=attachment["id"],
queue_before=queued_before,
queue_after=queued_after,
after=after,
)
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--base-url',required=True)
parser.add_argument('--ca-file')
parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all')
parser.add_argument('--output',default='evidence/jobs.json')
args=parser.parse_args()
client=Client(args.base_url,args.ca_file,timeout=40)
aliases={r.instance_id for r in ComposeProject().replicas() if r.running}
check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL')
actor,_,_,chat=client.bootstrap()
results=[]
operations={'persistence':persistence,'crash-after-object':lambda *x:crash(*x,'crash-after-object'),'crash-before-ack':lambda *x:crash(*x,'crash-before-ack'),'fail-transient':lambda *x:retry_replay(*x,False),'fail-permanent':lambda *x:retry_replay(*x,True)}
selected=operations.values() if args.scenario=='all' else [operations[args.scenario]]
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--ca-file")
parser.add_argument(
"--scenario",
choices=[
"all",
"persistence",
"crash-after-object",
"crash-before-ack",
"fail-transient",
"fail-permanent",
],
default="all",
)
parser.add_argument("--output", default="evidence/jobs.json")
args = parser.parse_args()
client = Client(args.base_url, args.ca_file, timeout=40)
aliases = {r.instance_id for r in ComposeProject().replicas() if r.running}
check(
client.call("GET", "/health/live", schema="Health")["headers"].get(
"X-Instance-Id"
)
in aliases,
"Wrong Compose project for BASE_URL",
)
actor, _, _, chat = client.bootstrap()
results = []
operations = {
"persistence": persistence,
"crash-after-object": lambda *x: crash(*x, "crash-after-object"),
"crash-before-ack": lambda *x: crash(*x, "crash-before-ack"),
"fail-transient": lambda *x: retry_replay(*x, False),
"fail-permanent": lambda *x: retry_replay(*x, True),
}
selected = (
operations.values() if args.scenario == "all" else [operations[args.scenario]]
)
for operation in selected:
result=operation(client,actor,chat);results.append(result)
Path(args.output).parent.mkdir(parents=True,exist_ok=True)
Path(args.output).write_text(json.dumps(dict(status='in_progress',results=results),indent=2)+'\n')
print('PASS: '+result['scenario'],flush=True)
Path(args.output).write_text(json.dumps(dict(status='passed',results=results),indent=2)+'\n')
result = operation(client, actor, chat)
results.append(result)
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text(
json.dumps(dict(status="in_progress", results=results), indent=2) + "\n"
)
print("PASS: " + result["scenario"], flush=True)
Path(args.output).write_text(
json.dumps(dict(status="passed", results=results), indent=2) + "\n"
)
if __name__=='__main__':main()
if __name__ == "__main__":
main()
+89 -25
View File
@@ -19,7 +19,13 @@ COMPOSE = ["docker", "compose", "--env-file", ".env"]
def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess:
return subprocess.run(COMPOSE + list(arguments), cwd=ROOT, text=True, capture_output=True, timeout=timeout)
return subprocess.run(
COMPOSE + list(arguments),
cwd=ROOT,
text=True,
capture_output=True,
timeout=timeout,
)
def require_compose(*arguments: str) -> str:
@@ -29,54 +35,112 @@ def require_compose(*arguments: str) -> str:
def main() -> None:
parser = argparse.ArgumentParser(description="Отказ publisher после COMMIT перед RabbitMQ publish")
parser = argparse.ArgumentParser(
description="Отказ publisher после COMMIT перед RabbitMQ publish"
)
parser.add_argument("--base-url", required=True)
parser.add_argument("--ca-file")
args = parser.parse_args()
running = set(require_compose("ps", "--status", "running", "--services").splitlines())
check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the application stack before this scenario")
running = set(
require_compose("ps", "--status", "running", "--services").splitlines()
)
check(
{"api", "postgres", "minio", "broker", "publisher", "worker"} <= running,
"Start the application stack before this scenario",
)
client = Client(args.base_url, args.ca_file, timeout=40)
project = ComposeProject()
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
live = client.call("GET", "/health/live", schema="Health")
check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project")
check(
live["headers"].get("X-Instance-Id") in aliases,
"BASE_URL does not belong to the selected Compose project",
)
alice, _, _, chat = client.bootstrap()
attachment = None
fault_container = "messenger-outbox-fault-" + uuid4().hex
try:
require_compose("stop", "publisher")
attachment, original = client.attachment(alice, chat)
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected=409, actor=alice)
killed = compose(
"run", "--rm", "--no-deps", "--name", fault_container, "publisher", "python", "-m",
"app.core.outbox_publisher", "--crash-before-publish", attachment["id"],
message = client.message(alice, chat, "", attachments=[attachment["id"]])[
"json"
]
client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}/content",
expected=409,
actor=alice,
)
marker = json.dumps({"fault": "before_publish", "attachment_id": attachment["id"]})
check(killed.returncode == 137 and marker in killed.stdout, "Expected targeted SIGKILL before publish was not observed")
pending = client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=alice, schema="Attachment")["json"]
killed = compose(
"run",
"--rm",
"--no-deps",
"--name",
fault_container,
"publisher",
"python",
"-m",
"app.core.outbox_publisher",
"--crash-before-publish",
attachment["id"],
)
marker = json.dumps(
{"fault": "before_publish", "attachment_id": attachment["id"]}
)
check(
killed.returncode == 137 and marker in killed.stdout,
"Expected targeted SIGKILL before publish was not observed",
)
pending = client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}",
actor=alice,
schema="Attachment",
)["json"]
check(pending["status"] == "queued", "Crash must leave accepted job queued")
finally:
# Если docker compose run оборвался по timeout, одноразовый контейнер тоже удаляется.
try:
subprocess.run(["docker", "rm", "-f", fault_container], capture_output=True, timeout=30)
subprocess.run(
["docker", "rm", "-f", fault_container], capture_output=True, timeout=30
)
finally:
require_compose("start", "publisher")
ready = client.wait_ready(alice, attachment, wait_seconds=60)
check(ready["id"] == attachment["id"], "Recovered a different attachment")
check({variant["name"] for variant in ready["variants"]} == {"thumbnail", "crop", "optimized"}, "Missing required variants")
check(
{variant["name"] for variant in ready["variants"]}
== {"thumbnail", "crop", "optimized"},
"Missing required variants",
)
path = f"/api/v1/attachments/{attachment['id']}/content"
check(client.call("GET", path, actor=alice)["bytes"] == original, "Original changed after recovery")
check(
client.call("GET", path, actor=alice)["bytes"] == original,
"Original changed after recovery",
)
for variant in ready["variants"]:
downloaded = client.call("GET", path + "?variant=" + variant["name"], actor=alice)["bytes"]
check(len(downloaded) == variant["size_bytes"] and hashlib.sha256(downloaded).hexdigest() == variant["sha256"], "Recovered variant checksum mismatch")
print(json.dumps({
"scenario": "commit_before_publish", "attachment_id": ready["id"],
"message_id": message["id"], "publisher_exit_code": killed.returncode,
"status_after_crash": pending["status"], "status_after_restart": ready["status"],
"variants": [variant["name"] for variant in ready["variants"]],
"reuploaded": False,
}))
downloaded = client.call(
"GET", path + "?variant=" + variant["name"], actor=alice
)["bytes"]
check(
len(downloaded) == variant["size_bytes"]
and hashlib.sha256(downloaded).hexdigest() == variant["sha256"],
"Recovered variant checksum mismatch",
)
print(
json.dumps(
{
"scenario": "commit_before_publish",
"attachment_id": ready["id"],
"message_id": message["id"],
"publisher_exit_code": killed.returncode,
"status_after_crash": pending["status"],
"status_after_restart": ready["status"],
"variants": [variant["name"] for variant in ready["variants"]],
"reuploaded": False,
}
)
)
if __name__ == "__main__":
+61 -30
View File
@@ -7,7 +7,7 @@ import time
from scripts.deployment import ComposeProject
from tests.client import Client, arguments, check, run
PROBE = r'''
PROBE = r"""
import asyncio, json, sys
from pathlib import Path
from uuid import UUID
@@ -62,60 +62,91 @@ async def main():
await storage.aclose()
await engine.dispose()
asyncio.run(main())
'''
"""
def main():
def configure(parser):
parser.add_argument('--with-outage', action='store_true', help='временно остановить MinIO выбранного проекта')
args = arguments('MinIO private bucket and failure checks', configure)
parser.add_argument(
"--with-outage",
action="store_true",
help="временно остановить MinIO выбранного проекта",
)
args = arguments("MinIO private bucket and failure checks", configure)
project = ComposeProject()
client = Client(args.base_url, args.ca_file, timeout=40)
alice, _, _, chat = client.bootstrap()
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
live = client.call('GET', '/health/live', schema='Health')
check(live['headers'].get('X-Instance-Id') in aliases, 'BASE_URL does not belong to the selected Compose project')
live = client.call("GET", "/health/live", schema="Health")
check(
live["headers"].get("X-Instance-Id") in aliases,
"BASE_URL does not belong to the selected Compose project",
)
attachment, original = client.attachment(alice, chat)
attachment = client.wait_ready(alice, attachment)
# Повторный bootstrap не должен очистить bucket или сменить используемые credentials.
for _ in range(2):
project.run('run', '--rm', '--no-deps', 'minio-bootstrap', timeout=100)
project.run("run", "--rm", "--no-deps", "minio-bootstrap", timeout=100)
replicas = project.replicas()
check(bool(replicas), 'API not running')
result = subprocess.run(['docker', 'exec', '-i', replicas[0].id, 'python', '-c', PROBE],
input=json.dumps({'attachment_id': attachment['id']}),
text=True, capture_output=True, timeout=20)
check(result.returncode == 0, 'private bucket probe failed (details hidden)')
check(bool(replicas), "API not running")
result = subprocess.run(
["docker", "exec", "-i", replicas[0].id, "python", "-c", PROBE],
input=json.dumps({"attachment_id": attachment["id"]}),
text=True,
capture_output=True,
timeout=20,
)
check(result.returncode == 0, "private bucket probe failed (details hidden)")
statuses = json.loads(result.stdout)
check(statuses['GET'] == statuses['PUT'] == 403, 'anonymous S3 access allowed')
check(statuses['admin_denied'] and statuses['other_buckets_denied'] and statuses['root_files_absent'], 'API has excessive access')
check(statuses['console_unreachable'], 'console reachable from API network')
check(statuses['temp_originals_absent'], 'API retained temporary originals')
check(not project.config['services']['minio'].get('ports'), 'MinIO publishes host ports')
env = project.config['services']['minio']['environment']
check(env['MINIO_BROWSER'] == 'off', 'browser enabled')
check('127.0.0.1:9001' in project.config['services']['minio']['command'], 'console is not bound to loopback')
path = '/api/v1/attachments/' + attachment['id'] + '/content'
check(client.call('GET', path, actor=alice)['bytes'] == original, 'bootstrap changed original')
check(statuses["GET"] == statuses["PUT"] == 403, "anonymous S3 access allowed")
check(
statuses["admin_denied"]
and statuses["other_buckets_denied"]
and statuses["root_files_absent"],
"API has excessive access",
)
check(statuses["console_unreachable"], "console reachable from API network")
check(statuses["temp_originals_absent"], "API retained temporary originals")
check(
not project.config["services"]["minio"].get("ports"),
"MinIO publishes host ports",
)
env = project.config["services"]["minio"]["environment"]
check(env["MINIO_BROWSER"] == "off", "browser enabled")
check(
"127.0.0.1:9001" in project.config["services"]["minio"]["command"],
"console is not bound to loopback",
)
path = "/api/v1/attachments/" + attachment["id"] + "/content"
check(
client.call("GET", path, actor=alice)["bytes"] == original,
"bootstrap changed original",
)
if args.with_outage:
try:
project.run('stop', 'minio')
client.call('GET', path, expected=503, actor=alice)
project.run("stop", "minio")
client.call("GET", path, expected=503, actor=alice)
finally:
project.run('start', 'minio')
project.run("start", "minio")
deadline = time.monotonic() + 60
while True:
try:
client.call('GET', '/health/ready', schema='Health')
client.call("GET", "/health/ready", schema="Health")
break
except Exception:
if time.monotonic() >= deadline:
raise
time.sleep(0.5)
check(client.call('GET', path, actor=alice)['bytes'] == original, 'original lost after MinIO restart')
print('PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console'
+ (', MinIO outage/recovery' if args.with_outage else ''))
check(
client.call("GET", path, actor=alice)["bytes"] == original,
"original lost after MinIO restart",
)
print(
"PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console"
+ (", MinIO outage/recovery" if args.with_outage else "")
)
if __name__ == '__main__':
if __name__ == "__main__":
run(main)
+17 -4
View File
@@ -31,8 +31,12 @@ class ComposeProject:
def docker(self, *args: str, timeout: int = 90, env=None) -> str:
"""Выполняет Docker-команду с ограничением времени."""
result = subprocess.run(
["docker", *args], cwd=ROOT, text=True, capture_output=True,
timeout=timeout, env={**os.environ, **(env or {})},
["docker", *args],
cwd=ROOT,
text=True,
capture_output=True,
timeout=timeout,
env={**os.environ, **(env or {})},
)
if result.returncode:
raise RuntimeError(result.stderr.strip() or result.stdout.strip())
@@ -53,8 +57,17 @@ class ComposeProject:
'"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}}'
)
rows = self.docker("inspect", "--format", template, *ids).splitlines()
configured = str(self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID") or "").strip()
configured = str(
self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID")
or ""
).strip()
return [
Replica(row["id"], row["image"], configured or row["hostname"], row["running"], row["health"])
Replica(
row["id"],
row["image"],
configured or row["hostname"],
row["running"],
row["health"],
)
for row in map(json.loads, rows)
]
+9 -1
View File
@@ -14,7 +14,15 @@ def main() -> None:
destination.chmod(0o600)
directory = root / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
for name in ("db_password", "redis_password", "broker_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"):
for name in (
"db_password",
"redis_password",
"broker_password",
"minio_root_user",
"minio_root_password",
"s3_access_key_id",
"s3_secret_access_key",
):
path = directory / name
try:
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444)
+12 -8
View File
@@ -1,23 +1,27 @@
#!/usr/bin/env python3
"""Full OpenAPI and JSON Schema checks; install requirements-dev.txt first."""
import json
from pathlib import Path
from openapi_spec_validator import validate
from jsonschema import Draft202012Validator
root = Path(__file__).resolve().parents[1]
spec = json.loads((root/'contracts/openapi.json').read_text())
spec = json.loads((root / "contracts/openapi.json").read_text())
validate(spec)
from referencing import Registry, Resource
# The base URI resolves local #/components references inside examples.
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
registry = Registry().with_resource('urn:messenger:openapi', Resource.from_contents(resource))
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
schema = {'$ref': 'urn:messenger:openapi#/components/schemas/' + name}
resource = dict(spec, **{"$schema": "https://json-schema.org/draft/2020-12/schema"})
registry = Registry().with_resource(
"urn:messenger:openapi", Resource.from_contents(resource)
)
for name, example in json.loads((root / "contracts/examples.json").read_text()).items():
schema = {"$ref": "urn:messenger:openapi#/components/schemas/" + name}
Draft202012Validator(schema, registry=registry).validate(example)
for path in (root/'contracts').glob('*.schema.json'):
for path in (root / "contracts").glob("*.schema.json"):
schema = json.loads(path.read_text())
Draft202012Validator.check_schema(schema)
for example in schema.get('examples',[]):
for example in schema.get("examples", []):
Draft202012Validator(schema).validate(example)
print('PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.')
print("PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.")
+164 -70
View File
@@ -28,7 +28,7 @@ from app.storage import attachments as repository
from app.storage.s3 import ObjectInfo
from helpers import bearer, create_chat, create_user, idempotency_header
PNG = Path('tests/fixtures/sample.png').read_bytes()
PNG = Path("tests/fixtures/sample.png").read_bytes()
class MemoryStorage:
@@ -55,12 +55,15 @@ class MemoryStorage:
info = None
if key in self.objects:
data, metadata = self.objects[key]
info = ObjectInfo(key, len(data), metadata['content_type'], datetime.now(timezone.utc))
info = ObjectInfo(
key, len(data), metadata["content_type"], datetime.now(timezone.utc)
)
return info
async def open_object(self, key):
async def chunks():
yield self.objects[key][0]
return chunks()
async def check_available(self):
@@ -76,23 +79,40 @@ async def context(client, app):
return storage, alice, bob, eve, chat
async def upload(client, actor, chat, data=PNG, filename='фото.png', content_type='image/png'):
async def upload(
client, actor, chat, data=PNG, filename="фото.png", content_type="image/png"
):
return await client.post(
f"/api/v1/chats/{chat['id']}/attachments", params={'filename': filename},
headers={**bearer(actor), 'Content-Type': content_type}, content=data,
f"/api/v1/chats/{chat['id']}/attachments",
params={"filename": filename},
headers={**bearer(actor), "Content-Type": content_type},
content=data,
)
async def finish_attachment(client, storage, actor, attachment_id):
engine = client._transport.app.state.engine
async with AsyncSession(engine) as session:
job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)))
message = JobMessage(job_id=job.id, attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version, attempt=job.attempt + 1, created_at=job.created_at)
job = await session.scalar(
select(AttachmentJob).where(
AttachmentJob.attachment_id == UUID(attachment_id)
)
)
message = JobMessage(
job_id=job.id,
attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version,
attempt=job.attempt + 1,
created_at=job.created_at,
)
await session.rollback()
assert await process_attachment_job(session, storage, message, JobSettings(_env_file=None))
result = await client.get(f'/api/v1/attachments/{attachment_id}', headers=bearer(actor))
assert result.json()['status'] == 'ready'
assert await process_attachment_job(
session, storage, message, JobSettings(_env_file=None)
)
result = await client.get(
f"/api/v1/attachments/{attachment_id}", headers=bearer(actor)
)
assert result.json()["status"] == "ready"
return result.json()
@@ -101,37 +121,52 @@ async def test_upload_download_and_attachment_only_message(client, context):
response = await upload(client, alice, chat)
assert response.status_code == 202, response.text
attachment = response.json()
assert attachment['status'] == 'queued'
before = await client.get(f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice))
assert attachment["status"] == "queued"
before = await client.get(
f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice)
)
assert before.status_code == 409
attachment = await finish_attachment(client, storage, alice, attachment['id'])
assert attachment['sha256'] == hashlib.sha256(PNG).hexdigest()
assert {v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'}
assert attachment['error_code'] is None
assert 'object_key' not in attachment
id = attachment['id']
attachment = await finish_attachment(client, storage, alice, attachment["id"])
assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest()
assert {v["name"] for v in attachment["variants"]} == {
"thumbnail",
"crop",
"optimized",
}
assert attachment["error_code"] is None
assert "object_key" not in attachment
id = attachment["id"]
for actor in (alice, bob):
content = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(actor))
content = await client.get(
f"/api/v1/attachments/{id}/content", headers=bearer(actor)
)
assert content.status_code == 200 and content.content == PNG
assert content.headers['content-type'] == 'image/png'
assert content.headers["content-type"] == "image/png"
key = idempotency_header()
body = {'text': '', 'attachment_ids': [id]}
body = {"text": "", "attachment_ids": [id]}
path = f"/api/v1/chats/{chat['id']}/messages"
message = await client.post(path, json=body, headers={**bearer(bob), **key})
assert message.status_code == 201, message.text
repeat = await client.post(path, json=body, headers={**bearer(bob), **key})
assert repeat.json() == message.json()
history = await client.get(path, headers=bearer(alice))
assert history.json()['items'][0]['attachment_ids'] == [id]
for suffix in ('', '/content'):
denied = await client.get(f'/api/v1/attachments/{id}{suffix}', headers=bearer(eve))
assert history.json()["items"][0]["attachment_ids"] == [id]
for suffix in ("", "/content"):
denied = await client.get(
f"/api/v1/attachments/{id}{suffix}", headers=bearer(eve)
)
assert denied.status_code == 404
@pytest.mark.parametrize(
'data,content_type',
[(b'not an image', 'image/png'), (PNG[:30], 'image/png'), (PNG, 'image/jpeg'), (PNG, 'application/octet-stream')],
ids=['not-image', 'truncated-png', 'mime-spoofing', 'unsupported-type'],
"data,content_type",
[
(b"not an image", "image/png"),
(PNG[:30], "image/png"),
(PNG, "image/jpeg"),
(PNG, "application/octet-stream"),
],
ids=["not-image", "truncated-png", "mime-spoofing", "unsupported-type"],
)
async def test_invalid_image_and_mime_are_415(client, context, data, content_type):
storage, alice, _, _, chat = context
@@ -143,17 +178,21 @@ async def test_invalid_image_and_mime_are_415(client, context, data, content_typ
async def test_jpeg_is_stored_without_reencoding(client, context):
storage, alice, _, _, chat = context
output = BytesIO()
Image.new('RGB', (32, 32), 'blue').save(output, format='JPEG')
Image.new("RGB", (32, 32), "blue").save(output, format="JPEG")
original = output.getvalue()
response = await upload(client, alice, chat, original, content_type='image/jpeg')
response = await upload(client, alice, chat, original, content_type="image/jpeg")
assert response.status_code == 202, response.text
id = response.json()['id']
id = response.json()["id"]
await finish_attachment(client, storage, alice, id)
downloaded = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice))
downloaded = await client.get(
f"/api/v1/attachments/{id}/content", headers=bearer(alice)
)
assert downloaded.content == original
@pytest.mark.parametrize('filename', ['../a.png', 'a\\b.png', 'a\x00.png', 'a\n.png', 'x' * 256])
@pytest.mark.parametrize(
"filename", ["../a.png", "a\\b.png", "a\x00.png", "a\n.png", "x" * 256]
)
async def test_unsafe_filename_is_400(client, context, filename):
storage, alice, _, _, chat = context
response = await upload(client, alice, chat, filename=filename)
@@ -163,9 +202,11 @@ async def test_unsafe_filename_is_400(client, context, filename):
async def test_oversize_chunked_upload_is_413(client, context):
storage, alice, _, _, chat = context
async def chunks():
for _ in range(11):
yield b'x' * 1048576
yield b"x" * 1048576
response = await upload(client, alice, chat, chunks())
assert response.status_code == 413
assert not storage.objects
@@ -175,7 +216,12 @@ async def test_oversize_content_length_is_413(client, context):
_, alice, _, _, chat = context
response = await client.post(
f"/api/v1/chats/{chat['id']}/attachments?filename=a.png",
headers={**bearer(alice), 'Content-Type': 'image/png', 'Content-Length': '10485761'}, content=PNG,
headers={
**bearer(alice),
"Content-Type": "image/png",
"Content-Length": "10485761",
},
content=PNG,
)
assert response.status_code == 413
@@ -185,30 +231,47 @@ async def test_exact_byte_limit_accepts_valid_png(client, context):
maximum = 10_485_760
# Валидный ancillary chunk вместо данных после IEND.
payload = bytes(maximum - len(PNG) - 12)
kind = b'npAD'
padding = struct.pack('!I', len(payload)) + kind + payload + struct.pack('!I', zlib.crc32(kind + payload))
kind = b"npAD"
padding = (
struct.pack("!I", len(payload))
+ kind
+ payload
+ struct.pack("!I", zlib.crc32(kind + payload))
)
image = PNG[:-12] + padding + PNG[-12:]
response = await upload(client, alice, chat, image)
assert response.status_code == 202, response.text
assert response.json()['size_bytes'] == maximum
assert response.json()['sha256'] == hashlib.sha256(image).hexdigest()
assert response.json()["size_bytes"] == maximum
assert response.json()["sha256"] == hashlib.sha256(image).hexdigest()
async def test_exact_pixel_limit_accepts_valid_png(client, context):
_, alice, _, _, chat = context
output = BytesIO()
with Image.new('RGB', (8000, 5000), 'blue') as image:
image.save(output, format='PNG')
with Image.new("RGB", (8000, 5000), "blue") as image:
image.save(output, format="PNG")
response = await upload(client, alice, chat, output.getvalue())
assert response.status_code == 202, response.text
async def test_pixel_limit_before_decoding(client, context):
_, alice, _, _, chat = context
def chunk(kind, data):
return struct.pack('!I', len(data)) + kind + data + struct.pack('!I', zlib.crc32(kind + data))
header = struct.pack('!IIBBBBB', 8000, 5001, 8, 2, 0, 0, 0)
image = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(b'\0')) + chunk(b'IEND', b'')
return (
struct.pack("!I", len(data))
+ kind
+ data
+ struct.pack("!I", zlib.crc32(kind + data))
)
header = struct.pack("!IIBBBBB", 8000, 5001, 8, 2, 0, 0, 0)
image = (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(b"\0"))
+ chunk(b"IEND", b"")
)
response = await upload(client, alice, chat, image)
assert response.status_code == 413, response.text
@@ -217,10 +280,10 @@ async def test_unknown_and_other_chat_attachment_are_404(client, context):
_, alice, bob, _, chat = context
attachment = (await upload(client, alice, chat)).json()
other_chat = await create_chat(client, alice, bob)
for id in (attachment['id'], str(uuid4())):
for id in (attachment["id"], str(uuid4())):
response = await client.post(
f"/api/v1/chats/{other_chat['id']}/messages",
json={'text': '', 'attachment_ids': [id]},
json={"text": "", "attachment_ids": [id]},
headers={**bearer(alice), **idempotency_header()},
)
assert response.status_code == 404
@@ -228,10 +291,12 @@ async def test_unknown_and_other_chat_attachment_are_404(client, context):
async def test_duplicate_ids_are_400(client, context):
_, alice, _, _, chat = context
id = (await upload(client, alice, chat)).json()['id']
response = await client.post(f"/api/v1/chats/{chat['id']}/messages",
json={'text': '', 'attachment_ids': [id, id]},
headers={**bearer(alice), **idempotency_header()})
id = (await upload(client, alice, chat)).json()["id"]
response = await client.post(
f"/api/v1/chats/{chat['id']}/messages",
json={"text": "", "attachment_ids": [id, id]},
headers={**bearer(alice), **idempotency_header()},
)
assert response.status_code == 400
@@ -239,20 +304,24 @@ async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context):
_, alice, _, _, chat = context
headers = {**bearer(alice), **idempotency_header()}
path = f"/api/v1/chats/{chat['id']}/messages"
first = await client.post(path, json={'text': 'same'}, headers=headers)
second = await client.post(path, json={'text': 'same', 'attachment_ids': []}, headers=headers)
first = await client.post(path, json={"text": "same"}, headers=headers)
second = await client.post(
path, json={"text": "same", "attachment_ids": []}, headers=headers
)
assert first.status_code == second.status_code == 201
assert first.json() == second.json()
async def test_s3_outage_is_503(client, context):
storage, alice, _, _, chat = context
id = (await upload(client, alice, chat)).json()['id']
id = (await upload(client, alice, chat)).json()["id"]
await finish_attachment(client, storage, alice, id)
storage.down = True
response = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice))
response = await client.get(
f"/api/v1/attachments/{id}/content", headers=bearer(alice)
)
assert response.status_code == 503
assert (await client.get('/health/ready')).status_code == 503
assert (await client.get("/health/ready")).status_code == 503
async def test_uncertain_put_is_compensated(client, context, db_engine):
@@ -267,25 +336,31 @@ async def test_uncertain_put_is_compensated(client, context, db_engine):
async def test_failure_after_s3_write_compensates(client, context, monkeypatch):
storage, alice, _, _, chat = context
async def fail(*args, **kwargs):
raise OperationalError('injected', {}, None)
monkeypatch.setattr(repository, 'create_attachment', fail)
raise OperationalError("injected", {}, None)
monkeypatch.setattr(repository, "create_attachment", fail)
response = await upload(client, alice, chat)
assert response.status_code == 503
assert not storage.objects
async def test_ambiguous_successful_commit_keeps_original(client, context, db_engine, monkeypatch):
async def test_ambiguous_successful_commit_keeps_original(
client, context, db_engine, monkeypatch
):
storage, alice, _, _, chat = context
original = AsyncSession.commit
count = 0
async def uncertain(session):
nonlocal count
count += 1
await original(session)
if count == 2:
raise OperationalError('lost commit acknowledgement', {}, None)
monkeypatch.setattr(AsyncSession, 'commit', uncertain)
raise OperationalError("lost commit acknowledgement", {}, None)
monkeypatch.setattr(AsyncSession, "commit", uncertain)
response = await upload(client, alice, chat)
assert response.status_code == 503
assert len(storage.objects) == 1
@@ -318,10 +393,16 @@ async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_
storage, _, _, _, _ = context
old_id, fresh_id = uuid4(), uuid4()
async with AsyncSession(db_engine) as session:
session.add_all([
UploadIntent(id=old_id, object_key='originals/old', created_at=datetime.now(timezone.utc) - timedelta(days=2)),
UploadIntent(id=fresh_id, object_key='originals/fresh'),
])
session.add_all(
[
UploadIntent(
id=old_id,
object_key="originals/old",
created_at=datetime.now(timezone.utc) - timedelta(days=2),
),
UploadIntent(id=fresh_id, object_key="originals/fresh"),
]
)
await session.commit()
async with AsyncSession(db_engine) as uploader, AsyncSession(db_engine) as cleaner:
await repository.lock_upload(uploader, old_id)
@@ -335,9 +416,11 @@ async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_
async def test_client_disconnect_before_put_has_no_side_effects(client, context):
storage, alice, _, _, chat = context
async def interrupted():
yield PNG[:16]
raise ClientDisconnect()
response = await upload(client, alice, chat, interrupted())
assert response.status_code == 400
assert not storage.objects
@@ -345,10 +428,14 @@ async def test_client_disconnect_before_put_has_no_side_effects(client, context)
async def test_upload_deadline_cancels_validation(client, context, monkeypatch):
storage, alice, _, _, chat = context
storage.settings = storage.settings.model_copy(update={'upload_timeout_seconds': 0.02})
storage.settings = storage.settings.model_copy(
update={"upload_timeout_seconds": 0.02}
)
async def stalled(*args):
await asyncio.sleep(10)
monkeypatch.setattr(domain, 'validate_image', stalled)
monkeypatch.setattr(domain, "validate_image", stalled)
response = await upload(client, alice, chat)
assert response.status_code == 400
assert not storage.objects
@@ -358,11 +445,13 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa
storage, alice, _, _, chat = context
entered = asyncio.Event()
original = storage.put_file
async def interrupted(*args, **kwargs):
await original(*args, **kwargs)
entered.set()
await asyncio.sleep(10)
monkeypatch.setattr(storage, 'put_file', interrupted)
monkeypatch.setattr(storage, "put_file", interrupted)
task = asyncio.create_task(upload(client, alice, chat))
await asyncio.wait_for(entered.wait(), timeout=10)
task.cancel()
@@ -375,14 +464,19 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa
async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine):
storage, alice, _, _, chat = context
id = (await upload(client, alice, chat)).json()['id']
id = (await upload(client, alice, chat)).json()["id"]
await finish_attachment(client, storage, alice, id)
async with AsyncSession(db_engine) as session:
attachment = await session.get(Attachment, UUID(id))
key = attachment.object_key
# Искусственно оставляем старый журнал у уже подтверждённого объекта.
session.add(UploadIntent(id=uuid4(), object_key=key,
created_at=datetime.now(timezone.utc) - timedelta(days=2)))
session.add(
UploadIntent(
id=uuid4(),
object_key=key,
created_at=datetime.now(timezone.utc) - timedelta(days=2),
)
)
await session.commit()
result = await domain.reconcile_attachments(session, storage, dry_run=False)
assert result.examined == 1 and result.deleted == 0
+8 -1
View File
@@ -8,7 +8,14 @@ from sqlalchemy import text
from app.main import create_app
from conftest import close_app, make_client
from helpers import PASSWORD, basic_header, bearer, create_user, idempotency_header, register
from helpers import (
PASSWORD,
basic_header,
bearer,
create_user,
idempotency_header,
register,
)
async def test_parallel_registration_of_same_username_creates_exactly_one_user(
+2 -4
View File
@@ -29,14 +29,12 @@ async def seed_messages(
) -> None:
async with engine.begin() as connection:
await connection.execute(
text(
"""
text("""
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), :chat_id, :sender_id, 'message ' || g,
now() - make_interval(secs => :count - g)
FROM generate_series(1, :count) AS g
"""
),
"""),
{"chat_id": chat_id, "sender_id": sender_id, "count": count},
)
await connection.execute(text("ANALYZE messages"))
+53 -19
View File
@@ -16,10 +16,15 @@ from helpers import bearer, create_chat, create_user, idempotency_header, send_m
@pytest.fixture
async def replicas(request, make_settings):
overrides = getattr(request, "param", {})
apps = [create_app(make_settings(instance_id=name, **overrides)) for name in ("api-a", "api-b")]
apps = [
create_app(make_settings(instance_id=name, **overrides))
for name in ("api-a", "api-b")
]
try:
async with AsyncExitStack() as stack:
clients = [await stack.enter_async_context(make_client(app)) for app in apps]
clients = [
await stack.enter_async_context(make_client(app)) for app in apps
]
yield clients
finally:
for app in apps:
@@ -54,7 +59,11 @@ async def test_replay_and_conflict_across_replicas(replicas, db_engine):
original = await send_message(first, user, chat, "Привет", "same-key")
repeated = await second.post(
f"/api/v1/chats/{chat['id']}/messages",
headers={**bearer(user), **idempotency_header("same-key"), "Content-Type": "application/json"},
headers={
**bearer(user),
**idempotency_header("same-key"),
"Content-Type": "application/json",
},
content='{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442" }',
)
conflict = await send_message(second, user, chat, "другой текст", "same-key")
@@ -63,23 +72,38 @@ async def test_replay_and_conflict_across_replicas(replicas, db_engine):
assert original.headers["X-Instance-Id"] != repeated.headers["X-Instance-Id"]
assert conflict.status_code == 409
async with db_engine.connect() as connection:
assert (await connection.execute(text("SELECT count(*) FROM messages"))).scalar_one() == 1
assert (await connection.execute(text("SELECT value FROM counters WHERE name='messages'"))).scalar_one() == 1
assert (
await connection.execute(text("SELECT count(*) FROM messages"))
).scalar_one() == 1
assert (
await connection.execute(
text("SELECT value FROM counters WHERE name='messages'")
)
).scalar_one() == 1
async def test_parallel_replays_create_one_message(replicas, db_engine):
user = await create_user(replicas[0])
chat = await create_chat(replicas[0], user)
responses = await asyncio.gather(*(
send_message(replicas[index % 2], user, chat, "parallel", "one-key")
for index in range(24)
))
responses = await asyncio.gather(
*(
send_message(replicas[index % 2], user, chat, "parallel", "one-key")
for index in range(24)
)
)
assert all(response.status_code == 201 for response in responses)
assert len({response.json()["id"] for response in responses}) == 1
assert {response.headers["X-Instance-Id"] for response in responses} == {"api-a", "api-b"}
assert {response.headers["X-Instance-Id"] for response in responses} == {
"api-a",
"api-b",
}
async with db_engine.connect() as connection:
assert (await connection.execute(text("SELECT count(*) FROM messages"))).scalar_one() == 1
assert (await connection.execute(text("SELECT count(*) FROM idempotency_keys"))).scalar_one() == 1
assert (
await connection.execute(text("SELECT count(*) FROM messages"))
).scalar_one() == 1
assert (
await connection.execute(text("SELECT count(*) FROM idempotency_keys"))
).scalar_one() == 1
async def test_key_scope_separates_users_and_chats(client):
@@ -101,9 +125,11 @@ async def test_expired_key_can_be_reused_without_removing_message(client, db_eng
chat = await create_chat(client, user)
first = await send_message(client, user, chat, "old", "expiring-key")
async with db_engine.begin() as connection:
await connection.execute(text(
"UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'"
))
await connection.execute(
text(
"UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'"
)
)
second = await send_message(client, user, chat, "new", "expiring-key")
assert first.status_code == second.status_code == 201
assert first.json()["id"] != second.json()["id"]
@@ -127,7 +153,9 @@ async def test_failed_transaction_does_not_poison_key(client, db_engine, monkeyp
assert failed.status_code == 503
async with db_engine.connect() as connection:
for table in ("messages", "idempotency_keys", "counters"):
assert (await connection.execute(text(f"SELECT count(*) FROM {table}"))).scalar_one() == 0
assert (
await connection.execute(text(f"SELECT count(*) FROM {table}"))
).scalar_one() == 0
monkeypatch.setattr(key_storage, "save_result", original)
recovered = await send_message(client, user, chat, "retry", "retry-key")
assert recovered.status_code == 201
@@ -148,9 +176,15 @@ async def test_replay_works_when_message_limit_is_full(replicas):
async def test_replay_checks_session_and_membership(client, db_engine):
user = await create_user(client)
chat = await create_chat(client, user)
assert (await send_message(client, user, chat, "private", "private-key")).status_code == 201
assert (
await send_message(client, user, chat, "private", "private-key")
).status_code == 201
async with db_engine.begin() as connection:
await connection.execute(text("DELETE FROM chat_members"))
assert (await send_message(client, user, chat, "private", "private-key")).status_code == 404
assert (
await send_message(client, user, chat, "private", "private-key")
).status_code == 404
await client.delete("/api/v1/auth/sessions/current", headers=bearer(user))
assert (await send_message(client, user, chat, "private", "private-key")).status_code == 401
assert (
await send_message(client, user, chat, "private", "private-key")
).status_code == 401
+82 -40
View File
@@ -18,54 +18,96 @@ from app.storage.jobs import JobWork
from test_attachments import MemoryStorage
@pytest.mark.parametrize('size,transparent', [((1600,800),False),((24,12),True)])
async def test_variant_geometry_metadata_and_alpha(tmp_path,size,transparent):
source=tmp_path/'source.png'
with Image.new('RGBA',size,(255,0,0,0 if transparent else 255)) as image:
exif=Image.Exif();exif[270]='private description'
image.save(source,format='PNG',exif=exif)
result=create_variants(source,tmp_path,'image/png',40_000_000,ImageSettings(_env_file=None))
variants={v.name:v for v in result.variants}
assert (variants['crop'].width,variants['crop'].height)==(128,128)
assert (variants['thumbnail'].width,variants['thumbnail'].height)==((256,128) if size[0]>256 else size)
assert (variants['optimized'].width,variants['optimized'].height)==((1280,640) if size[0]>1280 else size)
for name in ('thumbnail.jpg','crop.jpg','optimized.webp'):
with Image.open(tmp_path/name) as image:
assert not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp')
@pytest.mark.parametrize("size,transparent", [((1600, 800), False), ((24, 12), True)])
async def test_variant_geometry_metadata_and_alpha(tmp_path, size, transparent):
source = tmp_path / "source.png"
with Image.new("RGBA", size, (255, 0, 0, 0 if transparent else 255)) as image:
exif = Image.Exif()
exif[270] = "private description"
image.save(source, format="PNG", exif=exif)
result = create_variants(
source, tmp_path, "image/png", 40_000_000, ImageSettings(_env_file=None)
)
variants = {v.name: v for v in result.variants}
assert (variants["crop"].width, variants["crop"].height) == (128, 128)
assert (variants["thumbnail"].width, variants["thumbnail"].height) == (
(256, 128) if size[0] > 256 else size
)
assert (variants["optimized"].width, variants["optimized"].height) == (
(1280, 640) if size[0] > 1280 else size
)
for name in ("thumbnail.jpg", "crop.jpg", "optimized.webp"):
with Image.open(tmp_path / name) as image:
assert (
not image.getexif()
and not image.info.get("icc_profile")
and not image.info.get("xmp")
)
if transparent:
if name.endswith('webp'):assert image.getpixel((0,0))[3]==0
else:assert min(image.getpixel((0,0)))>=250
if name.endswith("webp"):
assert image.getpixel((0, 0))[3] == 0
else:
assert min(image.getpixel((0, 0))) >= 250
async def test_exif_orientation_applied(tmp_path):
source=tmp_path/'source.jpg'
with Image.new('RGB',(400,200),'blue') as image:
exif=Image.Exif();exif[274]=6
image.save(source,exif=exif)
result=create_variants(source,tmp_path,'image/jpeg',40_000_000,ImageSettings(_env_file=None))
assert (result.variants[0].width,result.variants[0].height)==(128,256)
source = tmp_path / "source.jpg"
with Image.new("RGB", (400, 200), "blue") as image:
exif = Image.Exif()
exif[274] = 6
image.save(source, exif=exif)
result = create_variants(
source, tmp_path, "image/jpeg", 40_000_000, ImageSettings(_env_file=None)
)
assert (result.variants[0].width, result.variants[0].height) == (128, 256)
@pytest.mark.parametrize('failure', ['memory','cpu'])
async def test_worker_os_limits(failure,tmp_path):
body='return bytearray(2 * 1024 ** 3)' if failure=='memory' else 'while True: pass'
code="from app.core import thumbnail_worker as w\ndef fail(*args):\n "+body+"\nw.create_variants=fail\nraise SystemExit(w.main())"
result=await asyncio.to_thread(subprocess.run,[sys.executable,'-c',code,'source',str(tmp_path),str(tmp_path/'metadata'),'image/png','40000000'],
env={**os.environ,'IMAGE_MEMORY_MIB':'256','IMAGE_CPU_SECONDS':'1'},capture_output=True,timeout=15)
assert result.returncode==(3 if failure=='memory' else -9)
assert not (tmp_path/'metadata').exists()
@pytest.mark.parametrize("failure", ["memory", "cpu"])
async def test_worker_os_limits(failure, tmp_path):
body = (
"return bytearray(2 * 1024 ** 3)" if failure == "memory" else "while True: pass"
)
code = (
"from app.core import thumbnail_worker as w\ndef fail(*args):\n "
+ body
+ "\nw.create_variants=fail\nraise SystemExit(w.main())"
)
result = await asyncio.to_thread(
subprocess.run,
[
sys.executable,
"-c",
code,
"source",
str(tmp_path),
str(tmp_path / "metadata"),
"image/png",
"40000000",
],
env={**os.environ, "IMAGE_MEMORY_MIB": "256", "IMAGE_CPU_SECONDS": "1"},
capture_output=True,
timeout=15,
)
assert result.returncode == (3 if failure == "memory" else -9)
assert not (tmp_path / "metadata").exists()
async def test_render_timeout_kills_child(monkeypatch,tmp_path):
create=asyncio.create_subprocess_exec
children=[]
async def slow_child(*args,**kwargs):
child=await create(sys.executable,'-c','import time;time.sleep(60)',**kwargs)
async def test_render_timeout_kills_child(monkeypatch, tmp_path):
create = asyncio.create_subprocess_exec
children = []
async def slow_child(*args, **kwargs):
child = await create(
sys.executable, "-c", "import time;time.sleep(60)", **kwargs
)
children.append(child)
return child
monkeypatch.setattr(asyncio,'create_subprocess_exec',slow_child)
work=JobWork(uuid4(),uuid4(),uuid4(),1,'staging/test','image/png',1,'0'*64)
monkeypatch.setattr(asyncio, "create_subprocess_exec", slow_child)
work = JobWork(
uuid4(), uuid4(), uuid4(), 1, "staging/test", "image/png", 1, "0" * 64
)
with pytest.raises(TimeoutError):
async with asyncio.timeout(.2):
await render_variants(MemoryStorage(),work,tmp_path/'source',tmp_path)
assert len(children)==1 and children[0].returncode is not None
async with asyncio.timeout(0.2):
await render_variants(MemoryStorage(), work, tmp_path / "source", tmp_path)
assert len(children) == 1 and children[0].returncode is not None
+6 -2
View File
@@ -92,7 +92,9 @@ async def test_instance_header_is_present_on_success_and_errors(
assert response.headers["X-Instance-Id"] == "api-test"
assert response.headers["X-Request-Id"]
if status >= 400:
assert response.json()["error"]["request_id"] == response.headers["X-Request-Id"]
assert (
response.json()["error"]["request_id"] == response.headers["X-Request-Id"]
)
async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_client):
@@ -100,7 +102,9 @@ async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_cli
second = await instance_client.get("/__test__/request-id")
assert first.status_code == second.status_code == 200
assert first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test"
assert (
first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test"
)
assert first.headers["X-Request-Id"] != second.headers["X-Request-Id"]
for response in (first, second):
assert response.json()["request_id"] == response.headers["X-Request-Id"]
+137 -48
View File
@@ -13,7 +13,12 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.core.jobs import JobSettings
from app.domain import attachment_jobs
from app.domain.attachment_jobs import claim_job, process_attachment_job, cleanup_staging, build_result
from app.domain.attachment_jobs import (
claim_job,
process_attachment_job,
cleanup_staging,
build_result,
)
from app.domain.job_admin import replay_failed_job
from app.domain.job_retry import classify_failure, retry_delay
from app.errors.exceptions import JobOperationError, ProcessingError
@@ -26,50 +31,85 @@ from test_attachments import context, upload, finish_attachment
async def envelope(session, attachment_id):
job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)))
return JobMessage(job_id=job.id, attachment_id=job.attachment_id, pipeline_version=1,
attempt=job.attempt + 1, created_at=job.created_at)
job = await session.scalar(
select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))
)
return JobMessage(
job_id=job.id,
attachment_id=job.attachment_id,
pipeline_version=1,
attempt=job.attempt + 1,
created_at=job.created_at,
)
async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_engine):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
settings = JobSettings(_env_file=None, job_max_attempts=3)
def fail(_):
raise TimeoutError
async with AsyncSession(db_engine, expire_on_commit=False) as session:
first = await envelope(session, attachment['id'])
first = await envelope(session, attachment["id"])
for attempt in range(1, 4):
message = await envelope(session, attachment['id'])
assert await process_attachment_job(session, storage, message, settings, before_processing=fail)
message = await envelope(session, attachment["id"])
assert await process_attachment_job(
session, storage, message, settings, before_processing=fail
)
job, item = await jobs.lock_job(session, first.job_id)
assert job.attempt == attempt
delay = (job.next_attempt_at - item.updated_at).total_seconds()
assert 5 * 2 ** (attempt - 1) / 2 <= delay <= 5 * 2 ** (attempt - 1)
assert item.status == ('failed' if attempt == 3 else 'queued')
assert item.status == ("failed" if attempt == 3 else "queued")
assert job.last_error_permanent is False
job.next_attempt_at = await jobs.database_time(session) - timedelta(seconds=1)
job.next_attempt_at = await jobs.database_time(session) - timedelta(
seconds=1
)
await session.commit()
assert item.error_code == 'retry_exhausted'
assert item.error_code == "retry_exhausted"
request_id = uuid4()
result = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id,
actor='test', reason='dependency_restored')
assert result['attempt_limit'] == 6
result = await replay_failed_job(
session,
storage,
settings,
first.job_id,
request_id=request_id,
actor="test",
reason="dependency_restored",
)
assert result["attempt_limit"] == 6
# Old envelope cannot consume the fresh replay budget.
assert await process_attachment_job(session, storage, first, settings)
job, item = await jobs.lock_job(session, first.job_id)
assert job.attempt == 3 and item.status == 'queued'
assert job.attempt == 3 and item.status == "queued"
await session.commit()
message = await envelope(session, attachment['id'])
message = await envelope(session, attachment["id"])
assert await process_attachment_job(session, storage, message, settings)
before = dict(storage.objects)
assert await process_attachment_job(session, storage, message, settings)
assert storage.objects == before
repeat = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id,
actor='test', reason='dependency_restored')
assert repeat['already_applied']
with pytest.raises(JobOperationError, match='job_not_failed'):
await replay_failed_job(session, storage, settings, first.job_id, request_id=uuid4(), actor='test', reason='manual_review')
repeat = await replay_failed_job(
session,
storage,
settings,
first.job_id,
request_id=request_id,
actor="test",
reason="dependency_restored",
)
assert repeat["already_applied"]
with pytest.raises(JobOperationError, match="job_not_failed"):
await replay_failed_job(
session,
storage,
settings,
first.job_id,
request_id=uuid4(),
actor="test",
reason="manual_review",
)
await session.rollback()
assert len((await session.scalars(select(JobReplay))).all()) == 1
@@ -77,26 +117,40 @@ async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_eng
async def test_permanent_failure_and_missing_source_replay(client, context, db_engine):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
def fail(_):
raise ProcessingError('image_limits')
raise ProcessingError("image_limits")
settings = JobSettings(_env_file=None)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
await process_attachment_job(session, storage, message, settings, before_processing=fail)
message = await envelope(session, attachment["id"])
await process_attachment_job(
session, storage, message, settings, before_processing=fail
)
job, item = await jobs.lock_job(session, message.job_id)
assert item.status == 'failed' and job.attempt == 1 and job.last_error_permanent
assert item.status == "failed" and job.attempt == 1 and job.last_error_permanent
job.staging_key = None
await session.commit()
with pytest.raises(JobOperationError, match='source_expired'):
await replay_failed_job(session, storage, settings, job.id, request_id=uuid4(), actor='test', reason='manual_review')
with pytest.raises(JobOperationError, match="source_expired"):
await replay_failed_job(
session,
storage,
settings,
job.id,
request_id=uuid4(),
actor="test",
reason="manual_review",
)
async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, context, db_engine):
async def test_stale_worker_cannot_write_and_cleanup_keeps_active(
client, context, db_engine
):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
settings = JobSettings(_env_file=None)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
message = await envelope(session, attachment["id"])
claim = await claim_job(session, message, settings)
job, item = await jobs.lock_job(session, message.job_id)
job.lease_token = uuid4()
@@ -112,18 +166,33 @@ async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, contex
async def test_cleanup_ready_preserves_final_objects(client, context, db_engine):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
await finish_attachment(client, storage, alice, attachment['id'])
await finish_attachment(client, storage, alice, attachment["id"])
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
message = await envelope(session, attachment["id"])
job, item = await jobs.lock_job(session, message.job_id)
staging = job.staging_key
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
await session.commit()
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=False) == 1
assert (
await cleanup_staging(
session, storage, JobSettings(_env_file=None), apply=False
)
== 1
)
assert staging in storage.objects
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 1
assert (
await cleanup_staging(
session, storage, JobSettings(_env_file=None), apply=True
)
== 1
)
assert staging not in storage.objects and len(storage.objects) == 4
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 0
assert (
await cleanup_staging(
session, storage, JobSettings(_env_file=None), apply=True
)
== 0
)
async def test_retry_classification_and_cap():
@@ -131,44 +200,64 @@ async def test_retry_classification_and_cap():
for attempt in (1, 2, 6, 100):
cap = min(120, 5 * 2 ** min(30, attempt - 1))
assert all(cap / 2 <= retry_delay(settings, attempt) <= cap for _ in range(100))
assert classify_failure(TimeoutError()) == ('processing_timeout', False)
assert classify_failure(OSError(errno.EACCES, 'denied'))[1]
assert not classify_failure(OSError(errno.ENOSPC, 'full'))[1]
assert classify_failure(TimeoutError()) == ("processing_timeout", False)
assert classify_failure(OSError(errno.EACCES, "denied"))[1]
assert not classify_failure(OSError(errno.ENOSPC, "full"))[1]
async def test_timeout_records_retry_and_clears_lease(client, context, db_engine, monkeypatch):
async def test_timeout_records_retry_and_clears_lease(
client, context, db_engine, monkeypatch
):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
async def stalled(*args, **kwargs):
await asyncio.sleep(60)
monkeypatch.setattr(attachment_jobs, 'build_result', stalled)
settings = JobSettings(_env_file=None).model_copy(update={'job_timeout_seconds': .05})
monkeypatch.setattr(attachment_jobs, "build_result", stalled)
settings = JobSettings(_env_file=None).model_copy(
update={"job_timeout_seconds": 0.05}
)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
message = await envelope(session, attachment["id"])
assert await process_attachment_job(session, storage, message, settings)
job, item = await jobs.lock_job(session, message.job_id)
assert item.status == 'queued' and job.last_error_code == 'processing_timeout'
assert item.status == "queued" and job.last_error_code == "processing_timeout"
assert job.lease_token is None and job.lease_until is None
async def test_failed_retention_and_source_checksum_protect_replay(client, context, db_engine):
async def test_failed_retention_and_source_checksum_protect_replay(
client, context, db_engine
):
storage, alice, _, _, chat = context
attachment = (await upload(client, alice, chat)).json()
def fail(_):
raise ProcessingError('invalid_image')
raise ProcessingError("invalid_image")
settings = JobSettings(_env_file=None)
async with AsyncSession(db_engine, expire_on_commit=False) as session:
message = await envelope(session, attachment['id'])
await process_attachment_job(session, storage, message, settings, before_processing=fail)
message = await envelope(session, attachment["id"])
await process_attachment_job(
session, storage, message, settings, before_processing=fail
)
job, item = await jobs.lock_job(session, message.job_id)
key = job.staging_key
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
await session.commit()
assert await cleanup_staging(session, storage, settings, apply=True) == 0
data, metadata = storage.objects[key]
storage.objects[key] = (b'x' * len(data), metadata)
with pytest.raises(JobOperationError, match='source_mismatch'):
await replay_failed_job(session, storage, settings, message.job_id, request_id=uuid4(), actor='test', reason='manual_review')
storage.objects[key] = (b"x" * len(data), metadata)
with pytest.raises(JobOperationError, match="source_mismatch"):
await replay_failed_job(
session,
storage,
settings,
message.job_id,
request_id=uuid4(),
actor="test",
reason="manual_review",
)
await session.rollback()
assert not (await session.scalars(select(JobReplay))).all()
job, item = await jobs.lock_job(session, message.job_id)
+6 -6
View File
@@ -20,9 +20,9 @@ async def test_draining_keeps_liveness_but_rejects_new_work(make_settings):
app.state.lifecycle.start_draining()
try:
async with make_client(app) as client:
assert (await client.get('/health/live')).status_code == 200
assert (await client.get('/health/ready')).status_code == 503
assert (await client.get('/api/v1/users/me')).status_code == 503
assert (await client.get("/health/live")).status_code == 200
assert (await client.get("/health/ready")).status_code == 503
assert (await client.get("/api/v1/users/me")).status_code == 503
finally:
await close_app(app)
@@ -36,13 +36,13 @@ async def test_readiness_timeout_cancels_dependency_checks(make_settings, monkey
finally:
cancelled.append(True)
monkeypatch.setattr(health, 'check_database', stalled)
monkeypatch.setattr(health, 'check_redis', stalled)
monkeypatch.setattr(health, "check_database", stalled)
monkeypatch.setattr(health, "check_redis", stalled)
app = create_app(make_settings(readiness_timeout_seconds=0.03))
try:
async with make_client(app) as client:
started = time.monotonic()
response = await client.get('/health/ready')
response = await client.get("/health/ready")
assert response.status_code == 503
assert time.monotonic() - started < 1
assert len(cancelled) == 2
+31 -7
View File
@@ -6,7 +6,13 @@ import httpx
from app.main import create_app
from conftest import close_app, make_client
from helpers import assert_request_id, bearer, create_chat, create_user, idempotency_header
from helpers import (
assert_request_id,
bearer,
create_chat,
create_user,
idempotency_header,
)
def _assert_error(response: httpx.Response, status: int, code: str) -> None:
@@ -25,12 +31,16 @@ async def test_message_limit_rejects_without_deleting_confirmed_messages(make_se
for index in range(2):
ok = await client.post(
url, headers={**bearer(user), **idempotency_header()}, json={"text": f"confirmed-{index}"}
url,
headers={**bearer(user), **idempotency_header()},
json={"text": f"confirmed-{index}"},
)
assert ok.status_code == 201
rejected = await client.post(
url, headers={**bearer(user), **idempotency_header()}, json={"text": "rejected"}
url,
headers={**bearer(user), **idempotency_header()},
json={"text": "rejected"},
)
_assert_error(rejected, 503, "unavailable")
@@ -65,12 +75,18 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat
user = await create_user(client)
chat = await create_chat(client, user)
monkeypatch.setattr(message_endpoint, "send_message_idempotent", blocking_send_message)
monkeypatch.setattr(
message_endpoint, "send_message_idempotent", blocking_send_message
)
url = f"/api/v1/chats/{chat['id']}/messages"
held = [
asyncio.create_task(
client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": f"held-{i}"})
client.post(
url,
headers={**bearer(user), **idempotency_header()},
json={"text": f"held-{i}"},
)
)
for i in range(2)
]
@@ -79,7 +95,11 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat
rejected = await asyncio.gather(
*(
client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": f"rejected-{i}"})
client.post(
url,
headers={**bearer(user), **idempotency_header()},
json={"text": f"rejected-{i}"},
)
for i in range(4)
)
)
@@ -92,7 +112,11 @@ async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypat
release.set()
assert all(r.status_code == 201 for r in await asyncio.gather(*held))
recovered = await client.post(url, headers={**bearer(user), **idempotency_header()}, json={"text": "after"})
recovered = await client.post(
url,
headers={**bearer(user), **idempotency_header()},
json={"text": "after"},
)
assert recovered.status_code == 201
assert_request_id(recovered)
await close_app(app)
+16 -4
View File
@@ -8,7 +8,19 @@ from sqlalchemy import inspect, text
from app.models import Base
from conftest import reset_schema, upgrade_head
EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents", "attachment_jobs", "job_replays"}
EXPECTED_TABLES = {
"users",
"chats",
"chat_members",
"messages",
"counters",
"idempotency_keys",
"attachments",
"message_attachments",
"upload_intents",
"attachment_jobs",
"job_replays",
}
@pytest.fixture
@@ -62,6 +74,6 @@ async def test_models_and_migrations_are_in_sync(migrated_engine):
async with migrated_engine.connect() as connection:
diff = await connection.run_sync(compare)
assert diff == [], (
f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}"
)
assert (
diff == []
), f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}"
+9 -2
View File
@@ -10,10 +10,12 @@ from app.storage.s3 import ObjectStream, run_sync
async def test_sdk_thread_finishes_before_cancellation_returns():
entered, release, finished = Event(), Event(), Event()
def operation():
entered.set()
release.wait(timeout=3)
finished.set()
task = asyncio.create_task(run_sync(operation))
await asyncio.to_thread(entered.wait, 2)
task.cancel()
@@ -31,10 +33,12 @@ async def test_response_discarded_when_get_is_cancelled():
entered, release = Event(), Event()
discarded = []
response = object()
def operation():
entered.set()
release.wait(timeout=3)
return response
task = asyncio.create_task(run_sync(operation, discard_result=discarded.append))
await asyncio.to_thread(entered.wait, 2)
task.cancel()
@@ -48,9 +52,12 @@ async def test_response_discarded_when_get_is_cancelled():
async def test_stream_closes_even_before_first_iteration():
calls = []
response = SimpleNamespace(close=lambda: calls.append('close'), release_conn=lambda: calls.append('release'))
response = SimpleNamespace(
close=lambda: calls.append("close"),
release_conn=lambda: calls.append("release"),
)
storage = SimpleNamespace(settings=S3Settings(_env_file=None))
stream = ObjectStream(storage, response)
await stream.aclose()
await stream.aclose()
assert calls == ['close', 'release']
assert calls == ["close", "release"]
+252 -120
View File
@@ -1,4 +1,5 @@
"""HTTP helpers for public black-box tests; Python 3.9+, standard library only."""
import base64
import datetime as dt
import hashlib
@@ -13,205 +14,336 @@ import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
LAB = json.loads((ROOT / 'lab.json').read_text())['number']
SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text())
LAB = json.loads((ROOT / "lab.json").read_text())["number"]
SPEC = json.loads((ROOT / "contracts/openapi.json").read_text())
class Failure(AssertionError):
pass
def check(condition, message):
if not condition:
raise Failure(message)
def timestamp(value):
check(isinstance(value, str) and value.endswith('Z'), 'timestamp must be UTC ending in Z')
try:
return dt.datetime.fromisoformat(value[:-1] + '+00:00')
except ValueError as exc:
raise Failure('invalid RFC3339 timestamp') from exc
def validate(value, schema, where='response'):
def timestamp(value):
check(
isinstance(value, str) and value.endswith("Z"),
"timestamp must be UTC ending in Z",
)
try:
return dt.datetime.fromisoformat(value[:-1] + "+00:00")
except ValueError as exc:
raise Failure("invalid RFC3339 timestamp") from exc
def validate(value, schema, where="response"):
"""Validate the JSON Schema subset used by the supplied contract, not arbitrary OpenAPI."""
if '$ref' in schema:
if "$ref" in schema:
target = SPEC
for part in schema['$ref'].split('/')[1:]:
for part in schema["$ref"].split("/")[1:]:
target = target[part]
return validate(value, target, where)
kinds = schema.get('type', [])
kinds = schema.get("type", [])
if isinstance(kinds, str):
kinds = [kinds]
matches = {'null':value is None, 'boolean':isinstance(value,bool), 'integer':isinstance(value,int) and not isinstance(value,bool), 'number':isinstance(value,(int,float)) and not isinstance(value,bool), 'string':isinstance(value,str), 'array':isinstance(value,list), 'object':isinstance(value,dict)}
matches = {
"null": value is None,
"boolean": isinstance(value, bool),
"integer": isinstance(value, int) and not isinstance(value, bool),
"number": isinstance(value, (int, float)) and not isinstance(value, bool),
"string": isinstance(value, str),
"array": isinstance(value, list),
"object": isinstance(value, dict),
}
if kinds:
check(any(matches.get(k,False) for k in kinds), where + ': incorrect type')
if 'enum' in schema:
check(value in schema['enum'], where + ': invalid enum')
if 'const' in schema:
check(value == schema['const'], where + ': invalid const')
if isinstance(value,dict):
props = schema.get('properties',{})
check(all(k in value for k in schema.get('required',[])), where + ': missing required fields')
if schema.get('additionalProperties') is False:
check(not set(value).difference(props), where + ': unexpected fields')
check(any(matches.get(k, False) for k in kinds), where + ": incorrect type")
if "enum" in schema:
check(value in schema["enum"], where + ": invalid enum")
if "const" in schema:
check(value == schema["const"], where + ": invalid const")
if isinstance(value, dict):
props = schema.get("properties", {})
check(
all(k in value for k in schema.get("required", [])),
where + ": missing required fields",
)
if schema.get("additionalProperties") is False:
check(not set(value).difference(props), where + ": unexpected fields")
for key, sub in props.items():
if key in value:
validate(value[key], sub, where + '.' + key)
if isinstance(value,list):
check(len(value) >= schema.get('minItems',0), where + ': too few items')
check(len(value) <= schema.get('maxItems',float('inf')), where + ': too many items')
if schema.get('uniqueItems'):
encoded = [json.dumps(x,sort_keys=True) for x in value]
check(len(encoded)==len(set(encoded)), where + ': duplicate items')
if 'items' in schema:
validate(value[key], sub, where + "." + key)
if isinstance(value, list):
check(len(value) >= schema.get("minItems", 0), where + ": too few items")
check(
len(value) <= schema.get("maxItems", float("inf")),
where + ": too many items",
)
if schema.get("uniqueItems"):
encoded = [json.dumps(x, sort_keys=True) for x in value]
check(len(encoded) == len(set(encoded)), where + ": duplicate items")
if "items" in schema:
for item in value:
validate(item, schema['items'], where + '[]')
if isinstance(value,str):
check(len(value) >= schema.get('minLength',0), where + ': too short')
check(len(value) <= schema.get('maxLength',float('inf')), where + ': too long')
if 'pattern' in schema:
check(re.search(schema['pattern'], value) is not None, where + ': pattern mismatch')
if schema.get('format') == 'uuid':
validate(item, schema["items"], where + "[]")
if isinstance(value, str):
check(len(value) >= schema.get("minLength", 0), where + ": too short")
check(len(value) <= schema.get("maxLength", float("inf")), where + ": too long")
if "pattern" in schema:
check(
re.search(schema["pattern"], value) is not None,
where + ": pattern mismatch",
)
if schema.get("format") == "uuid":
try:
uuid.UUID(value)
except ValueError as exc:
raise Failure(where + ': invalid UUID') from exc
if schema.get('format') == 'date-time':
raise Failure(where + ": invalid UUID") from exc
if schema.get("format") == "date-time":
timestamp(value)
if isinstance(value,(int,float)) and not isinstance(value,bool):
check(value >= schema.get('minimum',-float('inf')), where + ': below minimum')
check(value <= schema.get('maximum',float('inf')), where + ': above maximum')
if 'anyOf' in schema:
for alternative in schema['anyOf']:
if isinstance(value, (int, float)) and not isinstance(value, bool):
check(value >= schema.get("minimum", -float("inf")), where + ": below minimum")
check(value <= schema.get("maximum", float("inf")), where + ": above maximum")
if "anyOf" in schema:
for alternative in schema["anyOf"]:
try:
validate(value, alternative, where)
break
except Failure:
pass
else:
raise Failure(where + ': no anyOf branch matches')
raise Failure(where + ": no anyOf branch matches")
def shape(value, name):
validate(value, SPEC['components']['schemas'][name])
validate(value, SPEC["components"]["schemas"][name])
return value
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
class Client:
def __init__(self, base_url=None, ca_file=None, timeout=10):
self.base = (base_url or os.environ.get('BASE_URL') or ('https://localhost:8443' if LAB==7 else 'http://localhost:8080')).rstrip('/')
check(self.base.startswith(('http://','https://')), 'BASE_URL must be HTTP(S)')
check(LAB != 7 or self.base.startswith('https://'), 'lab7 requires HTTPS')
self.base = (
base_url
or os.environ.get("BASE_URL")
or ("https://localhost:8443" if LAB == 7 else "http://localhost:8080")
).rstrip("/")
check(self.base.startswith(("http://", "https://")), "BASE_URL must be HTTP(S)")
check(LAB != 7 or self.base.startswith("https://"), "lab7 requires HTTPS")
self.timeout = timeout
context = ssl.create_default_context(cafile=ca_file or os.environ.get('CA_FILE') or None)
self.opener = urllib.request.build_opener(NoRedirect(), urllib.request.HTTPSHandler(context=context))
context = ssl.create_default_context(
cafile=ca_file or os.environ.get("CA_FILE") or None
)
self.opener = urllib.request.build_opener(
NoRedirect(), urllib.request.HTTPSHandler(context=context)
)
def call(self, method, path, expected=200, body=None, actor=None, headers=None, raw=None, schema=None):
h = {'Accept':'application/json'}
def call(
self,
method,
path,
expected=200,
body=None,
actor=None,
headers=None,
raw=None,
schema=None,
):
h = {"Accept": "application/json"}
if actor:
h['X-User-Id' if LAB==1 else 'Authorization'] = actor['id'] if LAB==1 else 'Bearer ' + actor['token']
h["X-User-Id" if LAB == 1 else "Authorization"] = (
actor["id"] if LAB == 1 else "Bearer " + actor["token"]
)
if body is not None:
raw = json.dumps(body,ensure_ascii=False).encode('utf-8')
h['Content-Type'] = 'application/json'
raw = json.dumps(body, ensure_ascii=False).encode("utf-8")
h["Content-Type"] = "application/json"
h.update(headers or {})
req = urllib.request.Request(self.base + path, data=raw, headers=h, method=method)
req = urllib.request.Request(
self.base + path, data=raw, headers=h, method=method
)
start = time.monotonic()
try:
response = self.opener.open(req, timeout=self.timeout)
except urllib.error.HTTPError as error:
response = error
except (urllib.error.URLError,TimeoutError,OSError) as exc:
except (urllib.error.URLError, TimeoutError, OSError) as exc:
# Do not print URL query, headers, request body, or credentials.
raise Failure(method + ' ' + path.split('?')[0] + ': connection/TLS failure (' + type(exc).__name__ + ')') from None
raise Failure(
method
+ " "
+ path.split("?")[0]
+ ": connection/TLS failure ("
+ type(exc).__name__
+ ")"
) from None
with response:
status = response.code
rh = response.headers
data = response.read(16*1024*1024 + 1)
check(len(data) <= 16*1024*1024, 'response exceeds test safety limit')
allowed = [expected] if isinstance(expected,int) else expected
check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status))
check(bool(rh.get('X-Request-Id')), 'missing X-Request-Id')
if LAB>=3:
check(bool(rh.get('X-Instance-Id')), 'missing X-Instance-Id')
data = response.read(16 * 1024 * 1024 + 1)
check(len(data) <= 16 * 1024 * 1024, "response exceeds test safety limit")
allowed = [expected] if isinstance(expected, int) else expected
check(
status in allowed,
method
+ " "
+ path.split("?")[0]
+ ": expected "
+ str(allowed)
+ ", received "
+ str(status),
)
check(bool(rh.get("X-Request-Id")), "missing X-Request-Id")
if LAB >= 3:
check(bool(rh.get("X-Instance-Id")), "missing X-Instance-Id")
payload = None
if schema or status>=400:
check(rh.get_content_type()=='application/json', 'expected application/json')
if schema or status >= 400:
check(
rh.get_content_type() == "application/json", "expected application/json"
)
try:
payload = json.loads(data)
except (ValueError,UnicodeError):
raise Failure('invalid response JSON') from None
shape(payload, 'Error' if status>=400 else schema)
if status>=400:
check(payload['error']['request_id']==rh.get('X-Request-Id'),'error request_id differs from header')
if status==401 and LAB>=2:
scheme = 'Basic' if path=='/api/v1/auth/sessions' else 'Bearer'
check(rh.get('WWW-Authenticate','').lower().startswith(scheme.lower()), 'incorrect WWW-Authenticate challenge')
if status==204:
check(not data, '204 must not contain body')
return {'status':status,'headers':rh,'json':payload,'bytes':data,'elapsed':time.monotonic()-start}
except (ValueError, UnicodeError):
raise Failure("invalid response JSON") from None
shape(payload, "Error" if status >= 400 else schema)
if status >= 400:
check(
payload["error"]["request_id"] == rh.get("X-Request-Id"),
"error request_id differs from header",
)
if status == 401 and LAB >= 2:
scheme = "Basic" if path == "/api/v1/auth/sessions" else "Bearer"
check(
rh.get("WWW-Authenticate", "").lower().startswith(scheme.lower()),
"incorrect WWW-Authenticate challenge",
)
if status == 204:
check(not data, "204 must not contain body")
return {
"status": status,
"headers": rh,
"json": payload,
"bytes": data,
"elapsed": time.monotonic() - start,
}
def user(self, label):
username = 'u_' + label + '_' + uuid.uuid4().hex[:12]
password = 'T3st-' + uuid.uuid4().hex
body = {'username':username,'display_name':'Студент ' + label}
if LAB>=2:
body['password'] = password
user = self.call('POST','/api/v1/users',201,body=body,schema='User')['json']
check(user['username']==username,'username changed')
actor = {'id':user['id'],'user':user,'registration':body}
if LAB>=2:
credential = base64.b64encode((username+':'+password).encode()).decode()
session = self.call('POST','/api/v1/auth/sessions',201,headers={'Authorization':'Basic '+credential},schema='Session')['json']
check(session['user']==user,'session user mismatch')
check(timestamp(session['expires_at']) > dt.datetime.now(dt.timezone.utc),'session already expired')
actor['token'] = session['token']
username = "u_" + label + "_" + uuid.uuid4().hex[:12]
password = "T3st-" + uuid.uuid4().hex
body = {"username": username, "display_name": "Студент " + label}
if LAB >= 2:
body["password"] = password
user = self.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"]
check(user["username"] == username, "username changed")
actor = {"id": user["id"], "user": user, "registration": body}
if LAB >= 2:
credential = base64.b64encode((username + ":" + password).encode()).decode()
session = self.call(
"POST",
"/api/v1/auth/sessions",
201,
headers={"Authorization": "Basic " + credential},
schema="Session",
)["json"]
check(session["user"] == user, "session user mismatch")
check(
timestamp(session["expires_at"]) > dt.datetime.now(dt.timezone.utc),
"session already expired",
)
actor["token"] = session["token"]
return actor
def bootstrap(self):
self.call('GET','/health/live',schema='Health')
self.call('GET','/health/ready',schema='Health')
alice, bob, eve = (self.user(label) for label in ('alice','bob','eve'))
chat = self.call('POST','/api/v1/chats',201,actor=alice,body={'title':'Контрактный тест','member_ids':[alice['id'],bob['id']]},schema='Chat')['json']
check(set(chat['member_ids'])=={alice['id'],bob['id']},'chat member mismatch')
return alice,bob,eve,chat
self.call("GET", "/health/live", schema="Health")
self.call("GET", "/health/ready", schema="Health")
alice, bob, eve = (self.user(label) for label in ("alice", "bob", "eve"))
chat = self.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "Контрактный тест", "member_ids": [alice["id"], bob["id"]]},
schema="Chat",
)["json"]
check(
set(chat["member_ids"]) == {alice["id"], bob["id"]}, "chat member mismatch"
)
return alice, bob, eve, chat
def message(self, actor, chat, text, key=None, attachments=None):
body = {'text':text}
body = {"text": text}
if attachments is not None:
body['attachment_ids'] = attachments
headers = {'Idempotency-Key':key or uuid.uuid4().hex} if LAB>=3 else {}
return self.call('POST','/api/v1/chats/'+chat['id']+'/messages',201,actor=actor,body=body,headers=headers,schema='Message')
body["attachment_ids"] = attachments
headers = {"Idempotency-Key": key or uuid.uuid4().hex} if LAB >= 3 else {}
return self.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/messages",
201,
actor=actor,
body=body,
headers=headers,
schema="Message",
)
def attachment(self, actor, chat):
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
att = self.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',201 if LAB==4 else 202,actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')['json']
check(att['status']==('ready' if LAB==4 else 'queued'),'incorrect upload status')
check(att['chat_id']==chat['id'] and att['owner_id']==actor['id'],'attachment ownership mismatch')
check(att['sha256']==hashlib.sha256(content).hexdigest() and att['size_bytes']==len(content),'original checksum/size mismatch')
return att,content
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
att = self.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
201 if LAB == 4 else 202,
actor=actor,
raw=content,
headers={"Content-Type": "image/png"},
schema="Attachment",
)["json"]
check(
att["status"] == ("ready" if LAB == 4 else "queued"),
"incorrect upload status",
)
check(
att["chat_id"] == chat["id"] and att["owner_id"] == actor["id"],
"attachment ownership mismatch",
)
check(
att["sha256"] == hashlib.sha256(content).hexdigest()
and att["size_bytes"] == len(content),
"original checksum/size mismatch",
)
return att, content
def wait_ready(self, actor, att, wait_seconds=60):
deadline = time.monotonic()+wait_seconds
while time.monotonic()<deadline:
att = self.call('GET','/api/v1/attachments/'+att['id'],actor=actor,schema='Attachment')['json']
check(att['status']!='failed','attachment reached failed')
if att['status']=='ready':
deadline = time.monotonic() + wait_seconds
while time.monotonic() < deadline:
att = self.call(
"GET",
"/api/v1/attachments/" + att["id"],
actor=actor,
schema="Attachment",
)["json"]
check(att["status"] != "failed", "attachment reached failed")
if att["status"] == "ready":
return att
time.sleep(0.3)
raise Failure('attachment did not become ready within test deadline')
raise Failure("attachment did not become ready within test deadline")
def arguments(description, configure=None):
import argparse
p = argparse.ArgumentParser(description=description)
p.add_argument('--base-url', default=None)
p.add_argument('--ca-file', default=None)
p.add_argument("--base-url", default=None)
p.add_argument("--ca-file", default=None)
if configure:
configure(p)
return p.parse_args()
def run(main):
try:
main()
except (Failure,ValueError,OSError) as exc:
print('FAIL:', str(exc))
except (Failure, ValueError, OSError) as exc:
print("FAIL:", str(exc))
raise SystemExit(1)
+18 -13
View File
@@ -1,26 +1,31 @@
"""Read JPEG SOF dimensions without external libraries; not a full image decoder."""
from client import Failure, check
def jpeg_size(data):
check(data[:2]==b'\xff\xd8','thumbnail is not JPEG')
check(data[:2] == b"\xff\xd8", "thumbnail is not JPEG")
i = 2
sof = {0xc0,0xc1,0xc2,0xc3,0xc5,0xc6,0xc7,0xc9,0xca,0xcb,0xcd,0xce,0xcf}
sof = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}
while i < len(data):
check(data[i]==0xff,'invalid JPEG marker')
while i<len(data) and data[i]==0xff:
check(data[i] == 0xFF, "invalid JPEG marker")
while i < len(data) and data[i] == 0xFF:
i += 1
check(i<len(data),'truncated JPEG marker')
check(i < len(data), "truncated JPEG marker")
marker = data[i]
i += 1
if marker in {0xd9,0xda}:
if marker in {0xD9, 0xDA}:
break
if marker in {0x01,0xd8} or 0xd0<=marker<=0xd7:
if marker in {0x01, 0xD8} or 0xD0 <= marker <= 0xD7:
continue
check(i+2<=len(data),'truncated JPEG segment')
length = int.from_bytes(data[i:i+2],'big')
check(length>=2 and i+length<=len(data),'invalid JPEG segment length')
check(i + 2 <= len(data), "truncated JPEG segment")
length = int.from_bytes(data[i : i + 2], "big")
check(length >= 2 and i + length <= len(data), "invalid JPEG segment length")
if marker in sof:
check(length>=8,'invalid JPEG SOF')
return (int.from_bytes(data[i+5:i+7],'big'),int.from_bytes(data[i+3:i+5],'big'))
check(length >= 8, "invalid JPEG SOF")
return (
int.from_bytes(data[i + 5 : i + 7], "big"),
int.from_bytes(data[i + 3 : i + 5], "big"),
)
i += length
raise Failure('JPEG dimensions not found')
raise Failure("JPEG dimensions not found")
+58 -27
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Small closed-loop HTTP write benchmark; not an open-loop capacity proof."""
from concurrent.futures import ThreadPoolExecutor
import json
import math
@@ -11,48 +12,78 @@ from client import Client, Failure, LAB, arguments, check, run
def main():
def configure(p):
p.add_argument('--duration',type=float,default=30)
p.add_argument('--concurrency',type=int,default=4)
p.add_argument('--output',default=None,help='Optional JSON metrics, no session tokens')
args = arguments('Synthetic POST message load, closed loop; creates users/messages',configure)
check(1<=args.duration<=3600,'duration must be 1..3600 seconds')
check(1<=args.concurrency<=128,'concurrency must be 1..128')
c = Client(args.base_url,args.ca_file)
alice,bob,eve,chat = c.bootstrap()
p.add_argument("--duration", type=float, default=30)
p.add_argument("--concurrency", type=int, default=4)
p.add_argument(
"--output", default=None, help="Optional JSON metrics, no session tokens"
)
args = arguments(
"Synthetic POST message load, closed loop; creates users/messages", configure
)
check(1 <= args.duration <= 3600, "duration must be 1..3600 seconds")
check(1 <= args.concurrency <= 128, "concurrency must be 1..128")
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
barrier = threading.Barrier(args.concurrency)
def worker(worker_id):
local = Client(args.base_url,args.ca_file,timeout=5)
local = Client(args.base_url, args.ca_file, timeout=5)
barrier.wait()
deadline = time.monotonic()+args.duration
latencies,errors = [],0
deadline = time.monotonic() + args.duration
latencies, errors = [], 0
index = 0
while time.monotonic()<deadline:
while time.monotonic() < deadline:
start = time.monotonic()
try:
local.message(alice,chat,'load-%s-%s' % (worker_id,index))
latencies.append(time.monotonic()-start)
local.message(alice, chat, "load-%s-%s" % (worker_id, index))
latencies.append(time.monotonic() - start)
except Failure:
errors += 1
index += 1
return latencies,errors
return latencies, errors
started = time.monotonic()
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
results = list(pool.map(worker,range(args.concurrency)))
elapsed = time.monotonic()-started
latencies = sorted(x for values,_ in results for x in values)
errors = sum(n for _,n in results)
total = len(latencies)+errors
results = list(pool.map(worker, range(args.concurrency)))
elapsed = time.monotonic() - started
latencies = sorted(x for values, _ in results for x in values)
errors = sum(n for _, n in results)
total = len(latencies) + errors
def percentile(p):
return round(1000*latencies[max(0,math.ceil(p*len(latencies))-1)],3) if latencies else None
report = {'lab':LAB,'workload':'closed-loop POST messages; no automatic retries','requested_duration_seconds':args.duration,'elapsed_seconds':round(elapsed,3),'concurrency':args.concurrency,'successful_requests':len(latencies),'failed_requests':errors,'error_ratio':round(errors/total,6) if total else None,'successful_rps':round(len(latencies)/elapsed,3),'successful_latency_ms':{'p50':percentile(.50),'p95':percentile(.95),'p99':percentile(.99)},'note':'Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.'}
text = json.dumps(report,ensure_ascii=False,indent=2)
return (
round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 3)
if latencies
else None
)
report = {
"lab": LAB,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": args.duration,
"elapsed_seconds": round(elapsed, 3),
"concurrency": args.concurrency,
"successful_requests": len(latencies),
"failed_requests": errors,
"error_ratio": round(errors / total, 6) if total else None,
"successful_rps": round(len(latencies) / elapsed, 3),
"successful_latency_ms": {
"p50": percentile(0.50),
"p95": percentile(0.95),
"p99": percentile(0.99),
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.",
}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
dest = Path(args.output)
dest.parent.mkdir(parents=True,exist_ok=True)
dest.write_text(text+'\n')
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_text(text + "\n")
print(text)
check(bool(latencies),'no successful writes')
check(bool(latencies), "no successful writes")
# Errors are reported, not treated as a universal load-test threshold.
if __name__=='__main__':
if __name__ == "__main__":
run(main)
+142 -64
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Explicit operator-authored hook; tokens remain in memory, volumes are never removed by this test."""
import json
import os
from pathlib import Path
@@ -11,111 +12,187 @@ from client import Client, Failure, LAB, arguments, check, run
def history(c, actor, chat):
path = '/api/v1/chats/'+chat['id']+'/messages?limit=100'
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
result = []
seen = set()
for _ in range(1000):
page = c.call('GET',path,actor=actor,schema='MessagePage')['json']
result.extend(page['items'])
cursor = page['next_cursor']
page = c.call("GET", path, actor=actor, schema="MessagePage")["json"]
result.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return result
check(cursor not in seen,'cursor cycle')
check(cursor not in seen, "cursor cycle")
seen.add(cursor)
path = '/api/v1/chats/'+chat['id']+'/messages?limit=100&cursor='+quote(cursor,safe='')
raise Failure('history pagination exceeded safety bound')
path = (
"/api/v1/chats/"
+ chat["id"]
+ "/messages?limit=100&cursor="
+ quote(cursor, safe="")
)
raise Failure("history pagination exceeded safety bound")
def main():
def configure(p):
p.add_argument('--mode', choices=['persistence','failover'], required=True)
p.add_argument('--action-script',required=True,help='Executable script authored by student; restarts stack or stops ONE API replica')
args = arguments('Explicit restart/failover scenario; modifies only synthetic API data',configure)
check(LAB>=2,'persistence requires lab2+')
check(args.mode!='failover' or LAB>=3,'failover requires lab3+')
p.add_argument("--mode", choices=["persistence", "failover"], required=True)
p.add_argument(
"--action-script",
required=True,
help="Executable script authored by student; restarts stack or stops ONE API replica",
)
args = arguments(
"Explicit restart/failover scenario; modifies only synthetic API data",
configure,
)
check(LAB >= 2, "persistence requires lab2+")
check(args.mode != "failover" or LAB >= 3, "failover requires lab3+")
script = Path(args.action_script).expanduser().resolve()
check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)')
check('.example.' not in script.name,'copy and implement the example hook first')
c = Client(args.base_url,args.ca_file,timeout=2)
alice,bob,eve,chat = c.bootstrap()
sent = c.message(alice,chat,'Сохранить при отказе')
original = sent['json']
target = sent['headers'].get('X-Instance-Id','')
check(
script.is_file() and os.access(script, os.X_OK),
"action script must exist and be executable (chmod +x)",
)
check(".example." not in script.name, "copy and implement the example hook first")
c = Client(args.base_url, args.ca_file, timeout=2)
alice, bob, eve, chat = c.bootstrap()
sent = c.message(alice, chat, "Сохранить при отказе")
original = sent["json"]
target = sent["headers"].get("X-Instance-Id", "")
attachment = content = None
if LAB>=4:
attachment,content = c.attachment(alice,chat)
if LAB>=5:
attachment = c.wait_ready(alice,attachment)
if LAB >= 4:
attachment, content = c.attachment(alice, chat)
if LAB >= 5:
attachment = c.wait_ready(alice, attachment)
env = dict(os.environ)
env['TARGET_INSTANCE_ID'] = target
env["TARGET_INSTANCE_ID"] = target
observed = set()
if args.mode=='failover':
if args.mode == "failover":
for _ in range(40):
observed.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'])
check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity')
print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.')
observed.add(
c.call("GET", "/api/v1/users/me", actor=alice, schema="User")[
"headers"
]["X-Instance-Id"]
)
check(
len(observed) >= 2,
"less than two API instances observed before failure; verify LB/session affinity",
)
print(
"Executing explicit "
+ args.mode
+ " hook; hook output suppressed to avoid leaking secrets."
)
# No shell interpolation and no Docker commands in this test.
proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
proc = subprocess.Popen(
[str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)
started = time.monotonic()
successes,failures,after_instances = [],0,[]
successes, failures, after_instances = [], 0, []
try:
if args.mode=='persistence':
if args.mode == "persistence":
try:
code = proc.wait(timeout=90)
except subprocess.TimeoutExpired:
raise Failure('restart hook exceeded 90 seconds') from None
check(code==0,'restart hook failed (inspect your script locally)')
deadline = time.monotonic()+90
raise Failure("restart hook exceeded 90 seconds") from None
check(code == 0, "restart hook failed (inspect your script locally)")
deadline = time.monotonic() + 90
while True:
try:
c.call('GET','/health/ready',schema='Health')
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
c.call("GET", "/health/ready", schema="Health")
c.call("GET", "/api/v1/users/me", actor=alice, schema="User")
break
except Failure:
check(time.monotonic()<deadline,'stack/session did not recover within 90 seconds')
check(
time.monotonic() < deadline,
"stack/session did not recover within 90 seconds",
)
time.sleep(0.5)
else:
first_recovery = None
key = uuid.uuid4().hex
text = 'Запись во время отказа ' + key
while time.monotonic()-started < 15:
text = "Запись во время отказа " + key
while time.monotonic() - started < 15:
code = proc.poll()
check(code is None or code==0,'stop-one hook failed (inspect your script locally)')
check(code is not None or time.monotonic()-started<10,'stop-one hook must return within 10 seconds')
check(
code is None or code == 0,
"stop-one hook failed (inspect your script locally)",
)
check(
code is not None or time.monotonic() - started < 10,
"stop-one hook must return within 10 seconds",
)
try:
response = c.message(alice,chat,text,key=key)
msg = response['json']
if code==0:
response = c.message(alice, chat, text, key=key)
msg = response["json"]
if code == 0:
if first_recovery is None:
first_recovery = time.monotonic()-started
after_instances.append(response['headers']['X-Instance-Id'])
first_recovery = time.monotonic() - started
after_instances.append(response["headers"]["X-Instance-Id"])
successes.append(msg)
key = uuid.uuid4().hex
text = 'Запись во время отказа ' + key
text = "Запись во время отказа " + key
except Failure:
failures += 1
# Keep SAME key/body after an uncertain POST result.
time.sleep(0.1)
check(proc.poll()==0,'stop-one hook did not finish successfully')
check(first_recovery is not None and first_recovery<=10,'writes did not recover within 10 seconds from hook start')
check(len(after_instances)>=5,'too few successful writes after stop')
check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped')
print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False))
check(proc.poll() == 0, "stop-one hook did not finish successfully")
check(
first_recovery is not None and first_recovery <= 10,
"writes did not recover within 10 seconds from hook start",
)
check(len(after_instances) >= 5, "too few successful writes after stop")
check(
target not in after_instances[-5:],
"target replica still serves requests; hook must leave it stopped",
)
print(
json.dumps(
{
"observed_instances_before": len(observed),
"successful_writes": len(successes),
"failed_attempts": failures,
"first_recovery_seconds": round(first_recovery, 3),
},
ensure_ascii=False,
)
)
# Existing token, no re-login. Read all confirmations and reject duplicates.
messages = history(c,bob,chat)
ids = [m['id'] for m in messages]
check(len(ids)==len(set(ids)),'duplicated message IDs in history')
check(any(m==original for m in messages),'confirmed pre-failure message lost or altered')
messages = history(c, bob, chat)
ids = [m["id"] for m in messages]
check(len(ids) == len(set(ids)), "duplicated message IDs in history")
check(
any(m == original for m in messages),
"confirmed pre-failure message lost or altered",
)
for msg in successes:
check(any(m==msg for m in messages),'confirmed in-flight message lost or altered')
check(
any(m == msg for m in messages),
"confirmed in-flight message lost or altered",
)
# Bodies are unique in this scenario: an uncertain request may exist once, never twice.
texts = [m['text'] for m in messages]
check(len(texts)==len(set(texts)),'uncertain retry created duplicate message')
texts = [m["text"] for m in messages]
check(
len(texts) == len(set(texts)), "uncertain retry created duplicate message"
)
if attachment:
c.call('GET','/api/v1/attachments/'+attachment['id'],actor=bob,schema='Attachment')
downloaded = c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob)
check(downloaded['bytes']==content,'attachment lost or changed across failure')
print('PASS: '+args.mode+' preserved confirmed data and existing session. Hook scope needs operator evidence.')
c.call(
"GET",
"/api/v1/attachments/" + attachment["id"],
actor=bob,
schema="Attachment",
)
downloaded = c.call(
"GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob
)
check(
downloaded["bytes"] == content,
"attachment lost or changed across failure",
)
print(
"PASS: "
+ args.mode
+ " preserved confirmed data and existing session. Hook scope needs operator evidence."
)
finally:
if proc.poll() is None:
proc.terminate()
@@ -125,5 +202,6 @@ def main():
proc.kill()
proc.wait()
if __name__=='__main__':
if __name__ == "__main__":
run(main)
+159 -69
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""One complete public acceptance scenario, cumulative by lab number."""
import base64
import hashlib
import uuid
@@ -7,76 +8,165 @@ from client import Client, LAB, arguments, check, run, timestamp
def main():
args = arguments('Messenger public smoke; creates synthetic users and messages')
c = Client(args.base_url,args.ca_file)
alice,bob,eve,chat = c.bootstrap()
path = '/api/v1/chats/'+chat['id']
c.call('POST','/api/v1/users',409,body=alice['registration'])
c.call('GET','/api/v1/users/me',401)
c.call('GET',path,404,actor=eve)
me = c.call('GET','/api/v1/users/me',actor=alice,schema='User')['json']
check(me==alice['user'],'current user mismatch')
listed = c.call('GET','/api/v1/chats?limit=100',actor=bob,schema='ChatPage')['json']
check(any(x['id']==chat['id'] for x in listed['items']),'member cannot list chat')
hidden = c.call('GET','/api/v1/chats?limit=100',actor=eve,schema='ChatPage')['json']
check(all(x['id']!=chat['id'] for x in hidden['items']),'chat leaks to outsider')
args = arguments("Messenger public smoke; creates synthetic users and messages")
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
path = "/api/v1/chats/" + chat["id"]
c.call("POST", "/api/v1/users", 409, body=alice["registration"])
c.call("GET", "/api/v1/users/me", 401)
c.call("GET", path, 404, actor=eve)
me = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"]
check(me == alice["user"], "current user mismatch")
listed = c.call("GET", "/api/v1/chats?limit=100", actor=bob, schema="ChatPage")[
"json"
]
check(
any(x["id"] == chat["id"] for x in listed["items"]), "member cannot list chat"
)
hidden = c.call("GET", "/api/v1/chats?limit=100", actor=eve, schema="ChatPage")[
"json"
]
check(all(x["id"] != chat["id"] for x in hidden["items"]), "chat leaks to outsider")
key = uuid.uuid4().hex
first = c.message(alice,chat,'Привет, мир 👋',key=key)['json']
second = c.message(bob,chat,'Сообщение Боба')['json']
check(first['sender_id']==alice['id'] and second['sender_id']==bob['id'],'sender was not derived from actor')
check(first['chat_id']==second['chat_id']==chat['id'],'wrong chat in message')
check(first['text']=='Привет, мир 👋' and second['text']=='Сообщение Боба','message text changed')
page1 = c.call('GET',path+'/messages?limit=1',actor=bob,schema='MessagePage')['json']
check(len(page1['items'])==1 and page1['next_cursor'],'first page must have one item and cursor')
first = c.message(alice, chat, "Привет, мир 👋", key=key)["json"]
second = c.message(bob, chat, "Сообщение Боба")["json"]
check(
first["sender_id"] == alice["id"] and second["sender_id"] == bob["id"],
"sender was not derived from actor",
)
check(first["chat_id"] == second["chat_id"] == chat["id"], "wrong chat in message")
check(
first["text"] == "Привет, мир 👋" and second["text"] == "Сообщение Боба",
"message text changed",
)
page1 = c.call("GET", path + "/messages?limit=1", actor=bob, schema="MessagePage")[
"json"
]
check(
len(page1["items"]) == 1 and page1["next_cursor"],
"first page must have one item and cursor",
)
from urllib.parse import quote
page2 = c.call('GET',path+'/messages?limit=1&cursor='+quote(page1['next_cursor'],safe=''),actor=alice,schema='MessagePage')['json']
check(len(page2['items'])==1 and page2['next_cursor'] is None,'second page must finish history')
ordered = sorted([first,second],key=lambda x:(timestamp(x['created_at']),uuid.UUID(x['id']).int))
check(page1['items']+page2['items']==ordered,'pagination lost, duplicated or reordered a message')
h = {'Idempotency-Key':uuid.uuid4().hex} if LAB>=3 else {}
c.call('POST',path+'/messages',400,actor=alice,body={'text':''},headers=h)
c.call('POST',path+'/messages',404,actor=eve,body={'text':'чужое'},headers=h)
c.call('GET',path+'/messages?limit=0',400,actor=alice)
if LAB>=2:
bad = base64.b64encode((alice['registration']['username']+':wrong-password').encode()).decode()
c.call('POST','/api/v1/auth/sessions',401,headers={'Authorization':'Basic '+bad})
good = base64.b64encode((alice['registration']['username']+':'+alice['registration']['password']).encode()).decode()
c.call('GET','/api/v1/users/me',401,headers={'Authorization':'Basic '+good})
c.call('GET','/api/v1/users/me',401,headers={'X-User-Id':alice['id']})
me = c.call('GET','/api/v1/users/me',actor=alice,headers={'X-User-Id':eve['id']},schema='User')['json']
check(me['id']==alice['id'],'X-User-Id overrode bearer identity')
if LAB>=3:
replay = c.message(alice,chat,first['text'],key=key)['json']
check(replay==first,'idempotency replay differs')
c.call('POST',path+'/messages',409,actor=alice,body={'text':'другой body'},headers={'Idempotency-Key':key})
c.call('POST',path+'/messages',400,actor=alice,body={'text':'без ключа'})
if LAB>=4:
att,content = c.attachment(alice,chat)
# A queued attachment must already be linkable to its own chat.
attached = c.message(alice,chat,'',attachments=[att['id']])['json']
check(attached['attachment_ids']==[att['id']],'message lost attachment')
ready = c.wait_ready(bob,att) if LAB>=5 else att
check(ready['error_code'] is None,'ready attachment has an error')
download = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
check(download['bytes']==content,'download differs from original')
check(download['headers'].get_content_type()=='image/png','original content type mismatch')
for suffix in ('','/content'):
c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=eve)
if LAB>=5:
variants = [x for x in ready['variants'] if x['name']=='thumbnail']
check(len(variants)==1,'expected exactly one thumbnail')
v = variants[0]
check((v['width'],v['height'])==(256,160),'thumbnail metadata has wrong dimensions')
thumb = c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant=thumbnail',actor=bob)
check(thumb['headers'].get_content_type()=='image/jpeg','thumbnail content type mismatch')
check(len(thumb['bytes'])==v['size_bytes'] and hashlib.sha256(thumb['bytes']).hexdigest()==v['sha256'],'thumbnail checksum mismatch')
from image_probe import jpeg_size
check(jpeg_size(thumb['bytes'])==(256,160),'actual JPEG dimensions mismatch')
if LAB>=2:
c.call('DELETE','/api/v1/auth/sessions/current',204,actor=alice)
c.call('GET','/api/v1/users/me',401,actor=alice)
c.call('DELETE','/api/v1/auth/sessions/current',401,actor=alice)
print('PASS: lab%d public API scenario; infrastructure and grading evidence remain separate.' % LAB)
if __name__=='__main__':
page2 = c.call(
"GET",
path + "/messages?limit=1&cursor=" + quote(page1["next_cursor"], safe=""),
actor=alice,
schema="MessagePage",
)["json"]
check(
len(page2["items"]) == 1 and page2["next_cursor"] is None,
"second page must finish history",
)
ordered = sorted(
[first, second],
key=lambda x: (timestamp(x["created_at"]), uuid.UUID(x["id"]).int),
)
check(
page1["items"] + page2["items"] == ordered,
"pagination lost, duplicated or reordered a message",
)
h = {"Idempotency-Key": uuid.uuid4().hex} if LAB >= 3 else {}
c.call("POST", path + "/messages", 400, actor=alice, body={"text": ""}, headers=h)
c.call(
"POST", path + "/messages", 404, actor=eve, body={"text": "чужое"}, headers=h
)
c.call("GET", path + "/messages?limit=0", 400, actor=alice)
if LAB >= 2:
bad = base64.b64encode(
(alice["registration"]["username"] + ":wrong-password").encode()
).decode()
c.call(
"POST",
"/api/v1/auth/sessions",
401,
headers={"Authorization": "Basic " + bad},
)
good = base64.b64encode(
(
alice["registration"]["username"]
+ ":"
+ alice["registration"]["password"]
).encode()
).decode()
c.call(
"GET", "/api/v1/users/me", 401, headers={"Authorization": "Basic " + good}
)
c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]})
me = c.call(
"GET",
"/api/v1/users/me",
actor=alice,
headers={"X-User-Id": eve["id"]},
schema="User",
)["json"]
check(me["id"] == alice["id"], "X-User-Id overrode bearer identity")
if LAB >= 3:
replay = c.message(alice, chat, first["text"], key=key)["json"]
check(replay == first, "idempotency replay differs")
c.call(
"POST",
path + "/messages",
409,
actor=alice,
body={"text": "другой body"},
headers={"Idempotency-Key": key},
)
c.call("POST", path + "/messages", 400, actor=alice, body={"text": "без ключа"})
if LAB >= 4:
att, content = c.attachment(alice, chat)
# A queued attachment must already be linkable to its own chat.
attached = c.message(alice, chat, "", attachments=[att["id"]])["json"]
check(attached["attachment_ids"] == [att["id"]], "message lost attachment")
ready = c.wait_ready(bob, att) if LAB >= 5 else att
check(ready["error_code"] is None, "ready attachment has an error")
download = c.call(
"GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob
)
check(download["bytes"] == content, "download differs from original")
check(
download["headers"].get_content_type() == "image/png",
"original content type mismatch",
)
for suffix in ("", "/content"):
c.call("GET", "/api/v1/attachments/" + att["id"] + suffix, 404, actor=eve)
if LAB >= 5:
variants = [x for x in ready["variants"] if x["name"] == "thumbnail"]
check(len(variants) == 1, "expected exactly one thumbnail")
v = variants[0]
check(
(v["width"], v["height"]) == (256, 160),
"thumbnail metadata has wrong dimensions",
)
thumb = c.call(
"GET",
"/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail",
actor=bob,
)
check(
thumb["headers"].get_content_type() == "image/jpeg",
"thumbnail content type mismatch",
)
check(
len(thumb["bytes"]) == v["size_bytes"]
and hashlib.sha256(thumb["bytes"]).hexdigest() == v["sha256"],
"thumbnail checksum mismatch",
)
from image_probe import jpeg_size
check(
jpeg_size(thumb["bytes"]) == (256, 160),
"actual JPEG dimensions mismatch",
)
if LAB >= 2:
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice)
c.call("GET", "/api/v1/users/me", 401, actor=alice)
c.call("DELETE", "/api/v1/auth/sessions/current", 401, actor=alice)
print(
"PASS: lab%d public API scenario; infrastructure and grading evidence remain separate."
% LAB
)
if __name__ == "__main__":
run(main)