This commit is contained in:
2026-11-06 18:45:08 +03:00
parent 545d083049
commit 482e7ff477
17 changed files with 958 additions and 42 deletions
+3 -17
View File
@@ -1,30 +1,16 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
# Compose supplies internal PostgreSQL, Redis, broker and S3 addresses.
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
JOB_MEMORY_MB=768
STAGING_RETENTION_SECONDS=86400
# Local settings used by compose.yaml (application carried over from lab4).
# Choose credentials locally.
POSTGRES_DB=messenger
POSTGRES_USER=messenger
POSTGRES_PASSWORD=replace-with-local-password
+53 -1
View File
@@ -1,6 +1,5 @@
x-api: &api
image: mtusi2026lab5-api:local
build: .
dns_opt:
- timeout:1
- attempts:1
@@ -23,14 +22,66 @@ x-api: &api
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
BROKER_URL: redis://broker:6379/0
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_MEMORY_MB: ${JOB_MEMORY_MB:-768}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
networks: [backend]
cpus: 1.0
mem_limit: 512m
stop_grace_period: 10s
services:
broker:
image: redis:7.4.2-bookworm
command: [redis-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, 128mb, --maxmemory-policy, noeviction]
volumes:
- broker_data:/data
networks: [backend]
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 2s
retries: 30
cpus: 0.5
mem_limit: 256m
dispatcher:
<<: *api
command: [python, -m, src.jobs, dispatch]
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
restart: unless-stopped
mem_limit: 128m
worker:
<<: *api
command: [python, -m, src.worker]
init: true
healthcheck:
disable: true
depends_on:
migrate:
condition: service_completed_successfully
broker:
condition: service_healthy
s3-init:
condition: service_completed_successfully
restart: unless-stopped
cpus: 2.0
mem_limit: 2g
pids_limit: 64
api:
<<: *api
build: .
deploy:
replicas: 2
depends_on:
@@ -170,6 +221,7 @@ networks:
backend:
internal: true
volumes:
broker_data:
postgres_data:
redis_data:
s3_data:
+70
View File
@@ -0,0 +1,70 @@
# Локальный pipeline lab5
Запуск: скопировать `.env.example` в `.env`, задать локальные пароли,
выполнить `docker compose up -d --build`. API: `http://localhost:8080`.
Миграции запускаются сервисом `migrate` перед API/worker.
Upload проверяет формат, bytes и пиксели, сохраняет неизменный оригинал
в приватный S3 `staging/`, затем одной транзакцией записывает вложение и job.
Ответ 202 содержит queued. Изменение размера выполняется только worker.
Job в PostgreSQL одновременно служит outbox: dispatcher отправляет его
в отдельный Redis Streams, после ответа XADD отмечает публикацию.
Redis использует volume, AOF и `appendfsync always`, eviction отключён.
Сбой до/после XADD даёт повторную доставку, а не потерю принятого задания.
Readiness API зависит от PostgreSQL, сессионного Redis и S3;
недоступность broker/worker не мешает сохранять работу в outbox.
Worker: два потребителя, у каждого не больше одной выполняемой задачи.
Каждая попытка работает в отдельном процессе: timeout 60 секунд,
лимит адресного пространства 768 MiB, контейнер 2 GiB.
Настройки: `WORKER_CONCURRENCY` (1–4), `JOB_TIMEOUT_SECONDS`,
`JOB_MEMORY_MB`, `JOB_MAX_ATTEMPTS` (по умолчанию 5).
При повышении concurrency нужно соответственно увеличить память контейнера.
Lease равен timeout + 15 секунд. Dispatcher возвращает просроченный
processing в очередь. Токен lease и блокировка job защищают публикацию
от устаревшей попытки. ACK выполняется после устойчивого результата
либо сохранённого решения о retry/failed. Pending читаются через XAUTOCLAIM.
Оригинал публикуется в `originals/`, варианты — по стабильным ключам
`variants/<attachment_id>/v1/<variant>`. Готовность и метаданные трёх
вариантов фиксируются одной транзакцией после S3. Дубль ready — no-op.
JPEG thumbnail: вписывание в 256×256, округление вниз, без увеличения.
JPEG crop: центрированный квадрат 128×128. JPEG quality=85.
WebP optimized: вписывание в 1280×1280 без увеличения, quality=80.
Применяется EXIF orientation, прозрачность — на белом; EXIF/ICC удаляются.
Оригинал остаётся побайтно неизменным.
Временные ошибки получают backoff `min(60, 2^failures)` секунд с jitter
0.5–1.5. Повреждённое изображение/checksum — `invalid_image`, нехватка
памяти — `resource_limit`, без retry. Исчерпание попыток — `retry_exhausted`.
Последняя причина ошибки и время следующей попытки сохраняются в PostgreSQL.
Реестр failed и replay доступны только локальному оператору через Compose:
```sh
docker compose exec api python -m src.jobs failed
docker compose exec api python -m src.jobs replay JOB_UUID --reason "причина повтора"
```
Replay допускается только для failed с доступным staging, сохраняет audit,
job_id и общий номер попытки, выдаёт новый бюджет retry. Ready повторять нельзя.
Cleanup запускается раз в минуту: staging ready удаляется после публикации;
failed хранится `STAGING_RETENTION_SECONDS` (86400) после ошибки.
Частично записанные конечные объекты failed очищаются после того же срока.
Бесхозные объекты удаляются после того же retention. Queued/processing
не удаляются независимо от возраста. После удаления failed staging нужен
новый upload. Реестр ошибок остаётся в PostgreSQL.
Проверки (из корня проекта):
```sh
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
python -X utf8 student-tests/pipeline.py
python -X utf8 student-tests/edge_cases.py
docker compose run --rm --no-deps -v ./student-tests:/checks:ro worker python -m unittest discover -s /checks
```
Pipeline-тест останавливает/возвращает сервисы только этого Compose-проекта,
создаёт синтетические данные, проверяет разрыв COMMIT/publish, рестарт broker,
crash после S3 и после COMMIT до ack, lease, дубли, retry/replay и cleanup.
Точки `JOB_FAULT=after_objects|after_commit` используются только тестом,
в обычной конфигурации выключены. Контракты и публичные тесты не изменены.
+3 -1
View File
@@ -9,7 +9,9 @@
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/originals/*"]
"Resource": ["arn:aws:s3:::messenger-attachments/originals/*",
"arn:aws:s3:::messenger-attachments/staging/*",
"arn:aws:s3:::messenger-attachments/variants/*"]
}
]
}
+23
View File
@@ -0,0 +1,23 @@
ALTER TABLE attachments
ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
ADD COLUMN status text NOT NULL DEFAULT 'ready'
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
ADD COLUMN staging_key text,
ADD COLUMN variants jsonb NOT NULL DEFAULT '[]',
ADD COLUMN error_code text;
ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued';
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id),
pipeline_version bigint NOT NULL DEFAULT 1 CHECK (pipeline_version = 1),
attempt bigint NOT NULL DEFAULT 0,
failures bigint NOT NULL DEFAULT 0,
available_at timestamptz NOT NULL DEFAULT now(),
published_at timestamptz,
lease_until timestamptz,
lease_token uuid,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
replay_audit jsonb NOT NULL DEFAULT '[]'
);
CREATE INDEX attachment_jobs_due_idx ON attachment_jobs(available_at);
+1
View File
@@ -0,0 +1 @@
ALTER TABLE attachment_jobs ADD COLUMN last_error text;
+10 -7
View File
@@ -6,7 +6,7 @@ from PIL import Image, UnidentifiedImageError
from sqlalchemy import func, select
from .errors import APIError
from .models import Attachment, ChatMember
from .models import Attachment, AttachmentJob, ChatMember
from .object_storage import BUCKET, PREFIX, s3_client
from .store import ensure_uuid, serialize, store
@@ -48,25 +48,28 @@ def lock_object(session, key):
def metadata(attachment):
result = serialize(attachment, 'id', 'chat_id', 'owner_id', 'filename',
'content_type', 'size_bytes', 'sha256', 'created_at')
return {**result, 'updated_at': result['created_at'], 'status': 'ready',
'variants': [], 'error_code': None}
'content_type', 'size_bytes', 'sha256', 'created_at',
'updated_at', 'status', 'error_code')
return {**result, 'variants': [{k: v for k, v in item.items() if k != 'object_key'}
for item in attachment.variants]}
def save_attachment(source, chat_id, actor_id, filename, content_type, size, digest):
validate_image(source, content_type)
attachment_id = str(uuid4())
key = PREFIX + attachment_id
staging_key = 'staging/' + attachment_id
with store.sessions.begin() as session:
store._chat(session, chat_id, actor_id)
lock_object(session, key)
s3_client().put_object(Bucket=BUCKET, Key=key, Body=source,
lock_object(session, staging_key)
s3_client().put_object(Bucket=BUCKET, Key=staging_key, Body=source,
ContentLength=size, ContentType=content_type)
attachment = Attachment(id=attachment_id, chat_id=chat_id, owner_id=actor_id,
object_key=key, filename=filename, content_type=content_type,
size_bytes=size, sha256=digest)
size_bytes=size, sha256=digest, staging_key=staging_key)
session.add(attachment)
session.flush()
session.add(AttachmentJob(attachment_id=attachment_id))
result = metadata(attachment)
# An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later.
return result
+33 -8
View File
@@ -1,6 +1,7 @@
"""Remove old S3 objects without committed metadata. Never delete on a DB error."""
import argparse
import logging
import os
import time
from datetime import datetime, timedelta, timezone
@@ -10,23 +11,47 @@ from sqlalchemy.orm import sessionmaker
from .attachments import lock_object
from .database import create_db_engine
from .models import Attachment
from .object_storage import BUCKET, PREFIX, s3_client
from .object_storage import BUCKET, s3_client
GRACE = timedelta(hours=1)
GRACE = timedelta(seconds=max(1, int(os.getenv('STAGING_RETENTION_SECONDS', '86400'))))
def cleanup(sessions, client, now=None):
cutoff = (now or datetime.now(timezone.utc)) - GRACE
deleted = 0
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX):
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET):
for obj in page.get('Contents', []):
if obj['LastModified'] >= cutoff:
continue
key = obj['Key']
if not key.startswith(('staging/', 'originals/', 'variants/')):
continue
with sessions.begin() as session:
lock_object(session, key)
if session.scalar(select(Attachment.id).where(Attachment.object_key == key)):
continue
if key.startswith('staging/'):
attachment = session.scalar(select(Attachment).where(
Attachment.staging_key == key).with_for_update())
if attachment:
if attachment.status in ('queued', 'processing'):
continue
if attachment.status == 'failed' and attachment.updated_at >= cutoff:
continue
client.delete_object(Bucket=BUCKET, Key=key)
attachment.staging_key = None
deleted += 1
continue
else:
# All final keys belong to a server-generated attachment UUID.
from uuid import UUID
try:
attachment_id = str(UUID(key.split('/')[1]))
except (ValueError, IndexError):
continue
attachment = session.get(Attachment, attachment_id, with_for_update=True)
if attachment:
if attachment.status != 'failed' or attachment.updated_at >= cutoff:
continue
client.delete_object(Bucket=BUCKET, Key=key)
deleted += 1
continue
# Recheck age after taking the same lock as the uploader.
if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff:
continue
@@ -52,7 +77,7 @@ def main():
raise SystemExit(1) from None
if args.once:
break
time.sleep(3600)
time.sleep(60)
finally:
engine.dispose()
+116
View File
@@ -0,0 +1,116 @@
"""One isolated process per attempt; quality: JPEG 85, WebP 80; no EXIF/ICC."""
import hashlib
import io
import os
import sys
from PIL import Image, ImageOps, UnidentifiedImageError
from sqlalchemy.orm import sessionmaker
from .attachments import MAX_BYTES, MAX_PIXELS
from .database import create_db_engine
from .jobs import now
from .models import Attachment, AttachmentJob
from .object_storage import BUCKET, s3_client
def fit_size(size, bound):
width, height = size
if max(size) <= bound:
return size
return max(1, width * bound // max(size)), max(1, height * bound // max(size))
def variants(data):
with Image.open(io.BytesIO(data), formats=['PNG', 'JPEG']) as original:
if original.width * original.height > MAX_PIXELS:
raise ValueError('pixel limit')
original.load()
oriented = ImageOps.exif_transpose(original)
rgba = oriented.convert('RGBA')
rgb = Image.new('RGB', rgba.size, 'white')
rgb.paste(rgba, mask=rgba.getchannel('A'))
rgba.close()
oriented.close()
rgb.info.clear()
try:
for name, image, fmt, mime, quality in (
('thumbnail', rgb.resize(fit_size(rgb.size, 256), Image.Resampling.LANCZOS), 'JPEG', 'image/jpeg', 85),
('crop', ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS), 'JPEG', 'image/jpeg', 85),
('optimized', rgb.resize(fit_size(rgb.size, 1280), Image.Resampling.LANCZOS), 'WEBP', 'image/webp', 80),
):
with image, io.BytesIO() as output:
image.save(output, format=fmt, quality=quality)
blob = output.getvalue()
yield {'name': name, 'content_type': mime, 'width': image.width,
'height': image.height, 'size_bytes': len(blob),
'sha256': hashlib.sha256(blob).hexdigest()}, blob
finally:
rgb.close()
def process(sessions, job_id, token):
with sessions() as session:
job = session.get(AttachmentJob, job_id)
if job.lease_token != token:
return
attachment = session.get(Attachment, job.attachment_id)
staging_key = attachment.staging_key
client = s3_client()
response = client.get_object(Bucket=BUCKET, Key=staging_key)
with response['Body'] as body:
data = body.read(MAX_BYTES + 1)
if len(data) != attachment.size_bytes or hashlib.sha256(data).hexdigest() != attachment.sha256:
raise ValueError('staging checksum mismatch')
try:
rendered = list(variants(data))
except OSError as exc:
raise ValueError('invalid image') from exc
# Job lock fences the final writes, cleanup and publication against lease recovery.
# Work before this transaction is CPU-only. Keys are stable across retries.
with sessions.begin() as session:
job = session.get(AttachmentJob, job_id, with_for_update=True)
if job.lease_token != token or job.lease_until <= now():
return
current = session.get(Attachment, job.attachment_id, with_for_update=True)
if current.status != 'processing':
return
client.put_object(Bucket=BUCKET, Key=current.object_key, Body=data,
ContentType=current.content_type)
saved = []
for meta, blob in rendered:
key = f'variants/{current.id}/v{job.pipeline_version}/{meta["name"]}'
client.put_object(Bucket=BUCKET, Key=key, Body=blob, ContentType=meta['content_type'])
saved.append({**meta, 'object_key': key})
# Test-only crash injection; never enabled by compose.yaml.
if os.getenv('JOB_FAULT') == 'after_objects':
os._exit(91)
current.variants = saved
current.status, current.error_code = 'ready', None
current.updated_at = now()
job.lease_token = job.lease_until = None
if os.getenv('JOB_FAULT') == 'after_commit':
os._exit(92)
def main():
# Linux worker: bounded address space as well as the container memory limit.
import resource
memory = int(os.getenv('JOB_MEMORY_MB', '768')) * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
engine = create_db_engine()
try:
process(sessionmaker(engine, expire_on_commit=False), sys.argv[1], sys.argv[2])
except (UnidentifiedImageError, Image.DecompressionBombError, ValueError, SyntaxError):
return 20
except MemoryError:
return 21
except Exception:
return 22 # No credentials or decoder/database internals in logs.
finally:
engine.dispose()
return 0
if __name__ == '__main__':
sys.exit(main())
+176
View File
@@ -0,0 +1,176 @@
"""Small PostgreSQL outbox + Redis Streams queue; PostgreSQL owns job state."""
import argparse
import json
import logging
import os
import random
import time
from datetime import datetime, timedelta, timezone
from uuid import UUID, uuid4
from redis import Redis
from redis.exceptions import ResponseError
from sqlalchemy import and_, or_, select
from sqlalchemy.orm import sessionmaker
from .database import create_db_engine
from .models import Attachment, AttachmentJob
from .object_storage import BUCKET, s3_client
STREAM = 'attachment-jobs'
GROUP = 'image-workers'
MAX_ATTEMPTS = max(1, int(os.getenv('JOB_MAX_ATTEMPTS', '5')))
TIMEOUT = max(1, int(os.getenv('JOB_TIMEOUT_SECONDS', '60')))
LEASE_SECONDS = TIMEOUT + 15
def now():
return datetime.now(timezone.utc)
def broker():
return Redis.from_url(os.getenv('BROKER_URL', 'redis://broker:6379/0'),
decode_responses=True, socket_connect_timeout=2, socket_timeout=5)
def ensure_group(client):
try:
client.xgroup_create(STREAM, GROUP, id='0', mkstream=True)
except ResponseError as exc:
if not str(exc).startswith('BUSYGROUP'):
raise
def envelope(job):
return {'schema_version': 1, 'job_id': job.id, 'type': 'attachment.process.v1',
'attachment_id': job.attachment_id, 'pipeline_version': job.pipeline_version,
'attempt': job.attempt + 1,
'created_at': job.created_at.isoformat().replace('+00:00', 'Z')}
def fail(session, job, attachment, code, permanent=False):
job.failures += 1
job.last_error = code
job.lease_token = job.lease_until = job.published_at = None
attachment.updated_at = now()
if permanent or job.failures >= MAX_ATTEMPTS:
attachment.status = 'failed'
attachment.error_code = code if permanent else 'retry_exhausted'
else:
attachment.status = 'queued'
attachment.error_code = None
delay = min(60, 2 ** min(job.failures, 6)) * random.uniform(0.5, 1.5)
job.available_at = now() + timedelta(seconds=delay)
logging.info('job=%s attempt=%s result=%s code=%s', job.id, job.attempt, attachment.status, code)
def dispatch_once(sessions, client):
# A crash after XADD but before COMMIT deliberately results in redelivery.
with sessions.begin() as session:
jobs = list(session.scalars(select(AttachmentJob).join(Attachment).where(
or_(and_(Attachment.status == 'queued', AttachmentJob.published_at.is_(None)),
and_(Attachment.status == 'processing', AttachmentJob.lease_until <= now())),
AttachmentJob.available_at <= now()).order_by(AttachmentJob.available_at)
.with_for_update(skip_locked=True, of=AttachmentJob).limit(50)))
for job in jobs:
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
if attachment.status == 'processing':
if job.lease_until <= now():
fail(session, job, attachment, 'lease_expired')
continue
if job.published_at is None:
client.xadd(STREAM, {'job': json.dumps(envelope(job))})
job.published_at = now()
def claim(sessions, message):
if (message.get('schema_version') != 1 or message.get('type') != 'attachment.process.v1'
or message.get('pipeline_version') != 1):
raise ValueError('unsupported job envelope')
job_id = str(UUID(message['job_id']))
with sessions.begin() as session:
job = session.get(AttachmentJob, job_id, with_for_update=True)
if job is None or job.attachment_id != message.get('attachment_id'):
raise ValueError('unknown job')
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
if attachment.status != 'queued' or job.available_at > now():
return None
job.attempt += 1
job.lease_token = str(uuid4())
job.lease_until = now() + timedelta(seconds=LEASE_SECONDS)
attachment.status = 'processing'
attachment.updated_at = now()
return job.id, job.lease_token
def record_failure(sessions, job_id, token, code, permanent=False):
with sessions.begin() as session:
job = session.get(AttachmentJob, job_id, with_for_update=True)
if job.lease_token == token:
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
fail(session, job, attachment, code, permanent)
def replay(sessions, job_id, reason):
if not reason.strip() or len(reason) > 200:
raise ValueError('Give a nonempty reason of at most 200 characters')
with sessions.begin() as session:
job = session.get(AttachmentJob, str(UUID(job_id)), with_for_update=True)
if job is None:
raise ValueError('Job not found')
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
if attachment.status != 'failed':
raise ValueError('Replay is allowed only for failed jobs')
if not attachment.staging_key:
raise ValueError('Staging retention expired; upload the file again')
s3_client().head_object(Bucket=BUCKET, Key=attachment.staging_key)
job.replay_audit = job.replay_audit + [{'at': now().isoformat(), 'reason': reason,
'after_attempt': job.attempt,
'last_error': job.last_error}]
job.failures = 0
job.available_at = now()
job.published_at = job.lease_until = job.lease_token = None
attachment.status, attachment.error_code = 'queued', None
attachment.updated_at = now()
def main():
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest='command', required=True)
sub.add_parser('dispatch')
sub.add_parser('failed')
retry = sub.add_parser('replay')
retry.add_argument('job_id')
retry.add_argument('--reason', required=True)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
engine = create_db_engine()
sessions = sessionmaker(engine, expire_on_commit=False)
try:
if args.command == 'failed':
with sessions() as session:
for job, att in session.execute(select(AttachmentJob, Attachment).join(
Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(Attachment.status == 'failed')):
print(json.dumps({'job_id': job.id, 'attachment_id': att.id,
'attempt': job.attempt, 'error_code': att.error_code,
'last_error': job.last_error,
'replay_available': bool(att.staging_key)}))
elif args.command == 'replay':
replay(sessions, args.job_id, args.reason)
print('Job queued for replay')
else:
client = broker()
while True:
try:
ensure_group(client)
dispatch_once(sessions, client)
except Exception as exc:
logging.warning('Dispatch deferred: %s', type(exc).__name__)
time.sleep(1)
finally:
engine.dispose()
if __name__ == '__main__':
main()
+23 -1
View File
@@ -3,7 +3,7 @@ from uuid import uuid4
from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
from sqlalchemy.dialects.postgresql import ARRAY
from sqlalchemy.dialects.postgresql import ARRAY, JSONB
class Base(DeclarativeBase):
@@ -74,6 +74,28 @@ class Attachment(Base):
size_bytes: Mapped[int] = mapped_column(BigInteger)
sha256: Mapped[str] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
status: Mapped[str] = mapped_column(Text, server_default='queued')
staging_key: Mapped[str | None] = mapped_column(Text)
variants: Mapped[list] = mapped_column(JSONB, server_default='[]')
error_code: Mapped[str | None] = mapped_column(Text)
class AttachmentJob(Base):
"""The job row is also the transactional outbox and failed-job registry."""
__tablename__ = 'attachment_jobs'
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
attachment_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('attachments.id'), unique=True)
pipeline_version: Mapped[int] = mapped_column(BigInteger, server_default='1')
attempt: Mapped[int] = mapped_column(BigInteger, server_default='0')
failures: Mapped[int] = mapped_column(BigInteger, server_default='0')
last_error: Mapped[str | None] = mapped_column(Text)
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
lease_token: Mapped[str | None] = mapped_column(Uuid(as_uuid=False))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
replay_audit: Mapped[list] = mapped_column(JSONB, server_default='[]')
class IdempotencyRecord(Base):
+11 -5
View File
@@ -20,7 +20,7 @@ router = APIRouter(prefix='/api/v1', tags=['attachments'])
decode_slot = asyncio.Semaphore(1)
@router.post('/chats/{chat_id}/attachments', status_code=201)
@router.post('/chats/{chat_id}/attachments', status_code=202)
async def upload(chat_id: str, request: Request, filename: str = Query(...),
user: dict = Depends(current_user)):
chat_id = ensure_uuid(chat_id)
@@ -67,9 +67,15 @@ def attachment_info(attachment_id: str, user: dict = Depends(current_user)):
@router.get('/attachments/{attachment_id}/content')
def download(attachment_id: str, variant: str = 'original', user: dict = Depends(current_user)):
attachment = get_attachment(attachment_id, user['id'])
if attachment.status != 'ready':
raise APIError(409, 'conflict', 'Вложение ещё не готово')
key, content_type, size = attachment.object_key, attachment.content_type, attachment.size_bytes
if variant != 'original':
raise APIError(404, 'not_found', 'Вариант не найден')
body = s3_client().get_object(Bucket=BUCKET, Key=attachment.object_key)['Body']
item = next((v for v in attachment.variants if v['name'] == variant), None)
if item is None:
raise APIError(404, 'not_found', 'Вариант не найден')
key, content_type, size = item['object_key'], item['content_type'], item['size_bytes']
body = s3_client().get_object(Bucket=BUCKET, Key=key)['Body']
def chunks():
try:
@@ -77,8 +83,8 @@ def download(attachment_id: str, variant: str = 'original', user: dict = Depends
finally:
body.close()
return StreamingResponse(chunks(), media_type=attachment.content_type,
headers={'Content-Length': str(attachment.size_bytes),
return StreamingResponse(chunks(), media_type=content_type,
headers={'Content-Length': str(size),
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(attachment.filename, safe=''),
'X-Content-Type-Options': 'nosniff'},
background=BackgroundTask(body.close))
+5 -2
View File
@@ -136,8 +136,9 @@ class PostgresStore:
with self.sessions.begin() as session:
self._chat(session, chat_id, actor_id)
if attachment_ids:
found = list(session.scalars(select(Attachment.id).where(
Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id)))
found = list(session.scalars(select(Attachment).where(
Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id)
.order_by(Attachment.id).with_for_update()))
if len(found) != len(attachment_ids):
raise APIError(404, 'not_found', 'Объект не найден')
if idempotency_key is not None:
@@ -150,6 +151,8 @@ class PostgresStore:
if previous.text != text or previous.attachment_ids != attachment_ids:
raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
return serialize(previous, *MESSAGE_FIELDS)
if attachment_ids and any(item.status == 'failed' for item in found):
raise APIError(409, 'conflict', 'Обработка вложения завершилась ошибкой')
session.execute(update(Chat).where(Chat.id == chat_id).values(
history_version=Chat.history_version + 1))
message = Message(chat_id=chat_id, sender_id=actor_id, text=text, attachment_ids=attachment_ids)
+76
View File
@@ -0,0 +1,76 @@
"""Bounded consumers, one unacknowledged message per consumer, hard process timeout."""
import json
import logging
import os
import subprocess
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
from sqlalchemy.orm import sessionmaker
from .database import create_db_engine
from .jobs import GROUP, STREAM, LEASE_SECONDS, TIMEOUT, broker, claim, ensure_group, record_failure
def handle(sessions, client, message_id, fields):
try:
claimed = claim(sessions, json.loads(fields['job']))
except (ValueError, KeyError, TypeError):
logging.error('Invalid broker envelope: id=%s', message_id)
claimed = None
if claimed:
job_id, token = claimed
try:
result = subprocess.run([sys.executable, '-m', 'src.image_processor', job_id, token],
timeout=TIMEOUT, check=False)
code = result.returncode
except subprocess.TimeoutExpired:
code = 23
if code in (91, 92):
# Simulate a lost worker before ack: leave the Redis pending entry and lease.
return
if code:
record_failure(sessions, job_id, token,
{20: 'invalid_image', 21: 'resource_limit', 23: 'processing_timeout'}
.get(code, 'processing_failed'), permanent=code in (20, 21))
# Only after a durable ready/failed/retry (or an existing durable lease).
with client.pipeline(transaction=True) as pipe:
pipe.xack(STREAM, GROUP, message_id)
pipe.xdel(STREAM, message_id)
pipe.execute()
def consume(sessions):
client = broker()
consumer = str(uuid4())
cursor = '0-0'
while True:
try:
ensure_group(client)
pending = client.xautoclaim(STREAM, GROUP, consumer, LEASE_SECONDS * 1000,
start_id=cursor, count=1)
cursor, messages = pending[0], pending[1]
if not messages:
batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=1000)
messages = batches[0][1] if batches else []
for message_id, fields in messages:
handle(sessions, client, message_id, fields)
except Exception as exc:
logging.warning('Worker deferred: %s', type(exc).__name__)
time.sleep(1)
def main():
logging.basicConfig(level=logging.INFO)
engine = create_db_engine()
sessions = sessionmaker(engine, expire_on_commit=False)
concurrency = min(4, max(1, int(os.getenv('WORKER_CONCURRENCY', '2'))))
with ThreadPoolExecutor(max_workers=concurrency) as pool:
for task in [pool.submit(consume, sessions) for _ in range(concurrency)]:
task.result()
if __name__ == '__main__':
main()
+97
View File
@@ -0,0 +1,97 @@
"""Additional real-container checks: validation, permanent failure, timeout, retention."""
import os
from pipeline import Client, check, compose, remote, state, until
def main():
c = Client(os.getenv('BASE_URL', 'http://127.0.0.1:8080'))
alice, bob, eve, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/attachments'
c.call('POST', path + '?filename=bad.png', 415, actor=alice, raw=b'not a PNG',
headers={'Content-Type': 'image/png'})
c.call('POST', path + '?filename=../bad.png', 400, actor=alice, raw=b'bad',
headers={'Content-Type': 'image/png'})
try:
compose('stop', 'worker', 'cleanup')
att, data = c.attachment(alice, chat)
aid = att['id']
before = state(aid)
# A queued record cannot be collected even when its staging is old.
remote(f'''
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from src.cleanup import cleanup
client = s3_client()
class SingleObject:
def __getattr__(self, name):
return getattr(client, name)
def get_paginator(self, name):
obj = dict(client.head_object(Bucket=BUCKET, Key={before['staging']!r}), Key={before['staging']!r})
return SimpleNamespace(paginate=lambda **kw: [{{'Contents': [obj]}}])
cleanup(sessions, SingleObject(), now=datetime.now(timezone.utc) + timedelta(days=2))
s3_client().head_object(Bucket=BUCKET, Key={before['staging']!r})
''')
# Force a real checksum failure after upload, without changing the public contract.
remote(f"s3_client().put_object(Bucket=BUCKET, Key={before['staging']!r}, Body=b'corrupted')\n")
key = 'queued-before-failure-' + aid
message = c.message(alice, chat, '', key=key, attachments=[aid])['json']
compose('start', 'worker')
until(lambda: state(aid)['status'] == 'failed')
check(state(aid)['attempt'] == 1 and state(aid)['error'] == 'invalid_image', 'permanent error retried')
c.call('GET', '/api/v1/attachments/' + aid + '/content', 409, actor=bob)
check(c.message(alice, chat, '', key=key, attachments=[aid])['json'] == message,
'failed processing broke message idempotency')
# Expire this test job's failed retention; cleanup must remove its staging.
remote(f'''
from datetime import timedelta
from src.jobs import now, replay
from src.cleanup import cleanup
with sessions.begin() as session:
a = session.get(Attachment, {aid!r})
a.updated_at = now() - timedelta(days=2)
cleanup(sessions, s3_client())
try:
replay(sessions, {before['job_id']!r}, 'expired staging must reject')
except ValueError:
pass
else:
raise AssertionError('Replay accepted missing staging')
''')
check(state(aid)['staging'] is None, 'failed staging retention did not expire')
print('PASS: upload validation, active staging retention, permanent failure, message replay, expired staging', flush=True)
compose('stop', 'worker')
att, _ = c.attachment(alice, chat)
aid = att['id']
until(lambda: state(aid)['published'])
# Substitute a real sleeping child for the decoder, keeping the real timeout/kill path.
remote(f'''
import subprocess, sys
import src.worker as worker
from unittest.mock import patch
real_run = subprocess.run
def stalled(*args, **kwargs):
return real_run([sys.executable, '-c', 'import time; time.sleep(30)'], timeout=1)
with sessions() as session:
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {aid!r}))
fields = {{'job': json.dumps(envelope(j))}}
with patch.object(worker.subprocess, 'run', stalled):
worker.handle(sessions, broker(), '0-0', fields)
''')
check(state(aid)['status'] == 'queued' and state(aid)['failures'] == 1, 'timeout did not schedule retry')
remote(f'''
with sessions() as session:
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {aid!r}))
assert j.last_error == 'processing_timeout'
''')
compose('start', 'worker')
c.wait_ready(bob, att)
check(state(aid)['attempt'] == 2, 'timeout recovery did not complete')
print('PASS: real subprocess timeout, termination and successful retry', flush=True)
finally:
compose('start', 'worker', 'cleanup')
if __name__ == '__main__':
main()
+197
View File
@@ -0,0 +1,197 @@
"""Integration/fault checks for the local lab5 Compose project; restores stopped services."""
import base64
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def compose(*args, input=None):
result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, input=input,
text=True, encoding='utf-8', capture_output=True)
if result.returncode:
raise AssertionError('Compose failed: ' + ' '.join(args[:3]) + '\n' + result.stderr)
return result.stdout
PRELUDE = '''
import json
from sqlalchemy import select
from sqlalchemy.orm import sessionmaker
from src.database import create_db_engine
from src.models import Attachment, AttachmentJob
from src.jobs import broker, envelope, GROUP, STREAM
from src.object_storage import BUCKET, s3_client
engine = create_db_engine()
sessions = sessionmaker(engine, expire_on_commit=False)
'''
def remote(code, fault=None):
options = ['exec', '-T']
if fault:
options += ['-e', 'JOB_FAULT=' + fault]
return compose(*options, 'api', 'python', '-', input=PRELUDE + code).strip()
def state(attachment_id):
return json.loads(remote(f'''
with sessions() as session:
a = session.get(Attachment, {attachment_id!r})
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == a.id))
print(json.dumps(dict(status=a.status, error=a.error_code, job_id=j.id,
attempt=j.attempt, failures=j.failures, published=bool(j.published_at),
staging=a.staging_key, variants=a.variants, audit=j.replay_audit)))
'''))
def until(fn, timeout=130):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
result = fn()
if result:
return result
time.sleep(1)
raise AssertionError('Timed out waiting for durable state')
def ready(client):
def probe():
try:
return client.call('GET', '/health/ready')['status'] == 200
except Exception:
return False
until(probe, 40)
def inject(attachment_id, fault):
# Use a real consumer-group delivery, leaving the message pending at the crash point.
return remote(f'''
from src.worker import handle
client = broker()
with sessions() as session:
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {attachment_id!r}))
target = j.id
for _ in range(100):
batches = client.xreadgroup(GROUP, 'fault-test', {{STREAM: '>'}}, count=1, block=1000)
if batches:
mid, fields = batches[0][1][0]
handle(sessions, client, mid, fields)
if json.loads(fields['job'])['job_id'] == target:
print(mid)
break
else:
raise AssertionError('No delivery for target job')
''', fault=fault)
def main():
client = Client(os.getenv('BASE_URL', 'http://127.0.0.1:8080'))
alice, bob, eve, chat = client.bootstrap()
try:
compose('stop', 'worker', 'dispatcher', 'cleanup')
attachment, content = client.attachment(alice, chat)
aid = attachment['id']
check(attachment['status'] == 'queued', 'upload must return queued')
check(not state(aid)['published'], 'outbox was already published')
client.message(alice, chat, '', attachments=[aid])
client.call('GET', f'/api/v1/attachments/{aid}/content', 409, actor=bob)
compose('restart', 'api')
ready(client)
check(state(aid)['status'] == 'queued', 'API restart lost upload')
compose('start', 'dispatcher')
until(lambda: state(aid)['published'])
compose('restart', 'broker')
compose('start', 'worker')
result = client.wait_ready(bob, attachment)
check({v['name'] for v in result['variants']} == {'thumbnail', 'crop', 'optimized'}, 'missing variants')
for variant in result['variants']:
download = client.call('GET', f'/api/v1/attachments/{aid}/content?variant=' + variant['name'], actor=bob)
check(hashlib.sha256(download['bytes']).hexdigest() == variant['sha256'], 'variant checksum mismatch')
check(len(download['bytes']) == variant['size_bytes'], 'variant size mismatch')
check(download['headers'].get_content_type() == variant['content_type'], 'variant MIME mismatch')
check(client.call('GET', f'/api/v1/attachments/{aid}/content', actor=bob)['bytes'] == content, 'original changed')
print('PASS: outbox COMMIT/publish gap, stopped worker, API and broker restart', flush=True)
for fault in ('after_objects', 'after_commit'):
compose('stop', 'worker')
att, _ = client.attachment(alice, chat)
aid = att['id']
until(lambda: state(aid)['published'])
pending_id = inject(aid, fault)
snapshot = state(aid)
check(snapshot['status'] == ('processing' if fault == 'after_objects' else 'ready'), 'wrong crash state')
compose('start', 'worker')
until(lambda: state(aid)['status'] == 'ready')
until(lambda: remote(f"print(int(not broker().xpending_range(STREAM, GROUP, {pending_id!r}, {pending_id!r}, 1)))\n") == '1')
if fault == 'after_objects':
check(state(aid)['attempt'] == 2, 'expired lease was not retried')
else:
time.sleep(2)
check(state(aid)['attempt'] == 1, 'ready job was reprocessed')
# Force duplicate delivery; stable object keys and metadata must remain identical.
before = state(aid)
remote(f'''
with sessions() as session:
j = session.get(AttachmentJob, {before['job_id']!r})
broker().xadd(STREAM, {{'job': json.dumps(envelope(j))}})
''')
time.sleep(2)
after = state(aid)
check(before['variants'] == after['variants'] and before['attempt'] == after['attempt'], 'duplicate changed result')
count = remote(f"print(s3_client().list_objects_v2(Bucket=BUCKET, Prefix='variants/{aid}/')['KeyCount'])\n")
check(count == '3', 'redelivery produced extra S3 objects')
print('PASS: crash ' + fault + ', redelivery and stable variants', flush=True)
compose('stop', 'worker')
att, content = client.attachment(alice, chat)
aid = att['id']
until(lambda: state(aid)['published'])
# Remove only this test's staging object to cause a real temporary S3 error.
staging = state(aid)['staging']
remote(f"s3_client().delete_object(Bucket=BUCKET, Key={staging!r})\n")
compose('start', 'worker')
until(lambda: state(aid)['status'] == 'failed', 150)
failed = state(aid)
check(failed['error'] == 'retry_exhausted' and failed['attempt'] == 5, 'retry budget incorrect')
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
body={'text': '', 'attachment_ids': [aid]}, headers={'Idempotency-Key': aid})
registry = compose('exec', '-T', 'api', 'python', '-m', 'src.jobs', 'failed')
check(failed['job_id'] in registry, 'failed job not visible')
remote(f'''
import base64
s3_client().put_object(Bucket=BUCKET, Key={staging!r},
Body=base64.b64decode({base64.b64encode(content).decode()!r}), ContentType='image/png')
''')
compose('exec', '-T', 'api', 'python', '-m', 'src.jobs', 'replay', failed['job_id'],
'--reason', 'synthetic test: restored staging')
client.wait_ready(bob, att)
check(len(state(aid)['audit']) == 1 and state(aid)['attempt'] == 6, 'replay lost audit or attempt')
remote(f'''
from src.jobs import replay
try:
replay(sessions, {failed['job_id']!r}, 'must reject ready')
except ValueError:
pass
else:
raise AssertionError('Replayed a ready job')
''')
print('PASS: bounded retries, failed registry, explicit audited replay, ready replay rejected', flush=True)
compose('exec', '-T', 'api', 'python', '-m', 'src.cleanup', '--once')
check(state(aid)['staging'] is None, 'completed staging was not cleaned')
check(client.call('GET', f'/api/v1/attachments/{aid}/content', actor=bob)['bytes'] == content, 'cleanup deleted original')
print('PASS: staging cleanup preserves ready objects', flush=True)
finally:
compose('start', 'broker', 'dispatcher', 'worker', 'cleanup')
if __name__ == '__main__':
main()
+61
View File
@@ -0,0 +1,61 @@
"""Run inside the application image: python -m unittest discover -s /checks."""
import hashlib
import io
import unittest
from unittest.mock import patch
from PIL import Image
from src.image_processor import variants
class ImageTests(unittest.TestCase):
def render(self, image, fmt='PNG', **kwargs):
source = io.BytesIO()
image.save(source, format=fmt, **kwargs)
results = list(variants(source.getvalue()))
self.assertEqual([meta['name'] for meta, _ in results], ['thumbnail', 'crop', 'optimized'])
decoded = {}
for meta, data in results:
self.assertEqual(meta['sha256'], hashlib.sha256(data).hexdigest())
self.assertEqual(meta['size_bytes'], len(data))
with Image.open(io.BytesIO(data)) as item:
item.load()
self.assertEqual(item.size, (meta['width'], meta['height']))
self.assertFalse(item.getexif())
self.assertNotIn('icc_profile', item.info)
decoded[meta['name']] = item.copy()
return decoded
def test_dimensions_and_floor_rounding(self):
images = self.render(Image.new('RGB', (321, 200), 'red'))
self.assertEqual(images['thumbnail'].size, (256, 159))
self.assertEqual(images['crop'].size, (128, 128))
self.assertEqual(images['optimized'].size, (321, 200))
def test_no_upscale_and_white_alpha(self):
images = self.render(Image.new('RGBA', (20, 10), (255, 0, 0, 0)))
self.assertEqual(images['thumbnail'].size, (20, 10))
self.assertEqual(images['optimized'].size, (20, 10))
self.assertEqual(images['thumbnail'].getpixel((5, 5)), (255, 255, 255))
def test_exif_orientation_and_metadata_removal(self):
exif = Image.Exif()
exif[274] = 6
exif[270] = 'private description'
images = self.render(Image.new('RGB', (320, 200), 'blue'), 'JPEG', exif=exif)
self.assertEqual(images['thumbnail'].size, (160, 256))
self.assertEqual(images['optimized'].size, (200, 320))
def test_optimized_bound(self):
images = self.render(Image.new('RGB', (2000, 1000), 'green'))
self.assertEqual(images['optimized'].size, (1280, 640))
def test_worker_pixel_limit(self):
with patch('src.image_processor.MAX_PIXELS', 100):
with self.assertRaises(ValueError):
self.render(Image.new('RGB', (11, 10)))
if __name__ == '__main__':
unittest.main()