forked from mtusi2026/mtusi2026lab5
Grade 4
This commit is contained in:
+3
-17
@@ -1,30 +1,16 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
# Compose supplies internal PostgreSQL, Redis, broker and S3 addresses.
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_MEMORY_MB=768
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
|
||||
# Local settings used by compose.yaml (application carried over from lab4).
|
||||
# Choose credentials locally.
|
||||
POSTGRES_DB=messenger
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=replace-with-local-password
|
||||
|
||||
+53
-1
@@ -1,6 +1,5 @@
|
||||
x-api: &api
|
||||
image: mtusi2026lab5-api:local
|
||||
build: .
|
||||
dns_opt:
|
||||
- timeout:1
|
||||
- attempts:1
|
||||
@@ -23,14 +22,66 @@ x-api: &api
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
BROKER_URL: redis://broker:6379/0
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_MEMORY_MB: ${JOB_MEMORY_MB:-768}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
networks: [backend]
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
stop_grace_period: 10s
|
||||
|
||||
services:
|
||||
broker:
|
||||
image: redis:7.4.2-bookworm
|
||||
command: [redis-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, 128mb, --maxmemory-policy, noeviction]
|
||||
volumes:
|
||||
- broker_data:/data
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
|
||||
dispatcher:
|
||||
<<: *api
|
||||
command: [python, -m, src.jobs, dispatch]
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
mem_limit: 128m
|
||||
|
||||
worker:
|
||||
<<: *api
|
||||
command: [python, -m, src.worker]
|
||||
init: true
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
cpus: 2.0
|
||||
mem_limit: 2g
|
||||
pids_limit: 64
|
||||
|
||||
api:
|
||||
<<: *api
|
||||
build: .
|
||||
deploy:
|
||||
replicas: 2
|
||||
depends_on:
|
||||
@@ -170,6 +221,7 @@ networks:
|
||||
backend:
|
||||
internal: true
|
||||
volumes:
|
||||
broker_data:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
# Локальный pipeline lab5
|
||||
|
||||
Запуск: скопировать `.env.example` в `.env`, задать локальные пароли,
|
||||
выполнить `docker compose up -d --build`. API: `http://localhost:8080`.
|
||||
Миграции запускаются сервисом `migrate` перед API/worker.
|
||||
|
||||
Upload проверяет формат, bytes и пиксели, сохраняет неизменный оригинал
|
||||
в приватный S3 `staging/`, затем одной транзакцией записывает вложение и job.
|
||||
Ответ 202 содержит queued. Изменение размера выполняется только worker.
|
||||
Job в PostgreSQL одновременно служит outbox: dispatcher отправляет его
|
||||
в отдельный Redis Streams, после ответа XADD отмечает публикацию.
|
||||
Redis использует volume, AOF и `appendfsync always`, eviction отключён.
|
||||
Сбой до/после XADD даёт повторную доставку, а не потерю принятого задания.
|
||||
Readiness API зависит от PostgreSQL, сессионного Redis и S3;
|
||||
недоступность broker/worker не мешает сохранять работу в outbox.
|
||||
|
||||
Worker: два потребителя, у каждого не больше одной выполняемой задачи.
|
||||
Каждая попытка работает в отдельном процессе: timeout 60 секунд,
|
||||
лимит адресного пространства 768 MiB, контейнер 2 GiB.
|
||||
Настройки: `WORKER_CONCURRENCY` (1–4), `JOB_TIMEOUT_SECONDS`,
|
||||
`JOB_MEMORY_MB`, `JOB_MAX_ATTEMPTS` (по умолчанию 5).
|
||||
При повышении concurrency нужно соответственно увеличить память контейнера.
|
||||
Lease равен timeout + 15 секунд. Dispatcher возвращает просроченный
|
||||
processing в очередь. Токен lease и блокировка job защищают публикацию
|
||||
от устаревшей попытки. ACK выполняется после устойчивого результата
|
||||
либо сохранённого решения о retry/failed. Pending читаются через XAUTOCLAIM.
|
||||
|
||||
Оригинал публикуется в `originals/`, варианты — по стабильным ключам
|
||||
`variants/<attachment_id>/v1/<variant>`. Готовность и метаданные трёх
|
||||
вариантов фиксируются одной транзакцией после S3. Дубль ready — no-op.
|
||||
JPEG thumbnail: вписывание в 256×256, округление вниз, без увеличения.
|
||||
JPEG crop: центрированный квадрат 128×128. JPEG quality=85.
|
||||
WebP optimized: вписывание в 1280×1280 без увеличения, quality=80.
|
||||
Применяется EXIF orientation, прозрачность — на белом; EXIF/ICC удаляются.
|
||||
Оригинал остаётся побайтно неизменным.
|
||||
|
||||
Временные ошибки получают backoff `min(60, 2^failures)` секунд с jitter
|
||||
0.5–1.5. Повреждённое изображение/checksum — `invalid_image`, нехватка
|
||||
памяти — `resource_limit`, без retry. Исчерпание попыток — `retry_exhausted`.
|
||||
Последняя причина ошибки и время следующей попытки сохраняются в PostgreSQL.
|
||||
Реестр failed и replay доступны только локальному оператору через Compose:
|
||||
|
||||
```sh
|
||||
docker compose exec api python -m src.jobs failed
|
||||
docker compose exec api python -m src.jobs replay JOB_UUID --reason "причина повтора"
|
||||
```
|
||||
|
||||
Replay допускается только для failed с доступным staging, сохраняет audit,
|
||||
job_id и общий номер попытки, выдаёт новый бюджет retry. Ready повторять нельзя.
|
||||
Cleanup запускается раз в минуту: staging ready удаляется после публикации;
|
||||
failed хранится `STAGING_RETENTION_SECONDS` (86400) после ошибки.
|
||||
Частично записанные конечные объекты failed очищаются после того же срока.
|
||||
Бесхозные объекты удаляются после того же retention. Queued/processing
|
||||
не удаляются независимо от возраста. После удаления failed staging нужен
|
||||
новый upload. Реестр ошибок остаётся в PostgreSQL.
|
||||
|
||||
Проверки (из корня проекта):
|
||||
|
||||
```sh
|
||||
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
python -X utf8 student-tests/pipeline.py
|
||||
python -X utf8 student-tests/edge_cases.py
|
||||
docker compose run --rm --no-deps -v ./student-tests:/checks:ro worker python -m unittest discover -s /checks
|
||||
```
|
||||
|
||||
Pipeline-тест останавливает/возвращает сервисы только этого Compose-проекта,
|
||||
создаёт синтетические данные, проверяет разрыв COMMIT/publish, рестарт broker,
|
||||
crash после S3 и после COMMIT до ack, lease, дубли, retry/replay и cleanup.
|
||||
Точки `JOB_FAULT=after_objects|after_commit` используются только тестом,
|
||||
в обычной конфигурации выключены. Контракты и публичные тесты не изменены.
|
||||
@@ -9,7 +9,9 @@
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/originals/*"]
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/originals/*",
|
||||
"arn:aws:s3:::messenger-attachments/staging/*",
|
||||
"arn:aws:s3:::messenger-attachments/variants/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
ADD COLUMN status text NOT NULL DEFAULT 'ready'
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
ADD COLUMN staging_key text,
|
||||
ADD COLUMN variants jsonb NOT NULL DEFAULT '[]',
|
||||
ADD COLUMN error_code text;
|
||||
ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued';
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id),
|
||||
pipeline_version bigint NOT NULL DEFAULT 1 CHECK (pipeline_version = 1),
|
||||
attempt bigint NOT NULL DEFAULT 0,
|
||||
failures bigint NOT NULL DEFAULT 0,
|
||||
available_at timestamptz NOT NULL DEFAULT now(),
|
||||
published_at timestamptz,
|
||||
lease_until timestamptz,
|
||||
lease_token uuid,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
replay_audit jsonb NOT NULL DEFAULT '[]'
|
||||
);
|
||||
CREATE INDEX attachment_jobs_due_idx ON attachment_jobs(available_at);
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE attachment_jobs ADD COLUMN last_error text;
|
||||
+10
-7
@@ -6,7 +6,7 @@ from PIL import Image, UnidentifiedImageError
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from .errors import APIError
|
||||
from .models import Attachment, ChatMember
|
||||
from .models import Attachment, AttachmentJob, ChatMember
|
||||
from .object_storage import BUCKET, PREFIX, s3_client
|
||||
from .store import ensure_uuid, serialize, store
|
||||
|
||||
@@ -48,25 +48,28 @@ def lock_object(session, key):
|
||||
|
||||
def metadata(attachment):
|
||||
result = serialize(attachment, 'id', 'chat_id', 'owner_id', 'filename',
|
||||
'content_type', 'size_bytes', 'sha256', 'created_at')
|
||||
return {**result, 'updated_at': result['created_at'], 'status': 'ready',
|
||||
'variants': [], 'error_code': None}
|
||||
'content_type', 'size_bytes', 'sha256', 'created_at',
|
||||
'updated_at', 'status', 'error_code')
|
||||
return {**result, 'variants': [{k: v for k, v in item.items() if k != 'object_key'}
|
||||
for item in attachment.variants]}
|
||||
|
||||
|
||||
def save_attachment(source, chat_id, actor_id, filename, content_type, size, digest):
|
||||
validate_image(source, content_type)
|
||||
attachment_id = str(uuid4())
|
||||
key = PREFIX + attachment_id
|
||||
staging_key = 'staging/' + attachment_id
|
||||
with store.sessions.begin() as session:
|
||||
store._chat(session, chat_id, actor_id)
|
||||
lock_object(session, key)
|
||||
s3_client().put_object(Bucket=BUCKET, Key=key, Body=source,
|
||||
lock_object(session, staging_key)
|
||||
s3_client().put_object(Bucket=BUCKET, Key=staging_key, Body=source,
|
||||
ContentLength=size, ContentType=content_type)
|
||||
attachment = Attachment(id=attachment_id, chat_id=chat_id, owner_id=actor_id,
|
||||
object_key=key, filename=filename, content_type=content_type,
|
||||
size_bytes=size, sha256=digest)
|
||||
size_bytes=size, sha256=digest, staging_key=staging_key)
|
||||
session.add(attachment)
|
||||
session.flush()
|
||||
session.add(AttachmentJob(attachment_id=attachment_id))
|
||||
result = metadata(attachment)
|
||||
# An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later.
|
||||
return result
|
||||
|
||||
+33
-8
@@ -1,6 +1,7 @@
|
||||
"""Remove old S3 objects without committed metadata. Never delete on a DB error."""
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
@@ -10,23 +11,47 @@ from sqlalchemy.orm import sessionmaker
|
||||
from .attachments import lock_object
|
||||
from .database import create_db_engine
|
||||
from .models import Attachment
|
||||
from .object_storage import BUCKET, PREFIX, s3_client
|
||||
from .object_storage import BUCKET, s3_client
|
||||
|
||||
GRACE = timedelta(hours=1)
|
||||
GRACE = timedelta(seconds=max(1, int(os.getenv('STAGING_RETENTION_SECONDS', '86400'))))
|
||||
|
||||
|
||||
def cleanup(sessions, client, now=None):
|
||||
cutoff = (now or datetime.now(timezone.utc)) - GRACE
|
||||
deleted = 0
|
||||
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX):
|
||||
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET):
|
||||
for obj in page.get('Contents', []):
|
||||
if obj['LastModified'] >= cutoff:
|
||||
continue
|
||||
key = obj['Key']
|
||||
if not key.startswith(('staging/', 'originals/', 'variants/')):
|
||||
continue
|
||||
with sessions.begin() as session:
|
||||
lock_object(session, key)
|
||||
if session.scalar(select(Attachment.id).where(Attachment.object_key == key)):
|
||||
continue
|
||||
if key.startswith('staging/'):
|
||||
attachment = session.scalar(select(Attachment).where(
|
||||
Attachment.staging_key == key).with_for_update())
|
||||
if attachment:
|
||||
if attachment.status in ('queued', 'processing'):
|
||||
continue
|
||||
if attachment.status == 'failed' and attachment.updated_at >= cutoff:
|
||||
continue
|
||||
client.delete_object(Bucket=BUCKET, Key=key)
|
||||
attachment.staging_key = None
|
||||
deleted += 1
|
||||
continue
|
||||
else:
|
||||
# All final keys belong to a server-generated attachment UUID.
|
||||
from uuid import UUID
|
||||
try:
|
||||
attachment_id = str(UUID(key.split('/')[1]))
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
attachment = session.get(Attachment, attachment_id, with_for_update=True)
|
||||
if attachment:
|
||||
if attachment.status != 'failed' or attachment.updated_at >= cutoff:
|
||||
continue
|
||||
client.delete_object(Bucket=BUCKET, Key=key)
|
||||
deleted += 1
|
||||
continue
|
||||
# Recheck age after taking the same lock as the uploader.
|
||||
if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff:
|
||||
continue
|
||||
@@ -52,7 +77,7 @@ def main():
|
||||
raise SystemExit(1) from None
|
||||
if args.once:
|
||||
break
|
||||
time.sleep(3600)
|
||||
time.sleep(60)
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
"""One isolated process per attempt; quality: JPEG 85, WebP 80; no EXIF/ICC."""
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from .attachments import MAX_BYTES, MAX_PIXELS
|
||||
from .database import create_db_engine
|
||||
from .jobs import now
|
||||
from .models import Attachment, AttachmentJob
|
||||
from .object_storage import BUCKET, s3_client
|
||||
|
||||
|
||||
def fit_size(size, bound):
|
||||
width, height = size
|
||||
if max(size) <= bound:
|
||||
return size
|
||||
return max(1, width * bound // max(size)), max(1, height * bound // max(size))
|
||||
|
||||
|
||||
def variants(data):
|
||||
with Image.open(io.BytesIO(data), formats=['PNG', 'JPEG']) as original:
|
||||
if original.width * original.height > MAX_PIXELS:
|
||||
raise ValueError('pixel limit')
|
||||
original.load()
|
||||
oriented = ImageOps.exif_transpose(original)
|
||||
rgba = oriented.convert('RGBA')
|
||||
rgb = Image.new('RGB', rgba.size, 'white')
|
||||
rgb.paste(rgba, mask=rgba.getchannel('A'))
|
||||
rgba.close()
|
||||
oriented.close()
|
||||
rgb.info.clear()
|
||||
try:
|
||||
for name, image, fmt, mime, quality in (
|
||||
('thumbnail', rgb.resize(fit_size(rgb.size, 256), Image.Resampling.LANCZOS), 'JPEG', 'image/jpeg', 85),
|
||||
('crop', ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS), 'JPEG', 'image/jpeg', 85),
|
||||
('optimized', rgb.resize(fit_size(rgb.size, 1280), Image.Resampling.LANCZOS), 'WEBP', 'image/webp', 80),
|
||||
):
|
||||
with image, io.BytesIO() as output:
|
||||
image.save(output, format=fmt, quality=quality)
|
||||
blob = output.getvalue()
|
||||
yield {'name': name, 'content_type': mime, 'width': image.width,
|
||||
'height': image.height, 'size_bytes': len(blob),
|
||||
'sha256': hashlib.sha256(blob).hexdigest()}, blob
|
||||
finally:
|
||||
rgb.close()
|
||||
|
||||
|
||||
def process(sessions, job_id, token):
|
||||
with sessions() as session:
|
||||
job = session.get(AttachmentJob, job_id)
|
||||
if job.lease_token != token:
|
||||
return
|
||||
attachment = session.get(Attachment, job.attachment_id)
|
||||
staging_key = attachment.staging_key
|
||||
client = s3_client()
|
||||
response = client.get_object(Bucket=BUCKET, Key=staging_key)
|
||||
with response['Body'] as body:
|
||||
data = body.read(MAX_BYTES + 1)
|
||||
if len(data) != attachment.size_bytes or hashlib.sha256(data).hexdigest() != attachment.sha256:
|
||||
raise ValueError('staging checksum mismatch')
|
||||
try:
|
||||
rendered = list(variants(data))
|
||||
except OSError as exc:
|
||||
raise ValueError('invalid image') from exc
|
||||
# Job lock fences the final writes, cleanup and publication against lease recovery.
|
||||
# Work before this transaction is CPU-only. Keys are stable across retries.
|
||||
with sessions.begin() as session:
|
||||
job = session.get(AttachmentJob, job_id, with_for_update=True)
|
||||
if job.lease_token != token or job.lease_until <= now():
|
||||
return
|
||||
current = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||
if current.status != 'processing':
|
||||
return
|
||||
client.put_object(Bucket=BUCKET, Key=current.object_key, Body=data,
|
||||
ContentType=current.content_type)
|
||||
saved = []
|
||||
for meta, blob in rendered:
|
||||
key = f'variants/{current.id}/v{job.pipeline_version}/{meta["name"]}'
|
||||
client.put_object(Bucket=BUCKET, Key=key, Body=blob, ContentType=meta['content_type'])
|
||||
saved.append({**meta, 'object_key': key})
|
||||
# Test-only crash injection; never enabled by compose.yaml.
|
||||
if os.getenv('JOB_FAULT') == 'after_objects':
|
||||
os._exit(91)
|
||||
current.variants = saved
|
||||
current.status, current.error_code = 'ready', None
|
||||
current.updated_at = now()
|
||||
job.lease_token = job.lease_until = None
|
||||
if os.getenv('JOB_FAULT') == 'after_commit':
|
||||
os._exit(92)
|
||||
|
||||
|
||||
def main():
|
||||
# Linux worker: bounded address space as well as the container memory limit.
|
||||
import resource
|
||||
memory = int(os.getenv('JOB_MEMORY_MB', '768')) * 1024 * 1024
|
||||
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
|
||||
engine = create_db_engine()
|
||||
try:
|
||||
process(sessionmaker(engine, expire_on_commit=False), sys.argv[1], sys.argv[2])
|
||||
except (UnidentifiedImageError, Image.DecompressionBombError, ValueError, SyntaxError):
|
||||
return 20
|
||||
except MemoryError:
|
||||
return 21
|
||||
except Exception:
|
||||
return 22 # No credentials or decoder/database internals in logs.
|
||||
finally:
|
||||
engine.dispose()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
"""Small PostgreSQL outbox + Redis Streams queue; PostgreSQL owns job state."""
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from redis import Redis
|
||||
from redis.exceptions import ResponseError
|
||||
from sqlalchemy import and_, or_, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from .database import create_db_engine
|
||||
from .models import Attachment, AttachmentJob
|
||||
from .object_storage import BUCKET, s3_client
|
||||
|
||||
STREAM = 'attachment-jobs'
|
||||
GROUP = 'image-workers'
|
||||
MAX_ATTEMPTS = max(1, int(os.getenv('JOB_MAX_ATTEMPTS', '5')))
|
||||
TIMEOUT = max(1, int(os.getenv('JOB_TIMEOUT_SECONDS', '60')))
|
||||
LEASE_SECONDS = TIMEOUT + 15
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def broker():
|
||||
return Redis.from_url(os.getenv('BROKER_URL', 'redis://broker:6379/0'),
|
||||
decode_responses=True, socket_connect_timeout=2, socket_timeout=5)
|
||||
|
||||
|
||||
def ensure_group(client):
|
||||
try:
|
||||
client.xgroup_create(STREAM, GROUP, id='0', mkstream=True)
|
||||
except ResponseError as exc:
|
||||
if not str(exc).startswith('BUSYGROUP'):
|
||||
raise
|
||||
|
||||
|
||||
def envelope(job):
|
||||
return {'schema_version': 1, 'job_id': job.id, 'type': 'attachment.process.v1',
|
||||
'attachment_id': job.attachment_id, 'pipeline_version': job.pipeline_version,
|
||||
'attempt': job.attempt + 1,
|
||||
'created_at': job.created_at.isoformat().replace('+00:00', 'Z')}
|
||||
|
||||
|
||||
def fail(session, job, attachment, code, permanent=False):
|
||||
job.failures += 1
|
||||
job.last_error = code
|
||||
job.lease_token = job.lease_until = job.published_at = None
|
||||
attachment.updated_at = now()
|
||||
if permanent or job.failures >= MAX_ATTEMPTS:
|
||||
attachment.status = 'failed'
|
||||
attachment.error_code = code if permanent else 'retry_exhausted'
|
||||
else:
|
||||
attachment.status = 'queued'
|
||||
attachment.error_code = None
|
||||
delay = min(60, 2 ** min(job.failures, 6)) * random.uniform(0.5, 1.5)
|
||||
job.available_at = now() + timedelta(seconds=delay)
|
||||
logging.info('job=%s attempt=%s result=%s code=%s', job.id, job.attempt, attachment.status, code)
|
||||
|
||||
|
||||
def dispatch_once(sessions, client):
|
||||
# A crash after XADD but before COMMIT deliberately results in redelivery.
|
||||
with sessions.begin() as session:
|
||||
jobs = list(session.scalars(select(AttachmentJob).join(Attachment).where(
|
||||
or_(and_(Attachment.status == 'queued', AttachmentJob.published_at.is_(None)),
|
||||
and_(Attachment.status == 'processing', AttachmentJob.lease_until <= now())),
|
||||
AttachmentJob.available_at <= now()).order_by(AttachmentJob.available_at)
|
||||
.with_for_update(skip_locked=True, of=AttachmentJob).limit(50)))
|
||||
for job in jobs:
|
||||
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||
if attachment.status == 'processing':
|
||||
if job.lease_until <= now():
|
||||
fail(session, job, attachment, 'lease_expired')
|
||||
continue
|
||||
if job.published_at is None:
|
||||
client.xadd(STREAM, {'job': json.dumps(envelope(job))})
|
||||
job.published_at = now()
|
||||
|
||||
|
||||
def claim(sessions, message):
|
||||
if (message.get('schema_version') != 1 or message.get('type') != 'attachment.process.v1'
|
||||
or message.get('pipeline_version') != 1):
|
||||
raise ValueError('unsupported job envelope')
|
||||
job_id = str(UUID(message['job_id']))
|
||||
with sessions.begin() as session:
|
||||
job = session.get(AttachmentJob, job_id, with_for_update=True)
|
||||
if job is None or job.attachment_id != message.get('attachment_id'):
|
||||
raise ValueError('unknown job')
|
||||
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||
if attachment.status != 'queued' or job.available_at > now():
|
||||
return None
|
||||
job.attempt += 1
|
||||
job.lease_token = str(uuid4())
|
||||
job.lease_until = now() + timedelta(seconds=LEASE_SECONDS)
|
||||
attachment.status = 'processing'
|
||||
attachment.updated_at = now()
|
||||
return job.id, job.lease_token
|
||||
|
||||
|
||||
def record_failure(sessions, job_id, token, code, permanent=False):
|
||||
with sessions.begin() as session:
|
||||
job = session.get(AttachmentJob, job_id, with_for_update=True)
|
||||
if job.lease_token == token:
|
||||
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||
fail(session, job, attachment, code, permanent)
|
||||
|
||||
|
||||
def replay(sessions, job_id, reason):
|
||||
if not reason.strip() or len(reason) > 200:
|
||||
raise ValueError('Give a nonempty reason of at most 200 characters')
|
||||
with sessions.begin() as session:
|
||||
job = session.get(AttachmentJob, str(UUID(job_id)), with_for_update=True)
|
||||
if job is None:
|
||||
raise ValueError('Job not found')
|
||||
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||
if attachment.status != 'failed':
|
||||
raise ValueError('Replay is allowed only for failed jobs')
|
||||
if not attachment.staging_key:
|
||||
raise ValueError('Staging retention expired; upload the file again')
|
||||
s3_client().head_object(Bucket=BUCKET, Key=attachment.staging_key)
|
||||
job.replay_audit = job.replay_audit + [{'at': now().isoformat(), 'reason': reason,
|
||||
'after_attempt': job.attempt,
|
||||
'last_error': job.last_error}]
|
||||
job.failures = 0
|
||||
job.available_at = now()
|
||||
job.published_at = job.lease_until = job.lease_token = None
|
||||
attachment.status, attachment.error_code = 'queued', None
|
||||
attachment.updated_at = now()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest='command', required=True)
|
||||
sub.add_parser('dispatch')
|
||||
sub.add_parser('failed')
|
||||
retry = sub.add_parser('replay')
|
||||
retry.add_argument('job_id')
|
||||
retry.add_argument('--reason', required=True)
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
engine = create_db_engine()
|
||||
sessions = sessionmaker(engine, expire_on_commit=False)
|
||||
try:
|
||||
if args.command == 'failed':
|
||||
with sessions() as session:
|
||||
for job, att in session.execute(select(AttachmentJob, Attachment).join(
|
||||
Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(Attachment.status == 'failed')):
|
||||
print(json.dumps({'job_id': job.id, 'attachment_id': att.id,
|
||||
'attempt': job.attempt, 'error_code': att.error_code,
|
||||
'last_error': job.last_error,
|
||||
'replay_available': bool(att.staging_key)}))
|
||||
elif args.command == 'replay':
|
||||
replay(sessions, args.job_id, args.reason)
|
||||
print('Job queued for replay')
|
||||
else:
|
||||
client = broker()
|
||||
while True:
|
||||
try:
|
||||
ensure_group(client)
|
||||
dispatch_once(sessions, client)
|
||||
except Exception as exc:
|
||||
logging.warning('Dispatch deferred: %s', type(exc).__name__)
|
||||
time.sleep(1)
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+23
-1
@@ -3,7 +3,7 @@ from uuid import uuid4
|
||||
|
||||
from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
from sqlalchemy.dialects.postgresql import ARRAY
|
||||
from sqlalchemy.dialects.postgresql import ARRAY, JSONB
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
@@ -74,6 +74,28 @@ class Attachment(Base):
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
sha256: Mapped[str] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
status: Mapped[str] = mapped_column(Text, server_default='queued')
|
||||
staging_key: Mapped[str | None] = mapped_column(Text)
|
||||
variants: Mapped[list] = mapped_column(JSONB, server_default='[]')
|
||||
error_code: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
class AttachmentJob(Base):
|
||||
"""The job row is also the transactional outbox and failed-job registry."""
|
||||
__tablename__ = 'attachment_jobs'
|
||||
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||
attachment_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('attachments.id'), unique=True)
|
||||
pipeline_version: Mapped[int] = mapped_column(BigInteger, server_default='1')
|
||||
attempt: Mapped[int] = mapped_column(BigInteger, server_default='0')
|
||||
failures: Mapped[int] = mapped_column(BigInteger, server_default='0')
|
||||
last_error: Mapped[str | None] = mapped_column(Text)
|
||||
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
lease_token: Mapped[str | None] = mapped_column(Uuid(as_uuid=False))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
replay_audit: Mapped[list] = mapped_column(JSONB, server_default='[]')
|
||||
|
||||
|
||||
class IdempotencyRecord(Base):
|
||||
|
||||
@@ -20,7 +20,7 @@ router = APIRouter(prefix='/api/v1', tags=['attachments'])
|
||||
decode_slot = asyncio.Semaphore(1)
|
||||
|
||||
|
||||
@router.post('/chats/{chat_id}/attachments', status_code=201)
|
||||
@router.post('/chats/{chat_id}/attachments', status_code=202)
|
||||
async def upload(chat_id: str, request: Request, filename: str = Query(...),
|
||||
user: dict = Depends(current_user)):
|
||||
chat_id = ensure_uuid(chat_id)
|
||||
@@ -67,9 +67,15 @@ def attachment_info(attachment_id: str, user: dict = Depends(current_user)):
|
||||
@router.get('/attachments/{attachment_id}/content')
|
||||
def download(attachment_id: str, variant: str = 'original', user: dict = Depends(current_user)):
|
||||
attachment = get_attachment(attachment_id, user['id'])
|
||||
if attachment.status != 'ready':
|
||||
raise APIError(409, 'conflict', 'Вложение ещё не готово')
|
||||
key, content_type, size = attachment.object_key, attachment.content_type, attachment.size_bytes
|
||||
if variant != 'original':
|
||||
raise APIError(404, 'not_found', 'Вариант не найден')
|
||||
body = s3_client().get_object(Bucket=BUCKET, Key=attachment.object_key)['Body']
|
||||
item = next((v for v in attachment.variants if v['name'] == variant), None)
|
||||
if item is None:
|
||||
raise APIError(404, 'not_found', 'Вариант не найден')
|
||||
key, content_type, size = item['object_key'], item['content_type'], item['size_bytes']
|
||||
body = s3_client().get_object(Bucket=BUCKET, Key=key)['Body']
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
@@ -77,8 +83,8 @@ def download(attachment_id: str, variant: str = 'original', user: dict = Depends
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
return StreamingResponse(chunks(), media_type=attachment.content_type,
|
||||
headers={'Content-Length': str(attachment.size_bytes),
|
||||
return StreamingResponse(chunks(), media_type=content_type,
|
||||
headers={'Content-Length': str(size),
|
||||
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(attachment.filename, safe=''),
|
||||
'X-Content-Type-Options': 'nosniff'},
|
||||
background=BackgroundTask(body.close))
|
||||
|
||||
+5
-2
@@ -136,8 +136,9 @@ class PostgresStore:
|
||||
with self.sessions.begin() as session:
|
||||
self._chat(session, chat_id, actor_id)
|
||||
if attachment_ids:
|
||||
found = list(session.scalars(select(Attachment.id).where(
|
||||
Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id)))
|
||||
found = list(session.scalars(select(Attachment).where(
|
||||
Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id)
|
||||
.order_by(Attachment.id).with_for_update()))
|
||||
if len(found) != len(attachment_ids):
|
||||
raise APIError(404, 'not_found', 'Объект не найден')
|
||||
if idempotency_key is not None:
|
||||
@@ -150,6 +151,8 @@ class PostgresStore:
|
||||
if previous.text != text or previous.attachment_ids != attachment_ids:
|
||||
raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
|
||||
return serialize(previous, *MESSAGE_FIELDS)
|
||||
if attachment_ids and any(item.status == 'failed' for item in found):
|
||||
raise APIError(409, 'conflict', 'Обработка вложения завершилась ошибкой')
|
||||
session.execute(update(Chat).where(Chat.id == chat_id).values(
|
||||
history_version=Chat.history_version + 1))
|
||||
message = Message(chat_id=chat_id, sender_id=actor_id, text=text, attachment_ids=attachment_ids)
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Bounded consumers, one unacknowledged message per consumer, hard process timeout."""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from .database import create_db_engine
|
||||
from .jobs import GROUP, STREAM, LEASE_SECONDS, TIMEOUT, broker, claim, ensure_group, record_failure
|
||||
|
||||
|
||||
def handle(sessions, client, message_id, fields):
|
||||
try:
|
||||
claimed = claim(sessions, json.loads(fields['job']))
|
||||
except (ValueError, KeyError, TypeError):
|
||||
logging.error('Invalid broker envelope: id=%s', message_id)
|
||||
claimed = None
|
||||
if claimed:
|
||||
job_id, token = claimed
|
||||
try:
|
||||
result = subprocess.run([sys.executable, '-m', 'src.image_processor', job_id, token],
|
||||
timeout=TIMEOUT, check=False)
|
||||
code = result.returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
code = 23
|
||||
if code in (91, 92):
|
||||
# Simulate a lost worker before ack: leave the Redis pending entry and lease.
|
||||
return
|
||||
if code:
|
||||
record_failure(sessions, job_id, token,
|
||||
{20: 'invalid_image', 21: 'resource_limit', 23: 'processing_timeout'}
|
||||
.get(code, 'processing_failed'), permanent=code in (20, 21))
|
||||
# Only after a durable ready/failed/retry (or an existing durable lease).
|
||||
with client.pipeline(transaction=True) as pipe:
|
||||
pipe.xack(STREAM, GROUP, message_id)
|
||||
pipe.xdel(STREAM, message_id)
|
||||
pipe.execute()
|
||||
|
||||
|
||||
def consume(sessions):
|
||||
client = broker()
|
||||
consumer = str(uuid4())
|
||||
cursor = '0-0'
|
||||
while True:
|
||||
try:
|
||||
ensure_group(client)
|
||||
pending = client.xautoclaim(STREAM, GROUP, consumer, LEASE_SECONDS * 1000,
|
||||
start_id=cursor, count=1)
|
||||
cursor, messages = pending[0], pending[1]
|
||||
if not messages:
|
||||
batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=1000)
|
||||
messages = batches[0][1] if batches else []
|
||||
for message_id, fields in messages:
|
||||
handle(sessions, client, message_id, fields)
|
||||
except Exception as exc:
|
||||
logging.warning('Worker deferred: %s', type(exc).__name__)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
engine = create_db_engine()
|
||||
sessions = sessionmaker(engine, expire_on_commit=False)
|
||||
concurrency = min(4, max(1, int(os.getenv('WORKER_CONCURRENCY', '2'))))
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
for task in [pool.submit(consume, sessions) for _ in range(concurrency)]:
|
||||
task.result()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Additional real-container checks: validation, permanent failure, timeout, retention."""
|
||||
import os
|
||||
|
||||
from pipeline import Client, check, compose, remote, state, until
|
||||
|
||||
|
||||
def main():
|
||||
c = Client(os.getenv('BASE_URL', 'http://127.0.0.1:8080'))
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/attachments'
|
||||
c.call('POST', path + '?filename=bad.png', 415, actor=alice, raw=b'not a PNG',
|
||||
headers={'Content-Type': 'image/png'})
|
||||
c.call('POST', path + '?filename=../bad.png', 400, actor=alice, raw=b'bad',
|
||||
headers={'Content-Type': 'image/png'})
|
||||
try:
|
||||
compose('stop', 'worker', 'cleanup')
|
||||
att, data = c.attachment(alice, chat)
|
||||
aid = att['id']
|
||||
before = state(aid)
|
||||
# A queued record cannot be collected even when its staging is old.
|
||||
remote(f'''
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from src.cleanup import cleanup
|
||||
client = s3_client()
|
||||
class SingleObject:
|
||||
def __getattr__(self, name):
|
||||
return getattr(client, name)
|
||||
def get_paginator(self, name):
|
||||
obj = dict(client.head_object(Bucket=BUCKET, Key={before['staging']!r}), Key={before['staging']!r})
|
||||
return SimpleNamespace(paginate=lambda **kw: [{{'Contents': [obj]}}])
|
||||
cleanup(sessions, SingleObject(), now=datetime.now(timezone.utc) + timedelta(days=2))
|
||||
s3_client().head_object(Bucket=BUCKET, Key={before['staging']!r})
|
||||
''')
|
||||
# Force a real checksum failure after upload, without changing the public contract.
|
||||
remote(f"s3_client().put_object(Bucket=BUCKET, Key={before['staging']!r}, Body=b'corrupted')\n")
|
||||
key = 'queued-before-failure-' + aid
|
||||
message = c.message(alice, chat, '', key=key, attachments=[aid])['json']
|
||||
compose('start', 'worker')
|
||||
until(lambda: state(aid)['status'] == 'failed')
|
||||
check(state(aid)['attempt'] == 1 and state(aid)['error'] == 'invalid_image', 'permanent error retried')
|
||||
c.call('GET', '/api/v1/attachments/' + aid + '/content', 409, actor=bob)
|
||||
check(c.message(alice, chat, '', key=key, attachments=[aid])['json'] == message,
|
||||
'failed processing broke message idempotency')
|
||||
# Expire this test job's failed retention; cleanup must remove its staging.
|
||||
remote(f'''
|
||||
from datetime import timedelta
|
||||
from src.jobs import now, replay
|
||||
from src.cleanup import cleanup
|
||||
with sessions.begin() as session:
|
||||
a = session.get(Attachment, {aid!r})
|
||||
a.updated_at = now() - timedelta(days=2)
|
||||
cleanup(sessions, s3_client())
|
||||
try:
|
||||
replay(sessions, {before['job_id']!r}, 'expired staging must reject')
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError('Replay accepted missing staging')
|
||||
''')
|
||||
check(state(aid)['staging'] is None, 'failed staging retention did not expire')
|
||||
print('PASS: upload validation, active staging retention, permanent failure, message replay, expired staging', flush=True)
|
||||
|
||||
compose('stop', 'worker')
|
||||
att, _ = c.attachment(alice, chat)
|
||||
aid = att['id']
|
||||
until(lambda: state(aid)['published'])
|
||||
# Substitute a real sleeping child for the decoder, keeping the real timeout/kill path.
|
||||
remote(f'''
|
||||
import subprocess, sys
|
||||
import src.worker as worker
|
||||
from unittest.mock import patch
|
||||
real_run = subprocess.run
|
||||
def stalled(*args, **kwargs):
|
||||
return real_run([sys.executable, '-c', 'import time; time.sleep(30)'], timeout=1)
|
||||
with sessions() as session:
|
||||
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {aid!r}))
|
||||
fields = {{'job': json.dumps(envelope(j))}}
|
||||
with patch.object(worker.subprocess, 'run', stalled):
|
||||
worker.handle(sessions, broker(), '0-0', fields)
|
||||
''')
|
||||
check(state(aid)['status'] == 'queued' and state(aid)['failures'] == 1, 'timeout did not schedule retry')
|
||||
remote(f'''
|
||||
with sessions() as session:
|
||||
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {aid!r}))
|
||||
assert j.last_error == 'processing_timeout'
|
||||
''')
|
||||
compose('start', 'worker')
|
||||
c.wait_ready(bob, att)
|
||||
check(state(aid)['attempt'] == 2, 'timeout recovery did not complete')
|
||||
print('PASS: real subprocess timeout, termination and successful retry', flush=True)
|
||||
finally:
|
||||
compose('start', 'worker', 'cleanup')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,197 @@
|
||||
"""Integration/fault checks for the local lab5 Compose project; restores stopped services."""
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def compose(*args, input=None):
|
||||
result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, input=input,
|
||||
text=True, encoding='utf-8', capture_output=True)
|
||||
if result.returncode:
|
||||
raise AssertionError('Compose failed: ' + ' '.join(args[:3]) + '\n' + result.stderr)
|
||||
return result.stdout
|
||||
|
||||
|
||||
PRELUDE = '''
|
||||
import json
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from src.database import create_db_engine
|
||||
from src.models import Attachment, AttachmentJob
|
||||
from src.jobs import broker, envelope, GROUP, STREAM
|
||||
from src.object_storage import BUCKET, s3_client
|
||||
engine = create_db_engine()
|
||||
sessions = sessionmaker(engine, expire_on_commit=False)
|
||||
'''
|
||||
|
||||
|
||||
def remote(code, fault=None):
|
||||
options = ['exec', '-T']
|
||||
if fault:
|
||||
options += ['-e', 'JOB_FAULT=' + fault]
|
||||
return compose(*options, 'api', 'python', '-', input=PRELUDE + code).strip()
|
||||
|
||||
|
||||
def state(attachment_id):
|
||||
return json.loads(remote(f'''
|
||||
with sessions() as session:
|
||||
a = session.get(Attachment, {attachment_id!r})
|
||||
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == a.id))
|
||||
print(json.dumps(dict(status=a.status, error=a.error_code, job_id=j.id,
|
||||
attempt=j.attempt, failures=j.failures, published=bool(j.published_at),
|
||||
staging=a.staging_key, variants=a.variants, audit=j.replay_audit)))
|
||||
'''))
|
||||
|
||||
|
||||
def until(fn, timeout=130):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
result = fn()
|
||||
if result:
|
||||
return result
|
||||
time.sleep(1)
|
||||
raise AssertionError('Timed out waiting for durable state')
|
||||
|
||||
|
||||
def ready(client):
|
||||
def probe():
|
||||
try:
|
||||
return client.call('GET', '/health/ready')['status'] == 200
|
||||
except Exception:
|
||||
return False
|
||||
until(probe, 40)
|
||||
|
||||
|
||||
def inject(attachment_id, fault):
|
||||
# Use a real consumer-group delivery, leaving the message pending at the crash point.
|
||||
return remote(f'''
|
||||
from src.worker import handle
|
||||
client = broker()
|
||||
with sessions() as session:
|
||||
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {attachment_id!r}))
|
||||
target = j.id
|
||||
for _ in range(100):
|
||||
batches = client.xreadgroup(GROUP, 'fault-test', {{STREAM: '>'}}, count=1, block=1000)
|
||||
if batches:
|
||||
mid, fields = batches[0][1][0]
|
||||
handle(sessions, client, mid, fields)
|
||||
if json.loads(fields['job'])['job_id'] == target:
|
||||
print(mid)
|
||||
break
|
||||
else:
|
||||
raise AssertionError('No delivery for target job')
|
||||
''', fault=fault)
|
||||
|
||||
|
||||
def main():
|
||||
client = Client(os.getenv('BASE_URL', 'http://127.0.0.1:8080'))
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
try:
|
||||
compose('stop', 'worker', 'dispatcher', 'cleanup')
|
||||
attachment, content = client.attachment(alice, chat)
|
||||
aid = attachment['id']
|
||||
check(attachment['status'] == 'queued', 'upload must return queued')
|
||||
check(not state(aid)['published'], 'outbox was already published')
|
||||
client.message(alice, chat, '', attachments=[aid])
|
||||
client.call('GET', f'/api/v1/attachments/{aid}/content', 409, actor=bob)
|
||||
compose('restart', 'api')
|
||||
ready(client)
|
||||
check(state(aid)['status'] == 'queued', 'API restart lost upload')
|
||||
compose('start', 'dispatcher')
|
||||
until(lambda: state(aid)['published'])
|
||||
compose('restart', 'broker')
|
||||
compose('start', 'worker')
|
||||
result = client.wait_ready(bob, attachment)
|
||||
check({v['name'] for v in result['variants']} == {'thumbnail', 'crop', 'optimized'}, 'missing variants')
|
||||
for variant in result['variants']:
|
||||
download = client.call('GET', f'/api/v1/attachments/{aid}/content?variant=' + variant['name'], actor=bob)
|
||||
check(hashlib.sha256(download['bytes']).hexdigest() == variant['sha256'], 'variant checksum mismatch')
|
||||
check(len(download['bytes']) == variant['size_bytes'], 'variant size mismatch')
|
||||
check(download['headers'].get_content_type() == variant['content_type'], 'variant MIME mismatch')
|
||||
check(client.call('GET', f'/api/v1/attachments/{aid}/content', actor=bob)['bytes'] == content, 'original changed')
|
||||
print('PASS: outbox COMMIT/publish gap, stopped worker, API and broker restart', flush=True)
|
||||
|
||||
for fault in ('after_objects', 'after_commit'):
|
||||
compose('stop', 'worker')
|
||||
att, _ = client.attachment(alice, chat)
|
||||
aid = att['id']
|
||||
until(lambda: state(aid)['published'])
|
||||
pending_id = inject(aid, fault)
|
||||
snapshot = state(aid)
|
||||
check(snapshot['status'] == ('processing' if fault == 'after_objects' else 'ready'), 'wrong crash state')
|
||||
compose('start', 'worker')
|
||||
until(lambda: state(aid)['status'] == 'ready')
|
||||
until(lambda: remote(f"print(int(not broker().xpending_range(STREAM, GROUP, {pending_id!r}, {pending_id!r}, 1)))\n") == '1')
|
||||
if fault == 'after_objects':
|
||||
check(state(aid)['attempt'] == 2, 'expired lease was not retried')
|
||||
else:
|
||||
time.sleep(2)
|
||||
check(state(aid)['attempt'] == 1, 'ready job was reprocessed')
|
||||
# Force duplicate delivery; stable object keys and metadata must remain identical.
|
||||
before = state(aid)
|
||||
remote(f'''
|
||||
with sessions() as session:
|
||||
j = session.get(AttachmentJob, {before['job_id']!r})
|
||||
broker().xadd(STREAM, {{'job': json.dumps(envelope(j))}})
|
||||
''')
|
||||
time.sleep(2)
|
||||
after = state(aid)
|
||||
check(before['variants'] == after['variants'] and before['attempt'] == after['attempt'], 'duplicate changed result')
|
||||
count = remote(f"print(s3_client().list_objects_v2(Bucket=BUCKET, Prefix='variants/{aid}/')['KeyCount'])\n")
|
||||
check(count == '3', 'redelivery produced extra S3 objects')
|
||||
print('PASS: crash ' + fault + ', redelivery and stable variants', flush=True)
|
||||
|
||||
compose('stop', 'worker')
|
||||
att, content = client.attachment(alice, chat)
|
||||
aid = att['id']
|
||||
until(lambda: state(aid)['published'])
|
||||
# Remove only this test's staging object to cause a real temporary S3 error.
|
||||
staging = state(aid)['staging']
|
||||
remote(f"s3_client().delete_object(Bucket=BUCKET, Key={staging!r})\n")
|
||||
compose('start', 'worker')
|
||||
until(lambda: state(aid)['status'] == 'failed', 150)
|
||||
failed = state(aid)
|
||||
check(failed['error'] == 'retry_exhausted' and failed['attempt'] == 5, 'retry budget incorrect')
|
||||
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
|
||||
body={'text': '', 'attachment_ids': [aid]}, headers={'Idempotency-Key': aid})
|
||||
registry = compose('exec', '-T', 'api', 'python', '-m', 'src.jobs', 'failed')
|
||||
check(failed['job_id'] in registry, 'failed job not visible')
|
||||
remote(f'''
|
||||
import base64
|
||||
s3_client().put_object(Bucket=BUCKET, Key={staging!r},
|
||||
Body=base64.b64decode({base64.b64encode(content).decode()!r}), ContentType='image/png')
|
||||
''')
|
||||
compose('exec', '-T', 'api', 'python', '-m', 'src.jobs', 'replay', failed['job_id'],
|
||||
'--reason', 'synthetic test: restored staging')
|
||||
client.wait_ready(bob, att)
|
||||
check(len(state(aid)['audit']) == 1 and state(aid)['attempt'] == 6, 'replay lost audit or attempt')
|
||||
remote(f'''
|
||||
from src.jobs import replay
|
||||
try:
|
||||
replay(sessions, {failed['job_id']!r}, 'must reject ready')
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError('Replayed a ready job')
|
||||
''')
|
||||
print('PASS: bounded retries, failed registry, explicit audited replay, ready replay rejected', flush=True)
|
||||
|
||||
compose('exec', '-T', 'api', 'python', '-m', 'src.cleanup', '--once')
|
||||
check(state(aid)['staging'] is None, 'completed staging was not cleaned')
|
||||
check(client.call('GET', f'/api/v1/attachments/{aid}/content', actor=bob)['bytes'] == content, 'cleanup deleted original')
|
||||
print('PASS: staging cleanup preserves ready objects', flush=True)
|
||||
finally:
|
||||
compose('start', 'broker', 'dispatcher', 'worker', 'cleanup')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Run inside the application image: python -m unittest discover -s /checks."""
|
||||
import hashlib
|
||||
import io
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from src.image_processor import variants
|
||||
|
||||
|
||||
class ImageTests(unittest.TestCase):
|
||||
def render(self, image, fmt='PNG', **kwargs):
|
||||
source = io.BytesIO()
|
||||
image.save(source, format=fmt, **kwargs)
|
||||
results = list(variants(source.getvalue()))
|
||||
self.assertEqual([meta['name'] for meta, _ in results], ['thumbnail', 'crop', 'optimized'])
|
||||
decoded = {}
|
||||
for meta, data in results:
|
||||
self.assertEqual(meta['sha256'], hashlib.sha256(data).hexdigest())
|
||||
self.assertEqual(meta['size_bytes'], len(data))
|
||||
with Image.open(io.BytesIO(data)) as item:
|
||||
item.load()
|
||||
self.assertEqual(item.size, (meta['width'], meta['height']))
|
||||
self.assertFalse(item.getexif())
|
||||
self.assertNotIn('icc_profile', item.info)
|
||||
decoded[meta['name']] = item.copy()
|
||||
return decoded
|
||||
|
||||
def test_dimensions_and_floor_rounding(self):
|
||||
images = self.render(Image.new('RGB', (321, 200), 'red'))
|
||||
self.assertEqual(images['thumbnail'].size, (256, 159))
|
||||
self.assertEqual(images['crop'].size, (128, 128))
|
||||
self.assertEqual(images['optimized'].size, (321, 200))
|
||||
|
||||
def test_no_upscale_and_white_alpha(self):
|
||||
images = self.render(Image.new('RGBA', (20, 10), (255, 0, 0, 0)))
|
||||
self.assertEqual(images['thumbnail'].size, (20, 10))
|
||||
self.assertEqual(images['optimized'].size, (20, 10))
|
||||
self.assertEqual(images['thumbnail'].getpixel((5, 5)), (255, 255, 255))
|
||||
|
||||
def test_exif_orientation_and_metadata_removal(self):
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6
|
||||
exif[270] = 'private description'
|
||||
images = self.render(Image.new('RGB', (320, 200), 'blue'), 'JPEG', exif=exif)
|
||||
self.assertEqual(images['thumbnail'].size, (160, 256))
|
||||
self.assertEqual(images['optimized'].size, (200, 320))
|
||||
|
||||
def test_optimized_bound(self):
|
||||
images = self.render(Image.new('RGB', (2000, 1000), 'green'))
|
||||
self.assertEqual(images['optimized'].size, (1280, 640))
|
||||
|
||||
def test_worker_pixel_limit(self):
|
||||
with patch('src.image_processor.MAX_PIXELS', 100):
|
||||
with self.assertRaises(ValueError):
|
||||
self.render(Image.new('RGB', (11, 10)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user