From a0ec26d96dcc4cfb19981eb2f45f7203fe5b5058 Mon Sep 17 00:00:00 2001 From: Sergey Lyakhonov Date: Fri, 20 Nov 2026 06:15:06 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A0=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D1=82=D1=8C=20=D1=83=D1=81=D1=82=D0=BE=D0=B9=D1=87?= =?UTF-8?q?=D0=B8=D0=B2=D1=8B=D0=B9=20=D0=BF=D1=80=D0=B8=D1=91=D0=BC=20?= =?UTF-8?q?=D0=B2=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD=D0=B8=D0=B9,=20=D0=B1?= =?UTF-8?q?=D1=80=D0=BE=D0=BA=D0=B5=D1=80=20=D0=B8=20=D0=BE=D0=B1=D1=80?= =?UTF-8?q?=D0=B0=D0=B1=D0=BE=D1=82=D0=BA=D1=83=20=D0=BC=D0=B8=D0=BD=D0=B8?= =?UTF-8?q?=D0=B0=D1=82=D1=8E=D1=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- compose.yaml | 153 +++++++++++++++++++++++ infra/broker.sh | 5 + infra/bucket.sh | 10 +- infra/rabbitmq.conf | 3 + infra/s3-policy.json | 2 +- lab5.env.example | 7 ++ migrations/004_attachment_pipeline.sql | 49 ++++++++ requirements.lock.txt | 1 + requirements.txt | 1 + scripts/bootstrap.py | 33 +++++ src/broker.py | 36 ++++++ src/dispatch.py | 53 ++++++++ src/image_pipeline.py | 50 ++++++++ src/jobs.py | 161 +++++++++++++++++++++++++ src/objects.py | 19 +++ src/routes/attachments.py | 59 +++++---- src/routes/health.py | 2 +- src/storage.py | 2 +- src/worker.py | 118 ++++++++++++++++++ 19 files changed, 738 insertions(+), 26 deletions(-) create mode 100644 compose.yaml create mode 100644 infra/broker.sh create mode 100644 infra/rabbitmq.conf create mode 100644 lab5.env.example create mode 100644 migrations/004_attachment_pipeline.sql create mode 100644 scripts/bootstrap.py create mode 100644 src/broker.py create mode 100644 src/dispatch.py create mode 100644 src/image_pipeline.py create mode 100644 src/jobs.py create mode 100644 src/worker.py diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..c44c1cc4 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,153 @@ +name: lyakhonov-lab5 + +x-app: &app + image: lyakhonov-messenger:lab5 + build: . + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3:9000 + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + CACHE_URL: redis://redis:6379/0 + BROKER_URL_FILE: /run/secrets/broker_url + secrets: [database_url, s3_access_key, s3_secret_key, broker_url] + cpus: 0.5 + mem_limit: 640m + stop_grace_period: 12s + +x-dependencies: &dependencies + migrate: + condition: service_completed_successfully + bucket-init: + condition: service_completed_successfully + +services: + db: + image: postgres:16.6-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 30 + cpus: 1.0 + mem_limit: 512m + redis: + image: redis:7.4.1-alpine + command: [redis-server, --maxmemory, 64mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"] + cpus: 0.25 + mem_limit: 128m + s3: + image: lyakhonov-storage:lab5 + build: + context: . + dockerfile: Dockerfile.s3 + command: [server, /data, --console-address, '127.0.0.1:9001'] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password + secrets: [s3_root_user, s3_root_password] + volumes: [s3_data:/data] + cpus: 1.0 + mem_limit: 768m + bucket-init: + image: lyakhonov-storage:lab5 + pull_policy: never + entrypoint: [/bin/sh, /bootstrap/bucket.sh] + secrets: [s3_root_user, s3_root_password, s3_access_key, s3_secret_key] + volumes: [./infra/bucket.sh:/bootstrap/bucket.sh:ro, ./infra/s3-policy.json:/bootstrap/policy.json:ro] + depends_on: [s3] + cpus: 0.25 + mem_limit: 128m + broker: + image: rabbitmq:4.1.0-alpine + hostname: broker + entrypoint: [/bin/sh, /bootstrap/broker.sh] + volumes: [broker_data:/var/lib/rabbitmq, ./infra/broker.sh:/bootstrap/broker.sh:ro, + ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-lab5.conf:ro] + secrets: [broker_password] + healthcheck: + test: [CMD, rabbitmq-diagnostics, -q, ping] + interval: 5s + timeout: 4s + retries: 20 + cpus: 0.75 + mem_limit: 512m + migrate: + <<: *app + command: [python, -m, src.migrate] + depends_on: + db: + condition: service_healthy + healthcheck: + disable: true + api: + <<: *app + depends_on: *dependencies + deploy: + replicas: ${API_REPLICAS:-2} + dispatcher: + <<: *app + command: [python, -m, src.dispatch] + restart: unless-stopped + depends_on: + <<: *dependencies + broker: + condition: service_healthy + healthcheck: + disable: true + mem_limit: 256m + worker: + <<: *app + command: [python, -m, src.worker] + restart: unless-stopped + depends_on: + <<: *dependencies + broker: + condition: service_healthy + healthcheck: + disable: true + deploy: + replicas: ${WORKER_CONCURRENCY:-2} + cpus: 0.75 + mem_limit: 768m + stop_grace_period: 95s + lb: + image: haproxy:3.0.6-alpine + ports: ['127.0.0.1:${LAB5_PORT:-8085}:8080'] + volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro] + depends_on: [api] + cpus: 0.5 + mem_limit: 128m + stop_grace_period: 12s + +secrets: + s3_root_user: + file: ./secrets/s3_root_user + s3_root_password: + file: ./secrets/s3_root_password + s3_access_key: + file: ./secrets/s3_access_key + s3_secret_key: + file: ./secrets/s3_secret_key + db_password: + file: ./secrets/db_password + database_url: + file: ./secrets/database_url + broker_password: + file: ./secrets/broker_password + broker_url: + file: ./secrets/broker_url +volumes: + postgres_data: + s3_data: + broker_data: diff --git a/infra/broker.sh b/infra/broker.sh new file mode 100644 index 00000000..bb064230 --- /dev/null +++ b/infra/broker.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +export RABBITMQ_DEFAULT_USER=messenger +export RABBITMQ_DEFAULT_PASS="$(cat /run/secrets/broker_password)" +exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server diff --git a/infra/bucket.sh b/infra/bucket.sh index 295b0271..acf28ff1 100644 --- a/infra/bucket.sh +++ b/infra/bucket.sh @@ -4,7 +4,15 @@ set -eu # Не включать set -x: аргументы содержат учебные credentials. -until mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null 2>&1; do sleep 1; done +attempt=0 +until mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 30 ]; then + echo 'S3 bootstrap timed out' >&2 + exit 1 + fi + sleep 1 +done mc mb --ignore-existing local/attachments >/dev/null diff --git a/infra/rabbitmq.conf b/infra/rabbitmq.conf new file mode 100644 index 00000000..c150c13a --- /dev/null +++ b/infra/rabbitmq.conf @@ -0,0 +1,3 @@ +heartbeat = 180 +vm_memory_high_watermark.absolute = 300MiB +disk_free_limit.absolute = 200MiB diff --git a/infra/s3-policy.json b/infra/s3-policy.json index 6b7d394e..1bff5854 100644 --- a/infra/s3-policy.json +++ b/infra/s3-policy.json @@ -1 +1 @@ -{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::attachments"],"Condition":{"StringLike":{"s3:prefix":["originals/*"]}}},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::attachments/originals/*"]}]} \ No newline at end of file +{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::attachments"],"Condition":{"StringLike":{"s3:prefix":["staging/*"]}}},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::attachments/originals/*","arn:aws:s3:::attachments/variants/*"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::attachments/staging/*"]}]} diff --git a/lab5.env.example b/lab5.env.example new file mode 100644 index 00000000..7096d481 --- /dev/null +++ b/lab5.env.example @@ -0,0 +1,7 @@ +LAB5_PORT=8085 +API_REPLICAS=2 +SESSION_TTL_SECONDS=3600 +WORKER_CONCURRENCY=2 +JOB_MAX_ATTEMPTS=5 +JOB_TIMEOUT_SECONDS=60 +STAGING_RETENTION_SECONDS=86400 diff --git a/migrations/004_attachment_pipeline.sql b/migrations/004_attachment_pipeline.sql new file mode 100644 index 00000000..2e2eb707 --- /dev/null +++ b/migrations/004_attachment_pipeline.sql @@ -0,0 +1,49 @@ +ALTER TABLE attachments DROP CONSTRAINT attachments_state_check; +UPDATE attachments SET state = 'failed' WHERE state = 'missing'; +ALTER TABLE attachments ADD COLUMN error_code text; +UPDATE attachments SET error_code = 'original_unavailable' WHERE state = 'failed'; +ALTER TABLE attachments ADD CONSTRAINT attachments_pipeline_state CHECK ( + state IN ('queued', 'processing', 'ready', 'failed') + AND ((state = 'failed') = (error_code IS NOT NULL)) +); + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id), + staging_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + attempt_limit integer NOT NULL CHECK (attempt_limit >= 1), + state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued', 'processing', 'ready', 'failed')), + next_publish_at timestamptz, + published_at timestamptz, + publish_failures integer NOT NULL DEFAULT 0, + lease_until timestamptz, + lease_token uuid, + last_error text, + CHECK ((lease_until IS NULL) = (lease_token IS NULL)) +); +CREATE INDEX attachment_jobs_outbox ON attachment_jobs(next_publish_at) + WHERE next_publish_at IS NOT NULL; +CREATE INDEX attachment_jobs_leases ON attachment_jobs(lease_until) + WHERE lease_until IS NOT NULL; + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id), + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + object_key text NOT NULL UNIQUE, + content_type text NOT NULL, + size_bytes integer NOT NULL CHECK (size_bytes > 0), + sha256 char(64) NOT NULL, + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + PRIMARY KEY (attachment_id, name) +); + +CREATE TABLE attachment_replays ( + job_id uuid NOT NULL REFERENCES attachment_jobs(id), + attempt integer NOT NULL, + reason text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (job_id, attempt) +); diff --git a/requirements.lock.txt b/requirements.lock.txt index 6fccf885..4da3e7c2 100644 --- a/requirements.lock.txt +++ b/requirements.lock.txt @@ -26,6 +26,7 @@ redis==5.2.0 boto3==1.40.61 Pillow==12.0.0 +pika==1.3.2 botocore==1.40.76 s3transfer==0.14.0 jmespath==1.1.0 diff --git a/requirements.txt b/requirements.txt index 7010e5f2..d27c7f25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ redis==5.2.0 boto3==1.40.61 Pillow==12.0.0 +pika==1.3.2 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..1b4cc364 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,33 @@ +"""Создаёт только отсутствующие локальные секреты и конфигурацию.""" + +from pathlib import Path +from secrets import token_hex +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] + + +def create(path, value): + try: + with path.open('x', encoding='utf-8') as target: + target.write(value) + except FileExistsError: + pass + + +def main(): + directory = ROOT / 'secrets' + directory.mkdir(mode=0o700, exist_ok=True) + for name in ('db_password', 'broker_password', 's3_root_user', 's3_root_password', + 's3_access_key', 's3_secret_key'): + create(directory / name, token_hex(24)) + db = quote((directory / 'db_password').read_text(encoding='utf-8').strip(), safe='') + broker = quote((directory / 'broker_password').read_text(encoding='utf-8').strip(), safe='') + create(directory / 'database_url', f'postgresql://messenger:{db}@db:5432/messenger') + create(directory / 'broker_url', f'amqp://messenger:{broker}@broker:5672/%2F') + create(ROOT / '.env', (ROOT / 'lab5.env.example').read_text(encoding='utf-8')) + print('Локальные файлы готовы') + + +if __name__ == '__main__': + main() diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 00000000..68c10390 --- /dev/null +++ b/src/broker.py @@ -0,0 +1,36 @@ +"""RabbitMQ с подтверждаемой публикацией и ручным ack.""" + +import json +from contextlib import contextmanager + +import pika + +from src.config import setting + +QUEUE = 'attachment.process.v1' + + +@contextmanager +def channel(heartbeat=90): + params = pika.URLParameters(setting('BROKER_URL')) + params.heartbeat = heartbeat + params.socket_timeout = 4 + params.stack_timeout = 6 + params.blocked_connection_timeout = 5 + connection = pika.BlockingConnection(params) + try: + result = connection.channel() + result.queue_declare(queue=QUEUE, durable=True) + yield connection, result + finally: + if connection.is_open: + connection.close() + + +def publish(ch, message): + ch.basic_publish( + exchange='', routing_key=QUEUE, body=json.dumps(message, separators=(',', ':')).encode(), + properties=pika.BasicProperties(content_type='application/json', delivery_mode=2, + message_id=message['job_id']), + mandatory=True, + ) diff --git a/src/dispatch.py b/src/dispatch.py new file mode 100644 index 00000000..72611427 --- /dev/null +++ b/src/dispatch.py @@ -0,0 +1,53 @@ +"""Отдельный relay: lease публикации, publisher confirm, восстановление jobs.""" + +import logging +import signal +import threading +import time + +import pika +import psycopg +from psycopg_pool import PoolTimeout + +from src import broker, jobs +from src.storage import storage + + +def main(): + logging.basicConfig(level=logging.WARNING) + stopping = threading.Event() + for sig in (signal.SIGINT, signal.SIGTERM): + signal.signal(sig, lambda *_: stopping.set()) + storage.pool.open(wait=True) + next_recovery = 0.0 + try: + while not stopping.is_set(): + reserved = None + try: + with broker.channel(heartbeat=10) as (connection, channel): + channel.confirm_delivery() + while not stopping.is_set(): + if time.monotonic() >= next_recovery: + jobs.recover_expired() + next_recovery = time.monotonic() + 1 + reserved = jobs.reserve_publish() + if reserved: + broker.publish(channel, jobs.envelope(reserved)) + jobs.confirm_publish(reserved) + reserved = None + else: + connection.process_data_events(time_limit=0.2) + except (pika.exceptions.AMQPError, OSError, psycopg.Error, PoolTimeout) as exc: + logging.warning('dispatch temporarily unavailable: %s', type(exc).__name__) + if reserved: + try: + jobs.publish_failed(reserved) + except (psycopg.Error, PoolTimeout): + pass # expires through the durable publication deadline + stopping.wait(1) + finally: + storage.pool.close() + + +if __name__ == '__main__': + main() diff --git a/src/image_pipeline.py b/src/image_pipeline.py new file mode 100644 index 00000000..b2512e7a --- /dev/null +++ b/src/image_pipeline.py @@ -0,0 +1,50 @@ +"""Преобразования Pillow в отдельном процессе с жёстким таймаутом родителя.""" + +import json +import sys +from pathlib import Path + +from PIL import Image, ImageOps + +Image.MAX_IMAGE_PIXELS = 40_000_000 + + +def fit(image, limit): + width, height = image.size + longest = max(width, height) + if longest <= limit: + return image + size = (max(1, width * limit // longest), max(1, height * limit // longest)) + return image.resize(size, Image.Resampling.LANCZOS) + + +def render(source, target): + with Image.open(source) as opened: + if opened.width * opened.height > Image.MAX_IMAGE_PIXELS: + raise ValueError('pixel limit') + image = ImageOps.exif_transpose(opened) + if 'A' in image.getbands() or 'transparency' in image.info: + rgba = image.convert('RGBA') + rgb = Image.new('RGB', rgba.size, 'white') + rgb.paste(rgba, mask=rgba.getchannel('A')) + else: + rgb = image.convert('RGB') + rgb.info.clear() + del image + thumbnail = fit(rgb, 256) + thumbnail.save(target / 'thumbnail', 'JPEG', quality=85) + return [['thumbnail', 'image/jpeg', *thumbnail.size]] + + +def main(): + directory = Path(sys.argv[1]) + try: + result = render(directory / 'original', directory) + except (OSError, ValueError, SyntaxError, Image.DecompressionBombError): + return 2 + (directory / 'manifest.json').write_text(json.dumps(result), encoding='utf-8') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 00000000..9e4d7e10 --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,161 @@ +"""Устойчивое намерение обработки, lease и атомарная публикация manifest.""" + +import random +from uuid import UUID, uuid4 + +from src.config import setting +from src.storage import storage, timestamp + +JOB_TIMEOUT = int(setting('JOB_TIMEOUT_SECONDS', '60')) +LEASE_SECONDS = JOB_TIMEOUT + 30 + + +def envelope(job): + return { + 'schema_version': 1, + 'job_id': str(job['id']), + 'type': 'attachment.process.v1', + 'attachment_id': str(job['attachment_id']), + 'pipeline_version': 1, + 'attempt': job['attempt'], + 'created_at': timestamp(job['created_at']), + } + + +def validate(message): + if set(message) != {'schema_version', 'job_id', 'type', 'attachment_id', + 'pipeline_version', 'attempt', 'created_at'}: + raise ValueError('invalid fields') + if (type(message['schema_version']) is not int or message['schema_version'] != 1 + or message['type'] != 'attachment.process.v1' + or type(message['pipeline_version']) is not int or message['pipeline_version'] != 1 + or type(message['attempt']) is not int or message['attempt'] < 1): + raise ValueError('invalid job') + UUID(message['job_id']) + UUID(message['attachment_id']) + return message + + +def reserve_publish(): + with storage.pool.connection() as conn: + return conn.execute( + """WITH candidate AS ( + SELECT id FROM attachment_jobs WHERE state='queued' + AND next_publish_at <= clock_timestamp() + ORDER BY next_publish_at, id LIMIT 1 FOR UPDATE SKIP LOCKED + ) + UPDATE attachment_jobs j + SET next_publish_at=clock_timestamp()+interval '15 seconds' + FROM candidate WHERE j.id=candidate.id RETURNING j.*""" + ).fetchone() + + +def confirm_publish(job): + with storage.pool.connection() as conn: + conn.execute( + """UPDATE attachment_jobs SET next_publish_at=NULL, published_at=clock_timestamp(), + publish_failures=0 WHERE id=%s AND attempt=%s""", + (job['id'], job['attempt']), + ) + + +def publish_failed(job): + with storage.pool.connection() as conn: + conn.execute( + """UPDATE attachment_jobs SET publish_failures=publish_failures+1, + next_publish_at=clock_timestamp()+%s * interval '1 second' + WHERE id=%s AND attempt=%s AND state='queued' AND next_publish_at IS NOT NULL""", + (delay(job['publish_failures'] + 1), job['id'], job['attempt']), + ) + + +def locked_job(conn, job_id, attachment_id): + attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE', + (attachment_id,)).fetchone() + if not attachment: + return None, None + job = conn.execute('SELECT * FROM attachment_jobs WHERE id=%s AND attachment_id=%s FOR UPDATE', + (job_id, attachment_id)).fetchone() + return job, attachment + + +def claim(message): + with storage.pool.connection() as conn: + job, attachment = locked_job(conn, message['job_id'], message['attachment_id']) + if not job or envelope(job) != message or job['state'] != 'queued': + return None + token = uuid4() + conn.execute( + """UPDATE attachment_jobs SET state='processing', lease_token=%s, + lease_until=clock_timestamp()+%s * interval '1 second', next_publish_at=NULL + WHERE id=%s""", (token, LEASE_SECONDS, job['id']), + ) + conn.execute("UPDATE attachments SET state='processing',updated_at=clock_timestamp() WHERE id=%s", + (job['attachment_id'],)) + return job | {'lease_token': token, 'attachment': attachment} + + +def owns(conn, job, claimed): + return ( + job and (job['state'], job['lease_token']) == ('processing', claimed['lease_token']) + and conn.execute('SELECT clock_timestamp() < %s AS active', (job['lease_until'],)).fetchone()['active'] + ) + + +def complete(claimed, variants): + with storage.pool.connection() as conn: + job, _ = locked_job(conn, claimed['id'], claimed['attachment_id']) + if not owns(conn, job, claimed): + return False + for variant in variants: + conn.execute( + """INSERT INTO attachment_variants + (attachment_id,name,object_key,content_type,size_bytes,sha256,width,height) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""", + (job['attachment_id'], *variant), + ) + conn.execute("UPDATE attachments SET state='ready',updated_at=clock_timestamp() WHERE id=%s", + (job['attachment_id'],)) + conn.execute("UPDATE attachment_jobs SET state='ready',lease_token=NULL,lease_until=NULL," + "last_error=NULL WHERE id=%s", (job['id'],)) + return True + + +def delay(attempt): + return min(30, 2 ** min(attempt - 1, 5)) * random.uniform(0.75, 1.25) + + +def settle_error(conn, job, code, permanent=False): + exhausted = job['attempt'] >= job['attempt_limit'] + if permanent or exhausted: + error = code if permanent else 'retry_exhausted' + conn.execute("UPDATE attachments SET state='failed',error_code=%s,updated_at=clock_timestamp() " + "WHERE id=%s", (error, job['attachment_id'])) + conn.execute("UPDATE attachment_jobs SET state='failed',lease_token=NULL,lease_until=NULL," + "last_error=%s,next_publish_at=NULL WHERE id=%s", (code, job['id'])) + return + conn.execute("UPDATE attachments SET state='queued',updated_at=clock_timestamp() WHERE id=%s", + (job['attachment_id'],)) + conn.execute("""UPDATE attachment_jobs SET state='queued',lease_token=NULL,lease_until=NULL, + last_error=%s,attempt=attempt+1,published_at=NULL, + next_publish_at=clock_timestamp()+%s*interval '1 second' WHERE id=%s""", + (code, delay(job['attempt']), job['id'])) + + +def fail(claimed, code, permanent=False): + with storage.pool.connection() as conn: + job, _ = locked_job(conn, claimed['id'], claimed['attachment_id']) + if owns(conn, job, claimed): + settle_error(conn, job, code, permanent) + + +def recover_expired(): + with storage.pool.connection() as conn: + expired = conn.execute("SELECT id,attachment_id FROM attachment_jobs " + "WHERE lease_until <= clock_timestamp() ORDER BY lease_until LIMIT 16").fetchall() + for candidate in expired: + with storage.pool.connection() as conn: + job, _ = locked_job(conn, candidate['id'], candidate['attachment_id']) + if (job and job['state'] == 'processing' + and not owns(conn, job, job)): + settle_error(conn, job, 'lease_expired') diff --git a/src/objects.py b/src/objects.py index ef7f82ee..d3ec1dab 100644 --- a/src/objects.py +++ b/src/objects.py @@ -1,6 +1,7 @@ """S3 operations with bounded timeouts and immutable server-generated keys.""" import boto3 from botocore.config import Config +from botocore.exceptions import ClientError from src.config import setting def client(): @@ -11,6 +12,24 @@ def client(): BUCKET = setting('S3_BUCKET','attachments') + +class ObjectConflict(Exception): + """Неизменяемый ключ уже содержит другие данные.""" + + +def put_immutable(s3, key, stream, size, content_type, digest): + try: + s3.put_object(Bucket=BUCKET, Key=key, Body=stream, ContentLength=size, + ContentType=content_type, Metadata={'sha256': digest}, IfNoneMatch='*') + except ClientError as exc: + if exc.response['ResponseMetadata']['HTTPStatusCode'] != 412: + raise + existing = s3.head_object(Bucket=BUCKET, Key=key) + if (existing['ContentLength'], existing['ContentType'], existing['Metadata'].get('sha256')) != ( + size, content_type, digest): + raise ObjectConflict(key) from None + + def object_lock(conn, key, wait=True): fn = 'pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock' row = conn.execute('SELECT '+fn+'(hashtextextended(%s,42)) AS locked',(key,)).fetchone() diff --git a/src/routes/attachments.py b/src/routes/attachments.py index e3ca1325..0fd5ac4b 100644 --- a/src/routes/attachments.py +++ b/src/routes/attachments.py @@ -12,7 +12,8 @@ from starlette.responses import StreamingResponse from starlette.requests import ClientDisconnect from src.routes.common import require_user, make_error, get_request_id from src.storage import storage, timestamp -from src.objects import client, BUCKET, object_lock +from src.objects import client, BUCKET, object_lock, put_immutable +from src.config import setting router = APIRouter() slots = asyncio.Semaphore(1) @@ -47,26 +48,29 @@ def inspect_image(stream, declared): finally: stream.seek(0) -def metadata(row): +def metadata(row, variants=()): return {k: str(row[k]) for k in ('id','chat_id','owner_id')} | { k:row[k] for k in ('filename','content_type','size_bytes','sha256')} | { - 'status':'ready','variants':[],'error_code':None, + 'status':row['state'],'variants':list(variants),'error_code':row['error_code'], 'created_at':timestamp(row['created_at']),'updated_at':timestamp(row['updated_at'])} def save(stream, chat_id, owner, filename, content_type, size, digest): attachment_id=uuid.uuid4() - key='originals/'+str(attachment_id) - # Keep this lock through S3 PUT and DB COMMIT/ROLLBACK. Reconciler shares it. + key=f'originals/{attachment_id}' + staging_key=f'staging/{attachment_id}' with storage.pool.connection() as conn: - object_lock(conn,key) - client().put_object(Bucket=BUCKET,Key=key,Body=stream,ContentLength=size, - ContentType=content_type,Metadata={'sha256':digest},IfNoneMatch='*') - row=conn.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'ready') RETURNING *", + object_lock(conn, staging_key) + put_immutable(client(), staging_key, stream, size, content_type, digest) + row=conn.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *", (attachment_id,chat_id,owner,key,filename,content_type,size,digest)).fetchone() - # Never blindly delete after ambiguous COMMIT: grace-period reconciliation handles it. + conn.execute( + "INSERT INTO attachment_jobs(id,attachment_id,staging_key,attempt_limit,next_publish_at) " + "VALUES (%s,%s,%s,%s,clock_timestamp())", + (uuid.uuid4(), attachment_id, staging_key, int(setting('JOB_MAX_ATTEMPTS', '5'))), + ) return metadata(row) -@router.post('/api/v1/chats/{chat_id}/attachments',status_code=201) +@router.post('/api/v1/chats/{chat_id}/attachments',status_code=202) async def upload(request:Request,chat_id:uuid.UUID,filename:str='',actor:str=Depends(require_user)): if not await run_in_threadpool(storage.is_chat_member,str(chat_id),actor): raise error(request,404,'not_found','chat not found') @@ -110,26 +114,37 @@ async def upload(request:Request,chat_id:uuid.UUID,filename:str='',actor:str=Dep finally: slots.release() -def accessible(request,attachment_id,actor): - with storage.pool.connection() as conn: - row=conn.execute('SELECT a.* FROM attachments a JOIN chat_members m ON a.chat_id=m.chat_id WHERE a.id=%s AND m.user_id=%s',(attachment_id,actor)).fetchone() +def accessible(conn, request, attachment_id, actor): + # One consistent manifest for all reads in the enclosing transaction. + row=conn.execute('SELECT a.* FROM attachments a JOIN chat_members m ON a.chat_id=m.chat_id ' + 'WHERE a.id=%s AND m.user_id=%s FOR SHARE OF a',(attachment_id,actor)).fetchone() if not row: raise error(request,404,'not_found','attachment not found') - if row['state']!='ready': - raise error(request,503,'unavailable','original temporarily unavailable') return row @router.get('/api/v1/attachments/{attachment_id}') def get_attachment(request:Request,attachment_id:uuid.UUID,actor:str=Depends(require_user)): - return metadata(accessible(request,attachment_id,actor)) + with storage.pool.connection() as conn: + row = accessible(conn, request, attachment_id, actor) + variants = conn.execute( + 'SELECT name,content_type,size_bytes,sha256,width,height FROM attachment_variants ' + 'WHERE attachment_id=%s ORDER BY name', (attachment_id,) + ).fetchall() + return metadata(row, variants) @router.get('/api/v1/attachments/{attachment_id}/content') def content(request:Request,attachment_id:uuid.UUID,variant:str='original',actor:str=Depends(require_user)): - row=accessible(request,attachment_id,actor) - if variant not in ('original','thumbnail','crop','optimized'): - raise error(request,400,'invalid_request','invalid variant') - if variant!='original': - raise error(request,404,'not_found','variant not found') + with storage.pool.connection() as conn: + row = accessible(conn, request, attachment_id, actor) + if variant not in ('original','thumbnail','crop','optimized'): + raise error(request,400,'invalid_request','invalid variant') + if row['state'] != 'ready': + raise error(request,409,'conflict','attachment not ready') + if variant != 'original': + row = conn.execute('SELECT * FROM attachment_variants WHERE attachment_id=%s AND name=%s', + (attachment_id, variant)).fetchone() + if not row: + raise error(request,404,'not_found','variant not found') obj=client().get_object(Bucket=BUCKET,Key=row['object_key']) def chunks(): try: diff --git a/src/routes/health.py b/src/routes/health.py index 0e9b4b94..9e45f954 100644 --- a/src/routes/health.py +++ b/src/routes/health.py @@ -14,7 +14,7 @@ def liveness(): def readiness(): storage.ready() from src.objects import client, BUCKET - client().list_objects_v2(Bucket=BUCKET,Prefix="originals/",MaxKeys=1) + client().list_objects_v2(Bucket=BUCKET, Prefix="staging/", MaxKeys=1) return Health() diff --git a/src/storage.py b/src/storage.py index 995a4cb1..45ad1a94 100644 --- a/src/storage.py +++ b/src/storage.py @@ -216,7 +216,7 @@ class PostgresStorage: attachment = conn.execute('SELECT state FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE', (attachment_id,chat_id)).fetchone() if not attachment: raise NotFoundError('вложение не найдено') - if attachment['state'] != 'ready': + if attachment['state'] == 'failed': raise ConflictError('вложение недоступно') optimized = os.getenv('OPTIMIZED_WRITES','1') == '1' if not optimized: diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 00000000..38d12be5 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,118 @@ +"""Worker обрабатывает одну доставку за раз и подтверждает после COMMIT.""" + +import hashlib +import json +import logging +import signal +import subprocess +import sys +import tempfile +import threading +from contextlib import closing +from pathlib import Path + +import pika +import psycopg +from botocore.exceptions import BotoCoreError, ClientError +from psycopg_pool import PoolTimeout + +from src import broker, jobs +from src.objects import BUCKET, ObjectConflict, client, put_immutable +from src.storage import storage + + +class PermanentError(Exception): + pass + + +def fetch(s3, key, attachment, destination): + digest = hashlib.sha256() + size = 0 + obj = s3.get_object(Bucket=BUCKET, Key=key) + with closing(obj['Body']) as stream, destination.open('wb') as output: + for chunk in stream.iter_chunks(chunk_size=64 * 1024): + size += len(chunk) + if size > attachment['size_bytes']: + raise PermanentError('staging_integrity') + digest.update(chunk) + output.write(chunk) + if (size, digest.hexdigest()) != (attachment['size_bytes'], attachment['sha256']): + raise PermanentError('staging_integrity') + + +def process(job): + attachment = job['attachment'] + s3 = client() + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + original = directory / 'original' + fetch(s3, job['staging_key'], attachment, original) + result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', temp], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=jobs.JOB_TIMEOUT, check=False) + if result.returncode == 2: + raise PermanentError('invalid_image') + result.check_returncode() + with original.open('rb') as stream: + put_immutable(s3, attachment['object_key'], stream, attachment['size_bytes'], + attachment['content_type'], attachment['sha256']) + variants = [] + for name, mime, width, height in json.loads((directory / 'manifest.json').read_text()): + key = f"variants/{attachment['id']}/v1/{name}" + output = directory / name + size = output.stat().st_size + with output.open('rb') as stream: + digest = hashlib.file_digest(stream, 'sha256').hexdigest() + stream.seek(0) + put_immutable(s3, key, stream, size, mime, digest) + variants.append((name, key, mime, size, digest, width, height)) + jobs.complete(job, variants) + + +def handle(channel, delivery, body): + try: + message = jobs.validate(json.loads(body)) + except (ValueError, TypeError, AttributeError): + logging.warning('invalid broker job') + channel.basic_reject(delivery.delivery_tag, requeue=False) + return + + job = jobs.claim(message) + if job: + try: + process(job) + except PermanentError as exc: + jobs.fail(job, str(exc), permanent=True) + except ObjectConflict: + jobs.fail(job, 'object_conflict', permanent=True) + except (BotoCoreError, ClientError, OSError, subprocess.SubprocessError) as exc: + logging.warning('processing retry job=%s cause=%s', job['id'], type(exc).__name__) + jobs.fail(job, 'processing_timeout' if isinstance(exc, subprocess.TimeoutExpired) else 'processing_failed') + channel.basic_ack(delivery.delivery_tag) + + +def main(): + logging.basicConfig(level=logging.WARNING) + stopping = threading.Event() + for sig in (signal.SIGINT, signal.SIGTERM): + signal.signal(sig, lambda *_: stopping.set()) + storage.pool.open(wait=True) + try: + while not stopping.is_set(): + try: + with broker.channel(heartbeat=jobs.LEASE_SECONDS + 15) as (_, channel): + channel.basic_qos(prefetch_count=1) + for delivery, _, body in channel.consume(broker.QUEUE, inactivity_timeout=0.5): + if stopping.is_set(): + break + if delivery: + handle(channel, delivery, body) + except (pika.exceptions.AMQPError, OSError, psycopg.Error, PoolTimeout) as exc: + logging.warning('worker temporarily unavailable: %s', type(exc).__name__) + stopping.wait(1) + finally: + storage.pool.close() + + +if __name__ == '__main__': + main()