forked from mtusi2026/mtusi2026lab5
Реализовать устойчивый приём вложений, брокер и обработку миниатюр
This commit is contained in:
+153
@@ -0,0 +1,153 @@
|
||||
name: lyakhonov-lab5
|
||||
|
||||
x-app: &app
|
||||
image: lyakhonov-messenger:lab5
|
||||
build: .
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
secrets: [database_url, s3_access_key, s3_secret_key, broker_url]
|
||||
cpus: 0.5
|
||||
mem_limit: 640m
|
||||
stop_grace_period: 12s
|
||||
|
||||
x-dependencies: &dependencies
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
bucket-init:
|
||||
condition: service_completed_successfully
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:16.6-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
|
||||
secrets: [db_password]
|
||||
volumes: [postgres_data:/var/lib/postgresql/data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
redis:
|
||||
image: redis:7.4.1-alpine
|
||||
command: [redis-server, --maxmemory, 64mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"]
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
s3:
|
||||
image: lyakhonov-storage:lab5
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.s3
|
||||
command: [server, /data, --console-address, '127.0.0.1:9001']
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
|
||||
secrets: [s3_root_user, s3_root_password]
|
||||
volumes: [s3_data:/data]
|
||||
cpus: 1.0
|
||||
mem_limit: 768m
|
||||
bucket-init:
|
||||
image: lyakhonov-storage:lab5
|
||||
pull_policy: never
|
||||
entrypoint: [/bin/sh, /bootstrap/bucket.sh]
|
||||
secrets: [s3_root_user, s3_root_password, s3_access_key, s3_secret_key]
|
||||
volumes: [./infra/bucket.sh:/bootstrap/bucket.sh:ro, ./infra/s3-policy.json:/bootstrap/policy.json:ro]
|
||||
depends_on: [s3]
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
broker:
|
||||
image: rabbitmq:4.1.0-alpine
|
||||
hostname: broker
|
||||
entrypoint: [/bin/sh, /bootstrap/broker.sh]
|
||||
volumes: [broker_data:/var/lib/rabbitmq, ./infra/broker.sh:/bootstrap/broker.sh:ro,
|
||||
./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-lab5.conf:ro]
|
||||
secrets: [broker_password]
|
||||
healthcheck:
|
||||
test: [CMD, rabbitmq-diagnostics, -q, ping]
|
||||
interval: 5s
|
||||
timeout: 4s
|
||||
retries: 20
|
||||
cpus: 0.75
|
||||
mem_limit: 512m
|
||||
migrate:
|
||||
<<: *app
|
||||
command: [python, -m, src.migrate]
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
disable: true
|
||||
api:
|
||||
<<: *app
|
||||
depends_on: *dependencies
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
dispatcher:
|
||||
<<: *app
|
||||
command: [python, -m, src.dispatch]
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
<<: *dependencies
|
||||
broker:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
disable: true
|
||||
mem_limit: 256m
|
||||
worker:
|
||||
<<: *app
|
||||
command: [python, -m, src.worker]
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
<<: *dependencies
|
||||
broker:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
disable: true
|
||||
deploy:
|
||||
replicas: ${WORKER_CONCURRENCY:-2}
|
||||
cpus: 0.75
|
||||
mem_limit: 768m
|
||||
stop_grace_period: 95s
|
||||
lb:
|
||||
image: haproxy:3.0.6-alpine
|
||||
ports: ['127.0.0.1:${LAB5_PORT:-8085}:8080']
|
||||
volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
|
||||
depends_on: [api]
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
stop_grace_period: 12s
|
||||
|
||||
secrets:
|
||||
s3_root_user:
|
||||
file: ./secrets/s3_root_user
|
||||
s3_root_password:
|
||||
file: ./secrets/s3_root_password
|
||||
s3_access_key:
|
||||
file: ./secrets/s3_access_key
|
||||
s3_secret_key:
|
||||
file: ./secrets/s3_secret_key
|
||||
db_password:
|
||||
file: ./secrets/db_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
broker_password:
|
||||
file: ./secrets/broker_password
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
volumes:
|
||||
postgres_data:
|
||||
s3_data:
|
||||
broker_data:
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
export RABBITMQ_DEFAULT_USER=messenger
|
||||
export RABBITMQ_DEFAULT_PASS="$(cat /run/secrets/broker_password)"
|
||||
exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server
|
||||
+9
-1
@@ -4,7 +4,15 @@ set -eu
|
||||
|
||||
# Не включать set -x: аргументы содержат учебные credentials.
|
||||
|
||||
until mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null 2>&1; do sleep 1; done
|
||||
attempt=0
|
||||
until mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null 2>&1; do
|
||||
attempt=$((attempt + 1))
|
||||
if [ "$attempt" -ge 30 ]; then
|
||||
echo 'S3 bootstrap timed out' >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
mc mb --ignore-existing local/attachments >/dev/null
|
||||
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
heartbeat = 180
|
||||
vm_memory_high_watermark.absolute = 300MiB
|
||||
disk_free_limit.absolute = 200MiB
|
||||
@@ -1 +1 @@
|
||||
{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::attachments"],"Condition":{"StringLike":{"s3:prefix":["originals/*"]}}},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::attachments/originals/*"]}]}
|
||||
{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::attachments"],"Condition":{"StringLike":{"s3:prefix":["staging/*"]}}},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::attachments/originals/*","arn:aws:s3:::attachments/variants/*"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::attachments/staging/*"]}]}
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
LAB5_PORT=8085
|
||||
API_REPLICAS=2
|
||||
SESSION_TTL_SECONDS=3600
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
@@ -0,0 +1,49 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_state_check;
|
||||
UPDATE attachments SET state = 'failed' WHERE state = 'missing';
|
||||
ALTER TABLE attachments ADD COLUMN error_code text;
|
||||
UPDATE attachments SET error_code = 'original_unavailable' WHERE state = 'failed';
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_pipeline_state CHECK (
|
||||
state IN ('queued', 'processing', 'ready', 'failed')
|
||||
AND ((state = 'failed') = (error_code IS NOT NULL))
|
||||
);
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id),
|
||||
staging_key text NOT NULL UNIQUE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
|
||||
attempt_limit integer NOT NULL CHECK (attempt_limit >= 1),
|
||||
state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued', 'processing', 'ready', 'failed')),
|
||||
next_publish_at timestamptz,
|
||||
published_at timestamptz,
|
||||
publish_failures integer NOT NULL DEFAULT 0,
|
||||
lease_until timestamptz,
|
||||
lease_token uuid,
|
||||
last_error text,
|
||||
CHECK ((lease_until IS NULL) = (lease_token IS NULL))
|
||||
);
|
||||
CREATE INDEX attachment_jobs_outbox ON attachment_jobs(next_publish_at)
|
||||
WHERE next_publish_at IS NOT NULL;
|
||||
CREATE INDEX attachment_jobs_leases ON attachment_jobs(lease_until)
|
||||
WHERE lease_until IS NOT NULL;
|
||||
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
content_type text NOT NULL,
|
||||
size_bytes integer NOT NULL CHECK (size_bytes > 0),
|
||||
sha256 char(64) NOT NULL,
|
||||
width integer NOT NULL CHECK (width > 0),
|
||||
height integer NOT NULL CHECK (height > 0),
|
||||
PRIMARY KEY (attachment_id, name)
|
||||
);
|
||||
|
||||
CREATE TABLE attachment_replays (
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(id),
|
||||
attempt integer NOT NULL,
|
||||
reason text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (job_id, attempt)
|
||||
);
|
||||
@@ -26,6 +26,7 @@ redis==5.2.0
|
||||
|
||||
boto3==1.40.61
|
||||
Pillow==12.0.0
|
||||
pika==1.3.2
|
||||
botocore==1.40.76
|
||||
s3transfer==0.14.0
|
||||
jmespath==1.1.0
|
||||
|
||||
@@ -8,3 +8,4 @@ redis==5.2.0
|
||||
|
||||
boto3==1.40.61
|
||||
Pillow==12.0.0
|
||||
pika==1.3.2
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Создаёт только отсутствующие локальные секреты и конфигурацию."""
|
||||
|
||||
from pathlib import Path
|
||||
from secrets import token_hex
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def create(path, value):
|
||||
try:
|
||||
with path.open('x', encoding='utf-8') as target:
|
||||
target.write(value)
|
||||
except FileExistsError:
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
directory = ROOT / 'secrets'
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
for name in ('db_password', 'broker_password', 's3_root_user', 's3_root_password',
|
||||
's3_access_key', 's3_secret_key'):
|
||||
create(directory / name, token_hex(24))
|
||||
db = quote((directory / 'db_password').read_text(encoding='utf-8').strip(), safe='')
|
||||
broker = quote((directory / 'broker_password').read_text(encoding='utf-8').strip(), safe='')
|
||||
create(directory / 'database_url', f'postgresql://messenger:{db}@db:5432/messenger')
|
||||
create(directory / 'broker_url', f'amqp://messenger:{broker}@broker:5672/%2F')
|
||||
create(ROOT / '.env', (ROOT / 'lab5.env.example').read_text(encoding='utf-8'))
|
||||
print('Локальные файлы готовы')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,36 @@
|
||||
"""RabbitMQ с подтверждаемой публикацией и ручным ack."""
|
||||
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pika
|
||||
|
||||
from src.config import setting
|
||||
|
||||
QUEUE = 'attachment.process.v1'
|
||||
|
||||
|
||||
@contextmanager
|
||||
def channel(heartbeat=90):
|
||||
params = pika.URLParameters(setting('BROKER_URL'))
|
||||
params.heartbeat = heartbeat
|
||||
params.socket_timeout = 4
|
||||
params.stack_timeout = 6
|
||||
params.blocked_connection_timeout = 5
|
||||
connection = pika.BlockingConnection(params)
|
||||
try:
|
||||
result = connection.channel()
|
||||
result.queue_declare(queue=QUEUE, durable=True)
|
||||
yield connection, result
|
||||
finally:
|
||||
if connection.is_open:
|
||||
connection.close()
|
||||
|
||||
|
||||
def publish(ch, message):
|
||||
ch.basic_publish(
|
||||
exchange='', routing_key=QUEUE, body=json.dumps(message, separators=(',', ':')).encode(),
|
||||
properties=pika.BasicProperties(content_type='application/json', delivery_mode=2,
|
||||
message_id=message['job_id']),
|
||||
mandatory=True,
|
||||
)
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Отдельный relay: lease публикации, publisher confirm, восстановление jobs."""
|
||||
|
||||
import logging
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pika
|
||||
import psycopg
|
||||
from psycopg_pool import PoolTimeout
|
||||
|
||||
from src import broker, jobs
|
||||
from src.storage import storage
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
stopping = threading.Event()
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
signal.signal(sig, lambda *_: stopping.set())
|
||||
storage.pool.open(wait=True)
|
||||
next_recovery = 0.0
|
||||
try:
|
||||
while not stopping.is_set():
|
||||
reserved = None
|
||||
try:
|
||||
with broker.channel(heartbeat=10) as (connection, channel):
|
||||
channel.confirm_delivery()
|
||||
while not stopping.is_set():
|
||||
if time.monotonic() >= next_recovery:
|
||||
jobs.recover_expired()
|
||||
next_recovery = time.monotonic() + 1
|
||||
reserved = jobs.reserve_publish()
|
||||
if reserved:
|
||||
broker.publish(channel, jobs.envelope(reserved))
|
||||
jobs.confirm_publish(reserved)
|
||||
reserved = None
|
||||
else:
|
||||
connection.process_data_events(time_limit=0.2)
|
||||
except (pika.exceptions.AMQPError, OSError, psycopg.Error, PoolTimeout) as exc:
|
||||
logging.warning('dispatch temporarily unavailable: %s', type(exc).__name__)
|
||||
if reserved:
|
||||
try:
|
||||
jobs.publish_failed(reserved)
|
||||
except (psycopg.Error, PoolTimeout):
|
||||
pass # expires through the durable publication deadline
|
||||
stopping.wait(1)
|
||||
finally:
|
||||
storage.pool.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Преобразования Pillow в отдельном процессе с жёстким таймаутом родителя."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
|
||||
|
||||
def fit(image, limit):
|
||||
width, height = image.size
|
||||
longest = max(width, height)
|
||||
if longest <= limit:
|
||||
return image
|
||||
size = (max(1, width * limit // longest), max(1, height * limit // longest))
|
||||
return image.resize(size, Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def render(source, target):
|
||||
with Image.open(source) as opened:
|
||||
if opened.width * opened.height > Image.MAX_IMAGE_PIXELS:
|
||||
raise ValueError('pixel limit')
|
||||
image = ImageOps.exif_transpose(opened)
|
||||
if 'A' in image.getbands() or 'transparency' in image.info:
|
||||
rgba = image.convert('RGBA')
|
||||
rgb = Image.new('RGB', rgba.size, 'white')
|
||||
rgb.paste(rgba, mask=rgba.getchannel('A'))
|
||||
else:
|
||||
rgb = image.convert('RGB')
|
||||
rgb.info.clear()
|
||||
del image
|
||||
thumbnail = fit(rgb, 256)
|
||||
thumbnail.save(target / 'thumbnail', 'JPEG', quality=85)
|
||||
return [['thumbnail', 'image/jpeg', *thumbnail.size]]
|
||||
|
||||
|
||||
def main():
|
||||
directory = Path(sys.argv[1])
|
||||
try:
|
||||
result = render(directory / 'original', directory)
|
||||
except (OSError, ValueError, SyntaxError, Image.DecompressionBombError):
|
||||
return 2
|
||||
(directory / 'manifest.json').write_text(json.dumps(result), encoding='utf-8')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
"""Устойчивое намерение обработки, lease и атомарная публикация manifest."""
|
||||
|
||||
import random
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from src.config import setting
|
||||
from src.storage import storage, timestamp
|
||||
|
||||
JOB_TIMEOUT = int(setting('JOB_TIMEOUT_SECONDS', '60'))
|
||||
LEASE_SECONDS = JOB_TIMEOUT + 30
|
||||
|
||||
|
||||
def envelope(job):
|
||||
return {
|
||||
'schema_version': 1,
|
||||
'job_id': str(job['id']),
|
||||
'type': 'attachment.process.v1',
|
||||
'attachment_id': str(job['attachment_id']),
|
||||
'pipeline_version': 1,
|
||||
'attempt': job['attempt'],
|
||||
'created_at': timestamp(job['created_at']),
|
||||
}
|
||||
|
||||
|
||||
def validate(message):
|
||||
if set(message) != {'schema_version', 'job_id', 'type', 'attachment_id',
|
||||
'pipeline_version', 'attempt', 'created_at'}:
|
||||
raise ValueError('invalid fields')
|
||||
if (type(message['schema_version']) is not int or message['schema_version'] != 1
|
||||
or message['type'] != 'attachment.process.v1'
|
||||
or type(message['pipeline_version']) is not int or message['pipeline_version'] != 1
|
||||
or type(message['attempt']) is not int or message['attempt'] < 1):
|
||||
raise ValueError('invalid job')
|
||||
UUID(message['job_id'])
|
||||
UUID(message['attachment_id'])
|
||||
return message
|
||||
|
||||
|
||||
def reserve_publish():
|
||||
with storage.pool.connection() as conn:
|
||||
return conn.execute(
|
||||
"""WITH candidate AS (
|
||||
SELECT id FROM attachment_jobs WHERE state='queued'
|
||||
AND next_publish_at <= clock_timestamp()
|
||||
ORDER BY next_publish_at, id LIMIT 1 FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE attachment_jobs j
|
||||
SET next_publish_at=clock_timestamp()+interval '15 seconds'
|
||||
FROM candidate WHERE j.id=candidate.id RETURNING j.*"""
|
||||
).fetchone()
|
||||
|
||||
|
||||
def confirm_publish(job):
|
||||
with storage.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"""UPDATE attachment_jobs SET next_publish_at=NULL, published_at=clock_timestamp(),
|
||||
publish_failures=0 WHERE id=%s AND attempt=%s""",
|
||||
(job['id'], job['attempt']),
|
||||
)
|
||||
|
||||
|
||||
def publish_failed(job):
|
||||
with storage.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"""UPDATE attachment_jobs SET publish_failures=publish_failures+1,
|
||||
next_publish_at=clock_timestamp()+%s * interval '1 second'
|
||||
WHERE id=%s AND attempt=%s AND state='queued' AND next_publish_at IS NOT NULL""",
|
||||
(delay(job['publish_failures'] + 1), job['id'], job['attempt']),
|
||||
)
|
||||
|
||||
|
||||
def locked_job(conn, job_id, attachment_id):
|
||||
attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',
|
||||
(attachment_id,)).fetchone()
|
||||
if not attachment:
|
||||
return None, None
|
||||
job = conn.execute('SELECT * FROM attachment_jobs WHERE id=%s AND attachment_id=%s FOR UPDATE',
|
||||
(job_id, attachment_id)).fetchone()
|
||||
return job, attachment
|
||||
|
||||
|
||||
def claim(message):
|
||||
with storage.pool.connection() as conn:
|
||||
job, attachment = locked_job(conn, message['job_id'], message['attachment_id'])
|
||||
if not job or envelope(job) != message or job['state'] != 'queued':
|
||||
return None
|
||||
token = uuid4()
|
||||
conn.execute(
|
||||
"""UPDATE attachment_jobs SET state='processing', lease_token=%s,
|
||||
lease_until=clock_timestamp()+%s * interval '1 second', next_publish_at=NULL
|
||||
WHERE id=%s""", (token, LEASE_SECONDS, job['id']),
|
||||
)
|
||||
conn.execute("UPDATE attachments SET state='processing',updated_at=clock_timestamp() WHERE id=%s",
|
||||
(job['attachment_id'],))
|
||||
return job | {'lease_token': token, 'attachment': attachment}
|
||||
|
||||
|
||||
def owns(conn, job, claimed):
|
||||
return (
|
||||
job and (job['state'], job['lease_token']) == ('processing', claimed['lease_token'])
|
||||
and conn.execute('SELECT clock_timestamp() < %s AS active', (job['lease_until'],)).fetchone()['active']
|
||||
)
|
||||
|
||||
|
||||
def complete(claimed, variants):
|
||||
with storage.pool.connection() as conn:
|
||||
job, _ = locked_job(conn, claimed['id'], claimed['attachment_id'])
|
||||
if not owns(conn, job, claimed):
|
||||
return False
|
||||
for variant in variants:
|
||||
conn.execute(
|
||||
"""INSERT INTO attachment_variants
|
||||
(attachment_id,name,object_key,content_type,size_bytes,sha256,width,height)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||||
(job['attachment_id'], *variant),
|
||||
)
|
||||
conn.execute("UPDATE attachments SET state='ready',updated_at=clock_timestamp() WHERE id=%s",
|
||||
(job['attachment_id'],))
|
||||
conn.execute("UPDATE attachment_jobs SET state='ready',lease_token=NULL,lease_until=NULL,"
|
||||
"last_error=NULL WHERE id=%s", (job['id'],))
|
||||
return True
|
||||
|
||||
|
||||
def delay(attempt):
|
||||
return min(30, 2 ** min(attempt - 1, 5)) * random.uniform(0.75, 1.25)
|
||||
|
||||
|
||||
def settle_error(conn, job, code, permanent=False):
|
||||
exhausted = job['attempt'] >= job['attempt_limit']
|
||||
if permanent or exhausted:
|
||||
error = code if permanent else 'retry_exhausted'
|
||||
conn.execute("UPDATE attachments SET state='failed',error_code=%s,updated_at=clock_timestamp() "
|
||||
"WHERE id=%s", (error, job['attachment_id']))
|
||||
conn.execute("UPDATE attachment_jobs SET state='failed',lease_token=NULL,lease_until=NULL,"
|
||||
"last_error=%s,next_publish_at=NULL WHERE id=%s", (code, job['id']))
|
||||
return
|
||||
conn.execute("UPDATE attachments SET state='queued',updated_at=clock_timestamp() WHERE id=%s",
|
||||
(job['attachment_id'],))
|
||||
conn.execute("""UPDATE attachment_jobs SET state='queued',lease_token=NULL,lease_until=NULL,
|
||||
last_error=%s,attempt=attempt+1,published_at=NULL,
|
||||
next_publish_at=clock_timestamp()+%s*interval '1 second' WHERE id=%s""",
|
||||
(code, delay(job['attempt']), job['id']))
|
||||
|
||||
|
||||
def fail(claimed, code, permanent=False):
|
||||
with storage.pool.connection() as conn:
|
||||
job, _ = locked_job(conn, claimed['id'], claimed['attachment_id'])
|
||||
if owns(conn, job, claimed):
|
||||
settle_error(conn, job, code, permanent)
|
||||
|
||||
|
||||
def recover_expired():
|
||||
with storage.pool.connection() as conn:
|
||||
expired = conn.execute("SELECT id,attachment_id FROM attachment_jobs "
|
||||
"WHERE lease_until <= clock_timestamp() ORDER BY lease_until LIMIT 16").fetchall()
|
||||
for candidate in expired:
|
||||
with storage.pool.connection() as conn:
|
||||
job, _ = locked_job(conn, candidate['id'], candidate['attachment_id'])
|
||||
if (job and job['state'] == 'processing'
|
||||
and not owns(conn, job, job)):
|
||||
settle_error(conn, job, 'lease_expired')
|
||||
@@ -1,6 +1,7 @@
|
||||
"""S3 operations with bounded timeouts and immutable server-generated keys."""
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from src.config import setting
|
||||
|
||||
def client():
|
||||
@@ -11,6 +12,24 @@ def client():
|
||||
|
||||
BUCKET = setting('S3_BUCKET','attachments')
|
||||
|
||||
|
||||
class ObjectConflict(Exception):
|
||||
"""Неизменяемый ключ уже содержит другие данные."""
|
||||
|
||||
|
||||
def put_immutable(s3, key, stream, size, content_type, digest):
|
||||
try:
|
||||
s3.put_object(Bucket=BUCKET, Key=key, Body=stream, ContentLength=size,
|
||||
ContentType=content_type, Metadata={'sha256': digest}, IfNoneMatch='*')
|
||||
except ClientError as exc:
|
||||
if exc.response['ResponseMetadata']['HTTPStatusCode'] != 412:
|
||||
raise
|
||||
existing = s3.head_object(Bucket=BUCKET, Key=key)
|
||||
if (existing['ContentLength'], existing['ContentType'], existing['Metadata'].get('sha256')) != (
|
||||
size, content_type, digest):
|
||||
raise ObjectConflict(key) from None
|
||||
|
||||
|
||||
def object_lock(conn, key, wait=True):
|
||||
fn = 'pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock'
|
||||
row = conn.execute('SELECT '+fn+'(hashtextextended(%s,42)) AS locked',(key,)).fetchone()
|
||||
|
||||
+37
-22
@@ -12,7 +12,8 @@ from starlette.responses import StreamingResponse
|
||||
from starlette.requests import ClientDisconnect
|
||||
from src.routes.common import require_user, make_error, get_request_id
|
||||
from src.storage import storage, timestamp
|
||||
from src.objects import client, BUCKET, object_lock
|
||||
from src.objects import client, BUCKET, object_lock, put_immutable
|
||||
from src.config import setting
|
||||
|
||||
router = APIRouter()
|
||||
slots = asyncio.Semaphore(1)
|
||||
@@ -47,26 +48,29 @@ def inspect_image(stream, declared):
|
||||
finally:
|
||||
stream.seek(0)
|
||||
|
||||
def metadata(row):
|
||||
def metadata(row, variants=()):
|
||||
return {k: str(row[k]) for k in ('id','chat_id','owner_id')} | {
|
||||
k:row[k] for k in ('filename','content_type','size_bytes','sha256')} | {
|
||||
'status':'ready','variants':[],'error_code':None,
|
||||
'status':row['state'],'variants':list(variants),'error_code':row['error_code'],
|
||||
'created_at':timestamp(row['created_at']),'updated_at':timestamp(row['updated_at'])}
|
||||
|
||||
def save(stream, chat_id, owner, filename, content_type, size, digest):
|
||||
attachment_id=uuid.uuid4()
|
||||
key='originals/'+str(attachment_id)
|
||||
# Keep this lock through S3 PUT and DB COMMIT/ROLLBACK. Reconciler shares it.
|
||||
key=f'originals/{attachment_id}'
|
||||
staging_key=f'staging/{attachment_id}'
|
||||
with storage.pool.connection() as conn:
|
||||
object_lock(conn,key)
|
||||
client().put_object(Bucket=BUCKET,Key=key,Body=stream,ContentLength=size,
|
||||
ContentType=content_type,Metadata={'sha256':digest},IfNoneMatch='*')
|
||||
row=conn.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'ready') RETURNING *",
|
||||
object_lock(conn, staging_key)
|
||||
put_immutable(client(), staging_key, stream, size, content_type, digest)
|
||||
row=conn.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *",
|
||||
(attachment_id,chat_id,owner,key,filename,content_type,size,digest)).fetchone()
|
||||
# Never blindly delete after ambiguous COMMIT: grace-period reconciliation handles it.
|
||||
conn.execute(
|
||||
"INSERT INTO attachment_jobs(id,attachment_id,staging_key,attempt_limit,next_publish_at) "
|
||||
"VALUES (%s,%s,%s,%s,clock_timestamp())",
|
||||
(uuid.uuid4(), attachment_id, staging_key, int(setting('JOB_MAX_ATTEMPTS', '5'))),
|
||||
)
|
||||
return metadata(row)
|
||||
|
||||
@router.post('/api/v1/chats/{chat_id}/attachments',status_code=201)
|
||||
@router.post('/api/v1/chats/{chat_id}/attachments',status_code=202)
|
||||
async def upload(request:Request,chat_id:uuid.UUID,filename:str='',actor:str=Depends(require_user)):
|
||||
if not await run_in_threadpool(storage.is_chat_member,str(chat_id),actor):
|
||||
raise error(request,404,'not_found','chat not found')
|
||||
@@ -110,26 +114,37 @@ async def upload(request:Request,chat_id:uuid.UUID,filename:str='',actor:str=Dep
|
||||
finally:
|
||||
slots.release()
|
||||
|
||||
def accessible(request,attachment_id,actor):
|
||||
with storage.pool.connection() as conn:
|
||||
row=conn.execute('SELECT a.* FROM attachments a JOIN chat_members m ON a.chat_id=m.chat_id WHERE a.id=%s AND m.user_id=%s',(attachment_id,actor)).fetchone()
|
||||
def accessible(conn, request, attachment_id, actor):
|
||||
# One consistent manifest for all reads in the enclosing transaction.
|
||||
row=conn.execute('SELECT a.* FROM attachments a JOIN chat_members m ON a.chat_id=m.chat_id '
|
||||
'WHERE a.id=%s AND m.user_id=%s FOR SHARE OF a',(attachment_id,actor)).fetchone()
|
||||
if not row:
|
||||
raise error(request,404,'not_found','attachment not found')
|
||||
if row['state']!='ready':
|
||||
raise error(request,503,'unavailable','original temporarily unavailable')
|
||||
return row
|
||||
|
||||
@router.get('/api/v1/attachments/{attachment_id}')
|
||||
def get_attachment(request:Request,attachment_id:uuid.UUID,actor:str=Depends(require_user)):
|
||||
return metadata(accessible(request,attachment_id,actor))
|
||||
with storage.pool.connection() as conn:
|
||||
row = accessible(conn, request, attachment_id, actor)
|
||||
variants = conn.execute(
|
||||
'SELECT name,content_type,size_bytes,sha256,width,height FROM attachment_variants '
|
||||
'WHERE attachment_id=%s ORDER BY name', (attachment_id,)
|
||||
).fetchall()
|
||||
return metadata(row, variants)
|
||||
|
||||
@router.get('/api/v1/attachments/{attachment_id}/content')
|
||||
def content(request:Request,attachment_id:uuid.UUID,variant:str='original',actor:str=Depends(require_user)):
|
||||
row=accessible(request,attachment_id,actor)
|
||||
if variant not in ('original','thumbnail','crop','optimized'):
|
||||
raise error(request,400,'invalid_request','invalid variant')
|
||||
if variant!='original':
|
||||
raise error(request,404,'not_found','variant not found')
|
||||
with storage.pool.connection() as conn:
|
||||
row = accessible(conn, request, attachment_id, actor)
|
||||
if variant not in ('original','thumbnail','crop','optimized'):
|
||||
raise error(request,400,'invalid_request','invalid variant')
|
||||
if row['state'] != 'ready':
|
||||
raise error(request,409,'conflict','attachment not ready')
|
||||
if variant != 'original':
|
||||
row = conn.execute('SELECT * FROM attachment_variants WHERE attachment_id=%s AND name=%s',
|
||||
(attachment_id, variant)).fetchone()
|
||||
if not row:
|
||||
raise error(request,404,'not_found','variant not found')
|
||||
obj=client().get_object(Bucket=BUCKET,Key=row['object_key'])
|
||||
def chunks():
|
||||
try:
|
||||
|
||||
@@ -14,7 +14,7 @@ def liveness():
|
||||
def readiness():
|
||||
storage.ready()
|
||||
from src.objects import client, BUCKET
|
||||
client().list_objects_v2(Bucket=BUCKET,Prefix="originals/",MaxKeys=1)
|
||||
client().list_objects_v2(Bucket=BUCKET, Prefix="staging/", MaxKeys=1)
|
||||
return Health()
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -216,7 +216,7 @@ class PostgresStorage:
|
||||
attachment = conn.execute('SELECT state FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE', (attachment_id,chat_id)).fetchone()
|
||||
if not attachment:
|
||||
raise NotFoundError('вложение не найдено')
|
||||
if attachment['state'] != 'ready':
|
||||
if attachment['state'] == 'failed':
|
||||
raise ConflictError('вложение недоступно')
|
||||
optimized = os.getenv('OPTIMIZED_WRITES','1') == '1'
|
||||
if not optimized:
|
||||
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
"""Worker обрабатывает одну доставку за раз и подтверждает после COMMIT."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
import pika
|
||||
import psycopg
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from psycopg_pool import PoolTimeout
|
||||
|
||||
from src import broker, jobs
|
||||
from src.objects import BUCKET, ObjectConflict, client, put_immutable
|
||||
from src.storage import storage
|
||||
|
||||
|
||||
class PermanentError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def fetch(s3, key, attachment, destination):
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
obj = s3.get_object(Bucket=BUCKET, Key=key)
|
||||
with closing(obj['Body']) as stream, destination.open('wb') as output:
|
||||
for chunk in stream.iter_chunks(chunk_size=64 * 1024):
|
||||
size += len(chunk)
|
||||
if size > attachment['size_bytes']:
|
||||
raise PermanentError('staging_integrity')
|
||||
digest.update(chunk)
|
||||
output.write(chunk)
|
||||
if (size, digest.hexdigest()) != (attachment['size_bytes'], attachment['sha256']):
|
||||
raise PermanentError('staging_integrity')
|
||||
|
||||
|
||||
def process(job):
|
||||
attachment = job['attachment']
|
||||
s3 = client()
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
directory = Path(temp)
|
||||
original = directory / 'original'
|
||||
fetch(s3, job['staging_key'], attachment, original)
|
||||
result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', temp],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
timeout=jobs.JOB_TIMEOUT, check=False)
|
||||
if result.returncode == 2:
|
||||
raise PermanentError('invalid_image')
|
||||
result.check_returncode()
|
||||
with original.open('rb') as stream:
|
||||
put_immutable(s3, attachment['object_key'], stream, attachment['size_bytes'],
|
||||
attachment['content_type'], attachment['sha256'])
|
||||
variants = []
|
||||
for name, mime, width, height in json.loads((directory / 'manifest.json').read_text()):
|
||||
key = f"variants/{attachment['id']}/v1/{name}"
|
||||
output = directory / name
|
||||
size = output.stat().st_size
|
||||
with output.open('rb') as stream:
|
||||
digest = hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
stream.seek(0)
|
||||
put_immutable(s3, key, stream, size, mime, digest)
|
||||
variants.append((name, key, mime, size, digest, width, height))
|
||||
jobs.complete(job, variants)
|
||||
|
||||
|
||||
def handle(channel, delivery, body):
|
||||
try:
|
||||
message = jobs.validate(json.loads(body))
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
logging.warning('invalid broker job')
|
||||
channel.basic_reject(delivery.delivery_tag, requeue=False)
|
||||
return
|
||||
|
||||
job = jobs.claim(message)
|
||||
if job:
|
||||
try:
|
||||
process(job)
|
||||
except PermanentError as exc:
|
||||
jobs.fail(job, str(exc), permanent=True)
|
||||
except ObjectConflict:
|
||||
jobs.fail(job, 'object_conflict', permanent=True)
|
||||
except (BotoCoreError, ClientError, OSError, subprocess.SubprocessError) as exc:
|
||||
logging.warning('processing retry job=%s cause=%s', job['id'], type(exc).__name__)
|
||||
jobs.fail(job, 'processing_timeout' if isinstance(exc, subprocess.TimeoutExpired) else 'processing_failed')
|
||||
channel.basic_ack(delivery.delivery_tag)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
stopping = threading.Event()
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
signal.signal(sig, lambda *_: stopping.set())
|
||||
storage.pool.open(wait=True)
|
||||
try:
|
||||
while not stopping.is_set():
|
||||
try:
|
||||
with broker.channel(heartbeat=jobs.LEASE_SECONDS + 15) as (_, channel):
|
||||
channel.basic_qos(prefetch_count=1)
|
||||
for delivery, _, body in channel.consume(broker.QUEUE, inactivity_timeout=0.5):
|
||||
if stopping.is_set():
|
||||
break
|
||||
if delivery:
|
||||
handle(channel, delivery, body)
|
||||
except (pika.exceptions.AMQPError, OSError, psycopg.Error, PoolTimeout) as exc:
|
||||
logging.warning('worker temporarily unavailable: %s', type(exc).__name__)
|
||||
stopping.wait(1)
|
||||
finally:
|
||||
storage.pool.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user