Реализовать устойчивый приём вложений, брокер и обработку миниатюр

This commit is contained in:
2026-11-20 06:15:06 +03:00
parent 5894da9fd1
commit a0ec26d96d
19 changed files with 738 additions and 26 deletions
+153
View File
@@ -0,0 +1,153 @@
name: lyakhonov-lab5
x-app: &app
image: lyakhonov-messenger:lab5
build: .
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: http://s3:9000
S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key
S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
CACHE_URL: redis://redis:6379/0
BROKER_URL_FILE: /run/secrets/broker_url
secrets: [database_url, s3_access_key, s3_secret_key, broker_url]
cpus: 0.5
mem_limit: 640m
stop_grace_period: 12s
x-dependencies: &dependencies
migrate:
condition: service_completed_successfully
bucket-init:
condition: service_completed_successfully
services:
db:
image: postgres:16.6-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
secrets: [db_password]
volumes: [postgres_data:/var/lib/postgresql/data]
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 2s
retries: 30
cpus: 1.0
mem_limit: 512m
redis:
image: redis:7.4.1-alpine
command: [redis-server, --maxmemory, 64mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"]
cpus: 0.25
mem_limit: 128m
s3:
image: lyakhonov-storage:lab5
build:
context: .
dockerfile: Dockerfile.s3
command: [server, /data, --console-address, '127.0.0.1:9001']
environment:
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
secrets: [s3_root_user, s3_root_password]
volumes: [s3_data:/data]
cpus: 1.0
mem_limit: 768m
bucket-init:
image: lyakhonov-storage:lab5
pull_policy: never
entrypoint: [/bin/sh, /bootstrap/bucket.sh]
secrets: [s3_root_user, s3_root_password, s3_access_key, s3_secret_key]
volumes: [./infra/bucket.sh:/bootstrap/bucket.sh:ro, ./infra/s3-policy.json:/bootstrap/policy.json:ro]
depends_on: [s3]
cpus: 0.25
mem_limit: 128m
broker:
image: rabbitmq:4.1.0-alpine
hostname: broker
entrypoint: [/bin/sh, /bootstrap/broker.sh]
volumes: [broker_data:/var/lib/rabbitmq, ./infra/broker.sh:/bootstrap/broker.sh:ro,
./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-lab5.conf:ro]
secrets: [broker_password]
healthcheck:
test: [CMD, rabbitmq-diagnostics, -q, ping]
interval: 5s
timeout: 4s
retries: 20
cpus: 0.75
mem_limit: 512m
migrate:
<<: *app
command: [python, -m, src.migrate]
depends_on:
db:
condition: service_healthy
healthcheck:
disable: true
api:
<<: *app
depends_on: *dependencies
deploy:
replicas: ${API_REPLICAS:-2}
dispatcher:
<<: *app
command: [python, -m, src.dispatch]
restart: unless-stopped
depends_on:
<<: *dependencies
broker:
condition: service_healthy
healthcheck:
disable: true
mem_limit: 256m
worker:
<<: *app
command: [python, -m, src.worker]
restart: unless-stopped
depends_on:
<<: *dependencies
broker:
condition: service_healthy
healthcheck:
disable: true
deploy:
replicas: ${WORKER_CONCURRENCY:-2}
cpus: 0.75
mem_limit: 768m
stop_grace_period: 95s
lb:
image: haproxy:3.0.6-alpine
ports: ['127.0.0.1:${LAB5_PORT:-8085}:8080']
volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
depends_on: [api]
cpus: 0.5
mem_limit: 128m
stop_grace_period: 12s
secrets:
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_access_key:
file: ./secrets/s3_access_key
s3_secret_key:
file: ./secrets/s3_secret_key
db_password:
file: ./secrets/db_password
database_url:
file: ./secrets/database_url
broker_password:
file: ./secrets/broker_password
broker_url:
file: ./secrets/broker_url
volumes:
postgres_data:
s3_data:
broker_data:
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
export RABBITMQ_DEFAULT_USER=messenger
export RABBITMQ_DEFAULT_PASS="$(cat /run/secrets/broker_password)"
exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server
+9 -1
View File
@@ -4,7 +4,15 @@ set -eu
# Не включать set -x: аргументы содержат учебные credentials.
until mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null 2>&1; do sleep 1; done
attempt=0
until mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null 2>&1; do
attempt=$((attempt + 1))
if [ "$attempt" -ge 30 ]; then
echo 'S3 bootstrap timed out' >&2
exit 1
fi
sleep 1
done
mc mb --ignore-existing local/attachments >/dev/null
+3
View File
@@ -0,0 +1,3 @@
heartbeat = 180
vm_memory_high_watermark.absolute = 300MiB
disk_free_limit.absolute = 200MiB
+1 -1
View File
@@ -1 +1 @@
{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::attachments"],"Condition":{"StringLike":{"s3:prefix":["originals/*"]}}},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::attachments/originals/*"]}]}
{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::attachments"],"Condition":{"StringLike":{"s3:prefix":["staging/*"]}}},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::attachments/originals/*","arn:aws:s3:::attachments/variants/*"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::attachments/staging/*"]}]}
+7
View File
@@ -0,0 +1,7 @@
LAB5_PORT=8085
API_REPLICAS=2
SESSION_TTL_SECONDS=3600
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
+49
View File
@@ -0,0 +1,49 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_state_check;
UPDATE attachments SET state = 'failed' WHERE state = 'missing';
ALTER TABLE attachments ADD COLUMN error_code text;
UPDATE attachments SET error_code = 'original_unavailable' WHERE state = 'failed';
ALTER TABLE attachments ADD CONSTRAINT attachments_pipeline_state CHECK (
state IN ('queued', 'processing', 'ready', 'failed')
AND ((state = 'failed') = (error_code IS NOT NULL))
);
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id),
staging_key text NOT NULL UNIQUE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
attempt_limit integer NOT NULL CHECK (attempt_limit >= 1),
state text NOT NULL DEFAULT 'queued' CHECK (state IN ('queued', 'processing', 'ready', 'failed')),
next_publish_at timestamptz,
published_at timestamptz,
publish_failures integer NOT NULL DEFAULT 0,
lease_until timestamptz,
lease_token uuid,
last_error text,
CHECK ((lease_until IS NULL) = (lease_token IS NULL))
);
CREATE INDEX attachment_jobs_outbox ON attachment_jobs(next_publish_at)
WHERE next_publish_at IS NOT NULL;
CREATE INDEX attachment_jobs_leases ON attachment_jobs(lease_until)
WHERE lease_until IS NOT NULL;
CREATE TABLE attachment_variants (
attachment_id uuid NOT NULL REFERENCES attachments(id),
name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')),
object_key text NOT NULL UNIQUE,
content_type text NOT NULL,
size_bytes integer NOT NULL CHECK (size_bytes > 0),
sha256 char(64) NOT NULL,
width integer NOT NULL CHECK (width > 0),
height integer NOT NULL CHECK (height > 0),
PRIMARY KEY (attachment_id, name)
);
CREATE TABLE attachment_replays (
job_id uuid NOT NULL REFERENCES attachment_jobs(id),
attempt integer NOT NULL,
reason text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (job_id, attempt)
);
+1
View File
@@ -26,6 +26,7 @@ redis==5.2.0
boto3==1.40.61
Pillow==12.0.0
pika==1.3.2
botocore==1.40.76
s3transfer==0.14.0
jmespath==1.1.0
+1
View File
@@ -8,3 +8,4 @@ redis==5.2.0
boto3==1.40.61
Pillow==12.0.0
pika==1.3.2
+33
View File
@@ -0,0 +1,33 @@
"""Создаёт только отсутствующие локальные секреты и конфигурацию."""
from pathlib import Path
from secrets import token_hex
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
def create(path, value):
try:
with path.open('x', encoding='utf-8') as target:
target.write(value)
except FileExistsError:
pass
def main():
directory = ROOT / 'secrets'
directory.mkdir(mode=0o700, exist_ok=True)
for name in ('db_password', 'broker_password', 's3_root_user', 's3_root_password',
's3_access_key', 's3_secret_key'):
create(directory / name, token_hex(24))
db = quote((directory / 'db_password').read_text(encoding='utf-8').strip(), safe='')
broker = quote((directory / 'broker_password').read_text(encoding='utf-8').strip(), safe='')
create(directory / 'database_url', f'postgresql://messenger:{db}@db:5432/messenger')
create(directory / 'broker_url', f'amqp://messenger:{broker}@broker:5672/%2F')
create(ROOT / '.env', (ROOT / 'lab5.env.example').read_text(encoding='utf-8'))
print('Локальные файлы готовы')
if __name__ == '__main__':
main()
+36
View File
@@ -0,0 +1,36 @@
"""RabbitMQ с подтверждаемой публикацией и ручным ack."""
import json
from contextlib import contextmanager
import pika
from src.config import setting
QUEUE = 'attachment.process.v1'
@contextmanager
def channel(heartbeat=90):
params = pika.URLParameters(setting('BROKER_URL'))
params.heartbeat = heartbeat
params.socket_timeout = 4
params.stack_timeout = 6
params.blocked_connection_timeout = 5
connection = pika.BlockingConnection(params)
try:
result = connection.channel()
result.queue_declare(queue=QUEUE, durable=True)
yield connection, result
finally:
if connection.is_open:
connection.close()
def publish(ch, message):
ch.basic_publish(
exchange='', routing_key=QUEUE, body=json.dumps(message, separators=(',', ':')).encode(),
properties=pika.BasicProperties(content_type='application/json', delivery_mode=2,
message_id=message['job_id']),
mandatory=True,
)
+53
View File
@@ -0,0 +1,53 @@
"""Отдельный relay: lease публикации, publisher confirm, восстановление jobs."""
import logging
import signal
import threading
import time
import pika
import psycopg
from psycopg_pool import PoolTimeout
from src import broker, jobs
from src.storage import storage
def main():
logging.basicConfig(level=logging.WARNING)
stopping = threading.Event()
for sig in (signal.SIGINT, signal.SIGTERM):
signal.signal(sig, lambda *_: stopping.set())
storage.pool.open(wait=True)
next_recovery = 0.0
try:
while not stopping.is_set():
reserved = None
try:
with broker.channel(heartbeat=10) as (connection, channel):
channel.confirm_delivery()
while not stopping.is_set():
if time.monotonic() >= next_recovery:
jobs.recover_expired()
next_recovery = time.monotonic() + 1
reserved = jobs.reserve_publish()
if reserved:
broker.publish(channel, jobs.envelope(reserved))
jobs.confirm_publish(reserved)
reserved = None
else:
connection.process_data_events(time_limit=0.2)
except (pika.exceptions.AMQPError, OSError, psycopg.Error, PoolTimeout) as exc:
logging.warning('dispatch temporarily unavailable: %s', type(exc).__name__)
if reserved:
try:
jobs.publish_failed(reserved)
except (psycopg.Error, PoolTimeout):
pass # expires through the durable publication deadline
stopping.wait(1)
finally:
storage.pool.close()
if __name__ == '__main__':
main()
+50
View File
@@ -0,0 +1,50 @@
"""Преобразования Pillow в отдельном процессе с жёстким таймаутом родителя."""
import json
import sys
from pathlib import Path
from PIL import Image, ImageOps
Image.MAX_IMAGE_PIXELS = 40_000_000
def fit(image, limit):
width, height = image.size
longest = max(width, height)
if longest <= limit:
return image
size = (max(1, width * limit // longest), max(1, height * limit // longest))
return image.resize(size, Image.Resampling.LANCZOS)
def render(source, target):
with Image.open(source) as opened:
if opened.width * opened.height > Image.MAX_IMAGE_PIXELS:
raise ValueError('pixel limit')
image = ImageOps.exif_transpose(opened)
if 'A' in image.getbands() or 'transparency' in image.info:
rgba = image.convert('RGBA')
rgb = Image.new('RGB', rgba.size, 'white')
rgb.paste(rgba, mask=rgba.getchannel('A'))
else:
rgb = image.convert('RGB')
rgb.info.clear()
del image
thumbnail = fit(rgb, 256)
thumbnail.save(target / 'thumbnail', 'JPEG', quality=85)
return [['thumbnail', 'image/jpeg', *thumbnail.size]]
def main():
directory = Path(sys.argv[1])
try:
result = render(directory / 'original', directory)
except (OSError, ValueError, SyntaxError, Image.DecompressionBombError):
return 2
(directory / 'manifest.json').write_text(json.dumps(result), encoding='utf-8')
return 0
if __name__ == '__main__':
sys.exit(main())
+161
View File
@@ -0,0 +1,161 @@
"""Устойчивое намерение обработки, lease и атомарная публикация manifest."""
import random
from uuid import UUID, uuid4
from src.config import setting
from src.storage import storage, timestamp
JOB_TIMEOUT = int(setting('JOB_TIMEOUT_SECONDS', '60'))
LEASE_SECONDS = JOB_TIMEOUT + 30
def envelope(job):
return {
'schema_version': 1,
'job_id': str(job['id']),
'type': 'attachment.process.v1',
'attachment_id': str(job['attachment_id']),
'pipeline_version': 1,
'attempt': job['attempt'],
'created_at': timestamp(job['created_at']),
}
def validate(message):
if set(message) != {'schema_version', 'job_id', 'type', 'attachment_id',
'pipeline_version', 'attempt', 'created_at'}:
raise ValueError('invalid fields')
if (type(message['schema_version']) is not int or message['schema_version'] != 1
or message['type'] != 'attachment.process.v1'
or type(message['pipeline_version']) is not int or message['pipeline_version'] != 1
or type(message['attempt']) is not int or message['attempt'] < 1):
raise ValueError('invalid job')
UUID(message['job_id'])
UUID(message['attachment_id'])
return message
def reserve_publish():
with storage.pool.connection() as conn:
return conn.execute(
"""WITH candidate AS (
SELECT id FROM attachment_jobs WHERE state='queued'
AND next_publish_at <= clock_timestamp()
ORDER BY next_publish_at, id LIMIT 1 FOR UPDATE SKIP LOCKED
)
UPDATE attachment_jobs j
SET next_publish_at=clock_timestamp()+interval '15 seconds'
FROM candidate WHERE j.id=candidate.id RETURNING j.*"""
).fetchone()
def confirm_publish(job):
with storage.pool.connection() as conn:
conn.execute(
"""UPDATE attachment_jobs SET next_publish_at=NULL, published_at=clock_timestamp(),
publish_failures=0 WHERE id=%s AND attempt=%s""",
(job['id'], job['attempt']),
)
def publish_failed(job):
with storage.pool.connection() as conn:
conn.execute(
"""UPDATE attachment_jobs SET publish_failures=publish_failures+1,
next_publish_at=clock_timestamp()+%s * interval '1 second'
WHERE id=%s AND attempt=%s AND state='queued' AND next_publish_at IS NOT NULL""",
(delay(job['publish_failures'] + 1), job['id'], job['attempt']),
)
def locked_job(conn, job_id, attachment_id):
attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',
(attachment_id,)).fetchone()
if not attachment:
return None, None
job = conn.execute('SELECT * FROM attachment_jobs WHERE id=%s AND attachment_id=%s FOR UPDATE',
(job_id, attachment_id)).fetchone()
return job, attachment
def claim(message):
with storage.pool.connection() as conn:
job, attachment = locked_job(conn, message['job_id'], message['attachment_id'])
if not job or envelope(job) != message or job['state'] != 'queued':
return None
token = uuid4()
conn.execute(
"""UPDATE attachment_jobs SET state='processing', lease_token=%s,
lease_until=clock_timestamp()+%s * interval '1 second', next_publish_at=NULL
WHERE id=%s""", (token, LEASE_SECONDS, job['id']),
)
conn.execute("UPDATE attachments SET state='processing',updated_at=clock_timestamp() WHERE id=%s",
(job['attachment_id'],))
return job | {'lease_token': token, 'attachment': attachment}
def owns(conn, job, claimed):
return (
job and (job['state'], job['lease_token']) == ('processing', claimed['lease_token'])
and conn.execute('SELECT clock_timestamp() < %s AS active', (job['lease_until'],)).fetchone()['active']
)
def complete(claimed, variants):
with storage.pool.connection() as conn:
job, _ = locked_job(conn, claimed['id'], claimed['attachment_id'])
if not owns(conn, job, claimed):
return False
for variant in variants:
conn.execute(
"""INSERT INTO attachment_variants
(attachment_id,name,object_key,content_type,size_bytes,sha256,width,height)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""",
(job['attachment_id'], *variant),
)
conn.execute("UPDATE attachments SET state='ready',updated_at=clock_timestamp() WHERE id=%s",
(job['attachment_id'],))
conn.execute("UPDATE attachment_jobs SET state='ready',lease_token=NULL,lease_until=NULL,"
"last_error=NULL WHERE id=%s", (job['id'],))
return True
def delay(attempt):
return min(30, 2 ** min(attempt - 1, 5)) * random.uniform(0.75, 1.25)
def settle_error(conn, job, code, permanent=False):
exhausted = job['attempt'] >= job['attempt_limit']
if permanent or exhausted:
error = code if permanent else 'retry_exhausted'
conn.execute("UPDATE attachments SET state='failed',error_code=%s,updated_at=clock_timestamp() "
"WHERE id=%s", (error, job['attachment_id']))
conn.execute("UPDATE attachment_jobs SET state='failed',lease_token=NULL,lease_until=NULL,"
"last_error=%s,next_publish_at=NULL WHERE id=%s", (code, job['id']))
return
conn.execute("UPDATE attachments SET state='queued',updated_at=clock_timestamp() WHERE id=%s",
(job['attachment_id'],))
conn.execute("""UPDATE attachment_jobs SET state='queued',lease_token=NULL,lease_until=NULL,
last_error=%s,attempt=attempt+1,published_at=NULL,
next_publish_at=clock_timestamp()+%s*interval '1 second' WHERE id=%s""",
(code, delay(job['attempt']), job['id']))
def fail(claimed, code, permanent=False):
with storage.pool.connection() as conn:
job, _ = locked_job(conn, claimed['id'], claimed['attachment_id'])
if owns(conn, job, claimed):
settle_error(conn, job, code, permanent)
def recover_expired():
with storage.pool.connection() as conn:
expired = conn.execute("SELECT id,attachment_id FROM attachment_jobs "
"WHERE lease_until <= clock_timestamp() ORDER BY lease_until LIMIT 16").fetchall()
for candidate in expired:
with storage.pool.connection() as conn:
job, _ = locked_job(conn, candidate['id'], candidate['attachment_id'])
if (job and job['state'] == 'processing'
and not owns(conn, job, job)):
settle_error(conn, job, 'lease_expired')
+19
View File
@@ -1,6 +1,7 @@
"""S3 operations with bounded timeouts and immutable server-generated keys."""
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
from src.config import setting
def client():
@@ -11,6 +12,24 @@ def client():
BUCKET = setting('S3_BUCKET','attachments')
class ObjectConflict(Exception):
"""Неизменяемый ключ уже содержит другие данные."""
def put_immutable(s3, key, stream, size, content_type, digest):
try:
s3.put_object(Bucket=BUCKET, Key=key, Body=stream, ContentLength=size,
ContentType=content_type, Metadata={'sha256': digest}, IfNoneMatch='*')
except ClientError as exc:
if exc.response['ResponseMetadata']['HTTPStatusCode'] != 412:
raise
existing = s3.head_object(Bucket=BUCKET, Key=key)
if (existing['ContentLength'], existing['ContentType'], existing['Metadata'].get('sha256')) != (
size, content_type, digest):
raise ObjectConflict(key) from None
def object_lock(conn, key, wait=True):
fn = 'pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock'
row = conn.execute('SELECT '+fn+'(hashtextextended(%s,42)) AS locked',(key,)).fetchone()
+37 -22
View File
@@ -12,7 +12,8 @@ from starlette.responses import StreamingResponse
from starlette.requests import ClientDisconnect
from src.routes.common import require_user, make_error, get_request_id
from src.storage import storage, timestamp
from src.objects import client, BUCKET, object_lock
from src.objects import client, BUCKET, object_lock, put_immutable
from src.config import setting
router = APIRouter()
slots = asyncio.Semaphore(1)
@@ -47,26 +48,29 @@ def inspect_image(stream, declared):
finally:
stream.seek(0)
def metadata(row):
def metadata(row, variants=()):
return {k: str(row[k]) for k in ('id','chat_id','owner_id')} | {
k:row[k] for k in ('filename','content_type','size_bytes','sha256')} | {
'status':'ready','variants':[],'error_code':None,
'status':row['state'],'variants':list(variants),'error_code':row['error_code'],
'created_at':timestamp(row['created_at']),'updated_at':timestamp(row['updated_at'])}
def save(stream, chat_id, owner, filename, content_type, size, digest):
attachment_id=uuid.uuid4()
key='originals/'+str(attachment_id)
# Keep this lock through S3 PUT and DB COMMIT/ROLLBACK. Reconciler shares it.
key=f'originals/{attachment_id}'
staging_key=f'staging/{attachment_id}'
with storage.pool.connection() as conn:
object_lock(conn,key)
client().put_object(Bucket=BUCKET,Key=key,Body=stream,ContentLength=size,
ContentType=content_type,Metadata={'sha256':digest},IfNoneMatch='*')
row=conn.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'ready') RETURNING *",
object_lock(conn, staging_key)
put_immutable(client(), staging_key, stream, size, content_type, digest)
row=conn.execute("INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *",
(attachment_id,chat_id,owner,key,filename,content_type,size,digest)).fetchone()
# Never blindly delete after ambiguous COMMIT: grace-period reconciliation handles it.
conn.execute(
"INSERT INTO attachment_jobs(id,attachment_id,staging_key,attempt_limit,next_publish_at) "
"VALUES (%s,%s,%s,%s,clock_timestamp())",
(uuid.uuid4(), attachment_id, staging_key, int(setting('JOB_MAX_ATTEMPTS', '5'))),
)
return metadata(row)
@router.post('/api/v1/chats/{chat_id}/attachments',status_code=201)
@router.post('/api/v1/chats/{chat_id}/attachments',status_code=202)
async def upload(request:Request,chat_id:uuid.UUID,filename:str='',actor:str=Depends(require_user)):
if not await run_in_threadpool(storage.is_chat_member,str(chat_id),actor):
raise error(request,404,'not_found','chat not found')
@@ -110,26 +114,37 @@ async def upload(request:Request,chat_id:uuid.UUID,filename:str='',actor:str=Dep
finally:
slots.release()
def accessible(request,attachment_id,actor):
with storage.pool.connection() as conn:
row=conn.execute('SELECT a.* FROM attachments a JOIN chat_members m ON a.chat_id=m.chat_id WHERE a.id=%s AND m.user_id=%s',(attachment_id,actor)).fetchone()
def accessible(conn, request, attachment_id, actor):
# One consistent manifest for all reads in the enclosing transaction.
row=conn.execute('SELECT a.* FROM attachments a JOIN chat_members m ON a.chat_id=m.chat_id '
'WHERE a.id=%s AND m.user_id=%s FOR SHARE OF a',(attachment_id,actor)).fetchone()
if not row:
raise error(request,404,'not_found','attachment not found')
if row['state']!='ready':
raise error(request,503,'unavailable','original temporarily unavailable')
return row
@router.get('/api/v1/attachments/{attachment_id}')
def get_attachment(request:Request,attachment_id:uuid.UUID,actor:str=Depends(require_user)):
return metadata(accessible(request,attachment_id,actor))
with storage.pool.connection() as conn:
row = accessible(conn, request, attachment_id, actor)
variants = conn.execute(
'SELECT name,content_type,size_bytes,sha256,width,height FROM attachment_variants '
'WHERE attachment_id=%s ORDER BY name', (attachment_id,)
).fetchall()
return metadata(row, variants)
@router.get('/api/v1/attachments/{attachment_id}/content')
def content(request:Request,attachment_id:uuid.UUID,variant:str='original',actor:str=Depends(require_user)):
row=accessible(request,attachment_id,actor)
if variant not in ('original','thumbnail','crop','optimized'):
raise error(request,400,'invalid_request','invalid variant')
if variant!='original':
raise error(request,404,'not_found','variant not found')
with storage.pool.connection() as conn:
row = accessible(conn, request, attachment_id, actor)
if variant not in ('original','thumbnail','crop','optimized'):
raise error(request,400,'invalid_request','invalid variant')
if row['state'] != 'ready':
raise error(request,409,'conflict','attachment not ready')
if variant != 'original':
row = conn.execute('SELECT * FROM attachment_variants WHERE attachment_id=%s AND name=%s',
(attachment_id, variant)).fetchone()
if not row:
raise error(request,404,'not_found','variant not found')
obj=client().get_object(Bucket=BUCKET,Key=row['object_key'])
def chunks():
try:
+1 -1
View File
@@ -14,7 +14,7 @@ def liveness():
def readiness():
storage.ready()
from src.objects import client, BUCKET
client().list_objects_v2(Bucket=BUCKET,Prefix="originals/",MaxKeys=1)
client().list_objects_v2(Bucket=BUCKET, Prefix="staging/", MaxKeys=1)
return Health()
+1 -1
View File
@@ -216,7 +216,7 @@ class PostgresStorage:
attachment = conn.execute('SELECT state FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE', (attachment_id,chat_id)).fetchone()
if not attachment:
raise NotFoundError('вложение не найдено')
if attachment['state'] != 'ready':
if attachment['state'] == 'failed':
raise ConflictError('вложение недоступно')
optimized = os.getenv('OPTIMIZED_WRITES','1') == '1'
if not optimized:
+118
View File
@@ -0,0 +1,118 @@
"""Worker обрабатывает одну доставку за раз и подтверждает после COMMIT."""
import hashlib
import json
import logging
import signal
import subprocess
import sys
import tempfile
import threading
from contextlib import closing
from pathlib import Path
import pika
import psycopg
from botocore.exceptions import BotoCoreError, ClientError
from psycopg_pool import PoolTimeout
from src import broker, jobs
from src.objects import BUCKET, ObjectConflict, client, put_immutable
from src.storage import storage
class PermanentError(Exception):
pass
def fetch(s3, key, attachment, destination):
digest = hashlib.sha256()
size = 0
obj = s3.get_object(Bucket=BUCKET, Key=key)
with closing(obj['Body']) as stream, destination.open('wb') as output:
for chunk in stream.iter_chunks(chunk_size=64 * 1024):
size += len(chunk)
if size > attachment['size_bytes']:
raise PermanentError('staging_integrity')
digest.update(chunk)
output.write(chunk)
if (size, digest.hexdigest()) != (attachment['size_bytes'], attachment['sha256']):
raise PermanentError('staging_integrity')
def process(job):
attachment = job['attachment']
s3 = client()
with tempfile.TemporaryDirectory() as temp:
directory = Path(temp)
original = directory / 'original'
fetch(s3, job['staging_key'], attachment, original)
result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', temp],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
timeout=jobs.JOB_TIMEOUT, check=False)
if result.returncode == 2:
raise PermanentError('invalid_image')
result.check_returncode()
with original.open('rb') as stream:
put_immutable(s3, attachment['object_key'], stream, attachment['size_bytes'],
attachment['content_type'], attachment['sha256'])
variants = []
for name, mime, width, height in json.loads((directory / 'manifest.json').read_text()):
key = f"variants/{attachment['id']}/v1/{name}"
output = directory / name
size = output.stat().st_size
with output.open('rb') as stream:
digest = hashlib.file_digest(stream, 'sha256').hexdigest()
stream.seek(0)
put_immutable(s3, key, stream, size, mime, digest)
variants.append((name, key, mime, size, digest, width, height))
jobs.complete(job, variants)
def handle(channel, delivery, body):
try:
message = jobs.validate(json.loads(body))
except (ValueError, TypeError, AttributeError):
logging.warning('invalid broker job')
channel.basic_reject(delivery.delivery_tag, requeue=False)
return
job = jobs.claim(message)
if job:
try:
process(job)
except PermanentError as exc:
jobs.fail(job, str(exc), permanent=True)
except ObjectConflict:
jobs.fail(job, 'object_conflict', permanent=True)
except (BotoCoreError, ClientError, OSError, subprocess.SubprocessError) as exc:
logging.warning('processing retry job=%s cause=%s', job['id'], type(exc).__name__)
jobs.fail(job, 'processing_timeout' if isinstance(exc, subprocess.TimeoutExpired) else 'processing_failed')
channel.basic_ack(delivery.delivery_tag)
def main():
logging.basicConfig(level=logging.WARNING)
stopping = threading.Event()
for sig in (signal.SIGINT, signal.SIGTERM):
signal.signal(sig, lambda *_: stopping.set())
storage.pool.open(wait=True)
try:
while not stopping.is_set():
try:
with broker.channel(heartbeat=jobs.LEASE_SECONDS + 15) as (_, channel):
channel.basic_qos(prefetch_count=1)
for delivery, _, body in channel.consume(broker.QUEUE, inactivity_timeout=0.5):
if stopping.is_set():
break
if delivery:
handle(channel, delivery, body)
except (pika.exceptions.AMQPError, OSError, psycopg.Error, PoolTimeout) as exc:
logging.warning('worker temporarily unavailable: %s', type(exc).__name__)
stopping.wait(1)
finally:
storage.pool.close()
if __name__ == '__main__':
main()