ЛР 6 — БПИ2302 — Нурлыгаянова — на 3

This commit is contained in:
Nurlygayanova Ekaterina
2026-11-13 11:32:45 +03:00
parent 470d9d6f1e
commit 69a190be0e
164 changed files with 25947 additions and 29 deletions
+7
View File
@@ -0,0 +1,7 @@
FROM python:3.13.9-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY main.py telemetry.py ./
USER nobody
CMD ["python", "-u", "main.py"]
+257
View File
@@ -0,0 +1,257 @@
"""Durable DB intent -> Valkey Streams -> idempotent S3 effects -> DB -> ack."""
import hashlib
import io
import json
import math
import os
from pathlib import Path
import signal
import sys
import time
import uuid
from multiprocessing import Process
from datetime import datetime, timezone
import boto3
from botocore.config import Config
from PIL import Image, ImageOps, UnidentifiedImageError
import psycopg
from psycopg.rows import dict_row
import redis
import telemetry
STREAM = 'attachment-jobs-v1'
GROUP = 'image-workers-v1'
MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5'))
TIMEOUT = int(os.getenv('JOB_TIMEOUT_SECONDS', '60'))
Image.MAX_IMAGE_PIXELS = int(os.getenv('MAX_IMAGE_PIXELS', '40000000'))
def secret(name):
return Path(os.environ[name + '_FILE']).read_text().strip()
def database():
return telemetry.Database(psycopg.connect(secret('DATABASE_URL'), autocommit=True, row_factory=dict_row,
connect_timeout=5, options='-c statement_timeout=5000'))
def broker():
return redis.Redis.from_url(secret('BROKER_URL'), decode_responses=True,
socket_connect_timeout=5, socket_timeout=5)
def event(row):
return dict(schema_version=1, job_id=str(row['job_id']), type='attachment.process.v1',
attachment_id=str(row['attachment_id']), pipeline_version=1,
attempt=row['attempt'] + 1,
created_at=row['created_at'].isoformat().replace('+00:00', 'Z'))
def dispatch():
"""Publish before marking: crash can duplicate, but cannot lose committed intent."""
telemetry.init("messenger-dispatcher")
while True:
try:
with database() as db:
r = broker()
measured = 0
while True:
if time.monotonic()-measured >= 1:
q = db.execute("SELECT count(*) AS depth, extract(epoch from min(j.created_at)) AS oldest FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.status IN ('queued','processing')").fetchone()
telemetry.queue.update(depth=q['depth'],oldest=float(q['oldest']) if q['oldest'] else None,up=1)
measured = time.monotonic()
with db.transaction():
rows = db.execute('''SELECT j.* FROM attachment_jobs j
JOIN attachments a ON a.id=j.attachment_id
WHERE j.published_at IS NULL AND a.status='queued'
ORDER BY j.created_at LIMIT 100 FOR UPDATE OF j SKIP LOCKED''').fetchall()
for row in rows:
with telemetry.publishing(row) as carrier:
r.xadd(STREAM, {'job': json.dumps(event(row)), **carrier})
db.execute('UPDATE attachment_jobs SET published_at=now() WHERE job_id=%s',
(row['job_id'],))
time.sleep(0.25)
except Exception as exc:
telemetry.queue['up'] = 0
print('dispatcher reconnect:', type(exc).__name__, flush=True)
time.sleep(1)
def thumbnail(data):
with Image.open(io.BytesIO(data)) as source:
if source.width * source.height > Image.MAX_IMAGE_PIXELS:
raise ValueError('pixel limit')
source.load()
oriented = ImageOps.exif_transpose(source)
rgba = oriented.convert('RGBA')
white = Image.new('RGBA', rgba.size, 'white')
white.alpha_composite(rgba)
rgb = white.convert('RGB')
scale = min(1, 256 / rgb.width, 256 / rgb.height)
size = (max(1, math.floor(rgb.width * scale)), max(1, math.floor(rgb.height * scale)))
rgb = rgb.resize(size, Image.Resampling.LANCZOS)
out = io.BytesIO()
rgb.save(out, format='JPEG', quality=85, exif=b'')
return out.getvalue(), size
def alarm_handler(*_):
raise TimeoutError('job deadline')
def handle(db, s3, r, message_id, fields):
payload = json.loads(fields['job'])
job_id = str(uuid.UUID(payload['job_id']))
attachment_id = str(uuid.UUID(payload['attachment_id']))
# Session advisory lock survives small transaction boundaries, releases on process death.
locked = db.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok',
(attachment_id,)).fetchone()['ok']
if not locked:
return
try:
row = db.execute('''SELECT a.*, j.job_id, j.attempt, j.staging_key, j.created_at AS job_created_at
FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id
WHERE a.id=%s AND j.job_id=%s AND j.pipeline_version=%s''',
(attachment_id, job_id, payload['pipeline_version'])).fetchone()
if row is None:
raise ValueError('unknown job')
if row['status'] in ('ready', 'failed'):
r.xack(STREAM, GROUP, message_id)
return
if row['attempt'] >= MAX_ATTEMPTS:
db.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s", (attachment_id,))
r.xack(STREAM, GROUP, message_id)
return
with db.transaction():
db.execute('UPDATE attachment_jobs SET attempt=attempt+1 WHERE job_id=%s', (job_id,))
db.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (attachment_id,))
telemetry.event('job.processing')
outcome = 'success'
signal.alarm(TIMEOUT)
try:
bucket = os.environ['S3_BUCKET']
response = telemetry.call('s3.get_object', s3.get_object, Bucket=bucket, Key=row['staging_key'])
try:
data = response['Body'].read(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')) + 1)
finally:
response['Body'].close()
if len(data) != row['size_bytes'] or hashlib.sha256(data).hexdigest() != row['sha256']:
raise ValueError('original checksum mismatch')
thumb, (width, height) = thumbnail(data)
original_key = 'originals/' + attachment_id
thumb_key = 'variants/' + attachment_id + '/v1/thumbnail'
telemetry.call('s3.put_original', s3.put_object, Bucket=bucket, Key=original_key, Body=data, ContentType=row['content_type'])
telemetry.call('s3.put_thumbnail', s3.put_object, Bucket=bucket, Key=thumb_key, Body=thumb, ContentType='image/jpeg')
variants = [dict(name='thumbnail', content_type='image/jpeg', size_bytes=len(thumb),
width=width, height=height, sha256=hashlib.sha256(thumb).hexdigest())]
db.execute("""UPDATE attachments SET status='ready',object_key=%s,variants=%s::jsonb,
error_code=NULL,updated_at=now() WHERE id=%s""",
(original_key, json.dumps(variants), attachment_id))
if telemetry.ready_duration:
telemetry.ready_duration.record((datetime.now(timezone.utc)-row['job_created_at']).total_seconds(), {'job_type':'attachment.process.v1'})
telemetry.event('job.ready')
except Exception as exc:
permanent = isinstance(exc, (ValueError, UnidentifiedImageError, Image.DecompressionBombError))
failed = permanent or row['attempt'] + 1 >= MAX_ATTEMPTS
with db.transaction():
db.execute('UPDATE attachments SET status=%s,error_code=%s,updated_at=now() WHERE id=%s',
('failed' if failed else 'queued',
'invalid_image' if permanent else 'retry_exhausted' if failed else None,
attachment_id))
if not failed:
carrier = telemetry.carrier()
db.execute('UPDATE attachment_jobs SET published_at=NULL,traceparent=%s,tracestate=%s WHERE job_id=%s', (carrier.get('traceparent',''),carrier.get('tracestate',''),job_id))
outcome = 'failed' if failed else 'retry'
finally:
signal.alarm(0)
# Both success and retry intent are committed before ack.
r.xack(STREAM, GROUP, message_id)
return outcome
finally:
db.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))', (attachment_id,))
def consume():
telemetry.init()
signal.signal(signal.SIGALRM, alarm_handler)
consumer = 'worker-' + str(uuid.uuid4())
while True:
try:
with database() as db:
r = broker()
s3 = boto3.client('s3', endpoint_url=os.environ['S3_ENDPOINT'],
region_name=os.getenv('S3_REGION', 'us-east-1'),
aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'),
config=Config(connect_timeout=3, read_timeout=10,
retries={'max_attempts': 1},
s3={'addressing_style': 'path'}))
try:
r.xgroup_create(STREAM, GROUP, id='0', mkstream=True)
except redis.ResponseError as exc:
if 'BUSYGROUP' not in str(exc):
raise
while True:
# Reclaim abandoned deliveries. Advisory lock excludes a still active worker.
claimed = r.xautoclaim(STREAM, GROUP, consumer, min_idle_time=3000, count=1)[1]
batches = [(STREAM, claimed)] if claimed else r.xreadgroup(
GROUP, consumer, {STREAM: '>'}, count=1, block=1000)
for _, entries in batches:
for message_id, fields in entries:
telemetry.run_job(handle, db, s3, r, message_id, fields)
except Exception as exc:
print('worker reconnect:', type(exc).__name__, flush=True)
time.sleep(1)
def cleanup():
"""Explicit maintenance: only terminal jobs older than the retention interval."""
retention = int(os.getenv('STAGING_RETENTION_SECONDS', '86400'))
if retention < 3600:
raise ValueError('retention must be at least one hour')
s3 = boto3.client('s3', endpoint_url=os.environ['S3_ENDPOINT'],
region_name=os.getenv('S3_REGION', 'us-east-1'),
aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'))
with database() as db:
rows = db.execute("""SELECT j.staging_key FROM attachment_jobs j
JOIN attachments a ON a.id=j.attachment_id
WHERE a.status IN ('ready','failed')
AND a.updated_at < now()-make_interval(secs => %s)""", (retention,)).fetchall()
for row in rows:
s3.delete_object(Bucket=os.environ['S3_BUCKET'], Key=row['staging_key'])
print('terminal staging cleanup:', len(rows), 'eligible keys')
def shutdown(*_):
raise SystemExit(0)
if __name__ == '__main__':
signal.signal(signal.SIGTERM, shutdown)
if MAX_ATTEMPTS < 1 or TIMEOUT < 1:
raise ValueError('invalid retry configuration')
if len(sys.argv) > 1 and sys.argv[1] == 'dispatch':
dispatch()
elif len(sys.argv) > 1 and sys.argv[1] == 'cleanup':
cleanup()
else:
count = int(os.getenv('WORKER_CONCURRENCY', '1'))
if count < 1:
raise ValueError('invalid concurrency')
if count == 1:
consume()
else:
children = [Process(target=consume) for _ in range(count)]
for child in children:
child.start()
try:
for child in children:
child.join()
finally:
for child in children:
if child.is_alive():
child.terminate()
for child in children:
child.join(timeout=10)
+6
View File
@@ -0,0 +1,6 @@
Pillow==12.0.0
psycopg[binary]==3.2.10
redis==6.4.0
boto3==1.40.65
opentelemetry-sdk==1.38.0
opentelemetry-exporter-otlp-proto-http==1.38.0
+146
View File
@@ -0,0 +1,146 @@
"""Bounded OTLP exporters; explicit technical data only, never exception text."""
import atexit
import json
import resource
from contextlib import contextmanager
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from opentelemetry.metrics import Observation
import logging
import os
import socket
import time
from opentelemetry import trace, metrics
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
logger = logging.getLogger('jobs')
tracer = trace.get_tracer('messenger-worker')
counter = histogram = ready_duration = None
busy = 0
queue = {"depth":0,"oldest":None,"up":0}
propagator = TraceContextTextMapPropagator()
def init(service="messenger-worker"):
global tracer, counter, histogram, ready_duration
if not os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT') or os.getenv('OTEL_SDK_DISABLED') == 'true':
return
res = Resource.create({'service.name': service, 'service.version': 'lab6-1',
'service.instance.id': socket.gethostname(),
'deployment.environment.name': 'lab'})
tp = TracerProvider(resource=res, sampler=ParentBased(TraceIdRatioBased(float(os.getenv('OTEL_TRACES_SAMPLER_ARG','1')))))
tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(timeout=3), max_queue_size=1024, max_export_batch_size=128))
trace.set_tracer_provider(tp)
mp = MeterProvider(resource=res, metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter(timeout=3), export_interval_millis=5000)])
metrics.set_meter_provider(mp)
lp = LoggerProvider(resource=res)
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(timeout=3), max_queue_size=1024, max_export_batch_size=128))
logger.addHandler(LoggingHandler(logger_provider=lp)); logger.setLevel(logging.INFO); logger.propagate = False
tracer = trace.get_tracer('messenger-worker')
meter = metrics.get_meter('messenger-worker')
counter = meter.create_counter('worker_jobs')
histogram = meter.create_histogram('worker_duration', unit='s', explicit_bucket_boundaries_advisory=(.01,.05,.1,.25,.5,1,2.5,5,10,30,60))
ready_duration = meter.create_histogram('attachment_ready_duration', unit='s', explicit_bucket_boundaries_advisory=(.1,.5,1,2,5,10,30,60,120))
meter.create_observable_counter('process_cpu', unit='s', callbacks=[lambda _: [Observation(resource.getrusage(resource.RUSAGE_SELF).ru_utime + resource.getrusage(resource.RUSAGE_SELF).ru_stime)]])
meter.create_observable_gauge('process_memory', unit='By', callbacks=[lambda _: [Observation(int(open('/proc/self/statm').read().split()[1])*os.sysconf('SC_PAGE_SIZE'))]])
if service == 'messenger-worker':
meter.create_observable_gauge('worker_busy', callbacks=[lambda _: [Observation(busy)]])
meter.create_observable_gauge('worker_capacity', callbacks=[lambda _: [Observation(1)]])
else:
attrs = {'queue_name':'attachment-jobs-v1'}
meter.create_observable_gauge('queue_depth', callbacks=[lambda _: [Observation(queue['depth'],attrs)]])
meter.create_observable_gauge('queue_oldest_age', unit='s', callbacks=[lambda _: [Observation(max(0,time.time()-queue['oldest']) if queue['oldest'] else 0,attrs)]])
meter.create_observable_gauge('queue_monitor_up', callbacks=[lambda _: [Observation(queue['up'],attrs)]])
atexit.register(lp.shutdown); atexit.register(mp.shutdown); atexit.register(tp.shutdown)
def event(name):
span = trace.get_current_span()
sc = span.get_span_context()
logger.info(name, extra={'event': name, 'trace_id': format(sc.trace_id,'032x'), 'span_id': format(sc.span_id,'016x')})
def run_job(fn, *args):
global busy
parent = propagator.extract(args[-1])
payload = json.loads(args[-1]['job'])
with tracer.start_as_current_span('attachment.process', context=parent, kind=trace.SpanKind.CONSUMER, record_exception=False, set_status_on_exception=False) as span:
span.set_attribute('messaging.message.id', payload['job_id'])
span.set_attribute('job.attempt', payload['attempt'])
busy += 1
start = time.monotonic()
outcome = 'error'
try:
result = fn(*args)
outcome = result or 'skipped'
return result
finally:
busy -= 1
attrs = {'job_type': 'attachment.process.v1', 'outcome': outcome}
span.set_attributes(attrs)
if outcome in ('error','failed','retry'):
span.set_status(trace.Status(trace.StatusCode.ERROR, outcome))
event('job.' + outcome)
if counter:
counter.add(1, attrs); histogram.record(time.monotonic()-start, attrs)
def call(name, fn, **kwargs):
with tracer.start_as_current_span(name, kind=trace.SpanKind.CLIENT,
record_exception=False, set_status_on_exception=False) as span:
try:
return fn(**kwargs)
except Exception as exc:
span.set_attribute('error.type', type(exc).__name__)
span.set_status(trace.Status(trace.StatusCode.ERROR, 'S3 operation failed'))
event('s3.error')
raise
@contextmanager
def publishing(row):
parent = propagator.extract({'traceparent':row['traceparent'], 'tracestate':row['tracestate']})
with tracer.start_as_current_span('attachment.publish', context=parent, kind=trace.SpanKind.PRODUCER,
record_exception=False, set_status_on_exception=False):
carrier = {}
propagator.inject(carrier)
yield carrier
def carrier():
result = {}
propagator.inject(result)
return result
class Database:
"""Instrument only queries inside an active request/job, omit SQL and parameters."""
def __init__(self, connection):
self.connection = connection
def __enter__(self):
self.connection.__enter__()
return self
def __exit__(self, *args):
return self.connection.__exit__(*args)
def __getattr__(self, name):
return getattr(self.connection, name)
def execute(self, sql, *args, **kwargs):
if not trace.get_current_span().get_span_context().is_valid:
return self.connection.execute(sql,*args,**kwargs)
op = sql.strip().split()[0].upper()
if op not in ('SELECT','UPDATE','INSERT','DELETE'):
op = 'query'
with tracer.start_as_current_span('db.'+op, kind=trace.SpanKind.CLIENT,
attributes={'db.system.name':'postgresql'},
record_exception=False, set_status_on_exception=False) as span:
try:
return self.connection.execute(sql,*args,**kwargs)
except Exception as exc:
span.set_attribute('error.type', type(exc).__name__)
span.set_status(trace.Status(trace.StatusCode.ERROR,'database operation failed'))
event('db.error')
raise
+41
View File
@@ -0,0 +1,41 @@
import hashlib
import io
import unittest
from PIL import Image
from main import thumbnail
class ThumbnailTests(unittest.TestCase):
def test_size_floor_and_no_upscale(self):
for dimensions, expected in [((320,200),(256,160)), ((7,3),(7,3)), ((1000,333),(256,85))]:
source = io.BytesIO()
Image.new('RGB', dimensions, 'red').save(source, 'PNG')
data, size = thumbnail(source.getvalue())
self.assertEqual(size, expected)
with Image.open(io.BytesIO(data)) as result:
self.assertEqual(result.format, 'JPEG')
self.assertEqual(result.size, expected)
def test_transparency_is_white(self):
source = io.BytesIO()
Image.new('RGBA', (10,10), (0,0,0,0)).save(source, 'PNG')
result, _ = thumbnail(source.getvalue())
with Image.open(io.BytesIO(result)) as image:
self.assertTrue(all(value >= 250 for value in image.getpixel((5,5))))
def test_orientation_applied_and_exif_removed(self):
source = io.BytesIO()
exif = Image.Exif()
exif[274] = 6
Image.new('RGB', (320,200), 'blue').save(source, 'JPEG', exif=exif)
data = source.getvalue()
original_hash = hashlib.sha256(data).hexdigest()
result, size = thumbnail(data)
self.assertEqual(size, (160,256))
self.assertEqual(hashlib.sha256(data).hexdigest(), original_hash)
with Image.open(io.BytesIO(result)) as image:
self.assertFalse(image.getexif())
if __name__ == '__main__':
unittest.main()