ЛР 5: асинхронная обработка изображений, уровень 5

This commit is contained in:
Nurlygayanova Ekaterina
2026-11-12 20:32:55 +03:00
parent 276dbdad8a
commit 7b3eab5473
115 changed files with 8847 additions and 34 deletions
+145
View File
@@ -0,0 +1,145 @@
#!/usr/bin/env python3
"""Short local fault checks; requires a running Compose stand. No load test."""
import hashlib
import json
import subprocess
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, Failure, check
from image_probe import jpeg_size
def dc(*args, ok=True):
r = subprocess.run(['docker', 'compose', *args], capture_output=True, text=True, timeout=90)
if ok and r.returncode:
raise RuntimeError('compose action failed: ' + ' '.join(args[:2]))
return r
def sql(query):
return dc('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-v', 'ON_ERROR_STOP=1', '-c', query).stdout.strip()
def wait_for(predicate, message, timeout=100):
end = time.monotonic()+timeout
while time.monotonic()<end:
if predicate():
return
time.sleep(.3)
raise AssertionError(message)
def wait_http_ready(client):
# Probe the same address used by clients: a healthy API process may still
# be marked down by the load balancer after a restart.
def available():
try:
return client.call('GET', '/health/ready', expected=[200, 503])['status'] == 200
except Failure:
# Connection failures are expected while the listener restarts.
return False
wait_for(available, 'HTTP endpoint did not become ready: ' + client.base)
def ready(c, actor, att):
end = time.monotonic()+45
while time.monotonic()<end:
value=c.call('GET','/api/v1/attachments/'+att['id'],actor=actor,schema='Attachment')['json']
if value['status']=='ready':return value
check(value['status']!='failed','unexpected failed')
time.sleep(.3)
raise AssertionError('ready timeout')
def verify(c, actor, att, original):
value=ready(c,actor,att)
path='/api/v1/attachments/'+att['id']+'/content'
check(c.call('GET',path,actor=actor)['bytes']==original,'original changed')
variants={v['name']:v for v in value['variants']}
check(set(variants)=={'thumbnail','crop','optimized'},'missing variants')
for name,v in variants.items():
data=c.call('GET',path+'?variant='+name,actor=actor)['bytes']
check(hashlib.sha256(data).hexdigest()==v['sha256'],'checksum mismatch')
if name=='crop':check(jpeg_size(data)==(128,128),'crop dimensions')
if name=='optimized':check(data[:4]==b'RIFF' and data[8:12]==b'WEBP','not WebP')
return value
c=Client()
wait_http_ready(c)
a,b,e,chat=c.bootstrap()
try:
dc('stop','worker','broker')
# A separate API exits immediately after the upload transaction COMMIT.
dc('run','--rm','-d','--no-deps','--name','lab5-fault-api','-p','127.0.0.1:18081:8080','-e','FAULT_AFTER_UPLOAD_COMMIT=1','api')
fault=Client('http://127.0.0.1:18081')
wait_http_ready(fault)
try:
fault.attachment(a,chat)
raise AssertionError('fault did not terminate API')
except Failure:pass
aid=sql("SELECT id FROM attachments WHERE chat_id='%s' ORDER BY created_at DESC LIMIT 1"%chat['id'])
check(bool(aid),'upload COMMIT missing')
check(sql("SELECT count(*) FROM attachment_outbox WHERE attachment_id='%s'"%aid)=='1','outbox missing')
dc('start','broker','worker')
ready(c,a,{'id':aid})
print('PASS: crash after COMMIT before publish recovered from outbox')
dc('stop','worker')
att,content=c.attachment(a,chat)
def pending():
return att['id'] in dc('exec','-T','broker','valkey-cli','LRANGE','attachments:pending','0','-1').stdout
wait_for(pending,'job not published')
dc('restart','broker')
wait_for(lambda: dc('exec','-T','broker','valkey-cli','PING',ok=False).stdout.strip()=='PONG','broker unavailable')
check(pending(),'broker restart lost durable job')
c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=a,expected=409)
dc('restart','api')
wait_http_ready(c)
dc('start','worker')
verify(c,a,att,content)
print('PASS: queued job survives broker/API restart; content blocked until ready')
for flag in ('FAULT_AFTER_OBJECTS','FAULT_BEFORE_ACK'):
dc('stop','worker')
att,content=c.attachment(a,chat)
r=dc('run','--rm','--no-deps','-e',flag+'=1','worker',ok=False)
check(r.returncode==86,'worker fault not triggered')
# Advance expired lease in this controlled test instead of waiting 60 seconds.
sql("UPDATE attachments SET lease_until=now()-interval '1 second' WHERE id='%s' AND status='processing'"%att['id'])
dc('start','worker')
value=verify(c,a,att,content)
before=sql("SELECT attempts FROM attachments WHERE id='%s'"%att['id'])
sql("INSERT INTO attachment_outbox(attachment_id) VALUES('%s') ON CONFLICT DO NOTHING"%att['id'])
# Explicit duplicate of a previously successful logical job.
job={'schema_version':1,'job_id':att['id'],'type':'attachment.process.v1','attachment_id':att['id'],'pipeline_version':1,'attempt':1,'created_at':att['created_at']}
dc('exec','-T','broker','valkey-cli','LPUSH','attachments:pending',json.dumps(job))
time.sleep(1)
check(ready(c,a,att)==value,'duplicate changed result')
check(sql("SELECT attempts FROM attachments WHERE id='%s'"%att['id'])==before,'duplicate executed')
check(dc('exec','-T','worker','/usr/local/bin/worker','replay',att['id'],ok=False).returncode!=0,'ready replay allowed')
print('PASS: '+flag+', recovery, duplicate no-op, ready replay rejected')
# A real temporary storage failure must exhaust bounded retries, then replay safely.
dc('stop','worker')
att,content=c.attachment(a,chat)
dc('run','-d','--no-deps','--name','lab5-retry-worker','-e','JOB_MAX_ATTEMPTS=2',
'-e','S3_ENDPOINT=http://127.0.0.1:1','worker')
wait_for(lambda: sql("SELECT status FROM attachments WHERE id='%s'"%att['id'])=='failed','retry exhaustion timeout')
check(sql("SELECT attempts::text || ':' || error_code FROM attachments WHERE id='%s'"%att['id'])=='2:retry_exhausted','incorrect retry limit/error')
subprocess.run(['docker','stop','lab5-retry-worker'],check=True,capture_output=True)
subprocess.run(['docker','rm','lab5-retry-worker'],check=True,capture_output=True)
c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=a,expected=409)
dc('run','--rm','--no-deps','worker','replay',att['id'])
check(sql("SELECT count(*) FROM attachment_replays WHERE attachment_id='%s'"%att['id'])=='1','replay audit missing')
dc('start','worker')
verify(c,a,att,content)
print('PASS: temporary I/O failure, bounded retries, failed content 409, audited replay to ready')
worker_id=dc('ps','-q','worker').stdout.strip()
before=subprocess.run(['docker','inspect','--format','{{.RestartCount}}',worker_id],check=True,capture_output=True,text=True).stdout.strip()
check(sql("SELECT pg_terminate_backend(pid) FROM pg_locks WHERE locktype='advisory' AND objid=505001 AND granted")=='t','worker lock not found')
def restarted():
value=subprocess.run(['docker','inspect','--format','{{.RestartCount}}',worker_id],check=True,capture_output=True,text=True).stdout.strip()
return int(value)>int(before)
wait_for(restarted,'worker did not restart after losing DB session')
att,content=c.attachment(a,chat)
verify(c,a,att,content)
print('PASS: lost DB session automatically restarts worker and processing resumes')
finally:
subprocess.run(['docker','rm','-f','lab5-retry-worker'],capture_output=True)
dc('start','broker','worker',ok=False)
+108
View File
@@ -0,0 +1,108 @@
#!/bin/sh
# Создаёт локальные секреты стенда в каталоге secrets/ (он в .gitignore). Запускается один раз
# на чистом clone, до первого docker compose up: compose.yaml читает секреты из этих файлов.
#
# Запуск из любого каталога: sh scripts/init-secrets.sh
# Настройки через окружение: POSTGRES_USER, POSTGRES_DB (по умолчанию messenger).
# Повторный запуск безопасен: непустые файлы не перезаписываются, значения секретов не печатаются.
# Не удаляйте пароль при существующих томах: PostgreSQL задаёт пароль только при создании своего
# тома, и новый пароль с ним не совпадёт.
# -e: выйти при ошибке команды; -u: считать ошибкой неизвестную переменную.
set -eu
# Перейти в корень проекта: все пути ниже указаны относительно него.
cd "$(dirname "$0")/.."
DIR=secrets
# Должны совпадать с POSTGRES_USER и POSTGRES_DB сервиса postgres в compose.yaml.
DB_USER="${POSTGRES_USER:-messenger}"
DB_NAME="${POSTGRES_DB:-messenger}"
# Новые файлы и каталог сразу доступны только владельцу: секрет не открыт ни на миг.
umask 077
mkdir -p "$DIR"
# Пароль PostgreSQL: 24 случайных байта в hex. -s: файла нет или он пуст.
if [ ! -s "$DIR/postgres_password" ]; then
# hex, чтобы пароль не требовал URL-экранирования в DSN.
od -An -tx1 -N24 /dev/urandom | tr -d ' \n' > "$DIR/postgres_password"
echo "init-secrets: created $DIR/postgres_password"
fi
# Строка подключения к БД для API и migrate. Хост postgres — имя сервиса в сети Compose.
# TLS выключен: подключение идёт только внутри сети учебного стенда.
if [ ! -s "$DIR/database_url" ]; then
pw="$(cat "$DIR/postgres_password")"
printf 'postgres://%s:%s@postgres:5432/%s?sslmode=disable' "$DB_USER" "$pw" "$DB_NAME" > "$DIR/database_url"
echo "init-secrets: created $DIR/database_url"
fi
# Пароль Valkey, в котором хранятся сессии.
if [ ! -s "$DIR/valkey_password" ]; then
od -An -tx1 -N24 /dev/urandom | tr -d ' \n' > "$DIR/valkey_password"
echo "init-secrets: created $DIR/valkey_password"
fi
# Адрес хранилища сессий для API.
if [ ! -s "$DIR/cache_url" ]; then
pw="$(cat "$DIR/valkey_password")"
printf 'redis://:%s@valkey:6379/0' "$pw" > "$DIR/cache_url"
echo "init-secrets: created $DIR/cache_url"
fi
# Пароль отдельного Valkey для кэша истории (необязательный профиль). Перевод строки в кавычках
# ниже — это символ для удаления, как \n в блоках выше.
if [ ! -s "$DIR/valkey_cache_password" ]; then
od -An -tx1 -N24 /dev/urandom | tr -d '
' > "$DIR/valkey_cache_password"
echo "init-secrets: created $DIR/valkey_cache_password"
fi
# Адрес кэша истории для API.
if [ ! -s "$DIR/history_cache_url" ]; then
pw="$(cat "$DIR/valkey_cache_password")"
printf 'redis://:%s@valkey-cache:6379/0' "$pw" > "$DIR/history_cache_url"
echo "init-secrets: created $DIR/history_cache_url"
fi
# Ключи S3, отдельные на каждую роль. Hex: значения не требуют экранирования в JSON.
# s3_* — API; s3_janitor_* — очиститель; s3_admin_* — только подготовка bucket (s3-init).
for role in s3 s3_janitor s3_admin; do
if [ ! -s "$DIR/${role}_access_key_id" ]; then
od -An -tx1 -N10 /dev/urandom | tr -d ' \n' > "$DIR/${role}_access_key_id"
echo "init-secrets: created $DIR/${role}_access_key_id"
fi
if [ ! -s "$DIR/${role}_secret_access_key" ]; then
od -An -tx1 -N20 /dev/urandom | tr -d ' \n' > "$DIR/${role}_secret_access_key"
echo "init-secrets: created $DIR/${role}_secret_access_key"
fi
done
# Пользователи SeaweedFS. Выводится из ключей выше и перезаписывается всегда, поэтому изменение прав
# здесь применяется и к уже созданному стенду (после перезапуска сервиса s3). Секретов не добавляет.
# Отсутствие записи anonymous означает, что запрос без подписи получает отказ: bucket приватный.
# Права на запись ограничены префиксом объектов, а не bucket: в SeaweedFS Write:<bucket> разрешает
# и PUT политики bucket, то есть позволил бы API сделать bucket публичным (найдено cmd/s3check).
BUCKET="$(sed -n 's/^S3_BUCKET=//p' .env 2>/dev/null || true)"
BUCKET="${BUCKET:-messenger-attachments}"
identity() {
printf '{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":[%s]}' \
"$1" "$(cat "$DIR/$2_access_key_id")" "$(cat "$DIR/$2_secret_access_key")" "$3"
}
{
printf '{"identities":['
identity messenger-admin s3_admin '"Admin","Read","Write","List"'
printf ','
identity messenger-api s3 "\"Read:$BUCKET/*\",\"Write:$BUCKET/*\""
printf ','
identity messenger-janitor s3_janitor "\"List:$BUCKET\",\"Write:$BUCKET/originals/*\""
printf ']}'
} > "$DIR/s3_config"
# Каталог закрыт от других пользователей хоста. Сами файлы открыты на чтение: Compose монтирует
# их в контейнеры с правами хоста, а процессы там работают не от root (API — uid 100).
chmod 0700 "$DIR"
chmod 0644 "$DIR/postgres_password" "$DIR/database_url" "$DIR/valkey_password" "$DIR/cache_url" "$DIR/valkey_cache_password" "$DIR/history_cache_url" \
"$DIR/s3_access_key_id" "$DIR/s3_secret_access_key" \
"$DIR/s3_janitor_access_key_id" "$DIR/s3_janitor_secret_access_key" \
"$DIR/s3_admin_access_key_id" "$DIR/s3_admin_secret_access_key" "$DIR/s3_config"
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""Bounded track-1 experiment, stdlib only. Run on a dedicated Compose project.
JOB_MAX_INFLIGHT=12 must be configured in API. Six batches, 16 uploads each.
"""
import concurrent.futures
import datetime as dt
import json
import os
from pathlib import Path
import random
import struct
import subprocess
import sys
import threading
import time
import zlib
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, check
ROOT=Path(__file__).resolve().parents[1]
OUTPUT=ROOT/'evidence/overload-results.json'
CONFIGS=((1,1),(2,2),(4,8))
def dc(*args, env=None):
result=subprocess.run(['docker','compose',*args],env=env,capture_output=True,text=True,timeout=90)
if result.returncode: raise RuntimeError('compose failed: '+' '.join(args[:2])+': '+result.stderr[-600:])
return result.stdout.strip()
def sql(query):
return dc('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',query)
def png(w,h):
# Repeat deterministic textured rows: compact upload, real image decode/WebP work.
rng=random.Random(42)
rows=[rng.randbytes(w*3) for _ in range(32)]
raw=b''.join(b'\x00'+rows[y%32] for y in range(h))
def chunk(name,data):return struct.pack('!I',len(data))+name+data+struct.pack('!I',zlib.crc32(name+data)&0xffffffff)
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('!2I5B',w,h,8,2,0,0,0))+chunk(b'IDAT',zlib.compress(raw,1))+chunk(b'IEND',b'')
def percentile(values,p=.95):
import math
return sorted(values)[max(0,math.ceil(len(values)*p)-1)]
def queue():
return [int(dc('exec','-T','broker','valkey-cli','LLEN',name)) for name in ('attachments:pending','attachments:active')]
def live_overload():
data=json.loads(OUTPUT.read_text())
c=Client(); actor,_,_,chat=c.bootstrap()
content=png(2000,1400)
samples=[]; stop=threading.Event()
def monitor():
while not stop.is_set():
samples.append(int(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")))
stop.wait(.1)
def upload(_):
return Client().call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=live.png',
expected=[202,429],actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
thread=threading.Thread(target=monitor);thread.start();start=time.monotonic()
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
replies=list(executor.map(upload,range(32)))
ids=[r['json']['id'] for r in replies if r['status']==202]
rejected=sum(r['status']==429 for r in replies)
check(rejected>0,'live burst did not create backpressure')
deadline=time.monotonic()+75
while sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")!='0':
check(time.monotonic()<deadline,'live burst did not drain');time.sleep(.2)
for aid in ids:
meta=c.call('GET','/api/v1/attachments/'+aid,actor=actor,schema='Attachment')['json']
check(meta['status']=='ready','accepted live task failed/lost')
check(max(samples)<=12,'global cap exceeded with live worker')
probe,_=c.attachment(actor,chat);c.wait_ready(actor,probe)
data['live_overload']={'offered':32,'clients':8,'accepted':len(ids),'rejected_429':rejected,
'max_unfinished':max(samples),'unfinished_samples':samples,'elapsed_s':round(time.monotonic()-start,3),
'all_accepted_ready':True,'recovery_probe_ready':True,'concurrency':4,'prefetch':8}
OUTPUT.write_text(json.dumps(data,indent=2)+'\n')
print('PASS live overload: '+json.dumps({k:v for k,v in data['live_overload'].items() if k!='unfinished_samples'}),flush=True)
finally:stop.set();thread.join()
def run():
c=Client(); actor,_,_,chat=c.bootstrap()
data={'started_utc':dt.datetime.now(dt.timezone.utc).isoformat(),'configs':CONFIGS,
'cap':12,'offered_per_batch':16,'samples_interval_s':.25,'results':[],
'host':subprocess.check_output(['uname','-sm'],text=True).strip(),
'docker':json.loads(subprocess.check_output(['docker','info','--format','{{json .}}'],text=True))}
# Keep only non-sensitive machine capacity metadata.
data['docker']={k:data['docker'].get(k) for k in ('Architecture','NCPU','MemTotal','OperatingSystem','ServerVersion')}
for concurrency,prefetch in CONFIGS:
env=dict(os.environ,WORKER_CONCURRENCY=str(concurrency),WORKER_PREFETCH=str(prefetch))
dc('up','-d','--no-deps','worker',env=env)
# Warm one render per setting; its data is excluded from measurements.
warm,_=c.attachment(actor,chat); c.wait_ready(actor,warm)
for w,h in ((320,200),(2000,1400)):
dc('stop','worker')
check(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")=='0','stand is not isolated/drained')
content=png(w,h)
def upload(_):
client=Client()
return client.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=study.png',
expected=[202,429],actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
t0=time.monotonic()
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
replies=list(executor.map(upload,range(16)))
accepted=[r['json']['id'] for r in replies if r['status']==202]
rejected=sum(r['status']==429 for r in replies)
check(len(accepted)==12 and rejected==4,'global admission cap was not enforced')
check(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")=='12','cap exceeded')
ids=','.join("'"+value+"'" for value in accepted)
# Give dispatcher <=2s to publish; admission remains durable even before publish.
until=time.monotonic()+3
while queue()[0]<12 and time.monotonic()<until:time.sleep(.1)
samples=[]; stop=threading.Event()
def monitor():
while not stop.is_set():
try:
rss=int(dc('exec','-T','worker','sh','-c',"awk '/^VmRSS:/ {s+=$2} END {print s+0}' /proc/[0-9]*/status"))
pending,active=queue()
samples.append({'t_s':round(time.monotonic()-release,3),'rss_kib':rss,'pending':pending,'active':active})
except RuntimeError:pass
stop.wait(.25)
release=time.monotonic(); release_utc=dt.datetime.now(dt.timezone.utc)
dc('start','worker')
thread=threading.Thread(target=monitor);thread.start()
try:
deadline=time.monotonic()+75
while True:
rows=json.loads(sql("SELECT json_agg(row_to_json(t)) FROM (SELECT id,status,attempts,extract(epoch FROM (processing_started_at-created_at)) AS lag,extract(epoch FROM (updated_at-created_at)) AS latency,updated_at FROM attachments WHERE id IN ("+ids+")) t"))
check(not any(r['status']=='failed' for r in rows),'failed accepted job')
if all(r['status']=='ready' for r in rows):break
check(time.monotonic()<deadline,'drain exceeded 75s budget')
time.sleep(.15)
# Verify durable effects for every accepted task through authenticated API.
import hashlib
for aid in accepted:
meta=c.call('GET','/api/v1/attachments/'+aid,actor=actor,schema='Attachment')['json']
check({v['name'] for v in meta['variants']}=={'thumbnail','crop','optimized'},'variants missing')
check(meta['sha256']==hashlib.sha256(content).hexdigest(),'original checksum mismatch')
deadline=time.monotonic()+5
while queue()!=[0,0] and time.monotonic()<deadline:time.sleep(.1)
check(queue()==[0,0],'broker did not drain')
finally:
stop.set();thread.join()
# New upload is accepted after overload and its real objects are downloadable.
probe,probe_bytes=c.attachment(actor,chat)
probe=c.wait_ready(actor,probe)
path='/api/v1/attachments/'+probe['id']+'/content'
check(c.call('GET',path,actor=actor)['bytes']==probe_bytes,'recovery original changed')
for variant in probe['variants']:
actual=c.call('GET',path+'?variant='+variant['name'],actor=actor)['bytes']
check(hashlib.sha256(actual).hexdigest()==variant['sha256'],'recovery variant missing/corrupt')
last=max(dt.datetime.fromisoformat(r['updated_at'].replace('Z','+00:00')) for r in rows)
drain=(last-release_utc).total_seconds()
result={'concurrency':concurrency,'prefetch':prefetch,'dimensions':[w,h],'bytes':len(content),
'accepted':len(accepted),'rejected_429':rejected,'admission_s':round(release-t0,3),
'drain_s':round(drain,3),'throughput_jobs_s':round(12/drain,3),
'queue_lag_p95_s':round(percentile([float(r['lag']) for r in rows]),3),
'ready_p95_s':round(percentile([float(r['latency']) for r in rows]),3),
'peak_rss_mib':round(max(s['rss_kib'] for s in samples)/1024,2),
'peak_active':max(s['active'] for s in samples),'final_queue':queue(),'recovery_probe_ready':True,
'attempts': [r['attempts'] for r in rows], 'samples':samples}
check(result['peak_active']<=prefetch,'prefetch exceeded')
data['results'].append(result)
OUTPUT.write_text(json.dumps(data,indent=2)+'\n')
print(json.dumps({k:v for k,v in result.items() if k not in ('samples','attempts')}),flush=True)
print('PASS: six batches drained; accepted work preserved; cap and prefetch enforced',flush=True)
if __name__=='__main__':
if '--live-only' not in sys.argv:run()
live_overload()
Executable → Regular
View File
Executable → Regular
View File