forked from mtusi2026/mtusi2026lab5
ЛР 5: асинхронная обработка изображений, уровень 5
This commit is contained in:
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Short local fault checks; requires a running Compose stand. No load test."""
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, Failure, check
|
||||
from image_probe import jpeg_size
|
||||
|
||||
def dc(*args, ok=True):
|
||||
r = subprocess.run(['docker', 'compose', *args], capture_output=True, text=True, timeout=90)
|
||||
if ok and r.returncode:
|
||||
raise RuntimeError('compose action failed: ' + ' '.join(args[:2]))
|
||||
return r
|
||||
|
||||
def sql(query):
|
||||
return dc('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-v', 'ON_ERROR_STOP=1', '-c', query).stdout.strip()
|
||||
|
||||
def wait_for(predicate, message, timeout=100):
|
||||
end = time.monotonic()+timeout
|
||||
while time.monotonic()<end:
|
||||
if predicate():
|
||||
return
|
||||
time.sleep(.3)
|
||||
raise AssertionError(message)
|
||||
|
||||
def wait_http_ready(client):
|
||||
# Probe the same address used by clients: a healthy API process may still
|
||||
# be marked down by the load balancer after a restart.
|
||||
def available():
|
||||
try:
|
||||
return client.call('GET', '/health/ready', expected=[200, 503])['status'] == 200
|
||||
except Failure:
|
||||
# Connection failures are expected while the listener restarts.
|
||||
return False
|
||||
wait_for(available, 'HTTP endpoint did not become ready: ' + client.base)
|
||||
|
||||
def ready(c, actor, att):
|
||||
end = time.monotonic()+45
|
||||
while time.monotonic()<end:
|
||||
value=c.call('GET','/api/v1/attachments/'+att['id'],actor=actor,schema='Attachment')['json']
|
||||
if value['status']=='ready':return value
|
||||
check(value['status']!='failed','unexpected failed')
|
||||
time.sleep(.3)
|
||||
raise AssertionError('ready timeout')
|
||||
|
||||
def verify(c, actor, att, original):
|
||||
value=ready(c,actor,att)
|
||||
path='/api/v1/attachments/'+att['id']+'/content'
|
||||
check(c.call('GET',path,actor=actor)['bytes']==original,'original changed')
|
||||
variants={v['name']:v for v in value['variants']}
|
||||
check(set(variants)=={'thumbnail','crop','optimized'},'missing variants')
|
||||
for name,v in variants.items():
|
||||
data=c.call('GET',path+'?variant='+name,actor=actor)['bytes']
|
||||
check(hashlib.sha256(data).hexdigest()==v['sha256'],'checksum mismatch')
|
||||
if name=='crop':check(jpeg_size(data)==(128,128),'crop dimensions')
|
||||
if name=='optimized':check(data[:4]==b'RIFF' and data[8:12]==b'WEBP','not WebP')
|
||||
return value
|
||||
|
||||
c=Client()
|
||||
wait_http_ready(c)
|
||||
a,b,e,chat=c.bootstrap()
|
||||
try:
|
||||
dc('stop','worker','broker')
|
||||
# A separate API exits immediately after the upload transaction COMMIT.
|
||||
dc('run','--rm','-d','--no-deps','--name','lab5-fault-api','-p','127.0.0.1:18081:8080','-e','FAULT_AFTER_UPLOAD_COMMIT=1','api')
|
||||
fault=Client('http://127.0.0.1:18081')
|
||||
wait_http_ready(fault)
|
||||
try:
|
||||
fault.attachment(a,chat)
|
||||
raise AssertionError('fault did not terminate API')
|
||||
except Failure:pass
|
||||
aid=sql("SELECT id FROM attachments WHERE chat_id='%s' ORDER BY created_at DESC LIMIT 1"%chat['id'])
|
||||
check(bool(aid),'upload COMMIT missing')
|
||||
check(sql("SELECT count(*) FROM attachment_outbox WHERE attachment_id='%s'"%aid)=='1','outbox missing')
|
||||
dc('start','broker','worker')
|
||||
ready(c,a,{'id':aid})
|
||||
print('PASS: crash after COMMIT before publish recovered from outbox')
|
||||
|
||||
dc('stop','worker')
|
||||
att,content=c.attachment(a,chat)
|
||||
def pending():
|
||||
return att['id'] in dc('exec','-T','broker','valkey-cli','LRANGE','attachments:pending','0','-1').stdout
|
||||
wait_for(pending,'job not published')
|
||||
dc('restart','broker')
|
||||
wait_for(lambda: dc('exec','-T','broker','valkey-cli','PING',ok=False).stdout.strip()=='PONG','broker unavailable')
|
||||
check(pending(),'broker restart lost durable job')
|
||||
c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=a,expected=409)
|
||||
dc('restart','api')
|
||||
wait_http_ready(c)
|
||||
dc('start','worker')
|
||||
verify(c,a,att,content)
|
||||
print('PASS: queued job survives broker/API restart; content blocked until ready')
|
||||
for flag in ('FAULT_AFTER_OBJECTS','FAULT_BEFORE_ACK'):
|
||||
dc('stop','worker')
|
||||
att,content=c.attachment(a,chat)
|
||||
r=dc('run','--rm','--no-deps','-e',flag+'=1','worker',ok=False)
|
||||
check(r.returncode==86,'worker fault not triggered')
|
||||
# Advance expired lease in this controlled test instead of waiting 60 seconds.
|
||||
sql("UPDATE attachments SET lease_until=now()-interval '1 second' WHERE id='%s' AND status='processing'"%att['id'])
|
||||
dc('start','worker')
|
||||
value=verify(c,a,att,content)
|
||||
before=sql("SELECT attempts FROM attachments WHERE id='%s'"%att['id'])
|
||||
sql("INSERT INTO attachment_outbox(attachment_id) VALUES('%s') ON CONFLICT DO NOTHING"%att['id'])
|
||||
# Explicit duplicate of a previously successful logical job.
|
||||
job={'schema_version':1,'job_id':att['id'],'type':'attachment.process.v1','attachment_id':att['id'],'pipeline_version':1,'attempt':1,'created_at':att['created_at']}
|
||||
dc('exec','-T','broker','valkey-cli','LPUSH','attachments:pending',json.dumps(job))
|
||||
time.sleep(1)
|
||||
check(ready(c,a,att)==value,'duplicate changed result')
|
||||
check(sql("SELECT attempts FROM attachments WHERE id='%s'"%att['id'])==before,'duplicate executed')
|
||||
check(dc('exec','-T','worker','/usr/local/bin/worker','replay',att['id'],ok=False).returncode!=0,'ready replay allowed')
|
||||
print('PASS: '+flag+', recovery, duplicate no-op, ready replay rejected')
|
||||
|
||||
# A real temporary storage failure must exhaust bounded retries, then replay safely.
|
||||
dc('stop','worker')
|
||||
att,content=c.attachment(a,chat)
|
||||
dc('run','-d','--no-deps','--name','lab5-retry-worker','-e','JOB_MAX_ATTEMPTS=2',
|
||||
'-e','S3_ENDPOINT=http://127.0.0.1:1','worker')
|
||||
wait_for(lambda: sql("SELECT status FROM attachments WHERE id='%s'"%att['id'])=='failed','retry exhaustion timeout')
|
||||
check(sql("SELECT attempts::text || ':' || error_code FROM attachments WHERE id='%s'"%att['id'])=='2:retry_exhausted','incorrect retry limit/error')
|
||||
subprocess.run(['docker','stop','lab5-retry-worker'],check=True,capture_output=True)
|
||||
subprocess.run(['docker','rm','lab5-retry-worker'],check=True,capture_output=True)
|
||||
c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=a,expected=409)
|
||||
dc('run','--rm','--no-deps','worker','replay',att['id'])
|
||||
check(sql("SELECT count(*) FROM attachment_replays WHERE attachment_id='%s'"%att['id'])=='1','replay audit missing')
|
||||
dc('start','worker')
|
||||
verify(c,a,att,content)
|
||||
print('PASS: temporary I/O failure, bounded retries, failed content 409, audited replay to ready')
|
||||
|
||||
worker_id=dc('ps','-q','worker').stdout.strip()
|
||||
before=subprocess.run(['docker','inspect','--format','{{.RestartCount}}',worker_id],check=True,capture_output=True,text=True).stdout.strip()
|
||||
check(sql("SELECT pg_terminate_backend(pid) FROM pg_locks WHERE locktype='advisory' AND objid=505001 AND granted")=='t','worker lock not found')
|
||||
def restarted():
|
||||
value=subprocess.run(['docker','inspect','--format','{{.RestartCount}}',worker_id],check=True,capture_output=True,text=True).stdout.strip()
|
||||
return int(value)>int(before)
|
||||
wait_for(restarted,'worker did not restart after losing DB session')
|
||||
att,content=c.attachment(a,chat)
|
||||
verify(c,a,att,content)
|
||||
print('PASS: lost DB session automatically restarts worker and processing resumes')
|
||||
finally:
|
||||
subprocess.run(['docker','rm','-f','lab5-retry-worker'],capture_output=True)
|
||||
dc('start','broker','worker',ok=False)
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/bin/sh
|
||||
# Создаёт локальные секреты стенда в каталоге secrets/ (он в .gitignore). Запускается один раз
|
||||
# на чистом clone, до первого docker compose up: compose.yaml читает секреты из этих файлов.
|
||||
#
|
||||
# Запуск из любого каталога: sh scripts/init-secrets.sh
|
||||
# Настройки через окружение: POSTGRES_USER, POSTGRES_DB (по умолчанию messenger).
|
||||
# Повторный запуск безопасен: непустые файлы не перезаписываются, значения секретов не печатаются.
|
||||
# Не удаляйте пароль при существующих томах: PostgreSQL задаёт пароль только при создании своего
|
||||
# тома, и новый пароль с ним не совпадёт.
|
||||
|
||||
# -e: выйти при ошибке команды; -u: считать ошибкой неизвестную переменную.
|
||||
set -eu
|
||||
|
||||
# Перейти в корень проекта: все пути ниже указаны относительно него.
|
||||
cd "$(dirname "$0")/.."
|
||||
DIR=secrets
|
||||
# Должны совпадать с POSTGRES_USER и POSTGRES_DB сервиса postgres в compose.yaml.
|
||||
DB_USER="${POSTGRES_USER:-messenger}"
|
||||
DB_NAME="${POSTGRES_DB:-messenger}"
|
||||
|
||||
# Новые файлы и каталог сразу доступны только владельцу: секрет не открыт ни на миг.
|
||||
umask 077
|
||||
mkdir -p "$DIR"
|
||||
|
||||
# Пароль PostgreSQL: 24 случайных байта в hex. -s: файла нет или он пуст.
|
||||
if [ ! -s "$DIR/postgres_password" ]; then
|
||||
# hex, чтобы пароль не требовал URL-экранирования в DSN.
|
||||
od -An -tx1 -N24 /dev/urandom | tr -d ' \n' > "$DIR/postgres_password"
|
||||
echo "init-secrets: created $DIR/postgres_password"
|
||||
fi
|
||||
|
||||
# Строка подключения к БД для API и migrate. Хост postgres — имя сервиса в сети Compose.
|
||||
# TLS выключен: подключение идёт только внутри сети учебного стенда.
|
||||
if [ ! -s "$DIR/database_url" ]; then
|
||||
pw="$(cat "$DIR/postgres_password")"
|
||||
printf 'postgres://%s:%s@postgres:5432/%s?sslmode=disable' "$DB_USER" "$pw" "$DB_NAME" > "$DIR/database_url"
|
||||
echo "init-secrets: created $DIR/database_url"
|
||||
fi
|
||||
|
||||
# Пароль Valkey, в котором хранятся сессии.
|
||||
if [ ! -s "$DIR/valkey_password" ]; then
|
||||
od -An -tx1 -N24 /dev/urandom | tr -d ' \n' > "$DIR/valkey_password"
|
||||
echo "init-secrets: created $DIR/valkey_password"
|
||||
fi
|
||||
|
||||
# Адрес хранилища сессий для API.
|
||||
if [ ! -s "$DIR/cache_url" ]; then
|
||||
pw="$(cat "$DIR/valkey_password")"
|
||||
printf 'redis://:%s@valkey:6379/0' "$pw" > "$DIR/cache_url"
|
||||
echo "init-secrets: created $DIR/cache_url"
|
||||
fi
|
||||
|
||||
# Пароль отдельного Valkey для кэша истории (необязательный профиль). Перевод строки в кавычках
|
||||
# ниже — это символ для удаления, как \n в блоках выше.
|
||||
if [ ! -s "$DIR/valkey_cache_password" ]; then
|
||||
od -An -tx1 -N24 /dev/urandom | tr -d '
|
||||
' > "$DIR/valkey_cache_password"
|
||||
echo "init-secrets: created $DIR/valkey_cache_password"
|
||||
fi
|
||||
|
||||
# Адрес кэша истории для API.
|
||||
if [ ! -s "$DIR/history_cache_url" ]; then
|
||||
pw="$(cat "$DIR/valkey_cache_password")"
|
||||
printf 'redis://:%s@valkey-cache:6379/0' "$pw" > "$DIR/history_cache_url"
|
||||
echo "init-secrets: created $DIR/history_cache_url"
|
||||
fi
|
||||
|
||||
# Ключи S3, отдельные на каждую роль. Hex: значения не требуют экранирования в JSON.
|
||||
# s3_* — API; s3_janitor_* — очиститель; s3_admin_* — только подготовка bucket (s3-init).
|
||||
for role in s3 s3_janitor s3_admin; do
|
||||
if [ ! -s "$DIR/${role}_access_key_id" ]; then
|
||||
od -An -tx1 -N10 /dev/urandom | tr -d ' \n' > "$DIR/${role}_access_key_id"
|
||||
echo "init-secrets: created $DIR/${role}_access_key_id"
|
||||
fi
|
||||
if [ ! -s "$DIR/${role}_secret_access_key" ]; then
|
||||
od -An -tx1 -N20 /dev/urandom | tr -d ' \n' > "$DIR/${role}_secret_access_key"
|
||||
echo "init-secrets: created $DIR/${role}_secret_access_key"
|
||||
fi
|
||||
done
|
||||
|
||||
# Пользователи SeaweedFS. Выводится из ключей выше и перезаписывается всегда, поэтому изменение прав
|
||||
# здесь применяется и к уже созданному стенду (после перезапуска сервиса s3). Секретов не добавляет.
|
||||
# Отсутствие записи anonymous означает, что запрос без подписи получает отказ: bucket приватный.
|
||||
# Права на запись ограничены префиксом объектов, а не bucket: в SeaweedFS Write:<bucket> разрешает
|
||||
# и PUT политики bucket, то есть позволил бы API сделать bucket публичным (найдено cmd/s3check).
|
||||
BUCKET="$(sed -n 's/^S3_BUCKET=//p' .env 2>/dev/null || true)"
|
||||
BUCKET="${BUCKET:-messenger-attachments}"
|
||||
identity() {
|
||||
printf '{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":[%s]}' \
|
||||
"$1" "$(cat "$DIR/$2_access_key_id")" "$(cat "$DIR/$2_secret_access_key")" "$3"
|
||||
}
|
||||
{
|
||||
printf '{"identities":['
|
||||
identity messenger-admin s3_admin '"Admin","Read","Write","List"'
|
||||
printf ','
|
||||
identity messenger-api s3 "\"Read:$BUCKET/*\",\"Write:$BUCKET/*\""
|
||||
printf ','
|
||||
identity messenger-janitor s3_janitor "\"List:$BUCKET\",\"Write:$BUCKET/originals/*\""
|
||||
printf ']}'
|
||||
} > "$DIR/s3_config"
|
||||
|
||||
# Каталог закрыт от других пользователей хоста. Сами файлы открыты на чтение: Compose монтирует
|
||||
# их в контейнеры с правами хоста, а процессы там работают не от root (API — uid 100).
|
||||
chmod 0700 "$DIR"
|
||||
chmod 0644 "$DIR/postgres_password" "$DIR/database_url" "$DIR/valkey_password" "$DIR/cache_url" "$DIR/valkey_cache_password" "$DIR/history_cache_url" \
|
||||
"$DIR/s3_access_key_id" "$DIR/s3_secret_access_key" \
|
||||
"$DIR/s3_janitor_access_key_id" "$DIR/s3_janitor_secret_access_key" \
|
||||
"$DIR/s3_admin_access_key_id" "$DIR/s3_admin_secret_access_key" "$DIR/s3_config"
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded track-1 experiment, stdlib only. Run on a dedicated Compose project.
|
||||
JOB_MAX_INFLIGHT=12 must be configured in API. Six batches, 16 uploads each.
|
||||
"""
|
||||
import concurrent.futures
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
OUTPUT=ROOT/'evidence/overload-results.json'
|
||||
CONFIGS=((1,1),(2,2),(4,8))
|
||||
|
||||
def dc(*args, env=None):
|
||||
result=subprocess.run(['docker','compose',*args],env=env,capture_output=True,text=True,timeout=90)
|
||||
if result.returncode: raise RuntimeError('compose failed: '+' '.join(args[:2])+': '+result.stderr[-600:])
|
||||
return result.stdout.strip()
|
||||
|
||||
def sql(query):
|
||||
return dc('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',query)
|
||||
|
||||
def png(w,h):
|
||||
# Repeat deterministic textured rows: compact upload, real image decode/WebP work.
|
||||
rng=random.Random(42)
|
||||
rows=[rng.randbytes(w*3) for _ in range(32)]
|
||||
raw=b''.join(b'\x00'+rows[y%32] for y in range(h))
|
||||
def chunk(name,data):return struct.pack('!I',len(data))+name+data+struct.pack('!I',zlib.crc32(name+data)&0xffffffff)
|
||||
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('!2I5B',w,h,8,2,0,0,0))+chunk(b'IDAT',zlib.compress(raw,1))+chunk(b'IEND',b'')
|
||||
|
||||
def percentile(values,p=.95):
|
||||
import math
|
||||
return sorted(values)[max(0,math.ceil(len(values)*p)-1)]
|
||||
|
||||
def queue():
|
||||
return [int(dc('exec','-T','broker','valkey-cli','LLEN',name)) for name in ('attachments:pending','attachments:active')]
|
||||
|
||||
def live_overload():
|
||||
data=json.loads(OUTPUT.read_text())
|
||||
c=Client(); actor,_,_,chat=c.bootstrap()
|
||||
content=png(2000,1400)
|
||||
samples=[]; stop=threading.Event()
|
||||
def monitor():
|
||||
while not stop.is_set():
|
||||
samples.append(int(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")))
|
||||
stop.wait(.1)
|
||||
def upload(_):
|
||||
return Client().call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=live.png',
|
||||
expected=[202,429],actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
|
||||
thread=threading.Thread(target=monitor);thread.start();start=time.monotonic()
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
|
||||
replies=list(executor.map(upload,range(32)))
|
||||
ids=[r['json']['id'] for r in replies if r['status']==202]
|
||||
rejected=sum(r['status']==429 for r in replies)
|
||||
check(rejected>0,'live burst did not create backpressure')
|
||||
deadline=time.monotonic()+75
|
||||
while sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")!='0':
|
||||
check(time.monotonic()<deadline,'live burst did not drain');time.sleep(.2)
|
||||
for aid in ids:
|
||||
meta=c.call('GET','/api/v1/attachments/'+aid,actor=actor,schema='Attachment')['json']
|
||||
check(meta['status']=='ready','accepted live task failed/lost')
|
||||
check(max(samples)<=12,'global cap exceeded with live worker')
|
||||
probe,_=c.attachment(actor,chat);c.wait_ready(actor,probe)
|
||||
data['live_overload']={'offered':32,'clients':8,'accepted':len(ids),'rejected_429':rejected,
|
||||
'max_unfinished':max(samples),'unfinished_samples':samples,'elapsed_s':round(time.monotonic()-start,3),
|
||||
'all_accepted_ready':True,'recovery_probe_ready':True,'concurrency':4,'prefetch':8}
|
||||
OUTPUT.write_text(json.dumps(data,indent=2)+'\n')
|
||||
print('PASS live overload: '+json.dumps({k:v for k,v in data['live_overload'].items() if k!='unfinished_samples'}),flush=True)
|
||||
finally:stop.set();thread.join()
|
||||
|
||||
|
||||
def run():
|
||||
c=Client(); actor,_,_,chat=c.bootstrap()
|
||||
data={'started_utc':dt.datetime.now(dt.timezone.utc).isoformat(),'configs':CONFIGS,
|
||||
'cap':12,'offered_per_batch':16,'samples_interval_s':.25,'results':[],
|
||||
'host':subprocess.check_output(['uname','-sm'],text=True).strip(),
|
||||
'docker':json.loads(subprocess.check_output(['docker','info','--format','{{json .}}'],text=True))}
|
||||
# Keep only non-sensitive machine capacity metadata.
|
||||
data['docker']={k:data['docker'].get(k) for k in ('Architecture','NCPU','MemTotal','OperatingSystem','ServerVersion')}
|
||||
for concurrency,prefetch in CONFIGS:
|
||||
env=dict(os.environ,WORKER_CONCURRENCY=str(concurrency),WORKER_PREFETCH=str(prefetch))
|
||||
dc('up','-d','--no-deps','worker',env=env)
|
||||
# Warm one render per setting; its data is excluded from measurements.
|
||||
warm,_=c.attachment(actor,chat); c.wait_ready(actor,warm)
|
||||
for w,h in ((320,200),(2000,1400)):
|
||||
dc('stop','worker')
|
||||
check(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")=='0','stand is not isolated/drained')
|
||||
content=png(w,h)
|
||||
def upload(_):
|
||||
client=Client()
|
||||
return client.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=study.png',
|
||||
expected=[202,429],actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
|
||||
t0=time.monotonic()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
|
||||
replies=list(executor.map(upload,range(16)))
|
||||
accepted=[r['json']['id'] for r in replies if r['status']==202]
|
||||
rejected=sum(r['status']==429 for r in replies)
|
||||
check(len(accepted)==12 and rejected==4,'global admission cap was not enforced')
|
||||
check(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")=='12','cap exceeded')
|
||||
ids=','.join("'"+value+"'" for value in accepted)
|
||||
# Give dispatcher <=2s to publish; admission remains durable even before publish.
|
||||
until=time.monotonic()+3
|
||||
while queue()[0]<12 and time.monotonic()<until:time.sleep(.1)
|
||||
samples=[]; stop=threading.Event()
|
||||
def monitor():
|
||||
while not stop.is_set():
|
||||
try:
|
||||
rss=int(dc('exec','-T','worker','sh','-c',"awk '/^VmRSS:/ {s+=$2} END {print s+0}' /proc/[0-9]*/status"))
|
||||
pending,active=queue()
|
||||
samples.append({'t_s':round(time.monotonic()-release,3),'rss_kib':rss,'pending':pending,'active':active})
|
||||
except RuntimeError:pass
|
||||
stop.wait(.25)
|
||||
release=time.monotonic(); release_utc=dt.datetime.now(dt.timezone.utc)
|
||||
dc('start','worker')
|
||||
thread=threading.Thread(target=monitor);thread.start()
|
||||
try:
|
||||
deadline=time.monotonic()+75
|
||||
while True:
|
||||
rows=json.loads(sql("SELECT json_agg(row_to_json(t)) FROM (SELECT id,status,attempts,extract(epoch FROM (processing_started_at-created_at)) AS lag,extract(epoch FROM (updated_at-created_at)) AS latency,updated_at FROM attachments WHERE id IN ("+ids+")) t"))
|
||||
check(not any(r['status']=='failed' for r in rows),'failed accepted job')
|
||||
if all(r['status']=='ready' for r in rows):break
|
||||
check(time.monotonic()<deadline,'drain exceeded 75s budget')
|
||||
time.sleep(.15)
|
||||
# Verify durable effects for every accepted task through authenticated API.
|
||||
import hashlib
|
||||
for aid in accepted:
|
||||
meta=c.call('GET','/api/v1/attachments/'+aid,actor=actor,schema='Attachment')['json']
|
||||
check({v['name'] for v in meta['variants']}=={'thumbnail','crop','optimized'},'variants missing')
|
||||
check(meta['sha256']==hashlib.sha256(content).hexdigest(),'original checksum mismatch')
|
||||
deadline=time.monotonic()+5
|
||||
while queue()!=[0,0] and time.monotonic()<deadline:time.sleep(.1)
|
||||
check(queue()==[0,0],'broker did not drain')
|
||||
finally:
|
||||
stop.set();thread.join()
|
||||
# New upload is accepted after overload and its real objects are downloadable.
|
||||
probe,probe_bytes=c.attachment(actor,chat)
|
||||
probe=c.wait_ready(actor,probe)
|
||||
path='/api/v1/attachments/'+probe['id']+'/content'
|
||||
check(c.call('GET',path,actor=actor)['bytes']==probe_bytes,'recovery original changed')
|
||||
for variant in probe['variants']:
|
||||
actual=c.call('GET',path+'?variant='+variant['name'],actor=actor)['bytes']
|
||||
check(hashlib.sha256(actual).hexdigest()==variant['sha256'],'recovery variant missing/corrupt')
|
||||
last=max(dt.datetime.fromisoformat(r['updated_at'].replace('Z','+00:00')) for r in rows)
|
||||
drain=(last-release_utc).total_seconds()
|
||||
result={'concurrency':concurrency,'prefetch':prefetch,'dimensions':[w,h],'bytes':len(content),
|
||||
'accepted':len(accepted),'rejected_429':rejected,'admission_s':round(release-t0,3),
|
||||
'drain_s':round(drain,3),'throughput_jobs_s':round(12/drain,3),
|
||||
'queue_lag_p95_s':round(percentile([float(r['lag']) for r in rows]),3),
|
||||
'ready_p95_s':round(percentile([float(r['latency']) for r in rows]),3),
|
||||
'peak_rss_mib':round(max(s['rss_kib'] for s in samples)/1024,2),
|
||||
'peak_active':max(s['active'] for s in samples),'final_queue':queue(),'recovery_probe_ready':True,
|
||||
'attempts': [r['attempts'] for r in rows], 'samples':samples}
|
||||
check(result['peak_active']<=prefetch,'prefetch exceeded')
|
||||
data['results'].append(result)
|
||||
OUTPUT.write_text(json.dumps(data,indent=2)+'\n')
|
||||
print(json.dumps({k:v for k,v in result.items() if k not in ('samples','attempts')}),flush=True)
|
||||
print('PASS: six batches drained; accepted work preserved; cap and prefetch enforced',flush=True)
|
||||
|
||||
if __name__=='__main__':
|
||||
if '--live-only' not in sys.argv:run()
|
||||
live_overload()
|
||||
Executable → Regular
Executable → Regular
Reference in New Issue
Block a user