Files
mtusi2026lab5-fork/scripts/check_async_grade4.py
T
Dima 4bee6a5d4a Реализовать уровень 4 и упорядочить структуру backend
Добавить crop и WebP, изолированную обработку с таймаутом, lease token, backoff с jitter, реестр failed, безопасный replay и очистку staging. Добавить проверки crash-окон и восстановления. Разделить запуск API, CLI-команды, фоновые процессы и инфраструктуру по пакетам.
2026-11-04 20:14:49 +03:00

214 lines
13 KiB
Python

#!/usr/bin/env python3
"""Grade 4: real fault injection on synthetic jobs; restores the base stack."""
import datetime as dt
import hashlib
import hmac
import json
import os
from pathlib import Path
import struct
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
from image_probe import jpeg_size
FAULTS = ROOT / '.memory/grade4-faults'
PREFIX = ['docker', 'compose', '--project-name', 'highload-lab5', '-f', 'compose.yml', '-f', 'infra/compose.faults.yml']
def compose(*args):
return subprocess.check_output(PREFIX + list(args), cwd=ROOT, text=True, stderr=subprocess.DEVNULL).strip()
def sql(query):
return compose('exec', '-T', 'db', 'sh', '-c', 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"', 'sh', query)
def until(fn, seconds=120):
deadline = time.monotonic()+seconds
last = None
while time.monotonic()<deadline:
try:
value = fn()
if value:
return value
except (Failure, OSError, subprocess.CalledProcessError) as exc:
last = type(exc).__name__
time.sleep(.3)
raise AssertionError('condition timed out; last exception: '+str(last))
def job_id(att):
return sql("SELECT id FROM attachment_jobs WHERE attachment_id='"+att['id']+"'")
def admin(*args):
return json.loads(compose('exec', '-T', 'dispatcher', './messenger', 'jobs', *args))
def arm(point, att):
marker = FAULTS/(point+'.'+att['id'])
marker.write_text('operator-only synthetic fault\n')
return Path(str(marker)+'.hit')
def s3(method, key='', query=None):
# Sign with the existing restricted API identity, never the MinIO root user.
values = dict(line.split('=',1) for line in (ROOT/'.env').read_text().splitlines() if '=' in line and not line.startswith('#'))
bucket = values.get('S3_BUCKET', 'messenger-attachments')
access = (ROOT/'secrets/s3_api_access_key_id').read_text().strip()
secret = (ROOT/'secrets/s3_api_secret_access_key').read_text().strip()
host = '127.0.0.1:9000'
uri = '/'+bucket+'/'+key
query_string = urllib.parse.urlencode(sorted((query or {}).items()), quote_via=urllib.parse.quote)
now = dt.datetime.now(dt.timezone.utc)
timestamp, day = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d')
payload = hashlib.sha256(b'').hexdigest()
headers = 'host:'+host+'\nx-amz-content-sha256:'+payload+'\nx-amz-date:'+timestamp+'\n'
signed = 'host;x-amz-content-sha256;x-amz-date'
canonical = '\n'.join([method,uri,query_string,headers,signed,payload])
scope = day+'/us-east-1/s3/aws4_request'
message = 'AWS4-HMAC-SHA256\n'+timestamp+'\n'+scope+'\n'+hashlib.sha256(canonical.encode()).hexdigest()
signing = ('AWS4'+secret).encode()
for part in (day,'us-east-1','s3','aws4_request'):
signing = hmac.new(signing,part.encode(),hashlib.sha256).digest()
signature = hmac.new(signing,message.encode(),hashlib.sha256).hexdigest()
request = urllib.request.Request('http://'+host+uri+('?' + query_string if query_string else ''),method=method,headers={
'x-amz-date':timestamp,'x-amz-content-sha256':payload,
'Authorization':'AWS4-HMAC-SHA256 Credential='+access+'/'+scope+', SignedHeaders='+signed+', Signature='+signature})
try:
response=urllib.request.urlopen(request,timeout=10)
except urllib.error.HTTPError as exc:
response=exc
with response:
return response.code, response.read()
def webp_size(data):
check(data[:4]==b'RIFF' and data[8:12]==b'WEBP','invalid WebP container')
offset=12
while offset+8<=len(data):
kind=data[offset:offset+4];length=int.from_bytes(data[offset+4:offset+8],'little');body=data[offset+8:offset+8+length]
check(kind not in (b'EXIF',b'XMP ',b'ICCP'),'optimized contains metadata')
if kind==b'VP8 ':
dimensions=(int.from_bytes(body[6:8],'little')&16383,int.from_bytes(body[8:10],'little')&16383)
if kind==b'VP8L':
bits=int.from_bytes(body[1:5],'little');dimensions=((bits&16383)+1,((bits>>14)&16383)+1)
if kind==b'VP8X':
dimensions=(int.from_bytes(body[4:7],'little')+1,int.from_bytes(body[7:10],'little')+1)
offset+=8+length+(length%2)
return dimensions
def large_png(w,h):
def chunk(kind,data):
return struct.pack('>I',len(data))+kind+data+struct.pack('>I',zlib.crc32(kind+data)&0xffffffff)
encoder=zlib.compressobj();pieces=[];row=b'\0'+b'\x80\x80\x80\xff'*w
for _ in range(h):
pieces.append(encoder.compress(row))
pieces.append(encoder.flush())
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',w,h,8,6,0,0,0))+chunk(b'IDAT',b''.join(pieces))+chunk(b'IEND',b'')
def main():
FAULTS.mkdir(parents=True, exist_ok=True);os.chmod(FAULTS,0o777)
compose('up','-d','--wait')
c=Client(sys.argv[1] if len(sys.argv)>1 else 'http://localhost:8085')
alice,bob,eve,chat=c.bootstrap()
results={}
def metadata(att):
return c.call('GET','/api/v1/attachments/'+att['id'],actor=alice,schema='Attachment')['json']
def ready(att):
return until(lambda: (value:=metadata(att))['status']=='ready' and value)
def verify(att, original):
value=ready(att)
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==original,'original changed')
check({v['name'] for v in value['variants']}=={'thumbnail','crop','optimized'},'variant set incomplete')
for variant in value['variants']:
path='/api/v1/attachments/'+att['id']+'/content?variant='+variant['name']
response=c.call('GET',path,actor=bob)
check(response['headers'].get_content_type()==variant['content_type'],'wrong variant MIME')
check(hashlib.sha256(response['bytes']).hexdigest()==variant['sha256'] and len(response['bytes'])==variant['size_bytes'],'wrong variant checksum')
dimensions=webp_size(response['bytes']) if variant['name']=='optimized' else jpeg_size(response['bytes'])
expected={'thumbnail':(256,160),'crop':(128,128),'optimized':(320,200)}[variant['name']]
check(dimensions==expected==(variant['width'],variant['height']),'actual variant dimensions differ')
c.call('GET',path,404,actor=eve)
until(lambda: s3('HEAD','staging/jobs/'+att['id'])[0]==404)
code,body=s3('GET',query={'list-type':'2','prefix':'variants/'+att['id']+'/'})
check(code==200 and len(ET.fromstring(body).findall('{*}Contents'))==3,'duplicate object set')
return value
try:
compose('stop','dispatcher')
att,original=c.attachment(alice,chat)
marker=arm('after_commit_before_publish',att)
check(sql("SELECT published_at IS NULL FROM attachment_jobs WHERE id='"+job_id(att)+"'")=='t','job already published')
compose('start','dispatcher');until(marker.exists)
verify(att,original)
results['commit_publish_crash']={'attachment_id':att['id'],'ready':True,'same_job_count':int(sql("SELECT count(*) FROM attachment_jobs WHERE attachment_id='"+att['id']+"'"))}
print('PASS: crash between COMMIT and publish recovers durable intent.',flush=True)
for point in ('after_object_before_commit','after_commit_before_ack'):
compose('stop','worker');att,original=c.attachment(alice,chat);marker=arm(point,att)
compose('start','worker');until(marker.exists)
value=verify(att,original)
attempts=int(sql("SELECT attempts FROM attachment_jobs WHERE id='"+job_id(att)+"'"))
check(attempts==(2 if point=='after_object_before_commit' else 1),'unexpected attempt count after crash')
results[point]={'attachment_id':att['id'],'attempts':attempts,'variant_count':len(value['variants'])}
print('PASS: '+point+' recovers without duplicate effects.',flush=True)
compose('stop','worker');att,original=c.attachment(alice,chat);marker=arm('transform_delay',att)
compose('start','worker')
restarts_before=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}}','highload-lab5-worker-1'],text=True)
started=time.monotonic();until(marker.exists);verify(att,original)
elapsed=time.monotonic()-started
check(subprocess.check_output(['docker','inspect','--format','{{.RestartCount}}','highload-lab5-worker-1'],text=True)==restarts_before,'worker restarted during child timeout')
check(elapsed<30,'child timeout did not bound processing')
check(compose('exec','-T','worker','sh','-c',"find /tmp -maxdepth 1 -name 'image-job-*' -print")=="",'child scratch leaked')
reasons=sql("SELECT error_code FROM attachment_job_attempts WHERE job_id='"+job_id(att)+"' AND error_code IS NOT NULL")
check('processing_timeout' in reasons,'child timeout was not recorded')
results['child_timeout']={'attachment_id':att['id'],'audit_reason':'processing_timeout','elapsed_seconds':round(elapsed,3),'worker_survived':True,'scratch_removed':True}
print('PASS: stuck image child is killed and retry succeeds.',flush=True)
compose('stop','worker');att,original=c.attachment(alice,chat);job=job_id(att)
sql("UPDATE attachment_jobs SET staging_key='staging/jobs/missing-test-object' WHERE id='"+job+"'")
compose('start','worker')
until(lambda: metadata(att)['status']=='failed',180)
failures=admin('list-failed');check(any(item['job_id']==job and item['attempts']==5 for item in failures),'failed registry missing job')
timing=json.loads(sql("SELECT json_agg(json_build_object('attempt',attempt,'delay',EXTRACT(EPOCH FROM next_attempt_at-finished_at))) FROM attachment_job_attempts WHERE job_id='"+job+"' AND next_attempt_at IS NOT NULL"))
for item in timing:
ceiling=min(30,2**item['attempt']);check(ceiling/2-.15<=float(item['delay'])<=ceiling+.15,'backoff not in jitter window')
sql("UPDATE attachment_jobs SET staging_key='staging/jobs/"+att['id']+"' WHERE id='"+job+"'")
check(s3('HEAD','staging/jobs/'+att['id'])[0]==200,'failed staging was removed before retention')
check(admin('replay',job,'Synthetic storage failure resolved')['changed'],'failed replay not accepted')
verify(att,original)
check(not admin('replay',job,'Already successful job')['changed'],'ready replay reopened job')
check(sql("SELECT count(*) FROM attachment_job_replays WHERE job_id='"+job+"'")=='1','replay not audited exactly once')
results['retry_and_replay']={'attachment_id':att['id'],'attempts_before_replay':5,'retry_delays':timing,'audited_replays':1}
print('PASS: bounded exponential jitter, failed registry, audited safe replay.',flush=True)
data=large_png(3000,3000)
att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=large.png',202,actor=alice,raw=data,headers={'Content-Type':'image/png'},schema='Attachment')['json']
until(lambda: metadata(att)['status']=='failed')
check(metadata(att)['error_code']=='invalid_image','worker pixel limit not enforced')
check(sql("SELECT attempts FROM attachment_jobs WHERE id='"+job_id(att)+"'")=='1','permanent error retried')
check(s3('HEAD','staging/jobs/'+att['id'])[0]==200,'failed source removed early')
sql("UPDATE attachment_job_failures SET failed_at=now()-interval '25 hours' WHERE job_id='"+job_id(att)+"'")
admin('cleanup-staging',job_id(att))
check(s3('HEAD','staging/jobs/'+att['id'])[0]==404,'expired failed source not removed')
results['permanent_error_retention']={'attachment_id':att['id'],'pixels':9000000,'attempts':1,'expired_staging_removed':True}
compose('stop','worker');queued,_=c.attachment(alice,chat)
check(not admin('cleanup-staging',job_id(queued))['changed'],'cleanup deleted queued work')
check(s3('HEAD','staging/jobs/'+queued['id'])[0]==200,'queued source missing')
compose('start','worker');ready(queued)
config=json.loads(subprocess.check_output(['docker','inspect','highload-lab5-worker-1'],text=True))[0]
check(config['HostConfig']['Memory']==512*1024*1024,'worker hard memory limit absent')
check('size=64m' in config['HostConfig']['Tmpfs']['/tmp'],'temporary disk not bounded')
results['resources']={'memory_limit_bytes':config['HostConfig']['Memory'],'prefetch':1,'image_timeout_seconds':8,'worker_pixel_limit':8000000,'queued_staging_protected':True}
(ROOT/'.memory/grade4-results.json').write_text(json.dumps(results,ensure_ascii=False,indent=2)+'\n')
print('PASS: grade 4 variants, fault windows, timeout, retry/replay, retention and resource limits.',flush=True)
finally:
# Remove test-only mounts and restore regular timeout/configuration.
subprocess.run(['docker','compose','--project-name','highload-lab5','up','-d','--wait'],cwd=ROOT,stdout=subprocess.DEVNULL,check=True)
if __name__=='__main__':
main()