forked from mtusi2026/mtusi2026lab5
215 lines
13 KiB
Python
215 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Grade 4: real fault injection on synthetic jobs; restores the base stack."""
|
|
import datetime as dt
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import xml.etree.ElementTree as ET
|
|
import zlib
|
|
from local_config import api_base_url
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, check
|
|
from image_probe import jpeg_size
|
|
|
|
FAULTS = ROOT / '.memory/grade4-faults'
|
|
PREFIX = ['docker', 'compose', '--project-name', 'highload-lab5', '-f', 'compose.yml', '-f', 'infra/compose.faults.yml']
|
|
|
|
def compose(*args):
|
|
return subprocess.check_output(PREFIX + list(args), cwd=ROOT, text=True, stderr=subprocess.DEVNULL).strip()
|
|
|
|
def sql(query):
|
|
return compose('exec', '-T', 'db', 'sh', '-c', 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"', 'sh', query)
|
|
|
|
def until(fn, seconds=120):
|
|
deadline = time.monotonic()+seconds
|
|
last = None
|
|
while time.monotonic()<deadline:
|
|
try:
|
|
value = fn()
|
|
if value:
|
|
return value
|
|
except (Failure, OSError, subprocess.CalledProcessError) as exc:
|
|
last = type(exc).__name__
|
|
time.sleep(.3)
|
|
raise AssertionError('condition timed out; last exception: '+str(last))
|
|
|
|
def job_id(att):
|
|
return sql("SELECT id FROM attachment_jobs WHERE attachment_id='"+att['id']+"'")
|
|
|
|
def admin(*args):
|
|
return json.loads(compose('exec', '-T', 'dispatcher', './messenger', 'jobs', *args))
|
|
|
|
def arm(point, att):
|
|
marker = FAULTS/(point+'.'+att['id'])
|
|
marker.write_text('operator-only synthetic fault\n')
|
|
return Path(str(marker)+'.hit')
|
|
|
|
def s3(method, key='', query=None):
|
|
# Sign with the existing restricted API identity, never the MinIO root user.
|
|
values = dict(line.split('=',1) for line in (ROOT/'.env').read_text().splitlines() if '=' in line and not line.startswith('#'))
|
|
bucket = values.get('S3_BUCKET', 'messenger-attachments')
|
|
access = (ROOT/'secrets/s3_api_access_key_id').read_text().strip()
|
|
secret = (ROOT/'secrets/s3_api_secret_access_key').read_text().strip()
|
|
host = '127.0.0.1:9000'
|
|
uri = '/'+bucket+'/'+key
|
|
query_string = urllib.parse.urlencode(sorted((query or {}).items()), quote_via=urllib.parse.quote)
|
|
now = dt.datetime.now(dt.timezone.utc)
|
|
timestamp, day = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d')
|
|
payload = hashlib.sha256(b'').hexdigest()
|
|
headers = 'host:'+host+'\nx-amz-content-sha256:'+payload+'\nx-amz-date:'+timestamp+'\n'
|
|
signed = 'host;x-amz-content-sha256;x-amz-date'
|
|
canonical = '\n'.join([method,uri,query_string,headers,signed,payload])
|
|
scope = day+'/us-east-1/s3/aws4_request'
|
|
message = 'AWS4-HMAC-SHA256\n'+timestamp+'\n'+scope+'\n'+hashlib.sha256(canonical.encode()).hexdigest()
|
|
signing = ('AWS4'+secret).encode()
|
|
for part in (day,'us-east-1','s3','aws4_request'):
|
|
signing = hmac.new(signing,part.encode(),hashlib.sha256).digest()
|
|
signature = hmac.new(signing,message.encode(),hashlib.sha256).hexdigest()
|
|
request = urllib.request.Request('http://'+host+uri+('?' + query_string if query_string else ''),method=method,headers={
|
|
'x-amz-date':timestamp,'x-amz-content-sha256':payload,
|
|
'Authorization':'AWS4-HMAC-SHA256 Credential='+access+'/'+scope+', SignedHeaders='+signed+', Signature='+signature})
|
|
try:
|
|
response=urllib.request.urlopen(request,timeout=10)
|
|
except urllib.error.HTTPError as exc:
|
|
response=exc
|
|
with response:
|
|
return response.code, response.read()
|
|
|
|
def webp_size(data):
|
|
check(data[:4]==b'RIFF' and data[8:12]==b'WEBP','invalid WebP container')
|
|
offset=12
|
|
while offset+8<=len(data):
|
|
kind=data[offset:offset+4];length=int.from_bytes(data[offset+4:offset+8],'little');body=data[offset+8:offset+8+length]
|
|
check(kind not in (b'EXIF',b'XMP ',b'ICCP'),'optimized contains metadata')
|
|
if kind==b'VP8 ':
|
|
dimensions=(int.from_bytes(body[6:8],'little')&16383,int.from_bytes(body[8:10],'little')&16383)
|
|
if kind==b'VP8L':
|
|
bits=int.from_bytes(body[1:5],'little');dimensions=((bits&16383)+1,((bits>>14)&16383)+1)
|
|
if kind==b'VP8X':
|
|
dimensions=(int.from_bytes(body[4:7],'little')+1,int.from_bytes(body[7:10],'little')+1)
|
|
offset+=8+length+(length%2)
|
|
return dimensions
|
|
|
|
def large_png(w,h):
|
|
def chunk(kind,data):
|
|
return struct.pack('>I',len(data))+kind+data+struct.pack('>I',zlib.crc32(kind+data)&0xffffffff)
|
|
encoder=zlib.compressobj();pieces=[];row=b'\0'+b'\x80\x80\x80\xff'*w
|
|
for _ in range(h):
|
|
pieces.append(encoder.compress(row))
|
|
pieces.append(encoder.flush())
|
|
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',w,h,8,6,0,0,0))+chunk(b'IDAT',b''.join(pieces))+chunk(b'IEND',b'')
|
|
|
|
def main():
|
|
FAULTS.mkdir(parents=True, exist_ok=True);os.chmod(FAULTS,0o777)
|
|
compose('up','-d','--wait')
|
|
c=Client(sys.argv[1] if len(sys.argv)>1 else api_base_url())
|
|
alice,bob,eve,chat=c.bootstrap()
|
|
results={}
|
|
def metadata(att):
|
|
return c.call('GET','/api/v1/attachments/'+att['id'],actor=alice,schema='Attachment')['json']
|
|
def ready(att):
|
|
return until(lambda: (value:=metadata(att))['status']=='ready' and value)
|
|
def verify(att, original):
|
|
value=ready(att)
|
|
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==original,'original changed')
|
|
check({v['name'] for v in value['variants']}=={'thumbnail','crop','optimized'},'variant set incomplete')
|
|
for variant in value['variants']:
|
|
path='/api/v1/attachments/'+att['id']+'/content?variant='+variant['name']
|
|
response=c.call('GET',path,actor=bob)
|
|
check(response['headers'].get_content_type()==variant['content_type'],'wrong variant MIME')
|
|
check(hashlib.sha256(response['bytes']).hexdigest()==variant['sha256'] and len(response['bytes'])==variant['size_bytes'],'wrong variant checksum')
|
|
dimensions=webp_size(response['bytes']) if variant['name']=='optimized' else jpeg_size(response['bytes'])
|
|
expected={'thumbnail':(256,160),'crop':(128,128),'optimized':(320,200)}[variant['name']]
|
|
check(dimensions==expected==(variant['width'],variant['height']),'actual variant dimensions differ')
|
|
c.call('GET',path,404,actor=eve)
|
|
until(lambda: s3('HEAD','staging/jobs/'+att['id'])[0]==404)
|
|
code,body=s3('GET',query={'list-type':'2','prefix':'variants/'+att['id']+'/'})
|
|
check(code==200 and len(ET.fromstring(body).findall('{*}Contents'))==3,'duplicate object set')
|
|
return value
|
|
try:
|
|
compose('stop','dispatcher')
|
|
att,original=c.attachment(alice,chat)
|
|
marker=arm('after_commit_before_publish',att)
|
|
check(sql("SELECT published_at IS NULL FROM attachment_jobs WHERE id='"+job_id(att)+"'")=='t','job already published')
|
|
compose('start','dispatcher');until(marker.exists)
|
|
verify(att,original)
|
|
results['commit_publish_crash']={'attachment_id':att['id'],'ready':True,'same_job_count':int(sql("SELECT count(*) FROM attachment_jobs WHERE attachment_id='"+att['id']+"'"))}
|
|
print('PASS: crash between COMMIT and publish recovers durable intent.',flush=True)
|
|
|
|
for point in ('after_object_before_commit','after_commit_before_ack'):
|
|
compose('stop','worker');att,original=c.attachment(alice,chat);marker=arm(point,att)
|
|
compose('start','worker');until(marker.exists)
|
|
value=verify(att,original)
|
|
attempts=int(sql("SELECT attempts FROM attachment_jobs WHERE id='"+job_id(att)+"'"))
|
|
check(attempts==(2 if point=='after_object_before_commit' else 1),'unexpected attempt count after crash')
|
|
results[point]={'attachment_id':att['id'],'attempts':attempts,'variant_count':len(value['variants'])}
|
|
print('PASS: '+point+' recovers without duplicate effects.',flush=True)
|
|
|
|
compose('stop','worker');att,original=c.attachment(alice,chat);marker=arm('transform_delay',att)
|
|
compose('start','worker')
|
|
restarts_before=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}}','highload-lab5-worker-1'],text=True)
|
|
started=time.monotonic();until(marker.exists);verify(att,original)
|
|
elapsed=time.monotonic()-started
|
|
check(subprocess.check_output(['docker','inspect','--format','{{.RestartCount}}','highload-lab5-worker-1'],text=True)==restarts_before,'worker restarted during child timeout')
|
|
check(elapsed<30,'child timeout did not bound processing')
|
|
check(compose('exec','-T','worker','sh','-c',"find /tmp -maxdepth 1 -name 'image-job-*' -print")=="",'child scratch leaked')
|
|
reasons=sql("SELECT error_code FROM attachment_job_attempts WHERE job_id='"+job_id(att)+"' AND error_code IS NOT NULL")
|
|
check('processing_timeout' in reasons,'child timeout was not recorded')
|
|
results['child_timeout']={'attachment_id':att['id'],'audit_reason':'processing_timeout','elapsed_seconds':round(elapsed,3),'worker_survived':True,'scratch_removed':True}
|
|
print('PASS: stuck image child is killed and retry succeeds.',flush=True)
|
|
|
|
compose('stop','worker');att,original=c.attachment(alice,chat);job=job_id(att)
|
|
sql("UPDATE attachment_jobs SET staging_key='staging/jobs/missing-test-object' WHERE id='"+job+"'")
|
|
compose('start','worker')
|
|
until(lambda: metadata(att)['status']=='failed',180)
|
|
failures=admin('list-failed');check(any(item['job_id']==job and item['attempts']==5 for item in failures),'failed registry missing job')
|
|
timing=json.loads(sql("SELECT json_agg(json_build_object('attempt',attempt,'delay',EXTRACT(EPOCH FROM next_attempt_at-finished_at))) FROM attachment_job_attempts WHERE job_id='"+job+"' AND next_attempt_at IS NOT NULL"))
|
|
for item in timing:
|
|
ceiling=min(30,2**item['attempt']);check(ceiling/2-.15<=float(item['delay'])<=ceiling+.15,'backoff not in jitter window')
|
|
sql("UPDATE attachment_jobs SET staging_key='staging/jobs/"+att['id']+"' WHERE id='"+job+"'")
|
|
check(s3('HEAD','staging/jobs/'+att['id'])[0]==200,'failed staging was removed before retention')
|
|
check(admin('replay',job,'Synthetic storage failure resolved')['changed'],'failed replay not accepted')
|
|
verify(att,original)
|
|
check(not admin('replay',job,'Already successful job')['changed'],'ready replay reopened job')
|
|
check(sql("SELECT count(*) FROM attachment_job_replays WHERE job_id='"+job+"'")=='1','replay not audited exactly once')
|
|
results['retry_and_replay']={'attachment_id':att['id'],'attempts_before_replay':5,'retry_delays':timing,'audited_replays':1}
|
|
print('PASS: bounded exponential jitter, failed registry, audited safe replay.',flush=True)
|
|
|
|
data=large_png(3000,3000)
|
|
att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=large.png',202,actor=alice,raw=data,headers={'Content-Type':'image/png'},schema='Attachment')['json']
|
|
until(lambda: metadata(att)['status']=='failed')
|
|
check(metadata(att)['error_code']=='invalid_image','worker pixel limit not enforced')
|
|
check(sql("SELECT attempts FROM attachment_jobs WHERE id='"+job_id(att)+"'")=='1','permanent error retried')
|
|
check(s3('HEAD','staging/jobs/'+att['id'])[0]==200,'failed source removed early')
|
|
sql("UPDATE attachment_job_failures SET failed_at=now()-interval '25 hours' WHERE job_id='"+job_id(att)+"'")
|
|
admin('cleanup-staging',job_id(att))
|
|
check(s3('HEAD','staging/jobs/'+att['id'])[0]==404,'expired failed source not removed')
|
|
results['permanent_error_retention']={'attachment_id':att['id'],'pixels':9000000,'attempts':1,'expired_staging_removed':True}
|
|
compose('stop','worker');queued,_=c.attachment(alice,chat)
|
|
check(not admin('cleanup-staging',job_id(queued))['changed'],'cleanup deleted queued work')
|
|
check(s3('HEAD','staging/jobs/'+queued['id'])[0]==200,'queued source missing')
|
|
compose('start','worker');ready(queued)
|
|
config=json.loads(subprocess.check_output(['docker','inspect','highload-lab5-worker-1'],text=True))[0]
|
|
check(config['HostConfig']['Memory']==512*1024*1024,'worker hard memory limit absent')
|
|
check('size=64m' in config['HostConfig']['Tmpfs']['/tmp'],'temporary disk not bounded')
|
|
results['resources']={'memory_limit_bytes':config['HostConfig']['Memory'],'prefetch':1,'image_timeout_seconds':8,'worker_pixel_limit':8000000,'queued_staging_protected':True}
|
|
(ROOT/'.memory/grade4-results.json').write_text(json.dumps(results,ensure_ascii=False,indent=2)+'\n')
|
|
print('PASS: grade 4 variants, fault windows, timeout, retry/replay, retention and resource limits.',flush=True)
|
|
finally:
|
|
# Remove test-only mounts and restore regular timeout/configuration.
|
|
subprocess.run(['docker','compose','--project-name','highload-lab5','up','-d','--wait'],cwd=ROOT,stdout=subprocess.DEVNULL,check=True)
|
|
|
|
if __name__=='__main__':
|
|
main()
|