#!/usr/bin/env python3 """Grade 4: real fault injection on synthetic jobs; restores the base stack.""" import datetime as dt import hashlib import hmac import json import os from pathlib import Path import struct import subprocess import sys import time import urllib.error import urllib.parse import urllib.request import xml.etree.ElementTree as ET import zlib from local_config import api_base_url ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure, check from image_probe import jpeg_size FAULTS = ROOT / '.memory/grade4-faults' PREFIX = ['docker', 'compose', '--project-name', 'highload-lab5', '-f', 'compose.yml', '-f', 'infra/compose.faults.yml'] def compose(*args): return subprocess.check_output(PREFIX + list(args), cwd=ROOT, text=True, stderr=subprocess.DEVNULL).strip() def sql(query): return compose('exec', '-T', 'db', 'sh', '-c', 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"', 'sh', query) def until(fn, seconds=120): deadline = time.monotonic()+seconds last = None while time.monotonic()>14)&16383)+1) if kind==b'VP8X': dimensions=(int.from_bytes(body[4:7],'little')+1,int.from_bytes(body[7:10],'little')+1) offset+=8+length+(length%2) return dimensions def large_png(w,h): def chunk(kind,data): return struct.pack('>I',len(data))+kind+data+struct.pack('>I',zlib.crc32(kind+data)&0xffffffff) encoder=zlib.compressobj();pieces=[];row=b'\0'+b'\x80\x80\x80\xff'*w for _ in range(h): pieces.append(encoder.compress(row)) pieces.append(encoder.flush()) return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',w,h,8,6,0,0,0))+chunk(b'IDAT',b''.join(pieces))+chunk(b'IEND',b'') def main(): FAULTS.mkdir(parents=True, exist_ok=True);os.chmod(FAULTS,0o777) compose('up','-d','--wait') c=Client(sys.argv[1] if len(sys.argv)>1 else api_base_url()) alice,bob,eve,chat=c.bootstrap() results={} def metadata(att): return c.call('GET','/api/v1/attachments/'+att['id'],actor=alice,schema='Attachment')['json'] def ready(att): return until(lambda: (value:=metadata(att))['status']=='ready' and value) def verify(att, original): value=ready(att) check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==original,'original changed') check({v['name'] for v in value['variants']}=={'thumbnail','crop','optimized'},'variant set incomplete') for variant in value['variants']: path='/api/v1/attachments/'+att['id']+'/content?variant='+variant['name'] response=c.call('GET',path,actor=bob) check(response['headers'].get_content_type()==variant['content_type'],'wrong variant MIME') check(hashlib.sha256(response['bytes']).hexdigest()==variant['sha256'] and len(response['bytes'])==variant['size_bytes'],'wrong variant checksum') dimensions=webp_size(response['bytes']) if variant['name']=='optimized' else jpeg_size(response['bytes']) expected={'thumbnail':(256,160),'crop':(128,128),'optimized':(320,200)}[variant['name']] check(dimensions==expected==(variant['width'],variant['height']),'actual variant dimensions differ') c.call('GET',path,404,actor=eve) until(lambda: s3('HEAD','staging/jobs/'+att['id'])[0]==404) code,body=s3('GET',query={'list-type':'2','prefix':'variants/'+att['id']+'/'}) check(code==200 and len(ET.fromstring(body).findall('{*}Contents'))==3,'duplicate object set') return value try: compose('stop','dispatcher') att,original=c.attachment(alice,chat) marker=arm('after_commit_before_publish',att) check(sql("SELECT published_at IS NULL FROM attachment_jobs WHERE id='"+job_id(att)+"'")=='t','job already published') compose('start','dispatcher');until(marker.exists) verify(att,original) results['commit_publish_crash']={'attachment_id':att['id'],'ready':True,'same_job_count':int(sql("SELECT count(*) FROM attachment_jobs WHERE attachment_id='"+att['id']+"'"))} print('PASS: crash between COMMIT and publish recovers durable intent.',flush=True) for point in ('after_object_before_commit','after_commit_before_ack'): compose('stop','worker');att,original=c.attachment(alice,chat);marker=arm(point,att) compose('start','worker');until(marker.exists) value=verify(att,original) attempts=int(sql("SELECT attempts FROM attachment_jobs WHERE id='"+job_id(att)+"'")) check(attempts==(2 if point=='after_object_before_commit' else 1),'unexpected attempt count after crash') results[point]={'attachment_id':att['id'],'attempts':attempts,'variant_count':len(value['variants'])} print('PASS: '+point+' recovers without duplicate effects.',flush=True) compose('stop','worker');att,original=c.attachment(alice,chat);marker=arm('transform_delay',att) compose('start','worker') restarts_before=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}}','highload-lab5-worker-1'],text=True) started=time.monotonic();until(marker.exists);verify(att,original) elapsed=time.monotonic()-started check(subprocess.check_output(['docker','inspect','--format','{{.RestartCount}}','highload-lab5-worker-1'],text=True)==restarts_before,'worker restarted during child timeout') check(elapsed<30,'child timeout did not bound processing') check(compose('exec','-T','worker','sh','-c',"find /tmp -maxdepth 1 -name 'image-job-*' -print")=="",'child scratch leaked') reasons=sql("SELECT error_code FROM attachment_job_attempts WHERE job_id='"+job_id(att)+"' AND error_code IS NOT NULL") check('processing_timeout' in reasons,'child timeout was not recorded') results['child_timeout']={'attachment_id':att['id'],'audit_reason':'processing_timeout','elapsed_seconds':round(elapsed,3),'worker_survived':True,'scratch_removed':True} print('PASS: stuck image child is killed and retry succeeds.',flush=True) compose('stop','worker');att,original=c.attachment(alice,chat);job=job_id(att) sql("UPDATE attachment_jobs SET staging_key='staging/jobs/missing-test-object' WHERE id='"+job+"'") compose('start','worker') until(lambda: metadata(att)['status']=='failed',180) failures=admin('list-failed');check(any(item['job_id']==job and item['attempts']==5 for item in failures),'failed registry missing job') timing=json.loads(sql("SELECT json_agg(json_build_object('attempt',attempt,'delay',EXTRACT(EPOCH FROM next_attempt_at-finished_at))) FROM attachment_job_attempts WHERE job_id='"+job+"' AND next_attempt_at IS NOT NULL")) for item in timing: ceiling=min(30,2**item['attempt']);check(ceiling/2-.15<=float(item['delay'])<=ceiling+.15,'backoff not in jitter window') sql("UPDATE attachment_jobs SET staging_key='staging/jobs/"+att['id']+"' WHERE id='"+job+"'") check(s3('HEAD','staging/jobs/'+att['id'])[0]==200,'failed staging was removed before retention') check(admin('replay',job,'Synthetic storage failure resolved')['changed'],'failed replay not accepted') verify(att,original) check(not admin('replay',job,'Already successful job')['changed'],'ready replay reopened job') check(sql("SELECT count(*) FROM attachment_job_replays WHERE job_id='"+job+"'")=='1','replay not audited exactly once') results['retry_and_replay']={'attachment_id':att['id'],'attempts_before_replay':5,'retry_delays':timing,'audited_replays':1} print('PASS: bounded exponential jitter, failed registry, audited safe replay.',flush=True) data=large_png(3000,3000) att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=large.png',202,actor=alice,raw=data,headers={'Content-Type':'image/png'},schema='Attachment')['json'] until(lambda: metadata(att)['status']=='failed') check(metadata(att)['error_code']=='invalid_image','worker pixel limit not enforced') check(sql("SELECT attempts FROM attachment_jobs WHERE id='"+job_id(att)+"'")=='1','permanent error retried') check(s3('HEAD','staging/jobs/'+att['id'])[0]==200,'failed source removed early') sql("UPDATE attachment_job_failures SET failed_at=now()-interval '25 hours' WHERE job_id='"+job_id(att)+"'") admin('cleanup-staging',job_id(att)) check(s3('HEAD','staging/jobs/'+att['id'])[0]==404,'expired failed source not removed') results['permanent_error_retention']={'attachment_id':att['id'],'pixels':9000000,'attempts':1,'expired_staging_removed':True} compose('stop','worker');queued,_=c.attachment(alice,chat) check(not admin('cleanup-staging',job_id(queued))['changed'],'cleanup deleted queued work') check(s3('HEAD','staging/jobs/'+queued['id'])[0]==200,'queued source missing') compose('start','worker');ready(queued) config=json.loads(subprocess.check_output(['docker','inspect','highload-lab5-worker-1'],text=True))[0] check(config['HostConfig']['Memory']==512*1024*1024,'worker hard memory limit absent') check('size=64m' in config['HostConfig']['Tmpfs']['/tmp'],'temporary disk not bounded') results['resources']={'memory_limit_bytes':config['HostConfig']['Memory'],'prefetch':1,'image_timeout_seconds':8,'worker_pixel_limit':8000000,'queued_staging_protected':True} (ROOT/'.memory/grade4-results.json').write_text(json.dumps(results,ensure_ascii=False,indent=2)+'\n') print('PASS: grade 4 variants, fault windows, timeout, retry/replay, retention and resource limits.',flush=True) finally: # Remove test-only mounts and restore regular timeout/configuration. subprocess.run(['docker','compose','--project-name','highload-lab5','up','-d','--wait'],cwd=ROOT,stdout=subprocess.DEVNULL,check=True) if __name__=='__main__': main()