forked from mtusi2026/mtusi2026lab5
237 lines
14 KiB
Python
237 lines
14 KiB
Python
"""Lab5 level4 acceptance/fault checks. Operates only on local synthetic Compose.
|
|
|
|
Does not change the public tests. Stops services temporarily; restores them in finally.
|
|
Only IDs, assertions and timings are saved; no tokens/credentials/signed URLs.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, check
|
|
from image_probe import jpeg_size
|
|
|
|
|
|
def compose(*args, input=None, expected=0, timeout=120):
|
|
p = subprocess.run(['docker', 'compose', *args], cwd=ROOT, input=input,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
|
|
check(p.returncode == expected, 'compose ' + ' '.join(args[:3]) + ': exit ' + str(p.returncode) + ' ' + p.stderr.decode(errors='replace')[-1500:])
|
|
return p.stdout
|
|
|
|
|
|
def sql(query):
|
|
return compose('exec', '-T', 'postgres', 'sh', '-c',
|
|
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -v ON_ERROR_STOP=1',
|
|
input=query.encode()).decode().strip()
|
|
|
|
|
|
def eventually(predicate, seconds=90):
|
|
end = time.monotonic() + seconds
|
|
while time.monotonic() < end:
|
|
if predicate():
|
|
return
|
|
time.sleep(0.4)
|
|
raise Failure('condition did not recover within ' + str(seconds) + ' seconds')
|
|
|
|
|
|
def jid(att):
|
|
return sql(f"SELECT id FROM attachment_jobs WHERE attachment_id='{uuid.UUID(att['id'])}';")
|
|
|
|
|
|
def status(att):
|
|
return sql(f"SELECT status FROM attachment_jobs WHERE attachment_id='{uuid.UUID(att['id'])}';")
|
|
|
|
|
|
def envelope(job):
|
|
return sql(f"SELECT json_build_object('schema_version',1,'type','attachment.process.v1','job_id',id,'attachment_id',attachment_id,'pipeline_version',1,'attempt',attempt,'created_at',created_at) FROM attachment_jobs WHERE id='{uuid.UUID(job)}';")
|
|
|
|
|
|
def duplicate(job):
|
|
compose('exec', '-T', 'broker', 'redis-cli', 'XADD', 'attachment-jobs-v1', '*', 'payload', envelope(job))
|
|
|
|
|
|
def objects(prefix):
|
|
raw=compose('run','--rm','--no-deps','-T','--entrypoint','/bin/sh','s3-init','-c',
|
|
'export AWS_ACCESS_KEY_ID="$(cat /run/secrets/s3_admin_access_key)"; '
|
|
'export AWS_SECRET_ACCESS_KEY="$(cat /run/secrets/s3_admin_secret_key)"; '
|
|
'export AWS_DEFAULT_REGION=us-east-1; '
|
|
'exec aws --endpoint-url http://s3-proxy:8333 s3api list-objects-v2 --bucket attachments --prefix "$1"',
|
|
'sh',prefix)
|
|
return json.loads(raw).get('Contents',[])
|
|
|
|
|
|
def verify_images(c, actor, att, original, thumb_size=(256, 160)):
|
|
ready = c.wait_ready(actor, att, 90)
|
|
check(ready['error_code'] is None, 'ready error_code')
|
|
variants = {v['name']: v for v in ready['variants']}
|
|
check(set(variants) == {'thumbnail', 'crop', 'optimized'}, 'exactly 3 distinct variants')
|
|
raw = c.call('GET', f"/api/v1/attachments/{att['id']}/content", actor=actor)['bytes']
|
|
check(raw == original, 'original changed')
|
|
for name, v in variants.items():
|
|
data = c.call('GET', f"/api/v1/attachments/{att['id']}/content?variant={name}", actor=actor)['bytes']
|
|
check(hashlib.sha256(data).hexdigest() == v['sha256'] and len(data) == v['size_bytes'], name + ' checksum')
|
|
if name != 'optimized':
|
|
dims = jpeg_size(data)
|
|
check(dims == ((128, 128) if name == 'crop' else thumb_size), name + ' real dimensions')
|
|
else:
|
|
check(data[:4] == b'RIFF' and data[8:12] == b'WEBP', 'real WebP')
|
|
decoded = json.loads(compose('exec', '-T', 'worker', 'python3', '-c',
|
|
"from PIL import Image; import sys,io,json; i=Image.open(io.BytesIO(sys.stdin.buffer.read())); i.load(); print(json.dumps({'size':i.size,'exif':bool(i.getexif()),'icc':bool(i.info.get('icc_profile')),'corner':i.convert('RGB').getpixel((0,0))}))", input=data))
|
|
check(decoded['size'] == [v['width'], v['height']], name + ' decoded size')
|
|
check(not decoded['exif'] and not decoded['icc'], name + ' metadata stripped')
|
|
return ready
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--base-url', default='http://localhost:8080')
|
|
parser.add_argument('--output', default='evidence/lab5-pipeline.json')
|
|
parser.add_argument('--resource-only', action='store_true')
|
|
args = parser.parse_args()
|
|
c = Client(args.base_url, timeout=45)
|
|
results = []
|
|
|
|
def record(name, start, **observed):
|
|
row = dict(scenario=name, result='PASS', seconds=round(time.monotonic()-start, 3), **observed)
|
|
results.append(row)
|
|
print('PASS', name, json.dumps(observed, ensure_ascii=False), flush=True)
|
|
|
|
def health():
|
|
try:
|
|
c.call('GET', '/health/ready')
|
|
return True
|
|
except Failure:
|
|
return False
|
|
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
fixture = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
try:
|
|
if not args.resource_only:
|
|
start = time.monotonic()
|
|
att, raw = c.attachment(alice, chat)
|
|
ready = verify_images(c, bob, att, raw)
|
|
c.call('GET', f"/api/v1/attachments/{att['id']}/content?variant=crop", 404, actor=eve)
|
|
eventually(lambda: sql(f"SELECT staging_key IS NULL FROM attachments WHERE id='{att['id']}';") == 't')
|
|
check(not objects('staging/'+att['id']),'staging bytes not removed')
|
|
record('variants_checksums_authorization_staging_cleanup', start, attachment_id=att['id'], variants=[(v['name'],v['width'],v['height']) for v in ready['variants']])
|
|
|
|
start = time.monotonic()
|
|
generators = [
|
|
("from PIL import Image; import sys; Image.new('RGBA',(16,8),(255,0,0,0)).save(sys.stdout.buffer,'PNG')", 'image/png', (16,8), True),
|
|
("from PIL import Image; import sys; i=Image.new('RGB',(320,200),'blue'); e=i.getexif(); e[274]=6; i.save(sys.stdout.buffer,'JPEG',exif=e)", 'image/jpeg', (160,256), False),
|
|
]
|
|
for code, content_type, dims, white in generators:
|
|
raw = compose('exec','-T','worker','python3','-c',code)
|
|
a = c.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=synthetic.png",202,actor=alice,raw=raw,headers={'Content-Type':content_type},schema='Attachment')['json']
|
|
verify_images(c,bob,a,raw,dims)
|
|
if white:
|
|
thumb = c.call('GET',f"/api/v1/attachments/{a['id']}/content?variant=thumbnail",actor=bob)['bytes']
|
|
pixel=json.loads(compose('exec','-T','worker','python3','-c',"from PIL import Image; import io,json,sys; print(json.dumps(Image.open(io.BytesIO(sys.stdin.buffer.read())).getpixel((0,0))))",input=thumb))
|
|
check(min(pixel)>250,'alpha must become white')
|
|
record('alpha_small_image_exif_orientation',start)
|
|
|
|
start = time.monotonic()
|
|
path=f"/api/v1/chats/{chat['id']}/attachments?filename=bad.png"
|
|
c.call('POST',path,415,actor=alice,raw=fixture[:-20],headers={'Content-Type':'image/png'})
|
|
c.call('POST',path,413,actor=alice,raw=b'x'*(10*1024*1024+1),headers={'Content-Type':'image/png'})
|
|
import struct, zlib
|
|
ihdr=struct.pack('>IIBBBBB',8000,8000,8,2,0,0,0)
|
|
chunk=b'IHDR'+ihdr
|
|
bomb=b'\x89PNG\r\n\x1a\n'+struct.pack('>I',13)+chunk+struct.pack('>I',zlib.crc32(chunk)&0xffffffff)
|
|
c.call('POST',path,413,actor=alice,raw=bomb,headers={'Content-Type':'image/png'})
|
|
record('invalid_bytes_pixels_limits',start)
|
|
|
|
start = time.monotonic()
|
|
compose('stop','worker','outbox')
|
|
att, raw = c.attachment(alice, chat)
|
|
job = jid(att)
|
|
c.message(alice, chat, '', attachments=[att['id']])
|
|
c.call('GET',f"/api/v1/attachments/{att['id']}/content",409,actor=bob)
|
|
c.message(alice,chat,'text works while worker is stopped')
|
|
compose('run','--rm','--no-deps','-T','-e','PIPELINE_FAULT=before_publish','outbox',expected=86)
|
|
check(sql(f"SELECT count(*) FROM attachment_outbox WHERE job_id='{job}' AND published_at IS NULL;")=='1','outbox intent survived real process crash')
|
|
compose('run','--rm','--no-deps','-T','-e','PIPELINE_FAULT=after_publish','outbox',expected=86)
|
|
check(sql(f"SELECT count(*) FROM attachment_outbox WHERE job_id='{job}' AND published_at IS NULL;")=='1','publish-before-mark crash lost outbox')
|
|
compose('start','outbox')
|
|
eventually(lambda: sql(f"SELECT count(*) FROM attachment_outbox WHERE job_id='{job}' AND published_at IS NOT NULL;")=='1')
|
|
count=compose('exec','-T','broker','redis-cli','XLEN','attachment-jobs-v1').strip()
|
|
before=compose('ps','-q','api').decode().split()
|
|
compose('kill','-s','SIGKILL','broker')
|
|
compose('start','broker')
|
|
eventually(lambda: compose('exec','-T','broker','redis-cli','ping').strip()==b'PONG')
|
|
check(compose('exec','-T','broker','redis-cli','XLEN','attachment-jobs-v1').strip()==count,'broker lost durable stream entries')
|
|
compose('restart','api')
|
|
eventually(health)
|
|
compose('start','worker')
|
|
verify_images(c,bob,att,raw)
|
|
record('commit_publish_crash_worker_off_api_restart_broker_sigkill',start,attachment_id=att['id'],job_id=job,api_containers=before,stream_entries=int(count))
|
|
|
|
start = time.monotonic()
|
|
compose('stop','worker')
|
|
att, raw = c.attachment(alice,chat);job=jid(att)
|
|
compose('run','--rm','--no-deps','-T','-e','PIPELINE_FAULT=after_object','worker',expected=86)
|
|
check(status(att)=='processing','worker must crash after object before DB result')
|
|
key=sql(f"SELECT object_key FROM attachments WHERE id='{att['id']}';")
|
|
check(len(objects(key))==1,'crash boundary must contain exactly the persisted original')
|
|
c.call('GET',f"/api/v1/attachments/{att['id']}/content",409,actor=bob)
|
|
compose('start','worker')
|
|
ready=verify_images(c,bob,att,raw)
|
|
attempt=int(sql(f"SELECT attempt FROM attachment_jobs WHERE id='{job}';"))
|
|
check(attempt>=2,'expired lease not recovered')
|
|
duplicate(job);time.sleep(2)
|
|
check(int(sql(f"SELECT attempt FROM attachment_jobs WHERE id='{job}';"))==attempt,'ready duplicate executed again')
|
|
check(c.call('GET',f"/api/v1/attachments/{att['id']}",actor=bob,schema='Attachment')['json']==ready,'duplicate changed completed metadata')
|
|
check(len(objects(key))==4,'duplicate created a second set of S3 objects')
|
|
record('crash_after_s3_before_ack_lease_recovery_duplicate',start,attachment_id=att['id'],job_id=job,attempts=attempt)
|
|
|
|
start=time.monotonic()
|
|
compose('stop','worker')
|
|
att,raw=c.attachment(alice,chat);job=jid(att)
|
|
compose('run','--rm','--no-deps','-T','-e','S3_ENDPOINT=http://127.0.0.1:1','-e','JOB_MAX_ATTEMPTS=2','-e','PIPELINE_FAULT=before_ack','worker',expected=86)
|
|
check(status(att)=='queued','transient error must retry')
|
|
delay=float(sql(f"SELECT extract(epoch from(next_attempt_at-updated_at)) FROM attachment_jobs WHERE id='{job}';"))
|
|
check(0.45<=delay<=1.1,'first retry backoff is outside jitter bounds')
|
|
time.sleep(1.1)
|
|
compose('run','--rm','--no-deps','-T','-e','S3_ENDPOINT=http://127.0.0.1:1','-e','JOB_MAX_ATTEMPTS=2','-e','PIPELINE_FAULT=before_ack','worker',expected=86)
|
|
check(status(att)=='failed','attempts must be bounded')
|
|
failed=c.call('GET',f"/api/v1/attachments/{att['id']}",actor=bob,schema='Attachment')['json']
|
|
check(failed['error_code']=='retry_exhausted','safe exhausted error_code')
|
|
c.call('POST',f"/api/v1/chats/{chat['id']}/messages",409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
|
|
registry=json.loads(compose('run','--rm','--no-deps','-T','worker','./worker','-mode=failed'))
|
|
check(any(row['job_id']==job for row in registry),'failed registry must expose job')
|
|
compose('run','--rm','--no-deps','-T','worker','./worker','-mode=replay','-job='+job,'-reason=lab5-recovered-storage')
|
|
compose('start','worker')
|
|
verify_images(c,bob,att,raw)
|
|
check(sql(f"SELECT count(*) FROM job_replays WHERE job_id='{job}';")=='1','replay audit missing')
|
|
compose('run','--rm','--no-deps','-T','worker','./worker','-mode=replay','-job='+job,'-reason=must-refuse-ready',expected=1)
|
|
record('retry_jitter_exhaustion_failed_registry_audited_replay',start,attachment_id=att['id'],job_id=job,first_backoff_seconds=round(delay,3))
|
|
|
|
start=time.monotonic()
|
|
raw=compose('exec','-T','worker','python3','-c',"from PIL import Image; import sys; Image.new('RGB',(3000,3000),'blue').save(sys.stdout.buffer,'PNG')")
|
|
compose('stop','worker')
|
|
att=c.call('POST',f"/api/v1/chats/{chat['id']}/attachments?filename=memory.png",202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')['json'];job=jid(att)
|
|
compose('run','--rm','--no-deps','-T','-e','IMAGE_MEMORY_MB=64','-e','PIPELINE_FAULT=before_ack','worker',expected=86)
|
|
check(status(att)=='failed','memory limit must be a permanent controlled failure')
|
|
check(sql(f"SELECT error_code FROM attachment_jobs WHERE id='{job}';")=='image_resource_limit','memory failure code')
|
|
check(sql(f"SELECT attempt FROM attachment_jobs WHERE id='{job}';")=='1','permanent failure was retried')
|
|
compose('run','--rm','--no-deps','-T','worker','./worker','-mode=replay','-job='+job,'-reason=restore-normal-memory-limit')
|
|
compose('start','worker')
|
|
verify_images(c,bob,att,raw,(256,256))
|
|
record('worker_memory_limit_permanent_failure_replay',start,attachment_id=att['id'],job_id=job,decoder_limit_mb=64,pixels=9000000)
|
|
finally:
|
|
compose('start','broker','api','outbox','worker')
|
|
output=ROOT/args.output
|
|
output.parent.mkdir(parents=True,exist_ok=True)
|
|
output.write_text(json.dumps({'date':'2026-11-10','scenarios':results},ensure_ascii=False,indent=2)+'\n')
|
|
print('PASS all lab5 level4 scenarios',flush=True)
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|