Files
mtusi2026lab5/scripts/check_pipeline.py
2026-11-12 17:28:28 +03:00

237 lines
14 KiB
Python

"""Lab5 level4 acceptance/fault checks. Operates only on local synthetic Compose.
Does not change the public tests. Stops services temporarily; restores them in finally.
Only IDs, assertions and timings are saved; no tokens/credentials/signed URLs.
"""
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
from image_probe import jpeg_size
def compose(*args, input=None, expected=0, timeout=120):
p = subprocess.run(['docker', 'compose', *args], cwd=ROOT, input=input,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
check(p.returncode == expected, 'compose ' + ' '.join(args[:3]) + ': exit ' + str(p.returncode) + ' ' + p.stderr.decode(errors='replace')[-1500:])
return p.stdout
def sql(query):
return compose('exec', '-T', 'postgres', 'sh', '-c',
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -v ON_ERROR_STOP=1',
input=query.encode()).decode().strip()
def eventually(predicate, seconds=90):
end = time.monotonic() + seconds
while time.monotonic() < end:
if predicate():
return
time.sleep(0.4)
raise Failure('condition did not recover within ' + str(seconds) + ' seconds')
def jid(att):
return sql(f"SELECT id FROM attachment_jobs WHERE attachment_id='{uuid.UUID(att['id'])}';")
def status(att):
return sql(f"SELECT status FROM attachment_jobs WHERE attachment_id='{uuid.UUID(att['id'])}';")
def envelope(job):
return sql(f"SELECT json_build_object('schema_version',1,'type','attachment.process.v1','job_id',id,'attachment_id',attachment_id,'pipeline_version',1,'attempt',attempt,'created_at',created_at) FROM attachment_jobs WHERE id='{uuid.UUID(job)}';")
def duplicate(job):
compose('exec', '-T', 'broker', 'redis-cli', 'XADD', 'attachment-jobs-v1', '*', 'payload', envelope(job))
def objects(prefix):
raw=compose('run','--rm','--no-deps','-T','--entrypoint','/bin/sh','s3-init','-c',
'export AWS_ACCESS_KEY_ID="$(cat /run/secrets/s3_admin_access_key)"; '
'export AWS_SECRET_ACCESS_KEY="$(cat /run/secrets/s3_admin_secret_key)"; '
'export AWS_DEFAULT_REGION=us-east-1; '
'exec aws --endpoint-url http://s3-proxy:8333 s3api list-objects-v2 --bucket attachments --prefix "$1"',
'sh',prefix)
return json.loads(raw).get('Contents',[])
def verify_images(c, actor, att, original, thumb_size=(256, 160)):
ready = c.wait_ready(actor, att, 90)
check(ready['error_code'] is None, 'ready error_code')
variants = {v['name']: v for v in ready['variants']}
check(set(variants) == {'thumbnail', 'crop', 'optimized'}, 'exactly 3 distinct variants')
raw = c.call('GET', f"/api/v1/attachments/{att['id']}/content", actor=actor)['bytes']
check(raw == original, 'original changed')
for name, v in variants.items():
data = c.call('GET', f"/api/v1/attachments/{att['id']}/content?variant={name}", actor=actor)['bytes']
check(hashlib.sha256(data).hexdigest() == v['sha256'] and len(data) == v['size_bytes'], name + ' checksum')
if name != 'optimized':
dims = jpeg_size(data)
check(dims == ((128, 128) if name == 'crop' else thumb_size), name + ' real dimensions')
else:
check(data[:4] == b'RIFF' and data[8:12] == b'WEBP', 'real WebP')
decoded = json.loads(compose('exec', '-T', 'worker', 'python3', '-c',
"from PIL import Image; import sys,io,json; i=Image.open(io.BytesIO(sys.stdin.buffer.read())); i.load(); print(json.dumps({'size':i.size,'exif':bool(i.getexif()),'icc':bool(i.info.get('icc_profile')),'corner':i.convert('RGB').getpixel((0,0))}))", input=data))
check(decoded['size'] == [v['width'], v['height']], name + ' decoded size')
check(not decoded['exif'] and not decoded['icc'], name + ' metadata stripped')
return ready
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--output', default='evidence/lab5-pipeline.json')
parser.add_argument('--resource-only', action='store_true')
args = parser.parse_args()
c = Client(args.base_url, timeout=45)
results = []
def record(name, start, **observed):
row = dict(scenario=name, result='PASS', seconds=round(time.monotonic()-start, 3), **observed)
results.append(row)
print('PASS', name, json.dumps(observed, ensure_ascii=False), flush=True)
def health():
try:
c.call('GET', '/health/ready')
return True
except Failure:
return False
alice, bob, eve, chat = c.bootstrap()
fixture = (ROOT / 'tests/fixtures/sample.png').read_bytes()
try:
if not args.resource_only:
start = time.monotonic()
att, raw = c.attachment(alice, chat)
ready = verify_images(c, bob, att, raw)
c.call('GET', f"/api/v1/attachments/{att['id']}/content?variant=crop", 404, actor=eve)
eventually(lambda: sql(f"SELECT staging_key IS NULL FROM attachments WHERE id='{att['id']}';") == 't')
check(not objects('staging/'+att['id']),'staging bytes not removed')
record('variants_checksums_authorization_staging_cleanup', start, attachment_id=att['id'], variants=[(v['name'],v['width'],v['height']) for v in ready['variants']])
start = time.monotonic()
generators = [
("from PIL import Image; import sys; Image.new('RGBA',(16,8),(255,0,0,0)).save(sys.stdout.buffer,'PNG')", 'image/png', (16,8), True),
("from PIL import Image; import sys; i=Image.new('RGB',(320,200),'blue'); e=i.getexif(); e[274]=6; i.save(sys.stdout.buffer,'JPEG',exif=e)", 'image/jpeg', (160,256), False),
]
for code, content_type, dims, white in generators:
raw = compose('exec','-T','worker','python3','-c',code)
a = c.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=synthetic.png",202,actor=alice,raw=raw,headers={'Content-Type':content_type},schema='Attachment')['json']
verify_images(c,bob,a,raw,dims)
if white:
thumb = c.call('GET',f"/api/v1/attachments/{a['id']}/content?variant=thumbnail",actor=bob)['bytes']
pixel=json.loads(compose('exec','-T','worker','python3','-c',"from PIL import Image; import io,json,sys; print(json.dumps(Image.open(io.BytesIO(sys.stdin.buffer.read())).getpixel((0,0))))",input=thumb))
check(min(pixel)>250,'alpha must become white')
record('alpha_small_image_exif_orientation',start)
start = time.monotonic()
path=f"/api/v1/chats/{chat['id']}/attachments?filename=bad.png"
c.call('POST',path,415,actor=alice,raw=fixture[:-20],headers={'Content-Type':'image/png'})
c.call('POST',path,413,actor=alice,raw=b'x'*(10*1024*1024+1),headers={'Content-Type':'image/png'})
import struct, zlib
ihdr=struct.pack('>IIBBBBB',8000,8000,8,2,0,0,0)
chunk=b'IHDR'+ihdr
bomb=b'\x89PNG\r\n\x1a\n'+struct.pack('>I',13)+chunk+struct.pack('>I',zlib.crc32(chunk)&0xffffffff)
c.call('POST',path,413,actor=alice,raw=bomb,headers={'Content-Type':'image/png'})
record('invalid_bytes_pixels_limits',start)
start = time.monotonic()
compose('stop','worker','outbox')
att, raw = c.attachment(alice, chat)
job = jid(att)
c.message(alice, chat, '', attachments=[att['id']])
c.call('GET',f"/api/v1/attachments/{att['id']}/content",409,actor=bob)
c.message(alice,chat,'text works while worker is stopped')
compose('run','--rm','--no-deps','-T','-e','PIPELINE_FAULT=before_publish','outbox',expected=86)
check(sql(f"SELECT count(*) FROM attachment_outbox WHERE job_id='{job}' AND published_at IS NULL;")=='1','outbox intent survived real process crash')
compose('run','--rm','--no-deps','-T','-e','PIPELINE_FAULT=after_publish','outbox',expected=86)
check(sql(f"SELECT count(*) FROM attachment_outbox WHERE job_id='{job}' AND published_at IS NULL;")=='1','publish-before-mark crash lost outbox')
compose('start','outbox')
eventually(lambda: sql(f"SELECT count(*) FROM attachment_outbox WHERE job_id='{job}' AND published_at IS NOT NULL;")=='1')
count=compose('exec','-T','broker','redis-cli','XLEN','attachment-jobs-v1').strip()
before=compose('ps','-q','api').decode().split()
compose('kill','-s','SIGKILL','broker')
compose('start','broker')
eventually(lambda: compose('exec','-T','broker','redis-cli','ping').strip()==b'PONG')
check(compose('exec','-T','broker','redis-cli','XLEN','attachment-jobs-v1').strip()==count,'broker lost durable stream entries')
compose('restart','api')
eventually(health)
compose('start','worker')
verify_images(c,bob,att,raw)
record('commit_publish_crash_worker_off_api_restart_broker_sigkill',start,attachment_id=att['id'],job_id=job,api_containers=before,stream_entries=int(count))
start = time.monotonic()
compose('stop','worker')
att, raw = c.attachment(alice,chat);job=jid(att)
compose('run','--rm','--no-deps','-T','-e','PIPELINE_FAULT=after_object','worker',expected=86)
check(status(att)=='processing','worker must crash after object before DB result')
key=sql(f"SELECT object_key FROM attachments WHERE id='{att['id']}';")
check(len(objects(key))==1,'crash boundary must contain exactly the persisted original')
c.call('GET',f"/api/v1/attachments/{att['id']}/content",409,actor=bob)
compose('start','worker')
ready=verify_images(c,bob,att,raw)
attempt=int(sql(f"SELECT attempt FROM attachment_jobs WHERE id='{job}';"))
check(attempt>=2,'expired lease not recovered')
duplicate(job);time.sleep(2)
check(int(sql(f"SELECT attempt FROM attachment_jobs WHERE id='{job}';"))==attempt,'ready duplicate executed again')
check(c.call('GET',f"/api/v1/attachments/{att['id']}",actor=bob,schema='Attachment')['json']==ready,'duplicate changed completed metadata')
check(len(objects(key))==4,'duplicate created a second set of S3 objects')
record('crash_after_s3_before_ack_lease_recovery_duplicate',start,attachment_id=att['id'],job_id=job,attempts=attempt)
start=time.monotonic()
compose('stop','worker')
att,raw=c.attachment(alice,chat);job=jid(att)
compose('run','--rm','--no-deps','-T','-e','S3_ENDPOINT=http://127.0.0.1:1','-e','JOB_MAX_ATTEMPTS=2','-e','PIPELINE_FAULT=before_ack','worker',expected=86)
check(status(att)=='queued','transient error must retry')
delay=float(sql(f"SELECT extract(epoch from(next_attempt_at-updated_at)) FROM attachment_jobs WHERE id='{job}';"))
check(0.45<=delay<=1.1,'first retry backoff is outside jitter bounds')
time.sleep(1.1)
compose('run','--rm','--no-deps','-T','-e','S3_ENDPOINT=http://127.0.0.1:1','-e','JOB_MAX_ATTEMPTS=2','-e','PIPELINE_FAULT=before_ack','worker',expected=86)
check(status(att)=='failed','attempts must be bounded')
failed=c.call('GET',f"/api/v1/attachments/{att['id']}",actor=bob,schema='Attachment')['json']
check(failed['error_code']=='retry_exhausted','safe exhausted error_code')
c.call('POST',f"/api/v1/chats/{chat['id']}/messages",409,actor=alice,body={'text':'','attachment_ids':[att['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
registry=json.loads(compose('run','--rm','--no-deps','-T','worker','./worker','-mode=failed'))
check(any(row['job_id']==job for row in registry),'failed registry must expose job')
compose('run','--rm','--no-deps','-T','worker','./worker','-mode=replay','-job='+job,'-reason=lab5-recovered-storage')
compose('start','worker')
verify_images(c,bob,att,raw)
check(sql(f"SELECT count(*) FROM job_replays WHERE job_id='{job}';")=='1','replay audit missing')
compose('run','--rm','--no-deps','-T','worker','./worker','-mode=replay','-job='+job,'-reason=must-refuse-ready',expected=1)
record('retry_jitter_exhaustion_failed_registry_audited_replay',start,attachment_id=att['id'],job_id=job,first_backoff_seconds=round(delay,3))
start=time.monotonic()
raw=compose('exec','-T','worker','python3','-c',"from PIL import Image; import sys; Image.new('RGB',(3000,3000),'blue').save(sys.stdout.buffer,'PNG')")
compose('stop','worker')
att=c.call('POST',f"/api/v1/chats/{chat['id']}/attachments?filename=memory.png",202,actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')['json'];job=jid(att)
compose('run','--rm','--no-deps','-T','-e','IMAGE_MEMORY_MB=64','-e','PIPELINE_FAULT=before_ack','worker',expected=86)
check(status(att)=='failed','memory limit must be a permanent controlled failure')
check(sql(f"SELECT error_code FROM attachment_jobs WHERE id='{job}';")=='image_resource_limit','memory failure code')
check(sql(f"SELECT attempt FROM attachment_jobs WHERE id='{job}';")=='1','permanent failure was retried')
compose('run','--rm','--no-deps','-T','worker','./worker','-mode=replay','-job='+job,'-reason=restore-normal-memory-limit')
compose('start','worker')
verify_images(c,bob,att,raw,(256,256))
record('worker_memory_limit_permanent_failure_replay',start,attachment_id=att['id'],job_id=job,decoder_limit_mb=64,pixels=9000000)
finally:
compose('start','broker','api','outbox','worker')
output=ROOT/args.output
output.parent.mkdir(parents=True,exist_ok=True)
output.write_text(json.dumps({'date':'2026-11-10','scenarios':results},ensure_ascii=False,indent=2)+'\n')
print('PASS all lab5 level4 scenarios',flush=True)
if __name__=='__main__':
main()