Files
mtusi2026lab6/tests/lab5.py
T

292 lines
16 KiB
Python

"""Lab5 HTTP checks and real, opt-in Compose fault injection on an isolated stand."""
import hashlib
import io
import json
import os
import subprocess
import tempfile
import time
import uuid
from pathlib import Path
from PIL import Image
from client import Client, Failure, arguments, check, run
ROOT = Path(__file__).resolve().parents[1]
RESULTS = []
def passed(name, **details):
result = dict(case=name, result='PASS', **details)
RESULTS.append(result)
print(json.dumps(result, ensure_ascii=False), flush=True)
def wait(predicate, description, seconds=40):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
result = predicate()
if result:
return result
time.sleep(.25)
raise AssertionError('Timeout: ' + description)
def main():
def configure(parser):
parser.add_argument('--faults', action='store_true')
parser.add_argument('--project', default=os.environ.get('COMPOSE_PROJECT_NAME', ''))
args = arguments(__doc__, configure)
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
base = '/api/v1/attachments/'
def upload(data, mime='image/png'):
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=image.png',
202, actor=alice, raw=data, headers={'Content-Type': mime}, schema='Attachment')['json']
check(att['status'] == 'queued' and att['variants'] == [], '202 must be a queued snapshot')
return att
def fixture(size, mode='RGB', color='red', fmt='PNG', exif=None):
data = io.BytesIO()
options = {'exif': exif} if exif else {}
Image.new(mode, size, color).save(data, fmt, **options)
return data.getvalue()
cases = [((320, 200), (256, 160)), ((7, 3), (7, 3)), ((1001, 667), (256, 170)),
((1600, 1000), (256, 160))]
for size, thumb_size in cases:
data = fixture(size)
att = c.wait_ready(alice, upload(data))
variants = {v['name']: v for v in att['variants']}
check(set(variants) == {'thumbnail', 'crop', 'optimized'}, 'all variants required')
check(c.call('GET', base + att['id'] + '/content', actor=bob)['bytes'] == data, 'original changed')
for name, item in variants.items():
response = c.call('GET', base + att['id'] + '/content?variant=' + name, actor=bob)
encoded = response['bytes']
check(hashlib.sha256(encoded).hexdigest() == item['sha256'], 'variant checksum')
check(len(encoded) == item['size_bytes'], 'variant size')
with Image.open(io.BytesIO(encoded)) as image:
image.load()
check(image.size == (item['width'], item['height']), 'reported dimensions')
check(not image.getexif() and 'icc_profile' not in image.info, 'metadata leaked')
check(image.format == ('WEBP' if name == 'optimized' else 'JPEG'), 'format')
if name == 'thumbnail': check(image.size == thumb_size, 'floor/no-upscale')
if name == 'crop': check(image.size == (128, 128), 'crop dimensions')
if name == 'optimized':
check(max(image.size) <= 1280 and image.width <= size[0] and image.height <= size[1], 'optimized bounds')
c.call('GET', base + att['id'] + '/content?variant=' + name, 404, actor=eve)
passed('variants', original=list(size), thumbnail=list(thumb_size))
alpha = upload(fixture((20, 10), 'RGBA', (255, 0, 0, 0)))
alpha = c.wait_ready(alice, alpha)
encoded = c.call('GET', base + alpha['id'] + '/content?variant=thumbnail', actor=bob)['bytes']
with Image.open(io.BytesIO(encoded)) as image:
check(min(image.getpixel((0, 0))) >= 250, 'alpha must be white')
exif = Image.Exif(); exif[274] = 6; exif[270] = 'private metadata'
rotated = c.wait_ready(alice, upload(fixture((200, 100), fmt='JPEG', exif=exif), 'image/jpeg'))
thumbnail = next(v for v in rotated['variants'] if v['name'] == 'thumbnail')
check((thumbnail['width'], thumbnail['height']) == (100, 200), 'EXIF orientation')
stripes = Image.new('RGB', (300, 100), 'red')
stripes.paste('green', (100, 0, 200, 100)); stripes.paste('blue', (200, 0, 300, 100))
buf = io.BytesIO(); stripes.save(buf, 'PNG')
cropped = c.wait_ready(alice, upload(buf.getvalue()))
data = c.call('GET', base + cropped['id'] + '/content?variant=crop', actor=bob)['bytes']
with Image.open(io.BytesIO(data)) as image:
red, green, blue = image.getpixel((64, 64))
check(green > red + 50 and green > blue + 50, 'crop must be centered')
passed('alpha_EXIF_metadata_center_crop')
if not args.faults:
return
check(bool(args.project) and args.project != ROOT.name, 'Fault tests require an explicitly named isolated project')
override = Path(tempfile.mkdtemp(prefix='lab5-faults-')) / 'override.yaml'
override.write_text('''services:
worker:
environment: &faults
PIPELINE_FAULT_INJECTION: "true"
JOB_LEASE_SECONDS: "5"
JOB_MAX_ATTEMPTS: "3"
JOB_TIMEOUT_SECONDS: "1"
JOB_RETRY_BASE_SECONDS: "0.5"
JOB_RETRY_CAP_SECONDS: "2"
dispatcher:
environment: *faults
''')
command = ['docker', 'compose', '--env-file', os.environ.get('ENV_FILE', '.env'), '-p', args.project, '-f', str(ROOT / 'compose.yaml'), '-f', str(override)]
def compose(*parts, input=None, expected=0):
result = subprocess.run(command + list(parts), input=input, text=True, capture_output=True, timeout=100)
check(result.returncode == expected, 'Compose action failed: ' + ' '.join(parts[:3]) + ' ' + result.stderr[-400:])
return result.stdout.strip()
def sql(query):
return compose('exec', '-T', 'postgres', 'sh', '-c',
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -v ON_ERROR_STOP=1', input=query)
def job(att):
return json.loads(sql("SELECT row_to_json(j) FROM attachment_jobs j WHERE attachment_id='" + att['id'] + "';"))
def set_fault(att, value):
sql("UPDATE attachment_jobs SET fault=" + ("'" + value + "'" if value else 'NULL') + " WHERE attachment_id='" + att['id'] + "';")
def exited(service, code):
ids = compose('ps', '-aq', service).split()
if not ids: return False
inspect = subprocess.check_output(['docker', 'inspect', '--format', '{{json .State}}', ids[0]], text=True)
state = json.loads(inspect)
return state['Status'] == 'exited' and state['ExitCode'] == code
def pending():
return int(compose('exec', '-T', 'broker', 'redis-cli', 'XPENDING', 'attachment-jobs-v1', 'image-workers-v1').splitlines()[0])
def objects(att):
code = '''import json,sys
from src.core.storage import make_storage
from src.core.settings import settings
s=make_storage(role='worker');i=sys.argv[1]
print(json.dumps({o.object_name:o.etag for prefix in ['originals/'+i,'variants/'+i+'/'] for o in s.list_objects(settings.s3_bucket,prefix=prefix,recursive=True)}))'''
return json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code, att['id']))
def inject(att):
j = job(att)
payload = dict(schema_version=1, job_id=j['id'], type='attachment.process.v1', attachment_id=att['id'],
pipeline_version=1, attempt=j['attempt'], created_at=j['created_at'])
compose('exec', '-T', 'broker', 'redis-cli', 'XADD', 'attachment-jobs-v1', '*', 'job', json.dumps(payload))
try:
compose('up', '-d', '--no-deps', '--force-recreate', 'worker', 'dispatcher')
compose('stop', 'worker')
att, data = c.attachment(alice, chat)
c.message(alice, chat, '', attachments=[att['id']])
c.call('GET', base + att['id'] + '/content', 409, actor=bob)
c.message(alice, chat, 'API работает без worker')
wait(lambda: job(att)['published_at'], 'publish while worker stopped')
check(int(compose('exec', '-T', 'broker', 'redis-cli', 'XLEN', 'attachment-jobs-v1')) > 0, 'broker must contain work')
compose('restart', 'api', 'broker')
def ready():
try:
return c.call('GET', '/health/ready', [200, 503])['status'] == 200
except Failure:
return False
wait(ready, 'API ready')
compose('start', 'worker')
c.wait_ready(alice, att)
check(c.call('GET', base + att['id'] + '/content', actor=bob)['bytes'] == data, 'restart lost original')
passed('stopped_worker_API_broker_restart', attachment_id=att['id'])
compose('stop', 'broker')
unavailable = upload(fixture((324, 204)))
check(c.call('GET', '/health/ready')['status'] == 200, 'broker outage must not block API')
check(job(unavailable)['published_at'] is None, 'unsent job must remain in DB')
compose('start', 'broker')
c.wait_ready(alice, unavailable)
passed('broker_unavailable_outbox_recovered')
# COMMIT is already durable; kill the actual relay before and after XADD.
for fault, code in [('before_publish', 81), ('after_publish', 82)]:
compose('stop', 'dispatcher', 'worker')
att = upload(fixture((321, 201)))
set_fault(att, fault)
check(job(att)['published_at'] is None, 'must have unsent outbox')
compose('start', 'dispatcher')
wait(lambda: exited('dispatcher', code), fault)
check(job(att)['published_at'] is None, 'relay crash must roll back sent flag')
# Dispatcher is stopped; clear through postgres, then restore the same job.
set_fault(att, None)
compose('start', 'dispatcher', 'worker')
c.wait_ready(alice, att)
passed(fault + '_recovered', job_id=job(att)['id'])
for fault, code in [('after_objects', 83), ('after_commit', 84)]:
compose('stop', 'worker')
att = upload(fixture((322, 202)))
set_fault(att, fault)
compose('start', 'worker')
wait(lambda: exited('worker', code), fault)
before = objects(att)
check(len(before) == 4, 'one original and three objects must exist')
j = job(att)
if fault == 'after_objects':
check(j['state'] == 'processing', 'crash must leave a processing lease')
else:
check(j['state'] == 'ready', 'ready commit must precede ack')
check(pending() > 0, 'crash must leave an unacked delivery')
set_fault(att, None)
compose('start', 'worker')
c.wait_ready(alice, att)
wait(lambda: pending() == 0, 'pending delivery recovery')
check(objects(att) == before, 'redelivery changed/duplicated objects')
stable = job(att)
inject(att); inject(att)
wait(lambda: pending() == 0 and int(compose('exec', '-T', 'broker', 'redis-cli', 'XLEN', 'attachment-jobs-v1')) == 0, 'duplicate ack')
check(job(att)['attempt'] == stable['attempt'], 'ready duplicate must not start a new attempt')
check(objects(att) == before, 'ready duplicate changed output')
passed(fault + '_redelivery_idempotent', attempt=stable['attempt'], objects=len(before))
for fault in ['temporary', 'permanent', 'timeout']:
compose('stop', 'worker')
att = upload(fixture((323, 203)))
set_fault(att, fault)
compose('start', 'worker')
failed = wait(lambda: (a if (a := c.call('GET', base + att['id'], actor=alice, schema='Attachment')['json'])['status'] == 'failed' else None), 'terminal failed')
j = job(att)
check(j['cycle_attempt'] == (1 if fault == 'permanent' else 3), 'retry budget/classification')
check(failed['error_code'] == ('invalid_image' if fault == 'permanent' else 'retry_exhausted'), 'safe terminal code')
c.call('GET', base + att['id'] + '/content', 409, actor=bob)
c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
body={'text':'', 'attachment_ids':[att['id']]}, headers={'Idempotency-Key':uuid.uuid4().hex})
registry = json.loads(compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'failed'))
check(any(x['job_id'] == j['id'] for x in registry), 'failed registry missing')
set_fault(att, None)
compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', j['id'], '--operator', 'lab5-test', '--reason', 'dependency restored')
c.wait_ready(alice, att)
check(sql("SELECT count(*) FROM job_replays WHERE job_id='" + j['id'] + "';") == '1', 'audit missing')
compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', j['id'], '--operator', 'lab5-test', '--reason', 'duplicate replay', expected=1)
passed(fault + '_failed_audited_replay', attempts_before_replay=j['attempt'])
wait(lambda: sql("SELECT count(*) FROM attachments WHERE status='ready' AND staging_key IS NOT NULL;") == '0', 'ready staging cleanup')
passed('staging_cleanup_after_ready')
compose('stop', 'worker')
retained = upload(fixture((325, 205)))
set_fault(retained, 'permanent')
compose('start', 'worker')
wait(lambda: job(retained)['state'] == 'failed', 'failed retention fixture')
check(sql("SELECT staging_key IS NOT NULL FROM attachments WHERE id='" + retained['id'] + "';") == 't', 'fresh failed staging must remain')
sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '8 days' WHERE id='" + retained['id'] + "';")
wait(lambda: sql("SELECT staging_key IS NULL FROM attachments WHERE id='" + retained['id'] + "';") == 't', 'failed retention cleanup')
compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', job(retained)['id'], '--operator', 'lab5-test', '--reason', 'expired source', expected=1)
passed('failed_retention_and_expired_replay_rejected')
compose('stop', 'worker')
active = upload(fixture((326, 206)))
sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '8 days' WHERE id='" + active['id'] + "';")
orphan = 'staging/' + str(uuid.uuid4())
script = '''import io,sys
from src.core.storage import make_storage
from src.core.settings import settings
make_storage(role='worker').put_object(settings.s3_bucket,sys.argv[1],io.BytesIO(b'orphan'),6)
'''
compose('exec', '-T', 'dispatcher', 'python', '-c', script, orphan)
sql("INSERT INTO upload_intents(object_key,state,created_at) VALUES ('" + orphan + "','pending',clock_timestamp()-interval '8 days');")
wait(lambda: sql("SELECT state FROM upload_intents WHERE object_key='" + orphan + "';") == 'abandoned', 'orphan staging cleanup')
check(sql("SELECT staging_key IS NOT NULL FROM attachments WHERE id='" + active['id'] + "';") == 't', 'cleanup must protect queued work')
compose('start', 'worker')
c.wait_ready(alice, active)
passed('abandoned_staging_cleaned_active_job_protected')
finally:
# Disable every injection and restore the ordinary configuration even on failure.
try:
sql('UPDATE attachment_jobs SET fault=NULL WHERE fault IS NOT NULL;')
finally:
subprocess.run(['docker','compose','--env-file',os.environ.get('ENV_FILE','.env'),'-p',args.project,'up','-d','--no-deps','--force-recreate','worker','dispatcher'],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=100, check=True)
override.unlink(missing_ok=True)
override.parent.rmdir()
if __name__ == '__main__':
run(main)