forked from mtusi2026/mtusi2026lab5
292 lines
16 KiB
Python
292 lines
16 KiB
Python
"""Lab5 HTTP checks and real, opt-in Compose fault injection on an isolated stand."""
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
from PIL import Image
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
RESULTS = []
|
|
|
|
|
|
def passed(name, **details):
|
|
result = dict(case=name, result='PASS', **details)
|
|
RESULTS.append(result)
|
|
print(json.dumps(result, ensure_ascii=False), flush=True)
|
|
|
|
|
|
def wait(predicate, description, seconds=40):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
result = predicate()
|
|
if result:
|
|
return result
|
|
time.sleep(.25)
|
|
raise AssertionError('Timeout: ' + description)
|
|
|
|
|
|
def main():
|
|
def configure(parser):
|
|
parser.add_argument('--faults', action='store_true')
|
|
parser.add_argument('--project', default=os.environ.get('COMPOSE_PROJECT_NAME', ''))
|
|
args = arguments(__doc__, configure)
|
|
c = Client(args.base_url, args.ca_file)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
base = '/api/v1/attachments/'
|
|
|
|
def upload(data, mime='image/png'):
|
|
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=image.png',
|
|
202, actor=alice, raw=data, headers={'Content-Type': mime}, schema='Attachment')['json']
|
|
check(att['status'] == 'queued' and att['variants'] == [], '202 must be a queued snapshot')
|
|
return att
|
|
|
|
def fixture(size, mode='RGB', color='red', fmt='PNG', exif=None):
|
|
data = io.BytesIO()
|
|
options = {'exif': exif} if exif else {}
|
|
Image.new(mode, size, color).save(data, fmt, **options)
|
|
return data.getvalue()
|
|
|
|
cases = [((320, 200), (256, 160)), ((7, 3), (7, 3)), ((1001, 667), (256, 170)),
|
|
((1600, 1000), (256, 160))]
|
|
for size, thumb_size in cases:
|
|
data = fixture(size)
|
|
att = c.wait_ready(alice, upload(data))
|
|
variants = {v['name']: v for v in att['variants']}
|
|
check(set(variants) == {'thumbnail', 'crop', 'optimized'}, 'all variants required')
|
|
check(c.call('GET', base + att['id'] + '/content', actor=bob)['bytes'] == data, 'original changed')
|
|
for name, item in variants.items():
|
|
response = c.call('GET', base + att['id'] + '/content?variant=' + name, actor=bob)
|
|
encoded = response['bytes']
|
|
check(hashlib.sha256(encoded).hexdigest() == item['sha256'], 'variant checksum')
|
|
check(len(encoded) == item['size_bytes'], 'variant size')
|
|
with Image.open(io.BytesIO(encoded)) as image:
|
|
image.load()
|
|
check(image.size == (item['width'], item['height']), 'reported dimensions')
|
|
check(not image.getexif() and 'icc_profile' not in image.info, 'metadata leaked')
|
|
check(image.format == ('WEBP' if name == 'optimized' else 'JPEG'), 'format')
|
|
if name == 'thumbnail': check(image.size == thumb_size, 'floor/no-upscale')
|
|
if name == 'crop': check(image.size == (128, 128), 'crop dimensions')
|
|
if name == 'optimized':
|
|
check(max(image.size) <= 1280 and image.width <= size[0] and image.height <= size[1], 'optimized bounds')
|
|
c.call('GET', base + att['id'] + '/content?variant=' + name, 404, actor=eve)
|
|
passed('variants', original=list(size), thumbnail=list(thumb_size))
|
|
|
|
alpha = upload(fixture((20, 10), 'RGBA', (255, 0, 0, 0)))
|
|
alpha = c.wait_ready(alice, alpha)
|
|
encoded = c.call('GET', base + alpha['id'] + '/content?variant=thumbnail', actor=bob)['bytes']
|
|
with Image.open(io.BytesIO(encoded)) as image:
|
|
check(min(image.getpixel((0, 0))) >= 250, 'alpha must be white')
|
|
exif = Image.Exif(); exif[274] = 6; exif[270] = 'private metadata'
|
|
rotated = c.wait_ready(alice, upload(fixture((200, 100), fmt='JPEG', exif=exif), 'image/jpeg'))
|
|
thumbnail = next(v for v in rotated['variants'] if v['name'] == 'thumbnail')
|
|
check((thumbnail['width'], thumbnail['height']) == (100, 200), 'EXIF orientation')
|
|
stripes = Image.new('RGB', (300, 100), 'red')
|
|
stripes.paste('green', (100, 0, 200, 100)); stripes.paste('blue', (200, 0, 300, 100))
|
|
buf = io.BytesIO(); stripes.save(buf, 'PNG')
|
|
cropped = c.wait_ready(alice, upload(buf.getvalue()))
|
|
data = c.call('GET', base + cropped['id'] + '/content?variant=crop', actor=bob)['bytes']
|
|
with Image.open(io.BytesIO(data)) as image:
|
|
red, green, blue = image.getpixel((64, 64))
|
|
check(green > red + 50 and green > blue + 50, 'crop must be centered')
|
|
passed('alpha_EXIF_metadata_center_crop')
|
|
|
|
if not args.faults:
|
|
return
|
|
check(bool(args.project) and args.project != ROOT.name, 'Fault tests require an explicitly named isolated project')
|
|
override = Path(tempfile.mkdtemp(prefix='lab5-faults-')) / 'override.yaml'
|
|
override.write_text('''services:
|
|
worker:
|
|
environment: &faults
|
|
PIPELINE_FAULT_INJECTION: "true"
|
|
JOB_LEASE_SECONDS: "5"
|
|
JOB_MAX_ATTEMPTS: "3"
|
|
JOB_TIMEOUT_SECONDS: "1"
|
|
JOB_RETRY_BASE_SECONDS: "0.5"
|
|
JOB_RETRY_CAP_SECONDS: "2"
|
|
dispatcher:
|
|
environment: *faults
|
|
''')
|
|
command = ['docker', 'compose', '--env-file', os.environ.get('ENV_FILE', '.env'), '-p', args.project, '-f', str(ROOT / 'compose.yaml'), '-f', str(override)]
|
|
|
|
def compose(*parts, input=None, expected=0):
|
|
result = subprocess.run(command + list(parts), input=input, text=True, capture_output=True, timeout=100)
|
|
check(result.returncode == expected, 'Compose action failed: ' + ' '.join(parts[:3]) + ' ' + result.stderr[-400:])
|
|
return result.stdout.strip()
|
|
|
|
def sql(query):
|
|
return compose('exec', '-T', 'postgres', 'sh', '-c',
|
|
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -v ON_ERROR_STOP=1', input=query)
|
|
|
|
def job(att):
|
|
return json.loads(sql("SELECT row_to_json(j) FROM attachment_jobs j WHERE attachment_id='" + att['id'] + "';"))
|
|
|
|
def set_fault(att, value):
|
|
sql("UPDATE attachment_jobs SET fault=" + ("'" + value + "'" if value else 'NULL') + " WHERE attachment_id='" + att['id'] + "';")
|
|
|
|
def exited(service, code):
|
|
ids = compose('ps', '-aq', service).split()
|
|
if not ids: return False
|
|
inspect = subprocess.check_output(['docker', 'inspect', '--format', '{{json .State}}', ids[0]], text=True)
|
|
state = json.loads(inspect)
|
|
return state['Status'] == 'exited' and state['ExitCode'] == code
|
|
|
|
def pending():
|
|
return int(compose('exec', '-T', 'broker', 'redis-cli', 'XPENDING', 'attachment-jobs-v1', 'image-workers-v1').splitlines()[0])
|
|
|
|
def objects(att):
|
|
code = '''import json,sys
|
|
from src.core.storage import make_storage
|
|
from src.core.settings import settings
|
|
s=make_storage(role='worker');i=sys.argv[1]
|
|
print(json.dumps({o.object_name:o.etag for prefix in ['originals/'+i,'variants/'+i+'/'] for o in s.list_objects(settings.s3_bucket,prefix=prefix,recursive=True)}))'''
|
|
return json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code, att['id']))
|
|
|
|
def inject(att):
|
|
j = job(att)
|
|
payload = dict(schema_version=1, job_id=j['id'], type='attachment.process.v1', attachment_id=att['id'],
|
|
pipeline_version=1, attempt=j['attempt'], created_at=j['created_at'])
|
|
compose('exec', '-T', 'broker', 'redis-cli', 'XADD', 'attachment-jobs-v1', '*', 'job', json.dumps(payload))
|
|
|
|
try:
|
|
compose('up', '-d', '--no-deps', '--force-recreate', 'worker', 'dispatcher')
|
|
compose('stop', 'worker')
|
|
att, data = c.attachment(alice, chat)
|
|
c.message(alice, chat, '', attachments=[att['id']])
|
|
c.call('GET', base + att['id'] + '/content', 409, actor=bob)
|
|
c.message(alice, chat, 'API работает без worker')
|
|
wait(lambda: job(att)['published_at'], 'publish while worker stopped')
|
|
check(int(compose('exec', '-T', 'broker', 'redis-cli', 'XLEN', 'attachment-jobs-v1')) > 0, 'broker must contain work')
|
|
compose('restart', 'api', 'broker')
|
|
def ready():
|
|
try:
|
|
return c.call('GET', '/health/ready', [200, 503])['status'] == 200
|
|
except Failure:
|
|
return False
|
|
wait(ready, 'API ready')
|
|
compose('start', 'worker')
|
|
c.wait_ready(alice, att)
|
|
check(c.call('GET', base + att['id'] + '/content', actor=bob)['bytes'] == data, 'restart lost original')
|
|
passed('stopped_worker_API_broker_restart', attachment_id=att['id'])
|
|
|
|
compose('stop', 'broker')
|
|
unavailable = upload(fixture((324, 204)))
|
|
check(c.call('GET', '/health/ready')['status'] == 200, 'broker outage must not block API')
|
|
check(job(unavailable)['published_at'] is None, 'unsent job must remain in DB')
|
|
compose('start', 'broker')
|
|
c.wait_ready(alice, unavailable)
|
|
passed('broker_unavailable_outbox_recovered')
|
|
|
|
# COMMIT is already durable; kill the actual relay before and after XADD.
|
|
for fault, code in [('before_publish', 81), ('after_publish', 82)]:
|
|
compose('stop', 'dispatcher', 'worker')
|
|
att = upload(fixture((321, 201)))
|
|
set_fault(att, fault)
|
|
check(job(att)['published_at'] is None, 'must have unsent outbox')
|
|
compose('start', 'dispatcher')
|
|
wait(lambda: exited('dispatcher', code), fault)
|
|
check(job(att)['published_at'] is None, 'relay crash must roll back sent flag')
|
|
# Dispatcher is stopped; clear through postgres, then restore the same job.
|
|
set_fault(att, None)
|
|
compose('start', 'dispatcher', 'worker')
|
|
c.wait_ready(alice, att)
|
|
passed(fault + '_recovered', job_id=job(att)['id'])
|
|
|
|
for fault, code in [('after_objects', 83), ('after_commit', 84)]:
|
|
compose('stop', 'worker')
|
|
att = upload(fixture((322, 202)))
|
|
set_fault(att, fault)
|
|
compose('start', 'worker')
|
|
wait(lambda: exited('worker', code), fault)
|
|
before = objects(att)
|
|
check(len(before) == 4, 'one original and three objects must exist')
|
|
j = job(att)
|
|
if fault == 'after_objects':
|
|
check(j['state'] == 'processing', 'crash must leave a processing lease')
|
|
else:
|
|
check(j['state'] == 'ready', 'ready commit must precede ack')
|
|
check(pending() > 0, 'crash must leave an unacked delivery')
|
|
set_fault(att, None)
|
|
compose('start', 'worker')
|
|
c.wait_ready(alice, att)
|
|
wait(lambda: pending() == 0, 'pending delivery recovery')
|
|
check(objects(att) == before, 'redelivery changed/duplicated objects')
|
|
stable = job(att)
|
|
inject(att); inject(att)
|
|
wait(lambda: pending() == 0 and int(compose('exec', '-T', 'broker', 'redis-cli', 'XLEN', 'attachment-jobs-v1')) == 0, 'duplicate ack')
|
|
check(job(att)['attempt'] == stable['attempt'], 'ready duplicate must not start a new attempt')
|
|
check(objects(att) == before, 'ready duplicate changed output')
|
|
passed(fault + '_redelivery_idempotent', attempt=stable['attempt'], objects=len(before))
|
|
|
|
for fault in ['temporary', 'permanent', 'timeout']:
|
|
compose('stop', 'worker')
|
|
att = upload(fixture((323, 203)))
|
|
set_fault(att, fault)
|
|
compose('start', 'worker')
|
|
failed = wait(lambda: (a if (a := c.call('GET', base + att['id'], actor=alice, schema='Attachment')['json'])['status'] == 'failed' else None), 'terminal failed')
|
|
j = job(att)
|
|
check(j['cycle_attempt'] == (1 if fault == 'permanent' else 3), 'retry budget/classification')
|
|
check(failed['error_code'] == ('invalid_image' if fault == 'permanent' else 'retry_exhausted'), 'safe terminal code')
|
|
c.call('GET', base + att['id'] + '/content', 409, actor=bob)
|
|
c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
|
|
body={'text':'', 'attachment_ids':[att['id']]}, headers={'Idempotency-Key':uuid.uuid4().hex})
|
|
registry = json.loads(compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'failed'))
|
|
check(any(x['job_id'] == j['id'] for x in registry), 'failed registry missing')
|
|
set_fault(att, None)
|
|
compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', j['id'], '--operator', 'lab5-test', '--reason', 'dependency restored')
|
|
c.wait_ready(alice, att)
|
|
check(sql("SELECT count(*) FROM job_replays WHERE job_id='" + j['id'] + "';") == '1', 'audit missing')
|
|
compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', j['id'], '--operator', 'lab5-test', '--reason', 'duplicate replay', expected=1)
|
|
passed(fault + '_failed_audited_replay', attempts_before_replay=j['attempt'])
|
|
|
|
wait(lambda: sql("SELECT count(*) FROM attachments WHERE status='ready' AND staging_key IS NOT NULL;") == '0', 'ready staging cleanup')
|
|
passed('staging_cleanup_after_ready')
|
|
|
|
compose('stop', 'worker')
|
|
retained = upload(fixture((325, 205)))
|
|
set_fault(retained, 'permanent')
|
|
compose('start', 'worker')
|
|
wait(lambda: job(retained)['state'] == 'failed', 'failed retention fixture')
|
|
check(sql("SELECT staging_key IS NOT NULL FROM attachments WHERE id='" + retained['id'] + "';") == 't', 'fresh failed staging must remain')
|
|
sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '8 days' WHERE id='" + retained['id'] + "';")
|
|
wait(lambda: sql("SELECT staging_key IS NULL FROM attachments WHERE id='" + retained['id'] + "';") == 't', 'failed retention cleanup')
|
|
compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', job(retained)['id'], '--operator', 'lab5-test', '--reason', 'expired source', expected=1)
|
|
passed('failed_retention_and_expired_replay_rejected')
|
|
|
|
compose('stop', 'worker')
|
|
active = upload(fixture((326, 206)))
|
|
sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '8 days' WHERE id='" + active['id'] + "';")
|
|
orphan = 'staging/' + str(uuid.uuid4())
|
|
script = '''import io,sys
|
|
from src.core.storage import make_storage
|
|
from src.core.settings import settings
|
|
make_storage(role='worker').put_object(settings.s3_bucket,sys.argv[1],io.BytesIO(b'orphan'),6)
|
|
'''
|
|
compose('exec', '-T', 'dispatcher', 'python', '-c', script, orphan)
|
|
sql("INSERT INTO upload_intents(object_key,state,created_at) VALUES ('" + orphan + "','pending',clock_timestamp()-interval '8 days');")
|
|
wait(lambda: sql("SELECT state FROM upload_intents WHERE object_key='" + orphan + "';") == 'abandoned', 'orphan staging cleanup')
|
|
check(sql("SELECT staging_key IS NOT NULL FROM attachments WHERE id='" + active['id'] + "';") == 't', 'cleanup must protect queued work')
|
|
compose('start', 'worker')
|
|
c.wait_ready(alice, active)
|
|
passed('abandoned_staging_cleaned_active_job_protected')
|
|
|
|
finally:
|
|
# Disable every injection and restore the ordinary configuration even on failure.
|
|
try:
|
|
sql('UPDATE attachment_jobs SET fault=NULL WHERE fault IS NOT NULL;')
|
|
finally:
|
|
subprocess.run(['docker','compose','--env-file',os.environ.get('ENV_FILE','.env'),'-p',args.project,'up','-d','--no-deps','--force-recreate','worker','dispatcher'],
|
|
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=100, check=True)
|
|
override.unlink(missing_ok=True)
|
|
override.parent.rmdir()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|