"""Lab5 HTTP checks and real, opt-in Compose fault injection on an isolated stand.""" import hashlib import io import json import os import subprocess import tempfile import time import uuid from pathlib import Path from PIL import Image from client import Client, Failure, arguments, check, run ROOT = Path(__file__).resolve().parents[1] RESULTS = [] def passed(name, **details): result = dict(case=name, result='PASS', **details) RESULTS.append(result) print(json.dumps(result, ensure_ascii=False), flush=True) def wait(predicate, description, seconds=40): deadline = time.monotonic() + seconds while time.monotonic() < deadline: result = predicate() if result: return result time.sleep(.25) raise AssertionError('Timeout: ' + description) def main(): def configure(parser): parser.add_argument('--faults', action='store_true') parser.add_argument('--project', default=os.environ.get('COMPOSE_PROJECT_NAME', '')) args = arguments(__doc__, configure) c = Client(args.base_url, args.ca_file) alice, bob, eve, chat = c.bootstrap() base = '/api/v1/attachments/' def upload(data, mime='image/png'): att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=image.png', 202, actor=alice, raw=data, headers={'Content-Type': mime}, schema='Attachment')['json'] check(att['status'] == 'queued' and att['variants'] == [], '202 must be a queued snapshot') return att def fixture(size, mode='RGB', color='red', fmt='PNG', exif=None): data = io.BytesIO() options = {'exif': exif} if exif else {} Image.new(mode, size, color).save(data, fmt, **options) return data.getvalue() cases = [((320, 200), (256, 160)), ((7, 3), (7, 3)), ((1001, 667), (256, 170)), ((1600, 1000), (256, 160))] for size, thumb_size in cases: data = fixture(size) att = c.wait_ready(alice, upload(data)) variants = {v['name']: v for v in att['variants']} check(set(variants) == {'thumbnail', 'crop', 'optimized'}, 'all variants required') check(c.call('GET', base + att['id'] + '/content', actor=bob)['bytes'] == data, 'original changed') for name, item in variants.items(): response = c.call('GET', base + att['id'] + '/content?variant=' + name, actor=bob) encoded = response['bytes'] check(hashlib.sha256(encoded).hexdigest() == item['sha256'], 'variant checksum') check(len(encoded) == item['size_bytes'], 'variant size') with Image.open(io.BytesIO(encoded)) as image: image.load() check(image.size == (item['width'], item['height']), 'reported dimensions') check(not image.getexif() and 'icc_profile' not in image.info, 'metadata leaked') check(image.format == ('WEBP' if name == 'optimized' else 'JPEG'), 'format') if name == 'thumbnail': check(image.size == thumb_size, 'floor/no-upscale') if name == 'crop': check(image.size == (128, 128), 'crop dimensions') if name == 'optimized': check(max(image.size) <= 1280 and image.width <= size[0] and image.height <= size[1], 'optimized bounds') c.call('GET', base + att['id'] + '/content?variant=' + name, 404, actor=eve) passed('variants', original=list(size), thumbnail=list(thumb_size)) alpha = upload(fixture((20, 10), 'RGBA', (255, 0, 0, 0))) alpha = c.wait_ready(alice, alpha) encoded = c.call('GET', base + alpha['id'] + '/content?variant=thumbnail', actor=bob)['bytes'] with Image.open(io.BytesIO(encoded)) as image: check(min(image.getpixel((0, 0))) >= 250, 'alpha must be white') exif = Image.Exif(); exif[274] = 6; exif[270] = 'private metadata' rotated = c.wait_ready(alice, upload(fixture((200, 100), fmt='JPEG', exif=exif), 'image/jpeg')) thumbnail = next(v for v in rotated['variants'] if v['name'] == 'thumbnail') check((thumbnail['width'], thumbnail['height']) == (100, 200), 'EXIF orientation') stripes = Image.new('RGB', (300, 100), 'red') stripes.paste('green', (100, 0, 200, 100)); stripes.paste('blue', (200, 0, 300, 100)) buf = io.BytesIO(); stripes.save(buf, 'PNG') cropped = c.wait_ready(alice, upload(buf.getvalue())) data = c.call('GET', base + cropped['id'] + '/content?variant=crop', actor=bob)['bytes'] with Image.open(io.BytesIO(data)) as image: red, green, blue = image.getpixel((64, 64)) check(green > red + 50 and green > blue + 50, 'crop must be centered') passed('alpha_EXIF_metadata_center_crop') if not args.faults: return check(bool(args.project) and args.project != ROOT.name, 'Fault tests require an explicitly named isolated project') override = Path(tempfile.mkdtemp(prefix='lab5-faults-')) / 'override.yaml' override.write_text('''services: worker: environment: &faults PIPELINE_FAULT_INJECTION: "true" JOB_LEASE_SECONDS: "5" JOB_MAX_ATTEMPTS: "3" JOB_TIMEOUT_SECONDS: "1" JOB_RETRY_BASE_SECONDS: "0.5" JOB_RETRY_CAP_SECONDS: "2" dispatcher: environment: *faults ''') command = ['docker', 'compose', '--env-file', os.environ.get('ENV_FILE', '.env'), '-p', args.project, '-f', str(ROOT / 'compose.yaml'), '-f', str(override)] def compose(*parts, input=None, expected=0): result = subprocess.run(command + list(parts), input=input, text=True, capture_output=True, timeout=100) check(result.returncode == expected, 'Compose action failed: ' + ' '.join(parts[:3]) + ' ' + result.stderr[-400:]) return result.stdout.strip() def sql(query): return compose('exec', '-T', 'postgres', 'sh', '-c', 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -v ON_ERROR_STOP=1', input=query) def job(att): return json.loads(sql("SELECT row_to_json(j) FROM attachment_jobs j WHERE attachment_id='" + att['id'] + "';")) def set_fault(att, value): sql("UPDATE attachment_jobs SET fault=" + ("'" + value + "'" if value else 'NULL') + " WHERE attachment_id='" + att['id'] + "';") def exited(service, code): ids = compose('ps', '-aq', service).split() if not ids: return False inspect = subprocess.check_output(['docker', 'inspect', '--format', '{{json .State}}', ids[0]], text=True) state = json.loads(inspect) return state['Status'] == 'exited' and state['ExitCode'] == code def pending(): return int(compose('exec', '-T', 'broker', 'redis-cli', 'XPENDING', 'attachment-jobs-v1', 'image-workers-v1').splitlines()[0]) def objects(att): code = '''import json,sys from src.core.storage import make_storage from src.core.settings import settings s=make_storage(role='worker');i=sys.argv[1] print(json.dumps({o.object_name:o.etag for prefix in ['originals/'+i,'variants/'+i+'/'] for o in s.list_objects(settings.s3_bucket,prefix=prefix,recursive=True)}))''' return json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code, att['id'])) def inject(att): j = job(att) payload = dict(schema_version=1, job_id=j['id'], type='attachment.process.v1', attachment_id=att['id'], pipeline_version=1, attempt=j['attempt'], created_at=j['created_at']) compose('exec', '-T', 'broker', 'redis-cli', 'XADD', 'attachment-jobs-v1', '*', 'job', json.dumps(payload)) try: compose('up', '-d', '--no-deps', '--force-recreate', 'worker', 'dispatcher') compose('stop', 'worker') att, data = c.attachment(alice, chat) c.message(alice, chat, '', attachments=[att['id']]) c.call('GET', base + att['id'] + '/content', 409, actor=bob) c.message(alice, chat, 'API работает без worker') wait(lambda: job(att)['published_at'], 'publish while worker stopped') check(int(compose('exec', '-T', 'broker', 'redis-cli', 'XLEN', 'attachment-jobs-v1')) > 0, 'broker must contain work') compose('restart', 'api', 'broker') def ready(): try: return c.call('GET', '/health/ready', [200, 503])['status'] == 200 except Failure: return False wait(ready, 'API ready') compose('start', 'worker') c.wait_ready(alice, att) check(c.call('GET', base + att['id'] + '/content', actor=bob)['bytes'] == data, 'restart lost original') passed('stopped_worker_API_broker_restart', attachment_id=att['id']) compose('stop', 'broker') unavailable = upload(fixture((324, 204))) check(c.call('GET', '/health/ready')['status'] == 200, 'broker outage must not block API') check(job(unavailable)['published_at'] is None, 'unsent job must remain in DB') compose('start', 'broker') c.wait_ready(alice, unavailable) passed('broker_unavailable_outbox_recovered') # COMMIT is already durable; kill the actual relay before and after XADD. for fault, code in [('before_publish', 81), ('after_publish', 82)]: compose('stop', 'dispatcher', 'worker') att = upload(fixture((321, 201))) set_fault(att, fault) check(job(att)['published_at'] is None, 'must have unsent outbox') compose('start', 'dispatcher') wait(lambda: exited('dispatcher', code), fault) check(job(att)['published_at'] is None, 'relay crash must roll back sent flag') # Dispatcher is stopped; clear through postgres, then restore the same job. set_fault(att, None) compose('start', 'dispatcher', 'worker') c.wait_ready(alice, att) passed(fault + '_recovered', job_id=job(att)['id']) for fault, code in [('after_objects', 83), ('after_commit', 84)]: compose('stop', 'worker') att = upload(fixture((322, 202))) set_fault(att, fault) compose('start', 'worker') wait(lambda: exited('worker', code), fault) before = objects(att) check(len(before) == 4, 'one original and three objects must exist') j = job(att) if fault == 'after_objects': check(j['state'] == 'processing', 'crash must leave a processing lease') else: check(j['state'] == 'ready', 'ready commit must precede ack') check(pending() > 0, 'crash must leave an unacked delivery') set_fault(att, None) compose('start', 'worker') c.wait_ready(alice, att) wait(lambda: pending() == 0, 'pending delivery recovery') check(objects(att) == before, 'redelivery changed/duplicated objects') stable = job(att) inject(att); inject(att) wait(lambda: pending() == 0 and int(compose('exec', '-T', 'broker', 'redis-cli', 'XLEN', 'attachment-jobs-v1')) == 0, 'duplicate ack') check(job(att)['attempt'] == stable['attempt'], 'ready duplicate must not start a new attempt') check(objects(att) == before, 'ready duplicate changed output') passed(fault + '_redelivery_idempotent', attempt=stable['attempt'], objects=len(before)) for fault in ['temporary', 'permanent', 'timeout']: compose('stop', 'worker') att = upload(fixture((323, 203))) set_fault(att, fault) compose('start', 'worker') failed = wait(lambda: (a if (a := c.call('GET', base + att['id'], actor=alice, schema='Attachment')['json'])['status'] == 'failed' else None), 'terminal failed') j = job(att) check(j['cycle_attempt'] == (1 if fault == 'permanent' else 3), 'retry budget/classification') check(failed['error_code'] == ('invalid_image' if fault == 'permanent' else 'retry_exhausted'), 'safe terminal code') c.call('GET', base + att['id'] + '/content', 409, actor=bob) c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, body={'text':'', 'attachment_ids':[att['id']]}, headers={'Idempotency-Key':uuid.uuid4().hex}) registry = json.loads(compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'failed')) check(any(x['job_id'] == j['id'] for x in registry), 'failed registry missing') set_fault(att, None) compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', j['id'], '--operator', 'lab5-test', '--reason', 'dependency restored') c.wait_ready(alice, att) check(sql("SELECT count(*) FROM job_replays WHERE job_id='" + j['id'] + "';") == '1', 'audit missing') compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', j['id'], '--operator', 'lab5-test', '--reason', 'duplicate replay', expected=1) passed(fault + '_failed_audited_replay', attempts_before_replay=j['attempt']) wait(lambda: sql("SELECT count(*) FROM attachments WHERE status='ready' AND staging_key IS NOT NULL;") == '0', 'ready staging cleanup') passed('staging_cleanup_after_ready') compose('stop', 'worker') retained = upload(fixture((325, 205))) set_fault(retained, 'permanent') compose('start', 'worker') wait(lambda: job(retained)['state'] == 'failed', 'failed retention fixture') check(sql("SELECT staging_key IS NOT NULL FROM attachments WHERE id='" + retained['id'] + "';") == 't', 'fresh failed staging must remain') sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '8 days' WHERE id='" + retained['id'] + "';") wait(lambda: sql("SELECT staging_key IS NULL FROM attachments WHERE id='" + retained['id'] + "';") == 't', 'failed retention cleanup') compose('exec', '-T', 'dispatcher', 'python', '-m', 'src.pipeline.admin', 'replay', job(retained)['id'], '--operator', 'lab5-test', '--reason', 'expired source', expected=1) passed('failed_retention_and_expired_replay_rejected') compose('stop', 'worker') active = upload(fixture((326, 206))) sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '8 days' WHERE id='" + active['id'] + "';") orphan = 'staging/' + str(uuid.uuid4()) script = '''import io,sys from src.core.storage import make_storage from src.core.settings import settings make_storage(role='worker').put_object(settings.s3_bucket,sys.argv[1],io.BytesIO(b'orphan'),6) ''' compose('exec', '-T', 'dispatcher', 'python', '-c', script, orphan) sql("INSERT INTO upload_intents(object_key,state,created_at) VALUES ('" + orphan + "','pending',clock_timestamp()-interval '8 days');") wait(lambda: sql("SELECT state FROM upload_intents WHERE object_key='" + orphan + "';") == 'abandoned', 'orphan staging cleanup') check(sql("SELECT staging_key IS NOT NULL FROM attachments WHERE id='" + active['id'] + "';") == 't', 'cleanup must protect queued work') compose('start', 'worker') c.wait_ready(alice, active) passed('abandoned_staging_cleaned_active_job_protected') finally: # Disable every injection and restore the ordinary configuration even on failure. try: sql('UPDATE attachment_jobs SET fault=NULL WHERE fault IS NOT NULL;') finally: subprocess.run(['docker','compose','--env-file',os.environ.get('ENV_FILE','.env'),'-p',args.project,'up','-d','--no-deps','--force-recreate','worker','dispatcher'], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=100, check=True) override.unlink(missing_ok=True) override.parent.rmdir() if __name__ == '__main__': run(main)