#!/usr/bin/env python3 """Defense scenarios for background attachment processing; run inside the tools container.""" import atexit import base64 import hashlib import json import os import re import subprocess import sys import time import urllib.error import urllib.request import uuid BASE = os.environ.get('BASE_URL', 'http://localhost:8080') SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read() STARTED = time.monotonic() ANSI = re.compile(chr(27) + re.escape('[') + '[0-9;]*m') def log(*parts): print('[%6.1fs]' % (time.monotonic() - STARTED), *parts, flush=True) def call(method, path, body=None, headers=None, raw=None): data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None) all_headers = dict(headers or {}) if body is not None: all_headers['Content-Type'] = 'application/json' request = urllib.request.Request(BASE + path, data=data, method=method, headers=all_headers) try: with urllib.request.urlopen(request, timeout=15) as response: return response.status, response.read(), response.headers except urllib.error.HTTPError as error: return error.code, error.read(), error.headers except OSError as error: return 0, str(error).encode(), {} def json_of(raw): try: return json.loads(raw or b'null') except ValueError: return None def compose(*args): log('$ docker compose', ' '.join(args)) subprocess.run(['docker', 'compose', *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def start_only(service): """docker compose start also starts dependencies (worker -> s3-init -> s3); start just this container.""" log('$ docker start', service, '(without dependencies)') ids = subprocess.run(['docker', 'compose', 'ps', '-aq', service], check=True, capture_output=True, text=True).stdout.split() subprocess.run(['docker', 'start', *ids], check=True, stdout=subprocess.DEVNULL) def psql(sql): out = subprocess.run( ['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tA', '-F', ' | ', '-c', sql], check=True, capture_output=True, text=True, ) return out.stdout.strip() def queue_depth(): time.sleep(6) out = subprocess.run( ['docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_queues', 'name', 'type', 'durable', 'messages_ready', 'messages_unacknowledged'], capture_output=True, text=True, ) return out.stdout.strip() or out.stderr.strip() def wait_ready(timeout=90): deadline = time.monotonic() + timeout while time.monotonic() < deadline: if call('GET', '/health/ready')[0] == 200: return time.sleep(1) sys.exit('API did not become ready') def session(): username = 'job_' + uuid.uuid4().hex[:8] password = 'Job-scenario-' + uuid.uuid4().hex code, raw, _ = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Job scenario', 'password': password}) if code != 201: sys.exit('register failed: %s' % code) user = json_of(raw) credential = base64.b64encode((username + ':' + password).encode()).decode() code, raw, _ = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential}) if code != 201: sys.exit('login failed: %s' % code) auth = {'Authorization': 'Bearer ' + json_of(raw)['token']} code, raw, _ = call('POST', '/api/v1/chats', {'title': 'Job scenario', 'member_ids': [user['id']]}, auth) if code != 201: sys.exit('chat failed: %s' % code) chat = json_of(raw)['id'] log('user', user['id'], 'chat', chat) return auth, chat def upload(auth, chat, body=SAMPLE): code, raw, headers = call( 'POST', '/api/v1/chats/%s/attachments?filename=sample.png' % chat, headers={**auth, 'Content-Type': 'image/png'}, raw=body, ) attachment = json_of(raw) log('POST attachment ->', code, 'status=%s' % attachment.get('status'), 'id=%s' % attachment.get('id'), 'instance=%s' % headers.get('X-Instance-Id')) if code != 202: sys.exit('upload failed') return attachment['id'] def message(auth, chat, text, attachments=()): code, raw, _ = call( 'POST', '/api/v1/chats/%s/messages' % chat, {'text': text, 'attachment_ids': list(attachments)}, {**auth, 'Idempotency-Key': uuid.uuid4().hex}, ) log('POST message', 'with %d attachment(s)' % len(attachments), '->', code, 'id=%s' % (json_of(raw) or {}).get('id')) return code def metadata(auth, attachment): code, raw, _ = call('GET', '/api/v1/attachments/' + attachment, headers=auth) return code, json_of(raw) def content_status(auth, attachment, variant='original'): return call('GET', '/api/v1/attachments/%s/content?variant=%s' % (attachment, variant), headers=auth)[0] def wait_status(auth, attachment, wanted, timeout=150): deadline = time.monotonic() + timeout last = None while time.monotonic() < deadline: code, meta = metadata(auth, attachment) if code == 200 and meta['status'] != last: last = meta['status'] log('GET attachment -> status=%s error_code=%s' % (last, meta['error_code'])) if code == 200 and meta['status'] in wanted: return meta time.sleep(1) sys.exit('attachment did not reach %s' % '/'.join(wanted)) def check_ready(auth, attachment, meta): code, raw, _ = call('GET', '/api/v1/attachments/%s/content' % attachment, headers=auth) log('original ->', code, 'sha256 matches upload:', code == 200 and hashlib.sha256(raw).hexdigest() == hashlib.sha256(SAMPLE).hexdigest()) for variant in meta['variants']: code, raw, headers = call('GET', '/api/v1/attachments/%s/content?variant=%s' % (attachment, variant['name']), headers=auth) log('%-9s -> %s %s %dx%d sha256 matches metadata: %s' % ( variant['name'], code, headers.get('Content-Type'), variant['width'], variant['height'], hashlib.sha256(raw).hexdigest() == variant['sha256'])) def job_state(attachment): log('job:', psql("SELECT id, status, attempt, max_attempts, error_code FROM attachment_jobs WHERE attachment_id = '%s'" % attachment)) log('variants rows:', psql("SELECT count(*) FROM attachment_variants WHERE attachment_id = '%s'" % attachment)) log('events:', psql("SELECT string_agg(e.event, ' -> ' ORDER BY e.id) FROM job_events e JOIN attachment_jobs j ON j.id = e.job_id WHERE j.attachment_id = '%s'" % attachment)) def scenario_restart(): """Level 3: worker down -> upload 202 -> restart API and broker -> worker up -> ready.""" wait_ready() auth, chat = session() compose('stop', 'worker') attachment = upload(auth, chat) log('content while queued ->', content_status(auth, attachment)) message(auth, chat, '', [attachment]) message(auth, chat, 'other messages keep working') log('queues:', queue_depth()) compose('restart', 'api', 'rabbitmq') wait_ready() log('queues after broker restart:', queue_depth()) log('status after restarts:', metadata(auth, attachment)[1]['status']) compose('start', 'worker') meta = wait_status(auth, attachment, ['ready', 'failed']) check_ready(auth, attachment, meta) job_state(attachment) def remove_container(name): subprocess.run(['docker', 'rm', '-f', name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def plain(line): return ANSI.sub('', line).split('] ')[-1].strip() def faulty(service, env): """Run a one-off copy of the service with fault injection; no restart policy, so it dies once.""" name = 'lab6-fault-%s-%s' % (service, uuid.uuid4().hex[:8]) atexit.register(remove_container, name) args = ['docker', 'compose', 'run', '-d', '--no-deps', '--name', name] for key, value in env.items(): args += ['-e', '%s=%s' % (key, value)] log('$ docker compose run --no-deps', ' '.join('-e %s=%s' % item for item in env.items()), service) subprocess.run(args + [service], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) deadline = time.monotonic() + 30 while time.monotonic() < deadline: if 'connected to broker' in subprocess.run(['docker', 'logs', name], capture_output=True, text=True).stdout: return name time.sleep(0.5) sys.exit('%s with fault did not start' % service) def wait_exit(name, timeout=90): try: code = subprocess.run(['docker', 'wait', name], capture_output=True, text=True, timeout=timeout).stdout.strip() except subprocess.TimeoutExpired: code = 'timeout' for line in subprocess.run(['docker', 'logs', name], capture_output=True, text=True).stdout.splitlines(): if 'fault injection' in line or 'published' in line or 'job ' in line: log(' one-off:', plain(line)) remove_container(name) log('one-off process exit code', code, '(137 = killed without ack)') def service_logs(service, since, needle): out = subprocess.run(['docker', 'compose', 'logs', '--no-log-prefix', '--no-color', '--since', since, service], capture_output=True, text=True).stdout for line in out.splitlines(): if needle in line: log(' %s:' % service, plain(line)) def utc_now(): return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) def staging_key(attachment): return psql("SELECT coalesce(staging_key, 'NULL') FROM attachments WHERE id = '%s'" % attachment) def job_id(attachment): return psql("SELECT id FROM attachment_jobs WHERE attachment_id = '%s'" % attachment) def scenario_crash_after_store(): """Level 4: worker killed after writing S3 objects, before COMMIT and ack; lease expires; same result.""" wait_ready() since = utc_now() auth, chat = session() compose('stop', 'worker') proc = faulty('worker', {'WORKER_FAULT': 'exit-after-store', 'JOB_LEASE_SECONDS': '15'}) attachment = upload(auth, chat) wait_exit(proc) log('status after crash:', metadata(auth, attachment)[1]['status']) log('lease:', psql("SELECT status, attempt, lease_until > now() AS lease_active FROM attachment_jobs WHERE attachment_id = '%s'" % attachment)) log('queues (unacked message returned):', queue_depth()) compose('start', 'worker') meta = wait_status(auth, attachment, ['ready', 'failed']) check_ready(auth, attachment, meta) job_state(attachment) service_logs('worker', since, job_id(attachment)) service_logs('relay', since, 'lease expired') def scenario_crash_after_commit(): """Level 4: worker killed after COMMIT ready, before ack; redelivery is a no-op and cleans staging.""" wait_ready() since = utc_now() auth, chat = session() compose('stop', 'worker') proc = faulty('worker', {'WORKER_FAULT': 'exit-after-commit'}) attachment = upload(auth, chat) wait_exit(proc) log('status after crash:', metadata(auth, attachment)[1]['status'], 'staging_key:', staging_key(attachment)) log('queues (unacked message returned):', queue_depth()) compose('start', 'worker') deadline = time.monotonic() + 30 while staging_key(attachment) != 'NULL' and time.monotonic() < deadline: time.sleep(1) log('staging_key after redelivery:', staging_key(attachment)) check_ready(auth, attachment, metadata(auth, attachment)[1]) job_state(attachment) service_logs('worker', since, job_id(attachment)) def scenario_outbox_gap(): """Level 4: crash between COMMIT and publish; the outbox row is published after recovery.""" wait_ready() auth, chat = session() compose('stop', 'relay') attachment = upload(auth, chat) outbox = "SELECT o.id, coalesce(o.published_at::text, 'NOT PUBLISHED'), o.publish_attempts FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = '%s'" % attachment log('outbox after COMMIT:', psql(outbox)) log('queues:', queue_depth()) compose('kill', 'api') log('API killed (SIGKILL) after COMMIT, nothing was published') compose('start', 'api') wait_ready() log('status after API restart:', metadata(auth, attachment)[1]['status']) compose('start', 'relay') meta = wait_status(auth, attachment, ['ready', 'failed']) log('outbox after relay recovery:', psql(outbox)) check_ready(auth, attachment, meta) job_state(attachment) def scenario_relay_crash(): """Level 4: relay killed after broker confirm, before marking outbox; duplicate delivery is a no-op.""" wait_ready() since = utc_now() auth, chat = session() compose('stop', 'relay') proc = faulty('relay', {'RELAY_FAULT': 'exit-after-confirm'}) attachment = upload(auth, chat) wait_exit(proc) outbox = "SELECT o.id, coalesce(o.published_at::text, 'NOT MARKED'), o.publish_attempts FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = '%s'" % attachment log('outbox after relay crash:', psql(outbox)) compose('start', 'relay') meta = wait_status(auth, attachment, ['ready', 'failed']) time.sleep(3) log('outbox after relay restart:', psql(outbox)) check_ready(auth, attachment, meta) job_state(attachment) service_logs('worker', since, job_id(attachment)) def replay(*args): out = subprocess.run(['docker', 'compose', 'exec', '-T', '-e', 'OPERATOR=defense-demo', 'worker', 'node', 'dist/replay.js', *args], capture_output=True, text=True) for line in (out.stdout.strip() or out.stderr.strip()).splitlines(): log(' replay:', line) def scenario_retry_replay(): """Level 4: S3 down -> retries with backoff and jitter -> failed retry_exhausted -> explicit replay -> ready.""" wait_ready() auth, chat = session() compose('stop', 'worker') attachment = upload(auth, chat) compose('stop', 's3') start_only('worker') meta = wait_status(auth, attachment, ['failed'], timeout=120) log('failed attachment in a new message ->', message(auth, chat, '', [attachment])) log('retries:') for line in psql("SELECT e.attempt, e.event, coalesce(e.detail, ''), to_char(e.created_at, 'HH24:MI:SS.MS') FROM job_events e JOIN attachment_jobs j ON j.id = e.job_id WHERE j.attachment_id = '%s' ORDER BY e.id" % attachment).splitlines(): log(' ', line) log('$ replay.js --list') replay('--list') compose('start', 's3') wait_ready() log('$ replay.js', attachment) replay(attachment) meta = wait_status(auth, attachment, ['ready', 'failed']) check_ready(auth, attachment, meta) log('$ replay.js', attachment, '(already ready)') replay(attachment) job_state(attachment) def scenario_broker(): """Level 4: prefetch/concurrency, limits and dead letter queue for a malformed message.""" wait_ready() since = utc_now() log('consumers:', subprocess.run( ['docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_consumers', 'queue_name', 'prefetch_count', 'ack_required'], capture_output=True, text=True).stdout.strip()) log('worker limits:', subprocess.run( ['sh', '-c', "docker inspect -f 'cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' $(docker compose ps -q worker)"], capture_output=True, text=True).stdout.strip()) publish = ( "const amqp=require('amqplib');const fs=require('fs');(async()=>{" "const c=await amqp.connect(fs.readFileSync(process.env.BROKER_URL_FILE,'utf8').trim());" "const ch=await c.createConfirmChannel();" "ch.sendToQueue('attachment.jobs',Buffer.from(JSON.stringify({job_id:'not-a-uuid',url:'http://example.invalid/x'})),{persistent:true});" "await ch.waitForConfirms();await c.close();console.log('malformed message published');})()" ) log(subprocess.run(['docker', 'compose', 'exec', '-T', 'worker', 'node', '-e', publish], capture_output=True, text=True).stdout.strip()) log('queues:', queue_depth()) service_logs('worker', since, 'malformed') SCENARIOS = { 'restart': scenario_restart, 'crash-after-store': scenario_crash_after_store, 'crash-after-commit': scenario_crash_after_commit, 'outbox-gap': scenario_outbox_gap, 'relay-crash': scenario_relay_crash, 'retry-replay': scenario_retry_replay, 'broker': scenario_broker, } if __name__ == '__main__': if len(sys.argv) != 2 or sys.argv[1] not in SCENARIOS: sys.exit('usage: job-scenario.py %s' % '|'.join(SCENARIOS)) SCENARIOS[sys.argv[1]]()