488 lines
25 KiB
Python
488 lines
25 KiB
Python
#!/usr/bin/env python3
|
|
"""Repeatable grade 3/4 failure checks for the local synthetic lab5 Compose stand.
|
|
|
|
Run separately from other tests: scenarios deliberately restart/stop services.
|
|
Every scenario restores the original services in finally. No volumes, secrets or
|
|
application records are removed. Tokens live only in process memory.
|
|
"""
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, check
|
|
|
|
|
|
def command(*args):
|
|
result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True,
|
|
encoding='utf-8', errors='replace')
|
|
if result.returncode:
|
|
# Never print environment, response bodies, credentials or arbitrary Docker logs.
|
|
raise Failure('command failed: ' + ' '.join(args[:4]) + ' (exit %d)' % result.returncode)
|
|
return result.stdout.strip()
|
|
|
|
|
|
def compose(*args):
|
|
return command('docker', 'compose', *args)
|
|
|
|
|
|
def sql(query):
|
|
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query)
|
|
|
|
|
|
def rows(query):
|
|
value = sql("SELECT COALESCE(json_agg(t), '[]'::json) FROM (%s) t" % query)
|
|
return json.loads(value)
|
|
|
|
|
|
def identifier(value):
|
|
return str(uuid.UUID(value))
|
|
|
|
|
|
def job(attachment_id):
|
|
data = rows("SELECT * FROM attachment_jobs WHERE attachment_id='%s'" % identifier(attachment_id))
|
|
check(len(data) == 1, 'attachment must have exactly one logical job')
|
|
return data[0]
|
|
|
|
|
|
def audit(job_id):
|
|
return rows("SELECT action,actor,reason,details,created_at FROM job_audit WHERE job_id='%s' ORDER BY id"
|
|
% identifier(job_id))
|
|
|
|
|
|
def cli(*args):
|
|
# The explicit operator service has broker/worker permissions. The API does
|
|
# not receive broker credentials merely to make an evidence script easier.
|
|
output = compose('--profile', 'tools', 'run', '--rm', '--no-deps', 'jobs', *args)
|
|
return json.loads(output.splitlines()[-1]) if output else None
|
|
|
|
|
|
def inject(job_id, action, count=1):
|
|
return cli('inject', identifier(job_id), '--action', action, '--count', str(count),
|
|
'--actor', 'lab5-evidence', '--reason', 'repeatable synthetic grade 4 failure check')
|
|
|
|
|
|
def wait_until(predicate, description, seconds=180):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
value = predicate()
|
|
if value:
|
|
return value
|
|
time.sleep(0.4)
|
|
raise Failure('deadline exceeded: ' + description)
|
|
|
|
|
|
def ready(client):
|
|
def probe():
|
|
try:
|
|
client.call('GET', '/health/ready', schema='Health')
|
|
return True
|
|
except Failure:
|
|
return False
|
|
wait_until(probe, 'API ready', 90)
|
|
|
|
|
|
def restore(client, *services):
|
|
# start preserves stopped container configuration and all named volumes.
|
|
compose('start', *services)
|
|
ready(client)
|
|
|
|
|
|
def upload(client, actor, chat, label):
|
|
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
result = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + label + '.png',
|
|
202, actor=actor, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
attachment = result['json']
|
|
check(attachment['status'] == 'queued', '202 must carry a queued snapshot')
|
|
check(attachment['variants'] == [] and attachment['error_code'] is None, 'queued result fields')
|
|
check(attachment['sha256'] == hashlib.sha256(content).hexdigest(), 'accepted original checksum')
|
|
return attachment, content
|
|
|
|
|
|
def metadata(client, actor, attachment):
|
|
return client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json']
|
|
|
|
|
|
def wait_state(client, actor, attachment, status):
|
|
def probe():
|
|
current = metadata(client, actor, attachment)
|
|
if current['status'] == status:
|
|
return current
|
|
check(current['status'] != 'failed' or status == 'failed', 'unexpected terminal failed state')
|
|
return wait_until(probe, 'attachment ' + status)
|
|
|
|
|
|
def verified_ready(client, actor, attachment, content):
|
|
result = wait_state(client, actor, attachment, 'ready')
|
|
check(result['id'] == attachment['id'], 'logical attachment identity changed')
|
|
check(result['error_code'] is None, 'ready error_code must be null')
|
|
check({item['name'] for item in result['variants']} == {'thumbnail', 'crop', 'optimized'}, 'grade 4 variants')
|
|
check(len(result['variants']) == 3, 'duplicate result variants')
|
|
for name in ['original', 'thumbnail', 'crop', 'optimized']:
|
|
data = client.call('GET', '/api/v1/attachments/' + result['id'] + '/content?variant=' + name, actor=actor)
|
|
record = result if name == 'original' else next(v for v in result['variants'] if v['name'] == name)
|
|
check(len(data['bytes']) == record['size_bytes'], 'published object size mismatch')
|
|
check(hashlib.sha256(data['bytes']).hexdigest() == record['sha256'], 'published object checksum mismatch')
|
|
if name == 'original':
|
|
check(data['bytes'] == content, 'original bytes changed')
|
|
count = int(sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % result['id']))
|
|
check(count == 3, 'exactly three persistent variant records required')
|
|
return result
|
|
|
|
|
|
def object_state(attachment_id):
|
|
# Runs with the API's restricted bucket credentials; only selected facts return.
|
|
aid = identifier(attachment_id)
|
|
code = """import json
|
|
from messenger.config import read_secret
|
|
from messenger.db import create_pool
|
|
from messenger.objects import ObjectStore
|
|
from botocore.exceptions import ClientError
|
|
p=create_pool(read_secret('DATABASE_URL'),1);p.open(wait=True)
|
|
with p.connection() as c:
|
|
r=c.execute('SELECT object_key,staging_key,staging_deleted_at,status,pipeline_version FROM attachments WHERE id=%%s',(%r,)).fetchone()
|
|
s=ObjectStore()
|
|
def exists(key):
|
|
try: s.client.head_object(Bucket=s.bucket,Key=key);return True
|
|
except ClientError as e:
|
|
if e.response['ResponseMetadata']['HTTPStatusCode']==404:return False
|
|
raise
|
|
prefix='variants/'+%r.replace('-','')+'/'
|
|
objects=s.client.list_objects_v2(Bucket=s.bucket,Prefix=prefix).get('Contents',[])
|
|
print(json.dumps(dict(final=exists(r['object_key']),staging=exists(r['staging_key']),status=r['status'],staging_deleted=r['staging_deleted_at'] is not None,variant_objects=len(objects))))
|
|
p.close()
|
|
""" % (aid, aid)
|
|
return json.loads(compose('exec', '-T', 'api', 'python', '-c', code).splitlines()[-1])
|
|
|
|
|
|
def effect_snapshot(attachment_id):
|
|
aid = identifier(attachment_id)
|
|
return rows("SELECT a.status,a.object_key,a.sha256,a.updated_at,j.id AS job_id,j.attempt,j.updated_at AS job_updated_at,"
|
|
"(SELECT json_agg(v ORDER BY v.name) FROM attachment_variants v WHERE v.attachment_id=a.id) AS variants "
|
|
"FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.id='%s'" % aid)[0]
|
|
|
|
|
|
def fault_consumed(job_id, action):
|
|
return bool(rows("SELECT 1 AS consumed FROM job_faults WHERE job_id='%s' AND action='%s' AND remaining=0"
|
|
% (identifier(job_id), action)))
|
|
|
|
|
|
def stale_lease_fenced(attachment_id, old_job):
|
|
"""A dead invocation cannot commit success or failure over the recovered result."""
|
|
old_claim = dict(job_id=old_job['id'], lease_token=old_job['lease_token'],
|
|
attempt=old_job['attempt'], pipeline_version=old_job['pipeline_version'])
|
|
code = """import json
|
|
from messenger.config import read_secret
|
|
from messenger.db import create_pool
|
|
from messenger.jobs import Jobs
|
|
p=create_pool(read_secret('DATABASE_URL'),1);p.open(wait=True);j=Jobs(p)
|
|
claim=json.loads(%r)
|
|
with p.connection() as c:
|
|
a=c.execute('SELECT object_key FROM attachments WHERE id=%%s',(%r,)).fetchone()
|
|
variants=c.execute('SELECT * FROM attachment_variants WHERE attachment_id=%%s ORDER BY name',(%r,)).fetchall()
|
|
assert j.finish(claim,a['object_key'],variants) is False,'old lease committed success'
|
|
assert j.fail(claim,'stale_worker',False,5,0) is False,'old lease committed failure'
|
|
p.close();print(json.dumps(dict(old_finish=False,old_fail=False)))
|
|
""" % (json.dumps(old_claim), identifier(attachment_id), identifier(attachment_id))
|
|
return json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code).splitlines()[-1])
|
|
|
|
|
|
def stack(client):
|
|
config = json.loads(compose('config', '--format', 'json'))
|
|
services = config['services']
|
|
for name in ('api', 'worker', 'dispatcher', 'postgres', 'valkey', 'broker', 's3'):
|
|
check(name in services, 'missing independent service ' + name)
|
|
check(not services[name].get('ports'), name + ': internal ports must not be published')
|
|
check(services['api']['deploy']['replicas'] >= 2, 'two API replicas required')
|
|
check(any(v.get('type') == 'volume' for v in services['broker'].get('volumes', [])), 'broker needs persistent volume')
|
|
check(services['worker'].get('deploy', {}).get('resources', {}).get('limits', {}).get('memory'),
|
|
'worker memory limit required')
|
|
print('PASS stack: separate durable broker/worker/dispatcher, private internal ports, bounded worker memory', flush=True)
|
|
|
|
|
|
def durability(client):
|
|
actor, member, _, chat = client.bootstrap()
|
|
try:
|
|
compose('stop', 'worker')
|
|
attachment, content = upload(client, actor, chat, 'worker-down')
|
|
logical_job = job(attachment['id'])['id']
|
|
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=member)
|
|
message = client.message(member, chat, '', attachments=[attachment['id']])['json']
|
|
check(message['attachment_ids'] == [attachment['id']], 'queued attachment cannot be linked')
|
|
check(object_state(attachment['id'])['staging'], '202 lost durable staging')
|
|
wait_until(lambda: bool(rows("SELECT 1 FROM attachment_outbox WHERE job_id='%s' AND published_at IS NOT NULL"
|
|
% logical_job)), 'broker publication before restart')
|
|
compose('restart', 'api', 'broker')
|
|
ready(client)
|
|
check(metadata(client, member, attachment)['status'] == 'queued', 'worker-down status must remain queued')
|
|
history = client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=member, schema='MessagePage')['json']
|
|
check(any(item['id'] == message['id'] for item in history['items']), 'linked message lost across restart')
|
|
compose('start', 'worker')
|
|
verified_ready(client, member, attachment, content)
|
|
check(job(attachment['id'])['id'] == logical_job, 'restart created a replacement logical job')
|
|
print('PASS durability: worker down -> 202+linked message+409 -> API/broker restart with volumes -> same job ready; job_id=' + logical_job, flush=True)
|
|
finally:
|
|
restore(client, 'api', 'broker', 'worker')
|
|
|
|
|
|
def outbox(client):
|
|
actor, member, _, chat = client.bootstrap()
|
|
try:
|
|
compose('stop', 'dispatcher', 'worker')
|
|
attachment, content = upload(client, actor, chat, 'commit-publish-gap')
|
|
logical_job = job(attachment['id'])['id']
|
|
pending = rows("SELECT attempt,published_at FROM attachment_outbox WHERE job_id='%s'" % logical_job)
|
|
check(len(pending) == 1 and pending[0]['published_at'] is None, 'COMMIT/publish gap was not isolated')
|
|
# API is killed after its successful COMMIT while publisher is absent.
|
|
compose('kill', '-s', 'SIGKILL', 'api')
|
|
compose('start', 'api')
|
|
ready(client)
|
|
check(job(attachment['id'])['id'] == logical_job, 'COMMIT intent lost after API crash')
|
|
compose('start', 'dispatcher', 'worker')
|
|
verified_ready(client, member, attachment, content)
|
|
check(job(attachment['id'])['id'] == logical_job, 'outbox recovery recreated the job')
|
|
print('PASS outbox: COMMIT persisted while dispatcher stopped -> kill API before publish -> restart -> same job ready; job_id=' + logical_job, flush=True)
|
|
finally:
|
|
restore(client, 'api', 'dispatcher', 'worker')
|
|
|
|
|
|
def broker_outage(client):
|
|
actor, member, _, chat = client.bootstrap()
|
|
try:
|
|
compose('stop', 'broker', 'worker')
|
|
ready(client)
|
|
attachment, content = upload(client, actor, chat, 'broker-down')
|
|
logical_job = job(attachment['id'])['id']
|
|
client.message(member, chat, 'broker outage does not stop messages', attachments=[attachment['id']])
|
|
check(bool(rows("SELECT 1 FROM attachment_outbox WHERE job_id='%s' AND published_at IS NULL" % logical_job)),
|
|
'broker-down upload lacks durable outbox intent')
|
|
compose('start', 'broker', 'worker')
|
|
verified_ready(client, member, attachment, content)
|
|
print('PASS broker outage: API remains ready, upload 202 and messages commit, durable outbox later reaches ready; job_id=' + logical_job, flush=True)
|
|
finally:
|
|
restore(client, 'broker', 'dispatcher', 'worker')
|
|
|
|
|
|
def wait_delivery_ack(message_id):
|
|
code = """import json
|
|
from messenger.broker import Broker
|
|
b=Broker();target=tuple(map(int,%r.split('-')))
|
|
groups=b.client.xinfo_groups(b.stream)
|
|
group=next((g for g in groups if g['name']==b.group),None)
|
|
delivered=bool(group and tuple(map(int,group['last-delivered-id'].split('-')))>=target)
|
|
pending=b.client.xpending_range(b.stream,b.group,%r,%r,1)
|
|
print(json.dumps(dict(done=delivered and not pending)));b.close()
|
|
""" % (message_id, message_id, message_id)
|
|
wait_until(lambda: json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code))['done'], 'redelivery ack')
|
|
|
|
|
|
def redelivery(client):
|
|
actor, member, _, chat = client.bootstrap()
|
|
attachment, content = upload(client, actor, chat, 'redelivery')
|
|
completed = verified_ready(client, member, attachment, content)
|
|
before = effect_snapshot(attachment['id'])
|
|
published = cli('redeliver', before['job_id'])
|
|
message_id = published['message_id']
|
|
wait_delivery_ack(message_id)
|
|
after = effect_snapshot(attachment['id'])
|
|
check(before == after, 'ready duplicate mutated persistent result, job attempt or timestamps')
|
|
check(metadata(client, member, attachment) == completed, 'ready duplicate changed public metadata')
|
|
print('PASS redelivery: same ready job acked without mutation, same 3 variants/checksums; job_id=' + before['job_id'], flush=True)
|
|
|
|
|
|
def crash(client):
|
|
actor, member, _, chat = client.bootstrap()
|
|
try:
|
|
compose('stop', 'worker')
|
|
attachment, content = upload(client, actor, chat, 'crash-after-objects')
|
|
logical_job = job(attachment['id'])['id']
|
|
inject(logical_job, 'crash_after_objects')
|
|
compose('start', 'worker')
|
|
wait_until(lambda: fault_consumed(logical_job, 'crash_after_objects'), 'crash fault consumed')
|
|
at_crash = object_state(attachment['id'])
|
|
check(at_crash['final'] and at_crash['variant_objects'] == 3,
|
|
'injected crash occurred before all four final objects existed')
|
|
check(at_crash['status'] == 'processing' and at_crash['staging'],
|
|
'crash window must precede ready commit and retain staging')
|
|
check(not rows("SELECT 1 FROM attachment_variants WHERE attachment_id='%s'" % attachment['id']),
|
|
'crash window must precede publication of variant metadata')
|
|
# The durable hook is consumed before exit, so container restart cannot loop it.
|
|
compose('start', 'worker')
|
|
verified_ready(client, member, attachment, content)
|
|
result = job(attachment['id'])
|
|
check(result['id'] == logical_job and result['attempt'] >= 2, 'crashed job did not recover its expired lease')
|
|
check(object_state(attachment['id'])['variant_objects'] == 3, 'redelivery duplicated S3 variants')
|
|
print('PASS crash after 4 S3 writes before DB result/ack: same job recovered; no duplicate variants; job_id=%s attempts=%s window=%s'
|
|
% (logical_job, result['attempt'], json.dumps(at_crash, sort_keys=True)), flush=True)
|
|
finally:
|
|
restore(client, 'worker')
|
|
|
|
|
|
def lease(client):
|
|
actor, member, _, chat = client.bootstrap()
|
|
try:
|
|
compose('stop', 'worker')
|
|
attachment, content = upload(client, actor, chat, 'expired-processing')
|
|
logical_job = job(attachment['id'])['id']
|
|
inject(logical_job, 'stall_processing')
|
|
compose('start', 'worker')
|
|
wait_until(lambda: fault_consumed(logical_job, 'stall_processing'), 'processing stall hook')
|
|
held = job(attachment['id'])
|
|
check(held['status'] == 'processing' and held['lease_until'], 'stall did not hold a processing lease')
|
|
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=member)
|
|
compose('kill', '-s', 'SIGKILL', 'worker')
|
|
compose('start', 'worker')
|
|
verified_ready(client, member, attachment, content)
|
|
recovered = job(attachment['id'])
|
|
check(recovered['id'] == logical_job and recovered['attempt'] > held['attempt'], 'expired processing was not reclaimed')
|
|
snapshot = effect_snapshot(attachment['id'])
|
|
stale_lease_fenced(attachment['id'], held)
|
|
check(effect_snapshot(attachment['id']) == snapshot, 'stale lease changed the recovered ready result')
|
|
print('PASS expired lease: killed stalled processing worker -> automatic recovery, same job; job_id=%s old_lease_until=%s attempts=%d'
|
|
% (logical_job, held['lease_until'], recovered['attempt']), flush=True)
|
|
print('PASS fencing: old lease cannot finish or fail after recovered ready commit', flush=True)
|
|
finally:
|
|
restore(client, 'worker')
|
|
|
|
|
|
def failed_contract(client, actor, chat, attachment):
|
|
failed = wait_state(client, actor, attachment, 'failed')
|
|
check(bool(re.fullmatch(r'[a-z][a-z0-9_]{0,63}', failed['error_code'] or '')), 'safe terminal error code required')
|
|
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=actor)
|
|
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=actor,
|
|
body={'text': '', 'attachment_ids': [attachment['id']]}, headers={'Idempotency-Key': uuid.uuid4().hex})
|
|
return failed
|
|
|
|
|
|
def retries(client):
|
|
actor, member, _, chat = client.bootstrap()
|
|
config = json.loads(compose('config', '--format', 'json'))['services']['worker'].get('environment', {})
|
|
max_attempts = int(config.get('JOB_MAX_ATTEMPTS', 5))
|
|
retry_base = float(config.get('JOB_RETRY_BASE_SECONDS', 1))
|
|
retry_max = float(config.get('JOB_RETRY_MAX_SECONDS', 30))
|
|
check(max_attempts >= 3, 'retry demonstration requires a budget of at least three attempts')
|
|
try:
|
|
compose('stop', 'worker')
|
|
attachment, content = upload(client, actor, chat, 'transient-retry')
|
|
logical_job = job(attachment['id'])['id']
|
|
inject(logical_job, 'transient', 2)
|
|
compose('start', 'worker')
|
|
verified_ready(client, member, attachment, content)
|
|
retried = job(attachment['id'])
|
|
check(retried['attempt'] >= 2, 'transient failure was not retried')
|
|
events = audit(logical_job)
|
|
retry_events = [event for event in events if event['action'] == 'retry' and event['reason'] == 'injected_transient']
|
|
check(len(retry_events) == 2, 'both retry delays must be observable in durable audit')
|
|
randomized = False
|
|
for event in retry_events:
|
|
details = event['details']
|
|
cap = min(retry_max, retry_base * 2.0 ** (details['run_attempts'] - 1))
|
|
check(cap / 2 <= details['delay_seconds'] <= cap, 'retry is outside its equal-jitter exponential interval')
|
|
randomized |= details['delay_seconds'] not in (cap / 2, cap)
|
|
created = dt.datetime.fromisoformat(event['created_at'].replace('Z', '+00:00'))
|
|
available = dt.datetime.fromisoformat(details['available_at'].replace('Z', '+00:00'))
|
|
check(abs((available - created).total_seconds() - details['delay_seconds']) < 0.25,
|
|
'retry due time does not persist the chosen jitter delay')
|
|
check(randomized, 'retry evidence contains no randomized delay')
|
|
print('PASS transient errors retry with durable backoff/jitter; job_id=%s audit=%s'
|
|
% (logical_job, json.dumps(retry_events, sort_keys=True)), flush=True)
|
|
|
|
compose('stop', 'worker')
|
|
attachment, content = upload(client, actor, chat, 'retry-exhausted')
|
|
logical_job = job(attachment['id'])['id']
|
|
inject(logical_job, 'transient', max_attempts)
|
|
compose('start', 'worker')
|
|
failed = failed_contract(client, member, chat, attachment)
|
|
check(failed['error_code'] == 'retry_exhausted', 'exhausted job needs a distinct terminal error code')
|
|
exhausted = job(attachment['id'])
|
|
check(exhausted['run_attempts'] == max_attempts, 'retry budget not enforced')
|
|
check(fault_consumed(logical_job, 'transient'), 'retry fault count does not match exhausted attempts')
|
|
failed_jobs = cli('failed', '--limit', '1000')
|
|
check(any(item['job_id'] == logical_job for item in failed_jobs), 'failed job missing from operator registry')
|
|
check(object_state(attachment['id'])['staging'], 'failed job lost original needed for replay')
|
|
cli('cleanup')
|
|
check(object_state(attachment['id'])['staging'], 'cleanup removed failed input required for replay')
|
|
replay = cli('replay', attachment['id'], '--actor', 'lab5-evidence', '--reason', 'operator resolved synthetic failure')
|
|
check(replay['job_id'] == logical_job, 'operator replay created a different logical job')
|
|
verified_ready(client, member, attachment, content)
|
|
before = effect_snapshot(attachment['id'])
|
|
noop = cli('replay', attachment['id'], '--actor', 'lab5-evidence', '--reason', 'verify successful replay is a no-op')
|
|
check(effect_snapshot(attachment['id']) == before, 'replay re-executed a successful job')
|
|
actions = {event['action'] for event in audit(logical_job) if event['actor'] == 'lab5-evidence'}
|
|
check({'replay', 'replay_noop'} <= actions, 'explicit replay and successful no-op need operator audit records')
|
|
print('PASS retry exhaustion -> failed+409+retained staging -> explicit audited replay -> ready -> ready replay no-op; job_id=%s max_attempts=%d error_code=%s replay=%s'
|
|
% (logical_job, max_attempts, failed['error_code'], json.dumps(noop, sort_keys=True)), flush=True)
|
|
finally:
|
|
restore(client, 'worker')
|
|
|
|
|
|
def permanent(client):
|
|
actor, member, _, chat = client.bootstrap()
|
|
try:
|
|
compose('stop', 'worker')
|
|
attachment, _ = upload(client, actor, chat, 'permanent-failure')
|
|
logical_job = job(attachment['id'])['id']
|
|
inject(logical_job, 'permanent')
|
|
compose('start', 'worker')
|
|
failed = failed_contract(client, member, chat, attachment)
|
|
check(failed['error_code'] == 'invalid_image', 'permanent errors need a distinct safe error code')
|
|
check(job(attachment['id'])['run_attempts'] == 1, 'permanent errors must not consume all retries')
|
|
print('PASS permanent failure: failed after one attempt; content/message rejected with409; job_id=%s error_code=%s'
|
|
% (logical_job, failed['error_code']), flush=True)
|
|
finally:
|
|
restore(client, 'worker')
|
|
|
|
|
|
def staging(client):
|
|
actor, member, _, chat = client.bootstrap()
|
|
try:
|
|
compose('stop', 'worker')
|
|
attachment, content = upload(client, actor, chat, 'staging-safety')
|
|
before = object_state(attachment['id'])
|
|
check(before['staging'] and not before['final'], 'staging/final publication boundary is not visible')
|
|
cli('cleanup')
|
|
check(object_state(attachment['id']) == before, 'cleanup removed current queued staging')
|
|
compose('start', 'worker')
|
|
verified_ready(client, member, attachment, content)
|
|
cli('cleanup')
|
|
after = wait_until(lambda: (lambda s: s if not s['staging'] and s['staging_deleted'] else None)
|
|
(object_state(attachment['id'])), 'post-commit staging cleanup')
|
|
check(after['final'] and after['status'] == 'ready', 'cleanup removed committed original')
|
|
print('PASS staging cleanup: queued protected; staging deleted only after ready commit; final original retained; attachment_id=' + attachment['id'], flush=True)
|
|
finally:
|
|
restore(client, 'worker')
|
|
|
|
|
|
SCENARIOS = ['stack', 'durability', 'outbox', 'broker_outage', 'redelivery', 'crash', 'lease', 'retries', 'permanent', 'staging']
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://127.0.0.1:8080'))
|
|
parser.add_argument('--only', nargs='+', choices=SCENARIOS)
|
|
args = parser.parse_args()
|
|
client = Client(args.base_url, timeout=15)
|
|
ready(client)
|
|
for name in args.only or SCENARIOS:
|
|
ready(client)
|
|
print('START ' + name + ' ' + dt.datetime.now(dt.timezone.utc).isoformat(), flush=True)
|
|
globals()[name](client)
|
|
print('PASS selected lab5 grade 3/4 checks; no grade 5 track claimed', flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except (Failure, ValueError, OSError) as exc:
|
|
print('FAIL: ' + str(exc), flush=True)
|
|
sys.exit(1)
|