forked from mtusi2026/mtusi2026lab5
325 lines
19 KiB
Python
325 lines
19 KiB
Python
"""Реальный Compose-стенд: обязательный минимум и все критерии lab5 на 4."""
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
from tempfile import TemporaryDirectory
|
|
import time
|
|
from uuid import uuid4
|
|
|
|
from PIL import Image
|
|
|
|
from helpers import Client, ROOT
|
|
from verify_restart import verify as verify_restart
|
|
|
|
|
|
def main():
|
|
with socket.socket() as sock:
|
|
sock.bind(('127.0.0.1', 0))
|
|
port = sock.getsockname()[1]
|
|
project = 'lab5-check-' + uuid4().hex[:10]
|
|
base = 'http://127.0.0.1:' + str(port)
|
|
env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), BASE_URL=base,
|
|
API_REPLICAS='2', ENABLE_FAULT_INJECTION='1', JOB_TIMEOUT_SECONDS='5',
|
|
JOB_LEASE_SECONDS='15', JOB_MAX_ATTEMPTS='5', JOB_BACKOFF_SECONDS='0.15')
|
|
result = {'utc': datetime.now(timezone.utc).isoformat(), 'command': './scripts/check-docker.sh',
|
|
'project': project, 'host': platform.system() + ' ' + platform.machine(),
|
|
'source_commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(),
|
|
'working_tree': True, 'configuration': {'concurrency': 2, 'prefetch': 1, 'attempts': 5,
|
|
'timeout_seconds': 5, 'lease_seconds': 15, 'backoff_base_seconds': 0.15}, 'checks': {}}
|
|
|
|
def command(args, timeout=240, check=True):
|
|
return subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout, check=check)
|
|
|
|
def compose(*args, timeout=240, check=True):
|
|
return command(['docker', 'compose', *args], timeout, check)
|
|
|
|
def sql(query):
|
|
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-c', query).stdout.strip()
|
|
|
|
def worker_code(code, service='worker'):
|
|
return compose('exec', '-T', service, 'python', '-c', code).stdout.strip()
|
|
|
|
def wait(predicate, seconds=60, description='condition'):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
value = predicate()
|
|
if value:
|
|
return value
|
|
time.sleep(0.2)
|
|
raise AssertionError('Timeout: ' + description)
|
|
|
|
def fault(job_id, point, action='crash', service='worker'):
|
|
# docker cp позволяет установить failpoint также в остановленный publisher.
|
|
container = compose('ps', '-aq', service).stdout.strip()
|
|
with TemporaryDirectory(prefix='pipeline-fault-') as directory:
|
|
source = Path(directory) / 'fault'
|
|
source.write_text(action)
|
|
source.chmod(0o644)
|
|
command(['docker', 'cp', str(source), container + ':/tmp/pipeline-faults/' + job_id + '.' + point])
|
|
|
|
def hit(job_id, point, service='worker'):
|
|
probe = compose('exec', '-T', service, 'python', '-c', 'from pathlib import Path; print(Path(' + repr('/tmp/pipeline-faults/' + job_id + '.' + point + '.hit') + ').exists())', check=False)
|
|
return probe.returncode == 0 and probe.stdout.strip() == 'True'
|
|
|
|
def job_id(att):
|
|
return sql("SELECT id FROM attachment_jobs WHERE attachment_id='" + att['id'] + "'")
|
|
|
|
def objects(att):
|
|
prefix = 'attachments/' + att['id'].replace('-', '') + '/v1/'
|
|
return json.loads(worker_code('import json; from src import object_store as s; print(json.dumps(sorted(o.object_name for o in s.client.list_objects(s.BUCKET,prefix=' + repr(prefix) + ',recursive=True))))', service='cleanup'))
|
|
|
|
def stage_exists(att):
|
|
key = sql("SELECT staging_key FROM attachment_jobs WHERE attachment_id='" + att['id'] + "'")
|
|
return int(worker_code('from src import object_store as s; print(sum(1 for o in s.client.list_objects(s.BUCKET,prefix=' + repr(key) + ',recursive=True)))', service='cleanup')) == 1
|
|
|
|
def record(name, **data):
|
|
result['checks'][name] = {'passed': True, **data}
|
|
print('PASS:', name, json.dumps(data, ensure_ascii=False), flush=True)
|
|
|
|
def metadata(att):
|
|
return client.call('GET', '/api/v1/attachments/' + att['id'], actor=alice, schema='Attachment')['json']
|
|
|
|
def status(att, wanted, seconds=60):
|
|
return wait(lambda: (value if (value := metadata(att))['status'] == wanted else None), seconds, wanted)
|
|
|
|
def upload(data=None, mime='image/png'):
|
|
return client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', 202,
|
|
actor=alice, raw=data or png, headers={'Content-Type': mime}, schema='Attachment')['json']
|
|
|
|
def assert_variants(att, original):
|
|
ready = status(att, 'ready')
|
|
assert {v['name'] for v in ready['variants']} == {'thumbnail', 'crop', 'optimized'}
|
|
original_response = client.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
|
|
assert original_response['bytes'] == original
|
|
assert ready['sha256'] == hashlib.sha256(original).hexdigest()
|
|
variants = {}
|
|
for variant in ready['variants']:
|
|
response = client.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + variant['name'], actor=bob)
|
|
data = response['bytes']
|
|
assert len(data) == variant['size_bytes'] and hashlib.sha256(data).hexdigest() == variant['sha256']
|
|
with Image.open(io.BytesIO(data)) as image:
|
|
image.load()
|
|
assert image.size == (variant['width'], variant['height'])
|
|
assert image.format == ('WEBP' if variant['name'] == 'optimized' else 'JPEG')
|
|
assert not image.getexif()
|
|
variants[variant['name']] = {'size': list(image.size), 'bytes': len(data)}
|
|
assert len(objects(att)) == 4
|
|
return variants
|
|
|
|
def queue_payload(att):
|
|
return json.loads(worker_code('''import json
|
|
from src import db,broker
|
|
from src.storage import utc
|
|
db.pool.open(wait=True)
|
|
with db.connection() as c:
|
|
j=c.execute("SELECT * FROM attachment_jobs WHERE attachment_id=%s",(''' + repr(att['id']) + ''',)).fetchone()
|
|
payload={'schema_version':1,'job_id':str(j['id']),'attachment_id':str(j['attachment_id']),'type':'attachment.process.v1','pipeline_version':1,'attempt':j['attempt'],'created_at':utc(j['created_at'])}
|
|
print(json.dumps(payload))
|
|
db.pool.close()
|
|
'''))
|
|
|
|
def duplicate(att, count=2):
|
|
payload = queue_payload(att)
|
|
from jsonschema import Draft202012Validator, FormatChecker
|
|
Draft202012Validator(json.loads((ROOT / 'contracts/jobs.schema.json').read_text()), format_checker=FormatChecker()).validate(payload)
|
|
worker_code('import json; from src import broker; c=broker.connect(); p=' + repr(json.dumps(payload)) + '; [c.xadd(broker.STREAM,{"job":p}) for _ in range(' + str(count) + ')]')
|
|
|
|
try:
|
|
result['docker'] = command(['docker', 'info', '--format', '{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}']).stdout.strip()
|
|
command([str(ROOT / 'scripts/start.sh')], timeout=300)
|
|
for service in ('worker', 'publisher'):
|
|
worker_code("from pathlib import Path; Path('/tmp/pipeline-faults').mkdir(exist_ok=True)", service)
|
|
ids = compose('ps', '-aq').stdout.split()
|
|
infos = json.loads(command(['docker', 'inspect', *ids]).stdout)
|
|
result['containers'] = [{'id': info['Id'], 'service': info['Config']['Labels']['com.docker.compose.service'],
|
|
'image': info['Config']['Image'], 'memory_bytes': info['HostConfig']['Memory'],
|
|
'nano_cpus': info['HostConfig']['NanoCpus'],
|
|
'mounts': [{'destination': m['Destination'], 'name': m.get('Name')} for m in info['Mounts']]} for info in infos]
|
|
assert sum(i['Config']['Labels']['com.docker.compose.service'] == 'api' for i in infos) == 2
|
|
for info in infos:
|
|
assert info['HostConfig']['Memory'] and info['HostConfig']['NanoCpus']
|
|
if info['Config']['Labels']['com.docker.compose.service'] != 'lb':
|
|
assert not info['HostConfig']['PortBindings']
|
|
record('topology_and_limits', api_replicas=2, broker='Valkey Streams with AOF fsync always', worker_processes=2)
|
|
public = command(['make', 'check', 'validate', 'test', 'PYTHON=' + sys.executable])
|
|
print(public.stdout, end='', flush=True)
|
|
record('public_smoke_and_schema')
|
|
unit = command([sys.executable, '-m', 'unittest', 'discover', '-s', 'student-tests', '-v'])
|
|
print(unit.stdout + unit.stderr, end='', flush=True)
|
|
record('student_api_regressions')
|
|
client = Client(base)
|
|
alice, bob, eve, chat = client.bootstrap()
|
|
png = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
|
|
# Worker stopped: queued already linkable; API and broker restarts preserve accepted bytes/job.
|
|
compose('stop', '--timeout', '3', 'worker')
|
|
att = upload()
|
|
assert att['status'] == 'queued' and stage_exists(att)
|
|
client.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=alice)
|
|
msg = client.message(bob, chat, '', attachments=[att['id']])['json']
|
|
jid = job_id(att)
|
|
wait(lambda: sql("SELECT published_at IS NOT NULL FROM attachment_outbox WHERE job_id='" + jid + "'") == 't')
|
|
messages = json.loads(worker_code('from src import broker; import json; c=broker.connect(); print(json.dumps([json.loads(fields["job"]) for _,fields in c.xrange(broker.STREAM)]))', service='publisher'))
|
|
from jsonschema import Draft202012Validator, FormatChecker
|
|
validator = Draft202012Validator(json.loads((ROOT / 'contracts/jobs.schema.json').read_text()), format_checker=FormatChecker())
|
|
assert messages
|
|
for payload in messages:
|
|
validator.validate(payload)
|
|
compose('restart', '--timeout', '3', 'api', 'broker')
|
|
command([str(ROOT / 'scripts/wait-ready.sh')])
|
|
assert metadata(att)['status'] == 'queued'
|
|
compose('start', 'worker')
|
|
variants = assert_variants(att, png)
|
|
assert variants['thumbnail']['size'] == [256, 160] and variants['crop']['size'] == [128, 128]
|
|
record('worker_offline_api_broker_restart', attachment_id=att['id'], job_id=jid, message_id=msg['id'], variants=variants)
|
|
|
|
# COMMIT -> publish gap and XADD -> outbox COMMIT gap, each crash is automatic.
|
|
for point in ('before_publish', 'after_publish'):
|
|
compose('stop', '--timeout', '3', 'publisher')
|
|
att = upload(); jid = job_id(att)
|
|
assert sql("SELECT published_at IS NULL FROM attachment_outbox WHERE job_id='" + jid + "'") == 't'
|
|
fault(jid, point, service='publisher')
|
|
compose('start', 'publisher')
|
|
wait(lambda: hit(jid, point, 'publisher'), description=point)
|
|
assert_variants(att, png)
|
|
record('outbox_crash_' + point, job_id=jid, automatic_recovery=True)
|
|
|
|
# Worker crash after PUT before result COMMIT and after result COMMIT before ACK.
|
|
for point in ('after_objects', 'before_ack'):
|
|
compose('stop', '--timeout', '3', 'publisher')
|
|
att = upload(); jid = job_id(att)
|
|
fault(jid, point)
|
|
compose('start', 'publisher')
|
|
wait(lambda: hit(jid, point), description=point)
|
|
assert_variants(att, png)
|
|
if point == 'after_objects':
|
|
assert int(sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'")) >= 2
|
|
snapshot = metadata(att)
|
|
duplicate(att, 4)
|
|
wait(lambda: worker_code('from src import broker; c=broker.connect(); print(c.xpending(broker.STREAM,broker.GROUP)["pending"])') == '0', seconds=45, description='pending ACK')
|
|
assert metadata(att) == snapshot and len(objects(att)) == 4
|
|
record('worker_crash_' + point, job_id=jid, attempt=int(sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'")), canonical_objects=4, duplicate_noop=True)
|
|
|
|
# Transient S3 errors, jittered backoff, exhausted registry and explicit replay.
|
|
compose('stop', '--timeout', '3', 'publisher')
|
|
att = upload(); jid = job_id(att)
|
|
fault(jid, 'before_process', 'transient:5')
|
|
compose('start', 'publisher')
|
|
status(att, 'failed')
|
|
failed = metadata(att)
|
|
assert failed['error_code'] == 'retry_exhausted'
|
|
client.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=alice)
|
|
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=bob,
|
|
body={'text': '', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': uuid4().hex})
|
|
registry = json.loads(compose('exec', '-T', 'worker', 'python', '-m', 'src.job_admin', 'failed').stdout)
|
|
assert any(row['job_id'] == jid for row in registry)
|
|
compose('exec', '-T', 'worker', 'python', '-m', 'src.job_admin', 'replay', jid)
|
|
assert_variants(att, png)
|
|
attempt = sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'")
|
|
rejected = compose('exec', '-T', 'worker', 'python', '-m', 'src.job_admin', 'replay', jid, check=False)
|
|
assert rejected.returncode != 0
|
|
assert sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") == attempt
|
|
audit = json.loads(sql("SELECT json_agg(json_build_object('event',event,'attempt',attempt,'error_code',error_code,'at',created_at) ORDER BY id) FROM attachment_job_events WHERE job_id='" + jid + "'"))
|
|
record('bounded_retry_failed_registry_replay', job_id=jid, max_automatic_attempts=5, audit=audit, ready_replay_rejected=True)
|
|
|
|
# Permanent error from damaged staging: one attempt, safe code, no retry.
|
|
compose('stop', '--timeout', '3', 'publisher')
|
|
invalid = upload(); jid = job_id(invalid)
|
|
key = sql("SELECT staging_key FROM attachment_jobs WHERE id='" + jid + "'")
|
|
worker_code('import io; from src import object_store as s; s.put(' + repr(key) + ',io.BytesIO(b"bad"),3,"image/png")')
|
|
compose('start', 'publisher')
|
|
status(invalid, 'failed')
|
|
assert metadata(invalid)['error_code'] == 'invalid_image'
|
|
assert sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") == '1'
|
|
record('permanent_error', job_id=jid, attempt=1, error_code='invalid_image')
|
|
|
|
# Реальные timeout и MemoryError внутри ограниченного resize subprocess.
|
|
for action, error_code in [('timeout', 'processing_timeout'), ('memory', 'resource_limit')]:
|
|
compose('stop', '--timeout', '3', 'publisher')
|
|
att = upload(); jid = job_id(att)
|
|
fault(jid, 'transform', action)
|
|
compose('start', 'publisher')
|
|
assert_variants(att, png)
|
|
assert sql("SELECT count(*) FROM attachment_job_events WHERE job_id='" + jid + "' AND error_code='" + error_code + "'") == '1'
|
|
record('image_subprocess_' + action, job_id=jid, timeout_seconds=5, task_memory_mb=640, recovered=True)
|
|
|
|
# Alpha, small input, EXIF rotation and floor rounding in real HTTP downloads.
|
|
cases = []
|
|
alpha = io.BytesIO(); Image.new('RGBA', (17, 9), (255, 0, 0, 0)).save(alpha, format='PNG')
|
|
att = upload(alpha.getvalue())
|
|
data = assert_variants(att, alpha.getvalue())
|
|
assert data['thumbnail']['size'] == [17, 9] and data['optimized']['size'] == [17, 9]
|
|
thumb = client.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)['bytes']
|
|
with Image.open(io.BytesIO(thumb)) as image:
|
|
assert all(channel >= 250 for channel in image.getpixel((8, 4)))
|
|
cases.append({'case': 'small_alpha_on_white', 'variants': data})
|
|
photo = io.BytesIO(); exif = Image.Exif(); exif[274] = 6
|
|
Image.new('RGB', (321, 201), 'orange').save(photo, format='JPEG', exif=exif)
|
|
att = upload(photo.getvalue(), 'image/jpeg')
|
|
data = assert_variants(att, photo.getvalue())
|
|
assert data['thumbnail']['size'] == [160, 256]
|
|
cases.append({'case': 'exif_orientation_6_and_floor', 'variants': data})
|
|
record('image_semantics', cases=cases)
|
|
|
|
# Cleanup preserves active staging even if old; removes ready and expired failed.
|
|
compose('stop', '--timeout', '3', 'worker')
|
|
active = upload()
|
|
sql("UPDATE attachments SET created_at=clock_timestamp()-interval '2 days',updated_at=clock_timestamp()-interval '2 days' WHERE id='" + active['id'] + "'")
|
|
sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '2 days' WHERE id='" + invalid['id'] + "'")
|
|
orphan_id = str(uuid4())
|
|
orphan_key = 'attachments/staging/' + orphan_id.replace('-', '')
|
|
sql("INSERT INTO attachment_uploads(id,object_key,created_at) VALUES ('" + orphan_id + "','" + orphan_key + "',clock_timestamp()-interval '2 days')")
|
|
worker_code('import io; from src import object_store as s; s.put(' + repr(orphan_key) + ',io.BytesIO(b"orphan"),6,"image/png")', service='cleanup')
|
|
compose('run', '--rm', '--no-deps', 'cleanup', 'python', '-m', 'src.cleanup', '--once')
|
|
assert sql("SELECT count(*) FROM attachment_uploads WHERE id='" + orphan_id + "'") == '0'
|
|
assert worker_code('from src import object_store as s; print(sum(1 for o in s.client.list_objects(s.BUCKET,prefix=' + repr(orphan_key) + ',recursive=True)))', service='cleanup') == '0'
|
|
|
|
assert stage_exists(active)
|
|
assert not stage_exists(invalid)
|
|
assert not stage_exists(att)
|
|
rejected = compose('exec', '-T', 'cleanup', 'python', '-m', 'src.job_admin', 'replay', job_id(invalid), check=False)
|
|
assert rejected.returncode != 0
|
|
compose('start', 'worker'); assert_variants(active, png)
|
|
record('staging_cleanup', active_preserved=True, orphan_removed=True, ready_removed=True, expired_failed_removed=True, expired_replay_rejected=True)
|
|
|
|
# Broker outage doesn't force synchronous resize: outbox recovers after restart.
|
|
compose('stop', '--timeout', '3', 'broker')
|
|
att = upload()
|
|
assert att['status'] == 'queued'
|
|
client.message(alice, chat, 'Broker offline; text API still works')
|
|
compose('start', 'broker'); assert_variants(att, png)
|
|
record('broker_offline_outbox', attachment_id=att['id'], recovered=True)
|
|
|
|
record('public_persistence', restart_proof=verify_restart(env))
|
|
failover = command(['make', 'test-failover', 'PYTHON=' + sys.executable, 'ACTION_SCRIPT=scripts/stop-one.sh'])
|
|
print(failover.stdout, end='', flush=True)
|
|
compose('start', 'api')
|
|
command([str(ROOT / 'scripts/wait-ready.sh')])
|
|
record('public_api_failover')
|
|
result['passed'] = True
|
|
print('PASS: all lab5 grade-4 criteria completed', flush=True)
|
|
except subprocess.CalledProcessError as error:
|
|
# Команды не содержат credentials, но runtime logs не сохраняем автоматически.
|
|
print('Command failed:', error.cmd[:4], error.returncode, flush=True)
|
|
print(error.stdout, error.stderr, flush=True)
|
|
raise
|
|
finally:
|
|
output = ROOT / 'evidence/lab5-checks.json'
|
|
output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
|
|
if os.environ.get('KEEP_TEST_STACK') == '1' and not result.get('passed'):
|
|
print('Kept isolated failed test stack:', project, flush=True)
|
|
else:
|
|
compose('down', '--volumes', '--timeout', '5')
|
|
print('Removed isolated test stack:', project, flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|