Files
mtusi2026lab5/student-tests/lab5_pipeline.py
T
Savelev_Ivan fd501cbf68 fix: продление аренды задач и контроль зависшей обработки
- Добавлен heartbeat для продления lease долгих задач.
- Добавлен общий таймаут с остановкой дерева процессов.
- Сохранена защита от устаревших исполнителей.
- Добавлены тесты и обновлена документация.
2026-11-20 14:00:53 +03:00

361 lines
21 KiB
Python

"""Реальный Compose-стенд: обязательный минимум и все критерии lab5 на 4."""
from datetime import datetime, timezone
import hashlib
import io
import json
import os
from pathlib import Path
import platform
import socket
import subprocess
import sys
from tempfile import TemporaryDirectory
import time
from uuid import uuid4
from PIL import Image
from helpers import Client, ROOT
from verify_restart import verify as verify_restart
def main():
with socket.socket() as sock:
sock.bind(('127.0.0.1', 0))
port = sock.getsockname()[1]
project = 'lab5-check-' + uuid4().hex[:10]
base = 'http://127.0.0.1:' + str(port)
env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), BASE_URL=base,
API_REPLICAS='2', ENABLE_FAULT_INJECTION='1', JOB_TIMEOUT_SECONDS='5',
JOB_LEASE_SECONDS='15', JOB_TASK_TIMEOUT_SECONDS='25',
JOB_MAX_ATTEMPTS='5', JOB_BACKOFF_SECONDS='0.15')
result = {'utc': datetime.now(timezone.utc).isoformat(), 'command': './scripts/check-docker.sh',
'project': project, 'host': platform.system() + ' ' + platform.machine(),
'source_commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(),
'working_tree': True, 'configuration': {'concurrency': 2, 'prefetch': 1, 'attempts': 5,
'timeout_seconds': 5, 'task_timeout_seconds': 25, 'lease_seconds': 15,
'backoff_base_seconds': 0.15}, 'checks': {}}
def command(args, timeout=240, check=True):
return subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout, check=check)
def compose(*args, timeout=240, check=True):
return command(['docker', 'compose', *args], timeout, check)
def sql(query):
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-c', query).stdout.strip()
def worker_code(code, service='worker'):
return compose('exec', '-T', service, 'python', '-c', code).stdout.strip()
def wait(predicate, seconds=60, description='condition'):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
value = predicate()
if value:
return value
time.sleep(0.2)
raise AssertionError('Timeout: ' + description)
def fault(job_id, point, action='crash', service='worker'):
# docker cp позволяет установить failpoint также в остановленный publisher.
container = compose('ps', '-aq', service).stdout.strip()
with TemporaryDirectory(prefix='pipeline-fault-') as directory:
source = Path(directory) / 'fault'
source.write_text(action)
source.chmod(0o644)
command(['docker', 'cp', str(source), container + ':/tmp/pipeline-faults/' + job_id + '.' + point])
def hit(job_id, point, service='worker'):
probe = compose('exec', '-T', service, 'python', '-c', 'from pathlib import Path; print(Path(' + repr('/tmp/pipeline-faults/' + job_id + '.' + point + '.hit') + ').exists())', check=False)
return probe.returncode == 0 and probe.stdout.strip() == 'True'
def job_id(att):
return sql("SELECT id FROM attachment_jobs WHERE attachment_id='" + att['id'] + "'")
def objects(att):
prefix = 'attachments/' + att['id'].replace('-', '') + '/v1/'
return json.loads(worker_code('import json; from src import object_store as s; print(json.dumps(sorted(o.object_name for o in s.client.list_objects(s.BUCKET,prefix=' + repr(prefix) + ',recursive=True))))', service='cleanup'))
def stage_exists(att):
key = sql("SELECT staging_key FROM attachment_jobs WHERE attachment_id='" + att['id'] + "'")
return int(worker_code('from src import object_store as s; print(sum(1 for o in s.client.list_objects(s.BUCKET,prefix=' + repr(key) + ',recursive=True)))', service='cleanup')) == 1
def record(name, **data):
result['checks'][name] = {'passed': True, **data}
print('PASS:', name, json.dumps(data, ensure_ascii=False), flush=True)
def metadata(att):
return client.call('GET', '/api/v1/attachments/' + att['id'], actor=alice, schema='Attachment')['json']
def status(att, wanted, seconds=60):
return wait(lambda: (value if (value := metadata(att))['status'] == wanted else None), seconds, wanted)
def upload(data=None, mime='image/png'):
return client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', 202,
actor=alice, raw=data or png, headers={'Content-Type': mime}, schema='Attachment')['json']
def assert_variants(att, original):
ready = status(att, 'ready')
assert {v['name'] for v in ready['variants']} == {'thumbnail', 'crop', 'optimized'}
original_response = client.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
assert original_response['bytes'] == original
assert ready['sha256'] == hashlib.sha256(original).hexdigest()
variants = {}
for variant in ready['variants']:
response = client.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + variant['name'], actor=bob)
data = response['bytes']
assert len(data) == variant['size_bytes'] and hashlib.sha256(data).hexdigest() == variant['sha256']
with Image.open(io.BytesIO(data)) as image:
image.load()
assert image.size == (variant['width'], variant['height'])
assert image.format == ('WEBP' if variant['name'] == 'optimized' else 'JPEG')
assert not image.getexif()
variants[variant['name']] = {'size': list(image.size), 'bytes': len(data)}
assert len(objects(att)) == 4
return variants
def queue_payload(att):
return json.loads(worker_code('''import json
from src import db,broker
from src.storage import utc
db.pool.open(wait=True)
with db.connection() as c:
j=c.execute("SELECT * FROM attachment_jobs WHERE attachment_id=%s",(''' + repr(att['id']) + ''',)).fetchone()
payload={'schema_version':1,'job_id':str(j['id']),'attachment_id':str(j['attachment_id']),'type':'attachment.process.v1','pipeline_version':1,'attempt':j['attempt'],'created_at':utc(j['created_at'])}
print(json.dumps(payload))
db.pool.close()
'''))
def duplicate(att, count=2):
payload = queue_payload(att)
from jsonschema import Draft202012Validator, FormatChecker
Draft202012Validator(json.loads((ROOT / 'contracts/jobs.schema.json').read_text()), format_checker=FormatChecker()).validate(payload)
worker_code('import json; from src import broker; c=broker.connect(); p=' + repr(json.dumps(payload)) + '; [c.xadd(broker.STREAM,{"job":p}) for _ in range(' + str(count) + ')]')
try:
result['docker'] = command(['docker', 'info', '--format', '{{.ServerVersion}} {{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}']).stdout.strip()
command([str(ROOT / 'scripts/start.sh')], timeout=300)
for service in ('worker', 'publisher'):
worker_code("from pathlib import Path; Path('/tmp/pipeline-faults').mkdir(exist_ok=True)", service)
ids = compose('ps', '-aq').stdout.split()
infos = json.loads(command(['docker', 'inspect', *ids]).stdout)
result['containers'] = [{'id': info['Id'], 'service': info['Config']['Labels']['com.docker.compose.service'],
'image': info['Config']['Image'], 'memory_bytes': info['HostConfig']['Memory'],
'nano_cpus': info['HostConfig']['NanoCpus'],
'mounts': [{'destination': m['Destination'], 'name': m.get('Name')} for m in info['Mounts']]} for info in infos]
assert sum(i['Config']['Labels']['com.docker.compose.service'] == 'api' for i in infos) == 2
for info in infos:
assert info['HostConfig']['Memory'] and info['HostConfig']['NanoCpus']
if info['Config']['Labels']['com.docker.compose.service'] != 'lb':
assert not info['HostConfig']['PortBindings']
record('topology_and_limits', api_replicas=2, broker='Valkey Streams with AOF fsync always', worker_processes=2)
public = command(['make', 'check', 'validate', 'test', 'PYTHON=' + sys.executable])
print(public.stdout, end='', flush=True)
record('public_smoke_and_schema')
unit = command([sys.executable, '-m', 'unittest', 'discover', '-s', 'student-tests', '-v'])
print(unit.stdout + unit.stderr, end='', flush=True)
record('student_api_regressions')
client = Client(base)
alice, bob, eve, chat = client.bootstrap()
png = (ROOT / 'tests/fixtures/sample.png').read_bytes()
# Worker stopped: queued already linkable; API and broker restarts preserve accepted bytes/job.
compose('stop', '--timeout', '3', 'worker')
att = upload()
assert att['status'] == 'queued' and stage_exists(att)
client.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=alice)
msg = client.message(bob, chat, '', attachments=[att['id']])['json']
jid = job_id(att)
wait(lambda: sql("SELECT published_at IS NOT NULL FROM attachment_outbox WHERE job_id='" + jid + "'") == 't')
messages = json.loads(worker_code('from src import broker; import json; c=broker.connect(); print(json.dumps([json.loads(fields["job"]) for _,fields in c.xrange(broker.STREAM)]))', service='publisher'))
from jsonschema import Draft202012Validator, FormatChecker
validator = Draft202012Validator(json.loads((ROOT / 'contracts/jobs.schema.json').read_text()), format_checker=FormatChecker())
assert messages
for payload in messages:
validator.validate(payload)
compose('restart', '--timeout', '3', 'api', 'broker')
command([str(ROOT / 'scripts/wait-ready.sh')])
assert metadata(att)['status'] == 'queued'
compose('start', 'worker')
variants = assert_variants(att, png)
assert variants['thumbnail']['size'] == [256, 160] and variants['crop']['size'] == [128, 128]
record('worker_offline_api_broker_restart', attachment_id=att['id'], job_id=jid, message_id=msg['id'], variants=variants)
# Исправная попытка живёт дольше исходного lease без ложного retry.
compose('stop', '--timeout', '3', 'publisher')
att = upload(); jid = job_id(att)
fault(jid, 'before_download', 'delay:20')
compose('start', 'publisher')
wait(lambda: hit(jid, 'before_download'), description='long task started')
initial_lease = sql("SELECT lease_until FROM attachment_jobs WHERE id='" + jid + "'")
assert worker_code('''from uuid import uuid4
from src import db,jobs
db.pool.open(wait=True)
with db.connection() as c:
j=jobs.locked(c,''' + repr(jid) + ''')
j['lease_token']=uuid4()
print(jobs.renew(j))
db.pool.close()
''') == 'False'
wait(lambda: sql("SELECT lease_until FROM attachment_jobs WHERE id='" + jid + "'") != initial_lease,
seconds=12, description='heartbeat extended lease')
assert_variants(att, png)
assert sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") == '1'
assert sql("SELECT count(*) FROM attachment_job_events WHERE job_id='" + jid + "' AND event='retry'") == '0'
record('long_task_heartbeat', job_id=jid, delay_seconds=20, lease_seconds=15, attempt=1)
# Heartbeat не сохраняет зависшую работу бесконечно: общий deadline.
compose('stop', '--timeout', '3', 'publisher')
att = upload(); jid = job_id(att)
fault(jid, 'before_download', 'timeout')
compose('start', 'publisher')
assert_variants(att, png)
assert hit(jid, 'before_download')
assert sql("SELECT count(*) FROM attachment_job_events WHERE job_id='" + jid + "' AND error_code='processing_timeout'") == '1'
assert sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") == '2'
record('whole_task_timeout', job_id=jid, timeout_seconds=25, recovered=True)
# COMMIT -> publish gap and XADD -> outbox COMMIT gap, each crash is automatic.
for point in ('before_publish', 'after_publish'):
compose('stop', '--timeout', '3', 'publisher')
att = upload(); jid = job_id(att)
assert sql("SELECT published_at IS NULL FROM attachment_outbox WHERE job_id='" + jid + "'") == 't'
fault(jid, point, service='publisher')
compose('start', 'publisher')
wait(lambda: hit(jid, point, 'publisher'), description=point)
assert_variants(att, png)
record('outbox_crash_' + point, job_id=jid, automatic_recovery=True)
# Worker crash after PUT before result COMMIT and after result COMMIT before ACK.
for point in ('after_objects', 'before_ack'):
compose('stop', '--timeout', '3', 'publisher')
att = upload(); jid = job_id(att)
fault(jid, point)
compose('start', 'publisher')
wait(lambda: hit(jid, point), description=point)
assert_variants(att, png)
if point == 'after_objects':
assert int(sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'")) >= 2
snapshot = metadata(att)
duplicate(att, 4)
wait(lambda: worker_code('from src import broker; c=broker.connect(); print(c.xpending(broker.STREAM,broker.GROUP)["pending"])') == '0', seconds=45, description='pending ACK')
assert metadata(att) == snapshot and len(objects(att)) == 4
record('worker_crash_' + point, job_id=jid, attempt=int(sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'")), canonical_objects=4, duplicate_noop=True)
# Transient S3 errors, jittered backoff, exhausted registry and explicit replay.
compose('stop', '--timeout', '3', 'publisher')
att = upload(); jid = job_id(att)
fault(jid, 'before_process', 'transient:5')
compose('start', 'publisher')
status(att, 'failed')
failed = metadata(att)
assert failed['error_code'] == 'retry_exhausted'
client.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=alice)
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=bob,
body={'text': '', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': uuid4().hex})
registry = json.loads(compose('exec', '-T', 'worker', 'python', '-m', 'src.job_admin', 'failed').stdout)
assert any(row['job_id'] == jid for row in registry)
compose('exec', '-T', 'worker', 'python', '-m', 'src.job_admin', 'replay', jid)
assert_variants(att, png)
attempt = sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'")
rejected = compose('exec', '-T', 'worker', 'python', '-m', 'src.job_admin', 'replay', jid, check=False)
assert rejected.returncode != 0
assert sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") == attempt
audit = json.loads(sql("SELECT json_agg(json_build_object('event',event,'attempt',attempt,'error_code',error_code,'at',created_at) ORDER BY id) FROM attachment_job_events WHERE job_id='" + jid + "'"))
record('bounded_retry_failed_registry_replay', job_id=jid, max_automatic_attempts=5, audit=audit, ready_replay_rejected=True)
# Permanent error from damaged staging: one attempt, safe code, no retry.
compose('stop', '--timeout', '3', 'publisher')
invalid = upload(); jid = job_id(invalid)
key = sql("SELECT staging_key FROM attachment_jobs WHERE id='" + jid + "'")
worker_code('import io; from src import object_store as s; s.put(' + repr(key) + ',io.BytesIO(b"bad"),3,"image/png")')
compose('start', 'publisher')
status(invalid, 'failed')
assert metadata(invalid)['error_code'] == 'invalid_image'
assert sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") == '1'
record('permanent_error', job_id=jid, attempt=1, error_code='invalid_image')
# Реальные timeout и MemoryError внутри ограниченного resize subprocess.
for action, error_code in [('timeout', 'processing_timeout'), ('memory', 'resource_limit')]:
compose('stop', '--timeout', '3', 'publisher')
att = upload(); jid = job_id(att)
fault(jid, 'transform', action)
compose('start', 'publisher')
assert_variants(att, png)
assert sql("SELECT count(*) FROM attachment_job_events WHERE job_id='" + jid + "' AND error_code='" + error_code + "'") == '1'
record('image_subprocess_' + action, job_id=jid, timeout_seconds=5, task_memory_mb=640, recovered=True)
# Alpha, small input, EXIF rotation and floor rounding in real HTTP downloads.
cases = []
alpha = io.BytesIO(); Image.new('RGBA', (17, 9), (255, 0, 0, 0)).save(alpha, format='PNG')
att = upload(alpha.getvalue())
data = assert_variants(att, alpha.getvalue())
assert data['thumbnail']['size'] == [17, 9] and data['optimized']['size'] == [17, 9]
thumb = client.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)['bytes']
with Image.open(io.BytesIO(thumb)) as image:
assert all(channel >= 250 for channel in image.getpixel((8, 4)))
cases.append({'case': 'small_alpha_on_white', 'variants': data})
photo = io.BytesIO(); exif = Image.Exif(); exif[274] = 6
Image.new('RGB', (321, 201), 'orange').save(photo, format='JPEG', exif=exif)
att = upload(photo.getvalue(), 'image/jpeg')
data = assert_variants(att, photo.getvalue())
assert data['thumbnail']['size'] == [160, 256]
cases.append({'case': 'exif_orientation_6_and_floor', 'variants': data})
record('image_semantics', cases=cases)
# Cleanup preserves active staging even if old; removes ready and expired failed.
compose('stop', '--timeout', '3', 'worker')
active = upload()
sql("UPDATE attachments SET created_at=clock_timestamp()-interval '2 days',updated_at=clock_timestamp()-interval '2 days' WHERE id='" + active['id'] + "'")
sql("UPDATE attachments SET updated_at=clock_timestamp()-interval '2 days' WHERE id='" + invalid['id'] + "'")
orphan_id = str(uuid4())
orphan_key = 'attachments/staging/' + orphan_id.replace('-', '')
sql("INSERT INTO attachment_uploads(id,object_key,created_at) VALUES ('" + orphan_id + "','" + orphan_key + "',clock_timestamp()-interval '2 days')")
worker_code('import io; from src import object_store as s; s.put(' + repr(orphan_key) + ',io.BytesIO(b"orphan"),6,"image/png")', service='cleanup')
compose('run', '--rm', '--no-deps', 'cleanup', 'python', '-m', 'src.cleanup', '--once')
assert sql("SELECT count(*) FROM attachment_uploads WHERE id='" + orphan_id + "'") == '0'
assert worker_code('from src import object_store as s; print(sum(1 for o in s.client.list_objects(s.BUCKET,prefix=' + repr(orphan_key) + ',recursive=True)))', service='cleanup') == '0'
assert stage_exists(active)
assert not stage_exists(invalid)
assert not stage_exists(att)
rejected = compose('exec', '-T', 'cleanup', 'python', '-m', 'src.job_admin', 'replay', job_id(invalid), check=False)
assert rejected.returncode != 0
compose('start', 'worker'); assert_variants(active, png)
record('staging_cleanup', active_preserved=True, orphan_removed=True, ready_removed=True, expired_failed_removed=True, expired_replay_rejected=True)
# Broker outage doesn't force synchronous resize: outbox recovers after restart.
compose('stop', '--timeout', '3', 'broker')
att = upload()
assert att['status'] == 'queued'
client.message(alice, chat, 'Broker offline; text API still works')
compose('start', 'broker'); assert_variants(att, png)
record('broker_offline_outbox', attachment_id=att['id'], recovered=True)
record('public_persistence', restart_proof=verify_restart(env))
failover = command(['make', 'test-failover', 'PYTHON=' + sys.executable, 'ACTION_SCRIPT=scripts/stop-one.sh'])
print(failover.stdout, end='', flush=True)
compose('start', 'api')
command([str(ROOT / 'scripts/wait-ready.sh')])
record('public_api_failover')
result['passed'] = True
print('PASS: all lab5 grade-4 criteria completed', flush=True)
except subprocess.CalledProcessError as error:
# Команды не содержат credentials, но runtime logs не сохраняем автоматически.
print('Command failed:', error.cmd[:4], error.returncode, flush=True)
print(error.stdout, error.stderr, flush=True)
raise
finally:
output = ROOT / 'evidence/lab5-checks.json'
output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
if os.environ.get('KEEP_TEST_STACK') == '1' and not result.get('passed'):
print('Kept isolated failed test stack:', project, flush=True)
else:
compose('down', '--volumes', '--timeout', '5')
print('Removed isolated test stack:', project, flush=True)
if __name__ == '__main__':
main()