From 42bf6aa702dab9270fc13d7b80df13dfca089ef8 Mon Sep 17 00:00:00 2001 From: Sergey Lyakhonov Date: Fri, 20 Nov 2026 06:51:54 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A0=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D1=82=D1=8C=20=D0=B2=D0=B5=D1=80=D1=81=D0=B8=D0=B8?= =?UTF-8?q?=20pipeline,=20fencing=20=D0=B8=20=D0=B0=D0=B2=D1=82=D0=BE?= =?UTF-8?q?=D0=BC=D0=B0=D1=82=D0=B8=D1=87=D0=B5=D1=81=D0=BA=D0=B8=D0=B5=20?= =?UTF-8?q?=D1=81=D1=86=D0=B5=D0=BD=D0=B0=D1=80=D0=B8=D0=B8=20=D0=BE=D1=82?= =?UTF-8?q?=D0=BA=D0=B0=D0=B7=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile.s3 | 2 +- compose.yaml | 15 ++ migrations/005_versioned_pipeline.sql | 19 ++ migrations/006_experiment_faults.sql | 9 + scripts/audit.py | 55 ++++++ scripts/environment.py | 28 +++ scripts/plot_results.py | 34 ++++ scripts/reproduce.py | 58 ++++++ src/dispatch.py | 4 +- src/faults.py | 29 +++ src/image_pipeline.py | 12 +- src/jobs.py | 102 ++++++++--- src/operator.py | 24 +-- src/pipeline_versions.py | 6 + src/routes/attachments.py | 9 +- src/staging.py | 6 +- src/worker.py | 13 +- student-tests/load_pipeline.py | 94 ++++++++++ student-tests/pipeline.py | 242 ++++++++++++++++++++++++++ student-tests/probe.py | 55 ++++++ student-tests/resilience_driver.py | 82 +++++++++ 21 files changed, 845 insertions(+), 53 deletions(-) create mode 100644 migrations/005_versioned_pipeline.sql create mode 100644 migrations/006_experiment_faults.sql create mode 100644 scripts/audit.py create mode 100644 scripts/environment.py create mode 100644 scripts/plot_results.py create mode 100644 scripts/reproduce.py create mode 100644 src/faults.py create mode 100644 src/pipeline_versions.py create mode 100644 student-tests/load_pipeline.py create mode 100644 student-tests/pipeline.py create mode 100644 student-tests/probe.py create mode 100644 student-tests/resilience_driver.py diff --git a/Dockerfile.s3 b/Dockerfile.s3 index b813ecf0..5ec09f31 100644 --- a/Dockerfile.s3 +++ b/Dockerfile.s3 @@ -1,4 +1,4 @@ -# Reproducible fallback: official MinIO images are unavailable from registries. +# Build pinned upstream releases and verify archive checksums. FROM golang:1.24.6-bookworm@sha256:ab1d1823abb55a9504d2e3e003b75b36dbeb1cbcc4c92593d85a84ee46becc6c AS build WORKDIR /build ADD --checksum=sha256:8819e3e7817e46b7b3798f8f200ead208562e571563c2e040352378031abe9f2 https://github.com/minio/minio/archive/07c3a429bfed433e49018cb0f78a52145d4bedeb.tar.gz /tmp/minio.tar.gz diff --git a/compose.yaml b/compose.yaml index 8a20f196..a30fb4fc 100644 --- a/compose.yaml +++ b/compose.yaml @@ -11,6 +11,8 @@ x-app: &app SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90} + LAB5_FAULT_INJECTION: ${LAB5_FAULT_INJECTION:-0} STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} CACHE_URL: redis://redis:6379/0 BROKER_URL_FILE: /run/secrets/broker_url @@ -26,6 +28,19 @@ x-dependencies: &dependencies condition: service_completed_successfully services: + inspector: + <<: *app + profiles: [tools] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3:9000 + S3_ACCESS_KEY_FILE: /run/secrets/s3_root_user + S3_SECRET_KEY_FILE: /run/secrets/s3_root_password + secrets: [database_url, s3_root_user, s3_root_password] + healthcheck: + disable: true + cpus: 0.25 + mem_limit: 256m db: image: postgres:16.6-alpine environment: diff --git a/migrations/005_versioned_pipeline.sql b/migrations/005_versioned_pipeline.sql new file mode 100644 index 00000000..760c52b7 --- /dev/null +++ b/migrations/005_versioned_pipeline.sql @@ -0,0 +1,19 @@ +ALTER TABLE attachments + ADD COLUMN target_pipeline_version integer NOT NULL DEFAULT 1 CHECK (target_pipeline_version >= 1), + ADD COLUMN published_pipeline_version integer CHECK (published_pipeline_version >= 1), + ADD CHECK (published_pipeline_version <= target_pipeline_version); +UPDATE attachments SET published_pipeline_version = 1 WHERE state = 'ready'; + +ALTER TABLE attachment_jobs + DROP CONSTRAINT attachment_jobs_attachment_id_key, + DROP CONSTRAINT attachment_jobs_staging_key_key, + DROP CONSTRAINT attachment_jobs_state_check, + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1), + ADD COLUMN settled_at timestamptz, + ADD CHECK (state IN ('queued', 'processing', 'ready', 'failed', 'superseded')), + ADD UNIQUE (attachment_id, pipeline_version); +CREATE INDEX attachment_jobs_staging ON attachment_jobs(staging_key); + +ALTER TABLE attachment_variants ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1; +ALTER TABLE attachment_variants DROP CONSTRAINT attachment_variants_pkey; +ALTER TABLE attachment_variants ADD PRIMARY KEY (attachment_id, pipeline_version, name); diff --git a/migrations/006_experiment_faults.sql b/migrations/006_experiment_faults.sql new file mode 100644 index 00000000..3a5e556e --- /dev/null +++ b/migrations/006_experiment_faults.sql @@ -0,0 +1,9 @@ +CREATE TABLE experiment_faults ( + job_id uuid NOT NULL REFERENCES attachment_jobs(id), + point text NOT NULL, + action text NOT NULL CHECK (action IN ('crash', 'pause')), + hit_at timestamptz, + process_id integer, + released boolean NOT NULL DEFAULT false, + PRIMARY KEY (job_id, point) +); diff --git a/scripts/audit.py b/scripts/audit.py new file mode 100644 index 00000000..11e5a68a --- /dev/null +++ b/scripts/audit.py @@ -0,0 +1,55 @@ +"""Аудит обязательных материалов, локальных секретов и предыдущей лабораторной.""" +import hashlib +import json +import os +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +PREVIOUS = ROOT.parent / 'vysoko-nagr 4' +ENV = dict(os.environ, GIT_OPTIONAL_LOCKS='0') + + +def git(root, *args): + return subprocess.check_output(['git', '-C', str(root), *args], env=ENV).decode('utf-8').strip() + + +def previous(): + paths = git(PREVIOUS, 'ls-files', '-z').split('\0') + files = {p: hashlib.sha256((PREVIOUS / p).read_bytes()).hexdigest() for p in paths if p} + return dict(head=git(PREVIOUS, 'rev-parse', 'HEAD'), branch=git(PREVIOUS, 'branch', '--show-current'), + status=git(PREVIOUS, 'status', '--porcelain'), files=files) + + +def main(): + import sys + if sys.argv[1:] == ['baseline']: + (ROOT / 'evidence/lab4-baseline.json').write_text(json.dumps(previous(), ensure_ascii=False, indent=2), encoding='utf-8') + print('Сохранён read-only fingerprint лабораторной 4') + return + baseline = json.loads((ROOT / 'evidence/lab4-baseline.json').read_text(encoding='utf-8')) + unchanged = previous() == baseline + assert unchanged, 'Lab4 differs from baseline' + protected = ['README.md', 'COURSE.md', 'CONTRIBUTING.md', 'MAINTAINERS.md', 'contracts', 'tests', + 'examples', 'compose.example.yaml', '.env.example', '.github'] + assert not git(ROOT, 'diff', '276dbda', '--', *protected), 'Protected lab5 material changed' + paths = git(ROOT, 'ls-files', '-z').split('\0') + secret_values = [p.read_bytes().strip() for p in (ROOT / 'secrets').iterdir() if p.is_file()] + for path in paths: + if not path: + continue + assert not path.startswith('secrets/') and path != '.env', 'Tracked private state' + content = (ROOT / path).read_bytes() + assert not any(value and value in content for value in secret_values), 'Local secret leaked in ' + path + assert b'-----BEGIN PRIVATE KEY-----' not in content, 'Private key found' + authors = git(ROOT, 'log', '276dbda..HEAD', '--format=%h|%an|%ae|%cn|%ce').splitlines() + for row in authors: + assert row.split('|')[1:] == ['Sergey Lyakhonov', 'lyserg119@gmail.com'] * 2, 'Wrong author/committer' + report = dict(lab4_unchanged=unchanged, protected_lab5_unchanged=True, tracked_local_secrets=False, + scanned_files=len(paths), authors=authors, head=git(ROOT, 'rev-parse', 'HEAD')) + (ROOT / 'evidence/audit.json').write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding='utf-8') + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/environment.py b/scripts/environment.py new file mode 100644 index 00000000..12c4afd6 --- /dev/null +++ b/scripts/environment.py @@ -0,0 +1,28 @@ +"""Безопасная фиксация версий, лимитов и опубликованных портов.""" +import json +from pathlib import Path +import platform +import subprocess + +ROOT = Path(__file__).resolve().parents[1] + + +def output(*args): + return subprocess.check_output(args, cwd=ROOT, text=True, encoding='utf-8').strip() + + +ids = output('docker', 'compose', '-p', 'lyakhonov-lab5', 'ps', '-q').split() +containers = json.loads(output('docker', 'inspect', *ids)) +result = dict(commit=output('git', 'rev-parse', 'HEAD'), python=platform.python_version(), + docker=json.loads(output('docker', 'version', '--format', '{{json .}}')), + compose=output('docker', 'compose', 'version'), + engine=json.loads(output('docker', 'info', '--format', '{"memory_bytes":{{.MemTotal}},"cpus":{{.NCPU}}}')), + services=[]) +for row in containers: + result['services'].append(dict(service=row['Config']['Labels']['com.docker.compose.service'], id=row['Id'], + image=row['Config']['Image'], image_id=row['Image'], started_at=row['State']['StartedAt'], + memory_limit=row['HostConfig']['Memory'], nanocpus=row['HostConfig']['NanoCpus'], + ports=row['HostConfig']['PortBindings'], + settings=[x for x in row['Config']['Env'] if x.startswith(('JOB_', 'WORKER_', 'LAB5_FAULT_', 'STAGING_RETENTION_'))])) +(ROOT / 'evidence/environment.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8') +print('Saved versions, IDs, resource limits, ports and non-secret experiment settings') diff --git a/scripts/plot_results.py b/scripts/plot_results.py new file mode 100644 index 00000000..b0621deb --- /dev/null +++ b/scripts/plot_results.py @@ -0,0 +1,34 @@ +"""Статические графики только из сохранённых фактических агрегатов.""" +import json +from pathlib import Path +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt + +ROOT = Path(__file__).resolve().parents[1] +data = json.loads((ROOT / 'evidence/pipeline.json').read_text(encoding='utf-8')) +points = ('before_publish', 'after_publish', 'before_effect', 'after_original', 'after_variants', 'after_commit') +fig, ax = plt.subplots(figsize=(10, 4.5), layout='constrained') +for index, point in enumerate(points): + values = [row['seconds'] for row in data if row['scenario'] == point] + ax.scatter([index] * len(values), values, s=45) +ax.set_xticks(range(len(points)), points, rotation=15) +ax.set_ylabel('Full scenario duration, seconds') +ax.set_title('Actual crash scenarios: stop, upload, restart, verified downloads') +ax.grid(axis='y', alpha=.25) +fig.savefig(ROOT / 'evidence/faults.png', dpi=160) +plt.close(fig) +data = json.loads((ROOT / 'evidence/load-pipeline.json').read_text(encoding='utf-8')) +fig, axes = plt.subplots(1, 2, figsize=(10, 4), layout='constrained') +for index, profile in enumerate(sorted({row['profile'] for row in data})): + rows = [row for row in data if row['profile'] == profile] + for axis, metric in zip(axes, ('successful_rps', 'p95')): + axis.scatter([index] * len(rows), [row[metric] for row in rows], label=profile) +for axis, label in zip(axes, ('Completed images / second', 'Upload to ready p95, seconds')): + axis.set_xticks(range(len({row['profile'] for row in data})), sorted({row['profile'] for row in data})) + axis.set_ylabel(label) + axis.grid(axis='y', alpha=.25) +fig.suptitle('Closed-loop pipeline: 2 clients, 20 seconds, 3 repeats') +fig.savefig(ROOT / 'evidence/load.png', dpi=160) +plt.close(fig) +print('Saved evidence/faults.png and evidence/load.png') diff --git a/scripts/reproduce.py b/scripts/reproduce.py new file mode 100644 index 00000000..cc5baaef --- /dev/null +++ b/scripts/reproduce.py @@ -0,0 +1,58 @@ +"""Чистый Git-экспорт, новые секреты и отдельный временный Compose-проект.""" +import io +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import tempfile +import uuid +import zipfile + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + name = 'lyakhonov-lab5-repro-' + uuid.uuid4().hex[:8] + directory = Path(tempfile.mkdtemp(prefix=name + '-')) + archive = subprocess.check_output(['git', 'archive', '--format=zip', 'HEAD'], cwd=ROOT) + with zipfile.ZipFile(io.BytesIO(archive)) as contents: + contents.extractall(directory) + with socket.socket() as listener: + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + env = dict(os.environ, LAB5_PORT=str(port), LAB5_FAULT_INJECTION='0', JOB_TIMEOUT_SECONDS='60', JOB_LEASE_SECONDS='90') + services = ('api', 'migrate', 'worker', 'dispatcher', 'staging-cleaner') + images = [name + ':app', name + ':s3'] + override = {'services': {service: {'image': images[0]} for service in services}} + override['services'].update({service: {'image': images[1]} for service in ('s3', 'bucket-init')}) + (directory / 'repro-override.json').write_text(json.dumps(override), encoding='utf-8') + compose = ['docker', 'compose', '-p', name, '-f', 'compose.yaml', '-f', 'repro-override.json'] + events = [] + def run(args): + result = subprocess.run(args, cwd=directory, env=env, text=True, encoding='utf-8', capture_output=True) + # Только безопасные последние строки: ни inspect .Config.Env, ни secret files. + events.append(dict(command=args, exit_code=result.returncode, output=(result.stdout + result.stderr)[-5000:])) + if result.returncode: + raise RuntimeError('reproduction failed: ' + ' '.join(args)) + try: + run([sys.executable, '-X', 'utf8', 'scripts/bootstrap.py']) + run([sys.executable, '-X', 'utf8', 'scripts/check_contract.py']) + run(['docker', 'build', '-t', images[0], '.']) + run(['docker', 'build', '-t', images[1], '-f', 'Dockerfile.s3', '.']) + run([*compose, 'up', '-d', '--wait']) + run([sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', f'http://127.0.0.1:{port}']) + run([*compose, 'ps']) + finally: + run([*compose, 'down', '--volumes', '--remove-orphans']) + for tag in images: + run(['docker', 'image', 'rm', tag]) + result = dict(project=name, directory=str(directory), port=port, + commit=subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), events=events) + (ROOT / 'evidence/reproduction.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8') + print('PASS: clean export, fresh secrets, isolated project; temporary Docker resources removed') + + +if __name__ == '__main__': + main() diff --git a/src/dispatch.py b/src/dispatch.py index 72611427..25a78f34 100644 --- a/src/dispatch.py +++ b/src/dispatch.py @@ -9,7 +9,7 @@ import pika import psycopg from psycopg_pool import PoolTimeout -from src import broker, jobs +from src import broker, jobs, faults from src.storage import storage @@ -32,7 +32,9 @@ def main(): next_recovery = time.monotonic() + 1 reserved = jobs.reserve_publish() if reserved: + faults.hit('before_publish', reserved) broker.publish(channel, jobs.envelope(reserved)) + faults.hit('after_publish', reserved) jobs.confirm_publish(reserved) reserved = None else: diff --git a/src/faults.py b/src/faults.py new file mode 100644 index 00000000..7d26432d --- /dev/null +++ b/src/faults.py @@ -0,0 +1,29 @@ +"""Одноразовые точки отказа. Выключены без LAB5_FAULT_INJECTION=1.""" +import os +import time + +from src.storage import storage + + +def hit(point, job): + if os.getenv('LAB5_FAULT_INJECTION') != '1': + return + with storage.pool.connection() as conn: + fault = conn.execute( + "UPDATE experiment_faults SET hit_at=clock_timestamp(), process_id=%s " + "WHERE job_id=%s AND point=%s AND hit_at IS NULL RETURNING action", + (os.getpid(), job['id'], point), + ).fetchone() + if not fault: + return + if fault['action'] == 'crash': + os._exit(91) # Реальная смерть процесса без finally/ack. + while True: + with storage.pool.connection() as conn: + released = conn.execute( + 'SELECT released FROM experiment_faults WHERE job_id=%s AND point=%s', + (job['id'], point), + ).fetchone()['released'] + if released: + return + time.sleep(0.1) diff --git a/src/image_pipeline.py b/src/image_pipeline.py index 86f651e0..e85ef94a 100644 --- a/src/image_pipeline.py +++ b/src/image_pipeline.py @@ -5,6 +5,7 @@ import sys from pathlib import Path from PIL import Image, ImageOps +from src.pipeline_versions import ENCODING Image.MAX_IMAGE_PIXELS = 40_000_000 @@ -18,7 +19,8 @@ def fit(image, limit): return image.resize(size, Image.Resampling.LANCZOS) -def render(source, target): +def render(source, target, version=1): + jpeg_quality, webp_quality = ENCODING[version] with Image.open(source) as opened: if opened.width * opened.height > Image.MAX_IMAGE_PIXELS: raise ValueError('pixel limit') @@ -32,11 +34,11 @@ def render(source, target): rgb.info.clear() del image thumbnail = fit(rgb, 256) - thumbnail.save(target / 'thumbnail', 'JPEG', quality=85) + thumbnail.save(target / 'thumbnail', 'JPEG', quality=jpeg_quality) crop = ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS) - crop.save(target / 'crop', 'JPEG', quality=85) + crop.save(target / 'crop', 'JPEG', quality=jpeg_quality) optimized = fit(rgb, 1280) - optimized.save(target / 'optimized', 'WEBP', quality=82, method=4) + optimized.save(target / 'optimized', 'WEBP', quality=webp_quality, method=4) return [ ['thumbnail', 'image/jpeg', *thumbnail.size], ['crop', 'image/jpeg', *crop.size], @@ -47,7 +49,7 @@ def render(source, target): def main(): directory = Path(sys.argv[1]) try: - result = render(directory / 'original', directory) + result = render(directory / 'original', directory, int(sys.argv[2])) except (OSError, ValueError, SyntaxError, Image.DecompressionBombError): return 2 (directory / 'manifest.json').write_text(json.dumps(result), encoding='utf-8') diff --git a/src/jobs.py b/src/jobs.py index e4156e71..e5c9ecc5 100644 --- a/src/jobs.py +++ b/src/jobs.py @@ -1,14 +1,17 @@ """Устойчивое намерение обработки, lease и атомарная публикация manifest.""" import random +import re +from datetime import datetime from uuid import UUID, uuid4 from src.config import setting from src.objects import BUCKET, client, object_lock +from src.pipeline_versions import ENCODING from src.storage import storage, timestamp JOB_TIMEOUT = int(setting('JOB_TIMEOUT_SECONDS', '60')) -LEASE_SECONDS = JOB_TIMEOUT + 30 +LEASE_SECONDS = max(JOB_TIMEOUT + 5, int(setting('JOB_LEASE_SECONDS', str(JOB_TIMEOUT + 30)))) def envelope(job): @@ -17,24 +20,31 @@ def envelope(job): 'job_id': str(job['id']), 'type': 'attachment.process.v1', 'attachment_id': str(job['attachment_id']), - 'pipeline_version': 1, + 'pipeline_version': job['pipeline_version'], 'attempt': job['attempt'], 'created_at': timestamp(job['created_at']), } def validate(message): - if set(message) != {'schema_version', 'job_id', 'type', 'attachment_id', - 'pipeline_version', 'attempt', 'created_at'}: + required = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'} + if not isinstance(message, dict) or not required <= message.keys() <= required | {'traceparent', 'tracestate'}: raise ValueError('invalid fields') if (type(message['schema_version']) is not int or message['schema_version'] != 1 or message['type'] != 'attachment.process.v1' - or type(message['pipeline_version']) is not int or message['pipeline_version'] != 1 + or type(message['pipeline_version']) is not int or message['pipeline_version'] not in ENCODING or type(message['attempt']) is not int or message['attempt'] < 1): raise ValueError('invalid job') UUID(message['job_id']) UUID(message['attachment_id']) - return message + if datetime.fromisoformat(message['created_at'].replace('Z', '+00:00')).tzinfo is None: + raise ValueError('missing timezone') + if 'traceparent' in message and not re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', + message['traceparent']): + raise ValueError('invalid traceparent') + if 'tracestate' in message and (not isinstance(message['tracestate'], str) or len(message['tracestate']) > 512): + raise ValueError('invalid tracestate') + return {key: message[key] for key in required} def reserve_publish(): @@ -83,7 +93,8 @@ def locked_job(conn, job_id, attachment_id): def claim(message): with storage.pool.connection() as conn: job, attachment = locked_job(conn, message['job_id'], message['attachment_id']) - if not job or envelope(job) != message or job['state'] != 'queued': + if (not job or envelope(job) != message or job['state'] != 'queued' + or job['pipeline_version'] != attachment['target_pipeline_version']): return None token = uuid4() conn.execute( @@ -91,7 +102,8 @@ def claim(message): lease_until=clock_timestamp()+%s * interval '1 second', next_publish_at=NULL WHERE id=%s""", (token, LEASE_SECONDS, job['id']), ) - conn.execute("UPDATE attachments SET state='processing',updated_at=clock_timestamp() WHERE id=%s", + conn.execute("UPDATE attachments SET state='processing',updated_at=clock_timestamp() " + "WHERE id=%s AND published_pipeline_version IS NULL", (job['attachment_id'],)) return job | {'lease_token': token, 'attachment': attachment} @@ -105,20 +117,21 @@ def owns(conn, job, claimed): def complete(claimed, variants): with storage.pool.connection() as conn: - job, _ = locked_job(conn, claimed['id'], claimed['attachment_id']) - if not owns(conn, job, claimed): + job, attachment = locked_job(conn, claimed['id'], claimed['attachment_id']) + if not owns(conn, job, claimed) or job['pipeline_version'] != attachment['target_pipeline_version']: return False for variant in variants: conn.execute( """INSERT INTO attachment_variants - (attachment_id,name,object_key,content_type,size_bytes,sha256,width,height) - VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""", - (job['attachment_id'], *variant), + (attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)""", + (job['attachment_id'], job['pipeline_version'], *variant), ) - conn.execute("UPDATE attachments SET state='ready',updated_at=clock_timestamp() WHERE id=%s", - (job['attachment_id'],)) + conn.execute("UPDATE attachments SET state='ready',published_pipeline_version=%s," + "updated_at=clock_timestamp(),error_code=NULL WHERE id=%s", + (job['pipeline_version'], job['attachment_id'])) conn.execute("UPDATE attachment_jobs SET state='ready',lease_token=NULL,lease_until=NULL," - "last_error=NULL WHERE id=%s", (job['id'],)) + "last_error=NULL,settled_at=clock_timestamp() WHERE id=%s", (job['id'],)) return True @@ -131,11 +144,13 @@ def settle_error(conn, job, code, permanent=False): if permanent or exhausted: error = code if permanent else 'retry_exhausted' conn.execute("UPDATE attachments SET state='failed',error_code=%s,updated_at=clock_timestamp() " - "WHERE id=%s", (error, job['attachment_id'])) + "WHERE id=%s AND published_pipeline_version IS NULL", (error, job['attachment_id'])) conn.execute("UPDATE attachment_jobs SET state='failed',lease_token=NULL,lease_until=NULL," - "last_error=%s,next_publish_at=NULL WHERE id=%s", (code, job['id'])) + "last_error=%s,next_publish_at=NULL,settled_at=clock_timestamp() WHERE id=%s", + (code, job['id'])) return - conn.execute("UPDATE attachments SET state='queued',updated_at=clock_timestamp() WHERE id=%s", + conn.execute("UPDATE attachments SET state='queued',updated_at=clock_timestamp() " + "WHERE id=%s AND published_pipeline_version IS NULL", (job['attachment_id'],)) conn.execute("""UPDATE attachment_jobs SET state='queued',lease_token=NULL,lease_until=NULL, last_error=%s,attempt=attempt+1,published_at=NULL, @@ -164,25 +179,60 @@ def recover_expired(): def replay(attachment_id): with storage.pool.connection() as conn: - attachment = conn.execute('SELECT id FROM attachments WHERE id=%s FOR UPDATE', + attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,)).fetchone() if not attachment: return None - job = conn.execute('SELECT * FROM attachment_jobs WHERE attachment_id=%s FOR UPDATE', - (attachment_id,)).fetchone() + job = conn.execute('SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s FOR UPDATE', + (attachment_id, attachment['target_pipeline_version'])).fetchone() if not job or job['state'] != 'failed': return None - object_lock(conn, job['staging_key']) - client().head_object(Bucket=BUCKET, Key=job['staging_key']) + source = attachment['object_key'] if attachment['published_pipeline_version'] else job['staging_key'] + object_lock(conn, source) + client().head_object(Bucket=BUCKET, Key=source) new_attempt = job['attempt'] + 1 budget = int(setting('JOB_MAX_ATTEMPTS', '5')) conn.execute( """UPDATE attachment_jobs SET state='queued', attempt=%s, attempt_limit=%s, - next_publish_at=clock_timestamp(), published_at=NULL, publish_failures=0 + next_publish_at=clock_timestamp(), published_at=NULL, publish_failures=0, settled_at=NULL WHERE id=%s""", (new_attempt, new_attempt + budget - 1, job['id']), ) conn.execute("INSERT INTO attachment_replays(job_id,attempt,reason) VALUES (%s,%s,'operator')", (job['id'], new_attempt)) conn.execute("UPDATE attachments SET state='queued',error_code=NULL,updated_at=clock_timestamp() " - "WHERE id=%s", (attachment_id,)) + "WHERE id=%s AND published_pipeline_version IS NULL", (attachment_id,)) return job['id'], new_attempt + + +def upgrade(attachment_id, version): + with storage.pool.connection() as conn: + attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,)).fetchone() + if not attachment or version not in ENCODING or version <= attachment['target_pipeline_version']: + return None + previous = conn.execute( + 'SELECT staging_key FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s', + (attachment_id, attachment['target_pipeline_version']), + ).fetchone() + if not previous and not attachment['published_pipeline_version']: + return None + source = attachment['object_key'] if attachment['published_pipeline_version'] else previous['staging_key'] + object_lock(conn, source) + client().head_object(Bucket=BUCKET, Key=source) + conn.execute( + """UPDATE attachment_jobs SET state='superseded', lease_token=NULL, lease_until=NULL, + next_publish_at=NULL, settled_at=clock_timestamp() + WHERE attachment_id=%s AND state IN ('queued','processing')""", (attachment_id,), + ) + conn.execute( + """UPDATE attachments SET target_pipeline_version=%s, + state=CASE WHEN published_pipeline_version IS NULL THEN 'queued' ELSE 'ready' END, + error_code=NULL,updated_at=clock_timestamp() WHERE id=%s""", + (version, attachment_id), + ) + job_id = uuid4() + conn.execute( + """INSERT INTO attachment_jobs(id,attachment_id,staging_key,attempt_limit,next_publish_at,pipeline_version) + VALUES (%s,%s,%s,%s,clock_timestamp(),%s)""", + (job_id, attachment_id, f'staging/{attachment_id}', int(setting('JOB_MAX_ATTEMPTS', '5')), version), + ) + return job_id diff --git a/src/operator.py b/src/operator.py index 96508c49..622f2b82 100644 --- a/src/operator.py +++ b/src/operator.py @@ -12,27 +12,31 @@ from src.storage import storage, timestamp def main(): parser = argparse.ArgumentParser() - parser.add_argument('action', choices=('failed', 'replay')) + parser.add_argument('action', choices=('failed', 'replay', 'upgrade')) parser.add_argument('attachment_id', type=UUID, nargs='?') + parser.add_argument('--version', type=int, default=2) args = parser.parse_args() - if args.action == 'replay' and args.attachment_id is None: - parser.error('replay requires attachment_id') + if args.action in ('replay', 'upgrade') and args.attachment_id is None: + parser.error('operation requires attachment_id') storage.pool.open(wait=True) try: - if args.action == 'replay': + if args.action in ('replay', 'upgrade'): try: - result = jobs.replay(args.attachment_id) + if args.action == 'replay': + result = jobs.replay(args.attachment_id) + else: + job_id = jobs.upgrade(args.attachment_id, args.version) + result = (job_id, 1) if job_id else None except (BotoCoreError, ClientError): - raise SystemExit('Источник для replay недоступен') from None + raise SystemExit('Источник обработки недоступен') from None if result is None: - raise SystemExit('Нет подходящей failed-задачи') + raise SystemExit('Операция неприменима к текущему состоянию или версии') print(json.dumps({'job_id': str(result[0]), 'attempt': result[1]})) else: with storage.pool.connection() as conn: rows = conn.execute( - """SELECT j.id, j.attachment_id, j.attempt, j.last_error, a.updated_at - FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id - WHERE j.state='failed' ORDER BY a.updated_at DESC LIMIT 100""" + """SELECT id, attachment_id, pipeline_version, attempt, last_error, settled_at + FROM attachment_jobs WHERE state='failed' ORDER BY settled_at DESC NULLS LAST LIMIT 100""" ).fetchall() print(json.dumps(rows, default=lambda value: str(value) if isinstance(value, UUID) else timestamp(value))) finally: diff --git a/src/pipeline_versions.py b/src/pipeline_versions.py new file mode 100644 index 00000000..89ed8e1c --- /dev/null +++ b/src/pipeline_versions.py @@ -0,0 +1,6 @@ +"""Версии преобразований и операция перехода на следующий manifest.""" + +ENCODING = { + 1: (85, 82), + 2: (90, 88), +} diff --git a/src/routes/attachments.py b/src/routes/attachments.py index 0fd5ac4b..a2728201 100644 --- a/src/routes/attachments.py +++ b/src/routes/attachments.py @@ -128,7 +128,8 @@ def get_attachment(request:Request,attachment_id:uuid.UUID,actor:str=Depends(req row = accessible(conn, request, attachment_id, actor) variants = conn.execute( 'SELECT name,content_type,size_bytes,sha256,width,height FROM attachment_variants ' - 'WHERE attachment_id=%s ORDER BY name', (attachment_id,) + 'WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name', + (attachment_id, row['published_pipeline_version']), ).fetchall() return metadata(row, variants) @@ -141,8 +142,10 @@ def content(request:Request,attachment_id:uuid.UUID,variant:str='original',actor if row['state'] != 'ready': raise error(request,409,'conflict','attachment not ready') if variant != 'original': - row = conn.execute('SELECT * FROM attachment_variants WHERE attachment_id=%s AND name=%s', - (attachment_id, variant)).fetchone() + row = conn.execute( + 'SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s', + (attachment_id, row['published_pipeline_version'], variant), + ).fetchone() if not row: raise error(request,404,'not_found','variant not found') obj=client().get_object(Bucket=BUCKET,Key=row['object_key']) diff --git a/src/staging.py b/src/staging.py index 466269e6..26ec0ad9 100644 --- a/src/staging.py +++ b/src/staging.py @@ -25,10 +25,10 @@ def cleanup(s3, retention): if not object_lock(conn, key, wait=False): continue job = conn.execute( - 'SELECT j.state,a.updated_at FROM attachment_jobs j JOIN attachments a ' - 'ON a.id=j.attachment_id WHERE j.staging_key=%s', (key,), + 'SELECT state,settled_at FROM attachment_jobs WHERE staging_key=%s ' + 'ORDER BY pipeline_version DESC LIMIT 1', (key,), ).fetchone() - retention_start = job['updated_at'] if job else item['LastModified'] + retention_start = (job['settled_at'] if job else None) or item['LastModified'] age = (now - retention_start).total_seconds() if job and job['state'] in ('queued', 'processing'): continue diff --git a/src/worker.py b/src/worker.py index 38d12be5..53ba47d5 100644 --- a/src/worker.py +++ b/src/worker.py @@ -16,7 +16,7 @@ import psycopg from botocore.exceptions import BotoCoreError, ClientError from psycopg_pool import PoolTimeout -from src import broker, jobs +from src import broker, jobs, faults from src.objects import BUCKET, ObjectConflict, client, put_immutable from src.storage import storage @@ -41,13 +41,15 @@ def fetch(s3, key, attachment, destination): def process(job): + faults.hit('before_effect', job) attachment = job['attachment'] s3 = client() with tempfile.TemporaryDirectory() as temp: directory = Path(temp) original = directory / 'original' - fetch(s3, job['staging_key'], attachment, original) - result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', temp], + source = attachment['object_key'] if attachment['published_pipeline_version'] else job['staging_key'] + fetch(s3, source, attachment, original) + result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', temp, str(job['pipeline_version'])], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=jobs.JOB_TIMEOUT, check=False) if result.returncode == 2: @@ -57,8 +59,9 @@ def process(job): put_immutable(s3, attachment['object_key'], stream, attachment['size_bytes'], attachment['content_type'], attachment['sha256']) variants = [] + faults.hit('after_original', job) for name, mime, width, height in json.loads((directory / 'manifest.json').read_text()): - key = f"variants/{attachment['id']}/v1/{name}" + key = f"variants/{attachment['id']}/v{job['pipeline_version']}/{name}" output = directory / name size = output.stat().st_size with output.open('rb') as stream: @@ -66,7 +69,9 @@ def process(job): stream.seek(0) put_immutable(s3, key, stream, size, mime, digest) variants.append((name, key, mime, size, digest, width, height)) + faults.hit('after_variants', job) jobs.complete(job, variants) + faults.hit('after_commit', job) def handle(channel, delivery, body): diff --git a/student-tests/load_pipeline.py b/student-tests/load_pipeline.py new file mode 100644 index 00000000..138568a2 --- /dev/null +++ b/student-tests/load_pipeline.py @@ -0,0 +1,94 @@ +"""Closed-loop: полный путь upload -> ready; ограничение генератора явно в отчёте.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +from pathlib import Path +import random +import struct +import subprocess +import sys +import threading +import time +import zlib + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +def png(width, height): + def chunk(name, data): + return struct.pack('>I', len(data)) + name + data + struct.pack('>I', zlib.crc32(name + data)) + rng = random.Random(2301) + data = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height)) + return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(data)) + chunk(b'IEND', b'')) + + +def percentile(values, q): + return round(sorted(values)[min(len(values)-1, int((len(values)-1)*q))], 4) if values else None + + +def main(args): + client = Client(args.base_url) + alice, _, _, chat = client.bootstrap() + profiles = [('320x200', (ROOT / 'tests/fixtures/sample.png').read_bytes()), ('1024x768', png(1024, 768))] + rows = [] + for profile, raw in profiles: + for repeat in range(args.repeats): + samples = [] + failures = [] + stats = [] + stop = threading.Event() + def sample(): + while not stop.is_set(): + ids = subprocess.check_output(['docker', 'compose', '-p', args.project, 'ps', '-q'], text=True).split() + result = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids], text=True) + stats.append(dict(at_seconds=round(time.monotonic()-start, 3), containers=[json.loads(line) for line in result.splitlines()])) + stop.wait(2) + def one(): + c = Client(args.base_url) + begin = time.monotonic() + att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png', 202, + raw=raw, actor=alice, headers={'Content-Type': 'image/png'})['json'] + accepted = time.monotonic() + ready = c.wait_ready(alice, att) + check(len(ready['variants']) == 3 and ready['sha256'] == hashlib.sha256(raw).hexdigest(), 'bad load result') + return dict(upload_seconds=accepted-begin, ready_seconds=time.monotonic()-begin) + # Один завершённый прогрев на профиль/повтор, не входит в измерение. + one() + start = time.monotonic() + sampler = threading.Thread(target=sample) + sampler.start() + def loop(_): + while time.monotonic()-start < args.duration: + try: + samples.append(one()) + except Exception as exc: + failures.append(type(exc).__name__ + ': ' + str(exc)) + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + list(pool.map(loop, range(args.concurrency))) + elapsed = time.monotonic()-start + stop.set() + sampler.join() + times = [x['ready_seconds'] for x in samples] + row = dict(profile=profile, bytes_per_image=len(raw), repeat=repeat+1, concurrency=args.concurrency, + requested_duration=args.duration, elapsed_seconds=round(elapsed, 3), warmup_completed=1, + completed=len(samples), successful_rps=round(len(samples)/elapsed, 3), failures=failures, + error_fraction=len(failures)/max(1,len(samples)+len(failures)), + p50=percentile(times,.5), p95=percentile(times,.95), p99=percentile(times,.99), + upload_p95=percentile([x['upload_seconds'] for x in samples],.95), stats=stats) + rows.append(row) + (ROOT / 'evidence/load-pipeline.json').write_text(json.dumps(rows, indent=2), encoding='utf-8') + print(json.dumps({k:v for k,v in row.items() if k != 'stats'}), flush=True) + + +if __name__ == '__main__': + p = argparse.ArgumentParser() + p.add_argument('--base-url', default='http://127.0.0.1:8085') + p.add_argument('--project', default='lyakhonov-lab5') + p.add_argument('--duration', type=int, default=20) + p.add_argument('--concurrency', type=int, default=2) + p.add_argument('--repeats', type=int, default=3) + main(p.parse_args()) diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py new file mode 100644 index 00000000..3a6f6be5 --- /dev/null +++ b/student-tests/pipeline.py @@ -0,0 +1,242 @@ +"""Реальные fault-injection и инварианты. Запускать только на своём lab5 стенде.""" +import argparse +import base64 +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import os +from pathlib import Path +import struct +import subprocess +import sys +import time +import zlib + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +class Experiment: + def __init__(self, args): + self.args = args + self.compose = ['docker', 'compose', '-p', args.project] + self.c = Client(args.base_url) + self.alice, self.bob, self.eve, self.chat = self.c.bootstrap() + path = ROOT / 'evidence/pipeline.json' + self.rows = json.loads(path.read_text(encoding='utf-8')) if args.resume and path.exists() else [] + + def command(self, *args): + return subprocess.check_output([*self.compose, *args], cwd=ROOT, text=True, encoding='utf-8', stderr=subprocess.PIPE) + + def probe(self, action, **kwargs): + script = 'ARGS=' + repr(dict(action=action, **kwargs)) + '\n' + (ROOT / 'student-tests/probe.py').read_text(encoding='utf-8') + invocation = ([*self.compose, 'run', '--rm', '--no-deps', '-T', 'inspector', 'python', '-'] + if action == 'objects' else [*self.compose, 'exec', '-T', '--index', '1', 'api', 'python', '-']) + result = subprocess.run(invocation, + input=script, text=True, encoding='utf-8', capture_output=True, cwd=ROOT) + check(result.returncode == 0, 'probe failed: ' + result.stderr) + return json.loads(result.stdout) + + def sql(self, query, *params): + return self.probe('sql', query=query, params=list(params)) + + def job(self, att, version=1): + return self.sql('SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s', att['id'], version)[0] + + @staticmethod + def envelope(job): + from datetime import datetime, timezone + created = datetime.fromisoformat(job['created_at']).astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + return dict(schema_version=1, type='attachment.process.v1', job_id=job['id'], attachment_id=job['attachment_id'], + pipeline_version=job['pipeline_version'], attempt=job['attempt'], created_at=created) + + def upload(self): + return self.c.attachment(self.alice, self.chat)[0] + + def arm(self, job, point, action='crash'): + self.sql('INSERT INTO experiment_faults(job_id,point,action) VALUES (%s,%s,%s)', job['id'], point, action) + + def hit(self, job, point): + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + row = self.sql('SELECT * FROM experiment_faults WHERE job_id=%s AND point=%s', job['id'], point)[0] + if row['hit_at']: + return row + time.sleep(.3) + raise AssertionError('fault not hit: ' + point) + + def snapshot(self): + ids = self.command('ps', '-a', '-q').split() + raw = subprocess.check_output(['docker', 'inspect', *ids], text=True, encoding='utf-8') + return [dict(id=x['Id'], name=x['Name'], started_at=x['State']['StartedAt'], + running=x['State']['Running'], restart_count=x['RestartCount'], exit_code=x['State']['ExitCode']) for x in json.loads(raw)] + + def invariant(self, att, version=1): + ready = self.c.wait_ready(self.alice, att, wait_seconds=120) + check(len(ready['variants']) == 3, 'three variants required') + row = self.sql('SELECT published_pipeline_version,target_pipeline_version FROM attachments WHERE id=%s', att['id'])[0] + check(row['published_pipeline_version'] == version, 'wrong published version') + variants = self.sql('SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s', att['id'], version) + check(len(variants) == 3, 'duplicate/missing manifests') + digest = {} + for name in ('original', 'thumbnail', 'crop', 'optimized'): + response = self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + name, actor=self.alice) + sha = hashlib.sha256(response['bytes']).hexdigest() + expected = att['sha256'] if name == 'original' else next(v['sha256'] for v in variants if v['name'] == name) + check(sha == expected, 'download checksum mismatch') + digest[name] = sha + return digest + + def record(self, name, started, **details): + self.rows.append(dict(scenario=name, seconds=round(time.monotonic() - started, 3), **details)) + (ROOT / 'evidence/pipeline.json').write_text(json.dumps(self.rows, ensure_ascii=False, indent=2), encoding='utf-8') + print('PASS:', name, flush=True) + + def crashes(self): + for repeat in range(self.args.repeats): + for point in ('before_publish', 'after_publish', 'before_effect', 'after_original', 'after_variants', 'after_commit'): + start = time.monotonic() + service = 'dispatcher' if 'publish' in point else 'worker' + self.command('stop', '-t', '2', service) + att = self.upload() + job = self.job(att) + self.arm(job, point) + before = self.snapshot() + self.command('start', service) + hit = self.hit(job, point) + self.invariant(att) + self.record(point, start, repeat=repeat + 1, attachment_id=att['id'], fault=hit, + job_after=self.job(att), before=before, after=self.snapshot()) + + def durable(self): + start = time.monotonic() + self.command('stop', '-t', '2', 'worker') + att = self.upload() + self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=self.alice) + self.probe('cleanup') + check(any(obj['key'].startswith('staging/') for obj in self.probe('objects', id=att['id'])), 'active staging deleted') + self.c.message(self.alice, self.chat, '', attachments=[att['id']]) + time.sleep(2) + before = self.snapshot() + self.command('restart', '-t', '2', 'api', 'broker') + self.command('start', 'worker') + self.invariant(att) + self.record('accepted_survives_api_broker_restart', start, attachment_id=att['id'], before=before, after=self.snapshot()) + + def versions(self): + for repeat in range(self.args.repeats): + start = time.monotonic() + self.command('stop', '-t', '2', 'worker') + att = self.upload() + old = self.job(att) + self.arm(old, 'after_variants', 'pause') + self.command('start', 'worker') + self.hit(old, 'after_variants') + self.probe('upgrade', id=att['id']) + new = self.job(att, 2) + # Новая работа, затем старая, затем 24 чередующихся дубля. + self.probe('publish', messages=[self.envelope(new), self.envelope(old)] * 13) + first = self.invariant(att, 2) + objects = self.probe('objects', id=att['id']) + self.sql('UPDATE experiment_faults SET released=true WHERE job_id=%s', old['id']) + time.sleep(2) + check(self.invariant(att, 2) == first, 'stale worker changed new result') + self.probe('publish', messages=[self.envelope(old), self.envelope(new)] * 13) + time.sleep(2) + check(self.invariant(att, 2) == first, 'reorder changed new result') + current = self.probe('objects', id=att['id']) + check([x for x in current if not x['key'].startswith('staging/')] == + [x for x in objects if not x['key'].startswith('staging/')], 'duplicate wrote more objects') + check(self.probe('replay', id=att['id']) is None, 'ready replay must no-op') + self.record('stale_running_worker_and_reordered_duplicates', start, repeat=repeat + 1, + attachment_id=att['id'], before_digest=first, after_digest=self.invariant(att, 2), + versions=self.sql('SELECT pipeline_version,state,attempt FROM attachment_jobs WHERE attachment_id=%s ORDER BY pipeline_version', att['id']), objects=current) + start = time.monotonic() + att = self.upload() + old_digest = self.invariant(att) + self.probe('cleanup') + self.probe('upgrade', id=att['id']) + deadline = time.monotonic() + 60 + while self.job(att, 2)['state'] != 'ready': + check(time.monotonic() < deadline, 'ready upgrade timeout') + time.sleep(.3) + new_digest = self.invariant(att, 2) + check(new_digest['original'] == old_digest['original'], 'upgrade changed original') + check(new_digest['thumbnail'] != old_digest['thumbnail'], 'versions must have a real different effect') + self.record('upgrade_ready_after_staging_cleanup', start, repeat=repeat + 1, attachment_id=att['id'], before_digest=old_digest, after_digest=new_digest) + + def errors(self): + for kind in ('permanent', 'transient'): + start = time.monotonic() + self.command('stop', '-t', '2', 'worker') + att = self.upload() + if kind == 'permanent': + self.probe('corrupt_staging', id=att['id']) + else: + self.probe('remove_staging', id=att['id']) + self.command('start', 'worker') + observed = [] + deadline = time.monotonic() + 120 + while True: + job = self.job(att) + observed.append(dict(attempt=job['attempt'], state=job['state'], next_publish_at=job['next_publish_at'], last_error=job['last_error'])) + if job['state'] == 'failed': + break + check(time.monotonic() < deadline, 'failed registry timeout') + time.sleep(.3) + check(job['attempt'] == (1 if kind == 'permanent' else job['attempt_limit']), 'wrong retry budget') + self.c.call('POST', '/api/v1/chats/' + self.chat['id'] + '/messages', 409, actor=self.alice, + headers={'Idempotency-Key': os.urandom(12).hex()}, body={'text': '', 'attachment_ids': [att['id']]}) + self.probe('restore_staging', id=att['id'], data=base64.b64encode((ROOT / 'tests/fixtures/sample.png').read_bytes()).decode()) + check(self.probe('replay', id=att['id']) is not None, 'failed replay missing') + self.invariant(att) + self.record(kind + '_error_and_explicit_replay', start, attachment_id=att['id'], attempts=observed, + audit=self.sql('SELECT attempt,reason FROM attachment_replays WHERE job_id=%s', job['id'])) + + def negatives(self): + start = time.monotonic() + path = '/api/v1/chats/' + self.chat['id'] + '/attachments?filename=x.png' + def upload(raw, expected, path=path, actor=None, mime='image/png'): + self.c.call('POST', path, expected, actor=actor or self.alice, raw=raw, headers={'Content-Type': mime}) + upload(b'bad', 415) + upload(b'x' * (10 * 1024 * 1024 + 1), 413) + raw = (ROOT / 'tests/fixtures/sample.png').read_bytes() + upload(raw, 415, mime='image/jpeg') + upload(raw, 400, path=path.replace('x.png', '..%2Fx.png')) + upload(raw, 404, actor=self.eve) + # Валидный PNG header >40 MP: отклонение до декодирования большого bitmap. + def chunk(name, data): + return struct.pack('>I', len(data)) + name + data + struct.pack('>I', zlib.crc32(name + data)) + huge = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 10000, 10000, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(b'')) + chunk(b'IEND', b'') + upload(huge, 413) + att = self.upload() + self.invariant(att) + self.c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=self.eve) + self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=self.eve) + key = os.urandom(16).hex() + with ThreadPoolExecutor(max_workers=12) as pool: + responses = list(pool.map(lambda _: self.c.message(self.alice, self.chat, 'concurrent', key=key)['json']['id'], range(24))) + check(len(set(responses)) == 1, 'message idempotency violated') + self.record('negative_upload_acl_and_24_concurrent_messages', start, checks=9, message_ids=len(set(responses))) + + def main(self): + try: + if not self.args.resume: + self.negatives() + self.durable() + self.crashes() + self.versions() + self.errors() + finally: + self.sql('UPDATE experiment_faults SET released=true WHERE NOT released') + self.command('start', 'api', 'broker', 'dispatcher', 'worker') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--project', default='lyakhonov-lab5') + parser.add_argument('--base-url', default='http://127.0.0.1:8085') + parser.add_argument('--repeats', type=int, default=3) + parser.add_argument('--resume', action='store_true', help='Сохранить уже выполненные crash-результаты; выполнить version/error suite') + Experiment(parser.parse_args()).main() diff --git a/student-tests/probe.py b/student-tests/probe.py new file mode 100644 index 00000000..10b3cab5 --- /dev/null +++ b/student-tests/probe.py @@ -0,0 +1,55 @@ +"""Исполняется внутри API через stdin; наружу возвращает только безопасные данные.""" +import json +import sys +from uuid import UUID + +from src import broker, jobs +from src.objects import BUCKET, client +from src.storage import storage + + +def run(args): + action = args['action'] + if action == 'sql': + with storage.pool.connection() as conn: + cursor = conn.execute(args['query'], args.get('params', [])) + return cursor.fetchall() if cursor.description else [] + if action == 'publish': + with broker.channel() as (_, channel): + channel.confirm_delivery() + for message in args['messages']: + broker.publish(channel, message) + return True + if action == 'upgrade': + return jobs.upgrade(UUID(args['id']), args.get('version', 2)) + if action == 'replay': + return jobs.replay(UUID(args['id'])) + if action == 'cleanup': + from src.staging import cleanup + return cleanup(client(), 60) + if action == 'objects': + s3 = client() + result = [] + for prefix in (f"originals/{args['id']}", f"variants/{args['id']}/", f"staging/{args['id']}"): + for page in s3.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=prefix): + result.extend({'key': row['Key'], 'etag': row['ETag'], 'size': row['Size']} + for row in page.get('Contents', [])) + return result + if action == 'remove_staging': + client().delete_object(Bucket=BUCKET, Key=f"staging/{args['id']}") + return True + if action == 'corrupt_staging': + client().put_object(Bucket=BUCKET, Key=f"staging/{args['id']}", Body=b'broken') + return True + if action == 'restore_staging': + import base64 + client().put_object(Bucket=BUCKET, Key=f"staging/{args['id']}", Body=base64.b64decode(args['data'])) + return True + raise ValueError('unknown probe action') + + +storage.pool.open(wait=True) +try: + print(json.dumps(run(ARGS), default=str)) +finally: + storage.pool.close() diff --git a/student-tests/resilience_driver.py b/student-tests/resilience_driver.py new file mode 100644 index 00000000..aed561af --- /dev/null +++ b/student-tests/resilience_driver.py @@ -0,0 +1,82 @@ +"""Публичные resilience и отдельный принудительный отказ каждой API-реплики.""" +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check + +COMPOSE = ['docker', 'compose', '-p', 'lyakhonov-lab5'] + + +def docker(*args): + return subprocess.check_output(['docker', *args], cwd=ROOT, text=True, encoding='utf-8') + + +def snapshot(): + ids = docker('compose', '-p', 'lyakhonov-lab5', 'ps', '-a', '-q').split() + raw = json.loads(docker('inspect', *ids)) + return [dict(id=x['Id'], name=x['Name'], started_at=x['State']['StartedAt'], running=x['State']['Running']) for x in raw] + + +def public(mode): + before = snapshot() + script = ROOT / 'scripts' / (('restart' if mode == 'persistence' else 'stop-one') + ('.cmd' if sys.platform == 'win32' else '.sh')) + result = subprocess.run([sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', mode, + '--base-url', 'http://127.0.0.1:8085', '--action-script', str(script)], + cwd=ROOT, text=True, encoding='utf-8', capture_output=True) + after = snapshot() + (ROOT / f'evidence/{mode}.json').write_text(json.dumps(dict(command=result.args, exit_code=result.returncode, + output=result.stdout + result.stderr, before=before, after=after), ensure_ascii=False, indent=2), encoding='utf-8') + print(result.stdout, flush=True) + check(result.returncode == 0, result.stderr) + subprocess.check_call([*COMPOSE, 'start', 'api'], cwd=ROOT) + + +def targeted(): + c = Client('http://127.0.0.1:8085', timeout=2) + alice, _, _, chat = c.bootstrap() + ids = docker('compose', '-p', 'lyakhonov-lab5', 'ps', '-q', 'api').split() + evidence = [] + for index, target in enumerate(ids): + before = snapshot() + responses, errors = [], [] + first = None + begin = time.monotonic() + subprocess.check_call(['docker', 'stop', '-t', '2', target], stdout=subprocess.DEVNULL) + try: + while time.monotonic() - begin < 15: + key = f'explicit-{index}-{len(responses)}' + try: + response = c.message(alice, chat, 'replica failure', key=key) + first = first or time.monotonic()-begin + responses.append(response['json']) + except Failure as exc: + errors.append(str(exc)) + time.sleep(.05) + check(first is not None and first <= 10, 'slow API failover') + page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice)['json'] + messages = list(page['items']) + while page['next_cursor']: + from urllib.parse import quote + page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe=''), actor=alice)['json'] + messages.extend(page['items']) + found = {m['id'] for m in messages} + check(all(m['id'] in found for m in responses), 'acknowledged message lost') + check(len(found) == len(messages), 'duplicate messages') + evidence.append(dict(target=target, before=before, after=snapshot(), successes=len(responses), + errors=errors, recovery_seconds=round(first, 3))) + finally: + subprocess.check_call(['docker', 'start', target], stdout=subprocess.DEVNULL) + time.sleep(3) + (ROOT / 'evidence/failover-each.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2), encoding='utf-8') + print('PASS: both real API containers independently stopped and restored') + + +if __name__ == '__main__': + public('persistence') + public('failover') + targeted()