Реализовать версии pipeline, fencing и автоматические сценарии отказов

This commit is contained in:
2026-11-20 06:51:54 +03:00
parent e376ede0a2
commit 42bf6aa702
21 changed files with 845 additions and 53 deletions
+1 -1
View File
@@ -1,4 +1,4 @@
# Reproducible fallback: official MinIO images are unavailable from registries.
# Build pinned upstream releases and verify archive checksums.
FROM golang:1.24.6-bookworm@sha256:ab1d1823abb55a9504d2e3e003b75b36dbeb1cbcc4c92593d85a84ee46becc6c AS build
WORKDIR /build
ADD --checksum=sha256:8819e3e7817e46b7b3798f8f200ead208562e571563c2e040352378031abe9f2 https://github.com/minio/minio/archive/07c3a429bfed433e49018cb0f78a52145d4bedeb.tar.gz /tmp/minio.tar.gz
+15
View File
@@ -11,6 +11,8 @@ x-app: &app
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
LAB5_FAULT_INJECTION: ${LAB5_FAULT_INJECTION:-0}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
CACHE_URL: redis://redis:6379/0
BROKER_URL_FILE: /run/secrets/broker_url
@@ -26,6 +28,19 @@ x-dependencies: &dependencies
condition: service_completed_successfully
services:
inspector:
<<: *app
profiles: [tools]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: http://s3:9000
S3_ACCESS_KEY_FILE: /run/secrets/s3_root_user
S3_SECRET_KEY_FILE: /run/secrets/s3_root_password
secrets: [database_url, s3_root_user, s3_root_password]
healthcheck:
disable: true
cpus: 0.25
mem_limit: 256m
db:
image: postgres:16.6-alpine
environment:
+19
View File
@@ -0,0 +1,19 @@
ALTER TABLE attachments
ADD COLUMN target_pipeline_version integer NOT NULL DEFAULT 1 CHECK (target_pipeline_version >= 1),
ADD COLUMN published_pipeline_version integer CHECK (published_pipeline_version >= 1),
ADD CHECK (published_pipeline_version <= target_pipeline_version);
UPDATE attachments SET published_pipeline_version = 1 WHERE state = 'ready';
ALTER TABLE attachment_jobs
DROP CONSTRAINT attachment_jobs_attachment_id_key,
DROP CONSTRAINT attachment_jobs_staging_key_key,
DROP CONSTRAINT attachment_jobs_state_check,
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1),
ADD COLUMN settled_at timestamptz,
ADD CHECK (state IN ('queued', 'processing', 'ready', 'failed', 'superseded')),
ADD UNIQUE (attachment_id, pipeline_version);
CREATE INDEX attachment_jobs_staging ON attachment_jobs(staging_key);
ALTER TABLE attachment_variants ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1;
ALTER TABLE attachment_variants DROP CONSTRAINT attachment_variants_pkey;
ALTER TABLE attachment_variants ADD PRIMARY KEY (attachment_id, pipeline_version, name);
+9
View File
@@ -0,0 +1,9 @@
CREATE TABLE experiment_faults (
job_id uuid NOT NULL REFERENCES attachment_jobs(id),
point text NOT NULL,
action text NOT NULL CHECK (action IN ('crash', 'pause')),
hit_at timestamptz,
process_id integer,
released boolean NOT NULL DEFAULT false,
PRIMARY KEY (job_id, point)
);
+55
View File
@@ -0,0 +1,55 @@
"""Аудит обязательных материалов, локальных секретов и предыдущей лабораторной."""
import hashlib
import json
import os
from pathlib import Path
import subprocess
ROOT = Path(__file__).resolve().parents[1]
PREVIOUS = ROOT.parent / 'vysoko-nagr 4'
ENV = dict(os.environ, GIT_OPTIONAL_LOCKS='0')
def git(root, *args):
return subprocess.check_output(['git', '-C', str(root), *args], env=ENV).decode('utf-8').strip()
def previous():
paths = git(PREVIOUS, 'ls-files', '-z').split('\0')
files = {p: hashlib.sha256((PREVIOUS / p).read_bytes()).hexdigest() for p in paths if p}
return dict(head=git(PREVIOUS, 'rev-parse', 'HEAD'), branch=git(PREVIOUS, 'branch', '--show-current'),
status=git(PREVIOUS, 'status', '--porcelain'), files=files)
def main():
import sys
if sys.argv[1:] == ['baseline']:
(ROOT / 'evidence/lab4-baseline.json').write_text(json.dumps(previous(), ensure_ascii=False, indent=2), encoding='utf-8')
print('Сохранён read-only fingerprint лабораторной 4')
return
baseline = json.loads((ROOT / 'evidence/lab4-baseline.json').read_text(encoding='utf-8'))
unchanged = previous() == baseline
assert unchanged, 'Lab4 differs from baseline'
protected = ['README.md', 'COURSE.md', 'CONTRIBUTING.md', 'MAINTAINERS.md', 'contracts', 'tests',
'examples', 'compose.example.yaml', '.env.example', '.github']
assert not git(ROOT, 'diff', '276dbda', '--', *protected), 'Protected lab5 material changed'
paths = git(ROOT, 'ls-files', '-z').split('\0')
secret_values = [p.read_bytes().strip() for p in (ROOT / 'secrets').iterdir() if p.is_file()]
for path in paths:
if not path:
continue
assert not path.startswith('secrets/') and path != '.env', 'Tracked private state'
content = (ROOT / path).read_bytes()
assert not any(value and value in content for value in secret_values), 'Local secret leaked in ' + path
assert b'-----BEGIN PRIVATE KEY-----' not in content, 'Private key found'
authors = git(ROOT, 'log', '276dbda..HEAD', '--format=%h|%an|%ae|%cn|%ce').splitlines()
for row in authors:
assert row.split('|')[1:] == ['Sergey Lyakhonov', 'lyserg119@gmail.com'] * 2, 'Wrong author/committer'
report = dict(lab4_unchanged=unchanged, protected_lab5_unchanged=True, tracked_local_secrets=False,
scanned_files=len(paths), authors=authors, head=git(ROOT, 'rev-parse', 'HEAD'))
(ROOT / 'evidence/audit.json').write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding='utf-8')
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == '__main__':
main()
+28
View File
@@ -0,0 +1,28 @@
"""Безопасная фиксация версий, лимитов и опубликованных портов."""
import json
from pathlib import Path
import platform
import subprocess
ROOT = Path(__file__).resolve().parents[1]
def output(*args):
return subprocess.check_output(args, cwd=ROOT, text=True, encoding='utf-8').strip()
ids = output('docker', 'compose', '-p', 'lyakhonov-lab5', 'ps', '-q').split()
containers = json.loads(output('docker', 'inspect', *ids))
result = dict(commit=output('git', 'rev-parse', 'HEAD'), python=platform.python_version(),
docker=json.loads(output('docker', 'version', '--format', '{{json .}}')),
compose=output('docker', 'compose', 'version'),
engine=json.loads(output('docker', 'info', '--format', '{"memory_bytes":{{.MemTotal}},"cpus":{{.NCPU}}}')),
services=[])
for row in containers:
result['services'].append(dict(service=row['Config']['Labels']['com.docker.compose.service'], id=row['Id'],
image=row['Config']['Image'], image_id=row['Image'], started_at=row['State']['StartedAt'],
memory_limit=row['HostConfig']['Memory'], nanocpus=row['HostConfig']['NanoCpus'],
ports=row['HostConfig']['PortBindings'],
settings=[x for x in row['Config']['Env'] if x.startswith(('JOB_', 'WORKER_', 'LAB5_FAULT_', 'STAGING_RETENTION_'))]))
(ROOT / 'evidence/environment.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
print('Saved versions, IDs, resource limits, ports and non-secret experiment settings')
+34
View File
@@ -0,0 +1,34 @@
"""Статические графики только из сохранённых фактических агрегатов."""
import json
from pathlib import Path
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
ROOT = Path(__file__).resolve().parents[1]
data = json.loads((ROOT / 'evidence/pipeline.json').read_text(encoding='utf-8'))
points = ('before_publish', 'after_publish', 'before_effect', 'after_original', 'after_variants', 'after_commit')
fig, ax = plt.subplots(figsize=(10, 4.5), layout='constrained')
for index, point in enumerate(points):
values = [row['seconds'] for row in data if row['scenario'] == point]
ax.scatter([index] * len(values), values, s=45)
ax.set_xticks(range(len(points)), points, rotation=15)
ax.set_ylabel('Full scenario duration, seconds')
ax.set_title('Actual crash scenarios: stop, upload, restart, verified downloads')
ax.grid(axis='y', alpha=.25)
fig.savefig(ROOT / 'evidence/faults.png', dpi=160)
plt.close(fig)
data = json.loads((ROOT / 'evidence/load-pipeline.json').read_text(encoding='utf-8'))
fig, axes = plt.subplots(1, 2, figsize=(10, 4), layout='constrained')
for index, profile in enumerate(sorted({row['profile'] for row in data})):
rows = [row for row in data if row['profile'] == profile]
for axis, metric in zip(axes, ('successful_rps', 'p95')):
axis.scatter([index] * len(rows), [row[metric] for row in rows], label=profile)
for axis, label in zip(axes, ('Completed images / second', 'Upload to ready p95, seconds')):
axis.set_xticks(range(len({row['profile'] for row in data})), sorted({row['profile'] for row in data}))
axis.set_ylabel(label)
axis.grid(axis='y', alpha=.25)
fig.suptitle('Closed-loop pipeline: 2 clients, 20 seconds, 3 repeats')
fig.savefig(ROOT / 'evidence/load.png', dpi=160)
plt.close(fig)
print('Saved evidence/faults.png and evidence/load.png')
+58
View File
@@ -0,0 +1,58 @@
"""Чистый Git-экспорт, новые секреты и отдельный временный Compose-проект."""
import io
import json
import os
from pathlib import Path
import socket
import subprocess
import sys
import tempfile
import uuid
import zipfile
ROOT = Path(__file__).resolve().parents[1]
def main():
name = 'lyakhonov-lab5-repro-' + uuid.uuid4().hex[:8]
directory = Path(tempfile.mkdtemp(prefix=name + '-'))
archive = subprocess.check_output(['git', 'archive', '--format=zip', 'HEAD'], cwd=ROOT)
with zipfile.ZipFile(io.BytesIO(archive)) as contents:
contents.extractall(directory)
with socket.socket() as listener:
listener.bind(('127.0.0.1', 0))
port = listener.getsockname()[1]
env = dict(os.environ, LAB5_PORT=str(port), LAB5_FAULT_INJECTION='0', JOB_TIMEOUT_SECONDS='60', JOB_LEASE_SECONDS='90')
services = ('api', 'migrate', 'worker', 'dispatcher', 'staging-cleaner')
images = [name + ':app', name + ':s3']
override = {'services': {service: {'image': images[0]} for service in services}}
override['services'].update({service: {'image': images[1]} for service in ('s3', 'bucket-init')})
(directory / 'repro-override.json').write_text(json.dumps(override), encoding='utf-8')
compose = ['docker', 'compose', '-p', name, '-f', 'compose.yaml', '-f', 'repro-override.json']
events = []
def run(args):
result = subprocess.run(args, cwd=directory, env=env, text=True, encoding='utf-8', capture_output=True)
# Только безопасные последние строки: ни inspect .Config.Env, ни secret files.
events.append(dict(command=args, exit_code=result.returncode, output=(result.stdout + result.stderr)[-5000:]))
if result.returncode:
raise RuntimeError('reproduction failed: ' + ' '.join(args))
try:
run([sys.executable, '-X', 'utf8', 'scripts/bootstrap.py'])
run([sys.executable, '-X', 'utf8', 'scripts/check_contract.py'])
run(['docker', 'build', '-t', images[0], '.'])
run(['docker', 'build', '-t', images[1], '-f', 'Dockerfile.s3', '.'])
run([*compose, 'up', '-d', '--wait'])
run([sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', f'http://127.0.0.1:{port}'])
run([*compose, 'ps'])
finally:
run([*compose, 'down', '--volumes', '--remove-orphans'])
for tag in images:
run(['docker', 'image', 'rm', tag])
result = dict(project=name, directory=str(directory), port=port,
commit=subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), events=events)
(ROOT / 'evidence/reproduction.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
print('PASS: clean export, fresh secrets, isolated project; temporary Docker resources removed')
if __name__ == '__main__':
main()
+3 -1
View File
@@ -9,7 +9,7 @@ import pika
import psycopg
from psycopg_pool import PoolTimeout
from src import broker, jobs
from src import broker, jobs, faults
from src.storage import storage
@@ -32,7 +32,9 @@ def main():
next_recovery = time.monotonic() + 1
reserved = jobs.reserve_publish()
if reserved:
faults.hit('before_publish', reserved)
broker.publish(channel, jobs.envelope(reserved))
faults.hit('after_publish', reserved)
jobs.confirm_publish(reserved)
reserved = None
else:
+29
View File
@@ -0,0 +1,29 @@
"""Одноразовые точки отказа. Выключены без LAB5_FAULT_INJECTION=1."""
import os
import time
from src.storage import storage
def hit(point, job):
if os.getenv('LAB5_FAULT_INJECTION') != '1':
return
with storage.pool.connection() as conn:
fault = conn.execute(
"UPDATE experiment_faults SET hit_at=clock_timestamp(), process_id=%s "
"WHERE job_id=%s AND point=%s AND hit_at IS NULL RETURNING action",
(os.getpid(), job['id'], point),
).fetchone()
if not fault:
return
if fault['action'] == 'crash':
os._exit(91) # Реальная смерть процесса без finally/ack.
while True:
with storage.pool.connection() as conn:
released = conn.execute(
'SELECT released FROM experiment_faults WHERE job_id=%s AND point=%s',
(job['id'], point),
).fetchone()['released']
if released:
return
time.sleep(0.1)
+7 -5
View File
@@ -5,6 +5,7 @@ import sys
from pathlib import Path
from PIL import Image, ImageOps
from src.pipeline_versions import ENCODING
Image.MAX_IMAGE_PIXELS = 40_000_000
@@ -18,7 +19,8 @@ def fit(image, limit):
return image.resize(size, Image.Resampling.LANCZOS)
def render(source, target):
def render(source, target, version=1):
jpeg_quality, webp_quality = ENCODING[version]
with Image.open(source) as opened:
if opened.width * opened.height > Image.MAX_IMAGE_PIXELS:
raise ValueError('pixel limit')
@@ -32,11 +34,11 @@ def render(source, target):
rgb.info.clear()
del image
thumbnail = fit(rgb, 256)
thumbnail.save(target / 'thumbnail', 'JPEG', quality=85)
thumbnail.save(target / 'thumbnail', 'JPEG', quality=jpeg_quality)
crop = ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS)
crop.save(target / 'crop', 'JPEG', quality=85)
crop.save(target / 'crop', 'JPEG', quality=jpeg_quality)
optimized = fit(rgb, 1280)
optimized.save(target / 'optimized', 'WEBP', quality=82, method=4)
optimized.save(target / 'optimized', 'WEBP', quality=webp_quality, method=4)
return [
['thumbnail', 'image/jpeg', *thumbnail.size],
['crop', 'image/jpeg', *crop.size],
@@ -47,7 +49,7 @@ def render(source, target):
def main():
directory = Path(sys.argv[1])
try:
result = render(directory / 'original', directory)
result = render(directory / 'original', directory, int(sys.argv[2]))
except (OSError, ValueError, SyntaxError, Image.DecompressionBombError):
return 2
(directory / 'manifest.json').write_text(json.dumps(result), encoding='utf-8')
+76 -26
View File
@@ -1,14 +1,17 @@
"""Устойчивое намерение обработки, lease и атомарная публикация manifest."""
import random
import re
from datetime import datetime
from uuid import UUID, uuid4
from src.config import setting
from src.objects import BUCKET, client, object_lock
from src.pipeline_versions import ENCODING
from src.storage import storage, timestamp
JOB_TIMEOUT = int(setting('JOB_TIMEOUT_SECONDS', '60'))
LEASE_SECONDS = JOB_TIMEOUT + 30
LEASE_SECONDS = max(JOB_TIMEOUT + 5, int(setting('JOB_LEASE_SECONDS', str(JOB_TIMEOUT + 30))))
def envelope(job):
@@ -17,24 +20,31 @@ def envelope(job):
'job_id': str(job['id']),
'type': 'attachment.process.v1',
'attachment_id': str(job['attachment_id']),
'pipeline_version': 1,
'pipeline_version': job['pipeline_version'],
'attempt': job['attempt'],
'created_at': timestamp(job['created_at']),
}
def validate(message):
if set(message) != {'schema_version', 'job_id', 'type', 'attachment_id',
'pipeline_version', 'attempt', 'created_at'}:
required = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'}
if not isinstance(message, dict) or not required <= message.keys() <= required | {'traceparent', 'tracestate'}:
raise ValueError('invalid fields')
if (type(message['schema_version']) is not int or message['schema_version'] != 1
or message['type'] != 'attachment.process.v1'
or type(message['pipeline_version']) is not int or message['pipeline_version'] != 1
or type(message['pipeline_version']) is not int or message['pipeline_version'] not in ENCODING
or type(message['attempt']) is not int or message['attempt'] < 1):
raise ValueError('invalid job')
UUID(message['job_id'])
UUID(message['attachment_id'])
return message
if datetime.fromisoformat(message['created_at'].replace('Z', '+00:00')).tzinfo is None:
raise ValueError('missing timezone')
if 'traceparent' in message and not re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}',
message['traceparent']):
raise ValueError('invalid traceparent')
if 'tracestate' in message and (not isinstance(message['tracestate'], str) or len(message['tracestate']) > 512):
raise ValueError('invalid tracestate')
return {key: message[key] for key in required}
def reserve_publish():
@@ -83,7 +93,8 @@ def locked_job(conn, job_id, attachment_id):
def claim(message):
with storage.pool.connection() as conn:
job, attachment = locked_job(conn, message['job_id'], message['attachment_id'])
if not job or envelope(job) != message or job['state'] != 'queued':
if (not job or envelope(job) != message or job['state'] != 'queued'
or job['pipeline_version'] != attachment['target_pipeline_version']):
return None
token = uuid4()
conn.execute(
@@ -91,7 +102,8 @@ def claim(message):
lease_until=clock_timestamp()+%s * interval '1 second', next_publish_at=NULL
WHERE id=%s""", (token, LEASE_SECONDS, job['id']),
)
conn.execute("UPDATE attachments SET state='processing',updated_at=clock_timestamp() WHERE id=%s",
conn.execute("UPDATE attachments SET state='processing',updated_at=clock_timestamp() "
"WHERE id=%s AND published_pipeline_version IS NULL",
(job['attachment_id'],))
return job | {'lease_token': token, 'attachment': attachment}
@@ -105,20 +117,21 @@ def owns(conn, job, claimed):
def complete(claimed, variants):
with storage.pool.connection() as conn:
job, _ = locked_job(conn, claimed['id'], claimed['attachment_id'])
if not owns(conn, job, claimed):
job, attachment = locked_job(conn, claimed['id'], claimed['attachment_id'])
if not owns(conn, job, claimed) or job['pipeline_version'] != attachment['target_pipeline_version']:
return False
for variant in variants:
conn.execute(
"""INSERT INTO attachment_variants
(attachment_id,name,object_key,content_type,size_bytes,sha256,width,height)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""",
(job['attachment_id'], *variant),
(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
(job['attachment_id'], job['pipeline_version'], *variant),
)
conn.execute("UPDATE attachments SET state='ready',updated_at=clock_timestamp() WHERE id=%s",
(job['attachment_id'],))
conn.execute("UPDATE attachments SET state='ready',published_pipeline_version=%s,"
"updated_at=clock_timestamp(),error_code=NULL WHERE id=%s",
(job['pipeline_version'], job['attachment_id']))
conn.execute("UPDATE attachment_jobs SET state='ready',lease_token=NULL,lease_until=NULL,"
"last_error=NULL WHERE id=%s", (job['id'],))
"last_error=NULL,settled_at=clock_timestamp() WHERE id=%s", (job['id'],))
return True
@@ -131,11 +144,13 @@ def settle_error(conn, job, code, permanent=False):
if permanent or exhausted:
error = code if permanent else 'retry_exhausted'
conn.execute("UPDATE attachments SET state='failed',error_code=%s,updated_at=clock_timestamp() "
"WHERE id=%s", (error, job['attachment_id']))
"WHERE id=%s AND published_pipeline_version IS NULL", (error, job['attachment_id']))
conn.execute("UPDATE attachment_jobs SET state='failed',lease_token=NULL,lease_until=NULL,"
"last_error=%s,next_publish_at=NULL WHERE id=%s", (code, job['id']))
"last_error=%s,next_publish_at=NULL,settled_at=clock_timestamp() WHERE id=%s",
(code, job['id']))
return
conn.execute("UPDATE attachments SET state='queued',updated_at=clock_timestamp() WHERE id=%s",
conn.execute("UPDATE attachments SET state='queued',updated_at=clock_timestamp() "
"WHERE id=%s AND published_pipeline_version IS NULL",
(job['attachment_id'],))
conn.execute("""UPDATE attachment_jobs SET state='queued',lease_token=NULL,lease_until=NULL,
last_error=%s,attempt=attempt+1,published_at=NULL,
@@ -164,25 +179,60 @@ def recover_expired():
def replay(attachment_id):
with storage.pool.connection() as conn:
attachment = conn.execute('SELECT id FROM attachments WHERE id=%s FOR UPDATE',
attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',
(attachment_id,)).fetchone()
if not attachment:
return None
job = conn.execute('SELECT * FROM attachment_jobs WHERE attachment_id=%s FOR UPDATE',
(attachment_id,)).fetchone()
job = conn.execute('SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s FOR UPDATE',
(attachment_id, attachment['target_pipeline_version'])).fetchone()
if not job or job['state'] != 'failed':
return None
object_lock(conn, job['staging_key'])
client().head_object(Bucket=BUCKET, Key=job['staging_key'])
source = attachment['object_key'] if attachment['published_pipeline_version'] else job['staging_key']
object_lock(conn, source)
client().head_object(Bucket=BUCKET, Key=source)
new_attempt = job['attempt'] + 1
budget = int(setting('JOB_MAX_ATTEMPTS', '5'))
conn.execute(
"""UPDATE attachment_jobs SET state='queued', attempt=%s, attempt_limit=%s,
next_publish_at=clock_timestamp(), published_at=NULL, publish_failures=0
next_publish_at=clock_timestamp(), published_at=NULL, publish_failures=0, settled_at=NULL
WHERE id=%s""", (new_attempt, new_attempt + budget - 1, job['id']),
)
conn.execute("INSERT INTO attachment_replays(job_id,attempt,reason) VALUES (%s,%s,'operator')",
(job['id'], new_attempt))
conn.execute("UPDATE attachments SET state='queued',error_code=NULL,updated_at=clock_timestamp() "
"WHERE id=%s", (attachment_id,))
"WHERE id=%s AND published_pipeline_version IS NULL", (attachment_id,))
return job['id'], new_attempt
def upgrade(attachment_id, version):
with storage.pool.connection() as conn:
attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,)).fetchone()
if not attachment or version not in ENCODING or version <= attachment['target_pipeline_version']:
return None
previous = conn.execute(
'SELECT staging_key FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s',
(attachment_id, attachment['target_pipeline_version']),
).fetchone()
if not previous and not attachment['published_pipeline_version']:
return None
source = attachment['object_key'] if attachment['published_pipeline_version'] else previous['staging_key']
object_lock(conn, source)
client().head_object(Bucket=BUCKET, Key=source)
conn.execute(
"""UPDATE attachment_jobs SET state='superseded', lease_token=NULL, lease_until=NULL,
next_publish_at=NULL, settled_at=clock_timestamp()
WHERE attachment_id=%s AND state IN ('queued','processing')""", (attachment_id,),
)
conn.execute(
"""UPDATE attachments SET target_pipeline_version=%s,
state=CASE WHEN published_pipeline_version IS NULL THEN 'queued' ELSE 'ready' END,
error_code=NULL,updated_at=clock_timestamp() WHERE id=%s""",
(version, attachment_id),
)
job_id = uuid4()
conn.execute(
"""INSERT INTO attachment_jobs(id,attachment_id,staging_key,attempt_limit,next_publish_at,pipeline_version)
VALUES (%s,%s,%s,%s,clock_timestamp(),%s)""",
(job_id, attachment_id, f'staging/{attachment_id}', int(setting('JOB_MAX_ATTEMPTS', '5')), version),
)
return job_id
+14 -10
View File
@@ -12,27 +12,31 @@ from src.storage import storage, timestamp
def main():
parser = argparse.ArgumentParser()
parser.add_argument('action', choices=('failed', 'replay'))
parser.add_argument('action', choices=('failed', 'replay', 'upgrade'))
parser.add_argument('attachment_id', type=UUID, nargs='?')
parser.add_argument('--version', type=int, default=2)
args = parser.parse_args()
if args.action == 'replay' and args.attachment_id is None:
parser.error('replay requires attachment_id')
if args.action in ('replay', 'upgrade') and args.attachment_id is None:
parser.error('operation requires attachment_id')
storage.pool.open(wait=True)
try:
if args.action == 'replay':
if args.action in ('replay', 'upgrade'):
try:
result = jobs.replay(args.attachment_id)
if args.action == 'replay':
result = jobs.replay(args.attachment_id)
else:
job_id = jobs.upgrade(args.attachment_id, args.version)
result = (job_id, 1) if job_id else None
except (BotoCoreError, ClientError):
raise SystemExit('Источник для replay недоступен') from None
raise SystemExit('Источник обработки недоступен') from None
if result is None:
raise SystemExit('Нет подходящей failed-задачи')
raise SystemExit('Операция неприменима к текущему состоянию или версии')
print(json.dumps({'job_id': str(result[0]), 'attempt': result[1]}))
else:
with storage.pool.connection() as conn:
rows = conn.execute(
"""SELECT j.id, j.attachment_id, j.attempt, j.last_error, a.updated_at
FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id
WHERE j.state='failed' ORDER BY a.updated_at DESC LIMIT 100"""
"""SELECT id, attachment_id, pipeline_version, attempt, last_error, settled_at
FROM attachment_jobs WHERE state='failed' ORDER BY settled_at DESC NULLS LAST LIMIT 100"""
).fetchall()
print(json.dumps(rows, default=lambda value: str(value) if isinstance(value, UUID) else timestamp(value)))
finally:
+6
View File
@@ -0,0 +1,6 @@
"""Версии преобразований и операция перехода на следующий manifest."""
ENCODING = {
1: (85, 82),
2: (90, 88),
}
+6 -3
View File
@@ -128,7 +128,8 @@ def get_attachment(request:Request,attachment_id:uuid.UUID,actor:str=Depends(req
row = accessible(conn, request, attachment_id, actor)
variants = conn.execute(
'SELECT name,content_type,size_bytes,sha256,width,height FROM attachment_variants '
'WHERE attachment_id=%s ORDER BY name', (attachment_id,)
'WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',
(attachment_id, row['published_pipeline_version']),
).fetchall()
return metadata(row, variants)
@@ -141,8 +142,10 @@ def content(request:Request,attachment_id:uuid.UUID,variant:str='original',actor
if row['state'] != 'ready':
raise error(request,409,'conflict','attachment not ready')
if variant != 'original':
row = conn.execute('SELECT * FROM attachment_variants WHERE attachment_id=%s AND name=%s',
(attachment_id, variant)).fetchone()
row = conn.execute(
'SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s',
(attachment_id, row['published_pipeline_version'], variant),
).fetchone()
if not row:
raise error(request,404,'not_found','variant not found')
obj=client().get_object(Bucket=BUCKET,Key=row['object_key'])
+3 -3
View File
@@ -25,10 +25,10 @@ def cleanup(s3, retention):
if not object_lock(conn, key, wait=False):
continue
job = conn.execute(
'SELECT j.state,a.updated_at FROM attachment_jobs j JOIN attachments a '
'ON a.id=j.attachment_id WHERE j.staging_key=%s', (key,),
'SELECT state,settled_at FROM attachment_jobs WHERE staging_key=%s '
'ORDER BY pipeline_version DESC LIMIT 1', (key,),
).fetchone()
retention_start = job['updated_at'] if job else item['LastModified']
retention_start = (job['settled_at'] if job else None) or item['LastModified']
age = (now - retention_start).total_seconds()
if job and job['state'] in ('queued', 'processing'):
continue
+9 -4
View File
@@ -16,7 +16,7 @@ import psycopg
from botocore.exceptions import BotoCoreError, ClientError
from psycopg_pool import PoolTimeout
from src import broker, jobs
from src import broker, jobs, faults
from src.objects import BUCKET, ObjectConflict, client, put_immutable
from src.storage import storage
@@ -41,13 +41,15 @@ def fetch(s3, key, attachment, destination):
def process(job):
faults.hit('before_effect', job)
attachment = job['attachment']
s3 = client()
with tempfile.TemporaryDirectory() as temp:
directory = Path(temp)
original = directory / 'original'
fetch(s3, job['staging_key'], attachment, original)
result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', temp],
source = attachment['object_key'] if attachment['published_pipeline_version'] else job['staging_key']
fetch(s3, source, attachment, original)
result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', temp, str(job['pipeline_version'])],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
timeout=jobs.JOB_TIMEOUT, check=False)
if result.returncode == 2:
@@ -57,8 +59,9 @@ def process(job):
put_immutable(s3, attachment['object_key'], stream, attachment['size_bytes'],
attachment['content_type'], attachment['sha256'])
variants = []
faults.hit('after_original', job)
for name, mime, width, height in json.loads((directory / 'manifest.json').read_text()):
key = f"variants/{attachment['id']}/v1/{name}"
key = f"variants/{attachment['id']}/v{job['pipeline_version']}/{name}"
output = directory / name
size = output.stat().st_size
with output.open('rb') as stream:
@@ -66,7 +69,9 @@ def process(job):
stream.seek(0)
put_immutable(s3, key, stream, size, mime, digest)
variants.append((name, key, mime, size, digest, width, height))
faults.hit('after_variants', job)
jobs.complete(job, variants)
faults.hit('after_commit', job)
def handle(channel, delivery, body):
+94
View File
@@ -0,0 +1,94 @@
"""Closed-loop: полный путь upload -> ready; ограничение генератора явно в отчёте."""
import argparse
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path
import random
import struct
import subprocess
import sys
import threading
import time
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def png(width, height):
def chunk(name, data):
return struct.pack('>I', len(data)) + name + data + struct.pack('>I', zlib.crc32(name + data))
rng = random.Random(2301)
data = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height))
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0))
+ chunk(b'IDAT', zlib.compress(data)) + chunk(b'IEND', b''))
def percentile(values, q):
return round(sorted(values)[min(len(values)-1, int((len(values)-1)*q))], 4) if values else None
def main(args):
client = Client(args.base_url)
alice, _, _, chat = client.bootstrap()
profiles = [('320x200', (ROOT / 'tests/fixtures/sample.png').read_bytes()), ('1024x768', png(1024, 768))]
rows = []
for profile, raw in profiles:
for repeat in range(args.repeats):
samples = []
failures = []
stats = []
stop = threading.Event()
def sample():
while not stop.is_set():
ids = subprocess.check_output(['docker', 'compose', '-p', args.project, 'ps', '-q'], text=True).split()
result = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids], text=True)
stats.append(dict(at_seconds=round(time.monotonic()-start, 3), containers=[json.loads(line) for line in result.splitlines()]))
stop.wait(2)
def one():
c = Client(args.base_url)
begin = time.monotonic()
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png', 202,
raw=raw, actor=alice, headers={'Content-Type': 'image/png'})['json']
accepted = time.monotonic()
ready = c.wait_ready(alice, att)
check(len(ready['variants']) == 3 and ready['sha256'] == hashlib.sha256(raw).hexdigest(), 'bad load result')
return dict(upload_seconds=accepted-begin, ready_seconds=time.monotonic()-begin)
# Один завершённый прогрев на профиль/повтор, не входит в измерение.
one()
start = time.monotonic()
sampler = threading.Thread(target=sample)
sampler.start()
def loop(_):
while time.monotonic()-start < args.duration:
try:
samples.append(one())
except Exception as exc:
failures.append(type(exc).__name__ + ': ' + str(exc))
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
list(pool.map(loop, range(args.concurrency)))
elapsed = time.monotonic()-start
stop.set()
sampler.join()
times = [x['ready_seconds'] for x in samples]
row = dict(profile=profile, bytes_per_image=len(raw), repeat=repeat+1, concurrency=args.concurrency,
requested_duration=args.duration, elapsed_seconds=round(elapsed, 3), warmup_completed=1,
completed=len(samples), successful_rps=round(len(samples)/elapsed, 3), failures=failures,
error_fraction=len(failures)/max(1,len(samples)+len(failures)),
p50=percentile(times,.5), p95=percentile(times,.95), p99=percentile(times,.99),
upload_p95=percentile([x['upload_seconds'] for x in samples],.95), stats=stats)
rows.append(row)
(ROOT / 'evidence/load-pipeline.json').write_text(json.dumps(rows, indent=2), encoding='utf-8')
print(json.dumps({k:v for k,v in row.items() if k != 'stats'}), flush=True)
if __name__ == '__main__':
p = argparse.ArgumentParser()
p.add_argument('--base-url', default='http://127.0.0.1:8085')
p.add_argument('--project', default='lyakhonov-lab5')
p.add_argument('--duration', type=int, default=20)
p.add_argument('--concurrency', type=int, default=2)
p.add_argument('--repeats', type=int, default=3)
main(p.parse_args())
+242
View File
@@ -0,0 +1,242 @@
"""Реальные fault-injection и инварианты. Запускать только на своём lab5 стенде."""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
import os
from pathlib import Path
import struct
import subprocess
import sys
import time
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
class Experiment:
def __init__(self, args):
self.args = args
self.compose = ['docker', 'compose', '-p', args.project]
self.c = Client(args.base_url)
self.alice, self.bob, self.eve, self.chat = self.c.bootstrap()
path = ROOT / 'evidence/pipeline.json'
self.rows = json.loads(path.read_text(encoding='utf-8')) if args.resume and path.exists() else []
def command(self, *args):
return subprocess.check_output([*self.compose, *args], cwd=ROOT, text=True, encoding='utf-8', stderr=subprocess.PIPE)
def probe(self, action, **kwargs):
script = 'ARGS=' + repr(dict(action=action, **kwargs)) + '\n' + (ROOT / 'student-tests/probe.py').read_text(encoding='utf-8')
invocation = ([*self.compose, 'run', '--rm', '--no-deps', '-T', 'inspector', 'python', '-']
if action == 'objects' else [*self.compose, 'exec', '-T', '--index', '1', 'api', 'python', '-'])
result = subprocess.run(invocation,
input=script, text=True, encoding='utf-8', capture_output=True, cwd=ROOT)
check(result.returncode == 0, 'probe failed: ' + result.stderr)
return json.loads(result.stdout)
def sql(self, query, *params):
return self.probe('sql', query=query, params=list(params))
def job(self, att, version=1):
return self.sql('SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s', att['id'], version)[0]
@staticmethod
def envelope(job):
from datetime import datetime, timezone
created = datetime.fromisoformat(job['created_at']).astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
return dict(schema_version=1, type='attachment.process.v1', job_id=job['id'], attachment_id=job['attachment_id'],
pipeline_version=job['pipeline_version'], attempt=job['attempt'], created_at=created)
def upload(self):
return self.c.attachment(self.alice, self.chat)[0]
def arm(self, job, point, action='crash'):
self.sql('INSERT INTO experiment_faults(job_id,point,action) VALUES (%s,%s,%s)', job['id'], point, action)
def hit(self, job, point):
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
row = self.sql('SELECT * FROM experiment_faults WHERE job_id=%s AND point=%s', job['id'], point)[0]
if row['hit_at']:
return row
time.sleep(.3)
raise AssertionError('fault not hit: ' + point)
def snapshot(self):
ids = self.command('ps', '-a', '-q').split()
raw = subprocess.check_output(['docker', 'inspect', *ids], text=True, encoding='utf-8')
return [dict(id=x['Id'], name=x['Name'], started_at=x['State']['StartedAt'],
running=x['State']['Running'], restart_count=x['RestartCount'], exit_code=x['State']['ExitCode']) for x in json.loads(raw)]
def invariant(self, att, version=1):
ready = self.c.wait_ready(self.alice, att, wait_seconds=120)
check(len(ready['variants']) == 3, 'three variants required')
row = self.sql('SELECT published_pipeline_version,target_pipeline_version FROM attachments WHERE id=%s', att['id'])[0]
check(row['published_pipeline_version'] == version, 'wrong published version')
variants = self.sql('SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s', att['id'], version)
check(len(variants) == 3, 'duplicate/missing manifests')
digest = {}
for name in ('original', 'thumbnail', 'crop', 'optimized'):
response = self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + name, actor=self.alice)
sha = hashlib.sha256(response['bytes']).hexdigest()
expected = att['sha256'] if name == 'original' else next(v['sha256'] for v in variants if v['name'] == name)
check(sha == expected, 'download checksum mismatch')
digest[name] = sha
return digest
def record(self, name, started, **details):
self.rows.append(dict(scenario=name, seconds=round(time.monotonic() - started, 3), **details))
(ROOT / 'evidence/pipeline.json').write_text(json.dumps(self.rows, ensure_ascii=False, indent=2), encoding='utf-8')
print('PASS:', name, flush=True)
def crashes(self):
for repeat in range(self.args.repeats):
for point in ('before_publish', 'after_publish', 'before_effect', 'after_original', 'after_variants', 'after_commit'):
start = time.monotonic()
service = 'dispatcher' if 'publish' in point else 'worker'
self.command('stop', '-t', '2', service)
att = self.upload()
job = self.job(att)
self.arm(job, point)
before = self.snapshot()
self.command('start', service)
hit = self.hit(job, point)
self.invariant(att)
self.record(point, start, repeat=repeat + 1, attachment_id=att['id'], fault=hit,
job_after=self.job(att), before=before, after=self.snapshot())
def durable(self):
start = time.monotonic()
self.command('stop', '-t', '2', 'worker')
att = self.upload()
self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=self.alice)
self.probe('cleanup')
check(any(obj['key'].startswith('staging/') for obj in self.probe('objects', id=att['id'])), 'active staging deleted')
self.c.message(self.alice, self.chat, '', attachments=[att['id']])
time.sleep(2)
before = self.snapshot()
self.command('restart', '-t', '2', 'api', 'broker')
self.command('start', 'worker')
self.invariant(att)
self.record('accepted_survives_api_broker_restart', start, attachment_id=att['id'], before=before, after=self.snapshot())
def versions(self):
for repeat in range(self.args.repeats):
start = time.monotonic()
self.command('stop', '-t', '2', 'worker')
att = self.upload()
old = self.job(att)
self.arm(old, 'after_variants', 'pause')
self.command('start', 'worker')
self.hit(old, 'after_variants')
self.probe('upgrade', id=att['id'])
new = self.job(att, 2)
# Новая работа, затем старая, затем 24 чередующихся дубля.
self.probe('publish', messages=[self.envelope(new), self.envelope(old)] * 13)
first = self.invariant(att, 2)
objects = self.probe('objects', id=att['id'])
self.sql('UPDATE experiment_faults SET released=true WHERE job_id=%s', old['id'])
time.sleep(2)
check(self.invariant(att, 2) == first, 'stale worker changed new result')
self.probe('publish', messages=[self.envelope(old), self.envelope(new)] * 13)
time.sleep(2)
check(self.invariant(att, 2) == first, 'reorder changed new result')
current = self.probe('objects', id=att['id'])
check([x for x in current if not x['key'].startswith('staging/')] ==
[x for x in objects if not x['key'].startswith('staging/')], 'duplicate wrote more objects')
check(self.probe('replay', id=att['id']) is None, 'ready replay must no-op')
self.record('stale_running_worker_and_reordered_duplicates', start, repeat=repeat + 1,
attachment_id=att['id'], before_digest=first, after_digest=self.invariant(att, 2),
versions=self.sql('SELECT pipeline_version,state,attempt FROM attachment_jobs WHERE attachment_id=%s ORDER BY pipeline_version', att['id']), objects=current)
start = time.monotonic()
att = self.upload()
old_digest = self.invariant(att)
self.probe('cleanup')
self.probe('upgrade', id=att['id'])
deadline = time.monotonic() + 60
while self.job(att, 2)['state'] != 'ready':
check(time.monotonic() < deadline, 'ready upgrade timeout')
time.sleep(.3)
new_digest = self.invariant(att, 2)
check(new_digest['original'] == old_digest['original'], 'upgrade changed original')
check(new_digest['thumbnail'] != old_digest['thumbnail'], 'versions must have a real different effect')
self.record('upgrade_ready_after_staging_cleanup', start, repeat=repeat + 1, attachment_id=att['id'], before_digest=old_digest, after_digest=new_digest)
def errors(self):
for kind in ('permanent', 'transient'):
start = time.monotonic()
self.command('stop', '-t', '2', 'worker')
att = self.upload()
if kind == 'permanent':
self.probe('corrupt_staging', id=att['id'])
else:
self.probe('remove_staging', id=att['id'])
self.command('start', 'worker')
observed = []
deadline = time.monotonic() + 120
while True:
job = self.job(att)
observed.append(dict(attempt=job['attempt'], state=job['state'], next_publish_at=job['next_publish_at'], last_error=job['last_error']))
if job['state'] == 'failed':
break
check(time.monotonic() < deadline, 'failed registry timeout')
time.sleep(.3)
check(job['attempt'] == (1 if kind == 'permanent' else job['attempt_limit']), 'wrong retry budget')
self.c.call('POST', '/api/v1/chats/' + self.chat['id'] + '/messages', 409, actor=self.alice,
headers={'Idempotency-Key': os.urandom(12).hex()}, body={'text': '', 'attachment_ids': [att['id']]})
self.probe('restore_staging', id=att['id'], data=base64.b64encode((ROOT / 'tests/fixtures/sample.png').read_bytes()).decode())
check(self.probe('replay', id=att['id']) is not None, 'failed replay missing')
self.invariant(att)
self.record(kind + '_error_and_explicit_replay', start, attachment_id=att['id'], attempts=observed,
audit=self.sql('SELECT attempt,reason FROM attachment_replays WHERE job_id=%s', job['id']))
def negatives(self):
start = time.monotonic()
path = '/api/v1/chats/' + self.chat['id'] + '/attachments?filename=x.png'
def upload(raw, expected, path=path, actor=None, mime='image/png'):
self.c.call('POST', path, expected, actor=actor or self.alice, raw=raw, headers={'Content-Type': mime})
upload(b'bad', 415)
upload(b'x' * (10 * 1024 * 1024 + 1), 413)
raw = (ROOT / 'tests/fixtures/sample.png').read_bytes()
upload(raw, 415, mime='image/jpeg')
upload(raw, 400, path=path.replace('x.png', '..%2Fx.png'))
upload(raw, 404, actor=self.eve)
# Валидный PNG header >40 MP: отклонение до декодирования большого bitmap.
def chunk(name, data):
return struct.pack('>I', len(data)) + name + data + struct.pack('>I', zlib.crc32(name + data))
huge = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 10000, 10000, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(b'')) + chunk(b'IEND', b'')
upload(huge, 413)
att = self.upload()
self.invariant(att)
self.c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=self.eve)
self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=self.eve)
key = os.urandom(16).hex()
with ThreadPoolExecutor(max_workers=12) as pool:
responses = list(pool.map(lambda _: self.c.message(self.alice, self.chat, 'concurrent', key=key)['json']['id'], range(24)))
check(len(set(responses)) == 1, 'message idempotency violated')
self.record('negative_upload_acl_and_24_concurrent_messages', start, checks=9, message_ids=len(set(responses)))
def main(self):
try:
if not self.args.resume:
self.negatives()
self.durable()
self.crashes()
self.versions()
self.errors()
finally:
self.sql('UPDATE experiment_faults SET released=true WHERE NOT released')
self.command('start', 'api', 'broker', 'dispatcher', 'worker')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--project', default='lyakhonov-lab5')
parser.add_argument('--base-url', default='http://127.0.0.1:8085')
parser.add_argument('--repeats', type=int, default=3)
parser.add_argument('--resume', action='store_true', help='Сохранить уже выполненные crash-результаты; выполнить version/error suite')
Experiment(parser.parse_args()).main()
+55
View File
@@ -0,0 +1,55 @@
"""Исполняется внутри API через stdin; наружу возвращает только безопасные данные."""
import json
import sys
from uuid import UUID
from src import broker, jobs
from src.objects import BUCKET, client
from src.storage import storage
def run(args):
action = args['action']
if action == 'sql':
with storage.pool.connection() as conn:
cursor = conn.execute(args['query'], args.get('params', []))
return cursor.fetchall() if cursor.description else []
if action == 'publish':
with broker.channel() as (_, channel):
channel.confirm_delivery()
for message in args['messages']:
broker.publish(channel, message)
return True
if action == 'upgrade':
return jobs.upgrade(UUID(args['id']), args.get('version', 2))
if action == 'replay':
return jobs.replay(UUID(args['id']))
if action == 'cleanup':
from src.staging import cleanup
return cleanup(client(), 60)
if action == 'objects':
s3 = client()
result = []
for prefix in (f"originals/{args['id']}", f"variants/{args['id']}/", f"staging/{args['id']}"):
for page in s3.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=prefix):
result.extend({'key': row['Key'], 'etag': row['ETag'], 'size': row['Size']}
for row in page.get('Contents', []))
return result
if action == 'remove_staging':
client().delete_object(Bucket=BUCKET, Key=f"staging/{args['id']}")
return True
if action == 'corrupt_staging':
client().put_object(Bucket=BUCKET, Key=f"staging/{args['id']}", Body=b'broken')
return True
if action == 'restore_staging':
import base64
client().put_object(Bucket=BUCKET, Key=f"staging/{args['id']}", Body=base64.b64decode(args['data']))
return True
raise ValueError('unknown probe action')
storage.pool.open(wait=True)
try:
print(json.dumps(run(ARGS), default=str))
finally:
storage.pool.close()
+82
View File
@@ -0,0 +1,82 @@
"""Публичные resilience и отдельный принудительный отказ каждой API-реплики."""
import json
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
COMPOSE = ['docker', 'compose', '-p', 'lyakhonov-lab5']
def docker(*args):
return subprocess.check_output(['docker', *args], cwd=ROOT, text=True, encoding='utf-8')
def snapshot():
ids = docker('compose', '-p', 'lyakhonov-lab5', 'ps', '-a', '-q').split()
raw = json.loads(docker('inspect', *ids))
return [dict(id=x['Id'], name=x['Name'], started_at=x['State']['StartedAt'], running=x['State']['Running']) for x in raw]
def public(mode):
before = snapshot()
script = ROOT / 'scripts' / (('restart' if mode == 'persistence' else 'stop-one') + ('.cmd' if sys.platform == 'win32' else '.sh'))
result = subprocess.run([sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', mode,
'--base-url', 'http://127.0.0.1:8085', '--action-script', str(script)],
cwd=ROOT, text=True, encoding='utf-8', capture_output=True)
after = snapshot()
(ROOT / f'evidence/{mode}.json').write_text(json.dumps(dict(command=result.args, exit_code=result.returncode,
output=result.stdout + result.stderr, before=before, after=after), ensure_ascii=False, indent=2), encoding='utf-8')
print(result.stdout, flush=True)
check(result.returncode == 0, result.stderr)
subprocess.check_call([*COMPOSE, 'start', 'api'], cwd=ROOT)
def targeted():
c = Client('http://127.0.0.1:8085', timeout=2)
alice, _, _, chat = c.bootstrap()
ids = docker('compose', '-p', 'lyakhonov-lab5', 'ps', '-q', 'api').split()
evidence = []
for index, target in enumerate(ids):
before = snapshot()
responses, errors = [], []
first = None
begin = time.monotonic()
subprocess.check_call(['docker', 'stop', '-t', '2', target], stdout=subprocess.DEVNULL)
try:
while time.monotonic() - begin < 15:
key = f'explicit-{index}-{len(responses)}'
try:
response = c.message(alice, chat, 'replica failure', key=key)
first = first or time.monotonic()-begin
responses.append(response['json'])
except Failure as exc:
errors.append(str(exc))
time.sleep(.05)
check(first is not None and first <= 10, 'slow API failover')
page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice)['json']
messages = list(page['items'])
while page['next_cursor']:
from urllib.parse import quote
page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe=''), actor=alice)['json']
messages.extend(page['items'])
found = {m['id'] for m in messages}
check(all(m['id'] in found for m in responses), 'acknowledged message lost')
check(len(found) == len(messages), 'duplicate messages')
evidence.append(dict(target=target, before=before, after=snapshot(), successes=len(responses),
errors=errors, recovery_seconds=round(first, 3)))
finally:
subprocess.check_call(['docker', 'start', target], stdout=subprocess.DEVNULL)
time.sleep(3)
(ROOT / 'evidence/failover-each.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2), encoding='utf-8')
print('PASS: both real API containers independently stopped and restored')
if __name__ == '__main__':
public('persistence')
public('failover')
targeted()