Реализовать версии pipeline, fencing и автоматические сценарии отказов
This commit is contained in:
+1
-1
@@ -1,4 +1,4 @@
|
||||
# Reproducible fallback: official MinIO images are unavailable from registries.
|
||||
# Build pinned upstream releases and verify archive checksums.
|
||||
FROM golang:1.24.6-bookworm@sha256:ab1d1823abb55a9504d2e3e003b75b36dbeb1cbcc4c92593d85a84ee46becc6c AS build
|
||||
WORKDIR /build
|
||||
ADD --checksum=sha256:8819e3e7817e46b7b3798f8f200ead208562e571563c2e040352378031abe9f2 https://github.com/minio/minio/archive/07c3a429bfed433e49018cb0f78a52145d4bedeb.tar.gz /tmp/minio.tar.gz
|
||||
|
||||
@@ -11,6 +11,8 @@ x-app: &app
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
|
||||
LAB5_FAULT_INJECTION: ${LAB5_FAULT_INJECTION:-0}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
@@ -26,6 +28,19 @@ x-dependencies: &dependencies
|
||||
condition: service_completed_successfully
|
||||
|
||||
services:
|
||||
inspector:
|
||||
<<: *app
|
||||
profiles: [tools]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_ACCESS_KEY_FILE: /run/secrets/s3_root_user
|
||||
S3_SECRET_KEY_FILE: /run/secrets/s3_root_password
|
||||
secrets: [database_url, s3_root_user, s3_root_password]
|
||||
healthcheck:
|
||||
disable: true
|
||||
cpus: 0.25
|
||||
mem_limit: 256m
|
||||
db:
|
||||
image: postgres:16.6-alpine
|
||||
environment:
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN target_pipeline_version integer NOT NULL DEFAULT 1 CHECK (target_pipeline_version >= 1),
|
||||
ADD COLUMN published_pipeline_version integer CHECK (published_pipeline_version >= 1),
|
||||
ADD CHECK (published_pipeline_version <= target_pipeline_version);
|
||||
UPDATE attachments SET published_pipeline_version = 1 WHERE state = 'ready';
|
||||
|
||||
ALTER TABLE attachment_jobs
|
||||
DROP CONSTRAINT attachment_jobs_attachment_id_key,
|
||||
DROP CONSTRAINT attachment_jobs_staging_key_key,
|
||||
DROP CONSTRAINT attachment_jobs_state_check,
|
||||
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1),
|
||||
ADD COLUMN settled_at timestamptz,
|
||||
ADD CHECK (state IN ('queued', 'processing', 'ready', 'failed', 'superseded')),
|
||||
ADD UNIQUE (attachment_id, pipeline_version);
|
||||
CREATE INDEX attachment_jobs_staging ON attachment_jobs(staging_key);
|
||||
|
||||
ALTER TABLE attachment_variants ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1;
|
||||
ALTER TABLE attachment_variants DROP CONSTRAINT attachment_variants_pkey;
|
||||
ALTER TABLE attachment_variants ADD PRIMARY KEY (attachment_id, pipeline_version, name);
|
||||
@@ -0,0 +1,9 @@
|
||||
CREATE TABLE experiment_faults (
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(id),
|
||||
point text NOT NULL,
|
||||
action text NOT NULL CHECK (action IN ('crash', 'pause')),
|
||||
hit_at timestamptz,
|
||||
process_id integer,
|
||||
released boolean NOT NULL DEFAULT false,
|
||||
PRIMARY KEY (job_id, point)
|
||||
);
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Аудит обязательных материалов, локальных секретов и предыдущей лабораторной."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PREVIOUS = ROOT.parent / 'vysoko-nagr 4'
|
||||
ENV = dict(os.environ, GIT_OPTIONAL_LOCKS='0')
|
||||
|
||||
|
||||
def git(root, *args):
|
||||
return subprocess.check_output(['git', '-C', str(root), *args], env=ENV).decode('utf-8').strip()
|
||||
|
||||
|
||||
def previous():
|
||||
paths = git(PREVIOUS, 'ls-files', '-z').split('\0')
|
||||
files = {p: hashlib.sha256((PREVIOUS / p).read_bytes()).hexdigest() for p in paths if p}
|
||||
return dict(head=git(PREVIOUS, 'rev-parse', 'HEAD'), branch=git(PREVIOUS, 'branch', '--show-current'),
|
||||
status=git(PREVIOUS, 'status', '--porcelain'), files=files)
|
||||
|
||||
|
||||
def main():
|
||||
import sys
|
||||
if sys.argv[1:] == ['baseline']:
|
||||
(ROOT / 'evidence/lab4-baseline.json').write_text(json.dumps(previous(), ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print('Сохранён read-only fingerprint лабораторной 4')
|
||||
return
|
||||
baseline = json.loads((ROOT / 'evidence/lab4-baseline.json').read_text(encoding='utf-8'))
|
||||
unchanged = previous() == baseline
|
||||
assert unchanged, 'Lab4 differs from baseline'
|
||||
protected = ['README.md', 'COURSE.md', 'CONTRIBUTING.md', 'MAINTAINERS.md', 'contracts', 'tests',
|
||||
'examples', 'compose.example.yaml', '.env.example', '.github']
|
||||
assert not git(ROOT, 'diff', '276dbda', '--', *protected), 'Protected lab5 material changed'
|
||||
paths = git(ROOT, 'ls-files', '-z').split('\0')
|
||||
secret_values = [p.read_bytes().strip() for p in (ROOT / 'secrets').iterdir() if p.is_file()]
|
||||
for path in paths:
|
||||
if not path:
|
||||
continue
|
||||
assert not path.startswith('secrets/') and path != '.env', 'Tracked private state'
|
||||
content = (ROOT / path).read_bytes()
|
||||
assert not any(value and value in content for value in secret_values), 'Local secret leaked in ' + path
|
||||
assert b'-----BEGIN PRIVATE KEY-----' not in content, 'Private key found'
|
||||
authors = git(ROOT, 'log', '276dbda..HEAD', '--format=%h|%an|%ae|%cn|%ce').splitlines()
|
||||
for row in authors:
|
||||
assert row.split('|')[1:] == ['Sergey Lyakhonov', 'lyserg119@gmail.com'] * 2, 'Wrong author/committer'
|
||||
report = dict(lab4_unchanged=unchanged, protected_lab5_unchanged=True, tracked_local_secrets=False,
|
||||
scanned_files=len(paths), authors=authors, head=git(ROOT, 'rev-parse', 'HEAD'))
|
||||
(ROOT / 'evidence/audit.json').write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Безопасная фиксация версий, лимитов и опубликованных портов."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def output(*args):
|
||||
return subprocess.check_output(args, cwd=ROOT, text=True, encoding='utf-8').strip()
|
||||
|
||||
|
||||
ids = output('docker', 'compose', '-p', 'lyakhonov-lab5', 'ps', '-q').split()
|
||||
containers = json.loads(output('docker', 'inspect', *ids))
|
||||
result = dict(commit=output('git', 'rev-parse', 'HEAD'), python=platform.python_version(),
|
||||
docker=json.loads(output('docker', 'version', '--format', '{{json .}}')),
|
||||
compose=output('docker', 'compose', 'version'),
|
||||
engine=json.loads(output('docker', 'info', '--format', '{"memory_bytes":{{.MemTotal}},"cpus":{{.NCPU}}}')),
|
||||
services=[])
|
||||
for row in containers:
|
||||
result['services'].append(dict(service=row['Config']['Labels']['com.docker.compose.service'], id=row['Id'],
|
||||
image=row['Config']['Image'], image_id=row['Image'], started_at=row['State']['StartedAt'],
|
||||
memory_limit=row['HostConfig']['Memory'], nanocpus=row['HostConfig']['NanoCpus'],
|
||||
ports=row['HostConfig']['PortBindings'],
|
||||
settings=[x for x in row['Config']['Env'] if x.startswith(('JOB_', 'WORKER_', 'LAB5_FAULT_', 'STAGING_RETENTION_'))]))
|
||||
(ROOT / 'evidence/environment.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print('Saved versions, IDs, resource limits, ports and non-secret experiment settings')
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Статические графики только из сохранённых фактических агрегатов."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
data = json.loads((ROOT / 'evidence/pipeline.json').read_text(encoding='utf-8'))
|
||||
points = ('before_publish', 'after_publish', 'before_effect', 'after_original', 'after_variants', 'after_commit')
|
||||
fig, ax = plt.subplots(figsize=(10, 4.5), layout='constrained')
|
||||
for index, point in enumerate(points):
|
||||
values = [row['seconds'] for row in data if row['scenario'] == point]
|
||||
ax.scatter([index] * len(values), values, s=45)
|
||||
ax.set_xticks(range(len(points)), points, rotation=15)
|
||||
ax.set_ylabel('Full scenario duration, seconds')
|
||||
ax.set_title('Actual crash scenarios: stop, upload, restart, verified downloads')
|
||||
ax.grid(axis='y', alpha=.25)
|
||||
fig.savefig(ROOT / 'evidence/faults.png', dpi=160)
|
||||
plt.close(fig)
|
||||
data = json.loads((ROOT / 'evidence/load-pipeline.json').read_text(encoding='utf-8'))
|
||||
fig, axes = plt.subplots(1, 2, figsize=(10, 4), layout='constrained')
|
||||
for index, profile in enumerate(sorted({row['profile'] for row in data})):
|
||||
rows = [row for row in data if row['profile'] == profile]
|
||||
for axis, metric in zip(axes, ('successful_rps', 'p95')):
|
||||
axis.scatter([index] * len(rows), [row[metric] for row in rows], label=profile)
|
||||
for axis, label in zip(axes, ('Completed images / second', 'Upload to ready p95, seconds')):
|
||||
axis.set_xticks(range(len({row['profile'] for row in data})), sorted({row['profile'] for row in data}))
|
||||
axis.set_ylabel(label)
|
||||
axis.grid(axis='y', alpha=.25)
|
||||
fig.suptitle('Closed-loop pipeline: 2 clients, 20 seconds, 3 repeats')
|
||||
fig.savefig(ROOT / 'evidence/load.png', dpi=160)
|
||||
plt.close(fig)
|
||||
print('Saved evidence/faults.png and evidence/load.png')
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Чистый Git-экспорт, новые секреты и отдельный временный Compose-проект."""
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import uuid
|
||||
import zipfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
name = 'lyakhonov-lab5-repro-' + uuid.uuid4().hex[:8]
|
||||
directory = Path(tempfile.mkdtemp(prefix=name + '-'))
|
||||
archive = subprocess.check_output(['git', 'archive', '--format=zip', 'HEAD'], cwd=ROOT)
|
||||
with zipfile.ZipFile(io.BytesIO(archive)) as contents:
|
||||
contents.extractall(directory)
|
||||
with socket.socket() as listener:
|
||||
listener.bind(('127.0.0.1', 0))
|
||||
port = listener.getsockname()[1]
|
||||
env = dict(os.environ, LAB5_PORT=str(port), LAB5_FAULT_INJECTION='0', JOB_TIMEOUT_SECONDS='60', JOB_LEASE_SECONDS='90')
|
||||
services = ('api', 'migrate', 'worker', 'dispatcher', 'staging-cleaner')
|
||||
images = [name + ':app', name + ':s3']
|
||||
override = {'services': {service: {'image': images[0]} for service in services}}
|
||||
override['services'].update({service: {'image': images[1]} for service in ('s3', 'bucket-init')})
|
||||
(directory / 'repro-override.json').write_text(json.dumps(override), encoding='utf-8')
|
||||
compose = ['docker', 'compose', '-p', name, '-f', 'compose.yaml', '-f', 'repro-override.json']
|
||||
events = []
|
||||
def run(args):
|
||||
result = subprocess.run(args, cwd=directory, env=env, text=True, encoding='utf-8', capture_output=True)
|
||||
# Только безопасные последние строки: ни inspect .Config.Env, ни secret files.
|
||||
events.append(dict(command=args, exit_code=result.returncode, output=(result.stdout + result.stderr)[-5000:]))
|
||||
if result.returncode:
|
||||
raise RuntimeError('reproduction failed: ' + ' '.join(args))
|
||||
try:
|
||||
run([sys.executable, '-X', 'utf8', 'scripts/bootstrap.py'])
|
||||
run([sys.executable, '-X', 'utf8', 'scripts/check_contract.py'])
|
||||
run(['docker', 'build', '-t', images[0], '.'])
|
||||
run(['docker', 'build', '-t', images[1], '-f', 'Dockerfile.s3', '.'])
|
||||
run([*compose, 'up', '-d', '--wait'])
|
||||
run([sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', f'http://127.0.0.1:{port}'])
|
||||
run([*compose, 'ps'])
|
||||
finally:
|
||||
run([*compose, 'down', '--volumes', '--remove-orphans'])
|
||||
for tag in images:
|
||||
run(['docker', 'image', 'rm', tag])
|
||||
result = dict(project=name, directory=str(directory), port=port,
|
||||
commit=subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), events=events)
|
||||
(ROOT / 'evidence/reproduction.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print('PASS: clean export, fresh secrets, isolated project; temporary Docker resources removed')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+3
-1
@@ -9,7 +9,7 @@ import pika
|
||||
import psycopg
|
||||
from psycopg_pool import PoolTimeout
|
||||
|
||||
from src import broker, jobs
|
||||
from src import broker, jobs, faults
|
||||
from src.storage import storage
|
||||
|
||||
|
||||
@@ -32,7 +32,9 @@ def main():
|
||||
next_recovery = time.monotonic() + 1
|
||||
reserved = jobs.reserve_publish()
|
||||
if reserved:
|
||||
faults.hit('before_publish', reserved)
|
||||
broker.publish(channel, jobs.envelope(reserved))
|
||||
faults.hit('after_publish', reserved)
|
||||
jobs.confirm_publish(reserved)
|
||||
reserved = None
|
||||
else:
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Одноразовые точки отказа. Выключены без LAB5_FAULT_INJECTION=1."""
|
||||
import os
|
||||
import time
|
||||
|
||||
from src.storage import storage
|
||||
|
||||
|
||||
def hit(point, job):
|
||||
if os.getenv('LAB5_FAULT_INJECTION') != '1':
|
||||
return
|
||||
with storage.pool.connection() as conn:
|
||||
fault = conn.execute(
|
||||
"UPDATE experiment_faults SET hit_at=clock_timestamp(), process_id=%s "
|
||||
"WHERE job_id=%s AND point=%s AND hit_at IS NULL RETURNING action",
|
||||
(os.getpid(), job['id'], point),
|
||||
).fetchone()
|
||||
if not fault:
|
||||
return
|
||||
if fault['action'] == 'crash':
|
||||
os._exit(91) # Реальная смерть процесса без finally/ack.
|
||||
while True:
|
||||
with storage.pool.connection() as conn:
|
||||
released = conn.execute(
|
||||
'SELECT released FROM experiment_faults WHERE job_id=%s AND point=%s',
|
||||
(job['id'], point),
|
||||
).fetchone()['released']
|
||||
if released:
|
||||
return
|
||||
time.sleep(0.1)
|
||||
@@ -5,6 +5,7 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
from src.pipeline_versions import ENCODING
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
|
||||
@@ -18,7 +19,8 @@ def fit(image, limit):
|
||||
return image.resize(size, Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def render(source, target):
|
||||
def render(source, target, version=1):
|
||||
jpeg_quality, webp_quality = ENCODING[version]
|
||||
with Image.open(source) as opened:
|
||||
if opened.width * opened.height > Image.MAX_IMAGE_PIXELS:
|
||||
raise ValueError('pixel limit')
|
||||
@@ -32,11 +34,11 @@ def render(source, target):
|
||||
rgb.info.clear()
|
||||
del image
|
||||
thumbnail = fit(rgb, 256)
|
||||
thumbnail.save(target / 'thumbnail', 'JPEG', quality=85)
|
||||
thumbnail.save(target / 'thumbnail', 'JPEG', quality=jpeg_quality)
|
||||
crop = ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS)
|
||||
crop.save(target / 'crop', 'JPEG', quality=85)
|
||||
crop.save(target / 'crop', 'JPEG', quality=jpeg_quality)
|
||||
optimized = fit(rgb, 1280)
|
||||
optimized.save(target / 'optimized', 'WEBP', quality=82, method=4)
|
||||
optimized.save(target / 'optimized', 'WEBP', quality=webp_quality, method=4)
|
||||
return [
|
||||
['thumbnail', 'image/jpeg', *thumbnail.size],
|
||||
['crop', 'image/jpeg', *crop.size],
|
||||
@@ -47,7 +49,7 @@ def render(source, target):
|
||||
def main():
|
||||
directory = Path(sys.argv[1])
|
||||
try:
|
||||
result = render(directory / 'original', directory)
|
||||
result = render(directory / 'original', directory, int(sys.argv[2]))
|
||||
except (OSError, ValueError, SyntaxError, Image.DecompressionBombError):
|
||||
return 2
|
||||
(directory / 'manifest.json').write_text(json.dumps(result), encoding='utf-8')
|
||||
|
||||
+76
-26
@@ -1,14 +1,17 @@
|
||||
"""Устойчивое намерение обработки, lease и атомарная публикация manifest."""
|
||||
|
||||
import random
|
||||
import re
|
||||
from datetime import datetime
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from src.config import setting
|
||||
from src.objects import BUCKET, client, object_lock
|
||||
from src.pipeline_versions import ENCODING
|
||||
from src.storage import storage, timestamp
|
||||
|
||||
JOB_TIMEOUT = int(setting('JOB_TIMEOUT_SECONDS', '60'))
|
||||
LEASE_SECONDS = JOB_TIMEOUT + 30
|
||||
LEASE_SECONDS = max(JOB_TIMEOUT + 5, int(setting('JOB_LEASE_SECONDS', str(JOB_TIMEOUT + 30))))
|
||||
|
||||
|
||||
def envelope(job):
|
||||
@@ -17,24 +20,31 @@ def envelope(job):
|
||||
'job_id': str(job['id']),
|
||||
'type': 'attachment.process.v1',
|
||||
'attachment_id': str(job['attachment_id']),
|
||||
'pipeline_version': 1,
|
||||
'pipeline_version': job['pipeline_version'],
|
||||
'attempt': job['attempt'],
|
||||
'created_at': timestamp(job['created_at']),
|
||||
}
|
||||
|
||||
|
||||
def validate(message):
|
||||
if set(message) != {'schema_version', 'job_id', 'type', 'attachment_id',
|
||||
'pipeline_version', 'attempt', 'created_at'}:
|
||||
required = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'}
|
||||
if not isinstance(message, dict) or not required <= message.keys() <= required | {'traceparent', 'tracestate'}:
|
||||
raise ValueError('invalid fields')
|
||||
if (type(message['schema_version']) is not int or message['schema_version'] != 1
|
||||
or message['type'] != 'attachment.process.v1'
|
||||
or type(message['pipeline_version']) is not int or message['pipeline_version'] != 1
|
||||
or type(message['pipeline_version']) is not int or message['pipeline_version'] not in ENCODING
|
||||
or type(message['attempt']) is not int or message['attempt'] < 1):
|
||||
raise ValueError('invalid job')
|
||||
UUID(message['job_id'])
|
||||
UUID(message['attachment_id'])
|
||||
return message
|
||||
if datetime.fromisoformat(message['created_at'].replace('Z', '+00:00')).tzinfo is None:
|
||||
raise ValueError('missing timezone')
|
||||
if 'traceparent' in message and not re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}',
|
||||
message['traceparent']):
|
||||
raise ValueError('invalid traceparent')
|
||||
if 'tracestate' in message and (not isinstance(message['tracestate'], str) or len(message['tracestate']) > 512):
|
||||
raise ValueError('invalid tracestate')
|
||||
return {key: message[key] for key in required}
|
||||
|
||||
|
||||
def reserve_publish():
|
||||
@@ -83,7 +93,8 @@ def locked_job(conn, job_id, attachment_id):
|
||||
def claim(message):
|
||||
with storage.pool.connection() as conn:
|
||||
job, attachment = locked_job(conn, message['job_id'], message['attachment_id'])
|
||||
if not job or envelope(job) != message or job['state'] != 'queued':
|
||||
if (not job or envelope(job) != message or job['state'] != 'queued'
|
||||
or job['pipeline_version'] != attachment['target_pipeline_version']):
|
||||
return None
|
||||
token = uuid4()
|
||||
conn.execute(
|
||||
@@ -91,7 +102,8 @@ def claim(message):
|
||||
lease_until=clock_timestamp()+%s * interval '1 second', next_publish_at=NULL
|
||||
WHERE id=%s""", (token, LEASE_SECONDS, job['id']),
|
||||
)
|
||||
conn.execute("UPDATE attachments SET state='processing',updated_at=clock_timestamp() WHERE id=%s",
|
||||
conn.execute("UPDATE attachments SET state='processing',updated_at=clock_timestamp() "
|
||||
"WHERE id=%s AND published_pipeline_version IS NULL",
|
||||
(job['attachment_id'],))
|
||||
return job | {'lease_token': token, 'attachment': attachment}
|
||||
|
||||
@@ -105,20 +117,21 @@ def owns(conn, job, claimed):
|
||||
|
||||
def complete(claimed, variants):
|
||||
with storage.pool.connection() as conn:
|
||||
job, _ = locked_job(conn, claimed['id'], claimed['attachment_id'])
|
||||
if not owns(conn, job, claimed):
|
||||
job, attachment = locked_job(conn, claimed['id'], claimed['attachment_id'])
|
||||
if not owns(conn, job, claimed) or job['pipeline_version'] != attachment['target_pipeline_version']:
|
||||
return False
|
||||
for variant in variants:
|
||||
conn.execute(
|
||||
"""INSERT INTO attachment_variants
|
||||
(attachment_id,name,object_key,content_type,size_bytes,sha256,width,height)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||||
(job['attachment_id'], *variant),
|
||||
(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
|
||||
(job['attachment_id'], job['pipeline_version'], *variant),
|
||||
)
|
||||
conn.execute("UPDATE attachments SET state='ready',updated_at=clock_timestamp() WHERE id=%s",
|
||||
(job['attachment_id'],))
|
||||
conn.execute("UPDATE attachments SET state='ready',published_pipeline_version=%s,"
|
||||
"updated_at=clock_timestamp(),error_code=NULL WHERE id=%s",
|
||||
(job['pipeline_version'], job['attachment_id']))
|
||||
conn.execute("UPDATE attachment_jobs SET state='ready',lease_token=NULL,lease_until=NULL,"
|
||||
"last_error=NULL WHERE id=%s", (job['id'],))
|
||||
"last_error=NULL,settled_at=clock_timestamp() WHERE id=%s", (job['id'],))
|
||||
return True
|
||||
|
||||
|
||||
@@ -131,11 +144,13 @@ def settle_error(conn, job, code, permanent=False):
|
||||
if permanent or exhausted:
|
||||
error = code if permanent else 'retry_exhausted'
|
||||
conn.execute("UPDATE attachments SET state='failed',error_code=%s,updated_at=clock_timestamp() "
|
||||
"WHERE id=%s", (error, job['attachment_id']))
|
||||
"WHERE id=%s AND published_pipeline_version IS NULL", (error, job['attachment_id']))
|
||||
conn.execute("UPDATE attachment_jobs SET state='failed',lease_token=NULL,lease_until=NULL,"
|
||||
"last_error=%s,next_publish_at=NULL WHERE id=%s", (code, job['id']))
|
||||
"last_error=%s,next_publish_at=NULL,settled_at=clock_timestamp() WHERE id=%s",
|
||||
(code, job['id']))
|
||||
return
|
||||
conn.execute("UPDATE attachments SET state='queued',updated_at=clock_timestamp() WHERE id=%s",
|
||||
conn.execute("UPDATE attachments SET state='queued',updated_at=clock_timestamp() "
|
||||
"WHERE id=%s AND published_pipeline_version IS NULL",
|
||||
(job['attachment_id'],))
|
||||
conn.execute("""UPDATE attachment_jobs SET state='queued',lease_token=NULL,lease_until=NULL,
|
||||
last_error=%s,attempt=attempt+1,published_at=NULL,
|
||||
@@ -164,25 +179,60 @@ def recover_expired():
|
||||
|
||||
def replay(attachment_id):
|
||||
with storage.pool.connection() as conn:
|
||||
attachment = conn.execute('SELECT id FROM attachments WHERE id=%s FOR UPDATE',
|
||||
attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',
|
||||
(attachment_id,)).fetchone()
|
||||
if not attachment:
|
||||
return None
|
||||
job = conn.execute('SELECT * FROM attachment_jobs WHERE attachment_id=%s FOR UPDATE',
|
||||
(attachment_id,)).fetchone()
|
||||
job = conn.execute('SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s FOR UPDATE',
|
||||
(attachment_id, attachment['target_pipeline_version'])).fetchone()
|
||||
if not job or job['state'] != 'failed':
|
||||
return None
|
||||
object_lock(conn, job['staging_key'])
|
||||
client().head_object(Bucket=BUCKET, Key=job['staging_key'])
|
||||
source = attachment['object_key'] if attachment['published_pipeline_version'] else job['staging_key']
|
||||
object_lock(conn, source)
|
||||
client().head_object(Bucket=BUCKET, Key=source)
|
||||
new_attempt = job['attempt'] + 1
|
||||
budget = int(setting('JOB_MAX_ATTEMPTS', '5'))
|
||||
conn.execute(
|
||||
"""UPDATE attachment_jobs SET state='queued', attempt=%s, attempt_limit=%s,
|
||||
next_publish_at=clock_timestamp(), published_at=NULL, publish_failures=0
|
||||
next_publish_at=clock_timestamp(), published_at=NULL, publish_failures=0, settled_at=NULL
|
||||
WHERE id=%s""", (new_attempt, new_attempt + budget - 1, job['id']),
|
||||
)
|
||||
conn.execute("INSERT INTO attachment_replays(job_id,attempt,reason) VALUES (%s,%s,'operator')",
|
||||
(job['id'], new_attempt))
|
||||
conn.execute("UPDATE attachments SET state='queued',error_code=NULL,updated_at=clock_timestamp() "
|
||||
"WHERE id=%s", (attachment_id,))
|
||||
"WHERE id=%s AND published_pipeline_version IS NULL", (attachment_id,))
|
||||
return job['id'], new_attempt
|
||||
|
||||
|
||||
def upgrade(attachment_id, version):
|
||||
with storage.pool.connection() as conn:
|
||||
attachment = conn.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,)).fetchone()
|
||||
if not attachment or version not in ENCODING or version <= attachment['target_pipeline_version']:
|
||||
return None
|
||||
previous = conn.execute(
|
||||
'SELECT staging_key FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s',
|
||||
(attachment_id, attachment['target_pipeline_version']),
|
||||
).fetchone()
|
||||
if not previous and not attachment['published_pipeline_version']:
|
||||
return None
|
||||
source = attachment['object_key'] if attachment['published_pipeline_version'] else previous['staging_key']
|
||||
object_lock(conn, source)
|
||||
client().head_object(Bucket=BUCKET, Key=source)
|
||||
conn.execute(
|
||||
"""UPDATE attachment_jobs SET state='superseded', lease_token=NULL, lease_until=NULL,
|
||||
next_publish_at=NULL, settled_at=clock_timestamp()
|
||||
WHERE attachment_id=%s AND state IN ('queued','processing')""", (attachment_id,),
|
||||
)
|
||||
conn.execute(
|
||||
"""UPDATE attachments SET target_pipeline_version=%s,
|
||||
state=CASE WHEN published_pipeline_version IS NULL THEN 'queued' ELSE 'ready' END,
|
||||
error_code=NULL,updated_at=clock_timestamp() WHERE id=%s""",
|
||||
(version, attachment_id),
|
||||
)
|
||||
job_id = uuid4()
|
||||
conn.execute(
|
||||
"""INSERT INTO attachment_jobs(id,attachment_id,staging_key,attempt_limit,next_publish_at,pipeline_version)
|
||||
VALUES (%s,%s,%s,%s,clock_timestamp(),%s)""",
|
||||
(job_id, attachment_id, f'staging/{attachment_id}', int(setting('JOB_MAX_ATTEMPTS', '5')), version),
|
||||
)
|
||||
return job_id
|
||||
|
||||
+14
-10
@@ -12,27 +12,31 @@ from src.storage import storage, timestamp
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('action', choices=('failed', 'replay'))
|
||||
parser.add_argument('action', choices=('failed', 'replay', 'upgrade'))
|
||||
parser.add_argument('attachment_id', type=UUID, nargs='?')
|
||||
parser.add_argument('--version', type=int, default=2)
|
||||
args = parser.parse_args()
|
||||
if args.action == 'replay' and args.attachment_id is None:
|
||||
parser.error('replay requires attachment_id')
|
||||
if args.action in ('replay', 'upgrade') and args.attachment_id is None:
|
||||
parser.error('operation requires attachment_id')
|
||||
storage.pool.open(wait=True)
|
||||
try:
|
||||
if args.action == 'replay':
|
||||
if args.action in ('replay', 'upgrade'):
|
||||
try:
|
||||
result = jobs.replay(args.attachment_id)
|
||||
if args.action == 'replay':
|
||||
result = jobs.replay(args.attachment_id)
|
||||
else:
|
||||
job_id = jobs.upgrade(args.attachment_id, args.version)
|
||||
result = (job_id, 1) if job_id else None
|
||||
except (BotoCoreError, ClientError):
|
||||
raise SystemExit('Источник для replay недоступен') from None
|
||||
raise SystemExit('Источник обработки недоступен') from None
|
||||
if result is None:
|
||||
raise SystemExit('Нет подходящей failed-задачи')
|
||||
raise SystemExit('Операция неприменима к текущему состоянию или версии')
|
||||
print(json.dumps({'job_id': str(result[0]), 'attempt': result[1]}))
|
||||
else:
|
||||
with storage.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"""SELECT j.id, j.attachment_id, j.attempt, j.last_error, a.updated_at
|
||||
FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id
|
||||
WHERE j.state='failed' ORDER BY a.updated_at DESC LIMIT 100"""
|
||||
"""SELECT id, attachment_id, pipeline_version, attempt, last_error, settled_at
|
||||
FROM attachment_jobs WHERE state='failed' ORDER BY settled_at DESC NULLS LAST LIMIT 100"""
|
||||
).fetchall()
|
||||
print(json.dumps(rows, default=lambda value: str(value) if isinstance(value, UUID) else timestamp(value)))
|
||||
finally:
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Версии преобразований и операция перехода на следующий manifest."""
|
||||
|
||||
ENCODING = {
|
||||
1: (85, 82),
|
||||
2: (90, 88),
|
||||
}
|
||||
@@ -128,7 +128,8 @@ def get_attachment(request:Request,attachment_id:uuid.UUID,actor:str=Depends(req
|
||||
row = accessible(conn, request, attachment_id, actor)
|
||||
variants = conn.execute(
|
||||
'SELECT name,content_type,size_bytes,sha256,width,height FROM attachment_variants '
|
||||
'WHERE attachment_id=%s ORDER BY name', (attachment_id,)
|
||||
'WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',
|
||||
(attachment_id, row['published_pipeline_version']),
|
||||
).fetchall()
|
||||
return metadata(row, variants)
|
||||
|
||||
@@ -141,8 +142,10 @@ def content(request:Request,attachment_id:uuid.UUID,variant:str='original',actor
|
||||
if row['state'] != 'ready':
|
||||
raise error(request,409,'conflict','attachment not ready')
|
||||
if variant != 'original':
|
||||
row = conn.execute('SELECT * FROM attachment_variants WHERE attachment_id=%s AND name=%s',
|
||||
(attachment_id, variant)).fetchone()
|
||||
row = conn.execute(
|
||||
'SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s',
|
||||
(attachment_id, row['published_pipeline_version'], variant),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise error(request,404,'not_found','variant not found')
|
||||
obj=client().get_object(Bucket=BUCKET,Key=row['object_key'])
|
||||
|
||||
+3
-3
@@ -25,10 +25,10 @@ def cleanup(s3, retention):
|
||||
if not object_lock(conn, key, wait=False):
|
||||
continue
|
||||
job = conn.execute(
|
||||
'SELECT j.state,a.updated_at FROM attachment_jobs j JOIN attachments a '
|
||||
'ON a.id=j.attachment_id WHERE j.staging_key=%s', (key,),
|
||||
'SELECT state,settled_at FROM attachment_jobs WHERE staging_key=%s '
|
||||
'ORDER BY pipeline_version DESC LIMIT 1', (key,),
|
||||
).fetchone()
|
||||
retention_start = job['updated_at'] if job else item['LastModified']
|
||||
retention_start = (job['settled_at'] if job else None) or item['LastModified']
|
||||
age = (now - retention_start).total_seconds()
|
||||
if job and job['state'] in ('queued', 'processing'):
|
||||
continue
|
||||
|
||||
+9
-4
@@ -16,7 +16,7 @@ import psycopg
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from psycopg_pool import PoolTimeout
|
||||
|
||||
from src import broker, jobs
|
||||
from src import broker, jobs, faults
|
||||
from src.objects import BUCKET, ObjectConflict, client, put_immutable
|
||||
from src.storage import storage
|
||||
|
||||
@@ -41,13 +41,15 @@ def fetch(s3, key, attachment, destination):
|
||||
|
||||
|
||||
def process(job):
|
||||
faults.hit('before_effect', job)
|
||||
attachment = job['attachment']
|
||||
s3 = client()
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
directory = Path(temp)
|
||||
original = directory / 'original'
|
||||
fetch(s3, job['staging_key'], attachment, original)
|
||||
result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', temp],
|
||||
source = attachment['object_key'] if attachment['published_pipeline_version'] else job['staging_key']
|
||||
fetch(s3, source, attachment, original)
|
||||
result = subprocess.run([sys.executable, '-m', 'src.image_pipeline', temp, str(job['pipeline_version'])],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
timeout=jobs.JOB_TIMEOUT, check=False)
|
||||
if result.returncode == 2:
|
||||
@@ -57,8 +59,9 @@ def process(job):
|
||||
put_immutable(s3, attachment['object_key'], stream, attachment['size_bytes'],
|
||||
attachment['content_type'], attachment['sha256'])
|
||||
variants = []
|
||||
faults.hit('after_original', job)
|
||||
for name, mime, width, height in json.loads((directory / 'manifest.json').read_text()):
|
||||
key = f"variants/{attachment['id']}/v1/{name}"
|
||||
key = f"variants/{attachment['id']}/v{job['pipeline_version']}/{name}"
|
||||
output = directory / name
|
||||
size = output.stat().st_size
|
||||
with output.open('rb') as stream:
|
||||
@@ -66,7 +69,9 @@ def process(job):
|
||||
stream.seek(0)
|
||||
put_immutable(s3, key, stream, size, mime, digest)
|
||||
variants.append((name, key, mime, size, digest, width, height))
|
||||
faults.hit('after_variants', job)
|
||||
jobs.complete(job, variants)
|
||||
faults.hit('after_commit', job)
|
||||
|
||||
|
||||
def handle(channel, delivery, body):
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Closed-loop: полный путь upload -> ready; ограничение генератора явно в отчёте."""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import random
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def png(width, height):
|
||||
def chunk(name, data):
|
||||
return struct.pack('>I', len(data)) + name + data + struct.pack('>I', zlib.crc32(name + data))
|
||||
rng = random.Random(2301)
|
||||
data = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height))
|
||||
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0))
|
||||
+ chunk(b'IDAT', zlib.compress(data)) + chunk(b'IEND', b''))
|
||||
|
||||
|
||||
def percentile(values, q):
|
||||
return round(sorted(values)[min(len(values)-1, int((len(values)-1)*q))], 4) if values else None
|
||||
|
||||
|
||||
def main(args):
|
||||
client = Client(args.base_url)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
profiles = [('320x200', (ROOT / 'tests/fixtures/sample.png').read_bytes()), ('1024x768', png(1024, 768))]
|
||||
rows = []
|
||||
for profile, raw in profiles:
|
||||
for repeat in range(args.repeats):
|
||||
samples = []
|
||||
failures = []
|
||||
stats = []
|
||||
stop = threading.Event()
|
||||
def sample():
|
||||
while not stop.is_set():
|
||||
ids = subprocess.check_output(['docker', 'compose', '-p', args.project, 'ps', '-q'], text=True).split()
|
||||
result = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids], text=True)
|
||||
stats.append(dict(at_seconds=round(time.monotonic()-start, 3), containers=[json.loads(line) for line in result.splitlines()]))
|
||||
stop.wait(2)
|
||||
def one():
|
||||
c = Client(args.base_url)
|
||||
begin = time.monotonic()
|
||||
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png', 202,
|
||||
raw=raw, actor=alice, headers={'Content-Type': 'image/png'})['json']
|
||||
accepted = time.monotonic()
|
||||
ready = c.wait_ready(alice, att)
|
||||
check(len(ready['variants']) == 3 and ready['sha256'] == hashlib.sha256(raw).hexdigest(), 'bad load result')
|
||||
return dict(upload_seconds=accepted-begin, ready_seconds=time.monotonic()-begin)
|
||||
# Один завершённый прогрев на профиль/повтор, не входит в измерение.
|
||||
one()
|
||||
start = time.monotonic()
|
||||
sampler = threading.Thread(target=sample)
|
||||
sampler.start()
|
||||
def loop(_):
|
||||
while time.monotonic()-start < args.duration:
|
||||
try:
|
||||
samples.append(one())
|
||||
except Exception as exc:
|
||||
failures.append(type(exc).__name__ + ': ' + str(exc))
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
list(pool.map(loop, range(args.concurrency)))
|
||||
elapsed = time.monotonic()-start
|
||||
stop.set()
|
||||
sampler.join()
|
||||
times = [x['ready_seconds'] for x in samples]
|
||||
row = dict(profile=profile, bytes_per_image=len(raw), repeat=repeat+1, concurrency=args.concurrency,
|
||||
requested_duration=args.duration, elapsed_seconds=round(elapsed, 3), warmup_completed=1,
|
||||
completed=len(samples), successful_rps=round(len(samples)/elapsed, 3), failures=failures,
|
||||
error_fraction=len(failures)/max(1,len(samples)+len(failures)),
|
||||
p50=percentile(times,.5), p95=percentile(times,.95), p99=percentile(times,.99),
|
||||
upload_p95=percentile([x['upload_seconds'] for x in samples],.95), stats=stats)
|
||||
rows.append(row)
|
||||
(ROOT / 'evidence/load-pipeline.json').write_text(json.dumps(rows, indent=2), encoding='utf-8')
|
||||
print(json.dumps({k:v for k,v in row.items() if k != 'stats'}), flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--base-url', default='http://127.0.0.1:8085')
|
||||
p.add_argument('--project', default='lyakhonov-lab5')
|
||||
p.add_argument('--duration', type=int, default=20)
|
||||
p.add_argument('--concurrency', type=int, default=2)
|
||||
p.add_argument('--repeats', type=int, default=3)
|
||||
main(p.parse_args())
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Реальные fault-injection и инварианты. Запускать только на своём lab5 стенде."""
|
||||
import argparse
|
||||
import base64
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
class Experiment:
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.compose = ['docker', 'compose', '-p', args.project]
|
||||
self.c = Client(args.base_url)
|
||||
self.alice, self.bob, self.eve, self.chat = self.c.bootstrap()
|
||||
path = ROOT / 'evidence/pipeline.json'
|
||||
self.rows = json.loads(path.read_text(encoding='utf-8')) if args.resume and path.exists() else []
|
||||
|
||||
def command(self, *args):
|
||||
return subprocess.check_output([*self.compose, *args], cwd=ROOT, text=True, encoding='utf-8', stderr=subprocess.PIPE)
|
||||
|
||||
def probe(self, action, **kwargs):
|
||||
script = 'ARGS=' + repr(dict(action=action, **kwargs)) + '\n' + (ROOT / 'student-tests/probe.py').read_text(encoding='utf-8')
|
||||
invocation = ([*self.compose, 'run', '--rm', '--no-deps', '-T', 'inspector', 'python', '-']
|
||||
if action == 'objects' else [*self.compose, 'exec', '-T', '--index', '1', 'api', 'python', '-'])
|
||||
result = subprocess.run(invocation,
|
||||
input=script, text=True, encoding='utf-8', capture_output=True, cwd=ROOT)
|
||||
check(result.returncode == 0, 'probe failed: ' + result.stderr)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
def sql(self, query, *params):
|
||||
return self.probe('sql', query=query, params=list(params))
|
||||
|
||||
def job(self, att, version=1):
|
||||
return self.sql('SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s', att['id'], version)[0]
|
||||
|
||||
@staticmethod
|
||||
def envelope(job):
|
||||
from datetime import datetime, timezone
|
||||
created = datetime.fromisoformat(job['created_at']).astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
return dict(schema_version=1, type='attachment.process.v1', job_id=job['id'], attachment_id=job['attachment_id'],
|
||||
pipeline_version=job['pipeline_version'], attempt=job['attempt'], created_at=created)
|
||||
|
||||
def upload(self):
|
||||
return self.c.attachment(self.alice, self.chat)[0]
|
||||
|
||||
def arm(self, job, point, action='crash'):
|
||||
self.sql('INSERT INTO experiment_faults(job_id,point,action) VALUES (%s,%s,%s)', job['id'], point, action)
|
||||
|
||||
def hit(self, job, point):
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
row = self.sql('SELECT * FROM experiment_faults WHERE job_id=%s AND point=%s', job['id'], point)[0]
|
||||
if row['hit_at']:
|
||||
return row
|
||||
time.sleep(.3)
|
||||
raise AssertionError('fault not hit: ' + point)
|
||||
|
||||
def snapshot(self):
|
||||
ids = self.command('ps', '-a', '-q').split()
|
||||
raw = subprocess.check_output(['docker', 'inspect', *ids], text=True, encoding='utf-8')
|
||||
return [dict(id=x['Id'], name=x['Name'], started_at=x['State']['StartedAt'],
|
||||
running=x['State']['Running'], restart_count=x['RestartCount'], exit_code=x['State']['ExitCode']) for x in json.loads(raw)]
|
||||
|
||||
def invariant(self, att, version=1):
|
||||
ready = self.c.wait_ready(self.alice, att, wait_seconds=120)
|
||||
check(len(ready['variants']) == 3, 'three variants required')
|
||||
row = self.sql('SELECT published_pipeline_version,target_pipeline_version FROM attachments WHERE id=%s', att['id'])[0]
|
||||
check(row['published_pipeline_version'] == version, 'wrong published version')
|
||||
variants = self.sql('SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s', att['id'], version)
|
||||
check(len(variants) == 3, 'duplicate/missing manifests')
|
||||
digest = {}
|
||||
for name in ('original', 'thumbnail', 'crop', 'optimized'):
|
||||
response = self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + name, actor=self.alice)
|
||||
sha = hashlib.sha256(response['bytes']).hexdigest()
|
||||
expected = att['sha256'] if name == 'original' else next(v['sha256'] for v in variants if v['name'] == name)
|
||||
check(sha == expected, 'download checksum mismatch')
|
||||
digest[name] = sha
|
||||
return digest
|
||||
|
||||
def record(self, name, started, **details):
|
||||
self.rows.append(dict(scenario=name, seconds=round(time.monotonic() - started, 3), **details))
|
||||
(ROOT / 'evidence/pipeline.json').write_text(json.dumps(self.rows, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print('PASS:', name, flush=True)
|
||||
|
||||
def crashes(self):
|
||||
for repeat in range(self.args.repeats):
|
||||
for point in ('before_publish', 'after_publish', 'before_effect', 'after_original', 'after_variants', 'after_commit'):
|
||||
start = time.monotonic()
|
||||
service = 'dispatcher' if 'publish' in point else 'worker'
|
||||
self.command('stop', '-t', '2', service)
|
||||
att = self.upload()
|
||||
job = self.job(att)
|
||||
self.arm(job, point)
|
||||
before = self.snapshot()
|
||||
self.command('start', service)
|
||||
hit = self.hit(job, point)
|
||||
self.invariant(att)
|
||||
self.record(point, start, repeat=repeat + 1, attachment_id=att['id'], fault=hit,
|
||||
job_after=self.job(att), before=before, after=self.snapshot())
|
||||
|
||||
def durable(self):
|
||||
start = time.monotonic()
|
||||
self.command('stop', '-t', '2', 'worker')
|
||||
att = self.upload()
|
||||
self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=self.alice)
|
||||
self.probe('cleanup')
|
||||
check(any(obj['key'].startswith('staging/') for obj in self.probe('objects', id=att['id'])), 'active staging deleted')
|
||||
self.c.message(self.alice, self.chat, '', attachments=[att['id']])
|
||||
time.sleep(2)
|
||||
before = self.snapshot()
|
||||
self.command('restart', '-t', '2', 'api', 'broker')
|
||||
self.command('start', 'worker')
|
||||
self.invariant(att)
|
||||
self.record('accepted_survives_api_broker_restart', start, attachment_id=att['id'], before=before, after=self.snapshot())
|
||||
|
||||
def versions(self):
|
||||
for repeat in range(self.args.repeats):
|
||||
start = time.monotonic()
|
||||
self.command('stop', '-t', '2', 'worker')
|
||||
att = self.upload()
|
||||
old = self.job(att)
|
||||
self.arm(old, 'after_variants', 'pause')
|
||||
self.command('start', 'worker')
|
||||
self.hit(old, 'after_variants')
|
||||
self.probe('upgrade', id=att['id'])
|
||||
new = self.job(att, 2)
|
||||
# Новая работа, затем старая, затем 24 чередующихся дубля.
|
||||
self.probe('publish', messages=[self.envelope(new), self.envelope(old)] * 13)
|
||||
first = self.invariant(att, 2)
|
||||
objects = self.probe('objects', id=att['id'])
|
||||
self.sql('UPDATE experiment_faults SET released=true WHERE job_id=%s', old['id'])
|
||||
time.sleep(2)
|
||||
check(self.invariant(att, 2) == first, 'stale worker changed new result')
|
||||
self.probe('publish', messages=[self.envelope(old), self.envelope(new)] * 13)
|
||||
time.sleep(2)
|
||||
check(self.invariant(att, 2) == first, 'reorder changed new result')
|
||||
current = self.probe('objects', id=att['id'])
|
||||
check([x for x in current if not x['key'].startswith('staging/')] ==
|
||||
[x for x in objects if not x['key'].startswith('staging/')], 'duplicate wrote more objects')
|
||||
check(self.probe('replay', id=att['id']) is None, 'ready replay must no-op')
|
||||
self.record('stale_running_worker_and_reordered_duplicates', start, repeat=repeat + 1,
|
||||
attachment_id=att['id'], before_digest=first, after_digest=self.invariant(att, 2),
|
||||
versions=self.sql('SELECT pipeline_version,state,attempt FROM attachment_jobs WHERE attachment_id=%s ORDER BY pipeline_version', att['id']), objects=current)
|
||||
start = time.monotonic()
|
||||
att = self.upload()
|
||||
old_digest = self.invariant(att)
|
||||
self.probe('cleanup')
|
||||
self.probe('upgrade', id=att['id'])
|
||||
deadline = time.monotonic() + 60
|
||||
while self.job(att, 2)['state'] != 'ready':
|
||||
check(time.monotonic() < deadline, 'ready upgrade timeout')
|
||||
time.sleep(.3)
|
||||
new_digest = self.invariant(att, 2)
|
||||
check(new_digest['original'] == old_digest['original'], 'upgrade changed original')
|
||||
check(new_digest['thumbnail'] != old_digest['thumbnail'], 'versions must have a real different effect')
|
||||
self.record('upgrade_ready_after_staging_cleanup', start, repeat=repeat + 1, attachment_id=att['id'], before_digest=old_digest, after_digest=new_digest)
|
||||
|
||||
def errors(self):
|
||||
for kind in ('permanent', 'transient'):
|
||||
start = time.monotonic()
|
||||
self.command('stop', '-t', '2', 'worker')
|
||||
att = self.upload()
|
||||
if kind == 'permanent':
|
||||
self.probe('corrupt_staging', id=att['id'])
|
||||
else:
|
||||
self.probe('remove_staging', id=att['id'])
|
||||
self.command('start', 'worker')
|
||||
observed = []
|
||||
deadline = time.monotonic() + 120
|
||||
while True:
|
||||
job = self.job(att)
|
||||
observed.append(dict(attempt=job['attempt'], state=job['state'], next_publish_at=job['next_publish_at'], last_error=job['last_error']))
|
||||
if job['state'] == 'failed':
|
||||
break
|
||||
check(time.monotonic() < deadline, 'failed registry timeout')
|
||||
time.sleep(.3)
|
||||
check(job['attempt'] == (1 if kind == 'permanent' else job['attempt_limit']), 'wrong retry budget')
|
||||
self.c.call('POST', '/api/v1/chats/' + self.chat['id'] + '/messages', 409, actor=self.alice,
|
||||
headers={'Idempotency-Key': os.urandom(12).hex()}, body={'text': '', 'attachment_ids': [att['id']]})
|
||||
self.probe('restore_staging', id=att['id'], data=base64.b64encode((ROOT / 'tests/fixtures/sample.png').read_bytes()).decode())
|
||||
check(self.probe('replay', id=att['id']) is not None, 'failed replay missing')
|
||||
self.invariant(att)
|
||||
self.record(kind + '_error_and_explicit_replay', start, attachment_id=att['id'], attempts=observed,
|
||||
audit=self.sql('SELECT attempt,reason FROM attachment_replays WHERE job_id=%s', job['id']))
|
||||
|
||||
def negatives(self):
|
||||
start = time.monotonic()
|
||||
path = '/api/v1/chats/' + self.chat['id'] + '/attachments?filename=x.png'
|
||||
def upload(raw, expected, path=path, actor=None, mime='image/png'):
|
||||
self.c.call('POST', path, expected, actor=actor or self.alice, raw=raw, headers={'Content-Type': mime})
|
||||
upload(b'bad', 415)
|
||||
upload(b'x' * (10 * 1024 * 1024 + 1), 413)
|
||||
raw = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
upload(raw, 415, mime='image/jpeg')
|
||||
upload(raw, 400, path=path.replace('x.png', '..%2Fx.png'))
|
||||
upload(raw, 404, actor=self.eve)
|
||||
# Валидный PNG header >40 MP: отклонение до декодирования большого bitmap.
|
||||
def chunk(name, data):
|
||||
return struct.pack('>I', len(data)) + name + data + struct.pack('>I', zlib.crc32(name + data))
|
||||
huge = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 10000, 10000, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(b'')) + chunk(b'IEND', b'')
|
||||
upload(huge, 413)
|
||||
att = self.upload()
|
||||
self.invariant(att)
|
||||
self.c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=self.eve)
|
||||
self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=self.eve)
|
||||
key = os.urandom(16).hex()
|
||||
with ThreadPoolExecutor(max_workers=12) as pool:
|
||||
responses = list(pool.map(lambda _: self.c.message(self.alice, self.chat, 'concurrent', key=key)['json']['id'], range(24)))
|
||||
check(len(set(responses)) == 1, 'message idempotency violated')
|
||||
self.record('negative_upload_acl_and_24_concurrent_messages', start, checks=9, message_ids=len(set(responses)))
|
||||
|
||||
def main(self):
|
||||
try:
|
||||
if not self.args.resume:
|
||||
self.negatives()
|
||||
self.durable()
|
||||
self.crashes()
|
||||
self.versions()
|
||||
self.errors()
|
||||
finally:
|
||||
self.sql('UPDATE experiment_faults SET released=true WHERE NOT released')
|
||||
self.command('start', 'api', 'broker', 'dispatcher', 'worker')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--project', default='lyakhonov-lab5')
|
||||
parser.add_argument('--base-url', default='http://127.0.0.1:8085')
|
||||
parser.add_argument('--repeats', type=int, default=3)
|
||||
parser.add_argument('--resume', action='store_true', help='Сохранить уже выполненные crash-результаты; выполнить version/error suite')
|
||||
Experiment(parser.parse_args()).main()
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Исполняется внутри API через stdin; наружу возвращает только безопасные данные."""
|
||||
import json
|
||||
import sys
|
||||
from uuid import UUID
|
||||
|
||||
from src import broker, jobs
|
||||
from src.objects import BUCKET, client
|
||||
from src.storage import storage
|
||||
|
||||
|
||||
def run(args):
|
||||
action = args['action']
|
||||
if action == 'sql':
|
||||
with storage.pool.connection() as conn:
|
||||
cursor = conn.execute(args['query'], args.get('params', []))
|
||||
return cursor.fetchall() if cursor.description else []
|
||||
if action == 'publish':
|
||||
with broker.channel() as (_, channel):
|
||||
channel.confirm_delivery()
|
||||
for message in args['messages']:
|
||||
broker.publish(channel, message)
|
||||
return True
|
||||
if action == 'upgrade':
|
||||
return jobs.upgrade(UUID(args['id']), args.get('version', 2))
|
||||
if action == 'replay':
|
||||
return jobs.replay(UUID(args['id']))
|
||||
if action == 'cleanup':
|
||||
from src.staging import cleanup
|
||||
return cleanup(client(), 60)
|
||||
if action == 'objects':
|
||||
s3 = client()
|
||||
result = []
|
||||
for prefix in (f"originals/{args['id']}", f"variants/{args['id']}/", f"staging/{args['id']}"):
|
||||
for page in s3.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=prefix):
|
||||
result.extend({'key': row['Key'], 'etag': row['ETag'], 'size': row['Size']}
|
||||
for row in page.get('Contents', []))
|
||||
return result
|
||||
if action == 'remove_staging':
|
||||
client().delete_object(Bucket=BUCKET, Key=f"staging/{args['id']}")
|
||||
return True
|
||||
if action == 'corrupt_staging':
|
||||
client().put_object(Bucket=BUCKET, Key=f"staging/{args['id']}", Body=b'broken')
|
||||
return True
|
||||
if action == 'restore_staging':
|
||||
import base64
|
||||
client().put_object(Bucket=BUCKET, Key=f"staging/{args['id']}", Body=base64.b64decode(args['data']))
|
||||
return True
|
||||
raise ValueError('unknown probe action')
|
||||
|
||||
|
||||
storage.pool.open(wait=True)
|
||||
try:
|
||||
print(json.dumps(run(ARGS), default=str))
|
||||
finally:
|
||||
storage.pool.close()
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Публичные resilience и отдельный принудительный отказ каждой API-реплики."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check
|
||||
|
||||
COMPOSE = ['docker', 'compose', '-p', 'lyakhonov-lab5']
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.check_output(['docker', *args], cwd=ROOT, text=True, encoding='utf-8')
|
||||
|
||||
|
||||
def snapshot():
|
||||
ids = docker('compose', '-p', 'lyakhonov-lab5', 'ps', '-a', '-q').split()
|
||||
raw = json.loads(docker('inspect', *ids))
|
||||
return [dict(id=x['Id'], name=x['Name'], started_at=x['State']['StartedAt'], running=x['State']['Running']) for x in raw]
|
||||
|
||||
|
||||
def public(mode):
|
||||
before = snapshot()
|
||||
script = ROOT / 'scripts' / (('restart' if mode == 'persistence' else 'stop-one') + ('.cmd' if sys.platform == 'win32' else '.sh'))
|
||||
result = subprocess.run([sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', mode,
|
||||
'--base-url', 'http://127.0.0.1:8085', '--action-script', str(script)],
|
||||
cwd=ROOT, text=True, encoding='utf-8', capture_output=True)
|
||||
after = snapshot()
|
||||
(ROOT / f'evidence/{mode}.json').write_text(json.dumps(dict(command=result.args, exit_code=result.returncode,
|
||||
output=result.stdout + result.stderr, before=before, after=after), ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print(result.stdout, flush=True)
|
||||
check(result.returncode == 0, result.stderr)
|
||||
subprocess.check_call([*COMPOSE, 'start', 'api'], cwd=ROOT)
|
||||
|
||||
|
||||
def targeted():
|
||||
c = Client('http://127.0.0.1:8085', timeout=2)
|
||||
alice, _, _, chat = c.bootstrap()
|
||||
ids = docker('compose', '-p', 'lyakhonov-lab5', 'ps', '-q', 'api').split()
|
||||
evidence = []
|
||||
for index, target in enumerate(ids):
|
||||
before = snapshot()
|
||||
responses, errors = [], []
|
||||
first = None
|
||||
begin = time.monotonic()
|
||||
subprocess.check_call(['docker', 'stop', '-t', '2', target], stdout=subprocess.DEVNULL)
|
||||
try:
|
||||
while time.monotonic() - begin < 15:
|
||||
key = f'explicit-{index}-{len(responses)}'
|
||||
try:
|
||||
response = c.message(alice, chat, 'replica failure', key=key)
|
||||
first = first or time.monotonic()-begin
|
||||
responses.append(response['json'])
|
||||
except Failure as exc:
|
||||
errors.append(str(exc))
|
||||
time.sleep(.05)
|
||||
check(first is not None and first <= 10, 'slow API failover')
|
||||
page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice)['json']
|
||||
messages = list(page['items'])
|
||||
while page['next_cursor']:
|
||||
from urllib.parse import quote
|
||||
page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe=''), actor=alice)['json']
|
||||
messages.extend(page['items'])
|
||||
found = {m['id'] for m in messages}
|
||||
check(all(m['id'] in found for m in responses), 'acknowledged message lost')
|
||||
check(len(found) == len(messages), 'duplicate messages')
|
||||
evidence.append(dict(target=target, before=before, after=snapshot(), successes=len(responses),
|
||||
errors=errors, recovery_seconds=round(first, 3)))
|
||||
finally:
|
||||
subprocess.check_call(['docker', 'start', target], stdout=subprocess.DEVNULL)
|
||||
time.sleep(3)
|
||||
(ROOT / 'evidence/failover-each.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print('PASS: both real API containers independently stopped and restored')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
public('persistence')
|
||||
public('failover')
|
||||
targeted()
|
||||
Reference in New Issue
Block a user