This commit is contained in:
2026-10-29 15:00:36 +03:00
parent ec540a1a72
commit 1706ddca25
56 changed files with 5935 additions and 19 deletions
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env python3
import json
import secrets
import shutil
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRETS = ROOT / 'secrets'
def write(name, value):
path = SECRETS / name
if path.exists():
print('keep secrets/' + name)
return path.read_text().strip()
path.write_bytes((value + '\n').encode('utf-8'))
print('created secrets/' + name)
return value
def main():
SECRETS.mkdir(exist_ok=True)
pg = write('pg_password', secrets.token_urlsafe(24))
write('database_url', 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' % pg)
vk_conf = SECRETS / 'valkey_auth.conf'
if vk_conf.exists():
vk = vk_conf.read_text().split()[-1]
print('keep secrets/valkey_auth.conf')
else:
vk = secrets.token_urlsafe(24)
write('valkey_auth.conf', 'requirepass ' + vk)
write('cache_url', 'redis://:%s@valkey:6379/0' % vk)
# Two S3 identities: admin (bucket bootstrap only) and api (read/write/list
# of the one bucket). SeaweedFS has no anonymous identity -> bucket is private.
admin_id = write('s3_admin_access_key_id', 'admin' + secrets.token_hex(8))
admin_secret = write('s3_admin_secret_access_key', secrets.token_urlsafe(30))
api_id = write('s3_access_key_id', 'api' + secrets.token_hex(8))
api_secret = write('s3_secret_access_key', secrets.token_urlsafe(30))
bucket = 'messenger-attachments'
identities = {'identities': [
{'name': 'bootstrap-admin',
'credentials': [{'accessKey': admin_id, 'secretKey': admin_secret}],
'actions': ['Admin', 'Read', 'Write', 'List', 'Tagging']},
{'name': 'messenger-api',
'credentials': [{'accessKey': api_id, 'secretKey': api_secret}],
'actions': ['Read:' + bucket, 'Write:' + bucket, 'List:' + bucket]},
]}
write('s3_identities.json', json.dumps(identities, indent=2))
env = ROOT / '.env'
if not env.exists():
shutil.copy(ROOT / '.env.example', env)
print('created .env from .env.example')
if __name__ == '__main__':
main()
+269
View File
@@ -0,0 +1,269 @@
#!/usr/bin/env python3
"""Operator checks for lab4 that need Docker access (run on the host, explicitly).
python scripts/lab4ops.py private # bucket privacy outside the API
python scripts/lab4ops.py failover-upload # upload -> stop that replica -> download via the other
python scripts/lab4ops.py s3-outage # stop S3: uploads/downloads 503, text path keeps working
python scripts/lab4ops.py no-local-file # API containers have no volume and no stored original
python scripts/lab4ops.py bootstrap # migrate + bucket bootstrap are idempotent
python scripts/lab4ops.py memory # peak RSS under concurrent uploads
Each command prints JSON and, with --output, stores it in evidence/.
Synthetic data only; tokens/credentials are never printed.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'student-tests'))
sys.path.insert(0, str(ROOT / 'scripts'))
from client import Client, Failure, check # noqa: E402
import imagegen # noqa: E402
from lablib import api_containers, sh, wait_healthy # noqa: E402
PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
NETWORK = 'mtusi-lab4_data'
TOOL_IMAGE = 'python:3.12.8-alpine3.21'
def winpath(p):
s = str(p)
return s.replace('\\', '/')
def s3tool(*args, stdin=None):
"""Run s3tool.py in a throwaway container on the data network (S3 is not published)."""
cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK,
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', *args]
r = subprocess.run(cmd, input=stdin, capture_output=True, text=True, encoding='utf-8')
if r.returncode != 0:
raise RuntimeError('s3tool %s failed: %s' % (args[0], (r.stdout + r.stderr).strip()[-300:]))
return json.loads(r.stdout.strip().splitlines()[-1])
def psql(sql):
return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
'-At', '-v', 'ON_ERROR_STOP=1', '-c', sql)
def upload(c, actor, chat, data=PNG, name='sample.png', expected=201):
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + name, expected,
actor=actor, raw=data, headers={'Content-Type': 'image/png'},
schema='Attachment' if expected == 201 else None)
def cmd_private(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
att = upload(c, alice, chat)['json']
key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % att['id'])
res = {
'object_key_shape': 'attachments/<chat_id>/<attachment_id> (server generated)',
'key_contains_filename': 'sample' in key,
'anonymous_get': s3tool('get', key, '--as', 'anonymous')['status'],
'anonymous_list': None,
'api_identity_get': s3tool('get', key)['status'],
'api_identity_get_sha256_matches': s3tool('get', key)['sha256'] == att['sha256'],
'api_identity_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket')['status'],
'anonymous_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket', '--as', 'anonymous')['status'],
'published_s3_ports': sh('docker', 'compose', 'port', 's3', '8333', check=False) or None,
'outsider_via_api_metadata': c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=eve)['status'],
'outsider_via_api_content': c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=eve)['status'],
}
try:
s3tool('list', 'attachments/', '--as', 'anonymous')
res['anonymous_list'] = 200
except RuntimeError as exc:
res['anonymous_list'] = 'denied (%s)' % str(exc).split(':')[-1].strip()
# the API identity is bound to ONE bucket: the admin creates another one, the API cannot write there
s3tool('create-bucket', 'lab4-other-bucket', '--as', 'admin')
res['api_identity_put_other_bucket'] = subprocess.run(
['docker', 'run', '--rm', '-e', 'S3_BUCKET=lab4-other-bucket', '--network', NETWORK,
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', 'probe'],
capture_output=True, text=True).stdout.strip()
check(res['anonymous_get'] == 403 and res['api_identity_get'] == 200, 'unexpected bucket privacy')
check(res['api_identity_create_bucket'] == 403 and '"status": 403' in res['api_identity_put_other_bucket'],
'api identity has more rights than expected')
return res
def cmd_failover_upload(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
r = upload(c, alice, chat)
att, served_by = r['json'], r['headers']['X-Instance-Id']
target = [x for x in api_containers() if x['id'].startswith(served_by[:12])]
check(len(target) == 1, 'cannot map X-Instance-Id to one api container')
before = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}')
sh('docker', 'stop', '-t', '15', target[0]['id'])
t0 = time.monotonic()
reads = []
try:
for _ in range(10):
try:
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
reads.append({'instance': d['headers']['X-Instance-Id'],
'sha256_ok': hashlib.sha256(d['bytes']).hexdigest() == att['sha256']})
except Failure as exc:
reads.append({'error': str(exc)[-60:]})
time.sleep(0.3)
during = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}')
finally:
sh('docker', 'start', target[0]['id'])
wait_healthy(target[0]['id'])
ok = [x for x in reads if x.get('sha256_ok')]
check(len(ok) >= 8 and all(x['instance'] != served_by for x in ok), 'download did not move to the other replica')
return {'uploaded_via': served_by, 'stopped_container': target[0]['name'], 'reads_after_stop': reads,
'seconds_observed': round(time.monotonic() - t0, 1), 'ps_before': before, 'ps_during': during}
def cmd_s3_outage(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
att = upload(c, alice, chat)['json']
pending_before = psql("SELECT count(*) FROM attachments WHERE status = 'pending'")
sh('docker', 'compose', 'stop', 's3')
res = {}
try:
u = upload(c, alice, chat, expected=503)
res['upload_status'], res['upload_ms'] = u['status'], round(u['elapsed'] * 1000)
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob)
res['download_status'], res['download_ms'] = d['status'], round(d['elapsed'] * 1000)
res['metadata_status'] = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob)['status']
res['text_message_status'] = c.message(alice, chat, 'S3 лежит, текст работает')['status']
res['ready_status'] = c.call('GET', '/health/ready')['status']
# expected 0: compensation deletes the pending row; a possible object is an orphan for the reconciler
res['pending_rows_left'] = int(psql(
"SELECT count(*) FROM attachments WHERE status = 'pending'")) - int(pending_before)
finally:
sh('docker', 'compose', 'up', '-d', '--no-recreate', '--wait', 's3')
t0 = time.monotonic()
while True:
try:
upload(c, alice, chat)
break
except Failure:
check(time.monotonic() - t0 < 60, 'uploads did not recover after S3 start')
time.sleep(0.5)
res['upload_recovered_after_s3_healthy_s'] = round(time.monotonic() - t0, 1)
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
res['old_file_intact'] = hashlib.sha256(d['bytes']).hexdigest() == att['sha256']
check(res['upload_status'] == 503 and res['text_message_status'] == 201 and res['ready_status'] == 200,
'S3 outage is not isolated to attachment endpoints')
return res
def cmd_no_local_file(a):
out = {}
for r in api_containers():
mounts = json.loads(sh('docker', 'inspect', '-f', '{{json .Mounts}}', r['id']))
ro = sh('docker', 'inspect', '-f', '{{.HostConfig.ReadonlyRootfs}}', r['id'])
probe = sh('docker', 'exec', r['id'], 'sh', '-c', 'touch /app/probe 2>&1 || true')
# Docker itself bind-mounts /etc/hosts, /etc/hostname, /etc/resolv.conf
files = sh('docker', 'exec', r['id'], 'sh', '-c',
'find / -xdev -type f -newer /app/server ! -path "/proc/*" ! -path "/sys/*" '
'! -path "/run/secrets/*" ! -name hosts ! -name hostname ! -name resolv.conf '
'! -name .dockerenv 2>/dev/null | head -20')
out[r['name']] = {'readonly_rootfs': ro == 'true', 'write_probe': probe,
'mounts': [m.get('Destination') for m in mounts],
'files_written_since_start': files.splitlines()}
check(ro == 'true' and not files.strip(), 'API container can store files locally')
return out
def cmd_bootstrap(a):
runs = []
for _ in range(2):
for svc in ('migrate', 's3-init'):
log = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', svc)
runs.append({'service': svc, 'last_log': json.loads(log.strip().splitlines()[-1])})
return runs
def peak(cid):
v = sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory.current').split()
return int(v[0]) / 2**20, int(v[1]) / 2**20
def cmd_memory(a):
big = imagegen.near_limit_png()
c = Client(a.base_url, timeout=120)
alice, bob, eve, chat = c.bootstrap()
results = []
for conc in a.levels:
for rep in range(a.repeat):
sh('docker', 'compose', 'restart', 'api')
for r in api_containers():
check(wait_healthy(r['id'], 60), 'api not healthy')
time.sleep(1)
s3_id = sh('docker', 'compose', 'ps', '-q', 's3')
s3_restarts = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id)
upload(c, alice, chat, big) # warm-up allocations of one replica
base = {r['name']: peak(r['id']) for r in api_containers()}
barrier = threading.Barrier(conc)
def one(i):
local = Client(a.base_url, timeout=120)
barrier.wait()
try:
x = local.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=big.png',
[201, 503], actor=alice, raw=big, headers={'Content-Type': 'image/png'})
return x['status'], x['elapsed']
except Failure as exc:
return str(exc)[-40:], 0
t0 = time.monotonic()
with ThreadPoolExecutor(conc) as pool:
res = list(pool.map(one, range(conc)))
wall = time.monotonic() - t0
after = {r['name']: peak(r['id']) for r in api_containers()}
lat = sorted(e for s, e in res if s == 201)
row = {'concurrency': conc, 'repeat': rep + 1, 'file_bytes': len(big),
'ok': sum(1 for s, _ in res if s == 201), 'status_503': sum(1 for s, _ in res if s == 503),
'other': [s for s, _ in res if s not in (201, 503)], 'wall_s': round(wall, 2),
'p50_ms': round(1000 * lat[len(lat) // 2]) if lat else None,
'max_ms': round(1000 * lat[-1]) if lat else None,
'peak_mib_per_replica': {k: round(v[0], 1) for k, v in after.items()},
'baseline_current_mib': {k: round(v[1], 1) for k, v in base.items()}}
row['max_peak_mib'] = max(row['peak_mib_per_replica'].values())
row['s3_peak_mib_since_start'] = round(peak(s3_id)[0], 1)
row['s3_restarted_during_run'] = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id) != s3_restarts
print(json.dumps(row, ensure_ascii=False), flush=True)
results.append(row)
return {'api_limit': '512M x 2 replicas, 1 CPU each', 'gomemlimit': '400MiB',
'upload_concurrency_per_replica': 8, 'decode_budget_mib': 128,
's3_limit': sh('docker', 'inspect', '-f', '{{.HostConfig.Memory}}', sh('docker', 'compose', 'ps', '-q', 's3')),
'runs': results}
def main():
p = argparse.ArgumentParser()
p.add_argument('cmd', choices=['private', 'failover-upload', 's3-outage', 'no-local-file', 'bootstrap', 'memory'])
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--output')
p.add_argument('--levels', type=int, nargs='+', default=[1, 10, 20, 40])
p.add_argument('--repeat', type=int, default=2)
a = p.parse_args()
fn = globals()['cmd_' + a.cmd.replace('-', '_')]
result = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False), 'result': fn(a)}
text = json.dumps(result, ensure_ascii=False, indent=2)
print(text)
if a.output:
Path(a.output).write_text(text + '\n', encoding='utf-8')
print('PASS: ' + a.cmd)
if __name__ == '__main__':
main()
+130
View File
@@ -0,0 +1,130 @@
import json
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure
PROJECT = 'mtusi-lab4'
def sh(*args, check=True):
r = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding='utf-8')
if check and r.returncode != 0:
raise RuntimeError('%s failed: %s' % (' '.join(args[:3]), r.stderr.strip()[:300]))
return r.stdout.strip()
def api_containers(all_states=False):
out = sh('docker', 'ps', *(['-a'] if all_states else []), '--no-trunc',
'--filter', 'label=com.docker.compose.project=' + PROJECT,
'--filter', 'label=com.docker.compose.service=api',
'--format', '{{json .}}')
rows = []
for line in out.splitlines():
d = json.loads(line)
rows.append({'id': d['ID'][:12], 'name': d['Names'], 'state': d['State'], 'status': d['Status']})
return sorted(rows, key=lambda r: r['name'])
def health(cid):
return sh('docker', 'inspect', '-f', '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}', cid)
def wait_healthy(cid, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if health(cid) == 'healthy':
return True
time.sleep(0.25)
return False
def ps_snapshot():
return sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}}\t{{.State}}\t{{.Status}}')
class WriteLoad:
def __init__(self, base_url, workers=8, timeout=2.0, pause=0.0):
self.base = base_url
self.workers = workers
self.timeout = timeout
self.pause = pause
self.events = []
self.confirmed = []
self.retries = 0
self._lock = threading.Lock()
self._stop = threading.Event()
self._threads = []
c = Client(base_url, timeout=10)
self.alice, self.bob, _eve, self.chat = c.bootstrap()
self.reader = c
def _worker(self, wid):
c = Client(self.base, timeout=self.timeout)
key = text = None
n = 0
while not self._stop.is_set():
if key is None:
key = uuid.uuid4().hex
text = 'load-%d-%d-%s' % (wid, n, key[:8])
start = time.monotonic()
try:
r = c.message(self.alice, self.chat, text, key=key)
with self._lock:
self.events.append((start, True, r['headers']['X-Instance-Id'], time.monotonic() - start))
self.confirmed.append(r['json'])
key = None
n += 1
except Failure as exc:
with self._lock:
self.events.append((start, False, str(exc)[-60:], time.monotonic() - start))
self.retries += 1
time.sleep(0.05)
if self.pause:
time.sleep(self.pause)
def start(self):
for i in range(self.workers):
t = threading.Thread(target=self._worker, args=(i,), daemon=True)
t.start()
self._threads.append(t)
self.t0 = time.monotonic()
def stop(self):
self._stop.set()
for t in self._threads:
t.join(timeout=15)
def window(self, a, b):
ev = [e for e in self.events if a <= e[0] < b]
ok = [e for e in ev if e[1]]
lat = sorted(e[3] for e in ok)
pct = lambda p: round(1000 * lat[min(len(lat) - 1, int(p * len(lat)))], 1) if lat else None
return {'seconds': round(b - a, 2), 'ok': len(ok), 'errors': len(ev) - len(ok),
'ok_rps': round(len(ok) / (b - a), 1) if b > a else None,
'p50_ms': pct(.5), 'p95_ms': pct(.95), 'p99_ms': pct(.99)}
def verify_history(self):
items, cursor = [], None
path = '/api/v1/chats/' + self.chat['id'] + '/messages?limit=100'
while True:
page = self.reader.call('GET', path + ('&cursor=' + cursor if cursor else ''),
actor=self.bob, schema='MessagePage')['json']
items += page['items']
cursor = page['next_cursor']
if not cursor:
break
by_id = {m['id']: m for m in items}
texts = [m['text'] for m in items]
lost = [m for m in self.confirmed if by_id.get(m['id']) != m]
return {'stored_messages': len(items), 'confirmed_201': len(self.confirmed),
'lost_confirmed': len(lost), 'duplicate_ids': len(items) - len(by_id),
'duplicate_texts': len(texts) - len(set(texts)),
'unique_confirmed': len({m['id'] for m in self.confirmed})}
+6
View File
@@ -0,0 +1,6 @@
@echo off
cd /d "%~dp0.."
docker compose stop lb api reconciler || exit /b 1
docker compose restart postgres valkey s3 || exit /b 1
docker compose up -d --no-recreate --wait postgres valkey s3 || exit /b 1
docker compose start api reconciler lb || exit /b 1
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
# Restart API and ALL state stores (PostgreSQL, Valkey, S3) keeping volumes.
set -eu
cd "$(dirname "$0")/.."
docker compose stop lb api reconciler
docker compose restart postgres valkey s3
# no recreation, only waits until the stores report healthy again
docker compose up -d --no-recreate --wait postgres valkey s3
docker compose start api reconciler lb
+10
View File
@@ -0,0 +1,10 @@
@echo off
setlocal
cd /d "%~dp0.."
if "%TARGET_INSTANCE_ID%"=="" exit /b 2
echo %TARGET_INSTANCE_ID%| findstr /r /x "[0-9a-f][0-9a-f]*" >nul || exit /b 2
set CID=
for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.project=mtusi-lab4" --filter "label=com.docker.compose.service=api"') do set CID=%%i
if "%CID%"=="" exit /b 3
docker stop -t 15 %CID% >nul || exit /b 1
echo stopped api container %CID%
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
*[!0-9a-f]*) echo "unexpected instance id format" >&2; exit 2 ;;
esac
cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \
--filter "label=com.docker.compose.project=mtusi-lab4" \
--filter "label=com.docker.compose.service=api")
[ -n "$cid" ] || { echo "no running api replica matches" >&2; exit 3; }
[ "$(printf '%s\n' "$cid" | wc -l)" -eq 1 ] || { echo "ambiguous id" >&2; exit 3; }
docker stop -t 15 "$cid" >/dev/null
echo "stopped api container $cid"
+254
View File
@@ -0,0 +1,254 @@
#!/usr/bin/env python3
"""Track 2: DB<->S3 consistency. Fault injection on both sides + reconciler cost.
python scripts/track2/run_track2.py faults --output evidence/track2/faults.json
python scripts/track2/run_track2.py cost --sizes 1000 5000 20000 --repeat 3 --output evidence/track2/cost.json
The periodic `reconciler` service is stopped for the duration (it would race the
experiment) and started again at the end. Synthetic data only.
"""
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
from lab4ops import NETWORK, PNG, TOOL_IMAGE, psql, s3tool, upload, winpath # noqa: E402
from lablib import sh # noqa: E402
from client import Client, check # noqa: E402
def s3put_bytes(key, data):
cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK,
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', key, '--stdin']
r = subprocess.run(cmd, input=data, capture_output=True)
check(r.returncode == 0, 's3 put failed')
return json.loads(r.stdout.decode().strip().splitlines()[-1])
def reconcile(*args):
"""One reconciler pass in a throwaway container; also returns its cgroup memory peak."""
out = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'sh', 'reconciler', '-c',
'/app/server reconcile ' + ' '.join(args) + ' > /tmp/r.json && cat /tmp/r.json '
'&& echo && cat /sys/fs/cgroup/memory.peak')
lines = [x for x in out.splitlines() if x.strip()]
rep = json.loads(lines[-2])
rep['container_memory_peak_mib'] = round(int(lines[-1]) / 2**20, 1)
return rep
def counts(prefix='attachments/'):
return {'s3_objects': s3tool('count', prefix)['count'],
'db_rows': int(psql('SELECT count(*) FROM attachments')),
'db_pending': int(psql("SELECT count(*) FROM attachments WHERE status = 'pending'")),
'db_missing_marked': int(psql('SELECT count(*) FROM attachments WHERE missing_at IS NOT NULL'))}
def summary(rep):
keys = ('mode', 'objects_scanned', 'rows_scanned', 'orphan_objects', 'fresh_skipped', 'stale_pending',
'missing_objects', 'size_mismatch', 'recovered', 'fixed', 'errors', 'total_ms')
return {k: rep[k] for k in keys}
def scoped(rep, prefix, invert=False):
out = {}
for i in rep['items']:
if i.get('key', '').startswith(prefix) == invert:
continue
k = i['kind'] + ('/skip' if i['action'].startswith('skip') else '')
out[k] = out.get(k, 0) + 1
return out
def insert_pending(att_id, chat, owner, key, age):
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height,"
" object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'crash.png', 'image/png', 100,"
" repeat('0', 64), 1, 1, '%s', 'pending', now() - interval '%s', now() - interval '%s')"
% (att_id, chat, owner, key, age, age))
def exists(key):
return s3tool('get', key)['status'] == 200
def cmd_faults(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
cid, prefix = chat['id'], 'attachments/' + chat['id'] + '/'
key_of = lambda att_id: psql("SELECT object_key FROM attachments WHERE id = '%s'" % att_id)
# healthy, referenced attachment (must survive everything)
u1 = upload(c, alice, chat)['json']
# referenced attachment whose object will be lost (S3-side fault)
u2 = upload(c, alice, chat)['json']
msg = c.message(alice, chat, '', attachments=[u1['id'], u2['id']])['json']
# unreferenced ready attachment whose metadata row will be lost (DB-side fault)
u3 = upload(c, alice, chat)['json']
u3_key = key_of(u3['id'])
f = {}
f['F1_orphan_object_no_row'] = prefix + str(uuid.uuid4())
s3tool('put', f['F1_orphan_object_no_row'])
psql("DELETE FROM attachments WHERE id = '%s'" % u3['id'])
f['F2_metadata_row_lost(U3)'] = u3_key
s3tool('delete', key_of(u2['id']))
f['F3_object_lost_of_used(U2)'] = key_of(u2['id'])
f4_id = str(uuid.uuid4())
f['F4_crash_after_put_before_ready'] = prefix + f4_id
insert_pending(f4_id, cid, alice['id'], f['F4_crash_after_put_before_ready'], '1 hour')
s3tool('put', f['F4_crash_after_put_before_ready'])
f5_id = str(uuid.uuid4())
f['F5_crash_before_put'] = prefix + f5_id
insert_pending(f5_id, cid, alice['id'], f['F5_crash_before_put'], '1 hour')
print('waiting %ds so that injected objects are older than grace=%s' % (a.wait, a.grace), flush=True)
time.sleep(a.wait)
# fresh state that must NOT be touched: an object being uploaded right now and its pending row
f7_id = str(uuid.uuid4())
f['F6_F7_fresh_upload_in_progress'] = prefix + f7_id
insert_pending(f7_id, cid, alice['id'], f['F6_F7_fresh_upload_in_progress'], '0 seconds')
s3tool('put', f['F6_F7_fresh_upload_in_progress'])
f8 = prefix + str(uuid.uuid4())
f['F8_fresh_orphan_object'] = f8
s3tool('put', f8)
steps = []
before = counts()
dry = reconcile('--grace', a.grace)
after_dry = counts()
steps.append({'step': 'dry-run', 'report': summary(dry), 'items': dry['items'], 'state_before': before,
'state_after': after_dry, 'dry_run_changed_nothing': before == after_dry})
check(before == after_dry, 'dry-run modified state')
mine = scoped(dry, prefix)
steps[-1]['this_experiment'] = mine
steps[-1]['other_findings_outside_experiment'] = scoped(dry, prefix, invert=True)
check(mine == {'orphan_object': 2, 'stale_pending': 2, 'missing_object': 1, 'orphan_object/skip': 1},
'detection mismatch: %s' % mine)
ap = reconcile('--apply', '--grace', a.grace)
steps.append({'step': 'apply #1', 'report': summary(ap), 'this_experiment': scoped(ap, prefix),
'items': [i for i in ap['items'] if i.get('key', '').startswith(prefix)], 'state_after': counts()})
again = reconcile('--apply', '--grace', a.grace)
steps.append({'step': 'apply #2 (repeat)', 'report': summary(again), 'this_experiment': scoped(again, prefix),
'state_after': counts()})
check(again['fixed'] == 0 and again['errors'] == 0, 'second apply was not a no-op')
d1 = c.call('GET', '/api/v1/attachments/' + u1['id'] + '/content', actor=bob)
msgs = c.call('GET', '/api/v1/chats/' + cid + '/messages?limit=100', actor=bob, schema='MessagePage')['json']['items']
verify = {
'U1_used_healthy_download_sha_ok': hashlib.sha256(d1['bytes']).hexdigest() == u1['sha256'],
'U2_metadata_kept': c.call('GET', '/api/v1/attachments/' + u2['id'], actor=bob)['status'],
'U2_content_status': c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', 404, actor=bob)['status'],
'U2_marked_missing': psql("SELECT missing_at IS NOT NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't',
'message_still_references_U1_U2': [m for m in msgs if m['id'] == msg['id']][0]['attachment_ids'] == [u1['id'], u2['id']],
'F1_orphan_deleted': not exists(f['F1_orphan_object_no_row']),
'F2_orphan_of_lost_row_deleted': not exists(u3_key),
'F4_row_and_object_deleted': not exists(f['F4_crash_after_put_before_ready'])
and psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f4_id) == '0',
'F5_row_deleted': psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f5_id) == '0',
'F7_fresh_pending_row_kept': psql("SELECT status FROM attachments WHERE id = '%s'" % f7_id) == 'pending',
'F7_fresh_object_kept': exists(f['F6_F7_fresh_upload_in_progress']),
'F8_fresh_orphan_kept': exists(f8),
}
for k, v in verify.items():
check(v is True or v in (200, 404), k)
# restore U2 from a "backup" (same bytes): the reconciler clears the missing mark
s3put_bytes(key_of(u2['id']), PNG)
rec = reconcile('--apply', '--grace', a.grace)
d2 = c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', actor=bob)
verify['U2_restored_and_unmarked'] = (rec['recovered'] == 1 and
hashlib.sha256(d2['bytes']).hexdigest() == u2['sha256'] and
psql("SELECT missing_at IS NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't')
steps.append({'step': 'apply after restoring U2 object', 'report': summary(rec)})
check(verify['U2_restored_and_unmarked'], 'restore not detected')
# default production grace (15m): the same kind of fresh garbage is only reported as skipped
prod = reconcile('--grace', '15m')
steps.append({'step': 'dry-run with production grace 15m', 'report': summary(prod)})
# clean the fresh leftovers with the regular procedure so the next run starts clean
return {'grace_used': a.grace, 'faults': f, 'steps': steps, 'verify': verify}
def cmd_cost(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
run = uuid.uuid4().hex[:8]
keys_file = ROOT / 'evidence/track2/.bench-keys.txt'
created = 0
results = []
try:
for n in sorted(a.sizes):
add = n - created
keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created, n)]
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,"
" width, height, object_key, status, created_at, updated_at)"
" SELECT gen_random_uuid(), '%s', '%s', 'bench.png', 'image/png', 100, repeat('0', 64), 1, 1,"
" 'attachments/bench-%s/' || lpad(i::text, 7, '0'), 'ready', now() - interval '1 day', now()"
" FROM generate_series(%d, %d) AS i" % (chat['id'], alice['id'], run, created, n - 1))
t0 = time.monotonic()
put = json.loads(subprocess.run(
['docker', 'run', '--rm', '-i', '--network', NETWORK, '-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I',
'/tool/s3tool.py', 'bulk-put', '--size', '100', '--workers', '32'],
input='\n'.join(keys), capture_output=True, text=True).stdout.strip().splitlines()[-1])
check(put['ok'] == add, 'bulk put incomplete: %s' % put)
created = n
print('populated %d (+%d in %.1fs)' % (n, add, time.monotonic() - t0), flush=True)
for r in range(a.repeat):
rep = reconcile('--grace', '15m')
row = {'bench_objects': n, 'repeat': r + 1, 'objects_scanned': rep['objects_scanned'],
'rows_scanned': rep['rows_scanned'], 'list_ms': rep['list_ms'], 'db_ms': rep['db_ms'],
'total_ms': rep['total_ms'], 'per_object_us': round(rep['per_object_us'], 1),
'container_memory_peak_mib': rep['container_memory_peak_mib'],
'findings': rep['orphan_objects'] + rep['stale_pending'] + rep['missing_objects']}
print(json.dumps(row), flush=True)
results.append(row)
finally:
if created:
keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created)]
psql("DELETE FROM attachments WHERE object_key LIKE 'attachments/bench-%s/%%'" % run)
subprocess.run(['docker', 'run', '--rm', '-i', '--network', NETWORK,
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I',
'/tool/s3tool.py', 'bulk-delete', '--workers', '32'],
input='\n'.join(keys), capture_output=True, text=True)
if keys_file.exists():
keys_file.unlink()
return {'object_size_bytes': 100, 'reconciler_limits': '0.5 CPU, 128M', 's3': 'SeaweedFS 3.80, 1 CPU, 1G',
'runs': results}
def main():
p = argparse.ArgumentParser()
p.add_argument('cmd', choices=['faults', 'cost'])
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--grace', default='1m')
p.add_argument('--wait', type=int, default=65)
p.add_argument('--sizes', type=int, nargs='+', default=[1000, 5000, 20000])
p.add_argument('--repeat', type=int, default=3)
p.add_argument('--output')
a = p.parse_args()
sh('docker', 'compose', 'stop', 'reconciler')
try:
res = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False),
'result': (cmd_faults if a.cmd == 'faults' else cmd_cost)(a)}
finally:
sh('docker', 'compose', 'start', 'reconciler')
text = json.dumps(res, ensure_ascii=False, indent=2)
if a.output:
Path(a.output).parent.mkdir(parents=True, exist_ok=True)
Path(a.output).write_text(text + '\n', encoding='utf-8')
print(text if a.cmd == 'faults' else json.dumps(res['result']['runs'][-1]))
print('PASS: track2 ' + a.cmd)
if __name__ == '__main__':
main()
+143
View File
@@ -0,0 +1,143 @@
#!/usr/bin/env python3
"""Minimal S3 client (stdlib, SigV4) for checks and fault injection OUTSIDE the API.
Runs inside a throwaway container on the compose `data` network (S3 is not
published to the host), see run_in_net() in lab4ops.py. Credentials are read
from mounted secret files and never printed.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import datetime as dt
import hashlib
import hmac
import http.client
import json
import os
import sys
import threading
from urllib.parse import quote
import xml.etree.ElementTree as ET
HOST = os.environ.get('S3_HOST', 's3:8333')
REGION = os.environ.get('S3_REGION', 'us-east-1')
BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments')
SECRETS = os.environ.get('SECRETS_DIR', '/secrets')
NS = '{http://s3.amazonaws.com/doc/2006-03-01/}'
def creds(identity):
if identity == 'anonymous':
return None
prefix = 's3_admin_' if identity == 'admin' else 's3_'
read = lambda n: open(os.path.join(SECRETS, prefix + n)).read().strip()
return read('access_key_id'), read('secret_access_key')
def _h(key, msg):
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
_local = threading.local()
def request(method, path, identity='api', body=b'', query=None):
query = query or {}
headers = {'host': HOST}
payload_hash = hashlib.sha256(body).hexdigest()
c = creds(identity)
if c:
now = dt.datetime.now(dt.timezone.utc)
amzdate, date = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d')
headers.update({'x-amz-content-sha256': payload_hash, 'x-amz-date': amzdate})
signed = ';'.join(sorted(headers))
cq = '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
creq = '\n'.join([method, quote(path, safe='/-_.~'), cq,
''.join('%s:%s\n' % (k, headers[k]) for k in sorted(headers)), signed, payload_hash])
scope = '%s/%s/s3/aws4_request' % (date, REGION)
sts = '\n'.join(['AWS4-HMAC-SHA256', amzdate, scope, hashlib.sha256(creq.encode()).hexdigest()])
k = _h(_h(_h(_h(('AWS4' + c[1]).encode(), date), REGION), 's3'), 'aws4_request')
sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest()
headers['authorization'] = 'AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s' % (
c[0], scope, signed, sig)
url = quote(path, safe='/-_.~')
if query:
url += '?' + '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
conn = getattr(_local, 'conn', None)
if conn is None:
conn = _local.conn = http.client.HTTPConnection(HOST, timeout=30)
try:
conn.request(method, url, body=body or None, headers=headers)
r = conn.getresponse()
return r.status, r.read()
except (OSError, http.client.HTTPException):
_local.conn = None
raise
def obj(key):
return '/' + BUCKET + '/' + key
def list_keys(prefix, identity='api'):
keys, token = [], None
while True:
q = {'list-type': '2', 'prefix': prefix, 'max-keys': '1000'}
if token:
q['continuation-token'] = token
st, body = request('GET', '/' + BUCKET, identity, query=q)
if st != 200:
raise SystemExit('list failed: HTTP %d' % st)
root = ET.fromstring(body)
keys += [e.find(NS + 'Key').text for e in root.findall(NS + 'Contents')]
if root.findtext(NS + 'IsTruncated') != 'true':
return keys
token = root.findtext(NS + 'NextContinuationToken')
def main():
p = argparse.ArgumentParser()
p.add_argument('cmd', choices=['get', 'put', 'delete', 'create-bucket', 'list', 'count', 'bulk-put', 'bulk-delete'])
p.add_argument('arg', nargs='?', default='')
p.add_argument('--as', dest='identity', default='api', choices=['api', 'admin', 'anonymous'])
p.add_argument('--size', type=int, default=100)
p.add_argument('--workers', type=int, default=32)
p.add_argument('--stdin', action='store_true', help='put: body from stdin instead of random bytes')
a = p.parse_args()
out = {'cmd': a.cmd, 'as': a.identity}
if a.cmd == 'get':
st, body = request('GET', obj(a.arg), a.identity)
out.update(status=st, bytes=len(body) if st == 200 else 0,
sha256=hashlib.sha256(body).hexdigest() if st == 200 else None)
elif a.cmd == 'put':
body = sys.stdin.buffer.read() if a.stdin else os.urandom(a.size)
st, _ = request('PUT', obj(a.arg), a.identity, body=body)
out.update(status=st)
elif a.cmd == 'delete':
st, _ = request('DELETE', obj(a.arg), a.identity)
out.update(status=st)
elif a.cmd == 'create-bucket':
st, _ = request('PUT', '/' + a.arg, a.identity)
out.update(status=st)
elif a.cmd == 'list':
out.update(keys=list_keys(a.arg, a.identity))
elif a.cmd == 'count':
out.update(count=len(list_keys(a.arg, a.identity)))
elif a.cmd in ('bulk-put', 'bulk-delete'):
keys = [k.strip() for k in sys.stdin if k.strip()]
data = b'x' * a.size
def one(k):
if a.cmd == 'bulk-put':
return request('PUT', obj(k), a.identity, body=data)[0]
return request('DELETE', obj(k), a.identity)[0]
start = dt.datetime.now()
with ThreadPoolExecutor(a.workers) as pool:
codes = list(pool.map(one, keys))
out.update(count=len(keys), ok=sum(1 for c in codes if c in (200, 204)),
seconds=round((dt.datetime.now() - start).total_seconds(), 2))
print(json.dumps(out))
if __name__ == '__main__':
main()