forked from mtusi2026/mtusi2026lab4
Fixed
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import secrets
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / 'secrets'
|
||||
|
||||
|
||||
def write(name, value):
|
||||
path = SECRETS / name
|
||||
if path.exists():
|
||||
print('keep secrets/' + name)
|
||||
return path.read_text().strip()
|
||||
path.write_bytes((value + '\n').encode('utf-8'))
|
||||
print('created secrets/' + name)
|
||||
return value
|
||||
|
||||
|
||||
def main():
|
||||
SECRETS.mkdir(exist_ok=True)
|
||||
pg = write('pg_password', secrets.token_urlsafe(24))
|
||||
write('database_url', 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' % pg)
|
||||
vk_conf = SECRETS / 'valkey_auth.conf'
|
||||
if vk_conf.exists():
|
||||
vk = vk_conf.read_text().split()[-1]
|
||||
print('keep secrets/valkey_auth.conf')
|
||||
else:
|
||||
vk = secrets.token_urlsafe(24)
|
||||
write('valkey_auth.conf', 'requirepass ' + vk)
|
||||
write('cache_url', 'redis://:%s@valkey:6379/0' % vk)
|
||||
# Two S3 identities: admin (bucket bootstrap only) and api (read/write/list
|
||||
# of the one bucket). SeaweedFS has no anonymous identity -> bucket is private.
|
||||
admin_id = write('s3_admin_access_key_id', 'admin' + secrets.token_hex(8))
|
||||
admin_secret = write('s3_admin_secret_access_key', secrets.token_urlsafe(30))
|
||||
api_id = write('s3_access_key_id', 'api' + secrets.token_hex(8))
|
||||
api_secret = write('s3_secret_access_key', secrets.token_urlsafe(30))
|
||||
bucket = 'messenger-attachments'
|
||||
identities = {'identities': [
|
||||
{'name': 'bootstrap-admin',
|
||||
'credentials': [{'accessKey': admin_id, 'secretKey': admin_secret}],
|
||||
'actions': ['Admin', 'Read', 'Write', 'List', 'Tagging']},
|
||||
{'name': 'messenger-api',
|
||||
'credentials': [{'accessKey': api_id, 'secretKey': api_secret}],
|
||||
'actions': ['Read:' + bucket, 'Write:' + bucket, 'List:' + bucket]},
|
||||
]}
|
||||
write('s3_identities.json', json.dumps(identities, indent=2))
|
||||
env = ROOT / '.env'
|
||||
if not env.exists():
|
||||
shutil.copy(ROOT / '.env.example', env)
|
||||
print('created .env from .env.example')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator checks for lab4 that need Docker access (run on the host, explicitly).
|
||||
|
||||
python scripts/lab4ops.py private # bucket privacy outside the API
|
||||
python scripts/lab4ops.py failover-upload # upload -> stop that replica -> download via the other
|
||||
python scripts/lab4ops.py s3-outage # stop S3: uploads/downloads 503, text path keeps working
|
||||
python scripts/lab4ops.py no-local-file # API containers have no volume and no stored original
|
||||
python scripts/lab4ops.py bootstrap # migrate + bucket bootstrap are idempotent
|
||||
python scripts/lab4ops.py memory # peak RSS under concurrent uploads
|
||||
|
||||
Each command prints JSON and, with --output, stores it in evidence/.
|
||||
Synthetic data only; tokens/credentials are never printed.
|
||||
"""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
sys.path.insert(0, str(ROOT / 'student-tests'))
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
import imagegen # noqa: E402
|
||||
from lablib import api_containers, sh, wait_healthy # noqa: E402
|
||||
|
||||
PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
NETWORK = 'mtusi-lab4_data'
|
||||
TOOL_IMAGE = 'python:3.12.8-alpine3.21'
|
||||
|
||||
|
||||
def winpath(p):
|
||||
s = str(p)
|
||||
return s.replace('\\', '/')
|
||||
|
||||
|
||||
def s3tool(*args, stdin=None):
|
||||
"""Run s3tool.py in a throwaway container on the data network (S3 is not published)."""
|
||||
cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK,
|
||||
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
|
||||
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
|
||||
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', *args]
|
||||
r = subprocess.run(cmd, input=stdin, capture_output=True, text=True, encoding='utf-8')
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError('s3tool %s failed: %s' % (args[0], (r.stdout + r.stderr).strip()[-300:]))
|
||||
return json.loads(r.stdout.strip().splitlines()[-1])
|
||||
|
||||
|
||||
def psql(sql):
|
||||
return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
|
||||
'-At', '-v', 'ON_ERROR_STOP=1', '-c', sql)
|
||||
|
||||
|
||||
def upload(c, actor, chat, data=PNG, name='sample.png', expected=201):
|
||||
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + name, expected,
|
||||
actor=actor, raw=data, headers={'Content-Type': 'image/png'},
|
||||
schema='Attachment' if expected == 201 else None)
|
||||
|
||||
|
||||
def cmd_private(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
att = upload(c, alice, chat)['json']
|
||||
key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % att['id'])
|
||||
res = {
|
||||
'object_key_shape': 'attachments/<chat_id>/<attachment_id> (server generated)',
|
||||
'key_contains_filename': 'sample' in key,
|
||||
'anonymous_get': s3tool('get', key, '--as', 'anonymous')['status'],
|
||||
'anonymous_list': None,
|
||||
'api_identity_get': s3tool('get', key)['status'],
|
||||
'api_identity_get_sha256_matches': s3tool('get', key)['sha256'] == att['sha256'],
|
||||
'api_identity_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket')['status'],
|
||||
'anonymous_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket', '--as', 'anonymous')['status'],
|
||||
'published_s3_ports': sh('docker', 'compose', 'port', 's3', '8333', check=False) or None,
|
||||
'outsider_via_api_metadata': c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=eve)['status'],
|
||||
'outsider_via_api_content': c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=eve)['status'],
|
||||
}
|
||||
try:
|
||||
s3tool('list', 'attachments/', '--as', 'anonymous')
|
||||
res['anonymous_list'] = 200
|
||||
except RuntimeError as exc:
|
||||
res['anonymous_list'] = 'denied (%s)' % str(exc).split(':')[-1].strip()
|
||||
# the API identity is bound to ONE bucket: the admin creates another one, the API cannot write there
|
||||
s3tool('create-bucket', 'lab4-other-bucket', '--as', 'admin')
|
||||
res['api_identity_put_other_bucket'] = subprocess.run(
|
||||
['docker', 'run', '--rm', '-e', 'S3_BUCKET=lab4-other-bucket', '--network', NETWORK,
|
||||
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
|
||||
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', 'probe'],
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
check(res['anonymous_get'] == 403 and res['api_identity_get'] == 200, 'unexpected bucket privacy')
|
||||
check(res['api_identity_create_bucket'] == 403 and '"status": 403' in res['api_identity_put_other_bucket'],
|
||||
'api identity has more rights than expected')
|
||||
return res
|
||||
|
||||
|
||||
def cmd_failover_upload(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
r = upload(c, alice, chat)
|
||||
att, served_by = r['json'], r['headers']['X-Instance-Id']
|
||||
target = [x for x in api_containers() if x['id'].startswith(served_by[:12])]
|
||||
check(len(target) == 1, 'cannot map X-Instance-Id to one api container')
|
||||
before = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}')
|
||||
sh('docker', 'stop', '-t', '15', target[0]['id'])
|
||||
t0 = time.monotonic()
|
||||
reads = []
|
||||
try:
|
||||
for _ in range(10):
|
||||
try:
|
||||
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
|
||||
reads.append({'instance': d['headers']['X-Instance-Id'],
|
||||
'sha256_ok': hashlib.sha256(d['bytes']).hexdigest() == att['sha256']})
|
||||
except Failure as exc:
|
||||
reads.append({'error': str(exc)[-60:]})
|
||||
time.sleep(0.3)
|
||||
during = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}')
|
||||
finally:
|
||||
sh('docker', 'start', target[0]['id'])
|
||||
wait_healthy(target[0]['id'])
|
||||
ok = [x for x in reads if x.get('sha256_ok')]
|
||||
check(len(ok) >= 8 and all(x['instance'] != served_by for x in ok), 'download did not move to the other replica')
|
||||
return {'uploaded_via': served_by, 'stopped_container': target[0]['name'], 'reads_after_stop': reads,
|
||||
'seconds_observed': round(time.monotonic() - t0, 1), 'ps_before': before, 'ps_during': during}
|
||||
|
||||
|
||||
def cmd_s3_outage(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
att = upload(c, alice, chat)['json']
|
||||
pending_before = psql("SELECT count(*) FROM attachments WHERE status = 'pending'")
|
||||
sh('docker', 'compose', 'stop', 's3')
|
||||
res = {}
|
||||
try:
|
||||
u = upload(c, alice, chat, expected=503)
|
||||
res['upload_status'], res['upload_ms'] = u['status'], round(u['elapsed'] * 1000)
|
||||
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob)
|
||||
res['download_status'], res['download_ms'] = d['status'], round(d['elapsed'] * 1000)
|
||||
res['metadata_status'] = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob)['status']
|
||||
res['text_message_status'] = c.message(alice, chat, 'S3 лежит, текст работает')['status']
|
||||
res['ready_status'] = c.call('GET', '/health/ready')['status']
|
||||
# expected 0: compensation deletes the pending row; a possible object is an orphan for the reconciler
|
||||
res['pending_rows_left'] = int(psql(
|
||||
"SELECT count(*) FROM attachments WHERE status = 'pending'")) - int(pending_before)
|
||||
finally:
|
||||
sh('docker', 'compose', 'up', '-d', '--no-recreate', '--wait', 's3')
|
||||
t0 = time.monotonic()
|
||||
while True:
|
||||
try:
|
||||
upload(c, alice, chat)
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic() - t0 < 60, 'uploads did not recover after S3 start')
|
||||
time.sleep(0.5)
|
||||
res['upload_recovered_after_s3_healthy_s'] = round(time.monotonic() - t0, 1)
|
||||
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
|
||||
res['old_file_intact'] = hashlib.sha256(d['bytes']).hexdigest() == att['sha256']
|
||||
check(res['upload_status'] == 503 and res['text_message_status'] == 201 and res['ready_status'] == 200,
|
||||
'S3 outage is not isolated to attachment endpoints')
|
||||
return res
|
||||
|
||||
|
||||
def cmd_no_local_file(a):
|
||||
out = {}
|
||||
for r in api_containers():
|
||||
mounts = json.loads(sh('docker', 'inspect', '-f', '{{json .Mounts}}', r['id']))
|
||||
ro = sh('docker', 'inspect', '-f', '{{.HostConfig.ReadonlyRootfs}}', r['id'])
|
||||
probe = sh('docker', 'exec', r['id'], 'sh', '-c', 'touch /app/probe 2>&1 || true')
|
||||
# Docker itself bind-mounts /etc/hosts, /etc/hostname, /etc/resolv.conf
|
||||
files = sh('docker', 'exec', r['id'], 'sh', '-c',
|
||||
'find / -xdev -type f -newer /app/server ! -path "/proc/*" ! -path "/sys/*" '
|
||||
'! -path "/run/secrets/*" ! -name hosts ! -name hostname ! -name resolv.conf '
|
||||
'! -name .dockerenv 2>/dev/null | head -20')
|
||||
out[r['name']] = {'readonly_rootfs': ro == 'true', 'write_probe': probe,
|
||||
'mounts': [m.get('Destination') for m in mounts],
|
||||
'files_written_since_start': files.splitlines()}
|
||||
check(ro == 'true' and not files.strip(), 'API container can store files locally')
|
||||
return out
|
||||
|
||||
|
||||
def cmd_bootstrap(a):
|
||||
runs = []
|
||||
for _ in range(2):
|
||||
for svc in ('migrate', 's3-init'):
|
||||
log = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', svc)
|
||||
runs.append({'service': svc, 'last_log': json.loads(log.strip().splitlines()[-1])})
|
||||
return runs
|
||||
|
||||
|
||||
def peak(cid):
|
||||
v = sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory.current').split()
|
||||
return int(v[0]) / 2**20, int(v[1]) / 2**20
|
||||
|
||||
|
||||
def cmd_memory(a):
|
||||
big = imagegen.near_limit_png()
|
||||
c = Client(a.base_url, timeout=120)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
results = []
|
||||
for conc in a.levels:
|
||||
for rep in range(a.repeat):
|
||||
sh('docker', 'compose', 'restart', 'api')
|
||||
for r in api_containers():
|
||||
check(wait_healthy(r['id'], 60), 'api not healthy')
|
||||
time.sleep(1)
|
||||
s3_id = sh('docker', 'compose', 'ps', '-q', 's3')
|
||||
s3_restarts = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id)
|
||||
upload(c, alice, chat, big) # warm-up allocations of one replica
|
||||
base = {r['name']: peak(r['id']) for r in api_containers()}
|
||||
barrier = threading.Barrier(conc)
|
||||
|
||||
def one(i):
|
||||
local = Client(a.base_url, timeout=120)
|
||||
barrier.wait()
|
||||
try:
|
||||
x = local.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=big.png',
|
||||
[201, 503], actor=alice, raw=big, headers={'Content-Type': 'image/png'})
|
||||
return x['status'], x['elapsed']
|
||||
except Failure as exc:
|
||||
return str(exc)[-40:], 0
|
||||
|
||||
t0 = time.monotonic()
|
||||
with ThreadPoolExecutor(conc) as pool:
|
||||
res = list(pool.map(one, range(conc)))
|
||||
wall = time.monotonic() - t0
|
||||
after = {r['name']: peak(r['id']) for r in api_containers()}
|
||||
lat = sorted(e for s, e in res if s == 201)
|
||||
row = {'concurrency': conc, 'repeat': rep + 1, 'file_bytes': len(big),
|
||||
'ok': sum(1 for s, _ in res if s == 201), 'status_503': sum(1 for s, _ in res if s == 503),
|
||||
'other': [s for s, _ in res if s not in (201, 503)], 'wall_s': round(wall, 2),
|
||||
'p50_ms': round(1000 * lat[len(lat) // 2]) if lat else None,
|
||||
'max_ms': round(1000 * lat[-1]) if lat else None,
|
||||
'peak_mib_per_replica': {k: round(v[0], 1) for k, v in after.items()},
|
||||
'baseline_current_mib': {k: round(v[1], 1) for k, v in base.items()}}
|
||||
row['max_peak_mib'] = max(row['peak_mib_per_replica'].values())
|
||||
row['s3_peak_mib_since_start'] = round(peak(s3_id)[0], 1)
|
||||
row['s3_restarted_during_run'] = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id) != s3_restarts
|
||||
print(json.dumps(row, ensure_ascii=False), flush=True)
|
||||
results.append(row)
|
||||
return {'api_limit': '512M x 2 replicas, 1 CPU each', 'gomemlimit': '400MiB',
|
||||
'upload_concurrency_per_replica': 8, 'decode_budget_mib': 128,
|
||||
's3_limit': sh('docker', 'inspect', '-f', '{{.HostConfig.Memory}}', sh('docker', 'compose', 'ps', '-q', 's3')),
|
||||
'runs': results}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('cmd', choices=['private', 'failover-upload', 's3-outage', 'no-local-file', 'bootstrap', 'memory'])
|
||||
p.add_argument('--base-url', default='http://localhost:8080')
|
||||
p.add_argument('--output')
|
||||
p.add_argument('--levels', type=int, nargs='+', default=[1, 10, 20, 40])
|
||||
p.add_argument('--repeat', type=int, default=2)
|
||||
a = p.parse_args()
|
||||
fn = globals()['cmd_' + a.cmd.replace('-', '_')]
|
||||
result = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
||||
'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False), 'result': fn(a)}
|
||||
text = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
print(text)
|
||||
if a.output:
|
||||
Path(a.output).write_text(text + '\n', encoding='utf-8')
|
||||
print('PASS: ' + a.cmd)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,130 @@
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure
|
||||
|
||||
PROJECT = 'mtusi-lab4'
|
||||
|
||||
|
||||
def sh(*args, check=True):
|
||||
r = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding='utf-8')
|
||||
if check and r.returncode != 0:
|
||||
raise RuntimeError('%s failed: %s' % (' '.join(args[:3]), r.stderr.strip()[:300]))
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
def api_containers(all_states=False):
|
||||
out = sh('docker', 'ps', *(['-a'] if all_states else []), '--no-trunc',
|
||||
'--filter', 'label=com.docker.compose.project=' + PROJECT,
|
||||
'--filter', 'label=com.docker.compose.service=api',
|
||||
'--format', '{{json .}}')
|
||||
rows = []
|
||||
for line in out.splitlines():
|
||||
d = json.loads(line)
|
||||
rows.append({'id': d['ID'][:12], 'name': d['Names'], 'state': d['State'], 'status': d['Status']})
|
||||
return sorted(rows, key=lambda r: r['name'])
|
||||
|
||||
|
||||
def health(cid):
|
||||
return sh('docker', 'inspect', '-f', '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}', cid)
|
||||
|
||||
|
||||
def wait_healthy(cid, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if health(cid) == 'healthy':
|
||||
return True
|
||||
time.sleep(0.25)
|
||||
return False
|
||||
|
||||
|
||||
def ps_snapshot():
|
||||
return sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}}\t{{.State}}\t{{.Status}}')
|
||||
|
||||
|
||||
class WriteLoad:
|
||||
|
||||
def __init__(self, base_url, workers=8, timeout=2.0, pause=0.0):
|
||||
self.base = base_url
|
||||
self.workers = workers
|
||||
self.timeout = timeout
|
||||
self.pause = pause
|
||||
self.events = []
|
||||
self.confirmed = []
|
||||
self.retries = 0
|
||||
self._lock = threading.Lock()
|
||||
self._stop = threading.Event()
|
||||
self._threads = []
|
||||
c = Client(base_url, timeout=10)
|
||||
self.alice, self.bob, _eve, self.chat = c.bootstrap()
|
||||
self.reader = c
|
||||
|
||||
def _worker(self, wid):
|
||||
c = Client(self.base, timeout=self.timeout)
|
||||
key = text = None
|
||||
n = 0
|
||||
while not self._stop.is_set():
|
||||
if key is None:
|
||||
key = uuid.uuid4().hex
|
||||
text = 'load-%d-%d-%s' % (wid, n, key[:8])
|
||||
start = time.monotonic()
|
||||
try:
|
||||
r = c.message(self.alice, self.chat, text, key=key)
|
||||
with self._lock:
|
||||
self.events.append((start, True, r['headers']['X-Instance-Id'], time.monotonic() - start))
|
||||
self.confirmed.append(r['json'])
|
||||
key = None
|
||||
n += 1
|
||||
except Failure as exc:
|
||||
with self._lock:
|
||||
self.events.append((start, False, str(exc)[-60:], time.monotonic() - start))
|
||||
self.retries += 1
|
||||
time.sleep(0.05)
|
||||
if self.pause:
|
||||
time.sleep(self.pause)
|
||||
|
||||
def start(self):
|
||||
for i in range(self.workers):
|
||||
t = threading.Thread(target=self._worker, args=(i,), daemon=True)
|
||||
t.start()
|
||||
self._threads.append(t)
|
||||
self.t0 = time.monotonic()
|
||||
|
||||
def stop(self):
|
||||
self._stop.set()
|
||||
for t in self._threads:
|
||||
t.join(timeout=15)
|
||||
|
||||
def window(self, a, b):
|
||||
ev = [e for e in self.events if a <= e[0] < b]
|
||||
ok = [e for e in ev if e[1]]
|
||||
lat = sorted(e[3] for e in ok)
|
||||
pct = lambda p: round(1000 * lat[min(len(lat) - 1, int(p * len(lat)))], 1) if lat else None
|
||||
return {'seconds': round(b - a, 2), 'ok': len(ok), 'errors': len(ev) - len(ok),
|
||||
'ok_rps': round(len(ok) / (b - a), 1) if b > a else None,
|
||||
'p50_ms': pct(.5), 'p95_ms': pct(.95), 'p99_ms': pct(.99)}
|
||||
|
||||
def verify_history(self):
|
||||
items, cursor = [], None
|
||||
path = '/api/v1/chats/' + self.chat['id'] + '/messages?limit=100'
|
||||
while True:
|
||||
page = self.reader.call('GET', path + ('&cursor=' + cursor if cursor else ''),
|
||||
actor=self.bob, schema='MessagePage')['json']
|
||||
items += page['items']
|
||||
cursor = page['next_cursor']
|
||||
if not cursor:
|
||||
break
|
||||
by_id = {m['id']: m for m in items}
|
||||
texts = [m['text'] for m in items]
|
||||
lost = [m for m in self.confirmed if by_id.get(m['id']) != m]
|
||||
return {'stored_messages': len(items), 'confirmed_201': len(self.confirmed),
|
||||
'lost_confirmed': len(lost), 'duplicate_ids': len(items) - len(by_id),
|
||||
'duplicate_texts': len(texts) - len(set(texts)),
|
||||
'unique_confirmed': len({m['id'] for m in self.confirmed})}
|
||||
@@ -0,0 +1,6 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
docker compose stop lb api reconciler || exit /b 1
|
||||
docker compose restart postgres valkey s3 || exit /b 1
|
||||
docker compose up -d --no-recreate --wait postgres valkey s3 || exit /b 1
|
||||
docker compose start api reconciler lb || exit /b 1
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
# Restart API and ALL state stores (PostgreSQL, Valkey, S3) keeping volumes.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop lb api reconciler
|
||||
docker compose restart postgres valkey s3
|
||||
# no recreation, only waits until the stores report healthy again
|
||||
docker compose up -d --no-recreate --wait postgres valkey s3
|
||||
docker compose start api reconciler lb
|
||||
@@ -0,0 +1,10 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
if "%TARGET_INSTANCE_ID%"=="" exit /b 2
|
||||
echo %TARGET_INSTANCE_ID%| findstr /r /x "[0-9a-f][0-9a-f]*" >nul || exit /b 2
|
||||
set CID=
|
||||
for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.project=mtusi-lab4" --filter "label=com.docker.compose.service=api"') do set CID=%%i
|
||||
if "%CID%"=="" exit /b 3
|
||||
docker stop -t 15 %CID% >nul || exit /b 1
|
||||
echo stopped api container %CID%
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!0-9a-f]*) echo "unexpected instance id format" >&2; exit 2 ;;
|
||||
esac
|
||||
cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \
|
||||
--filter "label=com.docker.compose.project=mtusi-lab4" \
|
||||
--filter "label=com.docker.compose.service=api")
|
||||
[ -n "$cid" ] || { echo "no running api replica matches" >&2; exit 3; }
|
||||
[ "$(printf '%s\n' "$cid" | wc -l)" -eq 1 ] || { echo "ambiguous id" >&2; exit 3; }
|
||||
docker stop -t 15 "$cid" >/dev/null
|
||||
echo "stopped api container $cid"
|
||||
@@ -0,0 +1,254 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 2: DB<->S3 consistency. Fault injection on both sides + reconciler cost.
|
||||
|
||||
python scripts/track2/run_track2.py faults --output evidence/track2/faults.json
|
||||
python scripts/track2/run_track2.py cost --sizes 1000 5000 20000 --repeat 3 --output evidence/track2/cost.json
|
||||
|
||||
The periodic `reconciler` service is stopped for the duration (it would race the
|
||||
experiment) and started again at the end. Synthetic data only.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
from lab4ops import NETWORK, PNG, TOOL_IMAGE, psql, s3tool, upload, winpath # noqa: E402
|
||||
from lablib import sh # noqa: E402
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
|
||||
def s3put_bytes(key, data):
|
||||
cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK,
|
||||
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
|
||||
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', key, '--stdin']
|
||||
r = subprocess.run(cmd, input=data, capture_output=True)
|
||||
check(r.returncode == 0, 's3 put failed')
|
||||
return json.loads(r.stdout.decode().strip().splitlines()[-1])
|
||||
|
||||
|
||||
def reconcile(*args):
|
||||
"""One reconciler pass in a throwaway container; also returns its cgroup memory peak."""
|
||||
out = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'sh', 'reconciler', '-c',
|
||||
'/app/server reconcile ' + ' '.join(args) + ' > /tmp/r.json && cat /tmp/r.json '
|
||||
'&& echo && cat /sys/fs/cgroup/memory.peak')
|
||||
lines = [x for x in out.splitlines() if x.strip()]
|
||||
rep = json.loads(lines[-2])
|
||||
rep['container_memory_peak_mib'] = round(int(lines[-1]) / 2**20, 1)
|
||||
return rep
|
||||
|
||||
|
||||
def counts(prefix='attachments/'):
|
||||
return {'s3_objects': s3tool('count', prefix)['count'],
|
||||
'db_rows': int(psql('SELECT count(*) FROM attachments')),
|
||||
'db_pending': int(psql("SELECT count(*) FROM attachments WHERE status = 'pending'")),
|
||||
'db_missing_marked': int(psql('SELECT count(*) FROM attachments WHERE missing_at IS NOT NULL'))}
|
||||
|
||||
|
||||
def summary(rep):
|
||||
keys = ('mode', 'objects_scanned', 'rows_scanned', 'orphan_objects', 'fresh_skipped', 'stale_pending',
|
||||
'missing_objects', 'size_mismatch', 'recovered', 'fixed', 'errors', 'total_ms')
|
||||
return {k: rep[k] for k in keys}
|
||||
|
||||
|
||||
def scoped(rep, prefix, invert=False):
|
||||
out = {}
|
||||
for i in rep['items']:
|
||||
if i.get('key', '').startswith(prefix) == invert:
|
||||
continue
|
||||
k = i['kind'] + ('/skip' if i['action'].startswith('skip') else '')
|
||||
out[k] = out.get(k, 0) + 1
|
||||
return out
|
||||
|
||||
|
||||
def insert_pending(att_id, chat, owner, key, age):
|
||||
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height,"
|
||||
" object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'crash.png', 'image/png', 100,"
|
||||
" repeat('0', 64), 1, 1, '%s', 'pending', now() - interval '%s', now() - interval '%s')"
|
||||
% (att_id, chat, owner, key, age, age))
|
||||
|
||||
|
||||
def exists(key):
|
||||
return s3tool('get', key)['status'] == 200
|
||||
|
||||
|
||||
def cmd_faults(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
cid, prefix = chat['id'], 'attachments/' + chat['id'] + '/'
|
||||
key_of = lambda att_id: psql("SELECT object_key FROM attachments WHERE id = '%s'" % att_id)
|
||||
|
||||
# healthy, referenced attachment (must survive everything)
|
||||
u1 = upload(c, alice, chat)['json']
|
||||
# referenced attachment whose object will be lost (S3-side fault)
|
||||
u2 = upload(c, alice, chat)['json']
|
||||
msg = c.message(alice, chat, '', attachments=[u1['id'], u2['id']])['json']
|
||||
# unreferenced ready attachment whose metadata row will be lost (DB-side fault)
|
||||
u3 = upload(c, alice, chat)['json']
|
||||
u3_key = key_of(u3['id'])
|
||||
|
||||
f = {}
|
||||
f['F1_orphan_object_no_row'] = prefix + str(uuid.uuid4())
|
||||
s3tool('put', f['F1_orphan_object_no_row'])
|
||||
psql("DELETE FROM attachments WHERE id = '%s'" % u3['id'])
|
||||
f['F2_metadata_row_lost(U3)'] = u3_key
|
||||
s3tool('delete', key_of(u2['id']))
|
||||
f['F3_object_lost_of_used(U2)'] = key_of(u2['id'])
|
||||
f4_id = str(uuid.uuid4())
|
||||
f['F4_crash_after_put_before_ready'] = prefix + f4_id
|
||||
insert_pending(f4_id, cid, alice['id'], f['F4_crash_after_put_before_ready'], '1 hour')
|
||||
s3tool('put', f['F4_crash_after_put_before_ready'])
|
||||
f5_id = str(uuid.uuid4())
|
||||
f['F5_crash_before_put'] = prefix + f5_id
|
||||
insert_pending(f5_id, cid, alice['id'], f['F5_crash_before_put'], '1 hour')
|
||||
|
||||
print('waiting %ds so that injected objects are older than grace=%s' % (a.wait, a.grace), flush=True)
|
||||
time.sleep(a.wait)
|
||||
# fresh state that must NOT be touched: an object being uploaded right now and its pending row
|
||||
f7_id = str(uuid.uuid4())
|
||||
f['F6_F7_fresh_upload_in_progress'] = prefix + f7_id
|
||||
insert_pending(f7_id, cid, alice['id'], f['F6_F7_fresh_upload_in_progress'], '0 seconds')
|
||||
s3tool('put', f['F6_F7_fresh_upload_in_progress'])
|
||||
f8 = prefix + str(uuid.uuid4())
|
||||
f['F8_fresh_orphan_object'] = f8
|
||||
s3tool('put', f8)
|
||||
|
||||
steps = []
|
||||
before = counts()
|
||||
dry = reconcile('--grace', a.grace)
|
||||
after_dry = counts()
|
||||
steps.append({'step': 'dry-run', 'report': summary(dry), 'items': dry['items'], 'state_before': before,
|
||||
'state_after': after_dry, 'dry_run_changed_nothing': before == after_dry})
|
||||
check(before == after_dry, 'dry-run modified state')
|
||||
mine = scoped(dry, prefix)
|
||||
steps[-1]['this_experiment'] = mine
|
||||
steps[-1]['other_findings_outside_experiment'] = scoped(dry, prefix, invert=True)
|
||||
check(mine == {'orphan_object': 2, 'stale_pending': 2, 'missing_object': 1, 'orphan_object/skip': 1},
|
||||
'detection mismatch: %s' % mine)
|
||||
|
||||
ap = reconcile('--apply', '--grace', a.grace)
|
||||
steps.append({'step': 'apply #1', 'report': summary(ap), 'this_experiment': scoped(ap, prefix),
|
||||
'items': [i for i in ap['items'] if i.get('key', '').startswith(prefix)], 'state_after': counts()})
|
||||
again = reconcile('--apply', '--grace', a.grace)
|
||||
steps.append({'step': 'apply #2 (repeat)', 'report': summary(again), 'this_experiment': scoped(again, prefix),
|
||||
'state_after': counts()})
|
||||
check(again['fixed'] == 0 and again['errors'] == 0, 'second apply was not a no-op')
|
||||
|
||||
d1 = c.call('GET', '/api/v1/attachments/' + u1['id'] + '/content', actor=bob)
|
||||
msgs = c.call('GET', '/api/v1/chats/' + cid + '/messages?limit=100', actor=bob, schema='MessagePage')['json']['items']
|
||||
verify = {
|
||||
'U1_used_healthy_download_sha_ok': hashlib.sha256(d1['bytes']).hexdigest() == u1['sha256'],
|
||||
'U2_metadata_kept': c.call('GET', '/api/v1/attachments/' + u2['id'], actor=bob)['status'],
|
||||
'U2_content_status': c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', 404, actor=bob)['status'],
|
||||
'U2_marked_missing': psql("SELECT missing_at IS NOT NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't',
|
||||
'message_still_references_U1_U2': [m for m in msgs if m['id'] == msg['id']][0]['attachment_ids'] == [u1['id'], u2['id']],
|
||||
'F1_orphan_deleted': not exists(f['F1_orphan_object_no_row']),
|
||||
'F2_orphan_of_lost_row_deleted': not exists(u3_key),
|
||||
'F4_row_and_object_deleted': not exists(f['F4_crash_after_put_before_ready'])
|
||||
and psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f4_id) == '0',
|
||||
'F5_row_deleted': psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f5_id) == '0',
|
||||
'F7_fresh_pending_row_kept': psql("SELECT status FROM attachments WHERE id = '%s'" % f7_id) == 'pending',
|
||||
'F7_fresh_object_kept': exists(f['F6_F7_fresh_upload_in_progress']),
|
||||
'F8_fresh_orphan_kept': exists(f8),
|
||||
}
|
||||
for k, v in verify.items():
|
||||
check(v is True or v in (200, 404), k)
|
||||
|
||||
# restore U2 from a "backup" (same bytes): the reconciler clears the missing mark
|
||||
s3put_bytes(key_of(u2['id']), PNG)
|
||||
rec = reconcile('--apply', '--grace', a.grace)
|
||||
d2 = c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', actor=bob)
|
||||
verify['U2_restored_and_unmarked'] = (rec['recovered'] == 1 and
|
||||
hashlib.sha256(d2['bytes']).hexdigest() == u2['sha256'] and
|
||||
psql("SELECT missing_at IS NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't')
|
||||
steps.append({'step': 'apply after restoring U2 object', 'report': summary(rec)})
|
||||
check(verify['U2_restored_and_unmarked'], 'restore not detected')
|
||||
|
||||
# default production grace (15m): the same kind of fresh garbage is only reported as skipped
|
||||
prod = reconcile('--grace', '15m')
|
||||
steps.append({'step': 'dry-run with production grace 15m', 'report': summary(prod)})
|
||||
# clean the fresh leftovers with the regular procedure so the next run starts clean
|
||||
return {'grace_used': a.grace, 'faults': f, 'steps': steps, 'verify': verify}
|
||||
|
||||
|
||||
def cmd_cost(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
run = uuid.uuid4().hex[:8]
|
||||
keys_file = ROOT / 'evidence/track2/.bench-keys.txt'
|
||||
created = 0
|
||||
results = []
|
||||
try:
|
||||
for n in sorted(a.sizes):
|
||||
add = n - created
|
||||
keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created, n)]
|
||||
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,"
|
||||
" width, height, object_key, status, created_at, updated_at)"
|
||||
" SELECT gen_random_uuid(), '%s', '%s', 'bench.png', 'image/png', 100, repeat('0', 64), 1, 1,"
|
||||
" 'attachments/bench-%s/' || lpad(i::text, 7, '0'), 'ready', now() - interval '1 day', now()"
|
||||
" FROM generate_series(%d, %d) AS i" % (chat['id'], alice['id'], run, created, n - 1))
|
||||
t0 = time.monotonic()
|
||||
put = json.loads(subprocess.run(
|
||||
['docker', 'run', '--rm', '-i', '--network', NETWORK, '-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
|
||||
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I',
|
||||
'/tool/s3tool.py', 'bulk-put', '--size', '100', '--workers', '32'],
|
||||
input='\n'.join(keys), capture_output=True, text=True).stdout.strip().splitlines()[-1])
|
||||
check(put['ok'] == add, 'bulk put incomplete: %s' % put)
|
||||
created = n
|
||||
print('populated %d (+%d in %.1fs)' % (n, add, time.monotonic() - t0), flush=True)
|
||||
for r in range(a.repeat):
|
||||
rep = reconcile('--grace', '15m')
|
||||
row = {'bench_objects': n, 'repeat': r + 1, 'objects_scanned': rep['objects_scanned'],
|
||||
'rows_scanned': rep['rows_scanned'], 'list_ms': rep['list_ms'], 'db_ms': rep['db_ms'],
|
||||
'total_ms': rep['total_ms'], 'per_object_us': round(rep['per_object_us'], 1),
|
||||
'container_memory_peak_mib': rep['container_memory_peak_mib'],
|
||||
'findings': rep['orphan_objects'] + rep['stale_pending'] + rep['missing_objects']}
|
||||
print(json.dumps(row), flush=True)
|
||||
results.append(row)
|
||||
finally:
|
||||
if created:
|
||||
keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created)]
|
||||
psql("DELETE FROM attachments WHERE object_key LIKE 'attachments/bench-%s/%%'" % run)
|
||||
subprocess.run(['docker', 'run', '--rm', '-i', '--network', NETWORK,
|
||||
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
|
||||
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I',
|
||||
'/tool/s3tool.py', 'bulk-delete', '--workers', '32'],
|
||||
input='\n'.join(keys), capture_output=True, text=True)
|
||||
if keys_file.exists():
|
||||
keys_file.unlink()
|
||||
return {'object_size_bytes': 100, 'reconciler_limits': '0.5 CPU, 128M', 's3': 'SeaweedFS 3.80, 1 CPU, 1G',
|
||||
'runs': results}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('cmd', choices=['faults', 'cost'])
|
||||
p.add_argument('--base-url', default='http://localhost:8080')
|
||||
p.add_argument('--grace', default='1m')
|
||||
p.add_argument('--wait', type=int, default=65)
|
||||
p.add_argument('--sizes', type=int, nargs='+', default=[1000, 5000, 20000])
|
||||
p.add_argument('--repeat', type=int, default=3)
|
||||
p.add_argument('--output')
|
||||
a = p.parse_args()
|
||||
sh('docker', 'compose', 'stop', 'reconciler')
|
||||
try:
|
||||
res = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
||||
'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False),
|
||||
'result': (cmd_faults if a.cmd == 'faults' else cmd_cost)(a)}
|
||||
finally:
|
||||
sh('docker', 'compose', 'start', 'reconciler')
|
||||
text = json.dumps(res, ensure_ascii=False, indent=2)
|
||||
if a.output:
|
||||
Path(a.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(a.output).write_text(text + '\n', encoding='utf-8')
|
||||
print(text if a.cmd == 'faults' else json.dumps(res['result']['runs'][-1]))
|
||||
print('PASS: track2 ' + a.cmd)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Minimal S3 client (stdlib, SigV4) for checks and fault injection OUTSIDE the API.
|
||||
|
||||
Runs inside a throwaway container on the compose `data` network (S3 is not
|
||||
published to the host), see run_in_net() in lab4ops.py. Credentials are read
|
||||
from mounted secret files and never printed.
|
||||
"""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import hmac
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
from urllib.parse import quote
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
HOST = os.environ.get('S3_HOST', 's3:8333')
|
||||
REGION = os.environ.get('S3_REGION', 'us-east-1')
|
||||
BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments')
|
||||
SECRETS = os.environ.get('SECRETS_DIR', '/secrets')
|
||||
NS = '{http://s3.amazonaws.com/doc/2006-03-01/}'
|
||||
|
||||
|
||||
def creds(identity):
|
||||
if identity == 'anonymous':
|
||||
return None
|
||||
prefix = 's3_admin_' if identity == 'admin' else 's3_'
|
||||
read = lambda n: open(os.path.join(SECRETS, prefix + n)).read().strip()
|
||||
return read('access_key_id'), read('secret_access_key')
|
||||
|
||||
|
||||
def _h(key, msg):
|
||||
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
|
||||
|
||||
|
||||
_local = threading.local()
|
||||
|
||||
|
||||
def request(method, path, identity='api', body=b'', query=None):
|
||||
query = query or {}
|
||||
headers = {'host': HOST}
|
||||
payload_hash = hashlib.sha256(body).hexdigest()
|
||||
c = creds(identity)
|
||||
if c:
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
amzdate, date = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d')
|
||||
headers.update({'x-amz-content-sha256': payload_hash, 'x-amz-date': amzdate})
|
||||
signed = ';'.join(sorted(headers))
|
||||
cq = '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
|
||||
creq = '\n'.join([method, quote(path, safe='/-_.~'), cq,
|
||||
''.join('%s:%s\n' % (k, headers[k]) for k in sorted(headers)), signed, payload_hash])
|
||||
scope = '%s/%s/s3/aws4_request' % (date, REGION)
|
||||
sts = '\n'.join(['AWS4-HMAC-SHA256', amzdate, scope, hashlib.sha256(creq.encode()).hexdigest()])
|
||||
k = _h(_h(_h(_h(('AWS4' + c[1]).encode(), date), REGION), 's3'), 'aws4_request')
|
||||
sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest()
|
||||
headers['authorization'] = 'AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s' % (
|
||||
c[0], scope, signed, sig)
|
||||
url = quote(path, safe='/-_.~')
|
||||
if query:
|
||||
url += '?' + '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
|
||||
conn = getattr(_local, 'conn', None)
|
||||
if conn is None:
|
||||
conn = _local.conn = http.client.HTTPConnection(HOST, timeout=30)
|
||||
try:
|
||||
conn.request(method, url, body=body or None, headers=headers)
|
||||
r = conn.getresponse()
|
||||
return r.status, r.read()
|
||||
except (OSError, http.client.HTTPException):
|
||||
_local.conn = None
|
||||
raise
|
||||
|
||||
|
||||
def obj(key):
|
||||
return '/' + BUCKET + '/' + key
|
||||
|
||||
|
||||
def list_keys(prefix, identity='api'):
|
||||
keys, token = [], None
|
||||
while True:
|
||||
q = {'list-type': '2', 'prefix': prefix, 'max-keys': '1000'}
|
||||
if token:
|
||||
q['continuation-token'] = token
|
||||
st, body = request('GET', '/' + BUCKET, identity, query=q)
|
||||
if st != 200:
|
||||
raise SystemExit('list failed: HTTP %d' % st)
|
||||
root = ET.fromstring(body)
|
||||
keys += [e.find(NS + 'Key').text for e in root.findall(NS + 'Contents')]
|
||||
if root.findtext(NS + 'IsTruncated') != 'true':
|
||||
return keys
|
||||
token = root.findtext(NS + 'NextContinuationToken')
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('cmd', choices=['get', 'put', 'delete', 'create-bucket', 'list', 'count', 'bulk-put', 'bulk-delete'])
|
||||
p.add_argument('arg', nargs='?', default='')
|
||||
p.add_argument('--as', dest='identity', default='api', choices=['api', 'admin', 'anonymous'])
|
||||
p.add_argument('--size', type=int, default=100)
|
||||
p.add_argument('--workers', type=int, default=32)
|
||||
p.add_argument('--stdin', action='store_true', help='put: body from stdin instead of random bytes')
|
||||
a = p.parse_args()
|
||||
out = {'cmd': a.cmd, 'as': a.identity}
|
||||
if a.cmd == 'get':
|
||||
st, body = request('GET', obj(a.arg), a.identity)
|
||||
out.update(status=st, bytes=len(body) if st == 200 else 0,
|
||||
sha256=hashlib.sha256(body).hexdigest() if st == 200 else None)
|
||||
elif a.cmd == 'put':
|
||||
body = sys.stdin.buffer.read() if a.stdin else os.urandom(a.size)
|
||||
st, _ = request('PUT', obj(a.arg), a.identity, body=body)
|
||||
out.update(status=st)
|
||||
elif a.cmd == 'delete':
|
||||
st, _ = request('DELETE', obj(a.arg), a.identity)
|
||||
out.update(status=st)
|
||||
elif a.cmd == 'create-bucket':
|
||||
st, _ = request('PUT', '/' + a.arg, a.identity)
|
||||
out.update(status=st)
|
||||
elif a.cmd == 'list':
|
||||
out.update(keys=list_keys(a.arg, a.identity))
|
||||
elif a.cmd == 'count':
|
||||
out.update(count=len(list_keys(a.arg, a.identity)))
|
||||
elif a.cmd in ('bulk-put', 'bulk-delete'):
|
||||
keys = [k.strip() for k in sys.stdin if k.strip()]
|
||||
data = b'x' * a.size
|
||||
|
||||
def one(k):
|
||||
if a.cmd == 'bulk-put':
|
||||
return request('PUT', obj(k), a.identity, body=data)[0]
|
||||
return request('DELETE', obj(k), a.identity)[0]
|
||||
|
||||
start = dt.datetime.now()
|
||||
with ThreadPoolExecutor(a.workers) as pool:
|
||||
codes = list(pool.map(one, keys))
|
||||
out.update(count=len(keys), ok=sum(1 for c in codes if c in (200, 204)),
|
||||
seconds=round((dt.datetime.now() - start).total_seconds(), 2))
|
||||
print(json.dumps(out))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user