forked from mtusi2026/mtusi2026lab4
144 lines
5.6 KiB
Python
144 lines
5.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Minimal S3 client (stdlib, SigV4) for checks and fault injection OUTSIDE the API.
|
|
|
|
Runs inside a throwaway container on the compose `data` network (S3 is not
|
|
published to the host), see run_in_net() in lab4ops.py. Credentials are read
|
|
from mounted secret files and never printed.
|
|
"""
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import datetime as dt
|
|
import hashlib
|
|
import hmac
|
|
import http.client
|
|
import json
|
|
import os
|
|
import sys
|
|
import threading
|
|
from urllib.parse import quote
|
|
import xml.etree.ElementTree as ET
|
|
|
|
HOST = os.environ.get('S3_HOST', 's3:8333')
|
|
REGION = os.environ.get('S3_REGION', 'us-east-1')
|
|
BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments')
|
|
SECRETS = os.environ.get('SECRETS_DIR', '/secrets')
|
|
NS = '{http://s3.amazonaws.com/doc/2006-03-01/}'
|
|
|
|
|
|
def creds(identity):
|
|
if identity == 'anonymous':
|
|
return None
|
|
prefix = 's3_admin_' if identity == 'admin' else 's3_'
|
|
read = lambda n: open(os.path.join(SECRETS, prefix + n)).read().strip()
|
|
return read('access_key_id'), read('secret_access_key')
|
|
|
|
|
|
def _h(key, msg):
|
|
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
|
|
|
|
|
|
_local = threading.local()
|
|
|
|
|
|
def request(method, path, identity='api', body=b'', query=None):
|
|
query = query or {}
|
|
headers = {'host': HOST}
|
|
payload_hash = hashlib.sha256(body).hexdigest()
|
|
c = creds(identity)
|
|
if c:
|
|
now = dt.datetime.now(dt.timezone.utc)
|
|
amzdate, date = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d')
|
|
headers.update({'x-amz-content-sha256': payload_hash, 'x-amz-date': amzdate})
|
|
signed = ';'.join(sorted(headers))
|
|
cq = '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
|
|
creq = '\n'.join([method, quote(path, safe='/-_.~'), cq,
|
|
''.join('%s:%s\n' % (k, headers[k]) for k in sorted(headers)), signed, payload_hash])
|
|
scope = '%s/%s/s3/aws4_request' % (date, REGION)
|
|
sts = '\n'.join(['AWS4-HMAC-SHA256', amzdate, scope, hashlib.sha256(creq.encode()).hexdigest()])
|
|
k = _h(_h(_h(_h(('AWS4' + c[1]).encode(), date), REGION), 's3'), 'aws4_request')
|
|
sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest()
|
|
headers['authorization'] = 'AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s' % (
|
|
c[0], scope, signed, sig)
|
|
url = quote(path, safe='/-_.~')
|
|
if query:
|
|
url += '?' + '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
|
|
conn = getattr(_local, 'conn', None)
|
|
if conn is None:
|
|
conn = _local.conn = http.client.HTTPConnection(HOST, timeout=30)
|
|
try:
|
|
conn.request(method, url, body=body or None, headers=headers)
|
|
r = conn.getresponse()
|
|
return r.status, r.read()
|
|
except (OSError, http.client.HTTPException):
|
|
_local.conn = None
|
|
raise
|
|
|
|
|
|
def obj(key):
|
|
return '/' + BUCKET + '/' + key
|
|
|
|
|
|
def list_keys(prefix, identity='api'):
|
|
keys, token = [], None
|
|
while True:
|
|
q = {'list-type': '2', 'prefix': prefix, 'max-keys': '1000'}
|
|
if token:
|
|
q['continuation-token'] = token
|
|
st, body = request('GET', '/' + BUCKET, identity, query=q)
|
|
if st != 200:
|
|
raise SystemExit('list failed: HTTP %d' % st)
|
|
root = ET.fromstring(body)
|
|
keys += [e.find(NS + 'Key').text for e in root.findall(NS + 'Contents')]
|
|
if root.findtext(NS + 'IsTruncated') != 'true':
|
|
return keys
|
|
token = root.findtext(NS + 'NextContinuationToken')
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('cmd', choices=['get', 'put', 'delete', 'create-bucket', 'list', 'count', 'bulk-put', 'bulk-delete'])
|
|
p.add_argument('arg', nargs='?', default='')
|
|
p.add_argument('--as', dest='identity', default='api', choices=['api', 'admin', 'anonymous'])
|
|
p.add_argument('--size', type=int, default=100)
|
|
p.add_argument('--workers', type=int, default=32)
|
|
p.add_argument('--stdin', action='store_true', help='put: body from stdin instead of random bytes')
|
|
a = p.parse_args()
|
|
out = {'cmd': a.cmd, 'as': a.identity}
|
|
if a.cmd == 'get':
|
|
st, body = request('GET', obj(a.arg), a.identity)
|
|
out.update(status=st, bytes=len(body) if st == 200 else 0,
|
|
sha256=hashlib.sha256(body).hexdigest() if st == 200 else None)
|
|
elif a.cmd == 'put':
|
|
body = sys.stdin.buffer.read() if a.stdin else os.urandom(a.size)
|
|
st, _ = request('PUT', obj(a.arg), a.identity, body=body)
|
|
out.update(status=st)
|
|
elif a.cmd == 'delete':
|
|
st, _ = request('DELETE', obj(a.arg), a.identity)
|
|
out.update(status=st)
|
|
elif a.cmd == 'create-bucket':
|
|
st, _ = request('PUT', '/' + a.arg, a.identity)
|
|
out.update(status=st)
|
|
elif a.cmd == 'list':
|
|
out.update(keys=list_keys(a.arg, a.identity))
|
|
elif a.cmd == 'count':
|
|
out.update(count=len(list_keys(a.arg, a.identity)))
|
|
elif a.cmd in ('bulk-put', 'bulk-delete'):
|
|
keys = [k.strip() for k in sys.stdin if k.strip()]
|
|
data = b'x' * a.size
|
|
|
|
def one(k):
|
|
if a.cmd == 'bulk-put':
|
|
return request('PUT', obj(k), a.identity, body=data)[0]
|
|
return request('DELETE', obj(k), a.identity)[0]
|
|
|
|
start = dt.datetime.now()
|
|
with ThreadPoolExecutor(a.workers) as pool:
|
|
codes = list(pool.map(one, keys))
|
|
out.update(count=len(keys), ok=sum(1 for c in codes if c in (200, 204)),
|
|
seconds=round((dt.datetime.now() - start).total_seconds(), 2))
|
|
print(json.dumps(out))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|