#!/usr/bin/env python3 """Minimal S3 client (stdlib, SigV4) for checks and fault injection OUTSIDE the API. Runs inside a throwaway container on the compose `data` network (S3 is not published to the host), see run_in_net() in lab4ops.py. Credentials are read from mounted secret files and never printed. """ import argparse from concurrent.futures import ThreadPoolExecutor import datetime as dt import hashlib import hmac import http.client import json import os import sys import threading from urllib.parse import quote import xml.etree.ElementTree as ET HOST = os.environ.get('S3_HOST', 's3:8333') REGION = os.environ.get('S3_REGION', 'us-east-1') BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments') SECRETS = os.environ.get('SECRETS_DIR', '/secrets') NS = '{http://s3.amazonaws.com/doc/2006-03-01/}' def creds(identity): if identity == 'anonymous': return None prefix = 's3_admin_' if identity == 'admin' else 's3_' read = lambda n: open(os.path.join(SECRETS, prefix + n)).read().strip() return read('access_key_id'), read('secret_access_key') def _h(key, msg): return hmac.new(key, msg.encode(), hashlib.sha256).digest() _local = threading.local() def request(method, path, identity='api', body=b'', query=None): query = query or {} headers = {'host': HOST} payload_hash = hashlib.sha256(body).hexdigest() c = creds(identity) if c: now = dt.datetime.now(dt.timezone.utc) amzdate, date = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d') headers.update({'x-amz-content-sha256': payload_hash, 'x-amz-date': amzdate}) signed = ';'.join(sorted(headers)) cq = '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items())) creq = '\n'.join([method, quote(path, safe='/-_.~'), cq, ''.join('%s:%s\n' % (k, headers[k]) for k in sorted(headers)), signed, payload_hash]) scope = '%s/%s/s3/aws4_request' % (date, REGION) sts = '\n'.join(['AWS4-HMAC-SHA256', amzdate, scope, hashlib.sha256(creq.encode()).hexdigest()]) k = _h(_h(_h(_h(('AWS4' + c[1]).encode(), date), REGION), 's3'), 'aws4_request') sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest() headers['authorization'] = 'AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s' % ( c[0], scope, signed, sig) url = quote(path, safe='/-_.~') if query: url += '?' + '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items())) conn = getattr(_local, 'conn', None) if conn is None: conn = _local.conn = http.client.HTTPConnection(HOST, timeout=30) try: conn.request(method, url, body=body or None, headers=headers) r = conn.getresponse() return r.status, r.read() except (OSError, http.client.HTTPException): _local.conn = None raise def obj(key): return '/' + BUCKET + '/' + key def list_keys(prefix, identity='api'): keys, token = [], None while True: q = {'list-type': '2', 'prefix': prefix, 'max-keys': '1000'} if token: q['continuation-token'] = token st, body = request('GET', '/' + BUCKET, identity, query=q) if st != 200: raise SystemExit('list failed: HTTP %d' % st) root = ET.fromstring(body) keys += [e.find(NS + 'Key').text for e in root.findall(NS + 'Contents')] if root.findtext(NS + 'IsTruncated') != 'true': return keys token = root.findtext(NS + 'NextContinuationToken') def main(): p = argparse.ArgumentParser() p.add_argument('cmd', choices=['get', 'put', 'delete', 'create-bucket', 'list', 'count', 'bulk-put', 'bulk-delete']) p.add_argument('arg', nargs='?', default='') p.add_argument('--as', dest='identity', default='api', choices=['api', 'admin', 'anonymous']) p.add_argument('--size', type=int, default=100) p.add_argument('--workers', type=int, default=32) p.add_argument('--stdin', action='store_true', help='put: body from stdin instead of random bytes') a = p.parse_args() out = {'cmd': a.cmd, 'as': a.identity} if a.cmd == 'get': st, body = request('GET', obj(a.arg), a.identity) out.update(status=st, bytes=len(body) if st == 200 else 0, sha256=hashlib.sha256(body).hexdigest() if st == 200 else None) elif a.cmd == 'put': body = sys.stdin.buffer.read() if a.stdin else os.urandom(a.size) st, _ = request('PUT', obj(a.arg), a.identity, body=body) out.update(status=st) elif a.cmd == 'delete': st, _ = request('DELETE', obj(a.arg), a.identity) out.update(status=st) elif a.cmd == 'create-bucket': st, _ = request('PUT', '/' + a.arg, a.identity) out.update(status=st) elif a.cmd == 'list': out.update(keys=list_keys(a.arg, a.identity)) elif a.cmd == 'count': out.update(count=len(list_keys(a.arg, a.identity))) elif a.cmd in ('bulk-put', 'bulk-delete'): keys = [k.strip() for k in sys.stdin if k.strip()] data = b'x' * a.size def one(k): if a.cmd == 'bulk-put': return request('PUT', obj(k), a.identity, body=data)[0] return request('DELETE', obj(k), a.identity)[0] start = dt.datetime.now() with ThreadPoolExecutor(a.workers) as pool: codes = list(pool.map(one, keys)) out.update(count=len(keys), ok=sum(1 for c in codes if c in (200, 204)), seconds=round((dt.datetime.now() - start).total_seconds(), 2)) print(json.dumps(out)) if __name__ == '__main__': main()