#!/usr/bin/env python3 import argparse from concurrent.futures import ThreadPoolExecutor import datetime as dt import hashlib import hmac import http.client import json import os import sys import threading from urllib.parse import quote import xml.etree.ElementTree as ET HOST = os.environ.get('S3_HOST', 's3:8333') REGION = os.environ.get('S3_REGION', 'us-east-1') BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments') SECRETS = os.environ.get('SECRETS_DIR', '/secrets') NS = '{http://s3.amazonaws.com/doc/2006-03-01/}' def creds(identity): if identity == 'anonymous': return None prefix = 's3_admin_' if identity == 'admin' else 's3_' read = lambda n: open(os.path.join(SECRETS, prefix + n)).read().strip() return read('access_key_id'), read('secret_access_key') def _h(key, msg): return hmac.new(key, msg.encode(), hashlib.sha256).digest() _local = threading.local() def request(method, path, identity='api', body=b'', query=None): query = query or {} headers = {'host': HOST} payload_hash = hashlib.sha256(body).hexdigest() c = creds(identity) if c: now = dt.datetime.now(dt.timezone.utc) amzdate, date = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d') headers.update({'x-amz-content-sha256': payload_hash, 'x-amz-date': amzdate}) signed = ';'.join(sorted(headers)) cq = '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items())) creq = '\n'.join([method, quote(path, safe='/-_.~'), cq, ''.join('%s:%s\n' % (k, headers[k]) for k in sorted(headers)), signed, payload_hash]) scope = '%s/%s/s3/aws4_request' % (date, REGION) sts = '\n'.join(['AWS4-HMAC-SHA256', amzdate, scope, hashlib.sha256(creq.encode()).hexdigest()]) k = _h(_h(_h(_h(('AWS4' + c[1]).encode(), date), REGION), 's3'), 'aws4_request') sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest() headers['authorization'] = 'AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s' % ( c[0], scope, signed, sig) url = quote(path, safe='/-_.~') if query: url += '?' + '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items())) conn = getattr(_local, 'conn', None) if conn is None: conn = _local.conn = http.client.HTTPConnection(HOST, timeout=30) try: conn.request(method, url, body=body or None, headers=headers) r = conn.getresponse() return r.status, r.read() except (OSError, http.client.HTTPException): _local.conn = None raise def obj(key): return '/' + BUCKET + '/' + key def list_keys(prefix, identity='api'): keys, token = [], None while True: q = {'list-type': '2', 'prefix': prefix, 'max-keys': '1000'} if token: q['continuation-token'] = token st, body = request('GET', '/' + BUCKET, identity, query=q) if st != 200: raise SystemExit('list failed: HTTP %d' % st) root = ET.fromstring(body) keys += [e.find(NS + 'Key').text for e in root.findall(NS + 'Contents')] if root.findtext(NS + 'IsTruncated') != 'true': return keys token = root.findtext(NS + 'NextContinuationToken') def main(): p = argparse.ArgumentParser() p.add_argument('cmd', choices=['get', 'put', 'delete', 'create-bucket', 'list', 'count', 'bulk-put', 'bulk-delete']) p.add_argument('arg', nargs='?', default='') p.add_argument('--as', dest='identity', default='api', choices=['api', 'admin', 'anonymous']) p.add_argument('--size', type=int, default=100) p.add_argument('--workers', type=int, default=32) p.add_argument('--stdin', action='store_true', help='put: body from stdin instead of random bytes') a = p.parse_args() out = {'cmd': a.cmd, 'as': a.identity} if a.cmd == 'get': st, body = request('GET', obj(a.arg), a.identity) out.update(status=st, bytes=len(body) if st == 200 else 0, sha256=hashlib.sha256(body).hexdigest() if st == 200 else None) elif a.cmd == 'put': body = sys.stdin.buffer.read() if a.stdin else os.urandom(a.size) st, _ = request('PUT', obj(a.arg), a.identity, body=body) out.update(status=st) elif a.cmd == 'delete': st, _ = request('DELETE', obj(a.arg), a.identity) out.update(status=st) elif a.cmd == 'create-bucket': st, _ = request('PUT', '/' + a.arg, a.identity) out.update(status=st) elif a.cmd == 'list': out.update(keys=list_keys(a.arg, a.identity)) elif a.cmd == 'count': out.update(count=len(list_keys(a.arg, a.identity))) elif a.cmd in ('bulk-put', 'bulk-delete'): keys = [k.strip() for k in sys.stdin if k.strip()] data = b'x' * a.size def one(k): if a.cmd == 'bulk-put': return request('PUT', obj(k), a.identity, body=data)[0] return request('DELETE', obj(k), a.identity)[0] start = dt.datetime.now() with ThreadPoolExecutor(a.workers) as pool: codes = list(pool.map(one, keys)) out.update(count=len(keys), ok=sum(1 for c in codes if c in (200, 204)), seconds=round((dt.datetime.now() - start).total_seconds(), 2)) print(json.dumps(out)) if __name__ == '__main__': main()