Files
mtusi2026lab5/scripts/s3tool.py

138 lines
5.3 KiB
Python

#!/usr/bin/env python3
import argparse
from concurrent.futures import ThreadPoolExecutor
import datetime as dt
import hashlib
import hmac
import http.client
import json
import os
import sys
import threading
from urllib.parse import quote
import xml.etree.ElementTree as ET
HOST = os.environ.get('S3_HOST', 's3:8333')
REGION = os.environ.get('S3_REGION', 'us-east-1')
BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments')
SECRETS = os.environ.get('SECRETS_DIR', '/secrets')
NS = '{http://s3.amazonaws.com/doc/2006-03-01/}'
def creds(identity):
if identity == 'anonymous':
return None
prefix = 's3_admin_' if identity == 'admin' else 's3_'
read = lambda n: open(os.path.join(SECRETS, prefix + n)).read().strip()
return read('access_key_id'), read('secret_access_key')
def _h(key, msg):
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
_local = threading.local()
def request(method, path, identity='api', body=b'', query=None):
query = query or {}
headers = {'host': HOST}
payload_hash = hashlib.sha256(body).hexdigest()
c = creds(identity)
if c:
now = dt.datetime.now(dt.timezone.utc)
amzdate, date = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d')
headers.update({'x-amz-content-sha256': payload_hash, 'x-amz-date': amzdate})
signed = ';'.join(sorted(headers))
cq = '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
creq = '\n'.join([method, quote(path, safe='/-_.~'), cq,
''.join('%s:%s\n' % (k, headers[k]) for k in sorted(headers)), signed, payload_hash])
scope = '%s/%s/s3/aws4_request' % (date, REGION)
sts = '\n'.join(['AWS4-HMAC-SHA256', amzdate, scope, hashlib.sha256(creq.encode()).hexdigest()])
k = _h(_h(_h(_h(('AWS4' + c[1]).encode(), date), REGION), 's3'), 'aws4_request')
sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest()
headers['authorization'] = 'AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s' % (
c[0], scope, signed, sig)
url = quote(path, safe='/-_.~')
if query:
url += '?' + '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
conn = getattr(_local, 'conn', None)
if conn is None:
conn = _local.conn = http.client.HTTPConnection(HOST, timeout=30)
try:
conn.request(method, url, body=body or None, headers=headers)
r = conn.getresponse()
return r.status, r.read()
except (OSError, http.client.HTTPException):
_local.conn = None
raise
def obj(key):
return '/' + BUCKET + '/' + key
def list_keys(prefix, identity='api'):
keys, token = [], None
while True:
q = {'list-type': '2', 'prefix': prefix, 'max-keys': '1000'}
if token:
q['continuation-token'] = token
st, body = request('GET', '/' + BUCKET, identity, query=q)
if st != 200:
raise SystemExit('list failed: HTTP %d' % st)
root = ET.fromstring(body)
keys += [e.find(NS + 'Key').text for e in root.findall(NS + 'Contents')]
if root.findtext(NS + 'IsTruncated') != 'true':
return keys
token = root.findtext(NS + 'NextContinuationToken')
def main():
p = argparse.ArgumentParser()
p.add_argument('cmd', choices=['get', 'put', 'delete', 'create-bucket', 'list', 'count', 'bulk-put', 'bulk-delete'])
p.add_argument('arg', nargs='?', default='')
p.add_argument('--as', dest='identity', default='api', choices=['api', 'admin', 'anonymous'])
p.add_argument('--size', type=int, default=100)
p.add_argument('--workers', type=int, default=32)
p.add_argument('--stdin', action='store_true', help='put: body from stdin instead of random bytes')
a = p.parse_args()
out = {'cmd': a.cmd, 'as': a.identity}
if a.cmd == 'get':
st, body = request('GET', obj(a.arg), a.identity)
out.update(status=st, bytes=len(body) if st == 200 else 0,
sha256=hashlib.sha256(body).hexdigest() if st == 200 else None)
elif a.cmd == 'put':
body = sys.stdin.buffer.read() if a.stdin else os.urandom(a.size)
st, _ = request('PUT', obj(a.arg), a.identity, body=body)
out.update(status=st)
elif a.cmd == 'delete':
st, _ = request('DELETE', obj(a.arg), a.identity)
out.update(status=st)
elif a.cmd == 'create-bucket':
st, _ = request('PUT', '/' + a.arg, a.identity)
out.update(status=st)
elif a.cmd == 'list':
out.update(keys=list_keys(a.arg, a.identity))
elif a.cmd == 'count':
out.update(count=len(list_keys(a.arg, a.identity)))
elif a.cmd in ('bulk-put', 'bulk-delete'):
keys = [k.strip() for k in sys.stdin if k.strip()]
data = b'x' * a.size
def one(k):
if a.cmd == 'bulk-put':
return request('PUT', obj(k), a.identity, body=data)[0]
return request('DELETE', obj(k), a.identity)[0]
start = dt.datetime.now()
with ThreadPoolExecutor(a.workers) as pool:
codes = list(pool.map(one, keys))
out.update(count=len(keys), ok=sum(1 for c in codes if c in (200, 204)),
seconds=round((dt.datetime.now() - start).total_seconds(), 2))
print(json.dumps(out))
if __name__ == '__main__':
main()