Files
mtusi2026lab4/scripts/track2/s3tool.py
2026-10-29 15:00:36 +03:00

144 lines
5.6 KiB
Python

#!/usr/bin/env python3
"""Minimal S3 client (stdlib, SigV4) for checks and fault injection OUTSIDE the API.
Runs inside a throwaway container on the compose `data` network (S3 is not
published to the host), see run_in_net() in lab4ops.py. Credentials are read
from mounted secret files and never printed.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import datetime as dt
import hashlib
import hmac
import http.client
import json
import os
import sys
import threading
from urllib.parse import quote
import xml.etree.ElementTree as ET
HOST = os.environ.get('S3_HOST', 's3:8333')
REGION = os.environ.get('S3_REGION', 'us-east-1')
BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments')
SECRETS = os.environ.get('SECRETS_DIR', '/secrets')
NS = '{http://s3.amazonaws.com/doc/2006-03-01/}'
def creds(identity):
if identity == 'anonymous':
return None
prefix = 's3_admin_' if identity == 'admin' else 's3_'
read = lambda n: open(os.path.join(SECRETS, prefix + n)).read().strip()
return read('access_key_id'), read('secret_access_key')
def _h(key, msg):
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
_local = threading.local()
def request(method, path, identity='api', body=b'', query=None):
query = query or {}
headers = {'host': HOST}
payload_hash = hashlib.sha256(body).hexdigest()
c = creds(identity)
if c:
now = dt.datetime.now(dt.timezone.utc)
amzdate, date = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d')
headers.update({'x-amz-content-sha256': payload_hash, 'x-amz-date': amzdate})
signed = ';'.join(sorted(headers))
cq = '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
creq = '\n'.join([method, quote(path, safe='/-_.~'), cq,
''.join('%s:%s\n' % (k, headers[k]) for k in sorted(headers)), signed, payload_hash])
scope = '%s/%s/s3/aws4_request' % (date, REGION)
sts = '\n'.join(['AWS4-HMAC-SHA256', amzdate, scope, hashlib.sha256(creq.encode()).hexdigest()])
k = _h(_h(_h(_h(('AWS4' + c[1]).encode(), date), REGION), 's3'), 'aws4_request')
sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest()
headers['authorization'] = 'AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s' % (
c[0], scope, signed, sig)
url = quote(path, safe='/-_.~')
if query:
url += '?' + '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
conn = getattr(_local, 'conn', None)
if conn is None:
conn = _local.conn = http.client.HTTPConnection(HOST, timeout=30)
try:
conn.request(method, url, body=body or None, headers=headers)
r = conn.getresponse()
return r.status, r.read()
except (OSError, http.client.HTTPException):
_local.conn = None
raise
def obj(key):
return '/' + BUCKET + '/' + key
def list_keys(prefix, identity='api'):
keys, token = [], None
while True:
q = {'list-type': '2', 'prefix': prefix, 'max-keys': '1000'}
if token:
q['continuation-token'] = token
st, body = request('GET', '/' + BUCKET, identity, query=q)
if st != 200:
raise SystemExit('list failed: HTTP %d' % st)
root = ET.fromstring(body)
keys += [e.find(NS + 'Key').text for e in root.findall(NS + 'Contents')]
if root.findtext(NS + 'IsTruncated') != 'true':
return keys
token = root.findtext(NS + 'NextContinuationToken')
def main():
p = argparse.ArgumentParser()
p.add_argument('cmd', choices=['get', 'put', 'delete', 'create-bucket', 'list', 'count', 'bulk-put', 'bulk-delete'])
p.add_argument('arg', nargs='?', default='')
p.add_argument('--as', dest='identity', default='api', choices=['api', 'admin', 'anonymous'])
p.add_argument('--size', type=int, default=100)
p.add_argument('--workers', type=int, default=32)
p.add_argument('--stdin', action='store_true', help='put: body from stdin instead of random bytes')
a = p.parse_args()
out = {'cmd': a.cmd, 'as': a.identity}
if a.cmd == 'get':
st, body = request('GET', obj(a.arg), a.identity)
out.update(status=st, bytes=len(body) if st == 200 else 0,
sha256=hashlib.sha256(body).hexdigest() if st == 200 else None)
elif a.cmd == 'put':
body = sys.stdin.buffer.read() if a.stdin else os.urandom(a.size)
st, _ = request('PUT', obj(a.arg), a.identity, body=body)
out.update(status=st)
elif a.cmd == 'delete':
st, _ = request('DELETE', obj(a.arg), a.identity)
out.update(status=st)
elif a.cmd == 'create-bucket':
st, _ = request('PUT', '/' + a.arg, a.identity)
out.update(status=st)
elif a.cmd == 'list':
out.update(keys=list_keys(a.arg, a.identity))
elif a.cmd == 'count':
out.update(count=len(list_keys(a.arg, a.identity)))
elif a.cmd in ('bulk-put', 'bulk-delete'):
keys = [k.strip() for k in sys.stdin if k.strip()]
data = b'x' * a.size
def one(k):
if a.cmd == 'bulk-put':
return request('PUT', obj(k), a.identity, body=data)[0]
return request('DELETE', obj(k), a.identity)[0]
start = dt.datetime.now()
with ThreadPoolExecutor(a.workers) as pool:
codes = list(pool.map(one, keys))
out.update(count=len(keys), ok=sum(1 for c in codes if c in (200, 204)),
seconds=round((dt.datetime.now() - start).total_seconds(), 2))
print(json.dumps(out))
if __name__ == '__main__':
main()