forked from mtusi2026/mtusi2026lab4
feat: restricted S3 credentials, orphan reconciler, upload limits (grade 4)
This commit is contained in:
@@ -13,11 +13,16 @@ password=$(cat secrets/postgres_password)
|
||||
printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$password" > secrets/database_url
|
||||
[ -f secrets/cache_url ] ||
|
||||
printf 'redis://redis:6379/0\n' > secrets/cache_url
|
||||
[ -f secrets/s3_root_access_key ] ||
|
||||
head -c 10 /dev/urandom | od -An -tx1 | tr -d ' \n' > secrets/s3_root_access_key
|
||||
[ -f secrets/s3_root_secret_key ] ||
|
||||
head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n' > secrets/s3_root_secret_key
|
||||
[ -f secrets/s3_access_key_id ] ||
|
||||
head -c 10 /dev/urandom | od -An -tx1 | tr -d ' \n' > secrets/s3_access_key_id
|
||||
[ -f secrets/s3_secret_access_key ] ||
|
||||
head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n' > secrets/s3_secret_access_key
|
||||
|
||||
chmod 644 secrets/postgres_password secrets/database_url secrets/cache_url \
|
||||
secrets/s3_root_access_key secrets/s3_root_secret_key \
|
||||
secrets/s3_access_key_id secrets/s3_secret_access_key
|
||||
echo "secrets ready in ./secrets"
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
ENGINE=${ENGINE:-$(command -v podman >/dev/null 2>&1 && echo podman || echo docker)}
|
||||
PROJECT=${PROJECT:-messenger-lab4}
|
||||
IMAGE=${IMAGE:-localhost/messenger-api:lab4}
|
||||
|
||||
$ENGINE run --rm --network "${PROJECT}_backend" \
|
||||
--entrypoint /usr/local/bin/reconcile \
|
||||
-v "$PWD/secrets/database_url:/run/secrets/database_url:ro" \
|
||||
-v "$PWD/secrets/s3_access_key_id:/run/secrets/s3_access_key_id:ro" \
|
||||
-v "$PWD/secrets/s3_secret_access_key:/run/secrets/s3_secret_access_key:ro" \
|
||||
-e S3_ENDPOINT="${S3_ENDPOINT:-http://s3:9000}" \
|
||||
-e S3_REGION="${S3_REGION:-us-east-1}" \
|
||||
-e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" \
|
||||
-e DATABASE_URL_FILE=/run/secrets/database_url \
|
||||
-e S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id \
|
||||
-e S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key \
|
||||
"$IMAGE" "$@"
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
ENGINE=${ENGINE:-$(command -v podman >/dev/null 2>&1 && echo podman || echo docker)}
|
||||
PROJECT=${PROJECT:-messenger-lab4}
|
||||
IMAGE=${IMAGE:-localhost/messenger-api:lab4}
|
||||
|
||||
$ENGINE run --rm -i --network "${PROJECT}_backend" \
|
||||
--entrypoint /usr/local/bin/s3tool \
|
||||
-v "$PWD/secrets/s3_access_key_id:/run/secrets/s3_access_key_id:ro" \
|
||||
-v "$PWD/secrets/s3_secret_access_key:/run/secrets/s3_secret_access_key:ro" \
|
||||
-e S3_ENDPOINT="${S3_ENDPOINT:-http://s3:9000}" \
|
||||
-e S3_REGION="${S3_REGION:-us-east-1}" \
|
||||
-e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" \
|
||||
-e S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id \
|
||||
-e S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key \
|
||||
"$IMAGE" "$@"
|
||||
@@ -0,0 +1,12 @@
|
||||
CREATE OR REPLACE FUNCTION inject_attachment_insert_failure() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
RAISE EXCEPTION 'injected attachment insert failure' USING ERRCODE = '57014';
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS inject_attachment_insert_failure ON attachments;
|
||||
|
||||
CREATE TRIGGER inject_attachment_insert_failure
|
||||
BEFORE INSERT ON attachments
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION inject_attachment_insert_failure();
|
||||
@@ -0,0 +1,2 @@
|
||||
DROP TRIGGER IF EXISTS inject_attachment_insert_failure ON attachments;
|
||||
DROP FUNCTION IF EXISTS inject_attachment_insert_failure();
|
||||
Executable
+130
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Concurrent upload experiment with API memory measurement; creates synthetic data."""
|
||||
import json
|
||||
import math
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def engine():
|
||||
return 'podman' if shutil.which('podman') else 'docker'
|
||||
|
||||
|
||||
def api_containers(project):
|
||||
result = subprocess.run(
|
||||
[engine(), 'ps',
|
||||
'--filter', 'label=com.docker.compose.project=' + project,
|
||||
'--filter', 'label=com.docker.compose.service=api',
|
||||
'--format', '{{.Names}}'],
|
||||
capture_output=True, text=True, check=False)
|
||||
check(result.returncode == 0, 'cannot list api containers: ' + result.stderr.strip())
|
||||
return sorted(line.strip() for line in result.stdout.splitlines() if line.strip())
|
||||
|
||||
|
||||
def peak_bytes(container):
|
||||
for path in ('/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory/memory.max_usage_in_bytes'):
|
||||
result = subprocess.run([engine(), 'exec', container, 'cat', path], capture_output=True, text=True, check=False)
|
||||
if result.returncode == 0 and result.stdout.strip().isdigit():
|
||||
return int(result.stdout.strip())
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument('--concurrency', type=int, default=10)
|
||||
p.add_argument('--file', default=str(ROOT / 'scripts/fixtures/large.png'))
|
||||
p.add_argument('--project', default='messenger-lab4')
|
||||
p.add_argument('--output', default=None)
|
||||
|
||||
args = arguments('Concurrent attachment uploads with peak memory measurement', configure)
|
||||
check(1 <= args.concurrency <= 64, 'concurrency must be 1..64')
|
||||
source = Path(args.file)
|
||||
check(source.is_file(), 'upload fixture not found: ' + str(source))
|
||||
content = source.read_bytes()
|
||||
|
||||
containers = api_containers(args.project)
|
||||
check(bool(containers), 'no api containers found for project ' + args.project)
|
||||
baseline = {name: peak_bytes(name) for name in containers}
|
||||
|
||||
c = Client()
|
||||
alice, _, _, chat = c.bootstrap()
|
||||
actor = {'id': alice['id'], 'token': alice['token']}
|
||||
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + source.name
|
||||
|
||||
barrier = threading.Barrier(args.concurrency)
|
||||
lock = threading.Lock()
|
||||
latencies, failures = [], []
|
||||
|
||||
def worker(index):
|
||||
local = Client(timeout=120)
|
||||
barrier.wait()
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = local.call('POST', path, 201, actor=actor, raw=content,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
with lock:
|
||||
latencies.append(time.monotonic() - started)
|
||||
check(response['json']['size_bytes'] == len(content), 'uploaded size mismatch')
|
||||
except Failure as exc:
|
||||
with lock:
|
||||
failures.append(str(exc))
|
||||
|
||||
started = time.monotonic()
|
||||
threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
peaks = {name: peak_bytes(name) for name in containers}
|
||||
peak_delta = {}
|
||||
for name in containers:
|
||||
before = baseline[name]
|
||||
after = peaks[name]
|
||||
if before is None or after is None:
|
||||
peak_delta[name] = None
|
||||
else:
|
||||
peak_delta[name] = after - before
|
||||
|
||||
ordered = sorted(latencies)
|
||||
def percentile(p):
|
||||
return round(1000 * ordered[max(0, math.ceil(p * len(ordered)) - 1)], 1) if ordered else None
|
||||
|
||||
report = {
|
||||
'concurrency': args.concurrency,
|
||||
'uploads': args.concurrency,
|
||||
'fixture': source.name,
|
||||
'fixture_bytes': len(content),
|
||||
'elapsed_seconds': round(elapsed, 3),
|
||||
'successful': len(latencies),
|
||||
'failed': len(failures),
|
||||
'failures': failures[:3],
|
||||
'latency_ms': {'p50': percentile(0.50), 'p95': percentile(0.95), 'p99': percentile(0.99)},
|
||||
'api_containers': containers,
|
||||
'peak_bytes_before': baseline,
|
||||
'peak_bytes_after': peaks,
|
||||
'peak_delta_bytes': peak_delta,
|
||||
'peak_max_mb': round(max(v for v in peaks.values() if v is not None) / (1024 * 1024), 2) if any(v is not None for v in peaks.values()) else None,
|
||||
}
|
||||
text = json.dumps(report, indent=2)
|
||||
if args.output:
|
||||
destination = Path(args.output)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(text + '\n')
|
||||
print(text)
|
||||
check(not failures, 'some uploads failed')
|
||||
check(len(latencies) == args.concurrency, 'not all uploads succeeded')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Single binary upload for failure drills; asserts the expected status code."""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument('--expect', type=int, default=201)
|
||||
p.add_argument('--file', default=str(ROOT / 'scripts/fixtures/sample.jpg'))
|
||||
p.add_argument('--content-type', default='image/jpeg')
|
||||
|
||||
args = arguments('Single upload used by compensation drills; creates synthetic data', configure)
|
||||
source = Path(args.file)
|
||||
check(source.is_file(), 'file not found: ' + str(source))
|
||||
content = source.read_bytes()
|
||||
|
||||
c = Client()
|
||||
alice, _, _, chat = c.bootstrap()
|
||||
response = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + source.name,
|
||||
args.expect, actor=alice, raw=content,
|
||||
headers={'Content-Type': args.content_type}, schema='Attachment')
|
||||
print(f'status={response["status"]}')
|
||||
if response['status'] == 201:
|
||||
attachment = response['json']
|
||||
print(f"attachment={attachment['id']} size={attachment['size_bytes']}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Reference in New Issue
Block a user