forked from mtusi2026/mtusi2026lab4
lab4 s3 lvl4
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# Host-side: S3 (9000/9001) must not be published anywhere; the network must be internal.
|
||||
set -eu
|
||||
pub=$(docker compose ps s3 --format '{{.Publishers}}')
|
||||
echo "s3 publishers: ${pub:-none}"
|
||||
case "$pub" in ""|"[]") ;; *) echo "FAIL: s3 publishes ports" >&2; exit 1;; esac
|
||||
for port in 9000 9001; do
|
||||
if (exec 3<>/dev/tcp/127.0.0.1/$port) 2>/dev/null; then echo "FAIL: host can reach :$port" >&2; exit 1; fi
|
||||
done
|
||||
echo "PASS: S3 and console unreachable from host; run 'make check-s3' for in-network checks"
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Runs INSIDE the backend network but WITHOUT the API: anonymous access is denied, app credentials are least-privilege."""
|
||||
import os, secrets, sys, urllib.request, urllib.error, socket
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
EP, B = os.environ["S3_ENDPOINT"], os.environ["S3_BUCKET"]
|
||||
s3 = boto3.client("s3", endpoint_url=EP, region_name=os.getenv("S3_REGION", "us-east-1"), aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],
|
||||
aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"], config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, retries={"max_attempts": 1}))
|
||||
bad = []
|
||||
def expect(name, ok): print(("PASS " if ok else "FAIL ") + name); (ok or bad.append(name))
|
||||
def anon(method, path, data=None):
|
||||
try: return urllib.request.urlopen(urllib.request.Request(EP + path, method=method, data=data), timeout=5).status
|
||||
except urllib.error.HTTPError as e: return e.code
|
||||
def denied(fn):
|
||||
try: fn(); return False
|
||||
except ClientError as e: return e.response["Error"]["Code"] in ("AccessDenied", "403")
|
||||
key = "attachments/probe-" + secrets.token_hex(8)
|
||||
s3.put_object(Bucket=B, Key=key, Body=b"probe")
|
||||
try:
|
||||
expect("anonymous GET object -> 403", anon("GET", f"/{B}/{key}") == 403)
|
||||
expect("anonymous HEAD object -> 403", anon("HEAD", f"/{B}/{key}") == 403)
|
||||
expect("anonymous list bucket -> 403", anon("GET", f"/{B}/") == 403)
|
||||
expect("anonymous PUT -> 403", anon("PUT", f"/{B}/attachments/anon-{secrets.token_hex(4)}", b"x") == 403)
|
||||
finally:
|
||||
s3.delete_object(Bucket=B, Key=key)
|
||||
expect("app creds: ListBuckets denied", denied(lambda: s3.list_buckets()))
|
||||
expect("app creds: CreateBucket denied", denied(lambda: s3.create_bucket(Bucket="x-" + secrets.token_hex(4))))
|
||||
expect("app creds: DeleteBucket denied", denied(lambda: s3.delete_bucket(Bucket=B)))
|
||||
expect("app creds: PutBucketPolicy denied", denied(lambda: s3.put_bucket_policy(Bucket=B, Policy="{}")))
|
||||
expect("app creds: PutObject outside attachments/ denied", denied(lambda: s3.put_object(Bucket=B, Key="other/x", Body=b"x")))
|
||||
host = EP.split("//")[1].split(":")[0]
|
||||
for port in (9001, 9090):
|
||||
with socket.socket() as sk:
|
||||
sk.settimeout(2); expect(f"admin console port {port} closed", sk.connect_ex((host, port)) != 0)
|
||||
sys.exit(1 if bad else 0)
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Apply SQL migrations once, before any API replica is started."""
|
||||
from pathlib import Path
|
||||
import os
|
||||
import time
|
||||
import psycopg
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CONNINFO = os.getenv("DATABASE_URL", "")
|
||||
|
||||
def main():
|
||||
last_error = None
|
||||
for _ in range(30):
|
||||
try:
|
||||
with psycopg.connect(CONNINFO, connect_timeout=3) as conn:
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())")
|
||||
conn.commit()
|
||||
for path in sorted((ROOT / "migrations").glob("*.sql")):
|
||||
version = path.name
|
||||
if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (version,)).fetchone():
|
||||
continue
|
||||
sql = path.read_text(encoding="utf-8")
|
||||
with conn.transaction():
|
||||
for statement in sql.split(";"):
|
||||
if statement.strip():
|
||||
conn.execute(statement)
|
||||
conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (version,))
|
||||
print(f"APPLIED {version}", flush=True)
|
||||
print("PASS: migrations complete", flush=True)
|
||||
return
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
time.sleep(1)
|
||||
raise SystemExit(f"Migration failed after retries: {type(last_error).__name__}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Restart ALL state stores (PostgreSQL, Redis, S3), then every API replica; named volumes are preserved.
|
||||
docker compose restart postgres redis s3
|
||||
docker compose restart api
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Sequential replacement-equivalent exercise for Compose (which has no native rolling-update primitive).
|
||||
# Run a load generator in another terminal while this script executes.
|
||||
replicas="$(docker compose ps -q --all api)"
|
||||
[ -n "$replicas" ] || { echo "No API containers found; start the stack first" >&2; exit 1; }
|
||||
for id in $replicas; do
|
||||
# Remove one replica; the LB excludes it after health-check failure.
|
||||
docker stop --time 10 "$id" >/dev/null || true
|
||||
docker rm -f "$id" >/dev/null
|
||||
docker compose up -d --no-deps --scale api=2 api
|
||||
# Wait for at least two healthy API containers before moving to the next one.
|
||||
tries=0
|
||||
while [ "$tries" -lt 30 ]; do
|
||||
healthy=0
|
||||
for candidate in $(docker compose ps -q api); do
|
||||
status=$(docker inspect --format '{{.State.Health.Status}}' "$candidate" 2>/dev/null || echo starting)
|
||||
[ "$status" = healthy ] && healthy=$((healthy+1))
|
||||
done
|
||||
[ "$healthy" -ge 2 ] && break
|
||||
tries=$((tries+1)); sleep 1
|
||||
done
|
||||
[ "$healthy" -ge 2 ] || { echo "Fewer than two healthy replicas after replacement" >&2; exit 1; }
|
||||
done
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""TEST-ONLY helper: puts an orphan object (as if the API died between S3 PUT and COMMIT)."""
|
||||
import os, secrets, boto3
|
||||
from botocore.config import Config
|
||||
s3 = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name=os.getenv("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"], aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
|
||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}))
|
||||
key = "attachments/" + secrets.token_hex(16)
|
||||
s3.put_object(Bucket=os.environ["S3_BUCKET"], Key=key, Body=b"orphan-test", ContentType="image/png")
|
||||
print(key)
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Orphan sweeper: deletes S3 objects under attachments/ that have no row in `attachments`.
|
||||
|
||||
Safety: only objects older than --grace-seconds are candidates (default 3600, far above the few seconds between
|
||||
PUT and COMMIT), and the DB is re-checked immediately before each delete. Default mode is --dry-run.
|
||||
"""
|
||||
import argparse, os, sys, time
|
||||
from datetime import datetime, timezone
|
||||
import boto3, psycopg
|
||||
from botocore.config import Config
|
||||
|
||||
def run_once(s3, conn, bucket, grace, apply):
|
||||
now = datetime.now(timezone.utc)
|
||||
seen = old = orphans = deleted = 0
|
||||
for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix="attachments/"):
|
||||
for obj in page.get("Contents", []):
|
||||
seen += 1
|
||||
if obj["Key"] == "attachments/.ready":
|
||||
continue
|
||||
|
||||
if (now - obj["LastModified"]).total_seconds() < grace:
|
||||
continue # fresh: may still be between PUT and COMMIT
|
||||
old += 1
|
||||
if conn.execute("SELECT 1 FROM attachments WHERE s3_key=%s", (obj["Key"],)).fetchone():
|
||||
continue
|
||||
orphans += 1
|
||||
print(("DELETE " if apply else "WOULD-DELETE ") + obj["Key"], flush=True)
|
||||
if apply:
|
||||
# re-check right before delete (closes the race with a concurrent COMMIT)
|
||||
if not conn.execute("SELECT 1 FROM attachments WHERE s3_key=%s", (obj["Key"],)).fetchone():
|
||||
s3.delete_object(Bucket=bucket, Key=obj["Key"]); deleted += 1
|
||||
print(f"sweep: seen={seen} older_than_grace={old} orphans={orphans} deleted={deleted} apply={apply}", flush=True)
|
||||
return orphans, deleted
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--apply", action="store_true", help="really delete (default: dry-run)")
|
||||
p.add_argument("--grace-seconds", type=int, default=int(os.getenv("SWEEP_GRACE_SECONDS", "3600")))
|
||||
p.add_argument("--loop", action="store_true"); p.add_argument("--interval", type=int, default=int(os.getenv("SWEEP_INTERVAL_SECONDS", "600")))
|
||||
a = p.parse_args()
|
||||
if a.grace_seconds < 0: sys.exit("grace must be >= 0")
|
||||
s3 = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name=os.getenv("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"], aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
|
||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=3, read_timeout=15, retries={"max_attempts": 2}))
|
||||
bucket = os.environ["S3_BUCKET"]
|
||||
while True:
|
||||
try:
|
||||
with psycopg.connect(connect_timeout=5, autocommit=True) as conn:
|
||||
run_once(s3, conn, bucket, a.grace_seconds, a.apply)
|
||||
except Exception as exc: # keep looping through outages
|
||||
print("sweep failed:", type(exc).__name__, flush=True)
|
||||
if not a.loop: raise
|
||||
if not a.loop: return
|
||||
time.sleep(a.interval)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID must be supplied by tests/resilience.py}"
|
||||
found=""
|
||||
for id in $(docker compose ps -q api); do
|
||||
host=$(docker inspect --format '{{.Config.Hostname}}' "$id")
|
||||
if [ "$host" = "$TARGET_INSTANCE_ID" ]; then
|
||||
found="$id"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ -z "$found" ]; then
|
||||
echo "No running API container matches TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Stop exactly one API replica and intentionally leave it stopped for the failover check.
|
||||
docker stop --time 10 "$found" >/dev/null
|
||||
Reference in New Issue
Block a user