lab4 s3 lvl4

This commit is contained in:
2026-11-01 01:18:40 +03:00
parent ec540a1a72
commit 6c0a3d8912
27 changed files with 1945 additions and 0 deletions
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env bash
# Host-side: S3 (9000/9001) must not be published anywhere; the network must be internal.
set -eu
pub=$(docker compose ps s3 --format '{{.Publishers}}')
echo "s3 publishers: ${pub:-none}"
case "$pub" in ""|"[]") ;; *) echo "FAIL: s3 publishes ports" >&2; exit 1;; esac
for port in 9000 9001; do
if (exec 3<>/dev/tcp/127.0.0.1/$port) 2>/dev/null; then echo "FAIL: host can reach :$port" >&2; exit 1; fi
done
echo "PASS: S3 and console unreachable from host; run 'make check-s3' for in-network checks"
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
"""Runs INSIDE the backend network but WITHOUT the API: anonymous access is denied, app credentials are least-privilege."""
import os, secrets, sys, urllib.request, urllib.error, socket
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
EP, B = os.environ["S3_ENDPOINT"], os.environ["S3_BUCKET"]
s3 = boto3.client("s3", endpoint_url=EP, region_name=os.getenv("S3_REGION", "us-east-1"), aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],
aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"], config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, retries={"max_attempts": 1}))
bad = []
def expect(name, ok): print(("PASS " if ok else "FAIL ") + name); (ok or bad.append(name))
def anon(method, path, data=None):
try: return urllib.request.urlopen(urllib.request.Request(EP + path, method=method, data=data), timeout=5).status
except urllib.error.HTTPError as e: return e.code
def denied(fn):
try: fn(); return False
except ClientError as e: return e.response["Error"]["Code"] in ("AccessDenied", "403")
key = "attachments/probe-" + secrets.token_hex(8)
s3.put_object(Bucket=B, Key=key, Body=b"probe")
try:
expect("anonymous GET object -> 403", anon("GET", f"/{B}/{key}") == 403)
expect("anonymous HEAD object -> 403", anon("HEAD", f"/{B}/{key}") == 403)
expect("anonymous list bucket -> 403", anon("GET", f"/{B}/") == 403)
expect("anonymous PUT -> 403", anon("PUT", f"/{B}/attachments/anon-{secrets.token_hex(4)}", b"x") == 403)
finally:
s3.delete_object(Bucket=B, Key=key)
expect("app creds: ListBuckets denied", denied(lambda: s3.list_buckets()))
expect("app creds: CreateBucket denied", denied(lambda: s3.create_bucket(Bucket="x-" + secrets.token_hex(4))))
expect("app creds: DeleteBucket denied", denied(lambda: s3.delete_bucket(Bucket=B)))
expect("app creds: PutBucketPolicy denied", denied(lambda: s3.put_bucket_policy(Bucket=B, Policy="{}")))
expect("app creds: PutObject outside attachments/ denied", denied(lambda: s3.put_object(Bucket=B, Key="other/x", Body=b"x")))
host = EP.split("//")[1].split(":")[0]
for port in (9001, 9090):
with socket.socket() as sk:
sk.settimeout(2); expect(f"admin console port {port} closed", sk.connect_ex((host, port)) != 0)
sys.exit(1 if bad else 0)
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env python3
"""Apply SQL migrations once, before any API replica is started."""
from pathlib import Path
import os
import time
import psycopg
ROOT = Path(__file__).resolve().parents[1]
CONNINFO = os.getenv("DATABASE_URL", "")
def main():
last_error = None
for _ in range(30):
try:
with psycopg.connect(CONNINFO, connect_timeout=3) as conn:
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())")
conn.commit()
for path in sorted((ROOT / "migrations").glob("*.sql")):
version = path.name
if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (version,)).fetchone():
continue
sql = path.read_text(encoding="utf-8")
with conn.transaction():
for statement in sql.split(";"):
if statement.strip():
conn.execute(statement)
conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (version,))
print(f"APPLIED {version}", flush=True)
print("PASS: migrations complete", flush=True)
return
except Exception as exc:
last_error = exc
time.sleep(1)
raise SystemExit(f"Migration failed after retries: {type(last_error).__name__}")
if __name__ == "__main__":
main()
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
# Restart ALL state stores (PostgreSQL, Redis, S3), then every API replica; named volumes are preserved.
docker compose restart postgres redis s3
docker compose restart api
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
set -eu
# Sequential replacement-equivalent exercise for Compose (which has no native rolling-update primitive).
# Run a load generator in another terminal while this script executes.
replicas="$(docker compose ps -q --all api)"
[ -n "$replicas" ] || { echo "No API containers found; start the stack first" >&2; exit 1; }
for id in $replicas; do
# Remove one replica; the LB excludes it after health-check failure.
docker stop --time 10 "$id" >/dev/null || true
docker rm -f "$id" >/dev/null
docker compose up -d --no-deps --scale api=2 api
# Wait for at least two healthy API containers before moving to the next one.
tries=0
while [ "$tries" -lt 30 ]; do
healthy=0
for candidate in $(docker compose ps -q api); do
status=$(docker inspect --format '{{.State.Health.Status}}' "$candidate" 2>/dev/null || echo starting)
[ "$status" = healthy ] && healthy=$((healthy+1))
done
[ "$healthy" -ge 2 ] && break
tries=$((tries+1)); sleep 1
done
[ "$healthy" -ge 2 ] || { echo "Fewer than two healthy replicas after replacement" >&2; exit 1; }
done
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env python3
"""TEST-ONLY helper: puts an orphan object (as if the API died between S3 PUT and COMMIT)."""
import os, secrets, boto3
from botocore.config import Config
s3 = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name=os.getenv("S3_REGION", "us-east-1"),
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"], aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}))
key = "attachments/" + secrets.token_hex(16)
s3.put_object(Bucket=os.environ["S3_BUCKET"], Key=key, Body=b"orphan-test", ContentType="image/png")
print(key)
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
"""Orphan sweeper: deletes S3 objects under attachments/ that have no row in `attachments`.
Safety: only objects older than --grace-seconds are candidates (default 3600, far above the few seconds between
PUT and COMMIT), and the DB is re-checked immediately before each delete. Default mode is --dry-run.
"""
import argparse, os, sys, time
from datetime import datetime, timezone
import boto3, psycopg
from botocore.config import Config
def run_once(s3, conn, bucket, grace, apply):
now = datetime.now(timezone.utc)
seen = old = orphans = deleted = 0
for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix="attachments/"):
for obj in page.get("Contents", []):
seen += 1
if obj["Key"] == "attachments/.ready":
continue
if (now - obj["LastModified"]).total_seconds() < grace:
continue # fresh: may still be between PUT and COMMIT
old += 1
if conn.execute("SELECT 1 FROM attachments WHERE s3_key=%s", (obj["Key"],)).fetchone():
continue
orphans += 1
print(("DELETE " if apply else "WOULD-DELETE ") + obj["Key"], flush=True)
if apply:
# re-check right before delete (closes the race with a concurrent COMMIT)
if not conn.execute("SELECT 1 FROM attachments WHERE s3_key=%s", (obj["Key"],)).fetchone():
s3.delete_object(Bucket=bucket, Key=obj["Key"]); deleted += 1
print(f"sweep: seen={seen} older_than_grace={old} orphans={orphans} deleted={deleted} apply={apply}", flush=True)
return orphans, deleted
def main():
p = argparse.ArgumentParser()
p.add_argument("--apply", action="store_true", help="really delete (default: dry-run)")
p.add_argument("--grace-seconds", type=int, default=int(os.getenv("SWEEP_GRACE_SECONDS", "3600")))
p.add_argument("--loop", action="store_true"); p.add_argument("--interval", type=int, default=int(os.getenv("SWEEP_INTERVAL_SECONDS", "600")))
a = p.parse_args()
if a.grace_seconds < 0: sys.exit("grace must be >= 0")
s3 = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name=os.getenv("S3_REGION", "us-east-1"),
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"], aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=3, read_timeout=15, retries={"max_attempts": 2}))
bucket = os.environ["S3_BUCKET"]
while True:
try:
with psycopg.connect(connect_timeout=5, autocommit=True) as conn:
run_once(s3, conn, bucket, a.grace_seconds, a.apply)
except Exception as exc: # keep looping through outages
print("sweep failed:", type(exc).__name__, flush=True)
if not a.loop: raise
if not a.loop: return
time.sleep(a.interval)
if __name__ == "__main__":
main()
+17
View File
@@ -0,0 +1,17 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID must be supplied by tests/resilience.py}"
found=""
for id in $(docker compose ps -q api); do
host=$(docker inspect --format '{{.Config.Hostname}}' "$id")
if [ "$host" = "$TARGET_INSTANCE_ID" ]; then
found="$id"
break
fi
done
if [ -z "$found" ]; then
echo "No running API container matches TARGET_INSTANCE_ID" >&2
exit 1
fi
# Stop exactly one API replica and intentionally leave it stopped for the failover check.
docker stop --time 10 "$found" >/dev/null