forked from mtusi2026/mtusi2026lab4
Реализовать ЛР4: S3-хранилище и вложения на оценку 3
Стенд: PostgreSQL 16 + две API-реплики за HAProxy + SeaweedFS S3 в приватном bucket. Метаданные вложений в PostgreSQL, байты в S3, object key назначает сервер. Лимиты 10 MiB и 40 млн пикселей, MIME по магическим байтам, проверка filename. Добавлены smoke/негативные тесты, хуки restart/stop-one, сценарий отказа S3 и отчёт с доказательствами.
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
"""In-container probe: status and error code of attachment endpoints.
|
||||
|
||||
Used as evidence that a replica answers with the common JSON error format while
|
||||
S3 is unavailable (the load balancer may answer 503 on its own before this).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
TOKEN = os.environ["TOKEN"]
|
||||
ATTACHMENT_ID = os.environ["ATTACHMENT_ID"]
|
||||
|
||||
|
||||
def probe(url):
|
||||
request = urllib.request.Request(url, headers={"Authorization": "Bearer " + TOKEN})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
return {"status": response.status}
|
||||
except urllib.error.HTTPError as error:
|
||||
payload = json.loads(error.read().decode("utf-8"))
|
||||
return {
|
||||
"status": error.code,
|
||||
"error_code": payload["error"]["code"],
|
||||
"has_request_id": bool(error.headers.get("X-Request-Id")),
|
||||
}
|
||||
|
||||
|
||||
base = "http://127.0.0.1:8080/api/v1/attachments/" + ATTACHMENT_ID
|
||||
print(json.dumps({"metadata": probe(base), "content": probe(base + "/content")}))
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Explicit evidence scenario: behaviour when the S3 backend is unavailable.
|
||||
|
||||
Stops the S3 service, observes 503 responses through the load balancer and from
|
||||
an API replica, then starts it again and verifies that the previously uploaded
|
||||
original is still downloadable with the same checksum.
|
||||
This script is NOT part of `make test`: it deliberately stops infrastructure.
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, Failure # noqa: E402
|
||||
|
||||
|
||||
def compose(*arguments):
|
||||
command = ["docker", "compose", "--env-file", ".env", *arguments]
|
||||
result = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError("compose command failed: " + " ".join(arguments))
|
||||
return result
|
||||
|
||||
|
||||
def raw_status(method, url, token=None, body=None, content_type=None):
|
||||
headers = {}
|
||||
if token:
|
||||
headers["Authorization"] = "Bearer " + token
|
||||
if content_type:
|
||||
headers["Content-Type"] = content_type
|
||||
request = urllib.request.Request(url, data=body, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
return response.status
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code
|
||||
|
||||
|
||||
def main():
|
||||
base = os.environ.get("BASE_URL", "http://127.0.0.1:8081")
|
||||
client = Client(base)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
attachment, content = client.attachment(alice, chat)
|
||||
print(
|
||||
"stage=baseline attachment=%s sha256=%s"
|
||||
% (attachment["id"], hashlib.sha256(content).hexdigest())
|
||||
)
|
||||
|
||||
compose("cp", "scripts/check_api_error.py", "api1:/tmp/check_api_error.py")
|
||||
compose("stop", "seaweedfs")
|
||||
try:
|
||||
print("stage=s3_down lb_readiness_status=%d" % raw_status("GET", base + "/health/ready"))
|
||||
print(
|
||||
"stage=s3_down lb_upload_status=%d"
|
||||
% raw_status(
|
||||
"POST",
|
||||
base + "/api/v1/chats/" + chat["id"] + "/attachments?filename=after-outage.png",
|
||||
token=alice["token"],
|
||||
body=content,
|
||||
content_type="image/png",
|
||||
)
|
||||
)
|
||||
print(
|
||||
"stage=s3_down lb_download_status=%d"
|
||||
% raw_status(
|
||||
"GET",
|
||||
base + "/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
token=bob["token"],
|
||||
)
|
||||
)
|
||||
probe = compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"-e",
|
||||
"TOKEN=" + bob["token"],
|
||||
"-e",
|
||||
"ATTACHMENT_ID=" + attachment["id"],
|
||||
"api1",
|
||||
"python",
|
||||
"/tmp/check_api_error.py",
|
||||
)
|
||||
print("stage=s3_down api_replica_probe=" + probe.stdout.strip())
|
||||
finally:
|
||||
compose("start", "seaweedfs")
|
||||
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
if client.call("GET", "/health/ready", schema="Health")["status"] == 200:
|
||||
break
|
||||
except Failure:
|
||||
pass
|
||||
if time.monotonic() >= deadline:
|
||||
raise SystemExit("stack did not become ready within 90 seconds")
|
||||
time.sleep(1)
|
||||
|
||||
restored = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)
|
||||
if restored["bytes"] != content:
|
||||
raise SystemExit("original changed after S3 restart")
|
||||
print("stage=s3_restored readiness=200 download=identical")
|
||||
print("PASS: S3 outage surfaced as 503 and the original survived the restart")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,29 @@
|
||||
@echo off
|
||||
rem Windows equivalent of restart.sh for local evidence collection.
|
||||
rem Restarts every stateful service and the API replicas without removing volumes.
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
if not exist ".env" (
|
||||
echo missing .env; copy .env.example and fill local values 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
|
||||
set "LAB4_PORT="
|
||||
for /f "usebackq tokens=1,* delims==" %%a in (".env") do (
|
||||
if /i "%%a"=="LAB4_PORT" set "LAB4_PORT=%%b"
|
||||
)
|
||||
if "%LAB4_PORT%"=="" set "LAB4_PORT=8080"
|
||||
|
||||
docker compose --env-file .env restart postgres seaweedfs haproxy api1 api2
|
||||
if errorlevel 1 exit /b 1
|
||||
|
||||
rem /health/ready covers PostgreSQL and S3, so a 200 means the whole synchronous
|
||||
rem path is usable again. Wait for it before reporting success.
|
||||
rem ping is used instead of timeout: timeout fails when stdin is redirected.
|
||||
for /l %%i in (1,1,90) do (
|
||||
powershell -NoProfile -Command "try { if ((Invoke-WebRequest -Uri 'http://127.0.0.1:%LAB4_PORT%/health/ready' -UseBasicParsing -TimeoutSec 3).StatusCode -eq 200) { exit 0 } else { exit 1 } } catch { exit 1 }" >nul 2>&1
|
||||
if not errorlevel 1 exit /b 0
|
||||
ping -n 2 127.0.0.1 >nul
|
||||
)
|
||||
echo stack did not become ready within 90 seconds 1>&2
|
||||
exit /b 1
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# Restart every process that owns state, without removing volumes or re-seeding.
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [ ! -f .env ]; then
|
||||
echo "missing .env; copy .env.example and fill local values" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
docker compose --env-file .env restart postgres seaweedfs haproxy api1 api2
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
. ./.env
|
||||
port="${LAB4_PORT:-8080}"
|
||||
|
||||
# /health/ready covers PostgreSQL and S3, so a 200 means the whole synchronous
|
||||
# path is usable again. Wait for it before reporting success to the caller.
|
||||
i=0
|
||||
while [ "$i" -lt 90 ]; do
|
||||
if curl -fsS "http://127.0.0.1:${port}/health/ready" >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "stack did not become ready within 90 seconds" >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,14 @@
|
||||
@echo off
|
||||
rem Windows equivalent of stop-one.sh: stop exactly one API replica and leave it stopped.
|
||||
setlocal
|
||||
if "%TARGET_INSTANCE_ID%"=="" (
|
||||
echo test provides target API instance alias 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
if /i not "%TARGET_INSTANCE_ID%"=="api1" if /i not "%TARGET_INSTANCE_ID%"=="api2" (
|
||||
echo unexpected API instance alias 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
docker stop lab4-%TARGET_INSTANCE_ID% >nul
|
||||
if errorlevel 1 exit /b 1
|
||||
exit /b 0
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
api1|api2)
|
||||
;;
|
||||
*)
|
||||
echo "unexpected API instance alias" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
# Leave exactly one API replica stopped. The test restores it explicitly later.
|
||||
docker stop "lab4-${TARGET_INSTANCE_ID}" >/dev/null
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Evidence helper: the bucket must not serve objects to anonymous clients.
|
||||
|
||||
Runs inside an API container (it needs the API's S3 credentials and network
|
||||
access to the S3 endpoint). Prints a small JSON summary without secrets.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from urllib.parse import quote
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config as BotoConfig
|
||||
|
||||
|
||||
def main():
|
||||
endpoint = os.environ["S3_ENDPOINT"].rstrip("/")
|
||||
bucket = os.environ["S3_BUCKET"]
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
region_name=os.environ.get("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],
|
||||
aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
|
||||
config=BotoConfig(signature_version="s3v4", s3={"addressing_style": "path"}),
|
||||
)
|
||||
objects = client.list_objects_v2(Bucket=bucket).get("Contents", [])
|
||||
summary = {"objects_in_bucket": len(objects), "first_key": None}
|
||||
if not objects:
|
||||
print(json.dumps(summary))
|
||||
return 0
|
||||
key = objects[0]["Key"]
|
||||
summary["first_key"] = key
|
||||
|
||||
def anonymous(url):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=5) as response:
|
||||
return response.status
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code
|
||||
|
||||
summary["anonymous_object_get_status"] = anonymous(endpoint + "/" + bucket + "/" + quote(key, safe="/"))
|
||||
summary["anonymous_bucket_list_status"] = anonymous(endpoint + "/" + bucket + "?list-type=2")
|
||||
print(json.dumps(summary, indent=2))
|
||||
return 0 if summary["anonymous_object_get_status"] >= 400 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user