Добавил загрузку картинок в S3 и проверку доступа к вложениям

This commit is contained in:
2026-11-17 13:19:43 +03:00
parent ec540a1a72
commit 372471ae58
35 changed files with 2004 additions and 19 deletions
+4 -19
View File
@@ -1,20 +1,5 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_PORT=8084
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
SESSION_TTL_SECONDS=3600
APP_ENV=lab
APP_VERSION=lab4-v1
+6
View File
@@ -0,0 +1,6 @@
*.py text eol=lf
*.sh text eol=lf
*.sql text eol=lf
*.yaml text eol=lf
*.cfg text eol=lf
evidence/* -whitespace
+27
View File
@@ -0,0 +1,27 @@
FROM docker:28.5.1-cli AS docker_cli
FROM python:3.13.15-slim-bookworm AS runtime
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 \
APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt \
&& useradd --uid 10001 --create-home messenger
COPY app ./app
COPY migrations ./migrations
COPY contracts/openapi.json ./contracts/openapi.json
USER messenger
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=5 \
CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2)"
CMD ["python", "-m", "app"]
# Optional test operator. The API image does not contain the Docker client/socket.
FROM runtime AS checks
USER root
COPY --from=docker_cli /usr/local/bin/docker /usr/local/bin/docker
COPY --from=docker_cli /usr/local/libexec/docker/cli-plugins/docker-compose /usr/local/libexec/docker/cli-plugins/docker-compose
RUN apt-get update && apt-get install -y --no-install-recommends make \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /checks
CMD ["make", "test-student"]
+4
View File
@@ -28,3 +28,7 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: test-student
test-student:
$(PYTHON) -m unittest discover -s student-tests -v
+1
View File
@@ -0,0 +1 @@
"""Laboratory 1: a single-process, in-memory messenger."""
+17
View File
@@ -0,0 +1,17 @@
import logging
import os
import uvicorn
def main():
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
logging.getLogger("psycopg.pool").setLevel(logging.CRITICAL)
logging.info("Starting lab3 environment=%s with shared PostgreSQL and Redis", os.getenv("APP_ENV", "lab"))
uvicorn.run("app.api:app", host=os.getenv("APP_HOST", "0.0.0.0"),
port=int(os.getenv("APP_PORT", "8080")), workers=1,
access_log=False, timeout_graceful_shutdown=5)
if __name__ == "__main__":
main()
+196
View File
@@ -0,0 +1,196 @@
import asyncio
from contextlib import asynccontextmanager
import json
import logging
import os
from pathlib import Path
import re
from time import perf_counter
from uuid import uuid4
from fastapi import Depends, FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from fastapi.responses import Response
import psycopg
from psycopg_pool import PoolTimeout, TooManyRequests
from redis.exceptions import RedisError
from pydantic import ValidationError
from starlette.exceptions import HTTPException
from .config import Settings
from .errors import APIError, invalid, unavailable
from .models import CreateChat, CreateMessage, CreateUser
from .security import basic_credentials, identifier
from .store import Store
from .attachments import Attachments, metadata
from botocore.exceptions import BotoCoreError, ClientError
from starlette.requests import ClientDisconnect
logger = logging.getLogger("messenger")
def error_response(request, status, code, message, headers=None):
return JSONResponse(status_code=status, headers=headers, content={"error": {
"code": code, "message": message, "request_id": request.state.request_id}})
async def body(request, model):
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if content_type != "application/json":
raise APIError(415, "unsupported_media_type", "Use application/json")
try:
raw = (await request.body()).decode("utf-8")
def reject_constant(value):
raise ValueError("Nonstandard JSON constant")
data = json.loads(raw, parse_constant=reject_constant)
return model.model_validate(data)
except (ValueError, UnicodeError, ValidationError):
raise invalid("Body must be valid UTF-8 JSON with the required fields and types") from None
def pagination(request):
raw = request.query_params.get("limit", "50")
if re.fullmatch(r"[0-9]+", raw) is None or len(raw) > 3 or not 1 <= int(raw) <= 100:
raise invalid("Limit must be an integer from 1 to 100")
return int(raw), request.query_params.get("cursor")
def create_app(settings=None):
@asynccontextmanager
async def lifespan(app):
store = Store(settings or Settings.load())
app.state.store = store
app.state.attachments = Attachments(store)
try:
await store.open()
yield
finally:
await store.close()
instance_id = os.getenv("HOSTNAME", "api") + "-" + uuid4().hex[:12]
app = FastAPI(title="Lab 4 messenger", docs_url=None, redoc_url=None,
openapi_url=None, redirect_slashes=False, lifespan=lifespan)
@app.middleware("http")
async def request_context(request, call_next):
request.state.request_id = str(uuid4())
start = perf_counter()
try:
async with asyncio.timeout(35):
response = await call_next(request)
except (psycopg.Error, PoolTimeout, TooManyRequests, RedisError, BotoCoreError, ClientError, TimeoutError, ClientDisconnect):
error = unavailable()
logger.warning("Storage unavailable request_id=%s", request.state.request_id)
response = error_response(request, error.status, error.code, error.message)
except Exception:
logger.error("Unhandled error request_id=%s", request.state.request_id)
response = error_response(request, 500, "internal_error", "Internal server error")
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = instance_id
response.headers["X-App-Version"] = os.getenv("APP_VERSION", "lab3-v1")
logger.info("request_id=%s method=%s path=%s status=%s duration_ms=%.2f",
request.state.request_id, request.method, request.url.path,
response.status_code, (perf_counter() - start) * 1000)
return response
@app.exception_handler(APIError)
async def api_error(request, error):
return error_response(request, error.status, error.code, error.message, error.headers)
@app.exception_handler(RequestValidationError)
async def validation_error(request, error):
return error_response(request, 400, "invalid_request", "Invalid request parameters")
@app.exception_handler(HTTPException)
async def routing_error(request, error):
code = "not_found" if error.status_code == 404 else "invalid_request"
return error_response(request, error.status_code, code, "Unknown path or unsupported method")
async def current_user(request: Request):
user, token = await app.state.store.authenticate(request.headers.get("authorization"))
request.state.session_token = token
return user
@app.get("/health/live")
async def health():
return {"status": "ok"}
@app.get("/health/ready")
async def readiness():
await app.state.store.ready()
await asyncio.to_thread(app.state.attachments.s3.head_bucket, Bucket="attachments")
return {"status": "ok"}
@app.post("/api/v1/auth/sessions", status_code=201)
async def login(request: Request):
credentials = basic_credentials(request.headers.get("authorization"))
if await request.body():
raise invalid("Session creation does not accept a body")
return await app.state.store.login(*credentials)
@app.delete("/api/v1/auth/sessions/current", status_code=204)
async def logout(request: Request, user=Depends(current_user)):
await app.state.store.logout(request.state.session_token)
return Response(status_code=204)
@app.post("/api/v1/users", status_code=201)
async def create_user(request: Request):
return await app.state.store.create_user(await body(request, CreateUser))
@app.get("/api/v1/users/me")
async def me(user=Depends(current_user)):
return user
@app.post("/api/v1/chats", status_code=201)
async def create_chat(request: Request, user=Depends(current_user)):
return await app.state.store.create_chat(user["id"], await body(request, CreateChat))
@app.get("/api/v1/chats")
async def list_chats(request: Request, user=Depends(current_user)):
return await app.state.store.list_chats(user["id"], *pagination(request))
@app.get("/api/v1/chats/{chat_id}")
async def get_chat(chat_id: str, user=Depends(current_user)):
return await app.state.store.get_chat(user["id"], identifier(chat_id))
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
async def create_message(chat_id: str, request: Request, user=Depends(current_user)):
chat_id = identifier(chat_id)
await app.state.store.get_chat(user["id"], chat_id)
key = request.headers.get("idempotency-key", "")
if re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key, flags=re.ASCII) is None:
raise invalid("A valid Idempotency-Key is required")
return await app.state.store.create_message(user["id"], chat_id,
await body(request, CreateMessage), key)
@app.get("/api/v1/chats/{chat_id}/messages")
async def list_messages(chat_id: str, request: Request, user=Depends(current_user)):
chat_id = identifier(chat_id)
await app.state.store.get_chat(user["id"], chat_id)
return await app.state.store.list_messages(user["id"], chat_id, *pagination(request))
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
async def upload(chat_id: str, request: Request, user=Depends(current_user)):
return await app.state.attachments.upload(user["id"], identifier(chat_id), request)
@app.get("/api/v1/attachments/{attachment_id}")
async def attachment(attachment_id: str, user=Depends(current_user)):
return metadata(await app.state.attachments.get(user["id"], identifier(attachment_id)))
@app.get("/api/v1/attachments/{attachment_id}/content")
async def content(attachment_id: str, request: Request, user=Depends(current_user)):
row = await app.state.attachments.get(user["id"], identifier(attachment_id))
if request.query_params.get("variant", "original") != "original":
raise APIError(404, "not_found", "Variant not found")
return await app.state.attachments.download(row)
@app.get("/openapi.json", include_in_schema=False)
async def contract():
path = Path(__file__).resolve().parents[1] / "contracts" / "openapi.json"
return json.loads(path.read_text(encoding="utf-8"))
return app
app = create_app()
+186
View File
@@ -0,0 +1,186 @@
"""Private S3 originals and durable upload intents."""
import asyncio
from contextlib import asynccontextmanager
import hashlib
import os
from pathlib import Path
import tempfile
import warnings
from uuid import uuid4, UUID
import boto3
from botocore.config import Config
from PIL import Image, UnidentifiedImageError
from starlette.responses import StreamingResponse
from .errors import APIError, invalid
from .store import public
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
BUCKET = 'attachments'
def secret(name):
return Path('/run/secrets/' + name).read_text().strip()
def client():
return boto3.client('s3', endpoint_url='http://minio:9000', region_name='us-east-1',
aws_access_key_id='messenger-api', aws_secret_access_key=secret('s3_secret_key'),
config=Config(connect_timeout=2, read_timeout=3, retries={'max_attempts': 0},
s3={'addressing_style': 'path'}, max_pool_connections=4))
def validate_image(file, content_type):
try:
with warnings.catch_warnings():
warnings.simplefilter('error', Image.DecompressionBombWarning)
file.seek(0)
with Image.open(file) as image:
if image.width * image.height > MAX_PIXELS:
raise APIError(413, 'payload_too_large', 'Too many pixels')
if image.format not in ('PNG', 'JPEG') or Image.MIME[image.format] != content_type:
raise APIError(415, 'unsupported_media_type', 'Image MIME does not match')
image.verify()
file.seek(0)
with Image.open(file) as image:
image.load()
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
raise APIError(413, 'payload_too_large', 'Too many pixels') from None
except (UnidentifiedImageError, OSError, SyntaxError, ValueError):
raise APIError(415, 'unsupported_media_type', 'Invalid image') from None
finally:
file.seek(0)
def metadata(row):
row = dict(row)
row['owner_id'] = row['uploader_id']
row['updated_at'] = row['created_at']
return public({k: row[k] for k in ('id','chat_id','owner_id','filename','content_type',
'size_bytes','sha256','status','created_at','updated_at')}) | {'variants': [], 'error_code': None}
async def blocking(function, *args, **kwargs):
# Cancellation must not close the spool or release the SQL lock while a thread uses it.
task = asyncio.create_task(asyncio.to_thread(function, *args, **kwargs))
try:
return await asyncio.shield(task)
except asyncio.CancelledError:
await task
raise
class Attachments:
def __init__(self, store):
self.store = store
self.s3 = client()
# One decode per replica bounds decoded RGBA memory to about 160 MB.
self.slots = asyncio.Semaphore(1)
@asynccontextmanager
async def slot(self):
try:
await asyncio.wait_for(self.slots.acquire(), timeout=10)
except TimeoutError:
raise APIError(503, 'unavailable', 'Upload capacity is busy') from None
try:
yield
finally:
self.slots.release()
async def get(self, actor, attachment_id):
async with self.store.pool.connection() as conn:
cur = await conn.execute('''SELECT a.* FROM attachments a JOIN chat_members m
ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s''',
(attachment_id, UUID(actor)))
row = await cur.fetchone()
if row is None:
raise APIError(404, 'not_found', 'Attachment not found')
if row['status'] != 'ready':
raise APIError(409, 'conflict', 'Attachment is not ready')
return row
async def upload(self, actor, chat_id, request):
await self.store.get_chat(actor, chat_id)
name = request.query_params.get('filename', '')
if not 1 <= len(name) <= 255 or any(c in '/\\' or ord(c) < 32 or ord(c) == 127 for c in name):
raise invalid('Invalid filename')
mime = request.headers.get('content-type', '').strip().lower()
if mime not in ('image/png','image/jpeg'):
raise APIError(415, 'unsupported_media_type', 'Use PNG or JPEG')
length = request.headers.get('content-length')
if length is not None:
if not length.isascii() or not length.isdecimal():
raise invalid('Invalid Content-Length')
if int(length) > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'File exceeds 10 MiB')
async with self.slot():
# Temporary spool is only staging, never the source of confirmed originals.
with tempfile.TemporaryFile() as file:
size = 0
digest = hashlib.sha256()
async with asyncio.timeout(20):
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'File exceeds 10 MiB')
file.write(chunk)
digest.update(chunk)
await blocking(validate_image, file, mime)
attachment_id = uuid4()
key = 'original/' + attachment_id.hex
# Commit the intent BEFORE PUT: even a killed process leaves a tracked key.
async with self.store.pool.connection() as conn:
await conn.execute('''INSERT INTO attachments
(id,chat_id,uploader_id,object_key,filename,content_type,size_bytes,sha256,status)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'pending')''',
(attachment_id,chat_id,UUID(actor),key,name,mime,size,digest.hexdigest()))
# Row lock also excludes the cleaner, including slow or cancelled PUTs.
async with self.store.pool.connection() as conn:
await conn.execute('SELECT id FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,))
await blocking(self.s3.put_object, Bucket=BUCKET, Key=key,
Body=file, ContentLength=size, ContentType=mime)
cur = await conn.execute("UPDATE attachments SET status='ready' WHERE id=%s RETURNING *",
(attachment_id,))
result = metadata(await cur.fetchone())
return result
async def download(self, row):
obj = await asyncio.to_thread(self.s3.get_object, Bucket=BUCKET, Key=row['object_key'])
stream = obj['Body']
async def chunks():
try:
while chunk := await asyncio.to_thread(stream.read, 64 * 1024):
yield chunk
finally:
stream.close()
return StreamingResponse(chunks(), media_type=row['content_type'], headers={
'Content-Length': str(row['size_bytes']), 'X-Content-Type-Options': 'nosniff'})
async def cleanup(self):
removed = 0
async with self.store.pool.connection() as conn:
cur = await conn.execute("""SELECT id,object_key FROM attachments WHERE status='pending'
AND created_at < clock_timestamp()-interval '1 hour' ORDER BY created_at
LIMIT 100 FOR UPDATE SKIP LOCKED""")
for row in await cur.fetchall():
await asyncio.to_thread(self.s3.delete_object, Bucket=BUCKET, Key=row['object_key'])
await conn.execute("DELETE FROM attachments WHERE id=%s AND status='pending'", (row['id'],))
removed += 1
return removed
async def cleaner():
from .config import Settings
from .store import Store
store = Store(Settings.load())
await store.open()
attachments = Attachments(store)
try:
while True:
try:
print({'removed_pending': await attachments.cleanup()}, flush=True)
except Exception:
print('Cleanup deferred; storage unavailable', flush=True)
await asyncio.sleep(60)
finally:
await store.close()
if __name__ == '__main__':
asyncio.run(cleaner())
+28
View File
@@ -0,0 +1,28 @@
"""Runtime settings; credential values are never logged."""
from dataclasses import dataclass
import os
from pathlib import Path
def secret(name):
return Path(os.environ[name + "_FILE"]).read_text(encoding="utf-8").strip()
@dataclass(frozen=True)
class Settings:
database_url: str
cache_url: str
cursor_key: bytes
session_ttl: int
@classmethod
def load(cls):
ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
if not 1 <= ttl <= 604800:
raise ValueError("SESSION_TTL_SECONDS must be in 1..604800")
if os.getenv("SESSION_STORE", "redis") != "redis":
raise ValueError("This deployment uses SESSION_STORE=redis")
key = secret("CURSOR_KEY").encode()
if len(key) < 32:
raise ValueError("Cursor key must contain at least 32 bytes")
return cls(secret("DATABASE_URL"), secret("CACHE_URL"), key, ttl)
+22
View File
@@ -0,0 +1,22 @@
class APIError(Exception):
def __init__(self, status: int, code: str, message: str, headers=None):
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
super().__init__(message)
def invalid(message="Invalid request"):
return APIError(400, "invalid_request", message)
def unauthorized(basic=False):
challenge = ('Basic realm="messenger", charset="UTF-8"' if basic
else 'Bearer realm="messenger"')
return APIError(401, "unauthorized", "Valid credentials required",
{"WWW-Authenticate": challenge})
def unavailable():
return APIError(503, "unavailable", "Storage temporarily unavailable")
+37
View File
@@ -0,0 +1,37 @@
"""Apply numbered migrations atomically; safe to repeat and serialize."""
import hashlib
from pathlib import Path
import psycopg
from .config import secret
ROOT = Path(__file__).resolve().parents[1]
def migrate():
with psycopg.connect(secret("DATABASE_URL"), connect_timeout=2,
options="-c statement_timeout=10000 -c lock_timeout=5000") as conn:
conn.execute("SELECT pg_advisory_xact_lock(202602)")
conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations (
version text PRIMARY KEY, sha256 text NOT NULL,
applied_at timestamptz NOT NULL DEFAULT clock_timestamp())""")
for path in sorted((ROOT / "migrations").glob("[0-9]*.sql")):
# Git checkout may change LF to CRLF on Windows; SQL is unchanged.
content = path.read_text(encoding="utf-8").encode("utf-8")
checksum = hashlib.sha256(content).hexdigest()
row = conn.execute("SELECT sha256 FROM schema_migrations WHERE version=%s", (path.name,)).fetchone()
if row:
if row[0] != checksum:
raise RuntimeError("Applied migration checksum differs: " + path.name)
print("Unchanged:", path.name)
continue
conn.execute(content.decode("utf-8"))
conn.execute("INSERT INTO schema_migrations(version, sha256) VALUES (%s, %s)",
(path.name, checksum))
print("Applied:", path.name)
print("PASS: migrations committed")
if __name__ == "__main__":
migrate()
+73
View File
@@ -0,0 +1,73 @@
"""Strict request models; strings retain their original Unicode content."""
import re
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
class Input(BaseModel):
model_config = ConfigDict(strict=True, extra="forbid")
@field_validator("*", check_fields=False)
@classmethod
def valid_text(cls, value):
if isinstance(value, str):
if not value.strip():
raise ValueError("A string must not be blank")
value.encode("utf-8")
if "\x00" in value:
raise ValueError("NUL is not supported")
return value
class CreateUser(Input):
username: str
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator("username")
@classmethod
def username_format(cls, value):
if re.fullmatch(r"[a-z0-9_]{3,32}", value) is None:
raise ValueError("Invalid username")
return value
class CreateChat(Input):
title: str = Field(min_length=1, max_length=100)
member_ids: list[str] = Field(min_length=1, max_length=100)
@field_validator("member_ids")
@classmethod
def members(cls, values):
normalized = [str(UUID(value)) for value in values]
if len(set(normalized)) != len(normalized):
raise ValueError("Duplicate members")
return normalized
class CreateMessage(Input):
text: str = Field(max_length=4000)
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
@field_validator("text")
@classmethod
def valid_text(cls, value):
value.encode("utf-8")
if "\x00" in value:
raise ValueError("Invalid text")
return value
@field_validator("attachment_ids")
@classmethod
def attachments(cls, values):
normalized = [str(UUID(value)) for value in values]
if len(set(normalized)) != len(normalized):
raise ValueError("Duplicate attachments")
return normalized
@model_validator(mode="after")
def not_empty(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Message is empty")
return self
+93
View File
@@ -0,0 +1,93 @@
"""scrypt passwords, strict Basic credentials and persistent signed cursors."""
import base64
from datetime import datetime, timezone
import hashlib
import hmac
import json
import re
import secrets
from uuid import UUID
from .errors import invalid, unauthorized
def password_hash(password, salt=None):
salt = salt or secrets.token_bytes(16)
digest = hashlib.scrypt(password.encode("utf-8"), salt=salt, n=16384,
r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024)
return "scrypt$16384$8$1$" + salt.hex() + "$" + digest.hex()
def password_matches(password, encoded):
try:
algorithm, n, r, p, salt, digest = encoded.split("$")
if (algorithm, n, r, p) != ("scrypt", "16384", "8", "1"):
return False
actual = password_hash(password, bytes.fromhex(salt)).rsplit("$", 1)[1]
return hmac.compare_digest(actual, digest)
except (ValueError, UnicodeError):
return False
def basic_credentials(header):
try:
scheme, encoded = header.split(" ", 1)
if scheme.lower() != "basic" or len(encoded) > 2048:
raise ValueError
value = base64.b64decode(encoded, validate=True).decode("utf-8")
username, password = value.split(":", 1)
if re.fullmatch(r"[a-z0-9_]{3,32}", username) is None:
raise ValueError
if not 12 <= len(password) <= 128:
raise ValueError
return username, password
except (AttributeError, ValueError, UnicodeError):
raise unauthorized(basic=True) from None
def bearer(header):
try:
scheme, token = header.split(" ", 1)
if scheme.lower() != "bearer" or re.fullmatch(r"[A-Za-z0-9_-]{43}", token) is None:
raise ValueError
return token
except (AttributeError, ValueError):
raise unauthorized() from None
def identifier(value):
try:
return UUID(value)
except (ValueError, AttributeError, TypeError):
raise invalid("Invalid UUID") from None
def stamp(value):
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
class Cursors:
def __init__(self, key):
self.key = key
def encode(self, scope, item):
payload = json.dumps([scope, item["created_at"], item["id"]], separators=(",", ":")).encode()
signature = hmac.digest(self.key, payload, hashlib.sha256)
return base64.urlsafe_b64encode(signature + payload).decode().rstrip("=")
def decode(self, scope, token):
try:
if not token or len(token) > 1024:
raise ValueError
raw = base64.b64decode(token + "=" * (-len(token) % 4), altchars=b"-_", validate=True)
signature, payload = raw[:32], raw[32:]
if not hmac.compare_digest(signature, hmac.digest(self.key, payload, hashlib.sha256)):
raise ValueError
parts = json.loads(payload)
if not isinstance(parts, list) or len(parts) != 3 or parts[0] != scope:
raise ValueError
if not all(isinstance(x, str) for x in parts) or not parts[1].endswith("Z"):
raise ValueError
return datetime.fromisoformat(parts[1].replace("Z", "+00:00")), UUID(parts[2])
except (ValueError, TypeError, UnicodeError):
raise invalid("Invalid cursor for this resource") from None
+238
View File
@@ -0,0 +1,238 @@
"""Persistent business data and durable Redis sessions; no local state cache."""
import asyncio
from datetime import datetime, timezone
import hashlib
import json
import secrets
from uuid import UUID, uuid4
from psycopg.errors import ForeignKeyViolation, UniqueViolation
from psycopg.rows import dict_row
from psycopg_pool import AsyncConnectionPool
import redis.asyncio as redis
from redis.asyncio.retry import Retry
from redis.backoff import NoBackoff
from .errors import APIError, invalid, unauthorized
from .security import Cursors, bearer, password_hash, password_matches, stamp
USER_COLUMNS = "id, username, display_name, created_at"
MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at, attachment_ids"
def public(row):
if "attachment_ids" in row:
row["attachment_ids"] = [str(x) for x in row["attachment_ids"]]
return {key: (str(value) if isinstance(value, UUID) else
stamp(value) if isinstance(value, datetime) else value)
for key, value in row.items()}
class Store:
def __init__(self, settings):
self.settings = settings
self.cursors = Cursors(settings.cursor_key)
self.kdf_slots = asyncio.Semaphore(4)
self.dummy_hash = password_hash("dummy-password-for-missing-user", b"0" * 16)
self.pool = AsyncConnectionPool(
settings.database_url, min_size=1, max_size=10, max_waiting=20,
timeout=1, open=False, reconnect_timeout=5,
check=AsyncConnectionPool.check_connection,
kwargs={"row_factory": dict_row, "connect_timeout": 2,
"keepalives": 1, "keepalives_idle": 2, "keepalives_interval": 1,
"keepalives_count": 2, "tcp_user_timeout": 2000,
"options": "-c timezone=UTC -c statement_timeout=2000 -c lock_timeout=1000"})
self.sessions = redis.Redis.from_url(
settings.cache_url, decode_responses=True, max_connections=10,
socket_connect_timeout=1, socket_timeout=1,
retry=Retry(NoBackoff(), 0), retry_on_timeout=False)
async def open(self):
await self.pool.open()
await self.pool.wait(timeout=10)
await self.sessions.ping()
async def close(self):
await self.sessions.aclose()
await self.pool.close()
async def ready(self):
async with self.pool.connection() as conn:
await conn.execute("SELECT 1")
await self.sessions.ping()
async def create_user(self, data):
async with self.kdf_slots:
encoded = await asyncio.to_thread(password_hash, data.password)
try:
async with self.pool.connection() as conn:
cur = await conn.execute(
"INSERT INTO users(id, username, display_name, password_hash) "
"VALUES (%s, %s, %s, %s) RETURNING " + USER_COLUMNS,
(uuid4(), data.username, data.display_name, encoded))
result = public(await cur.fetchone())
return result
except UniqueViolation:
raise APIError(409, "conflict", "Username is already taken") from None
async def login(self, username, password):
async with self.pool.connection() as conn:
cur = await conn.execute("SELECT " + USER_COLUMNS + ", password_hash FROM users WHERE username=%s",
(username,))
row = await cur.fetchone()
async with self.kdf_slots:
valid = await asyncio.to_thread(password_matches, password,
row["password_hash"] if row else self.dummy_hash)
if not valid or row is None:
raise unauthorized(basic=True)
row.pop("password_hash")
token = secrets.token_urlsafe(32)
expires_ms = int(datetime.now(timezone.utc).timestamp() * 1000) + self.settings.session_ttl * 1000
await self.sessions.set(self.session_key(token), json.dumps(
{"user_id": str(row["id"]), "expires_ms": expires_ms}), pxat=expires_ms)
return {"token": token, "token_type": "Bearer",
"expires_at": stamp(datetime.fromtimestamp(expires_ms / 1000, timezone.utc)),
"user": public(row)}
@staticmethod
def session_key(token):
return "session:" + hashlib.sha256(token.encode("ascii")).hexdigest()
async def authenticate(self, header):
token = bearer(header)
value = await self.sessions.get(self.session_key(token))
if value is None:
raise unauthorized()
session = json.loads(value)
if session["expires_ms"] <= int(datetime.now(timezone.utc).timestamp() * 1000):
raise unauthorized()
async with self.pool.connection() as conn:
cur = await conn.execute("SELECT " + USER_COLUMNS + " FROM users WHERE id=%s",
(UUID(session["user_id"]),))
row = await cur.fetchone()
if row is None:
raise unauthorized()
return public(row), token
async def logout(self, token):
await self.sessions.delete(self.session_key(token))
async def create_chat(self, actor, data):
if actor not in data.member_ids:
raise invalid("Members must include the current user")
try:
async with self.pool.connection() as conn:
cur = await conn.execute("INSERT INTO chats(id, title) VALUES (%s, %s) "
"RETURNING id, title, created_at", (uuid4(), data.title))
row = await cur.fetchone()
async with conn.cursor() as cur:
await cur.executemany("INSERT INTO chat_members(chat_id, user_id) VALUES (%s, %s)",
[(row["id"], UUID(member)) for member in data.member_ids])
result = public(row)
result["member_ids"] = sorted(data.member_ids)
return result
except ForeignKeyViolation:
raise invalid("All members must exist") from None
async def accessible_chat(self, conn, actor, chat_id):
cur = await conn.execute("""SELECT c.id, c.title, c.created_at FROM chats c
JOIN chat_members m ON m.chat_id=c.id
WHERE c.id=%s AND m.user_id=%s""", (chat_id, UUID(actor)))
row = await cur.fetchone()
if row is None:
raise APIError(404, "not_found", "Chat not found")
return row
async def get_chat(self, actor, chat_id):
async with self.pool.connection() as conn:
row = await self.accessible_chat(conn, actor, chat_id)
cur = await conn.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id", (chat_id,))
result = public(row)
result["member_ids"] = [str(x["user_id"]) for x in await cur.fetchall()]
return result
async def create_message(self, actor, chat_id, data, key):
fingerprint = hashlib.sha256(json.dumps({"text": data.text, "attachment_ids": data.attachment_ids},
ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
async with self.pool.connection() as conn:
await self.accessible_chat(conn, actor, chat_id)
# Evaluate clock_timestamp AFTER obtaining the per-chat lock.
await conn.execute("SELECT id FROM chats WHERE id=%s FOR UPDATE", (chat_id,))
# All replicas share this lock. Replay and side effect commit together.
await conn.execute("DELETE FROM message_requests WHERE user_id=%s AND chat_id=%s "
"AND request_key=%s AND expires_at <= clock_timestamp()", (UUID(actor), chat_id, key))
replay = await conn.execute("SELECT request_hash, message_id FROM message_requests "
"WHERE user_id=%s AND chat_id=%s AND request_key=%s", (UUID(actor), chat_id, key))
previous = await replay.fetchone()
if previous:
if previous["request_hash"] != fingerprint:
raise APIError(409, "conflict", "Idempotency key was used with a different message")
saved = await conn.execute("SELECT " + MESSAGE_COLUMNS + " FROM messages WHERE id=%s",
(previous["message_id"],))
return public(await saved.fetchone())
for attachment_id in data.attachment_ids:
found = await conn.execute("SELECT status FROM attachments WHERE id=%s AND chat_id=%s",
(UUID(attachment_id), chat_id))
attachment = await found.fetchone()
if attachment is None:
raise APIError(404, "not_found", "Attachment not found")
if attachment["status"] != "ready":
raise APIError(409, "conflict", "Attachment is not ready")
cur = await conn.execute("""UPDATE chats SET last_message_at=
greatest(clock_timestamp(), last_message_at + interval '1 microsecond')
WHERE id=%s RETURNING last_message_at""", (chat_id,))
created_at = (await cur.fetchone())["last_message_at"]
cur = await conn.execute("INSERT INTO messages(id, chat_id, sender_id, text, created_at, attachment_ids) "
"VALUES (%s, %s, %s, %s, %s, %s) RETURNING " + MESSAGE_COLUMNS,
(uuid4(), chat_id, UUID(actor), data.text, created_at, [UUID(x) for x in data.attachment_ids]))
result = public(await cur.fetchone())
await conn.execute("INSERT INTO message_requests "
"(user_id, chat_id, request_key, request_hash, message_id) VALUES (%s,%s,%s,%s,%s)",
(UUID(actor), chat_id, key, fingerprint, UUID(result["id"])))
return result
def page(self, rows, scope, limit):
items = rows[:limit]
return {"items": items, "next_cursor": self.cursors.encode(scope, items[-1])
if len(rows) > limit else None}
async def list_messages(self, actor, chat_id, limit, cursor):
scope = "messages:" + str(chat_id)
boundary = self.cursors.decode(scope, cursor) if cursor is not None else None
async with self.pool.connection() as conn:
await self.accessible_chat(conn, actor, chat_id)
query = "SELECT " + MESSAGE_COLUMNS + " FROM messages WHERE chat_id=%s"
params = [chat_id]
if boundary:
query += " AND (created_at, id) > (%s, %s)"
params.extend(boundary)
query += " ORDER BY created_at, id LIMIT %s"
params.append(limit + 1)
cur = await conn.execute(query, params)
rows = [public(row) for row in await cur.fetchall()]
return self.page(rows, scope, limit)
async def list_chats(self, actor, limit, cursor):
scope = "chats:" + actor
boundary = self.cursors.decode(scope, cursor) if cursor is not None else None
query = """SELECT c.id, c.title, c.created_at FROM chats c
JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s"""
params = [UUID(actor)]
if boundary:
query += " AND (c.created_at, c.id) > (%s, %s)"
params.extend(boundary)
query += " ORDER BY c.created_at, c.id LIMIT %s"
params.append(limit + 1)
async with self.pool.connection() as conn:
cur = await conn.execute(query, params)
rows = [public(row) for row in await cur.fetchall()]
if rows:
ids = [UUID(row["id"]) for row in rows]
cur = await conn.execute("SELECT chat_id, user_id FROM chat_members WHERE chat_id=ANY(%s) ORDER BY user_id",
(ids,))
members = {row["id"]: [] for row in rows}
for member in await cur.fetchall():
members[str(member["chat_id"])].append(str(member["user_id"]))
for row in rows:
row["member_ids"] = members[row["id"]]
return self.page(rows, scope, limit)
+170
View File
@@ -0,0 +1,170 @@
name: mtusi-lab4
x-app: &app
image: mtusi-messenger:lab4
build: {context: ., target: runtime}
environment: &environment
APP_HOST: 0.0.0.0
APP_PORT: "8080"
APP_ENV: ${APP_ENV:-lab}
APP_VERSION: ${APP_VERSION:-lab4-v1}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: redis
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
CURSOR_KEY_FILE: /run/secrets/cursor_key
secrets: [database_url, cache_url, cursor_key, s3_secret_key]
networks: [data]
cpus: 1.0
mem_limit: 512m
stop_grace_period: 8s
services:
postgres:
image: postgres:17.6-bookworm
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql/data]
networks: [data]
healthcheck:
test: [CMD-SHELL, "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 30
cpus: 1.0
mem_limit: 512m
stop_grace_period: 10s
redis:
image: redis:7.4.5-bookworm
command: [sh, -ec, 'exec redis-server /usr/local/etc/redis/redis.conf --requirepass "$$(cat /run/secrets/redis_password)"']
secrets: [redis_password]
volumes:
- redis_data:/data
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
networks: [data]
healthcheck:
test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/redis_password)" redis-cli ping | grep -q PONG']
interval: 2s
timeout: 2s
retries: 30
cpus: 0.5
mem_limit: 192m
stop_grace_period: 10s
migrate:
<<: *app
command: [python, -m, app.migrate]
depends_on:
postgres: {condition: service_healthy}
restart: "no"
api:
<<: *app
deploy:
replicas: ${API_REPLICAS:-2}
depends_on:
migrate: {condition: service_completed_successfully}
redis: {condition: service_healthy}
bucket-init: {condition: service_completed_successfully}
networks: [edge, data]
minio:
image: mtusi-minio:2025-09-07
build: ./infra/s3
command: [server, /data, --console-address, "127.0.0.1:9001"]
environment:
MINIO_ROOT_USER: lab-admin
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
secrets: [s3_root_password]
volumes: [s3_data:/data]
networks: [data]
cpus: 1.0
mem_limit: 512m
healthcheck:
test: [CMD-SHELL, "mc alias set health http://127.0.0.1:9000 lab-admin $$(cat /run/secrets/s3_root_password) >/dev/null && mc ready health"]
interval: 2s
timeout: 2s
retries: 30
bucket-init:
image: mtusi-minio:2025-09-07
build: ./infra/s3
entrypoint: [/bin/sh, /bootstrap/bootstrap.sh]
secrets: [s3_root_password, s3_secret_key]
volumes: [./infra/s3:/bootstrap:ro]
networks: [data]
depends_on:
minio: {condition: service_healthy}
mem_limit: 128m
restart: "no"
cleanup:
<<: *app
command: [python, -m, app.attachments]
healthcheck:
test: [CMD, python, -c, "import os; os.kill(1, 0)"]
interval: 5s
timeout: 2s
retries: 3
depends_on:
migrate: {condition: service_completed_successfully}
redis: {condition: service_healthy}
bucket-init: {condition: service_completed_successfully}
restart: unless-stopped
lb:
image: haproxy:3.0.12-alpine
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "127.0.0.1:${API_PORT:-8084}:8080"
networks: [edge]
depends_on:
api: {condition: service_healthy}
cpus: 0.5
mem_limit: 128m
stop_grace_period: 8s
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, "http://127.0.0.1:8080/health/ready"]
interval: 2s
timeout: 2s
retries: 30
checks:
profiles: [tools]
image: mtusi-messenger:lab4-checks
build: {context: ., target: checks}
environment:
<<: *environment
BASE_URL: http://lb:8080
COMPOSE_PROJECT_NAME: ${COMPOSE_PROJECT_NAME:-mtusi-lab4}
secrets: [database_url, cache_url, cursor_key, s3_secret_key]
volumes:
- .:/checks
- /var/run/docker.sock:/var/run/docker.sock
networks: [edge, data]
cpus: 1.0
mem_limit: 512m
networks:
edge:
data:
internal: true
volumes:
postgres_data:
redis_data:
s3_data:
secrets:
postgres_password: {file: ./secrets/postgres_password}
redis_password: {file: ./secrets/redis_password}
database_url: {file: ./secrets/database_url}
cache_url: {file: ./secrets/cache_url}
cursor_key: {file: ./secrets/cursor_key}
s3_root_password: {file: ./secrets/s3_root_password}
s3_secret_key: {file: ./secrets/s3_secret_key}
+39
View File
@@ -0,0 +1,39 @@
global
log stdout format raw local0
maxconn 512
stats socket ipv4@0.0.0.0:9999 level admin
defaults
mode http
log global
option httplog
option http-server-close
timeout connect 1s
timeout client 40s
timeout server 40s
timeout http-request 5s
timeout check 1s
retries 0
resolvers docker
nameserver embedded 127.0.0.11:53
resolve_retries 2
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
hold timeout 1s
hold refused 1s
frontend public
bind :8080
default_backend messenger
backend messenger
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 1
server-template api 8 api:8080 check resolvers docker init-addr none resolve-prefer ipv4
+9
View File
@@ -0,0 +1,9 @@
bind 0.0.0.0
protected-mode yes
port 6379
dir /data
appendonly yes
appendfsync always
save ""
maxmemory 128mb
maxmemory-policy noeviction
+9
View File
@@ -0,0 +1,9 @@
FROM golang:1.24.7-bookworm AS build
ENV CGO_ENABLED=0 GOTOOLCHAIN=auto
RUN go install github.com/minio/minio@RELEASE.2025-09-07T16-13-09Z
RUN go install github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
FROM debian:bookworm-slim
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*
COPY --from=build /go/bin/minio /usr/local/bin/minio
COPY --from=build /go/bin/mc /usr/local/bin/mc
ENTRYPOINT ["minio"]
+1
View File
@@ -0,0 +1 @@
{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::attachments"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::attachments/original/*"]}]}
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
mc alias set local http://minio:9000 lab-admin "$(cat /run/secrets/s3_root_password)" >/dev/null
mc mb --ignore-existing local/attachments
mc anonymous set none local/attachments
mc admin user add local messenger-api "$(cat /run/secrets/s3_secret_key)" >/dev/null
mc admin policy create local messenger-api /bootstrap/api-policy.json
mc admin policy attach local messenger-api --user messenger-api
+32
View File
@@ -0,0 +1,32 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
last_message_at timestamptz NOT NULL DEFAULT '-infinity'
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_chat_idx ON chat_members(user_id, chat_id);
CREATE INDEX chats_created_id_idx ON chats(created_at, id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL,
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
);
CREATE INDEX messages_chat_created_id_idx ON messages(chat_id, created_at, id);
+11
View File
@@ -0,0 +1,11 @@
CREATE TABLE message_requests (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
request_key varchar(128) NOT NULL,
request_hash char(64) NOT NULL,
message_id uuid NOT NULL REFERENCES messages(id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL DEFAULT (clock_timestamp() + interval '24 hours'),
PRIMARY KEY (user_id, chat_id, request_key)
);
CREATE INDEX message_requests_expiry_idx ON message_requests(expires_at);
+11
View File
@@ -0,0 +1,11 @@
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000);
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
CREATE TABLE attachments (
id uuid PRIMARY KEY, chat_id uuid NOT NULL REFERENCES chats(id),
uploader_id uuid NOT NULL REFERENCES users(id), object_key text UNIQUE NOT NULL,
filename text NOT NULL, content_type text NOT NULL, size_bytes bigint NOT NULL,
sha256 text NOT NULL, status text NOT NULL CHECK(status IN ('pending','ready')),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX attachments_pending_idx ON attachments(created_at) WHERE status='pending';
+19
View File
@@ -0,0 +1,19 @@
annotated-doc==0.0.5
annotated-types==0.8.0
anyio==4.15.1
click==8.5.0
colorama==0.4.6; sys_platform == 'win32'
fastapi==0.141.1
h11==0.16.0
idna==3.20
pydantic==2.13.5
pydantic-core==2.46.5
starlette==1.6.0
typing-extensions==4.16.0
typing-inspection==0.4.4
uvicorn==0.53.0
psycopg[binary]==3.2.10
psycopg-pool==3.2.6
redis==6.4.0
boto3==1.40.61
Pillow==12.0.0
+44
View File
@@ -0,0 +1,44 @@
"""Stop the API that actually accepted the binary upload, then download via LB."""
import hashlib
import json
from pathlib import Path
import sys
import time
ROOT=Path(__file__).resolve().parents[1]
sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
from client import Client, Failure
from ops import containers, command, compose
def main():
c=Client(timeout=5)
alice,bob,eve,chat=c.bootstrap()
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=failover.png',201,
actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')
instance=response['headers']['X-Instance-Id']
matches=[r for r in containers() if instance.startswith(r['Config']['Hostname']+'-')]
assert len(matches)==1
victim=matches[0]['Id']; started=time.monotonic()
command(['docker','stop','--time','6',victim])
try:
deadline=time.monotonic()+10
while True:
try:
result=c.call('GET','/api/v1/attachments/'+response['json']['id']+'/content',actor=bob)
break
except Failure:
if time.monotonic()>deadline: raise
time.sleep(.2)
assert result['headers']['X-Instance-Id']!=instance
assert result['bytes']==raw
assert command(['docker','inspect','--format','{{.State.Running}}',victim])=='false'
report={'upload_instance':instance,'stopped_container':victim,
'download_instance':result['headers']['X-Instance-Id'],
'sha256':hashlib.sha256(raw).hexdigest(),'size_bytes':len(raw),
'stop_and_download_seconds':round(time.monotonic()-started,3),'status':'PASS'}
(ROOT/'evidence/uploader-failover.json').write_text(json.dumps(report,indent=2))
finally:
compose('start','api')
print('PASS: original downloaded through another replica after uploader stopped')
if __name__=='__main__': main()
+35
View File
@@ -0,0 +1,35 @@
"""Create local synthetic credentials once without printing secret values."""
from pathlib import Path
import secrets
ROOT = Path(__file__).resolve().parents[1]
def main():
directory = ROOT / "secrets"
directory.mkdir(exist_ok=True)
for name in ("postgres_password", "redis_password", "cursor_key", "s3_root_password", "s3_secret_key"):
path = directory / name
if not path.exists():
path.write_text(secrets.token_hex(32) + "\n", encoding="utf-8", newline="\n")
else:
# Shell command substitution on Linux must not retain a Windows CR.
value = path.read_text(encoding="utf-8").strip()
path.write_text(value + "\n", encoding="utf-8", newline="\n")
pg_password = (directory / "postgres_password").read_text().strip()
redis_password = (directory / "redis_password").read_text().strip()
for name, value in {
"database_url": f"postgresql://messenger:{pg_password}@postgres:5432/messenger",
"cache_url": f"redis://:{redis_password}@redis:6379/0",
}.items():
path = directory / name
if not path.exists():
path.write_text(value + "\n", encoding="utf-8", newline="\n")
env = ROOT / ".env"
if not env.exists():
env.write_bytes((ROOT / ".env.example").read_bytes())
print("PASS: local .env and secret files exist; existing values preserved")
if __name__ == "__main__":
main()
+94
View File
@@ -0,0 +1,94 @@
"""Docker operations scoped to the configured local Compose project."""
import csv
import io
import json
import os
from pathlib import Path
import socket
import subprocess
import time
ROOT = Path(__file__).resolve().parents[1]
def command(args, timeout=120, env=None):
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=timeout, env=env)
if result.returncode:
# Commands and stderr can contain secrets. Keep detailed errors local.
raise RuntimeError("Local command failed: " + " ".join(args[:3]) + "\n" + result.stderr[-1500:])
return result.stdout.strip()
def compose(*args, timeout=120, env=None):
return command(["docker", "compose", "--env-file", ".env", *args], timeout, env)
def project():
return os.getenv("COMPOSE_PROJECT_NAME") or json.loads(compose("config", "--format", "json"))["name"]
def containers(service="api", running=True):
args = ["docker", "ps"] + ([] if running else ["-a"])
ids = command(args + ["--filter", "label=com.docker.compose.project=" + project(),
"--filter", "label=com.docker.compose.service=" + service,
"--filter", "label=com.docker.compose.oneoff=False", "--format", "{{.ID}}"])
return json.loads(command(["docker", "inspect", *ids.split()])) if ids else []
def summary():
result = []
for service in ("api", "postgres", "redis", "minio", "lb"):
for row in containers(service, running=False):
result.append({"service": service, "id": row["Id"], "name": row["Name"].lstrip("/"),
"started_at": row["State"]["StartedAt"], "status": row["State"]["Status"],
"image": row["Image"], "user": row["Config"]["User"],
"memory_bytes": row["HostConfig"]["Memory"], "nano_cpus": row["HostConfig"]["NanoCpus"],
"published_ports": row["HostConfig"]["PortBindings"],
"volumes": [m["Name"] for m in row["Mounts"] if m["Type"] == "volume"]})
return result
def runtime(line):
with socket.create_connection(("lb", 9999), timeout=3) as sock:
sock.sendall((line + "\n").encode())
sock.shutdown(socket.SHUT_WR)
chunks = []
while True:
data = sock.recv(65536)
if not data:
return b"".join(chunks).decode()
chunks.append(data)
def drain(address):
lines = runtime("show servers state messenger").splitlines()
fields = next(line[2:].split() for line in lines if line.startswith("# "))
for line in lines:
values = line.split()
if not values or line.startswith("#") or len(values) != len(fields):
continue
row = dict(zip(fields, values))
if row["srv_addr"] == address:
slot = "messenger/" + row["srv_name"]
runtime("set server " + slot + " state drain")
# Let requests already assigned to this replica finish (HTTP budget 5 s).
deadline = time.monotonic() + 7
while time.monotonic() < deadline:
stats = list(csv.DictReader(io.StringIO(runtime("show stat").lstrip("# "))))
active = [s for s in stats if s["pxname"] == "messenger" and s["svname"] == row["srv_name"]]
if active and int(active[0]["scur"]) == 0:
return slot
time.sleep(.1)
raise RuntimeError("Replica did not drain within 7 seconds")
raise RuntimeError("Backend address was not found in HAProxy")
if __name__ == "__main__":
import sys
if sys.argv[1] == "drain":
print(drain(sys.argv[2]))
elif sys.argv[1] == "ready":
runtime("set server " + sys.argv[2] + " state ready")
elif sys.argv[1] == "summary":
print(json.dumps(summary(), indent=2))
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env restart --timeout 6 api postgres redis minio lb
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python scripts/stop_one.py
+24
View File
@@ -0,0 +1,24 @@
"""Stop exactly the replica selected by the public failover scenario."""
import json
import os
import time
from ops import ROOT, command, containers
def main():
target = os.environ["TARGET_INSTANCE_ID"]
matches = [row for row in containers() if target.startswith(row["Config"]["Hostname"] + "-")]
if len(matches) != 1:
raise RuntimeError("Target does not identify exactly one running API replica")
row = matches[0]
started = time.monotonic()
command(["docker", "stop", "--time", "6", row["Id"]], timeout=9)
state = json.loads(command(["docker", "inspect", row["Id"]]))[0]["State"]
assert not state["Running"] and state["ExitCode"] == 0, "Graceful shutdown failed"
report = {"target_instance": target, "container": row["Id"], "exit_code": state["ExitCode"],
"stop_seconds": round(time.monotonic() - started, 3), "remains_stopped": True}
(ROOT / "evidence" / "last-stop.json").write_text(json.dumps(report, indent=2) + "\n")
if __name__ == "__main__":
main()
+42
View File
@@ -0,0 +1,42 @@
"""Build once per image and run all required local acceptance checks."""
import json
from pathlib import Path
import subprocess
import sys
import time
from ops import ROOT, compose, summary
def run(name, args):
started=time.monotonic()
with (ROOT/'evidence'/name).open('w',encoding='utf-8') as output:
result=subprocess.run(args,cwd=ROOT,stdout=output,stderr=subprocess.STDOUT)
if result.returncode: raise RuntimeError('Check failed; see evidence/'+name)
return round(time.monotonic()-started,3)
def main():
run('init.txt',[sys.executable,'scripts/init_local.py'])
run('build.txt',['docker','compose','build','api','checks','minio'])
run('startup.txt',['docker','compose','up','-d','--no-build','--wait','--wait-timeout','180'])
checks={}
checks['smoke_seconds']=run('smoke.txt',['docker','compose','run','--rm','--no-deps','checks','make','check','test'])
checks['student_seconds']=run('student-tests.txt',['docker','compose','run','--rm','--no-deps','checks','make','test-student'])
before=summary()
checks['persistence_seconds']=run('persistence.txt',['docker','compose','run','--rm','--no-deps','checks','sh','-ec','chmod +x scripts/restart.sh; make test-persistence ACTION_SCRIPT=scripts/restart.sh'])
after=summary()
for row in before:
matching=next(x for x in after if x['id']==row['id'])
assert matching['started_at']!=row['started_at']
assert matching['volumes']==row['volumes']
(ROOT/'evidence/restart.json').write_text(json.dumps({'before':before,'after':after},indent=2))
for i in (1,2):
checks[f'failover_{i}_seconds']=run(f'failover-{i}.txt',['docker','compose','run','--rm','--no-deps','checks','sh','-ec','chmod +x scripts/stop-one.sh; make test-failover ACTION_SCRIPT=scripts/stop-one.sh'])
(ROOT/f'evidence/stop-{i}.json').write_bytes((ROOT/'evidence/last-stop.json').read_bytes())
compose('start','api'); time.sleep(7)
checks['uploader_failover_seconds']=run('uploader-failover.txt',['docker','compose','run','--rm','--no-deps','checks','python','scripts/check_uploader_failover.py'])
time.sleep(7)
checks['final_smoke_seconds']=run('smoke-final.txt',['docker','compose','run','--rm','--no-deps','checks','make','test'])
checks['status']='PASS'; checks['containers']=summary()
(ROOT/'evidence/verification.json').write_text(json.dumps(checks,indent=2))
print('PASS: lab4 acceptance checks completed')
if __name__=='__main__': main()
+13
View File
@@ -0,0 +1,13 @@
$ErrorActionPreference = 'Stop'
Set-Location -LiteralPath $PSScriptRoot
docker info --format '{{.ServerVersion}}'
if ($LASTEXITCODE -ne 0) { throw 'Start Docker Desktop first.' }
docker run --rm --mount "type=bind,source=$PSScriptRoot,target=/work" --workdir /work python:3.13.15-slim-bookworm python scripts/init_local.py
if ($LASTEXITCODE -ne 0) { throw 'Local configuration initialization failed.' }
docker compose --env-file .env build api checks minio
if ($LASTEXITCODE -ne 0) { throw 'Image build failed.' }
docker compose --env-file .env up -d --no-build --wait --wait-timeout 180
if ($LASTEXITCODE -ne 0) { throw 'Stack startup failed; inspect docker compose logs.' }
$taskConfig = docker compose --env-file .env config --format json | ConvertFrom-Json
$taskPort = $taskConfig.services.lb.ports[0].published
Write-Host "Ready: http://127.0.0.1:$taskPort/health/ready"
+222
View File
@@ -0,0 +1,222 @@
"""Integration tests against real HTTP, PostgreSQL and Redis on the local stack."""
from concurrent.futures import ThreadPoolExecutor
import base64
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import time
import unittest
from urllib.parse import quote
from uuid import uuid4
import psycopg
import redis
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
sys.path.insert(0, str(ROOT / "scripts"))
from client import Client
from ops import containers
from app.config import secret
class APITests(unittest.TestCase):
def setUp(self):
self.c = Client(timeout=8)
self.alice = self.c.user("student")
self.bob = self.c.user("member")
self.chat = self.c.call("POST", "/api/v1/chats", 201, actor=self.alice,
body={"title": "Учебный чат", "member_ids": [self.alice["id"], self.bob["id"]]},
schema="Chat")["json"]
self.path = "/api/v1/chats/" + self.chat["id"]
def scalar(self, query, params=()):
with psycopg.connect(secret("DATABASE_URL"), connect_timeout=2) as conn:
return conn.execute(query, params).fetchone()[0]
def history(self, limit=17):
result, cursor = [], None
for _ in range(100):
path = self.path + "/messages?limit=" + str(limit)
if cursor:
path += "&cursor=" + quote(cursor, safe="")
page = self.c.call("GET", path, actor=self.alice, schema="MessagePage")["json"]
result.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return result
self.fail("Pagination did not terminate")
def test_registration_race(self):
registration = {"username": "race_" + uuid4().hex[:16], "display_name": "Гонка",
"password": "Synthetic-password-12345"}
def register(_):
return self.c.call("POST", "/api/v1/users", [201, 409], body=registration,
schema="User")["status"]
with ThreadPoolExecutor(max_workers=10) as pool:
statuses = list(pool.map(register, range(10)))
self.assertEqual(statuses.count(201), 1)
self.assertEqual(statuses.count(409), 9)
self.assertEqual(self.scalar("SELECT count(*) FROM users WHERE username=%s",
(registration["username"],)), 1)
print("RACE: 10 requests, 1 created, 9 conflicts, 1 database row", flush=True)
def test_composite_chat_rollback(self):
before = self.scalar("SELECT count(*) FROM chats")
memberships = self.scalar("SELECT count(*) FROM chat_members")
self.c.call("POST", "/api/v1/chats", 400, actor=self.alice,
body={"title": "Rollback", "member_ids": [self.alice["id"], str(uuid4())]})
self.assertEqual(self.scalar("SELECT count(*) FROM chats"), before)
self.assertEqual(self.scalar("SELECT count(*) FROM chat_members"), memberships)
self.assertEqual(self.scalar("SELECT count(*) FROM chats c WHERE NOT EXISTS "
"(SELECT 1 FROM chat_members m WHERE m.chat_id=c.id)"), 0)
print("ROLLBACK: invalid second member left no chat or membership rows", flush=True)
def test_concurrent_messages(self):
def send(index):
return self.c.message(self.alice, self.chat, "parallel-" + str(index))["json"]["id"]
with ThreadPoolExecutor(max_workers=10) as pool:
ids = list(pool.map(send, range(100)))
messages = self.history()
self.assertEqual(len(messages), 100)
self.assertEqual(len(set(ids)), 100)
self.assertEqual({row["id"] for row in messages}, set(ids))
self.assertEqual({row["text"] for row in messages}, {"parallel-" + str(i) for i in range(100)})
self.assertEqual(messages, sorted(messages, key=lambda row: (row["created_at"], row["id"])))
print("CONCURRENCY: 10 clients, 100 confirmations, 100 unique messages", flush=True)
def test_validation_and_access(self):
for payload in [b"{", b"null", b"[]", b"\xff", b'{"text":NaN}']:
self.c.call("POST", self.path + "/messages", 400, actor=self.alice, raw=payload,
headers={"Content-Type": "application/json", "Idempotency-Key": uuid4().hex})
for body in [{"text": ""}, {"text": " \n\t"}, {"text": 123}, {"text": "x" * 4001},
{"text": "hello", "sender_id": self.bob["id"]}]:
self.c.call("POST", self.path + "/messages", 400, actor=self.alice, body=body,
headers={"Idempotency-Key": uuid4().hex})
outsider = self.c.user("outsider")
for suffix in ("", "/messages"):
self.c.call("GET", self.path + suffix, 404, actor=outsider)
self.c.call("POST", self.path + "/messages", 404, actor=outsider, body={"text": "hidden"})
for members in [[self.bob["id"]], [self.alice["id"], self.alice["id"]], ["bad"], []]:
self.c.call("POST", "/api/v1/chats", 400, actor=self.alice,
body={"title": "Invalid", "member_ids": members})
self.c.call("GET", "/api/v1/chats/not-uuid", 400, actor=self.alice)
for limit in ("0", "101", "1.1", "-1", "", "true"):
self.c.call("GET", self.path + "/messages?limit=" + limit, 400, actor=self.alice)
self.c.call("POST", "/api/v1/users", 409, body=self.alice["registration"])
for raw in (b"{}", b'{"username":123}', b'{"username":"ab","display_name":"x","password":"123456789012"}'):
self.c.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"})
def test_passwords_and_authentication(self):
registration = {"username": "unicode_" + uuid4().hex[:12], "display_name": "Имя 🧪",
"password": "я:🧪" + "密" * 125}
user = self.c.call("POST", "/api/v1/users", 201, body=registration, schema="User")["json"]
credential = base64.b64encode((registration["username"] + ":" + registration["password"]).encode()).decode()
session = self.c.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": "Basic " + credential}, schema="Session")["json"]
self.assertEqual(session["user"]["id"], user["id"])
encoded = self.scalar("SELECT password_hash FROM users WHERE id=%s", (user["id"],))
algorithm, n, r, p, salt, digest = encoded.split("$")
self.assertEqual(algorithm, "scrypt")
actual = hashlib.scrypt(registration["password"].encode(), salt=bytes.fromhex(salt),
n=int(n), r=int(r), p=int(p), dklen=32, maxmem=64 * 1024 * 1024).hex()
self.assertTrue(actual == digest)
self.assertNotIn("password", user)
for username in (registration["username"], "missing_" + uuid4().hex[:12]):
bad = base64.b64encode((username + ":wrong-password-123").encode()).decode()
response = self.c.call("POST", "/api/v1/auth/sessions", 401,
headers={"Authorization": "Basic " + bad})
self.assertEqual(response["headers"]["WWW-Authenticate"], 'Basic realm="messenger", charset="UTF-8"')
self.assertEqual(response["json"]["error"]["message"], "Valid credentials required")
self.c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Basic " + credential})
self.c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": user["id"]})
self.c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Bearer " + "x" * 43})
actor = {"id": user["id"], "token": session["token"]}
self.c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=actor)
self.c.call("GET", "/api/v1/users/me", 401, actor=actor)
self.c.call("DELETE", "/api/v1/auth/sessions/current", 401, actor=actor)
def test_cursor_binding_and_chat_pages(self):
sent = [self.c.message(self.alice, self.chat, "page-" + str(i))["json"] for i in range(7)]
self.assertEqual(self.history(limit=2), sent)
page = self.c.call("GET", self.path + "/messages?limit=2", actor=self.alice, schema="MessagePage")["json"]
cursor = page["next_cursor"]
other = self.c.call("POST", "/api/v1/chats", 201, actor=self.alice,
body={"title": "Other", "member_ids": [self.alice["id"]]}, schema="Chat")["json"]
self.c.call("GET", "/api/v1/chats/" + other["id"] + "/messages?cursor=" + quote(cursor), 400, actor=self.alice)
for token in ("bad", "", cursor[:-3] + "xxx"):
self.c.call("GET", self.path + "/messages?cursor=" + quote(token), 400, actor=self.alice)
later = self.c.message(self.alice, self.chat, "later")["json"]
continued = self.c.call("GET", self.path + "/messages?limit=100&cursor=" + quote(cursor),
actor=self.alice, schema="MessagePage")["json"]
self.assertEqual(continued["items"], sent[2:] + [later])
first = self.c.call("GET", "/api/v1/chats?limit=1", actor=self.alice, schema="ChatPage")["json"]
second = self.c.call("GET", "/api/v1/chats?limit=100&cursor=" + quote(first["next_cursor"]),
actor=self.alice, schema="ChatPage")["json"]
self.assertEqual([row["id"] for row in first["items"] + second["items"]],
[self.chat["id"], other["id"]])
def compose(self, *args):
result = subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT,
capture_output=True, text=True, timeout=45)
if result.returncode:
self.fail("Compose action failed: " + " ".join(args[:2]))
def wait_ready(self):
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
try:
self.c.call("GET", "/health/ready", schema="Health")
return
except AssertionError:
time.sleep(0.25)
self.fail("Stack did not recover")
def test_dependency_failures(self):
# With all backends unready, LB may generate its own 503. Diagnose
# the application error contract on an internal replica explicitly.
row = containers()[0]
self.c = Client("http://" + row["Name"].lstrip("/") + ":8080", timeout=8)
report = {}
for service in ("redis", "postgres"):
self.compose("stop", "--timeout", "2", service)
try:
self.c.call("GET", "/health/live", schema="Health")
ready = self.c.call("GET", "/health/ready", 503)
started = time.monotonic()
response = self.c.call("GET", "/api/v1/users/me", 503, actor=self.alice)
elapsed = time.monotonic() - started
self.assertEqual(response["json"]["error"]["code"], "unavailable")
self.assertLess(elapsed, 6)
self.c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": self.alice["id"]})
report[service] = {"ready_status": ready["status"], "protected_status": response["status"],
"elapsed_seconds": round(elapsed, 3), "anonymous_status": 401}
finally:
self.compose("start", service)
self.wait_ready()
self.c.call("GET", "/api/v1/users/me", actor=self.alice, schema="User")
self.c.message(self.alice, self.chat, "after-" + service)
(ROOT / "evidence" / "failures.json").write_text(json.dumps(report, indent=2) + "\n")
print("FAILURES: Redis/PostgreSQL -> bounded 503, live 200, anonymous 401, recovery with old session", flush=True)
def test_redis_crash_durability(self):
client = redis.Redis.from_url(secret("CACHE_URL"), decode_responses=True)
try:
self.assertEqual(client.config_get("appendonly")["appendonly"], "yes")
self.assertEqual(client.config_get("appendfsync")["appendfsync"], "always")
key = "session:" + hashlib.sha256(self.alice["token"].encode()).hexdigest()
deadline_before = client.pexpiretime(key)
self.compose("kill", "--signal", "SIGKILL", "redis")
self.compose("start", "redis")
self.wait_ready()
self.c.call("GET", "/api/v1/users/me", actor=self.alice, schema="User")
self.assertEqual(client.pexpiretime(key), deadline_before)
finally:
client.close()
print("REDIS CRASH: old session survived SIGKILL/AOF replay, expiry deadline unchanged", flush=True)
if __name__ == "__main__":
unittest.main()
+171
View File
@@ -0,0 +1,171 @@
"""Grade 4 integration checks against real stores, including fault injection."""
import asyncio
from concurrent.futures import ThreadPoolExecutor
import hashlib
import io
import json
import os
from pathlib import Path
import socket
import sys
import threading
import time
import unittest
from uuid import uuid4
import psycopg
from PIL import Image
from botocore.exceptions import ClientError
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT), str(ROOT/'tests'), str(ROOT/'scripts')]
from client import Client
from ops import compose, command, containers, summary
from app.config import Settings
from app.store import Store
from app.attachments import Attachments, client, BUCKET
class AttachmentTests(unittest.TestCase):
def setUp(self):
self.c = Client(timeout=40)
self.a,self.b,self.e,self.chat = self.c.bootstrap()
self.path = '/api/v1/chats/'+self.chat['id']+'/attachments?filename=photo.png'
self.png = (ROOT/'tests/fixtures/sample.png').read_bytes()
def upload(self, raw=None, mime='image/png', expected=201, path=None):
return self.c.call('POST', path or self.path, expected, actor=self.a,
raw=self.png if raw is None else raw, headers={'Content-Type':mime},
schema='Attachment' if expected==201 else None)
def sql(self, query, params=()):
with psycopg.connect(Settings.load().database_url, autocommit=True) as conn:
return conn.execute(query, params).fetchall() if query.lstrip().upper().startswith('SELECT') else conn.execute(query,params)
def test_validation_and_acl(self):
self.upload(b'x'*(10*1024*1024+1),expected=413)
self.upload(self.png[:30],expected=415)
self.upload(mime='image/jpeg',expected=415)
self.upload(path=self.path.replace('photo.png','..%2Fphoto.png'),expected=400)
bomb=io.BytesIO(); Image.new('1',(8000,5001)).save(bomb,format='PNG')
self.upload(bomb.getvalue(),expected=413)
jpg=io.BytesIO(); Image.new('RGB',(64,32),'red').save(jpg,format='JPEG')
self.upload(jpg.getvalue(),mime='image/jpeg')
item=self.upload()['json']; p='/api/v1/attachments/'+item['id']
for suffix in ('','/content'):
self.c.call('GET',p+suffix,404,actor=self.e)
self.c.call('GET',p,401)
self.c.call('GET',p+'/content',actor=self.b)
other=self.c.call('POST','/api/v1/chats',201,actor=self.a,body={
'title':'Other','member_ids':[self.a['id']]},schema='Chat')['json']
foreign=self.upload(path=self.path.replace(self.chat['id'],other['id']))['json']
for aid in (foreign['id'],str(uuid4())):
self.c.call(
'POST','/api/v1/chats/'+self.chat['id']+'/messages',404,actor=self.a,
body={'text':'foreign','attachment_ids':[aid]},headers={'Idempotency-Key':uuid4().hex})
message=self.c.message(self.a,self.chat,'',attachments=[item['id']])
self.assertEqual(message['json']['attachment_ids'],[item['id']])
self.c.call('POST','/api/v1/chats/'+self.chat['id']+'/messages',400,actor=self.a,
body={'text':' '},headers={'Idempotency-Key':uuid4().hex})
def test_private_bucket_and_bootstrap(self):
item=self.upload()['json']
import urllib.request, urllib.error
key=self.sql('SELECT object_key FROM attachments WHERE id=%s',(item['id'],))[0][0]
for path in ('attachments', 'attachments/'+key):
try:
urllib.request.urlopen('http://minio:9000/'+path,timeout=3)
self.fail('Anonymous access succeeded')
except urllib.error.HTTPError as error:
self.assertEqual(error.code,403)
s3=client()
for operation in (lambda:s3.create_bucket(Bucket='forbidden'),
lambda:s3.put_object(Bucket=BUCKET,Key='outside-prefix',Body=b'x')):
with self.assertRaises(ClientError) as caught:
operation()
self.assertEqual(caught.exception.response['ResponseMetadata']['HTTPStatusCode'],403)
bootstrap = containers('bucket-init', running=False)[0]['Id']
command(['docker','start','-a',bootstrap]); command(['docker','start','-a',bootstrap])
result=self.c.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=self.b)
self.assertEqual(result['bytes'],self.png)
def test_failed_commit_cleanup_and_abort(self):
# A partial client body cannot reach PUT.
before=self.sql('SELECT count(*) FROM attachments')[0][0]
sock=socket.create_connection(('lb',8080),timeout=3)
request=('POST '+self.path+' HTTP/1.1\r\nHost: lb\r\nAuthorization: Bearer '+self.a['token']+
'\r\nContent-Type: image/png\r\nContent-Length: 10000\r\n\r\n').encode()+self.png[:20]
sock.sendall(request); sock.close(); time.sleep(.3)
self.assertEqual(before,self.sql('SELECT count(*) FROM attachments')[0][0])
self.sql("""CREATE OR REPLACE FUNCTION lab4_fail_ready() RETURNS trigger LANGUAGE plpgsql AS
$$ BEGIN IF NEW.status='ready' AND NEW.filename='fault.png' THEN
RAISE EXCEPTION 'injected commit failure'; END IF; RETURN NEW; END $$""")
self.sql('CREATE TRIGGER lab4_fail_ready BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION lab4_fail_ready()')
try:
self.upload(path=self.path.replace('photo.png','fault.png'),expected=503)
finally:
self.sql('DROP TRIGGER lab4_fail_ready ON attachments'); self.sql('DROP FUNCTION lab4_fail_ready()')
aid,key=self.sql("SELECT id,object_key FROM attachments WHERE filename='fault.png' AND chat_id=%s",(self.chat['id'],))[0]
s3=client(); s3.head_object(Bucket=BUCKET,Key=key)
async def cleanup():
store=Store(Settings.load()); await store.open()
try: return await Attachments(store).cleanup()
finally: await store.close()
asyncio.run(cleanup())
s3.head_object(Bucket=BUCKET,Key=key) # grace protects the fresh pending object
self.sql("UPDATE attachments SET created_at=clock_timestamp()-interval '2 hours' WHERE id=%s",(aid,))
self.assertGreaterEqual(asyncio.run(cleanup()),1)
asyncio.run(cleanup()) # repeat is harmless
self.assertFalse(self.sql('SELECT id FROM attachments WHERE id=%s',(aid,)))
with self.assertRaises(ClientError): s3.head_object(Bucket=BUCKET,Key=key)
self.upload() # ready original survives cleanup
def test_s3_outage(self):
item=self.upload()['json']
direct=Client('http://api:8080',timeout=10)
compose('stop','minio')
try:
direct.call('GET','/health/live',schema='Health')
direct.call('GET','/health/ready',503)
direct.call('POST',self.path,503,actor=self.a,raw=self.png,headers={'Content-Type':'image/png'})
direct.call('GET','/api/v1/attachments/'+item['id']+'/content',503,actor=self.a)
finally:
compose('start','minio')
time.sleep(5)
self.assertEqual(self.c.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=self.b)['bytes'],self.png)
def test_ten_parallel_uploads_memory(self):
# RGB random image is poorly compressible, roughly 8 MiB.
image=Image.frombytes('RGB',(1700,1600),os.urandom(1700*1600*3))
out=io.BytesIO(); image.save(out,format='PNG'); raw=out.getvalue()
ids=[x['Id'] for x in containers()]
barrier=threading.Barrier(11); stop=threading.Event(); samples=[]
def sample():
while not stop.is_set():
rows=[]
for cid in ids:
value=command(['docker','exec',cid,'cat','/sys/fs/cgroup/memory.current'])
rows.append(int(value))
samples.append({'elapsed':round(time.monotonic()-start,3),'bytes':rows})
stop.wait(.1)
start=time.monotonic(); sampler=threading.Thread(target=sample); sampler.start()
def upload(_):
barrier.wait()
t=time.monotonic(); item=self.upload(raw)['json']
got=self.c.call('GET','/api/v1/attachments/'+item['id']+'/content',actor=self.b)['bytes']
self.assertEqual(hashlib.sha256(got).hexdigest(),hashlib.sha256(raw).hexdigest())
return time.monotonic()-t
try:
with ThreadPoolExecutor(max_workers=10) as pool:
futures=[pool.submit(upload,i) for i in range(10)]
barrier.wait(); durations=[f.result() for f in futures]
finally:
stop.set(); sampler.join()
result={'concurrency':10,'successful':len(durations),'size_bytes':len(raw),
'duration_seconds':time.monotonic()-start,'upload_and_download_seconds':durations,
'api_container_ids':ids,'memory_samples':samples,
'sampled_peak_bytes':[max(s['bytes'][i] for s in samples) for i in range(len(ids))],
'cgroup_memory_peak_bytes':[int(command(['docker','exec',cid,'cat','/sys/fs/cgroup/memory.peak'])) for cid in ids],
'memory_limit_bytes':512*1024*1024,
'note':'memory.current includes cache and runtime; memory.peak covers container lifetime, not only experiment'}
(ROOT/'evidence/uploads-memory.json').write_text(json.dumps(result,indent=2))
if __name__=='__main__': unittest.main(verbosity=2)
+110
View File
@@ -0,0 +1,110 @@
"""Cross-process idempotency, access and transaction boundaries."""
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import sys
import threading
import unittest
from uuid import uuid4
import psycopg
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
sys.path.insert(0, str(ROOT / "scripts"))
from client import Client
from ops import containers
from app.config import secret
class ReplicationTests(unittest.TestCase):
def setUp(self):
self.c = Client()
self.alice, self.bob, self.eve, self.chat = self.c.bootstrap()
self.path = "/api/v1/chats/" + self.chat["id"] + "/messages"
def sql(self, query, values=()):
with psycopg.connect(secret("DATABASE_URL"), connect_timeout=2) as conn:
cur = conn.execute(query, values)
return cur.fetchall() if cur.description else None
def test_parallel_replay_across_real_replicas(self):
replicas = containers()
self.assertGreaterEqual(len(replicas), 2)
clients = [Client("http://" + row["Name"].lstrip("/") + ":8080") for row in replicas]
barrier = threading.Barrier(10)
key = uuid4().hex
def send(index):
if index < 10:
barrier.wait()
return clients[index % len(clients)].message(self.alice, self.chat, "Один результат 🧪", key=key)
with ThreadPoolExecutor(max_workers=10) as pool:
responses = list(pool.map(send, range(100)))
self.assertEqual(len({r["json"]["id"] for r in responses}), 1)
self.assertGreaterEqual(len({r["headers"]["X-Instance-Id"] for r in responses}), 2)
self.assertTrue(all(r["json"] == responses[0]["json"] for r in responses))
self.assertEqual(self.sql("SELECT count(*) FROM messages WHERE chat_id=%s", (self.chat["id"],))[0][0], 1)
self.assertEqual(self.sql("SELECT count(*) FROM message_requests WHERE chat_id=%s", (self.chat["id"],))[0][0], 1)
for _ in range(12):
self.assertEqual(self.c.message(self.alice, self.chat, "Один результат 🧪", key=key)["json"], responses[0]["json"])
report = {"requests": 100, "clients": 10, "real_containers": len(replicas),
"observed_instances": len({r["headers"]["X-Instance-Id"] for r in responses}),
"messages": 1, "replay_records": 1, "lb_replays": 12}
(ROOT / "evidence" / "idempotency.json").write_text(json.dumps(report, indent=2) + "\n")
def test_key_validation_scope_conflict_and_logout(self):
for key in (None, "", "bad key", "bad/key", "a" * 129):
headers = {} if key is None else {"Idempotency-Key": key}
self.c.call("POST", self.path, 400, actor=self.alice, body={"text": "test"}, headers=headers)
key = uuid4().hex
first = self.c.message(self.alice, self.chat, "first", key=key)["json"]
replay = self.c.call("POST", self.path, 201, actor=self.alice,
raw=b'{\n "text" : "first"\n}', headers={"Content-Type": "application/json", "Idempotency-Key": key},
schema="Message")["json"]
self.assertEqual(first, replay)
self.c.call("POST", self.path, 409, actor=self.alice, body={"text": "changed"}, headers={"Idempotency-Key": key})
second = self.c.message(self.bob, self.chat, "first", key=key)["json"]
self.assertNotEqual(first["id"], second["id"])
other = self.c.call("POST", "/api/v1/chats", 201, actor=self.alice,
body={"title": "scope", "member_ids": [self.alice["id"]]}, schema="Chat")["json"]
self.assertNotEqual(self.c.message(self.alice, other, "first", key=key)["json"]["id"], first["id"])
self.c.call("POST", self.path, 404, actor=self.eve, body={"text": "first"}, headers={"Idempotency-Key": key})
self.c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=self.alice)
self.c.call("POST", self.path, 401, actor=self.alice, body={"text": "first"}, headers={"Idempotency-Key": key})
def test_failed_transaction_does_not_save_message_or_replay(self):
key = uuid4().hex
# Fail after INSERT messages, at INSERT replay, to check the whole
# transaction rather than merely validation before the first write.
self.sql("""CREATE FUNCTION lab3_test_reject() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN RAISE EXCEPTION 'synthetic transaction failure'; END $$""")
self.sql("CREATE TRIGGER lab3_test_reject BEFORE INSERT ON message_requests "
"FOR EACH ROW EXECUTE FUNCTION lab3_test_reject()")
try:
self.c.call("POST", self.path, 503, actor=self.alice, body={"text": "atomic"}, headers={"Idempotency-Key": key})
self.assertEqual(self.sql("SELECT count(*) FROM messages WHERE chat_id=%s", (self.chat["id"],))[0][0], 0)
self.assertEqual(self.sql("SELECT count(*) FROM message_requests WHERE chat_id=%s", (self.chat["id"],))[0][0], 0)
finally:
self.sql("DROP TRIGGER lab3_test_reject ON message_requests")
self.sql("DROP FUNCTION lab3_test_reject()")
first = self.c.message(self.alice, self.chat, "atomic", key=key)["json"]
self.assertEqual(self.c.message(self.alice, self.chat, "atomic", key=key)["json"], first)
def test_expired_key_and_same_session_on_all_instances(self):
observed = set()
for _ in range(30):
response = self.c.call("GET", "/api/v1/users/me", actor=self.alice, schema="User")
self.assertEqual(response["json"]["id"], self.alice["id"])
observed.add(response["headers"]["X-Instance-Id"])
self.assertGreaterEqual(len(observed), 2)
key = uuid4().hex
original = self.c.message(self.alice, self.chat, "before expiry", key=key)["json"]
self.sql("UPDATE message_requests SET expires_at=clock_timestamp()-interval '1 second' "
"WHERE user_id=%s AND chat_id=%s AND request_key=%s", (self.alice["id"], self.chat["id"], key))
fresh = self.c.message(self.alice, self.chat, "after expiry", key=key)["json"]
self.assertNotEqual(original["id"], fresh["id"])
self.assertEqual(self.c.message(self.alice, self.chat, "after expiry", key=key)["json"], fresh)
if __name__ == "__main__":
unittest.main()