готово

This commit is contained in:
2026-11-12 19:38:03 +03:00
parent ec540a1a72
commit f3ba37f736
142 changed files with 10257 additions and 0 deletions
+44
View File
@@ -0,0 +1,44 @@
"""Stop the exact API replica which accepted the image; preserve original bytes."""
import os
import subprocess
import time
from pathlib import Path
from client import Client, Failure, arguments, check, run
def main():
args = arguments('Attachment download after stopping the uploading replica')
c = Client(args.base_url, args.ca_file)
alice, bob, _, chat = c.bootstrap()
content = (Path(__file__).parent / 'fixtures/sample.png').read_bytes()
uploaded = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=failover.png',
201, actor=alice, raw=content, headers={'Content-Type': 'image/png'},
schema='Attachment')
att = uploaded['json']
c.message(alice, chat, '', attachments=[att['id']])
target = uploaded['headers']['X-Instance-Id']
subprocess.run(['scripts/stop-one.sh'], check=True,
env=dict(os.environ, TARGET_INSTANCE_ID=target),
stdout=subprocess.DEVNULL)
deadline = time.monotonic() + 15
try:
while True:
try:
downloaded = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
break
except Failure:
if time.monotonic() >= deadline:
raise
time.sleep(.2)
check(downloaded['bytes'] == content, 'original lost after uploader replica stopped')
check(downloaded['headers']['X-Instance-Id'] != target, 'download used stopped replica')
print('PASS: exact uploading replica stopped; another replica returns identical original')
finally:
# Restore the two-replica development stack for subsequent work.
subprocess.run(['docker', 'compose', 'up', '-d', '--wait', 'api'], check=True,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
if __name__ == '__main__':
run(main)
+98
View File
@@ -0,0 +1,98 @@
"""Lab4 black-box checks against the running Compose stack."""
import hashlib
import io
import struct
import zlib
from concurrent.futures import ThreadPoolExecutor
import uuid
from pathlib import Path
from PIL import Image
from client import Client, arguments, check, run
def main():
args = arguments('Attachment originals, ACL, validation and message limits')
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id']
content = (Path(__file__).parent / 'fixtures/sample.png').read_bytes()
def upload(data=content, name='photo.png', mime='image/png', status=201, actor=alice):
return c.call('POST', path + '/attachments?filename=' + name, status,
actor=actor, raw=data, headers={'Content-Type': mime},
schema='Attachment' if status == 201 else None)['json']
first = upload()
second = upload() # Same display name must not overwrite the first object.
check(first['id'] != second['id'], 'same filename overwrote attachment')
jpg = io.BytesIO()
Image.new('RGB', (31, 17), 'red').save(jpg, format='JPEG')
jpeg = upload(jpg.getvalue(), 'photo.jpg', 'image/jpeg')
for att, original in [(first, content), (second, content), (jpeg, jpg.getvalue())]:
check(att['sha256'] == hashlib.sha256(original).hexdigest(), 'incorrect digest')
check(att['status'] == 'ready' and att['variants'] == [], 'incorrect ready metadata')
for suffix in ('', '/content'):
c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=eve)
downloaded = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
check(downloaded['bytes'] == original, 'original changed')
ids = [first['id'], second['id'], jpeg['id']]
key = uuid.uuid4().hex
message = c.message(alice, chat, '', key=key, attachments=ids)['json']
check(message['attachment_ids'] == ids, 'attachment order lost')
check(c.message(alice, chat, '', key=key, attachments=ids)['json'] == message,
'idempotency replay changed message')
c.call('POST', path + '/messages', 409, actor=alice,
body={'text': '', 'attachment_ids': ids[:1]}, headers={'Idempotency-Key': key})
page = c.call('GET', path + '/messages', actor=bob, schema='MessagePage')['json']
check(any(m == message for m in page['items']), 'history lost attachments')
# The contract permits referencing the same accessible chat attachment again.
c.message(bob, chat, 'same image', attachments=[first['id']])
for body in ({'text': '', 'attachment_ids': []},
{'text': '', 'attachment_ids': [first['id']] * 2},
{'text': '', 'attachment_ids': [str(uuid.uuid4()) for _ in range(11)]}):
c.call('POST', path + '/messages', 400, actor=alice, body=body,
headers={'Idempotency-Key': uuid.uuid4().hex})
c.call('POST', path + '/messages', 404, actor=alice,
body={'text': '', 'attachment_ids': [str(uuid.uuid4())]},
headers={'Idempotency-Key': uuid.uuid4().hex})
other = c.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]},
schema='Chat')['json']
c.call('POST', '/api/v1/chats/' + other['id'] + '/messages', 404, actor=alice,
body={'text': '', 'attachment_ids': ids},
headers={'Idempotency-Key': uuid.uuid4().hex})
upload(actor=eve, status=404)
upload(name='..%2Fphoto.png', status=400)
upload(name='a%00b.png', status=400)
upload(data=b'not a PNG', status=415)
upload(data=content[:30], status=415)
upload(mime='image/jpeg', status=415)
upload(mime='application/octet-stream', status=415)
# Oversized dimensions with a valid PNG header; no huge bitmap allocation.
header = struct.pack('>IIBBBBB', 10000, 5000, 8, 2, 0, 0, 0)
bomb = (content[:8] + struct.pack('>I', len(header)) + b'IHDR' + header
+ struct.pack('>I', zlib.crc32(b'IHDR' + header)) + content[33:])
upload(data=bomb, status=413)
upload(data=b'x' * (10485760 + 1), status=413)
c.call('POST', path + '/attachments?filename=a.png', 401,
raw=content, headers={'Content-Type': 'image/png'})
# Exactly 10 distinct attachments must remain valid, including after reading history.
ten = ids + [upload()['id'] for _ in range(7)]
check(len(c.message(alice, chat, '', attachments=ten)['json']['attachment_ids']) == 10,
'10 attachments rejected')
replay_key = uuid.uuid4().hex
with ThreadPoolExecutor(max_workers=8) as pool:
responses = list(pool.map(lambda _: Client(args.base_url, args.ca_file).message(
alice, chat, 'Concurrent image', key=replay_key, attachments=ids
), range(8)))
check(all(r['json'] == responses[0]['json'] for r in responses),
'concurrent attachment retries differ')
print('PASS: PNG/JPEG bytes, metadata, ACL, chat boundaries, limits and idempotency')
if __name__ == '__main__':
run(main)
+36
View File
@@ -0,0 +1,36 @@
"""Проверка русских событий авторизации и отсутствия секретов в логах Compose."""
import subprocess
import base64
from datetime import datetime, timezone
from pathlib import Path
from client import Client, arguments, check, run
def main():
args = arguments('Проверка логов авторизации')
started = datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z')
client = Client(args.base_url, args.ca_file)
actor = client.user('auth_logs')
username = actor['registration']['username']
wrong_password = 'wrong-password-for-logging'
client.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': 'Basic ' + base64.b64encode((username + ':' + wrong_password).encode()).decode()})
missing_username = 'missing_' + actor['id'].replace('-', '')[:12]
client.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': 'Basic ' + base64.b64encode((missing_username + ':' + wrong_password).encode()).decode()})
client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
client.call('GET', '/api/v1/users/me', 401, actor=actor)
result = subprocess.run(
['docker', 'compose', 'logs', '--no-color', '--since', started, 'api'],
cwd=Path(__file__).resolve().parents[1], check=True, capture_output=True, text=True,
)
logs = result.stdout + result.stderr
check('src.service.auth' in logs, 'Не используется обычный логгер модуля')
for message in ('Сессия создана', 'Вход отклонён', 'Сессия завершена', 'Доступ отклонён'):
check(message in logs, 'В логах отсутствует событие авторизации')
for secret in (actor['token'], actor['registration']['password'], wrong_password):
check(secret not in logs, 'В логах обнаружены credentials')
print('PASS: русские события входа/отказа/logout; пароли и токены не попали в логи')
if __name__ == '__main__':
run(main)
+107
View File
@@ -0,0 +1,107 @@
"""docker compose exec -T api python < tests/callbacks.py."""
import asyncio
import base64
import json
from uuid import uuid4
from urllib.error import HTTPError
from urllib.request import Request, urlopen
from unittest.mock import patch
from pydantic import SecretStr
from sqlalchemy import delete, func, select
from sqlalchemy.ext.asyncio import async_sessionmaker
from src.api.v1.routers.auth.callback import create_admin
from src.core.db.models import User
from src.core.db.session import create_engine
from src.core.settings import settings
from src.service.exceptions import UserAlreadyExistsException
from src.service.user import UserService
def call(method, path, expected, headers=None, body=None):
data = None if body is None else json.dumps(body).encode()
request_headers = dict(headers or {})
if body is not None:
request_headers['Content-Type'] = 'application/json'
request = Request(f'http://127.0.0.1:{settings.app_port}' + path,
method=method, headers=request_headers, data=data)
try:
response = urlopen(request, timeout=10)
except HTTPError as error:
response = error
with response:
assert response.status == expected, response.status
return json.load(response) if response.status != 204 else None
async def main():
engine = create_engine()
factory = async_sessionmaker(engine, expire_on_commit=False)
try:
async with factory() as session:
admin = await session.scalar(select(User).where(User.username == settings.admin_username))
assert admin is not None and admin.is_admin
identity = (admin.id, admin.password_hash, admin.created_at)
await asyncio.gather(create_admin(factory), create_admin(factory))
async with factory() as session:
admin = await session.scalar(select(User).where(User.username == settings.admin_username))
assert (admin.id, admin.password_hash, admin.created_at) == identity
assert await session.scalar(select(func.count()).select_from(User).where(
User.username == settings.admin_username,
)) == 1
print('PASS: стартовый администратор создан; повторные callbacks не дублируют и не меняют пароль')
with patch.object(settings, 'app_env', 'production'), patch.object(
settings, 'admin_password', SecretStr('different-production-password-123')
):
try:
await create_admin(factory)
except RuntimeError:
pass
else:
raise AssertionError('Production запущен с несовпадающим паролем в БД')
print('PASS: production не принимает старый пароль администратора при изменённой конфигурации')
credential = base64.b64encode((settings.admin_username + ':' + settings.admin_password.get_secret_value()).encode()).decode()
result = call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': 'Basic ' + credential})
bearer = {'Authorization': 'Bearer ' + result['token']}
me = call('GET', '/api/v1/users/me', 200, bearer)
assert me['id'] == str(identity[0])
call('DELETE', '/api/v1/auth/sessions/current', 204, bearer)
print('PASS: администратор входит через Basic, читает профиль через Bearer и выходит')
username = 'seed_test_' + uuid4().hex[:12]
password = 'valid-password-for-seed-test'
normal = call('POST', '/api/v1/users', 201, body={
'username': username, 'display_name': 'Обычный пользователь', 'password': password,
})
try:
try:
async with factory.begin() as session:
await UserService(session).create_admin(
username=username, display_name='Администратор', password=password,
)
except UserAlreadyExistsException:
pass
else:
raise AssertionError('Обычный пользователь стал администратором')
async with factory() as session:
assert not await session.scalar(select(User.is_admin).where(User.username == username))
finally:
from uuid import UUID
async with factory.begin() as session:
await session.execute(delete(User).where(User.id == UUID(normal['id'])))
print('PASS: callback не повышает права обычного пользователя с совпавшим username')
base = {'username': 'constraints_' + uuid4().hex[:12], 'display_name': 'Проверка',
'password': password}
call('POST', '/api/v1/users', 400, body={**base, 'username': 'user@example.com'})
call('POST', '/api/v1/users', 400, body={**base, 'password': 'admin'})
call('POST', '/api/v1/users', 400, body={**base, 'is_admin': True})
print('PASS: ограничения username/password сохранены; is_admin нельзя передать при регистрации')
finally:
await engine.dispose()
asyncio.run(main())
+87
View File
@@ -0,0 +1,87 @@
"""docker compose exec -T api python < tests/database.py."""
import asyncio
import subprocess
import sys
from datetime import datetime, timedelta, timezone
from uuid import uuid4
from sqlalchemy import Column, MetaData, String, Table, func, insert, select, text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import async_sessionmaker
from src.core.db.models import Chat, ChatMember, Message, User
from src.core.db.repo.chat import create_chat
from src.core.db.repo.message import history_query
from src.core.db.session import create_engine
async def main():
engine = create_engine()
factory = async_sessionmaker(engine, expire_on_commit=False)
try:
async def counts():
async with factory() as session:
return [await session.scalar(select(func.count()).select_from(model))
for model in (User, Chat, ChatMember, Message)]
before = await counts()
for _ in range(2):
subprocess.run([sys.executable, '-m', 'alembic', 'upgrade', 'head'], check=True)
assert before == await counts(), 'Повтор миграций изменил данные'
versions = Table('alembic_version', MetaData(), Column('version_num', String(32), primary_key=True))
async with factory() as session:
assert (await session.scalars(select(versions.c.version_num))).all() == ['006']
print('PASS: два повторных запуска миграций не изменили данные; версия 006 одна')
user_id, chat_id = uuid4(), uuid4()
now = datetime.now(timezone.utc)
try:
async with factory.begin() as session:
await create_chat(session, chat_id, 'rollback', [user_id], now)
except IntegrityError as error:
assert getattr(error.orig, 'sqlstate', None) == '23503'
else:
raise AssertionError('Ожидался отказ внешнего ключа')
async with factory() as session:
assert await session.get(Chat, chat_id) is None
assert await session.scalar(
select(func.count()).select_from(ChatMember).where(ChatMember.chat_id == chat_id)
) == 0
print('PASS: ошибка участника откатила INSERT чата и участников')
async with factory() as session:
try:
session.add(User(id=user_id, username='plan_' + uuid4().hex[:16],
display_name='EXPLAIN', password_hash='test-not-for-login', created_at=now))
await session.flush()
await create_chat(session, chat_id, 'EXPLAIN', [user_id], now)
await session.execute(insert(Message), [
dict(id=uuid4(), chat_id=chat_id, sender_id=user_id, text=f'message {n}',
created_at=now + timedelta(milliseconds=n)) for n in range(1, 10001)
])
# ANALYZE и EXPLAIN — диагностические команды PostgreSQL, не запросы репозитория.
await session.execute(text('ANALYZE messages'))
count = await session.scalar(
select(func.count()).select_from(Message).where(Message.chat_id == chat_id)
)
print('Сообщений в исследуемом чате:', count)
after = (await session.execute(
select(Message.created_at, Message.id).where(Message.chat_id == chat_id)
.order_by(Message.created_at, Message.id).offset(8999).limit(1)
)).one()
# Исследуется именно выражение SQLAlchemy, используемое в репозитории.
compiled = history_query(chat_id, 51, tuple(after)).compile(
dialect=engine.dialect, compile_kwargs={'literal_binds': True},
)
rows = await session.execute(text('EXPLAIN (ANALYZE, BUFFERS) ' + str(compiled)))
plan = '\n'.join(row[0] for row in rows)
assert 'messages_history_idx' in plan, plan
print(plan)
print('PASS: индекс истории используется для глубокой страницы из 10000 сообщений')
finally:
await session.rollback()
finally:
await engine.dispose()
asyncio.run(main())
+160
View File
@@ -0,0 +1,160 @@
"""Проверка отказов только локального Compose-стенда; сервисы восстанавливаются."""
import hashlib
import json
import subprocess
import time
from pathlib import Path
from client import Client, Failure, arguments, run
ROOT = Path(__file__).resolve().parents[1]
def compose(*args):
subprocess.run(
["docker", "compose", *args], cwd=ROOT, check=True, stdout=subprocess.DEVNULL, timeout=60
)
def main():
args = arguments("Отказы Redis и PostgreSQL")
c = Client(args.base_url, args.ca_file)
actor = c.user("outage")
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
key = "user:v1:" + actor["id"]
def redis(*args):
return subprocess.check_output(
["docker", "compose", "exec", "-T", "redis", "redis-cli", "--raw", *args],
cwd=ROOT,
text=True,
timeout=10,
).strip()
assert redis("EXISTS", key) == "1", "Профиль не попал в Redis"
assert 0 < int(redis("TTL", key)) <= 60
def hits():
return int(
next(
line.split(":")[1]
for line in redis("INFO", "stats").splitlines()
if line.startswith("keyspace_hits:")
)
)
before = hits()
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
assert hits() > before, "Повторное чтение не использовало Redis"
redis("DEL", key)
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
assert redis("EXISTS", key) == "1", "Кэш не восстановился из PostgreSQL"
token_key = "session:v1:" + hashlib.sha256(actor["token"].encode()).hexdigest()
assert redis("EXISTS", token_key) == "1", "Сессия отсутствует в Redis"
ttl_before = int(redis("PTTL", token_key))
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
assert 0 < int(redis("PTTL", token_key)) <= ttl_before, "Чтение продлило сессию"
print("PASS: сессия хранится в Redis с TTL; чтение не продлевает срок")
print(
"PASS: профиль кэшируется с TTL<=60; повторный GET даёт cache hit; потеря ключа восполняется"
)
lost = c.user("lost")
c.call("GET", "/api/v1/users/me", actor=lost, schema="User")
lost_key = "session:v1:" + hashlib.sha256(lost["token"].encode()).hexdigest()
redis("DEL", lost_key)
c.call("GET", "/api/v1/users/me", 401, actor=lost)
assert redis("EXISTS", lost_key) == "0", "Удалённая сессия воскресла"
print("PASS: отсутствующая сессия не восстанавливается из профиля/БД; Redis — источник истины")
# При общей неисправности LB исключает все API. Проверку отдельного
# процесса выполняем из контейнера; внешние API-порты не открываем.
container = subprocess.check_output(
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
).split()[0]
def probe(method, path, expected=200, actor=None, body=None, schema=None):
code = r"""
import json, sys, urllib.request, urllib.error
p=json.load(sys.stdin)
h={"Content-Type":"application/json"}
if p["actor"]: h["Authorization"]="Bearer "+p["actor"]["token"]
r=urllib.request.Request("http://127.0.0.1:8080"+p["path"], method=p["method"], headers=h,
data=json.dumps(p["body"]).encode() if p["body"] is not None else None)
try: response=urllib.request.urlopen(r, timeout=6)
except urllib.error.HTTPError as e: response=e
with response:
assert response.headers.get("X-Request-Id") and response.headers.get("X-Instance-Id")
print(response.status)
"""
result = subprocess.check_output(
["docker", "exec", "-i", container, "python", "-c", code],
cwd=ROOT,
text=True,
input=json.dumps(dict(method=method, path=path, actor=actor, body=body)),
)
assert int(result.strip()) == expected, (method, path, expected, result)
def wait_lb():
deadline = time.monotonic() + 30
while True:
try:
c.call("GET", "/health/ready", schema="Health")
return
except Failure:
if time.monotonic() > deadline:
raise
time.sleep(0.25)
try:
compose("stop", "redis")
probe("GET", "/health/live", schema="Health")
probe("GET", "/api/v1/users/me", 503, actor=actor)
probe("DELETE", "/api/v1/auth/sessions/current", 503, actor=actor)
probe("GET", "/health/ready", 503)
probe("GET", "/api/v1/users/me", 503, actor={"token": "invalid-token"})
probe("GET", "/api/v1/users/me", 401)
print(
"PASS: Redis выключен; live=200, ready=503; проверка сессий недоступна, доступ не выдаётся"
)
finally:
compose("up", "-d", "--wait", "redis")
wait_lb()
probe("GET", "/api/v1/users/me", actor=actor, schema="User")
print("PASS: сессия сохранилась после перезапуска Redis")
revoked = c.user("revoked")
probe("GET", "/api/v1/users/me", actor=revoked, schema="User")
probe("DELETE", "/api/v1/auth/sessions/current", 204, actor=revoked)
compose("restart", "redis")
compose("up", "-d", "--wait", "redis")
probe("GET", "/api/v1/users/me", 401, actor=revoked)
probe(
"POST",
"/api/v1/chats",
401,
actor=revoked,
body={"title": "Нет доступа", "member_ids": [revoked["id"]]},
)
print(
"PASS: logout при отказе Redis возвращает 503; успешный отзыв переживает AOF-рестарт и не даёт доступ"
)
try:
compose("stop", "postgres")
probe("GET", "/health/live", schema="Health")
probe("GET", "/health/ready", 503)
probe("GET", "/api/v1/chats", 503, actor=actor)
probe("GET", "/api/v1/users/me", 401)
probe("GET", "/api/v1/users/me", 401, actor={"token": "invalid-token"})
probe("GET", "/api/v1/users/me", actor=actor, schema="User")
print(
"PASS: PostgreSQL выключен; live=200, ready=503, чаты=503; действующая сессия и прогретый профиль доступны в Redis"
)
finally:
compose("up", "-d", "--wait", "postgres")
probe("GET", "/api/v1/users/me", actor=actor, schema="User")
wait_lb()
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
print("PASS: после восстановления БД прежняя сессия снова работает через HAProxy")
if __name__ == "__main__":
run(main)
+53
View File
@@ -0,0 +1,53 @@
"""Неожиданное исключение не раскрывает детали и сохраняет обязательные headers.
Маршрут добавлен только в этот изолированный процесс, не в работающий API.
"""
import asyncio
import json
from src.main import INSTANCE_ID, app
@app.get("/_audit_unexpected_error")
async def unexpected():
raise RuntimeError("audit-internal-detail")
async def main():
messages = []
async def receive():
await asyncio.Event().wait()
async def send(message):
messages.append(message)
scope = {
"type": "http",
"asgi": {"version": "3.0", "spec_version": "2.4"},
"http_version": "1.1",
"method": "GET",
"scheme": "http",
"path": "/_audit_unexpected_error",
"raw_path": b"/_audit_unexpected_error",
"query_string": b"",
"root_path": "",
"headers": [],
"server": ("localhost", 80),
"client": ("127.0.0.1", 1),
}
await app(scope, receive, send)
start = next(m for m in messages if m["type"] == "http.response.start")
payload = json.loads(
b"".join(m.get("body", b"") for m in messages if m["type"] == "http.response.body")
)
headers = dict(start["headers"])
assert start["status"] == 500
assert headers[b"x-instance-id"].decode() == INSTANCE_ID
assert payload["error"]["request_id"] == headers[b"x-request-id"].decode()
assert payload["error"]["code"] == "internal_error"
assert "audit-internal-detail" not in json.dumps(payload)
print("PASS: 500 has X-Request-Id/X-Instance-Id and hides internal exception details")
asyncio.run(main())
+17
View File
@@ -0,0 +1,17 @@
"""Отдельный стенд/запуск с SESSION_TTL_SECONDS=2, не обычный smoke."""
import time
from client import Client, arguments, run
def main():
args = arguments('Истечение короткой сессии')
c = Client(args.base_url, args.ca_file)
actor = c.user('expiry')
c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
time.sleep(3)
c.call('GET', '/api/v1/users/me', 401, actor=actor)
print('PASS: действующая сессия стала 401 через 3 секунды при TTL=2')
if __name__ == '__main__':
run(main)
+111
View File
@@ -0,0 +1,111 @@
from concurrent.futures import ThreadPoolExecutor
import json
import base64
import uuid
from urllib.parse import quote
from client import Client, arguments, check, run
def main():
args = arguments("Собственные тесты мессенджера")
client = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = client.bootstrap()
chat_path = "/api/v1/chats/" + chat["id"]
client.call(
"POST",
"/api/v1/users",
400,
raw=b'{"username":',
headers={"Content-Type": "application/json"},
)
client.call("POST", "/api/v1/users", 409, body=alice["registration"])
client.call(
"POST",
chat_path + "/messages",
400,
actor=alice,
body={"text": " "},
)
client.call("GET", chat_path, 404, actor=eve)
# Уникальность защищена ограничением БД, в том числе при гонке.
registration = {
'username': 'race_' + uuid.uuid4().hex[:16],
'display_name': 'Гонка регистрации',
'password': 'Пароль:с-двоеточием-123',
}
def register(_):
local = Client(args.base_url, args.ca_file)
return local.call('POST', '/api/v1/users', [201, 409], body=registration)['status']
with ThreadPoolExecutor(max_workers=6) as pool:
statuses = list(pool.map(register, range(6)))
check(statuses.count(201) == 1 and statuses.count(409) == 5, 'гонка регистрации не защищена')
session = client.call('POST', '/api/v1/auth/sessions', 201,
headers={'Authorization': 'Basic ' + base64.b64encode((registration['username'] + ':' + registration['password']).encode()).decode()}, schema='Session')['json']
actor = {'token': session['token']}
client.call('GET', '/api/v1/users/me', actor=actor, schema='User')
client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
# Даже прогретый Redis не должен разрешать отозванную сессию.
client.call('GET', '/api/v1/users/me', 401, actor=actor)
def send_batch(worker_id):
local = Client(args.base_url, args.ca_file)
result = []
for index in range(10):
response = local.message(
alice,
chat,
"параллельное-%d-%d" % (worker_id, index),
)["json"]
result.append(response["id"])
return result
with ThreadPoolExecutor(max_workers=10) as pool:
created_ids = [
message_id
for batch in pool.map(send_batch, range(10))
for message_id in batch
]
check(len(created_ids) == 100, "не все параллельные запросы вернули ответ")
check(len(set(created_ids)) == 100, "параллельные запросы вернули повторяющиеся ID")
received = []
cursor = None
while True:
path = chat_path + "/messages?limit=17"
if cursor is not None:
path += "&cursor=" + quote(cursor, safe="")
page = client.call("GET", path, actor=bob, schema="MessagePage")["json"]
received.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
break
concurrent_messages = [
message for message in received if message["text"].startswith("параллельное-")
]
received_ids = [message["id"] for message in concurrent_messages]
check(len(received_ids) == 100, "в истории нет ровно 100 параллельно созданных сообщений")
check(len(set(received_ids)) == 100, "в истории есть повторяющиеся сообщения")
check(set(received_ids) == set(created_ids), "ID созданных и прочитанных сообщений не совпадают")
print(
json.dumps(
{
"результат": "ПРОЙДЕНО",
"гонка_регистрации": "1 создан, 5 конфликтов",
"unicode_пароль_и_logout_с_кэшем": "ПРОЙДЕНО",
"параллельных_клиентов": 10,
"создано_сообщений": len(created_ids),
"уникальных_сообщений": len(set(received_ids)),
},
ensure_ascii=False,
)
)
if __name__ == "__main__":
run(main)
+79
View File
@@ -0,0 +1,79 @@
"""Запуск внутри API: rollback и 24h expiry без ожидания суток."""
import asyncio
from datetime import datetime, timezone
from uuid import uuid4
from sqlalchemy import delete, select, text, update
from sqlalchemy.ext.asyncio import async_sessionmaker
from src.core.db.models import Chat, ChatMember, Message, MessageRequest, User
from src.core.db.repo.message import create_idempotent_message
from src.core.db.session import create_engine
async def main():
engine = create_engine()
factory = async_sessionmaker(engine, expire_on_commit=False)
user_id, chat_id = uuid4(), uuid4()
now = datetime.now(timezone.utc)
async with factory.begin() as s:
s.add(
User(
id=user_id,
username="idem_" + user_id.hex[:20],
display_name="Fixture",
password_hash="not-for-login",
created_at=now,
)
)
await s.flush()
s.add(Chat(id=chat_id, title="Idempotency fixture", created_at=now))
await s.flush()
s.add(ChatMember(chat_id=chat_id, user_id=user_id))
try:
try:
async with factory.begin() as s:
await create_idempotent_message(
s, uuid4(), chat_id, user_id, "rollback", now, "same"
)
raise RuntimeError("simulated failure before COMMIT")
except RuntimeError:
pass
async with factory() as s:
assert not list(await s.scalars(select(Message).where(Message.chat_id == chat_id)))
assert not list(
await s.scalars(select(MessageRequest).where(MessageRequest.chat_id == chat_id))
)
async with factory.begin() as s:
first = await create_idempotent_message(
s, uuid4(), chat_id, user_id, "committed", now, "same"
)
async with factory.begin() as s:
replay = await create_idempotent_message(
s, uuid4(), chat_id, user_id, "committed", now, "same"
)
assert replay == first
await s.execute(
update(MessageRequest)
.where(MessageRequest.chat_id == chat_id)
.values(expires_at=text("clock_timestamp() - interval '1 second'"))
)
async with factory.begin() as s:
fresh = await create_idempotent_message(
s, uuid4(), chat_id, user_id, "new after expiry", now, "same"
)
assert fresh.id != first.id
print(
"PASS: rollback removes both message/key; committed result replays; expired key can be reused"
)
finally:
async with factory.begin() as s:
await s.execute(delete(MessageRequest).where(MessageRequest.chat_id == chat_id))
await s.execute(delete(Message).where(Message.chat_id == chat_id))
await s.execute(delete(Chat).where(Chat.id == chat_id))
await s.execute(delete(User).where(User.id == user_id))
await engine.dispose()
asyncio.run(main())
+208
View File
@@ -0,0 +1,208 @@
"""Replay concrete reviewer cases through real HTTP and controlled dependency freezes."""
import io
import json
import struct
import subprocess
import time
import zlib
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from client import Client, arguments, check, run
ROOT=Path(__file__).resolve().parents[1]
def compose(*args):
return subprocess.check_output(['docker','compose',*args],cwd=ROOT,text=True,stderr=subprocess.DEVNULL).strip()
def png(width,height):
# Complete PNG, including all scanlines and CRCs, without allocating a bitmap.
def chunk(kind,data):
return struct.pack('>I',len(data))+kind+data+struct.pack('>I',zlib.crc32(kind+data))
compressor=zlib.compressobj()
packed=io.BytesIO()
row=b'\0'*(width+1)
for _ in range(height): packed.write(compressor.compress(row))
packed.write(compressor.flush())
return (b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',width,height,8,0,0,0,0))
+chunk(b'IDAT',packed.getvalue())+chunk(b'IEND',b''))
DIRECT='''import json,sys,time,urllib.request,urllib.error
p=json.load(sys.stdin);h={}
if p.get('token'):h['Authorization']='Bearer '+p['token']
r=urllib.request.Request('http://127.0.0.1:8080'+p['path'],headers=h)
t=time.monotonic()
try:r=urllib.request.urlopen(r,timeout=10)
except urllib.error.HTTPError as e:r=e
with r:
data=json.load(r)
if r.code>=400:
assert data['error']['request_id']==r.headers['X-Request-Id']
assert data['error']['code']=='unavailable'
print(json.dumps({'status':r.code,'seconds':round(time.monotonic()-t,4),
'request_id_present':bool(r.headers.get('X-Request-Id')),'instance_id_present':bool(r.headers.get('X-Instance-Id'))}))
'''
LOCKER='''import asyncio,sys,asyncpg
from src.core.settings import settings
async def f():
c=await asyncpg.connect(settings.database_url.get_secret_value())
await c.execute('BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE')
print('LOCKED',flush=True)
await asyncio.to_thread(sys.stdin.readline)
await c.execute('ROLLBACK');await c.close()
asyncio.run(f())
'''
PENDING='''import asyncio,asyncpg
from src.core.settings import settings
async def f():
c=await asyncpg.connect(settings.database_url.get_secret_value())
print(await c.fetchval("SELECT count(*) FROM pg_stat_activity WHERE wait_event_type='Lock' AND query LIKE '%FROM messages%'"))
await c.close()
asyncio.run(f())
'''
def main():
args=arguments('Lab4 reviewer regression scenarios')
c=Client(args.base_url,args.ca_file,timeout=15)
alice,bob,_,chat=c.bootstrap()
history='/api/v1/chats/'+chat['id']+'/messages'
c.message(alice,chat,'Контроль до отказов')
results=[]
for name in ('review123\n','review123\r','review123\x00','review123\t'):
c.call('POST','/api/v1/users',400,body={'username':name,'display_name':'Review','password':'valid-password-123'})
results.append({'case':'username_control_characters','result':'PASS','status':400})
data=(ROOT/'tests/fixtures/sample.png').read_bytes()
upload='/api/v1/chats/'+chat['id']+'/attachments'
for label,filename,payload,mime,status in [
('truncated_png','sample.png',data[:-12],'image/png',415),
('header_only_png','sample.png',data[:33],'image/png',415),
('mismatched_mime','sample.png',data,'image/jpeg',415),
('DEL_filename','a%7F.png',data,'image/png',400),
('missing_filename',None,data,'image/png',400),
]:
c.call('POST',upload+('' if filename is None else '?filename='+filename),status,
actor=alice,raw=payload,headers={'Content-Type':mime})
results.append({'case':label,'result':'PASS','status':status})
for width,height,status in ((8000,5000,201),(8000,5001,413),(20000,10000,413)):
payload=png(width,height)
reply=c.call('POST',upload+'?filename=dimensions.png',status,actor=alice,raw=payload,
headers={'Content-Type':'image/png'},schema='Attachment' if status==201 else None)
if status==201:
check(c.call('GET','/api/v1/attachments/'+reply['json']['id']+'/content',actor=bob)['bytes']==payload,
'40 MP original changed')
results.append({'case':'pixels','dimensions':[width,height],'result':'PASS','status':status})
api=compose('ps','-q','api').split()[0]
# Actual application credentials must not permit creating unrelated buckets.
code='''from src.core.storage import get_storage
from minio.error import S3Error
import uuid
s=get_storage();b='forbidden-'+uuid.uuid4().hex
try:s.make_bucket(b)
except S3Error as e:assert e.code=='AccessDenied';print('PASS')
else:s.remove_bucket(b);raise AssertionError('API can create foreign buckets')
'''
check(subprocess.check_output(['docker','exec',api,'python','-c',code],text=True).strip()=='PASS',
'API has excessive S3 rights')
results.append({'case':'API_cannot_create_foreign_bucket','result':'PASS'})
ids=compose('ps','-a','-q','api','reconciler','migration').split()
containers=json.loads(subprocess.check_output(['docker','inspect',*ids],text=True))
for container in containers:
service=container['Config']['Labels']['com.docker.compose.service']
environment=dict(value.split('=',1) for value in container['Config']['Env'] if '=' in value)
forbidden=['MINIO_ROOT_USER','MINIO_ROOT_PASSWORD']
if service in ('api','migration'):
forbidden+=['S3_RECONCILE_ACCESS_KEY_ID','S3_RECONCILE_SECRET_ACCESS_KEY']
if service in ('reconciler','migration'):
forbidden+=['S3_ACCESS_KEY_ID','S3_SECRET_ACCESS_KEY']
for key in forbidden:
check(not environment.get(key),service+' inherited an unnecessary S3 credential: '+key)
results.append({'case':'S3_credentials_confined_to_intended_services','result':'PASS'})
def direct(path,token=None):
reply=subprocess.run(['docker','exec','-i',api,'python','-c',DIRECT],
input=json.dumps({'path':path,'token':token}),text=True,capture_output=True,timeout=13,check=True)
return json.loads(reply.stdout)
transfer_code="""import asyncio,http.client,json,sys,time
from sqlalchemy import func,select
from sqlalchemy.ext.asyncio import async_sessionmaker
from src.core.db.session import create_engine
from src.core.db.models import Attachment,UploadIntent
p=json.load(sys.stdin)
async def counts():
e=create_engine();f=async_sessionmaker(e)
try:
async with f() as s:
return [await s.scalar(select(func.count()).select_from(m)) for m in (Attachment,UploadIntent)]
finally:await e.dispose()
before=asyncio.run(counts())
c=http.client.HTTPConnection('127.0.0.1',8080,timeout=8)
c.putrequest('POST',p['path'])
c.putheader('Authorization','Bearer '+p['token'])
c.putheader('Content-Type','image/png');c.putheader('Content-Length','1000');c.endheaders()
c.send(b'\\x89PNG\\r\\n\\x1a\\nshort');c.close();time.sleep(.3)
assert before==asyncio.run(counts()),'interrupted client created durable state'
c=http.client.HTTPConnection('127.0.0.1',8080,timeout=8)
c.request('POST',p['path'],body=(b'x'*65536 for _ in range(161)),
headers={'Authorization':'Bearer '+p['token'],'Content-Type':'image/png'},encode_chunked=True)
r=c.getresponse();body=json.loads(r.read())
assert r.status==413 and body['error']['code']=='payload_too_large'
assert body['error']['request_id']==r.getheader('X-Request-Id')
c.close();assert before==asyncio.run(counts()),'oversized chunked body created durable state'
print('PASS')
"""
transfer=subprocess.run(['docker','exec','-i',api,'python','-c',transfer_code],
input=json.dumps({'path':upload+'?filename=interrupted.png','token':alice['token']}),
text=True,capture_output=True,timeout=20,check=True)
check(transfer.stdout.strip()=='PASS','transfer limits failed')
results.append({'case':'client_abort_and_oversized_chunked_body','result':'PASS',
'oversized_status':413,'durable_state_unchanged':True})
direct(history,alice['token'])
# Start an actual history SELECT, hold it on a table lock, then freeze PG.
locker=subprocess.Popen(['docker','exec','-i',api,'python','-u','-c',LOCKER],
stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
check(locker.stdout.readline().strip()=='LOCKED','test table lock was not acquired')
paused=False
pool=ThreadPoolExecutor(max_workers=2)
query=pool.submit(direct,history,alice['token'])
try:
deadline=time.monotonic()+2
while True:
pending=int(subprocess.check_output(['docker','exec',api,'python','-c',PENDING],text=True))
if pending:break
check(time.monotonic()<deadline,'history did not enter SQL before the freeze')
time.sleep(.02)
compose('pause','postgres');paused=True
ready=pool.submit(direct,'/health/ready')
history_result=query.result(timeout=8)
readiness_result=ready.result(timeout=8)
for result in (history_result,readiness_result):
check(result['status']==503 and result['seconds']<7,'frozen PostgreSQL exceeded HTTP budget')
check(result['request_id_present'] and result['instance_id_present'],'missing response headers')
results.append({'case':'PostgreSQL_frozen_after_SQL_started','result':'PASS',
'history':history_result,'readiness':readiness_result})
finally:
if paused:compose('unpause','postgres')
try:locker.stdin.write('release\n');locker.stdin.flush()
except BrokenPipeError:pass
locker.wait(timeout=8)
pool.shutdown(wait=True)
deadline=time.monotonic()+12
while True:
try:
response=direct(history,alice['token'])
check(response['status']==200,'history did not recover')
check(direct('/health/ready')['status']==200,'readiness did not recover')
break
except Exception:
if time.monotonic()>deadline:raise
time.sleep(.2)
results.append({'case':'PostgreSQL_resume_without_API_restart','result':'PASS'})
print(json.dumps({'result':'PASS','checks':results},ensure_ascii=False,indent=2))
if __name__=='__main__':
run(main)
+68
View File
@@ -0,0 +1,68 @@
"""docker compose exec -T api python < tests/passwords.py."""
import asyncio
import base64
import json
from datetime import datetime, timezone
from uuid import uuid4
from urllib.error import HTTPError
from urllib.request import Request, urlopen
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import async_sessionmaker
from src.core import cache, session_store
from src.core.db.models import User
from src.core.db.session import create_engine
from src.core.security import hash_password, hash_token, verify_password
def login(username, password, expected):
credential = base64.b64encode((username + ':' + password).encode()).decode()
request = Request('http://127.0.0.1:8080/api/v1/auth/sessions', method='POST',
headers={'Authorization': 'Basic ' + credential})
try:
response = urlopen(request, timeout=10)
except HTTPError as error:
response = error
with response:
assert response.status == expected, response.status
return json.load(response)
async def main():
password = 'Я:' + 'ю' * 126
hashed = hash_password(password)
assert hashed.startswith('$argon2id$')
assert hash_password(password) != hashed
assert verify_password(password, hashed)
assert not verify_password('неверный-пароль', hashed)
assert not verify_password(password, 'unsupported$hash')
print('PASS: только Argon2id, случайная соль, Unicode 128 символов, неверный пароль')
engine = create_engine()
factory = async_sessionmaker(engine, expire_on_commit=False)
user_id, username = uuid4(), 'password_' + uuid4().hex[:16]
tokens = []
cache.open_cache()
try:
async with factory.begin() as session:
session.add(User(id=user_id, username=username, display_name='Password test',
password_hash=hashed, created_at=datetime.now(timezone.utc)))
login(username, 'неверный-пароль', 401)
for _ in range(2):
result = login(username, password, 201)
tokens.append(result['token'])
stored = await session_store.fetch_session(hash_token(result['token']))
assert stored is not None and stored.user_id == user_id
async with factory() as session:
assert await session.scalar(select(User.password_hash).where(User.id == user_id)) == hashed
print('PASS: повторный вход с Argon2id; сессии находятся в Redis; хеш пароля не изменяется')
finally:
for token in tokens:
await session_store.delete_session(hash_token(token))
await cache.close_cache()
async with factory.begin() as session:
await session.execute(delete(User).where(User.id == user_id))
await engine.dispose()
asyncio.run(main())
+187
View File
@@ -0,0 +1,187 @@
"""Негативные HTTP/Redis проверки для production-аудита; только локальный Compose."""
import base64
from concurrent.futures import ThreadPoolExecutor
import hashlib
import http.client
import json
from pathlib import Path
import subprocess
import threading
import uuid
from urllib.parse import urlsplit, quote
from client import Client, arguments, check, run
ROOT = Path(__file__).resolve().parents[1]
def redis(*args, raw=None):
command = ['docker', 'compose', 'exec', '-T', 'redis', 'redis-cli', '--raw']
if raw is not None:
command += ['-x']
return subprocess.check_output(command + list(args), cwd=ROOT, input=raw, timeout=10).rstrip(b'\n')
def main():
args = arguments('Production-аудит входных данных, Redis и изоляции сессий')
c = Client(args.base_url, args.ca_file)
alice, bob, _, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/messages'
failures = []
def expect(label, method, path, status, **kwargs):
response = c.call(method, path, [200, 201, 400, 401, 413, 415, 500, 503], **kwargs)
if response['status'] != status:
failures.append(f'{label}: ожидался {status}, получен {response["status"]}')
return response
payload = {'kind': 'messages', 'owner_id': chat['id'],
'created_at': '2026-01-01T00:00:00+00:00', 'item_id': str(uuid.uuid4())}
for field, value in [('owner_id', 123), ('item_id', 123), ('owner_id', []),
('created_at', '9999-12-31T23:59:59-23:59')]:
cursor = base64.urlsafe_b64encode(json.dumps({**payload, field: value}).encode()).decode()
expect('Типы/диапазон курсора ' + field, 'GET', path + '?cursor=' + cursor, 400, actor=alice)
expect('Длина курсора', 'GET', path + '?cursor=' + 'a' * 5000, 400, actor=alice)
for text in ('\ud800', '\udfff'):
expect('Непредставимый текст PostgreSQL', 'POST', path, 400, actor=alice,
raw=json.dumps({'text': 'valid' + text}).encode(), headers={'Content-Type': 'application/json'})
registration = {'username': 'unicode_' + uuid.uuid4().hex[:16],
'password': 'valid-password-123', 'display_name': 'Valid'}
for field, value in [('display_name', '\ud800'),
('password', 'secret-audit-\ud800')]:
expect('Непредставимая строка ' + field, 'POST', '/api/v1/users', 400,
raw=json.dumps({**registration, field: value}).encode(), headers={'Content-Type': 'application/json'})
expect('Большое тело с допустимым коротким текстом', 'POST', path, 413, actor=alice,
raw=b'{"text":"valid"}' + b' ' * 70000, headers={'Content-Type': 'application/json'})
expect('Повторяющиеся ключи JSON', 'POST', path, 400, actor=alice,
raw=b'{"text":"first","text":"second"}', headers={'Content-Type': 'application/json'})
expect('JSON не в UTF-8', 'POST', path, 400, actor=alice,
raw='{"text":"valid"}'.encode('utf-16'), headers={'Content-Type': 'application/json'})
expect('Нестандартный NaN', 'POST', path, 400, actor=alice,
raw=b'{"text":"valid","extra":NaN}', headers={'Content-Type': 'application/json'})
expect('Глубокий JSON', 'POST', path, 400, actor=alice,
raw=b'[' * 1200 + b'0' + b']' * 1200, headers={'Content-Type': 'application/json'})
# Максимально допустимый текст остаётся допустимым и в escaped JSON.
emoji = '😀' * 4000
maximum = c.call('POST', path, 201, actor=alice, schema='Message',
raw=json.dumps({'text': emoji}, ensure_ascii=True).encode(),
headers={'Content-Type': 'application/json', 'Idempotency-Key':uuid.uuid4().hex})
check(maximum['json']['text'] == emoji, 'Лимит тела отверг/исказил 4000 Unicode-символов')
check(maximum['headers'].get('Cache-Control') == 'no-store', 'Ответ API может попасть в кэш')
check(maximum['headers'].get('X-Content-Type-Options') == 'nosniff', 'Нет nosniff')
parsed = urlsplit(c.base)
check(parsed.scheme == 'http', 'Сетевые проверки этого Compose-сценария рассчитаны на localhost HTTP')
def wire(duplicate=False, chunked=False, slow=False, started=None):
connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=10)
try:
if chunked:
connection.request('POST', path, body=iter([b'{"text":"valid"}', b' ' * 40000, b' ' * 40000]),
headers={'Authorization': 'Bearer ' + alice['token'],
'Content-Type': 'application/json'}, encode_chunked=True)
elif slow:
connection.putrequest('POST', path)
connection.putheader('Authorization', 'Bearer ' + alice['token'])
connection.putheader('Content-Type', 'application/json')
connection.putheader('Content-Length', '100')
connection.endheaders(b'{\"text\":')
started.set()
else:
connection.putrequest('GET', '/api/v1/users/me')
connection.putheader('Authorization', 'Bearer ' + alice['token'])
if duplicate:
connection.putheader('Authorization', 'Bearer ' + bob['token'])
connection.endheaders()
response = connection.getresponse()
data = json.loads(response.read())
check(data['error']['request_id'] == response.getheader('X-Request-Id'), 'Сетевой запрос потерял request_id')
return response.status
finally:
connection.close()
check(wire(chunked=True) == 413, 'Chunked обходит лимит размера тела')
check(wire(duplicate=True) == 401, 'Два Authorization принимаются неоднозначно')
print('PASS: реальный chunked=413, два Authorization=401, no-store; 4000 emoji в escaped JSON допустимы')
started = threading.Event()
with ThreadPoolExecutor(max_workers=1) as pool:
pending = pool.submit(wire, slow=True, started=started)
check(started.wait(2), 'Не началась медленная передача')
health = c.call('GET', '/health/live', schema='Health')
check(health['elapsed'] < 2, 'Медленная передача заблокировала event loop')
check(pending.result(timeout=10) == 408, 'У медленной передачи нет deadline')
print('PASS: реальное незавершённое тело получает 408; health работает во время ожидания')
created = [chat]
for index in range(12):
created.append(c.call('POST', '/api/v1/chats', 201, actor=alice, schema='Chat',
body={'title': 'SQL-page-' + str(index), 'member_ids': [alice['id']]})['json'])
chats, cursor = [], None
while True:
url = '/api/v1/chats?limit=3' + ('&cursor=' + quote(cursor, safe='') if cursor else '')
page = c.call('GET', url, actor=alice, schema='ChatPage')['json']
chats.extend(item['id'] for item in page['items'])
cursor = page['next_cursor']
if cursor is None:
break
c.call('GET', '/api/v1/chats?cursor=' + quote(cursor, safe=''), 400, actor=bob)
check(len(chats) <= len(created), 'Зациклилась пагинация чатов')
check(chats == [item['id'] for item in sorted(created, key=lambda x: (x['created_at'], uuid.UUID(x['id']).int))],
'SQL-пагинация чатов потеряла/повторила объекты')
bob_chats = c.call('GET', '/api/v1/chats', actor=bob, schema='ChatPage')['json']
check([item['id'] for item in bob_chats['items']] == [chat['id']], 'В список попали чужие чаты')
print('PASS: SQL-пагинация 13 чатов по 3; курсор принадлежит пользователю, чужие чаты скрыты')
profile_key = 'user:v1:' + alice['id']
for raw in (b'\xff\xfe', b'null', b'{"id":123}'):
redis('SET', profile_key, raw=raw)
expect('Повреждённый профиль Redis', 'GET', '/api/v1/users/me', 200, actor=alice)
session_key = 'session:v1:' + hashlib.sha256(alice['token'].encode()).hexdigest()
saved = redis('GET', session_key)
ttl = int(redis('PTTL', session_key))
for raw in (b'\xff', json.dumps({'user_id': 123, 'expires_at': '2099-01-01T00:00:00+00:00'}).encode(),
b'[]', b'null', json.dumps({'user_id': alice['id'], 'expires_at': '2020-01-01T00:00:00+00:00'}).encode()):
try:
redis('SET', session_key, raw=raw)
redis('PEXPIRE', session_key, str(ttl))
expect('Повреждённая/истёкшая сессия Redis', 'GET', '/api/v1/users/me', 401, actor=alice)
finally:
redis('SET', session_key, raw=saved)
redis('PEXPIRE', session_key, str(ttl))
c.call('GET', '/api/v1/users/me', actor=alice, schema='User')
# NUL недопустим в PostgreSQL text, но пароль хранится как Argon2-хеш.
special = {**registration, 'username': 'nulpass_' + uuid.uuid4().hex[:16],
'password': ' пароль:\x00с пробелами '}
c.call('POST', '/api/v1/users', 201, body=special, schema='User')
special_basic = 'Basic ' + base64.b64encode((special['username'] + ':' + special['password']).encode()).decode()
special_session = c.call('POST', '/api/v1/auth/sessions', 201,
headers={'Authorization': special_basic}, schema='Session')['json']
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor={'token': special_session['token']})
print('PASS: пароль с Unicode, пробелами, двоеточием и NUL не усекается')
basic = 'Basic ' + base64.b64encode((alice['registration']['username'] + ':' + alice['registration']['password']).encode()).decode()
another = c.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': basic}, schema='Session')['json']
check(another['token'] != alice['token'], 'Повторный вход вернул старый токен')
def get_profile(_):
return Client(args.base_url, args.ca_file).call('GET', '/api/v1/users/me', actor=alice, schema='User')
with ThreadPoolExecutor(max_workers=8) as pool:
profiles = list(pool.map(get_profile, range(24)))
check(all(p['json']['id'] == alice['id'] for p in profiles), 'Смешаны пользователи конкурентных запросов')
check(len({p['headers']['X-Request-Id'] for p in profiles}) == 24, 'Повторяется request_id')
with ThreadPoolExecutor(max_workers=4) as pool:
list(pool.map(lambda _: Client(args.base_url, args.ca_file).call(
'DELETE', '/api/v1/auth/sessions/current', [204, 401], actor=alice), range(4)))
c.call('GET', '/api/v1/users/me', 401, actor=alice)
c.call('GET', '/api/v1/users/me', actor={'token': another['token']}, schema='User')
c.call('GET', '/api/v1/users/me', actor=bob, schema='User')
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor={'token': another['token']})
print('PASS: независимые токены, конкурентные logout/GET, уникальные request_id')
if failures:
for failure in failures:
print('FAIL:', failure)
check(False, 'Production-аудит нашёл ошибки: ' + str(len(failures)))
print('PASS: типы/диапазон курсора, Unicode, размер/формат JSON и повреждённый Redis')
if __name__ == '__main__':
run(main)
+137
View File
@@ -0,0 +1,137 @@
"""Изолированные проверки ASGI-лимитов и реального пула Argon2 в образе API."""
import asyncio
import errno
import json
import os
from pathlib import Path
from tempfile import TemporaryDirectory
import threading
import time
from unittest.mock import patch
from src.core import security
from src.core.request_limits import RequestLimitsMiddleware
from src.core.settings import settings
def scope(path='/api/v1/users', headers=()):
return {'type': 'http', 'asgi': {'version': '3.0'}, 'http_version': '1.1',
'method': 'POST', 'scheme': 'http', 'path': path, 'raw_path': path.encode(),
'query_string': b'', 'headers': [(b'content-type', b'application/json'), *headers],
'state': {'request_id': 'limit-test'}, 'server': ('localhost', 80), 'client': ('localhost', 1)}
async def invoke(app, messages=(), path='/api/v1/users', headers=(), delay=0):
sent = []
pending = iter(messages or [{'type': 'http.request', 'body': b'{}', 'more_body': False}])
async def receive():
if delay:
await asyncio.sleep(delay)
return next(pending)
async def send(message):
sent.append(message)
await app(scope(path, headers), receive, send)
if not sent:
return None
status = next(m['status'] for m in sent if m['type'] == 'http.response.start')
data = json.loads(b''.join(m.get('body', b'') for m in sent if m['type'] == 'http.response.body'))
if status >= 400:
assert data['error']['request_id'] == 'limit-test'
return status, data
async def main():
assert os.geteuid() != 0, 'API работает от root'
status = Path('/proc/self/status').read_text()
assert 'NoNewPrivs:\t1' in status
assert 'CapEff:\t0000000000000000' in status
probe = Path('/app/.production-write-probe')
try:
probe.write_text('probe')
except OSError as error:
assert error.errno == errno.EROFS, error.errno
else:
probe.unlink()
raise AssertionError('Корневая файловая система API доступна для записи')
with TemporaryDirectory() as directory:
Path(directory, 'probe').write_text('temporary')
print('PASS: non-root, read-only rootfs, пустые capabilities, no-new-privileges; /tmp доступен')
called = 0
async def endpoint(scope, receive, send):
nonlocal called
called += 1
message = await receive()
await send({'type': 'http.response.start', 'status': 200, 'headers': []})
await send({'type': 'http.response.body', 'body': json.dumps({'bytes': len(message['body'])}).encode()})
app = RequestLimitsMiddleware(endpoint)
maximum = settings.http_max_body_bytes
assert (await invoke(app, headers=[(b'content-length', str(maximum+1).encode())]))[0] == 413
assert called == 0
chunks = [{'type': 'http.request', 'body': b' ' * 40000, 'more_body': True},
{'type': 'http.request', 'body': b' ' * 40000, 'more_body': False}]
assert (await invoke(app, messages=chunks))[0] == 413
assert (await invoke(app, messages=chunks, headers=[(b'content-length', b'1')]))[0] == 413
assert (await invoke(app, headers=[(b'content-length', b'-1')]))[0] == 400
with patch.object(settings, 'http_body_timeout_seconds', 0.02):
assert (await invoke(app, delay=0.1))[0] == 408
assert await invoke(app, messages=[{'type': 'http.disconnect'}]) is None
assert called == 0
assert (await invoke(app))[0] == 200
assert app.limiter.borrowed_tokens == 0
print('PASS: Content-Length/chunked/ложный размер, timeout, disconnect; слот всегда освобождается')
gate = asyncio.Event()
entered = asyncio.Event()
active = 0
async def blocked(scope, receive, send):
nonlocal active
if scope['path'].startswith('/api/'):
active += 1
if active == 2:
entered.set()
await gate.wait()
await endpoint(scope, receive, send)
with patch.object(settings, 'http_concurrency_limit', 2):
limited = RequestLimitsMiddleware(blocked)
tasks = [asyncio.create_task(invoke(limited)) for _ in range(2)]
try:
await asyncio.wait_for(entered.wait(), 1)
assert (await invoke(limited))[0] == 503
assert (await invoke(limited, path='/health/live'))[0] == 200
tasks[0].cancel()
await asyncio.gather(tasks[0], return_exceptions=True)
assert limited.limiter.borrowed_tokens == 1
gate.set()
assert (await tasks[1])[0] == 200
assert (await invoke(limited))[0] == 200
finally:
gate.set()
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
assert limited.limiter.borrowed_tokens == 0
print('PASS: перегрузка=503 в общем JSON, health доступен, отмена запроса не теряет слот')
lock = threading.Lock()
active = peak = 0
def slow_hash(password):
nonlocal active, peak
with lock:
active += 1
peak = max(peak, active)
try:
time.sleep(0.03)
return 'hashed-' + password
finally:
with lock:
active -= 1
count = settings.password_hash_concurrency * 4
with patch.object(security, 'hash_password', slow_hash):
result = await asyncio.gather(*(security.hash_password_async(str(i)) for i in range(count)))
assert result == ['hashed-' + str(i) for i in range(count)]
assert peak == settings.password_hash_concurrency, peak
assert security.password_limiter.borrowed_tokens == 0
print('PASS: очередь Argon2 соблюдает лимит параллелизма; результаты не смешиваются')
asyncio.run(main())
+53
View File
@@ -0,0 +1,53 @@
"""Изолированная проверка HTTP-бюджета readiness при медленной отмене драйвера."""
import asyncio
import time
from unittest.mock import patch
from src.core import readiness
from src.core.errors import ApiError
from src.core.settings import settings
async def main():
cancelled = asyncio.Event()
release = asyncio.Event()
calls = 0
async def stalled_driver():
nonlocal calls
calls += 1
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancelled.set()
await release.wait()
with (
patch.object(settings, "healthcheck_timeout_seconds", 0.03),
patch.object(readiness, "_probe", stalled_driver),
):
for _ in range(3):
began = time.monotonic()
try:
await readiness.check()
except ApiError as error:
assert error.status_code == 503
else:
raise AssertionError("Stalled dependency was declared ready")
assert time.monotonic() - began < 0.15, "HTTP waited for slow driver cancellation"
assert cancelled.is_set() and calls == 1, "Readiness spawned accumulating probes"
release.set()
await asyncio.sleep(0)
await readiness.close()
async def healthy():
pass
with patch.object(readiness, "_probe", healthy):
await readiness.check()
await readiness.close()
print("PASS: bounded readiness response, one pending probe, recovery after cancellation")
asyncio.run(main())
+418
View File
@@ -0,0 +1,418 @@
"""Integration research on isolated PostgreSQL schema and private MinIO bucket.
Run through scripts/test-reconciliation.sh; credentials stay in environment.
Only UUID-named test resources are removed on completion.
"""
import asyncio
import hashlib
import io
import json
import subprocess
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta, timezone
from unittest.mock import patch
from uuid import uuid4
from minio import Minio
from minio.error import S3Error
from urllib.parse import urlsplit
from PIL import Image
from sqlalchemy import event, func, select, text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from src.core.db.models import (
Attachment, Base, Chat, ChatMember, Message, MessageAttachment,
StorageFinding, UploadIntent, User,
)
from src.core.errors import ApiError
from src.core.db.repo.message import create_idempotent_message
from src.core.settings import settings
from src.core.storage import StorageMetrics, make_storage
from src.core.upload_coordination import lock_object
from src.maintenance.reconcile import Reconciler
from src.service.attachment import AttachmentService
schema = 'reconcile_' + uuid4().hex
bucket = 'reconcile-' + uuid4().hex
original_bucket = settings.s3_bucket
root = make_storage(role='bootstrap')
content_buffer = io.BytesIO()
Image.new('RGB', (32, 24), 'green').save(content_buffer, format='PNG')
content = content_buffer.getvalue()
user_id, chat_id = uuid4(), uuid4()
checks = []
def passed(name):
checks.append(name)
print(json.dumps({'check': name, 'result': 'PASS'}), flush=True)
def fake_request():
from starlette.requests import Request
delivered = False
async def receive():
nonlocal delivered
if delivered:
return {'type': 'http.disconnect'}
delivered = True
return {'type': 'http.request', 'body': content, 'more_body': False}
return Request({'type':'http', 'method':'POST', 'path':'/',
'headers':[(b'content-type', b'image/png'),
(b'content-length', str(len(content)).encode())]}, receive)
def error(code):
return S3Error(None, code, 'injected', 'test-resource', 'test-request', 'test-host')
class FailPut:
def put_object(self, *args, **kwargs):
raise OSError('injected S3 failure')
class FailCommit(AsyncSession):
async def commit(self):
raise OSError('injected commit failure')
async def snapshot(factory):
async with factory() as s:
attachments = [(str(a.id), a.object_key, str(a.storage_missing_since))
for a in (await s.scalars(select(Attachment).order_by(Attachment.id)))]
intents = [(a.object_key,a.state) for a in (await s.scalars(select(UploadIntent).order_by(UploadIntent.object_key)))]
findings = [(f.object_key,f.kind,str(f.first_seen),str(f.last_seen),str(f.resolved_at),f.fingerprint)
for f in (await s.scalars(select(StorageFinding).order_by(StorageFinding.object_key,StorageFinding.kind)))]
objects = sorted((o.object_name,o.etag) for o in root.list_objects(bucket,recursive=True))
return attachments,intents,findings,objects
async def main():
global schema, bucket
admin_engine = create_async_engine(settings.sqlalchemy_database_url)
async with admin_engine.begin() as conn:
await conn.execute(text(f'CREATE SCHEMA {schema}'))
engine = create_async_engine(settings.sqlalchemy_database_url,
connect_args={'server_settings': {'search_path':schema}})
factory = async_sessionmaker(engine, expire_on_commit=False)
# Verify the deployed application's credentials against its actual bucket.
api_client=make_storage(role='api')
cleaner_client=make_storage(role='reconciler')
probe='originals/permission-probe/'+uuid4().hex
outside='outside/'+uuid4().hex
root.put_object(original_bucket,probe,io.BytesIO(content),len(content))
root.put_object(original_bucket,outside,io.BytesIO(content),len(content))
try:
response=api_client.get_object(original_bucket,probe)
assert response.read()==content
response.close();response.release_conn()
for operation in [
lambda: api_client.remove_object(original_bucket,probe),
lambda: api_client.get_object(original_bucket,outside),
lambda: cleaner_client.put_object(original_bucket,probe,io.BytesIO(content),len(content)),
lambda: cleaner_client.remove_object(original_bucket,outside),
]:
try: operation()
except S3Error as exc: assert exc.code=='AccessDenied'
else: raise AssertionError('role/prefix policy allowed a forbidden operation')
endpoint=urlsplit(settings.s3_endpoint)
anonymous=Minio(endpoint.netloc,secure=endpoint.scheme=='https',region=settings.s3_region)
try: anonymous.get_object(original_bucket,probe)
except S3Error as exc: assert exc.code=='AccessDenied' and exc.response.status==403
else: raise AssertionError('private bucket allowed an anonymous read')
passed('direct unsigned GET of an existing S3 object returns 403')
cleaner_client.remove_object(original_bucket,probe)
passed('deployed API cannot delete; reconciler cannot upload; both are confined to originals/')
finally:
root.remove_object(original_bucket,probe)
root.remove_object(original_bucket,outside)
root.make_bucket(bucket)
settings.s3_bucket = bucket
try:
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
now = datetime.now(timezone.utc)
async with factory.begin() as s:
s.add(User(id=user_id,username='test_user',display_name='Test',password_hash='test',is_admin=False,created_at=now))
await s.flush()
s.add(Chat(id=chat_id,title='Test',created_at=now))
await s.flush()
s.add(ChatMember(chat_id=chat_id,user_id=user_id))
with patch('src.service.attachment.get_storage', return_value=root):
async with factory.begin() as s:
good = await AttachmentService(s).upload(fake_request(),chat_id,user_id,'photo.png')
async with factory.begin() as s:
intent = await s.get(UploadIntent,good.object_key)
assert intent.state=='committed'
s.add(Message(id=uuid4(),chat_id=chat_id,sender_id=user_id,text='',created_at=now,
attachments=[MessageAttachment(attachment_id=good.id,chat_id=chat_id,position=0)]))
passed('real upload atomically commits Attachment and intent; image is referenced by a message')
async with factory.begin() as s:
confirmed_message=await create_idempotent_message(s,message_id=uuid4(),chat_id=chat_id,
sender_id=user_id,text='confirmed',created_at=now,idempotency_key='confirmed-image',
attachment_ids=[good.id])
# Failure on S3 side: durable intent remains, no ready attachment is published.
with patch('src.service.attachment.get_storage', return_value=FailPut()):
try:
async with factory.begin() as s:
await AttachmentService(s).upload(fake_request(),chat_id,user_id,'fail.png')
except ApiError as exc:
assert exc.status_code==503
else:
raise AssertionError('S3 failure accepted')
async with factory() as s:
failed_put_key = await s.scalar(select(UploadIntent.object_key).where(UploadIntent.state=='pending'))
assert failed_put_key
passed('S3 PUT failure leaves an accounted pending attempt, no ready attachment')
# Failure on DB side AFTER the actual S3 PUT; registration uses independent session.
broken_factory = async_sessionmaker(engine, class_=FailCommit, expire_on_commit=False)
with patch('src.service.attachment.get_storage', return_value=root):
try:
async with broken_factory.begin() as s:
await AttachmentService(s).upload(fake_request(),chat_id,user_id,'db-fail.png')
except OSError:
pass
else:
raise AssertionError('DB failure accepted')
async with factory() as s:
pending = list(await s.scalars(select(UploadIntent.object_key).where(UploadIntent.state=='pending')))
db_fail_key = next(k for k in pending if k!=failed_put_key)
assert root.stat_object(bucket,db_fail_key).size==len(content)
passed('DB commit failure after S3 PUT leaves a durable discoverable orphan')
# Kill the actual upload implementation after PUT, before its final COMMIT.
import base64
child = """
import asyncio,base64,json,os,sys
from unittest.mock import patch
from starlette.requests import Request
from sqlalchemy.ext.asyncio import AsyncSession,async_sessionmaker,create_async_engine
from src.core.settings import settings
from src.core.storage import make_storage
from src.service.attachment import AttachmentService
from uuid import UUID
p=json.load(sys.stdin)
settings.s3_bucket=p['bucket']
class CrashSession(AsyncSession):
async def commit(self): os._exit(23)
async def main():
engine=create_async_engine(settings.sqlalchemy_database_url,connect_args={'server_settings':{'search_path':p['schema']}})
factory=async_sessionmaker(engine,class_=CrashSession,expire_on_commit=False)
data=base64.b64decode(p['content'])
async def receive(): return {'type':'http.request','body':data,'more_body':False}
request=Request({'type':'http','method':'POST','path':'/','headers':[(b'content-type',b'image/png')]},receive)
with patch('src.service.attachment.get_storage',return_value=make_storage(role='bootstrap')):
async with factory.begin() as s:
await AttachmentService(s).upload(request,UUID(p['chat']),UUID(p['user']),'crash.png')
asyncio.run(main())
"""
crash = subprocess.run([sys.executable,'-c',child],input=json.dumps({
'bucket':bucket,'schema':schema,'chat':str(chat_id),'user':str(user_id),
'content':base64.b64encode(content).decode()}),text=True,capture_output=True)
assert crash.returncode==23, 'upload child did not reach the injected crash'
async with factory() as s:
crashed_keys=set(await s.scalars(select(UploadIntent.object_key).where(UploadIntent.state=='pending')))-set(pending)
assert len(crashed_keys)==1
assert root.stat_object(bucket,crashed_keys.pop()).size==len(content)
passed('process death after real PUT preserves intent; PostgreSQL releases the upload lock')
# Existing journal plus old object, with a live writer's lock.
active_key='originals/active-'+uuid4().hex
late_key='originals/late-'+uuid4().hex
orphan_key='originals/orphan-'+uuid4().hex
for key in (active_key,late_key,orphan_key):
root.put_object(bucket,key,io.BytesIO(content),len(content))
async with factory.begin() as s:
for key in (active_key,late_key):
s.add(UploadIntent(object_key=key,state='pending',created_at=now-timedelta(hours=1)))
root.put_object(bucket,'outside/keep',io.BytesIO(b'keep'),4)
await asyncio.sleep(1.2)
before=await snapshot(factory)
report=await Reconciler(factory,root,grace_seconds=1).run()
assert before==await snapshot(factory)
assert report['counts']['would_delete_orphan']>=3
passed('dry-run changes neither DB records nor S3 objects')
# Publish one candidate after dry-run, before apply. Cleaner must recheck.
async with factory.begin() as s:
s.add(Attachment(id=uuid4(),chat_id=chat_id,owner_id=user_id,filename='late.png',
object_key=late_key,content_type='image/png',size_bytes=len(content),
sha256=hashlib.sha256(content).hexdigest(),created_at=now,updated_at=now))
(await s.get(UploadIntent,late_key)).state='committed'
fresh_key='originals/fresh-'+uuid4().hex
root.put_object(bucket,fresh_key,io.BytesIO(content),len(content))
async with factory.begin() as s:
s.add(UploadIntent(object_key=fresh_key,state='pending',created_at=datetime.now(timezone.utc)))
async with factory.begin() as held:
await lock_object(held,active_key,wait=True)
report=await Reconciler(factory,root,apply=True,grace_seconds=1).run()
assert report['counts']['protected_active']>=1
assert report['counts']['protected_fresh']>=1
assert report['counts']['deleted_orphan']>=2
assert root.stat_object(bucket,active_key)
assert root.stat_object(bucket,good.object_key)
assert root.stat_object(bucket,late_key)
assert root.stat_object(bucket,fresh_key)
assert root.stat_object(bucket,'outside/keep')
async with factory() as s:
assert (await s.get(UploadIntent,failed_put_key)).state=='abandoned'
passed('apply cleans old orphan/empty attempt; preserves active, fresh, referenced and late-committed objects')
async with factory.begin() as s:
s.add(Attachment(id=uuid4(),chat_id=chat_id,owner_id=user_id,filename='fresh.png',
object_key=fresh_key,content_type='image/png',size_bytes=len(content),
sha256=hashlib.sha256(content).hexdigest(),created_at=now,updated_at=now))
(await s.get(UploadIntent,fresh_key)).state='committed'
# Missing metadata counterpart: do not delete messages or fabricate bytes.
root.remove_object(bucket,good.object_key)
before=await snapshot(factory)
report=await Reconciler(factory,root,grace_seconds=1).run()
assert report['counts']['would_mark_missing']==1
assert before==await snapshot(factory)
await Reconciler(factory,root,apply=True,grace_seconds=1).run()
async with factory() as s:
assert (await s.get(Attachment,good.id)).storage_missing_since is not None
assert await s.scalar(select(func.count()).select_from(MessageAttachment))==2
try:
await AttachmentService(s).get(good.id,user_id)
except ApiError as exc:
assert exc.status_code==503
else:
raise AssertionError('missing original reported ready')
passed('missing original is quarantined with 503; metadata and message link survive')
async with factory.begin() as s:
replay=await create_idempotent_message(s,message_id=uuid4(),chat_id=chat_id,
sender_id=user_id,text='confirmed',created_at=now,idempotency_key='confirmed-image',
attachment_ids=[good.id])
assert replay==confirmed_message
try:
async with factory.begin() as s:
await create_idempotent_message(s,message_id=uuid4(),chat_id=chat_id,
sender_id=user_id,text='new',created_at=now,idempotency_key='new-missing-image',
attachment_ids=[good.id])
except ApiError as exc: assert exc.status_code==409
else: raise AssertionError('new message accepted a missing attachment')
passed('missing content blocks new links, but confirmed idempotency replays stay intact')
root.put_object(bucket,good.object_key,io.BytesIO(b'x'*len(content)),len(content))
report=await Reconciler(factory,root,apply=True,grace_seconds=1).run()
assert report['counts']['restore_checksum_mismatch']==1
root.put_object(bucket,good.object_key,io.BytesIO(content),len(content))
report=await Reconciler(factory,root,apply=True,grace_seconds=1).run()
assert report['counts']['recovered']==1
async with factory() as s:
assert (await AttachmentService(s).get(good.id,user_id)).storage_missing_since is None
report=await Reconciler(factory,root,apply=True,grace_seconds=1).run()
assert report['counts'].get('recovered',0)==0
assert report['counts'].get('deleted_orphan',0)==0
passed('recovery requires exact SHA-256; repeat apply is safe')
# Authorization failure is not absence. No metadata quarantine or deletions.
class DeniedHead:
def list_objects(self,*args,**kwargs): return iter(())
def stat_object(self,*args,**kwargs): raise error('AccessDenied')
before=await snapshot(factory)
try:
await Reconciler(factory,DeniedHead(),apply=True,grace_seconds=1).run()
except S3Error as exc:
assert exc.code=='AccessDenied'
else: raise AssertionError('AccessDenied was interpreted as missing')
assert before==await snapshot(factory)
with patch('src.maintenance.reconcile.lock_object',side_effect=OSError('DB unavailable')):
try:
await Reconciler(factory,root,apply=True,grace_seconds=1).run()
except OSError: pass
else: raise AssertionError('DB unavailable did not stop repair')
assert before==await snapshot(factory)
passed('S3 AccessDenied and unavailable DB abort safely, without false missing/deletion')
# Lost DELETE response: S3 changed, DB transaction rolls back. Next scan resolves it.
lost_key='originals/delete-loss/'+uuid4().hex
root.put_object(bucket,lost_key,io.BytesIO(content),len(content))
async with factory.begin() as s:
s.add(UploadIntent(object_key=lost_key,state='pending',created_at=now-timedelta(hours=1)))
await asyncio.sleep(1.2)
class LostDelete:
def __getattr__(self,name): return getattr(root,name)
def remove_object(self,*args,**kwargs):
root.remove_object(*args,**kwargs)
raise OSError('injected lost DELETE response')
try:
await Reconciler(factory,LostDelete(),apply=True,grace_seconds=1,prefix='originals/delete-loss/').run()
except OSError: pass
else: raise AssertionError('lost DELETE response was ignored')
await Reconciler(factory,root,apply=True,grace_seconds=1,prefix='originals/delete-loss/').run()
async with factory() as s:
assert (await s.get(UploadIntent,lost_key)).state=='abandoned'
passed('lost DELETE response is repaired on repeat; no live object is affected')
# Failure BEFORE deletion must retain the ability to retry after recovery.
retry_key='originals/delete-retry/'+uuid4().hex
root.put_object(bucket,retry_key,io.BytesIO(content),len(content))
async with factory.begin() as s:
s.add(UploadIntent(object_key=retry_key,state='pending',created_at=now-timedelta(hours=1)))
await asyncio.sleep(1.2)
class FailedDelete:
def __getattr__(self,name): return getattr(root,name)
def remove_object(self,*args,**kwargs): raise OSError('injected S3 delete outage')
try:
await Reconciler(factory,FailedDelete(),apply=True,grace_seconds=1,prefix='originals/delete-retry/').run()
except OSError: pass
else: raise AssertionError('failed delete was reported successful')
assert root.stat_object(bucket,retry_key)
async with factory() as s:
assert (await s.get(UploadIntent,retry_key)).state=='pending'
await Reconciler(factory,root,apply=True,grace_seconds=1,prefix='originals/delete-retry/').run()
async with factory() as s:
assert (await s.get(UploadIntent,retry_key)).state=='abandoned'
try: root.stat_object(bucket,retry_key)
except S3Error as exc: assert exc.code=='NoSuchKey'
else: raise AssertionError('delete was not retried after S3 recovery')
passed('failed S3 DELETE preserves pending; recovery retries and actually removes the orphan')
# Cost experiment: bounded listing must cross the 1000-object page boundary.
metrics=StorageMetrics()
measured=make_storage(role='bootstrap',metrics=metrics)
statement_count=0
@event.listens_for(engine.sync_engine,'before_cursor_execute')
def count(*_):
nonlocal statement_count
statement_count+=1
results=[]
for n in (10,100,1005):
prefix=f'originals/benchmark/{n}/'
def put(i):
root.put_object(bucket,prefix+f'{i:06}',io.BytesIO(content),len(content))
with ThreadPoolExecutor(max_workers=16) as pool:
list(pool.map(put,range(n)))
for repeat in range(3):
metrics.requests.clear(); statement_count=0; started=time.perf_counter()
report=await Reconciler(factory,measured,prefix=prefix,grace_seconds=300,sample_limit=0).run()
elapsed=time.perf_counter()-started
assert report['counts']['objects_scanned']==n
assert metrics.requests['LIST']==(2 if n>1000 else 1)
results.append({'objects':n,'repeat':repeat+1,'seconds':round(elapsed,4),
's3_requests':dict(metrics.requests),'db_statements':statement_count})
print(json.dumps({'experiment':'scan_cost','runs':results}),flush=True)
passed('1005 objects cross ListObjectsV2 pagination; cost measured at 3 sizes x 3 runs')
print(json.dumps({'result':'PASS','checks':checks,'isolated_resources':True}),flush=True)
finally:
# These resources belong exclusively to this test run.
for obj in root.list_objects(bucket,recursive=True):
root.remove_object(bucket,obj.object_name)
root.remove_bucket(bucket)
settings.s3_bucket=original_bucket
await engine.dispose()
async with admin_engine.begin() as conn:
await conn.execute(text(f'DROP SCHEMA {schema} CASCADE'))
await admin_engine.dispose()
asyncio.run(main())
+163
View File
@@ -0,0 +1,163 @@
"""Проверки замечаний преподавателя: HTTP, повреждённый Redis и точный курсор."""
import base64
import json
import subprocess
import uuid
from pathlib import Path
from urllib.parse import quote
from client import Client, arguments, check, run
ROOT = Path(__file__).resolve().parents[1]
def database(code):
container = subprocess.check_output(['docker', 'compose', 'ps', '-q', 'api'], cwd=ROOT, text=True).split()[0]
result = subprocess.run(['docker', 'exec', '-i', container, 'python'],
input=code, cwd=ROOT, text=True, capture_output=True, timeout=30)
check(result.returncode == 0, 'Ошибка подготовки/удаления ORM-фикстуры')
return json.loads(result.stdout) if result.stdout.strip() else None
def main():
args = arguments('Регрессии по замечаниям преподавателя')
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/messages'
registration = {**alice['registration'], 'username': 'blank_' + uuid.uuid4().hex[:16]}
for field, value in [('password', ' ' * 12), ('password', '\t\n' * 6),
('display_name', ' \t\n')]:
c.call('POST', '/api/v1/users', 400, body={**registration, field: value})
for body in ({'title': ' \t\n', 'member_ids': [alice['id']]},
{'title': 'Duplicate', 'member_ids': [alice['id'], alice['id']]},
{'title': 'Unknown', 'member_ids': [alice['id'], str(uuid.uuid4())]},
{'title': 'Missing actor', 'member_ids': [bob['id']]}):
c.call('POST', '/api/v1/chats', 400, actor=alice, body=body)
basic = {'Authorization': 'Basic ' + base64.b64encode((
alice['registration']['username'] + ':' + alice['registration']['password']
).encode()).decode()}
for raw in (b'{"extra":"forbidden"}', b'{}', b'null', b' ', b'{'):
c.call('POST', '/api/v1/auth/sessions', 400, headers=basic, raw=raw)
c.call('POST', '/api/v1/auth/sessions', 401)
for credential in ('Basic !!!', 'Basic /w==', 'Bearer something',
'Basic ' + base64.b64encode(b'no-colon').decode()):
c.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': credential})
for payload in ('null', '[]', '{}', '123', '{broken'):
cursor = base64.urlsafe_b64encode(payload.encode()).decode().rstrip('=')
c.call('GET', path + '?cursor=' + quote(cursor, safe=''), 400, actor=alice)
for endpoint, body, actor in (
('/api/v1/users', registration, None),
('/api/v1/chats', {'title': 'MIME', 'member_ids': [alice['id']]}, alice),
(path, {'text': 'valid'}, alice),
):
response = c.call('POST', endpoint, 415, actor=actor, body=body,
headers={'Content-Type': 'text/plain'})
check(response['json']['error']['code'] == 'unsupported_media_type', 'Неверный код MIME-ошибки')
for raw in (b'{"text":"valid"}}', b'{"text":"valid"}]', b'{"text":"valid"}{}'):
c.call('POST', path, 400, actor=alice, raw=raw, headers={'Content-Type': 'application/json'})
for body in ({'text': ' ' * 12}, {'text': ' ' + 'a' * 4000}, {'text': 123},
{'text': 'valid', 'sender_id': eve['id']}):
c.call('POST', path, 400, actor=alice, body=body)
check(c.call('GET', path, actor=alice, schema='MessagePage')['json']['items'] == [],
'Отклонённый запрос сохранил сообщение')
print('PASS: обязательные пробельные строки, malformed JSON, типы/лишние поля, MIME 415, Basic без тела')
# Создаём только синтетические строки через SQLAlchemy; пароли/токены в команды не передаются.
fixture = database(f'''
import asyncio, json
from datetime import datetime, timezone
from uuid import UUID, uuid4
from sqlalchemy.ext.asyncio import async_sessionmaker
from src.core.db.session import create_engine
from src.core.db.models import User, Message
async def main():
engine = create_engine()
ids = [uuid4() for _ in range(100)]
base = uuid4().int & ~65535
message_ids = [UUID(int=base+4), UUID(int=base+3), UUID(int=base+2), UUID(int=base+1)]
times = [100000, 200000, 200000, 200000]
async with async_sessionmaker(engine).begin() as session:
session.add_all([User(id=i, username='fixture_' + i.hex[:20], display_name='Fixture',
password_hash='not-for-login', created_at=datetime.now(timezone.utc)) for i in ids])
session.add_all([Message(id=i, chat_id=UUID('{chat['id']}'), sender_id=UUID('{alice['id']}'),
text='precision', created_at=datetime(2026, 1, 1, 0, 0, 0, us, timezone.utc))
for i, us in zip(message_ids, times)])
await engine.dispose()
print(json.dumps({{'users': [str(i) for i in ids], 'messages': [str(i) for i in message_ids]}}))
asyncio.run(main())
''')
try:
c.call('POST', '/api/v1/chats', 400, actor=alice,
body={'title': '101 участников', 'member_ids': [alice['id'], *fixture['users']]})
hundred = c.call('POST', '/api/v1/chats', 201, actor=alice, schema='Chat',
body={'title': '100 участников',
'member_ids': [alice['id'], *fixture['users'][:99]]})['json']
check(len(hundred['member_ids']) == 100, 'Допустимые 100 участников не сохранены')
received = []
cursor = None
while True:
url = path + '?limit=1' + ('&cursor=' + quote(cursor, safe='') if cursor else '')
page = c.call('GET', url, actor=bob, schema='MessagePage')['json']
received.extend(item['id'] for item in page['items'])
if cursor is None:
first_cursor = page['next_cursor']
cursor = page['next_cursor']
if cursor is None:
break
check(len(received) <= 4, 'Курсор зациклился')
expected = [fixture['messages'][0], *reversed(fixture['messages'][1:])]
check(received == expected, 'Потеря/повтор сообщений при микросекундах или равном времени')
other = c.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Other', 'member_ids': [alice['id']]}, schema='Chat')['json']
c.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + quote(first_cursor, safe=''),
400, actor=alice)
print('PASS: 101 существующий участник отклонён; курсор сохраняет микросекунды и UUID-порядок, привязан к чату')
finally:
database(f'''
import asyncio
from uuid import UUID
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import async_sessionmaker
from src.core.db.session import create_engine
from src.core.db.models import User, Chat, ChatMember, Message
async def main():
engine = create_engine()
ids = [UUID(i) for i in {fixture['users']!r}]
async with async_sessionmaker(engine).begin() as session:
chats = list(await session.scalars(select(ChatMember.chat_id).where(ChatMember.user_id.in_(ids))))
await session.execute(delete(Message).where(Message.chat_id.in_(chats)))
await session.execute(delete(Chat).where(Chat.id.in_(chats)))
await session.execute(delete(User).where(User.id.in_(ids)))
await engine.dispose()
asyncio.run(main())
''')
text = ' строка\nс пробелами '
saved = c.message(alice, chat, text)['json']
check(saved['text'] == text, 'Текст обрезан до сохранения')
history = c.call('GET', path, actor=alice, schema='MessagePage')['json']
check(next(m for m in history['items'] if m['id'] == saved['id'])['text'] == text,
'Сохранённый текст изменён')
print('PASS: исходный текст и его длина сохраняются без strip')
key = 'user:v1:' + alice['id']
def redis(*args):
return subprocess.check_output(['docker', 'compose', 'exec', '-T', 'redis',
'redis-cli', '--raw', *args], cwd=ROOT, text=True, timeout=10).strip()
for corrupted in ('null', '[]', '{}', '{broken', '"text"', '{"id":123}',
json.dumps({**alice['user'], 'id': bob['id']})):
redis('SET', key, corrupted)
me = c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['json']
check(me == alice['user'], 'Повреждённый профиль отвергает сессию или меняет пользователя')
check(json.loads(redis('GET', key))['id'] == alice['id'], 'Повреждённый кэш не заменён')
check(0 < int(redis('TTL', key)) <= 60, 'Восстановленному профилю не назначен TTL')
print('PASS: null/повреждённый/чужой профиль заменяется из PostgreSQL с TTL; сессия работает')
spec = json.loads(c.call('GET', '/openapi.json')['bytes'])
login = spec['paths']['/api/v1/auth/sessions']['post']
check('requestBody' not in login and {'basicAuth': []} in login['security'], 'Swagger вход не Basic')
check({'bearerAuth': []} in spec['paths']['/api/v1/users/me']['get']['security'], 'Swagger не передаёт Bearer')
print('PASS: Swagger объявляет Basic для входа и Bearer для защищённых методов')
if __name__ == '__main__':
run(main)
+47
View File
@@ -0,0 +1,47 @@
"""The HTTP budget must not await a stuck driver's cancellation/rollback."""
import asyncio
from unittest.mock import patch
from src.core.request_deadline import RequestDeadlineMiddleware
from src.core.settings import settings
async def main():
release=asyncio.Event()
calls=0
async def stuck(scope,receive,send):
nonlocal calls
calls+=1
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
await release.wait()
# A misbehaving cancelled handler must not emit a late success.
await send({'type':'http.response.start','status':201,'headers':[]})
await send({'type':'http.response.body','body':b'late'})
middleware=RequestDeadlineMiddleware(stuck)
scope={'type':'http','method':'GET','path':'/api/v1/chats','headers':[], 'query_string':b''}
async def receive():
return {'type':'http.request','body':b'','more_body':False}
responses=[]
async def send(message): responses.append(message)
with patch.object(settings,'http_request_timeout_seconds',.02),patch.object(settings,'http_concurrency_limit',2):
for _ in range(3):
await asyncio.wait_for(middleware(scope,receive,send),timeout=.15)
assert calls==2 and len(middleware.pending)==2
assert [m['status'] for m in responses if m['type']=='http.response.start']==[503]*3
release.set()
await asyncio.gather(*middleware.pending)
await asyncio.sleep(0)
assert not middleware.pending
assert [m['status'] for m in responses if m['type']=='http.response.start']==[503]*3
assert all(m.get('body')!=b'late' for m in responses)
async def healthy(scope,receive,send):
await send({'type':'http.response.start','status':200,'headers':[]})
await send({'type':'http.response.body','body':b'ok'})
middleware.app=healthy
responses.clear()
await middleware(scope,receive,send)
assert responses[-1]['body']==b'ok'
print('PASS: bounded response before slow rollback, retained capacity, no late response, recovery')
asyncio.run(main())
+53
View File
@@ -0,0 +1,53 @@
"""docker compose exec -T api python < tests/settings.py."""
import os
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import patch
from pydantic import ValidationError
from src.core.settings import AppSettings
# Только искусственные значения, соединения с этими адресами не открываются.
with patch.dict(os.environ, {
'DATABASE_URL': 'postgresql://settings_user:settings_test_password@postgres/settings_db',
'REDIS_URL': 'redis://redis:6379/0',
}, clear=True):
with TemporaryDirectory() as directory:
path = Path(directory) / '.env'
path.write_text('SESSION_TTL_SECONDS=2\nDATABASE_POOL_SIZE=3\n', encoding='utf-8')
config = AppSettings(_env_file=path)
assert config.session_ttl_seconds == 2
assert config.database_pool_size == 3
assert config.sqlalchemy_database_url.startswith('postgresql+asyncpg://')
assert 'settings_test_password' not in repr(config)
with patch.dict(os.environ, {'SESSION_TTL_SECONDS': '4'}):
assert AppSettings(_env_file=path).session_ttl_seconds == 4
print('PASS: AppSettings читает .env, окружение имеет приоритет, URL скрыт в repr')
for values, expected_field in (
({'session_ttl_seconds': 0}, 'session_ttl_seconds'),
({'session_ttl_seconds': 'invalid'}, 'session_ttl_seconds'),
({'database_pool_size': 0}, 'database_pool_size'),
({'redis_read_timeout_seconds': -1}, 'redis_read_timeout_seconds'),
({'session_token_bytes': 16}, 'session_token_bytes'),
):
try:
AppSettings(_env_file=None, **values)
except ValidationError as error:
assert any(item['loc'] == (expected_field,) for item in error.errors())
else:
raise AssertionError('Некорректная настройка принята: ' + expected_field)
print('PASS: Pydantic отклоняет неверные TTL, pool, timeout и недостаточную длину токена')
for values in ({'app_env': 'production'}, {'admin_password': ' ' * 12},
{'admin_display_name': 'bad\x00'}):
try:
AppSettings(_env_file=None, **values)
except ValidationError as error:
assert 'settings_test_password' not in str(error)
assert 'input_value' not in str(error)
else:
raise AssertionError('Небезопасная конфигурация принята')
assert AppSettings(_env_file=None, app_env='production',
admin_password='custom-production-password-123').app_env == 'production'
print('PASS: production отвергает учебный пароль; ошибки конфигурации скрывают входные значения')
+78
View File
@@ -0,0 +1,78 @@
"""10 simultaneous binary uploads; sample per-container cgroup memory including tmpfs."""
import hashlib
import io
import json
import os
import subprocess
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from PIL import Image
from client import Client, arguments, check, run
def main():
args=arguments('Upload concurrency and memory experiment')
c=Client(args.base_url,args.ca_file,timeout=30)
alice,bob,_,chat=c.bootstrap()
containers=subprocess.check_output(['docker','compose','ps','-q','api'],text=True).split()
check(len(containers)==2,'experiment requires two API replicas')
# One sampler process per container, not one docker invocation per sample.
sampler='''import json,time
from pathlib import Path
values=[]
for _ in range(750):
values.append(int(Path('/sys/fs/cgroup/memory.current').read_text()))
time.sleep(.02)
print(json.dumps({'baseline_bytes':values[0],'sampled_peak_bytes':max(values),'samples':len(values),'interval_seconds':.02,'cgroup_limit_bytes':int(Path('/sys/fs/cgroup/memory.max').read_text())}))
'''
samplers=[subprocess.Popen(['docker','exec',cid,'python','-c',sampler],stdout=subprocess.PIPE,text=True)
for cid in containers]
time.sleep(.2)
results=[]
try:
for side in (256,1280):
source=io.BytesIO()
Image.frombytes('RGB',(side,side),os.urandom(side*side*3)).save(source,format='PNG')
content=source.getvalue()
barrier=threading.Barrier(10)
def upload(_):
local=Client(args.base_url,args.ca_file,timeout=30)
barrier.wait()
started=time.perf_counter()
response=local.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=memory.png',
expected=[201,503],actor=alice,raw=content,headers={'Content-Type':'image/png'})
if response['status']==201:
att=json.loads(response['bytes'])
check(att['sha256']==hashlib.sha256(content).hexdigest(),'upload checksum differs')
downloaded=local.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
check(downloaded['bytes']==content,'concurrent upload lost original')
return {'status':response['status'],'seconds':round(time.perf_counter()-started,4)}
with ThreadPoolExecutor(max_workers=10) as pool:
responses=list(pool.map(upload,range(10)))
check(all(r['status']==201 for r in responses),'configured 10-upload profile rejected a request')
results.append({'dimensions':[side,side],'file_bytes':len(content),
'concurrent_uploads':10,'responses':responses})
memory=[]
for process in samplers:
output,_=process.communicate(timeout=25)
check(process.returncode==0,'memory sampler failed')
memory.append(json.loads(output))
check(all(m['sampled_peak_bytes']<m['cgroup_limit_bytes'] for m in memory),'memory reached container limit')
print(json.dumps({'result':'PASS','api_replicas':2,'uploads_per_replica_limit':5,
'decode_concurrency_per_replica':1,'temporary_storage_per_replica_bytes':64*1024*1024,
'profiles':results,'memory':memory,'limitations':[
'20ms sampling can miss short peaks',
'two synthetic image sizes do not cover the worst-case 40MP input',
'cgroup memory includes Python, image decoding and tmpfs; experiment is not a throughput benchmark',
'deployment restores the default admission limit of 1 after this experiment'
]}),flush=True)
finally:
for process in samplers:
if process.poll() is None:
process.terminate();process.wait()
if __name__=='__main__':
run(main)