Добавить вложения: S3, метаданные в PostgreSQL и очистка незавершённых загрузок

This commit is contained in:
2026-10-30 23:10:44 +03:00
parent ec540a1a72
commit d812d8f8d8
75 changed files with 5968 additions and 16 deletions
+102
View File
@@ -0,0 +1,102 @@
"""Маленький HTTP-клиент для собственных тестов. Только стандартная библиотека."""
import base64
import json
import os
import urllib.error
import urllib.parse
import urllib.request
import uuid
BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/")
def call(method, path, body=None, user=None, raw=None, headers=None, base_url=None, timeout=15):
"""Возвращает (код ответа, разобранный JSON или None, заголовки)."""
request_headers = {"Accept": "application/json"}
if body is not None:
raw = json.dumps(body, ensure_ascii=False).encode("utf-8")
if raw is not None:
request_headers["Content-Type"] = "application/json"
if user is not None:
request_headers["Authorization"] = "Bearer " + user["token"]
request_headers.update(headers or {})
request = urllib.request.Request((base_url or BASE_URL) + path, data=raw, headers=request_headers, method=method)
try:
response = urllib.request.urlopen(request, timeout=timeout)
except urllib.error.HTTPError as error:
response = error
with response:
data = response.read()
return response.status, json.loads(data) if data else None, response.headers
def basic(username, password):
return {"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode("utf-8")).decode()}
def login(username, password, base_url=None):
return call("POST", "/api/v1/auth/sessions", headers=basic(username, password), base_url=base_url)
def user_body(**fields):
"""Корректное тело регистрации со случайным username; поля можно переопределить."""
body = {"username": "u_" + uuid.uuid4().hex[:12], "display_name": "Тест", "password": "Pw-" + uuid.uuid4().hex}
body.update(fields)
return body
def new_user(prefix="t", base_url=None):
"""Регистрирует синтетического пользователя и сразу входит. В словаре id, username, password и token."""
body = user_body(username=prefix + "_" + uuid.uuid4().hex[:12], display_name="Тест " + prefix)
status, user, _ = call("POST", "/api/v1/users", body, base_url=base_url)
assert status == 201, status
status, session, _ = login(body["username"], body["password"], base_url=base_url)
assert status == 201, status
return {**user, "password": body["password"], "token": session["token"], "user": user}
def new_chat(owner, *others, title="Тестовый чат"):
member_ids = [owner["id"]] + [other["id"] for other in others]
status, chat, _ = call("POST", "/api/v1/chats", {"title": title, "member_ids": member_ids}, user=owner)
assert status == 201, status
return chat
def new_key():
return uuid.uuid4().hex
def send(user, chat, text, key=None, attachment_ids=None, **options):
headers = {"Idempotency-Key": key or new_key()}
body = {"text": text}
if attachment_ids is not None:
body["attachment_ids"] = attachment_ids
return call("POST", "/api/v1/chats/" + chat["id"] + "/messages", body, user=user, headers=headers, **options)
def request_bytes(method, path, user=None, base_url=None, timeout=15):
"""Как call, но ответ не разбирается как JSON: для скачивания файлов."""
headers = {"Authorization": "Bearer " + user["token"]} if user else {}
request = urllib.request.Request((base_url or BASE_URL) + path, headers=headers, method=method)
try:
response = urllib.request.urlopen(request, timeout=timeout)
except urllib.error.HTTPError as error:
response = error
with response:
return response.status, response.read(), response.headers
def history(user, chat, limit):
"""Обходит всю историю чата страницами по limit, возвращает сообщения и размеры страниц."""
items, sizes, cursor = [], [], None
while True:
path = "/api/v1/chats/%s/messages?limit=%d" % (chat["id"], limit)
if cursor:
path += "&cursor=" + urllib.parse.quote(cursor, safe="")
status, page, _ = call("GET", path, user=user)
assert status == 200, status
items += page["items"]
sizes.append(len(page["items"]))
cursor = page["next_cursor"]
if cursor is None:
return items, sizes
+49
View File
@@ -0,0 +1,49 @@
"""Управление стендом для тестов отказов: останавливает и запускает сервисы через docker compose."""
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "student-tests"))
from api import BASE_URL, call # noqa: E402
def compose(*args):
command = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", "compose.test.yaml", *args]
result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True)
if result.returncode:
raise AssertionError("compose %s: %s" % (" ".join(args[:4]), result.stderr[-600:]))
return result.stdout
def sql(query):
"""Выполняет запрос через psql внутри контейнера postgres и возвращает вывод без оформления."""
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", query).strip()
def wait_ready(base_url=BASE_URL, seconds=60):
"""Ждёт устойчивой готовности: балансировщик мог ещё не отметить отказ, поэтому одного ответа 200 мало."""
deadline = time.monotonic() + seconds
stable_since = None
while time.monotonic() < deadline:
try:
ready = call("GET", "/health/ready", base_url=base_url)[0] == 200
except OSError:
ready = False
if not ready:
stable_since = None
elif stable_since is None:
stable_since = time.monotonic()
elif time.monotonic() - stable_since >= 3:
return
time.sleep(0.3)
raise AssertionError("стенд не стал готов за %d секунд" % seconds)
def timed(*args, **kwargs):
"""Как call, но первым элементом возвращает время ответа в секундах."""
started = time.monotonic()
result = call(*args, **kwargs)
return (time.monotonic() - started, *result)
@@ -0,0 +1,91 @@
"""Недоступность Redis и PostgreSQL через балансировщик.
Пока реплики не признаны неготовыми, 503 отвечает API, потом сам балансировщик. Формат ошибки у них один.
"""
import unittest
from stand import call, compose, timed, wait_ready
from api import basic, history, new_chat, new_key, new_user, send, user_body
# таймауты в .env.example: 2 с на Redis, 3 с на ожидание соединения с БД; берём запас
MAX_SECONDS = 6
class DependencyDownTest(unittest.TestCase):
def setUp(self):
wait_ready()
self.user = new_user("dep")
self.chat = new_chat(self.user)
send(self.user, self.chat, "сообщение до отказа")
self.messages = "/api/v1/chats/" + self.chat["id"] + "/messages"
def assert_unavailable(self, result):
elapsed, status, body, headers = result
self.assertEqual(status, 503)
self.assertEqual(body["error"]["code"], "unavailable")
self.assertEqual(body["error"]["request_id"], headers["X-Request-Id"])
self.assertLess(elapsed, MAX_SECONDS)
def test_redis_down(self):
compose("stop", "redis")
self.addCleanup(wait_ready)
self.addCleanup(compose, "start", "redis")
# сессию проверить нечем: запрос с настоящим токеном не выполняется, но и чужим не становится
self.assert_unavailable(timed("GET", "/api/v1/users/me", user=self.user))
self.assert_unavailable(timed("GET", self.messages, user=self.user))
key = {"Idempotency-Key": new_key()}
self.assert_unavailable(timed("POST", self.messages, {"text": "во время отказа"}, user=self.user, headers=key))
# поддельный токен тоже не принимается за действующий
self.assert_unavailable(timed("GET", "/api/v1/users/me", user={"token": "x" * 43}))
# без токена Redis не нужен: API ответит 401, балансировщик без готовых реплик 503, но не данные
self.assertIn(call("GET", "/api/v1/users/me")[0], (401, 503))
self.assertIn(call("GET", self.messages)[0], (401, 503))
# новую сессию создать негде
self.assert_unavailable(timed("POST", "/api/v1/auth/sessions",
headers=basic(self.user["username"], self.user["password"])))
self.assert_unavailable(timed("GET", "/health/ready"))
compose("start", "redis")
wait_ready()
# прежний токен снова работает: сессия пережила остановку Redis, а сообщение во время отказа не записалось
self.assertEqual(call("GET", "/api/v1/users/me", user=self.user)[1], self.user["user"])
self.assertEqual([item["text"] for item in history(self.user, self.chat, 50)[0]], ["сообщение до отказа"])
def test_uncertain_write_can_be_repeated_with_same_key(self):
compose("stop", "postgres")
self.addCleanup(wait_ready)
self.addCleanup(compose, "start", "postgres")
key = new_key()
self.assertEqual(send(self.user, self.chat, "важное", key=key)[0], 503)
compose("start", "postgres")
wait_ready()
first = send(self.user, self.chat, "важное", key=key)
second = send(self.user, self.chat, "важное", key=key)
self.assertEqual((first[0], second[0]), (201, 201))
self.assertEqual(first[1], second[1])
def test_postgres_down(self):
compose("stop", "postgres")
self.addCleanup(wait_ready)
self.addCleanup(compose, "start", "postgres")
self.assert_unavailable(timed("GET", "/api/v1/users/me", user=self.user))
self.assert_unavailable(timed("GET", self.messages, user=self.user))
key = {"Idempotency-Key": new_key()}
self.assert_unavailable(timed("POST", self.messages, {"text": "во время отказа"}, user=self.user, headers=key))
self.assert_unavailable(timed("POST", "/api/v1/users", user_body()))
self.assert_unavailable(timed("POST", "/api/v1/auth/sessions",
headers=basic(self.user["username"], self.user["password"])))
self.assertIn(call("GET", "/api/v1/users/me")[0], (401, 503))
self.assert_unavailable(timed("GET", "/health/ready"))
compose("start", "postgres")
wait_ready()
self.assertEqual(call("GET", "/api/v1/users/me", user=self.user)[1], self.user["user"])
self.assertEqual([item["text"] for item in history(self.user, self.chat, 50)[0]], ["сообщение до отказа"])
self.assertEqual(send(self.user, self.chat, "после восстановления")[0], 201)
if __name__ == "__main__":
unittest.main()
+75
View File
@@ -0,0 +1,75 @@
"""Истечение сессии и ключа. Проверяется на отдельном API с TTL 2 секунды (порт 8081, только compose.test.yaml)."""
import datetime
import time
import unittest
from stand import call, compose, wait_ready
from api import login, new_key, new_user, send
SHORT_TTL_URL = "http://localhost:8081"
class SessionExpiryTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
compose("up", "-d", "--no-deps", "--no-build", "api-short-ttl")
cls.addClassCleanup(compose, "rm", "-sf", "api-short-ttl")
wait_ready(SHORT_TTL_URL)
def test_session_expires(self):
user = new_user("ttl", base_url=SHORT_TTL_URL)
self.assertEqual(call("GET", "/api/v1/users/me", user=user, base_url=SHORT_TTL_URL)[0], 200)
time.sleep(3)
status, body, headers = call("GET", "/api/v1/users/me", user=user, base_url=SHORT_TTL_URL)
self.assertEqual(status, 401)
self.assertEqual(body["error"]["code"], "unauthorized")
self.assertTrue(headers["WWW-Authenticate"].startswith("Bearer "))
# logout с истёкшим токеном тоже 401
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", user=user, base_url=SHORT_TTL_URL)[0], 401)
def test_idempotency_key_is_free_after_ttl(self):
user = new_user("idem", base_url=SHORT_TTL_URL)
chat = new_chat_short(user)
key = new_key()
first = send(user, chat, "первое", key=key, base_url=SHORT_TTL_URL)
repeat = send(user, chat, "первое", key=key, base_url=SHORT_TTL_URL)
self.assertEqual((first[0], repeat[1]), (201, first[1]))
time.sleep(2.2)
# сессия живёт столько же, поэтому нужен новый вход
_, session, _ = login(user["username"], user["password"], base_url=SHORT_TTL_URL)
again = send({**user, "token": session["token"]}, chat, "уже другое", key=key, base_url=SHORT_TTL_URL)
self.assertEqual(again[0], 201)
self.assertNotEqual(again[1]["id"], first[1]["id"])
def test_expires_at_matches_ttl(self):
user = new_user("ttl", base_url=SHORT_TTL_URL)
_, session, _ = login(user["username"], user["password"], base_url=SHORT_TTL_URL)
expires_at = datetime.datetime.fromisoformat(session["expires_at"][:-1] + "+00:00")
left = (expires_at - datetime.datetime.now(datetime.timezone.utc)).total_seconds()
self.assertTrue(0 < left <= 2.5, left)
def test_requests_do_not_extend_session(self):
user = new_user("ttl", base_url=SHORT_TTL_URL)
deadline = time.monotonic() + 3
while time.monotonic() < deadline:
call("GET", "/api/v1/users/me", user=user, base_url=SHORT_TTL_URL)
time.sleep(0.2)
self.assertEqual(call("GET", "/api/v1/users/me", user=user, base_url=SHORT_TTL_URL)[0], 401)
def test_login_after_expiry(self):
user = new_user("ttl", base_url=SHORT_TTL_URL)
time.sleep(3)
status, session, _ = login(user["username"], user["password"], base_url=SHORT_TTL_URL)
self.assertEqual(status, 201)
self.assertEqual(call("GET", "/api/v1/users/me", user=session, base_url=SHORT_TTL_URL)[0], 200)
def new_chat_short(owner):
body = {"title": "Чат", "member_ids": [owner["id"]]}
status, chat, _ = call("POST", "/api/v1/chats", body, user=owner, base_url=SHORT_TTL_URL)
assert status == 201, status
return chat
if __name__ == "__main__":
unittest.main()
+263
View File
@@ -0,0 +1,263 @@
"""Согласованность S3 и PostgreSQL, отказ S3, приватность bucket. Состояние после сбоя создаётся настоящими сбоями."""
import hashlib
import socket
import subprocess
import threading
import time
import unittest
import urllib.parse
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from stand import BASE_URL, call, compose, sql, timed, wait_ready
from api import new_chat, new_user, request_bytes, send
from samples import noisy_png
FIXTURE = (Path(__file__).resolve().parents[2] / "tests/fixtures/sample.png").read_bytes()
LIST_OBJECTS = """
import config
from objects import ObjectStore
store = ObjectStore(config.read_secret("S3_JANITOR_ACCESS_KEY_ID"), config.read_secret("S3_JANITOR_SECRET_ACCESS_KEY"))
for page in store.pages():
for item in page:
print(item["Key"])
"""
PROBE_API_KEY = """
import urllib.error, urllib.request
import config
from botocore.exceptions import ClientError
from objects import ObjectStore
store = ObjectStore(config.read_secret("S3_ACCESS_KEY_ID"), config.read_secret("S3_SECRET_ACCESS_KEY"))
def attempt(name, action):
try:
action()
print(name, "allowed")
except ClientError as error:
print(name, error.response["Error"]["Code"])
attempt("list", lambda: store.client.list_objects_v2(Bucket=store.bucket))
attempt("create_bucket", lambda: store.client.create_bucket(Bucket="other"))
attempt("delete_bucket", lambda: store.client.delete_bucket(Bucket=store.bucket))
try:
urllib.request.urlopen(config.S3_ENDPOINT + "/" + store.bucket + "/KEY", timeout=3)
print("anonymous allowed")
except urllib.error.HTTPError as error:
print("anonymous", error.code)
"""
def upload(user, chat, data=FIXTURE):
path = "/api/v1/chats/%s/attachments?filename=a.png" % chat["id"]
return call("POST", path, raw=data, user=user, headers={"Content-Type": "image/png"}, timeout=30)
def bucket_keys():
return set(compose("exec", "-T", "janitor", "python", "-c", LIST_OBJECTS).split())
def janitor(*args):
# логи идут в stdout контейнера; сервисный проход на проверки не влияет: его период 300 секунд
return compose("run", "--rm", "--no-deps", "janitor", "python", "janitor.py", "--once", *args)
def in_api_replica(*command):
# по id контейнера: после поочерёдной замены номера реплик уже не 1 и 2
container = compose("ps", "-q", "api").split()[0]
return subprocess.run(["docker", "exec", container, *command], check=True, capture_output=True, text=True).stdout
def wait_s3(seconds=90):
"""S3 готов, когда контейнер healthy: том данных зарегистрирован, запросы к объектам не зависают."""
deadline = time.monotonic() + seconds
while compose("ps", "--format", "{{.Health}}", "s3").strip() != "healthy":
if time.monotonic() > deadline:
raise AssertionError("S3 не стал готов за %d секунд" % seconds)
time.sleep(1)
def rows(condition="true"):
output = sql("SELECT object_key FROM attachments WHERE " + condition)
return set(output.split())
class StorageBase(unittest.TestCase):
def setUp(self):
wait_ready()
self.user = new_user("files")
self.chat = new_chat(self.user)
def tearDown(self):
# следы теста не должны влиять на сверку в следующем; S3 к этому моменту может ещё запускаться
compose("start", "s3")
wait_s3()
janitor("--grace-seconds", "0")
class ConsistencyTest(StorageBase):
def test_commit_lost_after_s3_put(self):
"""Состояние как после падения между PUT и подтверждением: объект есть, строка pending, клиент не получил 201."""
attachment = upload(self.user, self.chat)[1]
key = "originals/" + attachment["id"]
self.assertIn(key, bucket_keys())
sql("UPDATE attachments SET state = 'pending', created_at = now() - interval '2 hours' WHERE id = '%s'" % attachment["id"])
janitor()
self.assertNotIn(key, bucket_keys())
self.assertEqual(rows("id = '%s'" % attachment["id"]), set())
def test_grace_period_protects_fresh_upload(self):
attachment = upload(self.user, self.chat)[1]
key = "originals/" + attachment["id"]
sql("UPDATE attachments SET state = 'pending' WHERE id = '%s'" % attachment["id"])
janitor() # строка свежая, загрузка могла ещё идти
self.assertIn(key, bucket_keys())
self.assertEqual(rows("id = '%s'" % attachment["id"]), {key})
# строку не видно снаружи, пока она pending
self.assertEqual(call("GET", "/api/v1/attachments/" + attachment["id"], user=self.user)[0], 404)
def test_ready_attachments_are_never_removed(self):
attachment = upload(self.user, self.chat)[1]
janitor("--grace-seconds", "0")
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
self.assertEqual((status, body), (200, FIXTURE))
def test_orphan_object_is_removed_only_after_grace(self):
attachment = upload(self.user, self.chat)[1]
key = "originals/" + attachment["id"]
sql("DELETE FROM attachments WHERE id = '%s'" % attachment["id"])
janitor()
self.assertIn(key, bucket_keys())
janitor("--grace-seconds", "0", "--dry-run")
self.assertIn(key, bucket_keys())
janitor("--grace-seconds", "0")
self.assertNotIn(key, bucket_keys())
def test_second_pass_changes_nothing(self):
attachment = upload(self.user, self.chat)[1]
sql("UPDATE attachments SET state = 'pending', created_at = now() - interval '2 hours' WHERE id = '%s'" % attachment["id"])
janitor()
before = bucket_keys(), rows()
janitor()
self.assertEqual((bucket_keys(), rows()), before)
def test_interrupted_upload_leaves_nothing(self):
before = bucket_keys(), rows()
address = urllib.parse.urlsplit(BASE_URL)
path = "/api/v1/chats/%s/attachments?filename=a.png" % self.chat["id"]
head = ("POST %s HTTP/1.1\r\nHost: x\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n"
"Authorization: Bearer %s\r\n\r\n" % (path, self.user["token"])).encode()
for _ in range(4):
with socket.create_connection((address.hostname, address.port), timeout=5) as connection:
connection.sendall(head + b"\x89PNG" + b"x" * 200000)
time.sleep(0.2)
time.sleep(1)
self.assertEqual((bucket_keys(), rows()), before)
def test_postgres_lost_during_parallel_uploads(self):
"""Настоящий сбой: база пропадает, пока идут загрузки. Часть объектов попадает в S3 без подтверждения."""
data = noisy_png(3 * 1024 * 1024)
# подтверждённые до сбоя файлы: они обязаны уцелеть при любом исходе гонки ниже
confirmed_before = [upload(self.user, self.chat, data)[1] for _ in range(3)]
before_keys = bucket_keys()
barrier = threading.Barrier(13)
def post(_):
barrier.wait()
confirmed = []
for _ in range(6):
try:
status, attachment, _ = upload(self.user, self.chat, data)
except OSError:
continue
if status == 201:
confirmed.append(attachment)
return confirmed
with ThreadPoolExecutor(max_workers=12) as pool:
jobs = [pool.submit(post, number) for number in range(12)]
barrier.wait()
time.sleep(0.25)
compose("kill", "postgres")
confirmed = confirmed_before + [item for job in jobs for item in job.result()]
compose("start", "postgres")
self.addCleanup(wait_ready)
wait_ready()
leftovers = bucket_keys() - before_keys - {"originals/" + item["id"] for item in confirmed}
janitor("--grace-seconds", "0")
self.assertEqual(rows("state = 'pending'"), set())
self.assertEqual(bucket_keys(), rows("state = 'ready'"))
self.assertGreaterEqual(len(confirmed), 3)
for attachment in confirmed:
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
self.assertEqual((status, hashlib.sha256(body).hexdigest()), (200, attachment["sha256"]))
print("\nподтверждено %d, в S3 без подтверждения %d, после очистки расхождений нет" % (len(confirmed), len(leftovers)))
class S3DownTest(StorageBase):
def test_s3_down(self):
attachment = upload(self.user, self.chat)[1]
compose("stop", "s3")
self.addCleanup(wait_ready)
self.addCleanup(compose, "start", "s3")
path = "/api/v1/chats/%s/attachments?filename=a.png" % self.chat["id"]
elapsed, status, body, headers = timed("POST", path, raw=FIXTURE, user=self.user, headers={"Content-Type": "image/png"})
self.assertEqual((status, body["error"]["code"]), (503, "unavailable"))
self.assertEqual(body["error"]["request_id"], headers["X-Request-Id"])
self.assertLess(elapsed, 10)
self.assertEqual(request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)[0], 503)
# без S3 мессенджер продолжает работать: текст, метаданные и готовность реплик от него не зависят
self.assertEqual(send(self.user, self.chat, "текст без S3")[0], 201)
self.assertEqual(call("GET", "/api/v1/attachments/" + attachment["id"], user=self.user)[0], 200)
self.assertEqual(call("GET", "/health/ready")[0], 200)
compose("start", "s3")
wait_s3()
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
self.assertEqual((status, body), (200, FIXTURE))
def test_failed_upload_is_cleaned(self):
compose("stop", "s3")
self.addCleanup(wait_ready)
self.addCleanup(compose, "start", "s3")
self.assertEqual(upload(self.user, self.chat)[0], 503)
self.assertTrue(rows("state = 'pending'")) # строка осталась, клиент получил ошибку
compose("start", "s3")
wait_s3()
janitor("--grace-seconds", "0")
self.assertEqual(rows("state = 'pending'"), set())
class PrivateBucketTest(StorageBase):
def test_bucket_is_private_and_api_key_is_limited(self):
attachment = upload(self.user, self.chat)[1]
script = PROBE_API_KEY.replace("/KEY", "/originals/" + attachment["id"])
output = in_api_replica("python", "-c", script)
self.assertIn("list AccessDenied", output)
self.assertIn("create_bucket AccessDenied", output)
self.assertIn("delete_bucket AccessDenied", output)
self.assertIn("anonymous 403", output)
def test_bucket_bootstrap_can_be_repeated(self):
before = bucket_keys()
output = compose("run", "--rm", "--no-deps", "s3-init")
self.assertIn("bucket уже есть", output)
self.assertIn("пробная запись прошла", output)
self.assertEqual(bucket_keys(), before)
def test_no_file_on_api_replica(self):
attachment = upload(self.user, self.chat)[1]
output = in_api_replica("sh", "-c", "find / -xdev -name '*%s*' 2>/dev/null | wc -l" % attachment["id"])
self.assertEqual(output.split()[-1], "0")
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,90 @@
"""Проверки на уровне БД: откат составной операции, гонка регистрации, повторный запуск миграций."""
import threading
import unittest
import uuid
from concurrent.futures import ThreadPoolExecutor
from stand import call, compose, sql, wait_ready
from api import new_chat, new_key, new_user, send, user_body
class TransactionTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
wait_ready()
cls.alice, cls.bob = new_user("alice"), new_user("bob")
def create_chat(self, title, member_ids):
return call("POST", "/api/v1/chats", {"title": title, "member_ids": member_ids}, user=self.alice)
def test_chat_with_unknown_member_is_rolled_back(self):
title = "rollback-" + uuid.uuid4().hex
# Алиса и Боб существуют, третьего участника нет: вставка чата и двух участников уже выполнена,
# на третьем срабатывает внешний ключ, и транзакция должна откатить всё
status, body, _ = self.create_chat(title, [self.alice["id"], self.bob["id"], str(uuid.uuid4())])
self.assertEqual(status, 400)
self.assertEqual(body["error"]["code"], "invalid_request")
self.assertEqual(sql("SELECT count(*) FROM chats WHERE title = '%s'" % title), "0")
self.assertEqual(sql("SELECT count(*) FROM chat_members m LEFT JOIN chats c ON c.id = m.chat_id WHERE c.id IS NULL"), "0")
def test_valid_chat_is_committed_whole(self):
title = "commit-" + uuid.uuid4().hex
status, chat, _ = self.create_chat(title, [self.alice["id"], self.bob["id"]])
self.assertEqual(status, 201)
self.assertEqual(sql("SELECT count(*) FROM chats WHERE title = '%s'" % title), "1")
self.assertEqual(sql("SELECT count(*) FROM chat_members WHERE chat_id = '%s'" % chat["id"]), "2")
def test_username_race_creates_one_row(self):
body = user_body(username="race_" + uuid.uuid4().hex[:8])
barrier = threading.Barrier(10)
def register(_):
barrier.wait()
return call("POST", "/api/v1/users", body)[0]
with ThreadPoolExecutor(max_workers=10) as pool:
statuses = sorted(pool.map(register, range(10)))
self.assertEqual(statuses, [201] + [409] * 9)
self.assertEqual(sql("SELECT count(*) FROM users WHERE username = '%s'" % body["username"]), "1")
def test_same_key_in_parallel_creates_one_row(self):
chat = new_chat(self.alice)
key = new_key()
barrier = threading.Barrier(20)
def post(_):
barrier.wait()
return send(self.alice, chat, "один раз", key=key)
with ThreadPoolExecutor(max_workers=20) as pool:
results = list(pool.map(post, range(20)))
self.assertEqual({status for status, _, _ in results}, {201})
self.assertEqual(len({message["id"] for _, message, _ in results}), 1)
self.assertEqual(sql("SELECT count(*) FROM messages WHERE chat_id = '%s'" % chat["id"]), "1")
self.assertEqual(sql("SELECT count(*) FROM idempotency_keys WHERE chat_id = '%s'" % chat["id"]), "1")
def test_failed_message_leaves_no_key(self):
# чужой чат: сообщение не создано, и ключ тоже не занят
eve = new_user("eve")
chat = new_chat(self.alice)
key = new_key()
self.assertEqual(send(eve, chat, "чужое", key=key)[0], 404)
self.assertEqual(sql("SELECT count(*) FROM idempotency_keys WHERE chat_id = '%s'" % chat["id"]), "0")
def test_password_is_stored_as_argon2id_hash(self):
stored = sql("SELECT password_hash FROM users WHERE id = '%s'" % self.alice["id"])
self.assertTrue(stored.startswith("$argon2id$"), stored[:12])
self.assertNotIn(self.alice["password"], stored)
def test_migrations_can_be_repeated(self):
before = sql("SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM chats), (SELECT count(*) FROM messages)")
output = compose("run", "--rm", "migrate")
self.assertIn("уже применена: 0001_init.sql", output)
self.assertIn("уже применена: 0003_attachments.sql", output)
self.assertEqual(sql("SELECT count(*) FROM schema_migrations"), "3")
after = sql("SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM chats), (SELECT count(*) FROM messages)")
self.assertEqual(before, after)
if __name__ == "__main__":
unittest.main()
+103
View File
@@ -0,0 +1,103 @@
"""Настоящие маленькие изображения, созданы Pillow, и сборка синтетических PNG."""
import base64
import struct
import zlib
from random import Random
REAL_IMAGES = {
"jpeg_baseline": ("image/jpeg", base64.b64decode(
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAoHBwgHBgoICAgLCgoLDhgQDg0NDh0VFhEYIx8lJCIfIiEmKzcvJik0KSEiMEExNDk7"
"Pj4+JS5ESUM8SDc9Pjv/2wBDAQoLCw4NDhwQEBw7KCIoOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7"
"Ozs7Ozs7Ozv/wAARCAAQABgDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUF"
"BAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVW"
"V1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi"
"4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAEC"
"AxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVm"
"Z2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq"
"8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDz220fp8ta9to/T5a6W20fp8ta9to/T5a5YYg4sBm22pzNto/T5aK7y20fp8tFdKxGh9nQ"
"zb3Nz//Z"
)),
"jpeg_progressive": ("image/jpeg", base64.b64decode(
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAoHBwgHBgoICAgLCgoLDhgQDg0NDh0VFhEYIx8lJCIfIiEmKzcvJik0KSEiMEExNDk7"
"Pj4+JS5ESUM8SDc9Pjv/2wBDAQoLCw4NDhwQEBw7KCIoOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7"
"Ozs7Ozs7Ozv/wgARCAAQABgDASIAAhEBAxEB/8QAFwAAAwEAAAAAAAAAAAAAAAAAAAQFBv/EABUBAQEAAAAAAAAAAAAAAAAAAAMF"
"/9oADAMBAAIQAxAAAAHPN02yCYXhLX//xAAWEAADAAAAAAAAAAAAAAAAAAAAAgP/2gAIAQEAAQUCWIsRYixFiLE//8QAGBEAAgMA"
"AAAAAAAAAAAAAAAAAAQCAyH/2gAIAQMBAT8BobIN4f/EABcRAQADAAAAAAAAAAAAAAAAAAABAxP/2gAIAQIBAT8Bixo//8QAFBAB"
"AAAAAAAAAAAAAAAAAAAAIP/aAAgBAQAGPwJf/8QAFRABAQAAAAAAAAAAAAAAAAAAAGH/2gAIAQEAAT8hgggggg//2gAMAwEAAgAD"
"AAAAEFfv/8QAFxEBAAMAAAAAAAAAAAAAAAAAACExUf/aAAgBAwEBPxCqW1//xAAWEQEBAQAAAAAAAAAAAAAAAAAAASH/2gAIAQIB"
"AT8QUw//xAAVEAEBAAAAAAAAAAAAAAAAAAAAMf/aAAgBAQABPxCRIkSJEj//2Q=="
)),
"jpeg_gray": ("image/jpeg", base64.b64decode(
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAx"
"NDQ0Hyc5PTgyPC4zNDL/wAALCAAQABgBAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUF"
"BAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVW"
"V1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi"
"4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APLLTQ+nyVu2mh9PkrctND6fJWlaaH0+St200Pp8lblpofT5K//Z"
)),
"jpeg_cmyk": ("image/jpeg", base64.b64decode(
"/9j/7gAOQWRvYmUAZAAAAAAA/9sAQwAIBgYHBgUIBwcHCQkICgwUDQwLCwwZEhMPFB0aHx4dGhwcICQuJyAiLCMcHCg3KSwwMTQ0"
"NB8nOT04MjwuMzQy/8AAFAgAEAAYBEMRAE0RAFkRAEsRAP/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIB"
"AwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZH"
"SElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV"
"1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/aAA4EQwBNAFkASwAAPwDyLQf4a8z/AOER/wCmf6V5FYWHTivf69Q0H+Gj/hEf+mf6"
"V1FhYdOKK9Q0H+Gj/hEf+mf6V1FhYdOKK+X9B/hr3n/hEf8Apn+lcvYWHTiivUNB/ho/4RH/AKZ/pXUWFh04or1DQf4aP+ER/wCm"
"f6V1FhYdOKK//9k="
)),
"png_rgb": ("image/png", base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAABgAAAAQCAIAAACDRijCAAAAIklEQVR4nGNkYGDgYmClHLEw8LMyMFABjRo0atCoQSPVIACIwAmA"
"4cyUgAAAAABJRU5ErkJggg=="
)),
"png_rgba": ("image/png", base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAABgAAAAQCAYAAAAMJL+VAAAAKUlEQVR4nGNkYGD4z8XAykArzMLAz8rAwEA7PGrBqAWjFoxaMGoB"
"NTAAUr4Kf23u9hgAAAAASUVORK5CYII="
)),
"png_palette": ("image/png", base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAABgAAAAQCAMAAAA7+k+nAAACplBMVEUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAz"
"AABmAACZAADMAAD/AAAAMwAzMwBmMwCZMwDMMwD/MwAAZgAzZgBmZgCZZgDMZgD/ZgAAmQAzmQBmmQCZmQDMmQD/mQAAzAAzzABm"
"zACZzADMzAD/zAAA/wAz/wBm/wCZ/wDM/wD//wAAADMzADNmADOZADPMADP/ADMAMzMzMzNmMzOZMzPMMzP/MzMAZjMzZjNmZjOZ"
"ZjPMZjP/ZjMAmTMzmTNmmTOZmTPMmTP/mTMAzDMzzDNmzDOZzDPMzDP/zDMA/zMz/zNm/zOZ/zPM/zP//zMAAGYzAGZmAGaZAGbM"
"AGb/AGYAM2YzM2ZmM2aZM2bMM2b/M2YAZmYzZmZmZmaZZmbMZmb/ZmYAmWYzmWZmmWaZmWbMmWb/mWYAzGYzzGZmzGaZzGbMzGb/"
"zGYA/2Yz/2Zm/2aZ/2bM/2b//2YAAJkzAJlmAJmZAJnMAJn/AJkAM5kzM5lmM5mZM5nMM5n/M5kAZpkzZplmZpmZZpnMZpn/ZpkA"
"mZkzmZlmmZmZmZnMmZn/mZkAzJkzzJlmzJmZzJnMzJn/zJkA/5kz/5lm/5mZ/5nM/5n//5kAAMwzAMxmAMyZAMzMAMz/AMwAM8wz"
"M8xmM8yZM8zMM8z/M8wAZswzZsxmZsyZZszMZsz/ZswAmcwzmcxmmcyZmczMmcz/mcwAzMwzzMxmzMyZzMzMzMz/zMwA/8wz/8xm"
"/8yZ/8zM/8z//8wAAP8zAP9mAP+ZAP/MAP//AP8AM/8zM/9mM/+ZM//MM///M/8AZv8zZv9mZv+ZZv/MZv//Zv8Amf8zmf9mmf+Z"
"mf/Mmf//mf8AzP8zzP9mzP+ZzP/MzP//zP8A//8z//9m//+Z///M//////91g91EAAABH0lEQVR4nC3LMW6DMBTGce8ssOQaBAkT"
"KVAJOsAFSiIZqmJUoMJGart2IUOcwVTC7pBzdMnU3q3P0G/8/95DCCHLtVx3a+Z5uZcfzI7QXQRgbYG8By/PvYOhD2TbNnYwDnAQ"
"BGRHdkVRwsYjsn3Hx7C1F9BBxvIE4DsrkMDcGxnL8RH5G+yHGEeYRIRQUtCiFOVYn9Bmvw/NoqeouqsopfW5FrW4IOhGIMOomaBC"
"XATA8lGFVVRRkDPkWohPFMdJnDRJ05p1nezki5ymyUCcGLgH6lro3TT9Q/KcNE27vEiQSa6Qxn3WZKzhvOWqVYNUWmr9rVHaZz1j"
"jDPOZ64GpZSG3b4AUpYZgT4PM3Slbwu8sv4N4J3N/Ar9CvKjb79/rgyKjTEBaRcAAAAASUVORK5CYII="
)),
"png_1bit": ("image/png", base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAABgAAAAQAQAAAAAkX4I4AAAAS0lEQVR4nAFAAL//AAAAAgAAEqkBAkLgARABRAEFRQAES9UEAQVM"
"BQJN/P8BCRwwAaoLuQKo9UcC2AMmAaoMAQKr9bcAqt37AV8Y6K/aEe5QdGjgAAAAAElFTkSuQmCC"
)),
}
def png_chunk(kind, data):
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
def make_png(width, height, raw=None):
"""PNG RGB с верными CRC. Без raw пиксели настоящие, с raw можно подложить любые данные."""
if raw is None:
row = b"\x00" + bytes(i % 251 for i in range(width * 3))
raw = row * height
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", header) + png_chunk(b"IDAT", zlib.compress(raw)) + png_chunk(b"IEND", b"")
def noisy_png(size):
"""Настоящий PNG ровно из size байт. Случайные пиксели не сжимаются, поэтому файл действительно большой."""
width = 1000
row = 1 + width * 3
height = (size - size // 500 - 200) // row
random = Random(size)
raw = b"".join(b"\x00" + random.randbytes(width * 3) for _ in range(height))
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
start = b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", header) + png_chunk(b"IDAT", zlib.compress(raw, 1))
end = png_chunk(b"IEND", b"")
# остаток добирается служебным блоком: он не меняет изображение
padding = size - len(start) - len(end) - 12
assert padding >= 0, "образец больше запрошенного размера"
return start + png_chunk(b"tEXt", b"x" * padding) + end
+85
View File
@@ -0,0 +1,85 @@
import unittest
import uuid
from api import call, new_chat, new_user, send
class ForeignChatTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.alice, cls.bob, cls.eve = new_user("alice"), new_user("bob"), new_user("eve")
cls.chat = new_chat(cls.alice, cls.bob)
cls.path = "/api/v1/chats/" + cls.chat["id"]
send(cls.alice, cls.chat, "секрет")
def test_outsider_gets_404_everywhere(self):
self.assertEqual(call("GET", self.path, user=self.eve)[0], 404)
self.assertEqual(call("GET", self.path + "/messages", user=self.eve)[0], 404)
status, body, _ = send(self.eve, self.chat, "я тут")
self.assertEqual(status, 404)
self.assertEqual(body["error"]["code"], "not_found")
def test_foreign_chat_looks_like_missing_chat(self):
_, foreign, _ = call("GET", self.path, user=self.eve)
_, missing, _ = call("GET", "/api/v1/chats/" + str(uuid.uuid4()), user=self.eve)
self.assertEqual(foreign["error"]["message"], missing["error"]["message"])
def test_outsider_message_is_not_saved(self):
send(self.eve, self.chat, "не должно появиться")
_, page, _ = call("GET", self.path + "/messages", user=self.alice)
self.assertNotIn("не должно появиться", [item["text"] for item in page["items"]])
def test_chat_is_not_listed_for_outsider(self):
_, page, _ = call("GET", "/api/v1/chats?limit=100", user=self.eve)
self.assertNotIn(self.chat["id"], [item["id"] for item in page["items"]])
_, page, _ = call("GET", "/api/v1/chats?limit=100", user=self.bob)
self.assertIn(self.chat["id"], [item["id"] for item in page["items"]])
def test_members_can_read_and_write(self):
self.assertEqual(send(self.bob, self.chat, "ответ")[0], 201)
self.assertEqual(call("GET", self.path, user=self.bob)[0], 200)
def test_bad_chat_id(self):
chat_id = self.chat["id"]
for bad in ("123", chat_id.replace("-", ""), "urn:uuid:" + chat_id):
with self.subTest(chat_id=bad):
self.assertEqual(call("GET", "/api/v1/chats/" + bad, user=self.alice)[0], 400)
self.assertEqual(call("GET", "/api/v1/chats/" + bad + "/messages", user=self.alice)[0], 400)
def test_member_id_must_be_canonical_uuid(self):
body = {"title": "Чат", "member_ids": ["urn:uuid:" + self.alice["id"]]}
self.assertEqual(call("POST", "/api/v1/chats", body, user=self.alice)[0], 400)
class CreateChatTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.alice, cls.bob = new_user("alice"), new_user("bob")
def create(self, member_ids, title="Чат"):
return call("POST", "/api/v1/chats", {"title": title, "member_ids": member_ids}, user=self.alice)[0]
def test_creator_must_be_member(self):
self.assertEqual(self.create([self.bob["id"]]), 400)
def test_duplicate_members(self):
self.assertEqual(self.create([self.alice["id"], self.bob["id"], self.bob["id"]]), 400)
def test_unknown_member(self):
self.assertEqual(self.create([self.alice["id"], str(uuid.uuid4())]), 400)
def test_empty_members(self):
self.assertEqual(self.create([]), 400)
def test_blank_title(self):
self.assertEqual(self.create([self.alice["id"]], title=" "), 400)
def test_failed_create_leaves_no_chat(self):
_, before, _ = call("GET", "/api/v1/chats?limit=100", user=self.bob)
self.create([self.alice["id"], self.bob["id"], str(uuid.uuid4())])
_, after, _ = call("GET", "/api/v1/chats?limit=100", user=self.bob)
self.assertEqual(before, after)
if __name__ == "__main__":
unittest.main()
+314
View File
@@ -0,0 +1,314 @@
import hashlib
import socket
import unittest
import urllib.parse
import uuid
from pathlib import Path
from api import BASE_URL, call, history, new_chat, new_key, new_user, request_bytes, send
from samples import REAL_IMAGES, make_png, noisy_png
FIXTURE = (Path(__file__).resolve().parents[1] / "tests/fixtures/sample.png").read_bytes()
MIB = 1024 * 1024
def upload(user, chat, data, content_type="image/png", filename="photo.png", headers=None):
path = "/api/v1/chats/%s/attachments" % chat["id"]
if filename is not None:
path += "?filename=" + urllib.parse.quote(filename, safe="")
return call("POST", path, raw=data, user=user, headers={"Content-Type": content_type, **(headers or {})})
def content(user, attachment_id, query=""):
return request_bytes("GET", "/api/v1/attachments/%s/content%s" % (attachment_id, query), user=user)
class Base(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.alice, cls.bob, cls.eve = new_user("alice"), new_user("bob"), new_user("eve")
cls.chat = new_chat(cls.alice, cls.bob)
def assert_error(self, result, status, code):
got, body, headers = result
self.assertEqual(got, status)
self.assertEqual(body["error"]["code"], code)
self.assertEqual(body["error"]["request_id"], headers["X-Request-Id"])
class UploadTest(Base):
def test_upload_and_download_real_images(self):
for name, (content_type, data) in REAL_IMAGES.items():
with self.subTest(name):
status, attachment, _ = upload(self.alice, self.chat, data, content_type, name)
self.assertEqual(status, 201)
self.assertEqual(attachment["content_type"], content_type)
self.assertEqual(attachment["sha256"], hashlib.sha256(data).hexdigest())
got, body, headers = content(self.bob, attachment["id"])
self.assertEqual(got, 200)
self.assertEqual(body, data)
self.assertEqual(headers.get_content_type(), content_type)
def test_attachment_fields(self):
status, attachment, _ = upload(self.alice, self.chat, FIXTURE, filename="отчёт 👋.png")
self.assertEqual(status, 201)
self.assertEqual(attachment["status"], "ready")
self.assertEqual(attachment["variants"], [])
self.assertIsNone(attachment["error_code"])
self.assertEqual(attachment["filename"], "отчёт 👋.png")
self.assertEqual(attachment["owner_id"], self.alice["id"])
self.assertEqual(attachment["size_bytes"], len(FIXTURE))
self.assertEqual(call("GET", "/api/v1/attachments/" + attachment["id"], user=self.bob)[1], attachment)
def test_download_headers(self):
_, attachment, _ = upload(self.alice, self.chat, FIXTURE, filename="отчёт.png")
_, _, headers = content(self.alice, attachment["id"])
self.assertEqual(headers["X-Content-Type-Options"], "nosniff")
self.assertIn("filename*=UTF-8''", headers["Content-Disposition"])
self.assertEqual(headers["Content-Length"], str(len(FIXTURE)))
def test_file_of_exactly_10_mib(self):
data = noisy_png(10 * MIB)
self.assertEqual(len(data), 10 * MIB)
status, attachment, _ = upload(self.alice, self.chat, data)
self.assertEqual(status, 201)
got, body, _ = content(self.bob, attachment["id"])
self.assertEqual((got, hashlib.sha256(body).hexdigest()), (200, attachment["sha256"]))
def test_one_byte_over_10_mib(self):
self.assert_error(upload(self.alice, self.chat, noisy_png(10 * MIB + 1)), 413, "payload_too_large")
class LimitsTest(Base):
def test_too_large_with_length(self):
self.assert_error(upload(self.alice, self.chat, b"\x89PNG" + b"x" * (10 * MIB)), 413, "payload_too_large")
def test_too_large_for_balancer(self):
self.assert_error(upload(self.alice, self.chat, b"x" * (12 * MIB)), 413, "payload_too_large")
def test_too_large_chunked(self):
# без Content-Length размер известен только по ходу чтения
address = urllib.parse.urlsplit(BASE_URL)
path = "/api/v1/chats/%s/attachments?filename=a.png" % self.chat["id"]
head = ("POST %s HTTP/1.1\r\nHost: x\r\nContent-Type: image/png\r\nAuthorization: Bearer %s\r\n"
"Transfer-Encoding: chunked\r\n\r\n" % (path, self.alice["token"])).encode()
with socket.create_connection((address.hostname, address.port), timeout=15) as connection:
connection.sendall(head)
block = b"x" * MIB
try:
for _ in range(12):
connection.sendall(b"100000\r\n" + block + b"\r\n")
connection.sendall(b"0\r\n\r\n")
except OSError:
pass # сервер закрыл соединение, не дочитав: это и нужно
self.assertIn(b" 413 ", connection.recv(4096).split(b"\r\n")[0])
def test_decompression_bomb(self):
data = make_png(50000, 50000, b"\x00" * 1000)
self.assert_error(upload(self.alice, self.chat, data), 413, "payload_too_large")
def test_pixels_over_limit_with_small_file(self):
self.assert_error(upload(self.alice, self.chat, make_png(8000, 8000, b"\x00" * 100)), 413, "payload_too_large")
def test_empty_file(self):
self.assert_error(upload(self.alice, self.chat, b""), 415, "unsupported_media_type")
class ContentCheckTest(Base):
def test_broken_images(self):
png, jpeg = FIXTURE, REAL_IMAGES["jpeg_baseline"][1]
flipped = bytearray(png)
flipped[len(png) // 2] ^= 0xFF
cases = {
"текст": (b"not an image at all", "image/png"),
"обрезанный PNG": (png[: len(png) // 2], "image/png"),
"обрезанный JPEG": (jpeg[: len(jpeg) // 2], "image/jpeg"),
"только сигнатура PNG": (png[:8], "image/png"),
"только сигнатура JPEG": (jpeg[:3], "image/jpeg"),
"испорченный байт в PNG": (bytes(flipped), "image/png"),
"GIF": (b"GIF89a" + b"\x00" * 100, "image/png"),
"SVG": (b"<svg xmlns='http://www.w3.org/2000/svg'/>", "image/png"),
}
for name, (data, content_type) in cases.items():
with self.subTest(name):
self.assert_error(upload(self.alice, self.chat, data, content_type), 415, "unsupported_media_type")
def test_mime_spoofing(self):
png, jpeg = FIXTURE, REAL_IMAGES["jpeg_baseline"][1]
self.assert_error(upload(self.alice, self.chat, png, "image/jpeg"), 415, "unsupported_media_type")
self.assert_error(upload(self.alice, self.chat, jpeg, "image/png"), 415, "unsupported_media_type")
def test_unsupported_content_type(self):
for content_type in ("image/gif", "text/html", "application/octet-stream", "multipart/form-data"):
with self.subTest(content_type):
self.assert_error(upload(self.alice, self.chat, FIXTURE, content_type), 415, "unsupported_media_type")
def test_extension_is_not_trusted(self):
status, attachment, _ = upload(self.alice, self.chat, FIXTURE, filename="evil.exe")
self.assertEqual((status, attachment["content_type"]), (201, "image/png"))
class FilenameTest(Base):
def test_bad_filenames(self):
names = ("", " ", "a/b.png", "../../etc/passwd", "a\\b.png", "a\x00b.png", "a\nb.png", "a\tb.png", "a\x7fb.png",
".", "..", "x" * 256, "photo‮gnp.exe")
for name in names:
with self.subTest(name=ascii(name[:20])):
self.assert_error(upload(self.alice, self.chat, FIXTURE, filename=name), 400, "invalid_request")
def test_missing_filename(self):
self.assert_error(upload(self.alice, self.chat, FIXTURE, filename=None), 400, "invalid_request")
def test_good_filenames(self):
for name in ("a", "x" * 255, "я" * 255, "two words.png", "семья 👨‍👩‍👧.jpg", ".hidden.png", "'; DROP TABLE users; --.png"):
with self.subTest(name=name[:20]):
status, attachment, _ = upload(self.alice, self.chat, FIXTURE, filename=name)
self.assertEqual((status, attachment["filename"]), (201, name))
class AccessTest(Base):
@classmethod
def setUpClass(cls):
super().setUpClass()
_, cls.attachment, _ = upload(cls.alice, cls.chat, FIXTURE)
cls.other_chat = new_chat(cls.eve)
def test_outsider_gets_404(self):
attachment_id = self.attachment["id"]
self.assert_error(call("GET", "/api/v1/attachments/" + attachment_id, user=self.eve), 404, "not_found")
self.assertEqual(content(self.eve, attachment_id)[0], 404)
def test_outsider_looks_the_same_as_missing(self):
_, foreign, _ = call("GET", "/api/v1/attachments/" + self.attachment["id"], user=self.eve)
_, missing, _ = call("GET", "/api/v1/attachments/" + str(uuid.uuid4()), user=self.eve)
self.assertEqual(foreign["error"]["message"], missing["error"]["message"])
def test_no_token(self):
self.assertEqual(call("GET", "/api/v1/attachments/" + self.attachment["id"])[0], 401)
self.assertEqual(content({"token": "x" * 43}, self.attachment["id"])[0], 401)
self.assertEqual(upload({"token": "x" * 43}, self.chat, FIXTURE)[0], 401)
def test_outsider_cannot_upload(self):
self.assert_error(upload(self.eve, self.chat, FIXTURE), 404, "not_found")
def test_bad_ids(self):
self.assertEqual(call("GET", "/api/v1/attachments/123", user=self.alice)[0], 400)
self.assertEqual(call("GET", "/api/v1/attachments/urn:uuid:" + self.attachment["id"], user=self.alice)[0], 400)
def test_download_never_redirects(self):
status, body, headers = content(self.bob, self.attachment["id"])
self.assertEqual(status, 200)
self.assertIsNone(headers.get("Location"))
self.assertNotIn(b"X-Amz", body[:200])
def test_variants(self):
attachment_id = self.attachment["id"]
self.assertEqual(content(self.bob, attachment_id, "?variant=original")[0], 200)
self.assertEqual(content(self.bob, attachment_id, "?variant=thumbnail")[0], 404)
self.assertEqual(content(self.bob, attachment_id, "?variant=nonsense")[0], 400)
self.assertEqual(content(self.eve, attachment_id, "?variant=nonsense")[0], 404)
class MessageLinkTest(Base):
def test_message_with_attachments(self):
ids = [upload(self.alice, self.chat, FIXTURE)[1]["id"] for _ in range(2)]
status, message, _ = send(self.alice, self.chat, "смотри", attachment_ids=ids)
self.assertEqual((status, message["attachment_ids"]), (201, ids))
items, _ = history(self.bob, self.chat, 100)
self.assertIn(message, items)
def test_attachment_only_message(self):
attachment_id = upload(self.alice, self.chat, FIXTURE)[1]["id"]
status, message, _ = send(self.bob, self.chat, "", attachment_ids=[attachment_id])
self.assertEqual((status, message["text"]), (201, ""))
def test_text_message_has_empty_list(self):
self.assertEqual(send(self.alice, self.chat, "текст")[1]["attachment_ids"], [])
def test_empty_message_without_attachments(self):
self.assert_error(send(self.alice, self.chat, "", attachment_ids=[]), 400, "invalid_request")
self.assert_error(send(self.alice, self.chat, " ", attachment_ids=[]), 400, "invalid_request")
def test_foreign_attachment(self):
foreign_chat = new_chat(self.eve)
foreign = upload(self.eve, foreign_chat, FIXTURE)[1]["id"]
self.assert_error(send(self.alice, self.chat, "чужое", attachment_ids=[foreign]), 404, "not_found")
def test_attachment_of_my_other_chat(self):
other = new_chat(self.alice)
attachment_id = upload(self.alice, other, FIXTURE)[1]["id"]
self.assert_error(send(self.alice, self.chat, "не тот чат", attachment_ids=[attachment_id]), 404, "not_found")
def test_unknown_attachment(self):
self.assert_error(send(self.alice, self.chat, "нет", attachment_ids=[str(uuid.uuid4())]), 404, "not_found")
def test_failed_link_saves_nothing_and_frees_key(self):
key = new_key()
attachment_id = upload(self.alice, self.chat, FIXTURE)[1]["id"]
self.assertEqual(send(self.alice, self.chat, "повтор", key=key, attachment_ids=[str(uuid.uuid4())])[0], 404)
self.assertEqual(send(self.alice, self.chat, "повтор", key=key, attachment_ids=[attachment_id])[0], 201)
def test_limits(self):
one = str(uuid.uuid4())
self.assert_error(send(self.alice, self.chat, "x", attachment_ids=[one, one]), 400, "invalid_request")
many = [str(uuid.uuid4()) for _ in range(11)]
self.assert_error(send(self.alice, self.chat, "x", attachment_ids=many), 400, "invalid_request")
self.assert_error(send(self.alice, self.chat, "x", attachment_ids=["abc"]), 400, "invalid_request")
def test_ten_attachments(self):
ids = [upload(self.alice, self.chat, FIXTURE)[1]["id"] for _ in range(10)]
self.assertEqual(send(self.alice, self.chat, "десять", attachment_ids=ids)[1]["attachment_ids"], ids)
def test_idempotent_repeat(self):
attachment_id = upload(self.alice, self.chat, FIXTURE)[1]["id"]
key = new_key()
first = send(self.alice, self.chat, "с файлом", key=key, attachment_ids=[attachment_id])
second = send(self.alice, self.chat, "с файлом", key=key, attachment_ids=[attachment_id])
self.assertEqual(first[1], second[1])
self.assertEqual(send(self.alice, self.chat, "с файлом", key=key)[0], 409)
def test_missing_list_equals_empty_list(self):
key = new_key()
first = send(self.alice, self.chat, "без файла", key=key)
second = send(self.alice, self.chat, "без файла", key=key, attachment_ids=[])
self.assertEqual(first[1], second[1])
def test_one_file_in_two_messages(self):
attachment_id = upload(self.alice, self.chat, FIXTURE)[1]["id"]
self.assertEqual(send(self.alice, self.chat, "первое", attachment_ids=[attachment_id])[0], 201)
self.assertEqual(send(self.bob, self.chat, "второе", attachment_ids=[attachment_id])[0], 201)
class ParallelLimitTest(Base):
def test_too_many_parallel_uploads_get_429(self):
address = urllib.parse.urlsplit(BASE_URL)
path = "/api/v1/chats/%s/attachments?filename=a.png" % self.chat["id"]
head = ("POST %s HTTP/1.1\r\nHost: x\r\nContent-Type: image/png\r\nContent-Length: 1000\r\n"
"Authorization: Bearer %s\r\n\r\nabc" % (path, self.alice["token"])).encode()
# начатые загрузки, которые ничего не досылают; на каждую реплику лимит 32
connections = []
answers = []
try:
for _ in range(80):
connection = socket.create_connection((address.hostname, address.port), timeout=5)
connection.sendall(head)
connections.append(connection)
import time
time.sleep(1.5)
for connection in connections:
connection.setblocking(False)
try:
answers.append(connection.recv(4096).split(b"\r\n")[0])
except BlockingIOError:
pass
finally:
for connection in connections:
connection.close()
self.assertTrue(answers, "ни одна лишняя загрузка не получила ответ")
self.assertTrue(all(b" 429 " in answer for answer in answers), answers[:3])
self.assertEqual(upload(self.alice, self.chat, FIXTURE)[0], 201)
if __name__ == "__main__":
unittest.main()
+144
View File
@@ -0,0 +1,144 @@
import datetime
import unittest
import uuid
from api import basic, call, login, new_user, user_body
class LoginTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.body = user_body()
_, cls.user, _ = call("POST", "/api/v1/users", cls.body)
def assert_basic_challenge(self, result):
status, body, headers = result
self.assertEqual(status, 401)
self.assertEqual(body["error"]["code"], "unauthorized")
self.assertTrue(headers["WWW-Authenticate"].startswith("Basic "))
return body
def test_login(self):
status, session, _ = login(self.body["username"], self.body["password"])
self.assertEqual(status, 201)
self.assertEqual(set(session), {"token", "token_type", "expires_at", "user"})
self.assertEqual(session["token_type"], "Bearer")
self.assertEqual(session["user"], self.user)
self.assertGreaterEqual(len(session["token"]), 32)
expires_at = datetime.datetime.fromisoformat(session["expires_at"][:-1] + "+00:00")
left = (expires_at - datetime.datetime.now(datetime.timezone.utc)).total_seconds()
self.assertTrue(300 <= left <= 3600 + 5, left)
def test_token_is_opaque_and_unique(self):
first = login(self.body["username"], self.body["password"])[1]["token"]
second = login(self.body["username"], self.body["password"])[1]["token"]
self.assertNotEqual(first, second)
for token in (first, second):
self.assertNotIn(self.user["id"], token)
self.assertNotIn(self.body["username"], token)
self.assertEqual(call("GET", "/api/v1/users/me", user={"token": token})[0], 200)
def test_wrong_password_and_unknown_user_look_the_same(self):
wrong = self.assert_basic_challenge(login(self.body["username"], "wrong-password-123"))
unknown = self.assert_basic_challenge(login("nobody_" + uuid.uuid4().hex[:8], self.body["password"]))
self.assertEqual(wrong["error"]["message"], unknown["error"]["message"])
def test_malformed_credentials(self):
token = login(self.body["username"], self.body["password"])[1]["token"]
for header in (None, "Basic", "Basic !!!not-base64!!!", "Basic bm8tY29sb24=", "Bearer " + token, "Basic 0KU6"):
with self.subTest(header=header):
headers = {"Authorization": header} if header else {}
self.assert_basic_challenge(call("POST", "/api/v1/auth/sessions", headers=headers))
def test_garbage_after_basic_is_401_not_500(self):
# значение заголовка приходит на сервер как latin-1, поэтому после Basic может оказаться не ASCII
values = ("\u00e9", "\u00e9\u00e9\u00e9=", "\u00ff\u00fe", "//46cHc=", "=", "====", "a", " ", "A" * 5000, "Og==",
"dXNlcjpwYXNz dXNlcjpwYXNz")
for value in values:
with self.subTest(value=ascii(value[:16])):
result = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + value})
self.assert_basic_challenge(result)
def test_username_that_cannot_exist(self):
for username in ("", "Has Space", "ab\u0000cd", "имя_кириллицей"):
with self.subTest(username=username):
self.assert_basic_challenge(login(username, self.body["password"]))
def test_colon_and_unicode_in_password(self):
body = user_body(password="пароль:с:двоеточиями")
self.assertEqual(call("POST", "/api/v1/users", body)[0], 201)
self.assertEqual(login(body["username"], body["password"])[0], 201)
self.assert_basic_challenge(login(body["username"], "пароль"))
class BearerTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.alice, cls.eve = new_user("alice"), new_user("eve")
def assert_bearer_challenge(self, result):
status, body, headers = result
self.assertEqual(status, 401)
self.assertEqual(body["error"]["code"], "unauthorized")
self.assertTrue(headers["WWW-Authenticate"].startswith("Bearer "))
def test_no_token(self):
self.assert_bearer_challenge(call("GET", "/api/v1/users/me"))
def test_unknown_tokens(self):
for token in ("x" * 43, self.alice["id"], self.alice["username"], self.alice["token"][:-1]):
with self.subTest(token=token[:8]):
self.assert_bearer_challenge(call("GET", "/api/v1/users/me", user={"token": token}))
def test_garbage_token_is_401_not_500(self):
for token in ("\u00e9", "\u00ff\u00fe", "a b", "x" * 10000):
with self.subTest(token=ascii(token[:16])):
self.assert_bearer_challenge(call("GET", "/api/v1/users/me", user={"token": token}))
def test_basic_is_not_accepted_on_business_endpoints(self):
headers = basic(self.alice["username"], self.alice["password"])
self.assert_bearer_challenge(call("GET", "/api/v1/users/me", headers=headers))
self.assert_bearer_challenge(call("GET", "/api/v1/chats", headers=headers))
def test_x_user_id_gives_no_access(self):
self.assert_bearer_challenge(call("GET", "/api/v1/users/me", headers={"X-User-Id": self.alice["id"]}))
def test_x_user_id_does_not_change_identity(self):
status, me, _ = call("GET", "/api/v1/users/me", user=self.alice, headers={"X-User-Id": self.eve["id"]})
self.assertEqual(status, 200)
self.assertEqual(me, self.alice["user"])
def test_token_is_checked_before_anything_else(self):
self.assert_bearer_challenge(call("GET", "/api/v1/chats/not-a-uuid"))
self.assert_bearer_challenge(call("GET", "/api/v1/chats?limit=0"))
self.assert_bearer_challenge(call("POST", "/api/v1/chats", {"title": 5}))
class LogoutTest(unittest.TestCase):
def test_logout_revokes_only_current_session(self):
user = new_user()
other_token = login(user["username"], user["password"])[1]["token"]
status, body, _ = call("DELETE", "/api/v1/auth/sessions/current", user=user)
self.assertEqual(status, 204)
self.assertIsNone(body)
self.assertEqual(call("GET", "/api/v1/users/me", user=user)[0], 401)
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", user=user)[0], 401)
# вторая сессия того же пользователя продолжает работать
self.assertEqual(call("GET", "/api/v1/users/me", user={"token": other_token})[0], 200)
def test_login_again_after_logout(self):
user = new_user()
call("DELETE", "/api/v1/auth/sessions/current", user=user)
status, session, _ = login(user["username"], user["password"])
self.assertEqual(status, 201)
self.assertNotEqual(session["token"], user["token"])
self.assertEqual(call("GET", "/api/v1/users/me", user=session)[0], 200)
def test_logout_requires_token(self):
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current")[0], 401)
if __name__ == "__main__":
unittest.main()
+94
View File
@@ -0,0 +1,94 @@
import json
import socket
import unittest
import urllib.parse
import uuid
from api import BASE_URL, basic, call, history, new_chat, new_user, send
class ErrorsFromBalancerTest(unittest.TestCase):
"""Запросы, которые до API не доходят: ответ всё равно в формате контракта."""
def assert_contract_error(self, raw_request):
address = urllib.parse.urlsplit(BASE_URL)
with socket.create_connection((address.hostname, address.port), timeout=5) as connection:
connection.sendall(raw_request)
data = b""
while chunk := connection.recv(65536):
data += chunk
head, _, body = data.partition(b"\r\n\r\n")
lines = head.decode("latin-1").split("\r\n")
headers = {line.split(":")[0].lower(): line.split(":", 1)[1].strip() for line in lines[1:]}
self.assertEqual(headers["content-type"], "application/json")
error = json.loads(body)["error"]
self.assertEqual(error["request_id"], headers["x-request-id"])
self.assertEqual(error["code"], "invalid_request")
return int(lines[0].split()[1])
def test_invalid_http(self):
self.assertEqual(self.assert_contract_error(b"GET /a b HTTP/1.1\r\nHost: x\r\n\r\n"), 400)
def test_too_long_uri(self):
self.assertEqual(self.assert_contract_error(b"GET /" + b"a" * 20000 + b" HTTP/1.1\r\nHost: x\r\n\r\n"), 414)
def test_too_large_headers(self):
raw = b"GET /x HTTP/1.1\r\nHost: x\r\nX-Big: " + b"a" * 20000 + b"\r\n\r\n"
self.assertEqual(self.assert_contract_error(raw), 431)
class ReplicasTest(unittest.TestCase):
def test_every_answer_has_instance_id(self):
user = new_user()
answers = (
call("GET", "/health/live"),
call("GET", "/health/ready"),
call("GET", "/api/v1/users/me"),
call("GET", "/api/v1/users/me", user=user),
call("GET", "/api/v1/chats/" + str(uuid.uuid4()), user=user),
call("GET", "/nowhere"),
call("POST", "/api/v1/users", raw=b"{"),
)
for status, _, headers in answers:
with self.subTest(status=status):
self.assertTrue(headers["X-Instance-Id"])
self.assertTrue(headers["X-Request-Id"])
def test_one_session_is_served_by_different_replicas(self):
user = new_user()
instances = set()
for _ in range(20):
status, me, headers = call("GET", "/api/v1/users/me", user=user)
self.assertEqual((status, me), (200, user["user"]))
instances.add(headers["X-Instance-Id"])
self.assertGreaterEqual(len(instances), 2)
def test_session_created_on_one_replica_works_on_all(self):
user = new_user()
for _ in range(6):
status, session, _ = call("POST", "/api/v1/auth/sessions", headers=basic(user["username"], user["password"]))
self.assertEqual(status, 201)
seen = {call("GET", "/api/v1/users/me", user=session)[2]["X-Instance-Id"] for _ in range(4)}
self.assertGreaterEqual(len(seen), 2)
def test_read_after_write_on_any_replica(self):
user = new_user()
chat = new_chat(user)
for number in range(20):
_, message, _ = send(user, chat, "запись %d" % number)
items, _ = history(user, chat, 100)
self.assertEqual(items[-1], message)
def test_logout_works_on_all_replicas(self):
user = new_user()
call("DELETE", "/api/v1/auth/sessions/current", user=user)
for _ in range(4):
self.assertEqual(call("GET", "/api/v1/users/me", user=user)[0], 401)
def test_no_sticky_cookie(self):
_, _, headers = call("GET", "/health/live")
self.assertIsNone(headers.get("Set-Cookie"), BASE_URL)
if __name__ == "__main__":
unittest.main()
+63
View File
@@ -0,0 +1,63 @@
"""Проверка гонок: 10 клиентов параллельно отправляют 100 сообщений в один чат."""
import threading
import unittest
import uuid
from concurrent.futures import ThreadPoolExecutor
from api import call, history, new_chat, new_user, send, user_body
CLIENTS = 10
MESSAGES_PER_CLIENT = 10
class ConcurrencyTest(unittest.TestCase):
def test_100_messages_from_10_clients(self):
users = [new_user("c%d" % number) for number in range(CLIENTS)]
chat = new_chat(users[0], *users[1:])
barrier = threading.Barrier(CLIENTS)
def client(number):
barrier.wait() # все потоки стартуют одновременно
results = []
for index in range(MESSAGES_PER_CLIENT):
results.append(send(users[number], chat, "клиент %d сообщение %d" % (number, index)))
return results
with ThreadPoolExecutor(max_workers=CLIENTS) as pool:
results = [result for chunk in pool.map(client, range(CLIENTS)) for result in chunk]
total = CLIENTS * MESSAGES_PER_CLIENT
self.assertEqual([status for status, _, _ in results], [201] * total)
confirmed = {message["id"]: message for _, message, _ in results}
self.assertEqual(len(confirmed), total)
items, _ = history(users[0], chat, limit=7)
self.assertEqual(len(items), total)
self.assertEqual(len({item["id"] for item in items}), total)
self.assertEqual(len({item["text"] for item in items}), total)
# каждое подтверждённое сообщение есть в истории без изменений
self.assertEqual({item["id"]: item for item in items}, confirmed)
keys = [(item["created_at"], item["id"]) for item in items]
self.assertEqual(keys, sorted(keys))
# сообщения одного клиента идут в том порядке, в котором он их отправлял
for number in range(CLIENTS):
own = [item["text"] for item in items if item["sender_id"] == users[number]["id"]]
self.assertEqual(own, ["клиент %d сообщение %d" % (number, index) for index in range(MESSAGES_PER_CLIENT)])
print("\nотправлено %d, подтверждено %d, в истории %d, уникальных id %d"
% (total, len(confirmed), len(items), len({item["id"] for item in items})))
def test_same_username_registered_once(self):
body = user_body(username="race_" + uuid.uuid4().hex[:8])
barrier = threading.Barrier(CLIENTS)
def register(_):
barrier.wait()
return call("POST", "/api/v1/users", body)[0]
with ThreadPoolExecutor(max_workers=CLIENTS) as pool:
statuses = sorted(pool.map(register, range(CLIENTS)))
self.assertEqual(statuses, [201] + [409] * (CLIENTS - 1))
if __name__ == "__main__":
unittest.main()
+98
View File
@@ -0,0 +1,98 @@
import threading
import unittest
from concurrent.futures import ThreadPoolExecutor
from api import call, history, new_chat, new_key, new_user, send
class IdempotencyTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.alice, cls.bob, cls.eve = new_user("alice"), new_user("bob"), new_user("eve")
cls.chat = new_chat(cls.alice, cls.bob)
cls.path = "/api/v1/chats/" + cls.chat["id"] + "/messages"
def test_key_is_required(self):
status, body, _ = call("POST", self.path, {"text": "без ключа"}, user=self.alice)
self.assertEqual(status, 400)
self.assertEqual(body["error"]["code"], "invalid_request")
def test_bad_keys(self):
for key in ("", "a" * 129, "с пробелом", "кириллица", "a/b", "a,b"):
with self.subTest(key=key[:12]):
headers = {"Idempotency-Key": key.encode("utf-8").decode("latin-1")}
self.assertEqual(call("POST", self.path, {"text": "x"}, user=self.alice, headers=headers)[0], 400)
def test_good_key_alphabet(self):
for key in ("a", "A.b_c:d-1", "x" * 128):
with self.subTest(key=key[:12]):
self.assertEqual(send(self.alice, self.chat, "ключ " + key, key=key)[0], 201)
def test_repeat_returns_same_message(self):
key = new_key()
first = send(self.alice, self.chat, "Привет 👋", key=key)
second = send(self.alice, self.chat, "Привет 👋", key=key)
self.assertEqual((first[0], second[0]), (201, 201))
self.assertEqual(first[1], second[1])
def test_key_order_in_json_does_not_matter(self):
key = new_key()
headers = {"Idempotency-Key": key}
first = call("POST", self.path, {"text": "порядок"}, user=self.alice, headers=headers)
second = call("POST", self.path, raw=' { "text" : "порядок" } '.encode(), user=self.alice, headers=headers)
self.assertEqual(first[1], second[1])
def test_other_text_with_same_key_is_409(self):
key = new_key()
first = send(self.alice, self.chat, "первый текст", key=key)
status, body, _ = send(self.alice, self.chat, "другой текст", key=key)
self.assertEqual(status, 409)
self.assertEqual(body["error"]["code"], "conflict")
# конфликт ничего не меняет: исходный текст по-прежнему возвращается
self.assertEqual(send(self.alice, self.chat, "первый текст", key=key)[1], first[1])
def test_same_key_for_other_user_and_other_chat(self):
key = new_key()
other_chat = new_chat(self.alice)
first = send(self.alice, self.chat, "общий ключ", key=key)[1]
by_bob = send(self.bob, self.chat, "общий ключ", key=key)[1]
in_other_chat = send(self.alice, other_chat, "общий ключ", key=key)[1]
self.assertEqual(len({first["id"], by_bob["id"], in_other_chat["id"]}), 3)
self.assertEqual(by_bob["sender_id"], self.bob["id"])
def test_repeat_checks_access_again(self):
key = new_key()
send(self.alice, self.chat, "только участникам", key=key)
self.assertEqual(send(self.eve, self.chat, "только участникам", key=key)[0], 404)
self.assertEqual(send({"token": "x" * 43}, self.chat, "только участникам", key=key)[0], 401)
def test_invalid_body_does_not_take_key(self):
key = new_key()
self.assertEqual(send(self.alice, self.chat, "", key=key)[0], 400)
self.assertEqual(send(self.alice, self.chat, "теперь верно", key=key)[0], 201)
def test_parallel_repeat_through_different_replicas(self):
user = new_user("idem")
chat = new_chat(user)
instances = set()
for round_number in range(10):
key = new_key()
barrier = threading.Barrier(8)
def post(_):
barrier.wait()
return send(user, chat, "сообщение %d" % round_number, key=key)
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(post, range(8)))
self.assertEqual({status for status, _, _ in results}, {201})
self.assertEqual(len({message["id"] for _, message, _ in results}), 1)
instances |= {headers["X-Instance-Id"] for _, _, headers in results}
self.assertGreaterEqual(len(instances), 2, "запросы не дошли до разных реплик")
items, _ = history(user, chat, 100)
self.assertEqual(len(items), 10)
print("\n10 ключей по 8 параллельных запросов: реплик %d, сообщений в истории %d" % (len(instances), len(items)))
if __name__ == "__main__":
unittest.main()
+119
View File
@@ -0,0 +1,119 @@
"""Проверка разбора изображений без сервера: файлы настоящие, а испорченные версии не должны ронять разбор."""
import io
import random
import struct
import sys
import unittest
import zlib
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
import images # noqa: E402
from samples import REAL_IMAGES, make_png, png_chunk # noqa: E402
LIMIT = 40_000_000
def inspect(data, limit=LIMIT):
return images.inspect(io.BytesIO(data), limit)
class RealImagesTest(unittest.TestCase):
def test_real_images_are_accepted(self):
for name, (content_type, data) in REAL_IMAGES.items():
with self.subTest(name):
self.assertEqual(inspect(data), content_type)
def test_course_fixture(self):
path = Path(__file__).resolve().parents[1] / "tests/fixtures/sample.png"
self.assertEqual(inspect(path.read_bytes()), "image/png")
def test_jpeg_with_data_after_end(self):
# камеры дописывают после конца изображения превью
self.assertEqual(inspect(REAL_IMAGES["jpeg_baseline"][1] + b"\xff\xd8" + b"x" * 50), "image/jpeg")
def test_marker_on_block_border(self):
data = bytearray(REAL_IMAGES["jpeg_baseline"][1])
start = data.index(b"\xff\xda")
length = struct.unpack(">H", data[start + 2:start + 4])[0]
padded = data[:start + 2 + length] + b"\x12" * (images.BLOCK - 3) + b"\xff\x00" * 40 + data[start + 2 + length:]
self.assertEqual(inspect(bytes(padded)), "image/jpeg")
class BrokenImagesTest(unittest.TestCase):
def assert_not_image(self, data):
with self.assertRaises(images.NotAnImage):
inspect(data)
def test_truncated(self):
for name, (_, data) in REAL_IMAGES.items():
for size in (0, 1, 2, 3, 8, 12, len(data) // 2, len(data) - 1):
with self.subTest(name=name, size=size):
self.assert_not_image(data[:size])
def test_not_images(self):
for data in (b"", b"GIF89a" + b"\x00" * 20, b"%PDF-1.7", b"<svg></svg>", b"\xff\xd8", b"\x89PNG\r\n\x1a\n"):
with self.subTest(start=data[:8]):
self.assert_not_image(data)
def test_png_bad_crc(self):
data = bytearray(make_png(8, 8))
data[-20] ^= 0xFF
self.assert_not_image(bytes(data))
def test_png_data_after_end(self):
self.assert_not_image(make_png(8, 8) + b"<script>")
def test_png_with_wrong_amount_of_pixels(self):
self.assert_not_image(make_png(8, 8, b"\x00" * 10))
self.assert_not_image(make_png(8, 8, b"\x00" * 10000))
def test_png_data_is_not_zlib(self):
header = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)
data = b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", header) + png_chunk(b"IDAT", b"not zlib") + png_chunk(b"IEND", b"")
self.assert_not_image(data)
def test_png_zero_size(self):
self.assert_not_image(make_png(0, 8, b""))
def test_jpeg_without_frame_or_scan(self):
self.assert_not_image(b"\xff\xd8\xff\xd9")
self.assert_not_image(b"\xff\xd8\xff\xe0\x00\x04ab\xff\xd9")
def test_random_corruption_never_crashes(self):
random.seed(20261030)
for name, (_, original) in REAL_IMAGES.items():
for _ in range(1500):
data = bytearray(original)
for _ in range(random.randint(1, 8)):
data[random.randrange(len(data))] = random.randrange(256)
try:
inspect(bytes(data))
except (images.NotAnImage, images.TooManyPixels):
pass
class PixelLimitTest(unittest.TestCase):
def test_png_bomb_is_rejected_before_unpacking(self):
header = struct.pack(">IIBBBBB", 50000, 50000, 8, 2, 0, 0, 0)
# данные заголовка верны, а распаковать пришлось бы 7,5 ГБ
data = b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", header) + png_chunk(b"IDAT", zlib.compress(b"\x00" * 1000))
with self.assertRaises(images.TooManyPixels):
inspect(data)
def test_jpeg_over_limit(self):
data = bytearray(REAL_IMAGES["jpeg_baseline"][1])
start = data.index(b"\xff\xc0")
data[start + 5:start + 9] = struct.pack(">HH", 20000, 20000)
with self.assertRaises(images.TooManyPixels):
inspect(bytes(data))
def test_limit_is_exact(self):
self.assertEqual(inspect(make_png(10, 10), limit=100), "image/png")
with self.assertRaises(images.TooManyPixels):
inspect(make_png(10, 10), limit=99)
if __name__ == "__main__":
unittest.main()
+145
View File
@@ -0,0 +1,145 @@
import base64
import json
import unittest
import urllib.parse
from api import call, history, new_chat, new_user, send
class MessagePaginationTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.user = new_user()
cls.chat = new_chat(cls.user)
cls.sent = [send(cls.user, cls.chat, "сообщение %d" % number)[1] for number in range(25)]
cls.path = "/api/v1/chats/" + cls.chat["id"] + "/messages"
def test_walk_several_pages(self):
items, sizes = history(self.user, self.chat, limit=10)
self.assertEqual(sizes, [10, 10, 5])
self.assertEqual(items, self.sent)
def test_no_duplicates_and_order(self):
items, _ = history(self.user, self.chat, limit=7)
ids = [item["id"] for item in items]
self.assertEqual(len(ids), len(set(ids)))
keys = [(item["created_at"], item["id"]) for item in items]
self.assertEqual(keys, sorted(keys))
def test_exact_page_has_no_cursor(self):
_, page, _ = call("GET", self.path + "?limit=25", user=self.user)
self.assertEqual(len(page["items"]), 25)
self.assertIsNone(page["next_cursor"])
def test_default_limit(self):
_, page, _ = call("GET", self.path, user=self.user)
self.assertEqual(len(page["items"]), 25)
def test_limit_can_change_between_pages(self):
_, first, _ = call("GET", self.path + "?limit=3", user=self.user)
cursor = urllib.parse.quote(first["next_cursor"], safe="")
_, second, _ = call("GET", self.path + "?limit=100&cursor=" + cursor, user=self.user)
self.assertEqual(first["items"] + second["items"], self.sent)
def test_bad_limit(self):
for limit in ("0", "101", "-1", "abc", "1.5", ""):
with self.subTest(limit=limit):
self.assertEqual(call("GET", self.path + "?limit=" + limit, user=self.user)[0], 400)
def test_bad_cursor(self):
for cursor in ("garbage", "", "e30=", "%D0%B9"):
with self.subTest(cursor=cursor):
self.assertEqual(call("GET", self.path + "?cursor=" + cursor, user=self.user)[0], 400)
def test_cursor_format_is_checked(self):
_, first, _ = call("GET", self.path + "?limit=3", user=self.user)
good = first["next_cursor"]
scope, created_at, item_id = json.loads(base64.urlsafe_b64decode(good))
def encode(value, **options):
return base64.urlsafe_b64encode(json.dumps(value, **options).encode()).decode()
# тест знает внутренний формат cursor: собранный заново правильный cursor совпадает с выданным
self.assertEqual(encode([scope, created_at, item_id]), good)
bad = {
"посторонний символ внутри base64": good[:5] + "!" + good[5:],
"пробел внутри base64": good[:5] + " " + good[5:],
"перевод строки в конце": good + "\n",
"лишний знак =": good + "=",
"обрезанный": good[:-4],
"время не в формате сервера": encode([scope, "вчера", item_id]),
"время без микросекунд": encode([scope, created_at[:19] + "Z", item_id]),
"время с неполными микросекундами": encode([scope, created_at[:21] + "Z", item_id]),
"время со смещением вместо Z": encode([scope, created_at[:-1] + "+00:00", item_id]),
"несуществующая дата": encode([scope, "2026-13-45T25:61:61.000000Z", item_id]),
"число вместо времени": encode([scope, 1234567890, item_id]),
"id не UUID": encode([scope, created_at, "abc"]),
"UUID в верхнем регистре": encode([scope, created_at, item_id.upper()]),
"UUID без дефисов": encode([scope, created_at, item_id.replace("-", "")]),
"null вместо id": encode([scope, created_at, None]),
"лишний элемент": encode([scope, created_at, item_id, "x"]),
"не хватает элемента": encode([scope, created_at]),
"объект вместо списка": encode({"scope": scope, "created_at": created_at, "id": item_id}),
"тот же JSON без пробелов": encode([scope, created_at, item_id], separators=(",", ":")),
"пустой список": encode([]),
}
for name, cursor in bad.items():
with self.subTest(name):
path = self.path + "?cursor=" + urllib.parse.quote(cursor, safe="")
status, body, _ = call("GET", path, user=self.user)
self.assertEqual(status, 400)
self.assertEqual(body["error"]["code"], "invalid_request")
# правильный cursor при этом продолжает работать
self.assertEqual(call("GET", self.path + "?cursor=" + urllib.parse.quote(good, safe=""), user=self.user)[0], 200)
def test_cursor_from_another_chat(self):
other = new_chat(self.user, title="Другой чат")
send(self.user, other, "раз")
send(self.user, other, "два")
_, page, _ = call("GET", "/api/v1/chats/" + other["id"] + "/messages?limit=1", user=self.user)
cursor = urllib.parse.quote(page["next_cursor"], safe="")
self.assertEqual(call("GET", self.path + "?cursor=" + cursor, user=self.user)[0], 400)
def test_new_message_appears_after_last_page(self):
user = new_user()
chat = new_chat(user)
send(user, chat, "раз")
send(user, chat, "два")
path = "/api/v1/chats/" + chat["id"] + "/messages"
_, first, _ = call("GET", path + "?limit=1", user=user)
_, created, _ = send(user, chat, "три")
cursor = urllib.parse.quote(first["next_cursor"], safe="")
_, rest, _ = call("GET", path + "?limit=100&cursor=" + cursor, user=user)
self.assertEqual([item["text"] for item in rest["items"]], ["два", "три"])
self.assertEqual(rest["items"][-1], created)
class ChatPaginationTest(unittest.TestCase):
def test_walk_chat_list(self):
user = new_user()
created = [new_chat(user, title="Чат %d" % number) for number in range(5)]
items, cursor = [], None
while True:
path = "/api/v1/chats?limit=2"
if cursor:
path += "&cursor=" + urllib.parse.quote(cursor, safe="")
status, page, _ = call("GET", path, user=user)
self.assertEqual(status, 200)
self.assertLessEqual(len(page["items"]), 2)
items += page["items"]
cursor = page["next_cursor"]
if cursor is None:
break
self.assertEqual(items, created)
def test_chat_cursor_belongs_to_user(self):
alice, bob = new_user("alice"), new_user("bob")
for number in range(2):
new_chat(alice, bob, title="Общий %d" % number)
_, page, _ = call("GET", "/api/v1/chats?limit=1", user=alice)
cursor = urllib.parse.quote(page["next_cursor"], safe="")
self.assertEqual(call("GET", "/api/v1/chats?cursor=" + cursor, user=bob)[0], 400)
if __name__ == "__main__":
unittest.main()
+153
View File
@@ -0,0 +1,153 @@
import unittest
from api import call, login, new_chat, new_key, new_user, send, user_body
class ErrorFormatTest(unittest.TestCase):
def assert_error(self, result, status, code):
got_status, body, headers = result
self.assertEqual(got_status, status)
self.assertEqual(set(body), {"error"})
self.assertEqual(set(body["error"]), {"code", "message", "request_id"})
self.assertEqual(body["error"]["code"], code)
self.assertTrue(body["error"]["message"])
self.assertEqual(body["error"]["request_id"], headers["X-Request-Id"])
class InvalidJsonTest(ErrorFormatTest):
def test_malformed_json(self):
self.assert_error(call("POST", "/api/v1/users", raw=b'{"username": "abc", '), 400, "invalid_request")
def test_not_an_object(self):
self.assert_error(call("POST", "/api/v1/users", raw=b'["abc"]'), 400, "invalid_request")
def test_extra_field(self):
self.assert_error(call("POST", "/api/v1/users", user_body(is_admin=True)), 400, "invalid_request")
def test_wrong_type(self):
self.assert_error(call("POST", "/api/v1/users", user_body(username=12345)), 400, "invalid_request")
def test_missing_field(self):
body = user_body()
del body["password"]
self.assert_error(call("POST", "/api/v1/users", body), 400, "invalid_request")
def test_malformed_json_in_message(self):
user = new_user()
chat = new_chat(user)
path = "/api/v1/chats/" + chat["id"] + "/messages"
headers = {"Idempotency-Key": new_key()}
self.assert_error(call("POST", path, raw=b"{text: no quotes}", user=user, headers=headers), 400, "invalid_request")
def test_only_utf8_is_accepted(self):
user = new_user()
chat = new_chat(user)
path = "/api/v1/chats/" + chat["id"] + "/messages"
text = '{"text": "привет"}'
for raw in (text.encode("utf-16"), text.encode("utf-32"), b"\xef\xbb\xbf" + text.encode(), text.encode("cp1251")):
with self.subTest(start=raw[:4]):
headers = {"Idempotency-Key": new_key()}
self.assert_error(call("POST", path, raw=raw, user=user, headers=headers), 400, "invalid_request")
def test_content_type_must_be_json(self):
body = str(user_body()).replace("'", '"').encode()
for content_type in ("text/plain", "application/xml"):
with self.subTest(content_type=content_type):
headers = {"Content-Type": content_type}
self.assert_error(call("POST", "/api/v1/users", raw=body, headers=headers), 400, "invalid_request")
def test_too_large_json(self):
body = b'{"username": "' + b"x" * (2 * 1024 * 1024) + b'"}'
self.assert_error(call("POST", "/api/v1/users", raw=body), 413, "payload_too_large")
def test_deeply_nested_json(self):
self.assert_error(call("POST", "/api/v1/users", raw=b"[" * 100000), 400, "invalid_request")
def test_broken_json_without_token_is_401(self):
# сначала проверяется токен, и только потом разбирается тело
status, _, _ = call("POST", "/api/v1/chats", raw=b"{broken")
self.assertEqual(status, 401)
class UserValidationTest(ErrorFormatTest):
def test_bad_usernames(self):
for username in ("ab", "a" * 33, "Alice", "with space", "имя", "abc\n", ""):
with self.subTest(username=username):
self.assert_error(call("POST", "/api/v1/users", user_body(username=username)), 400, "invalid_request")
def test_bad_display_names(self):
for display_name in ("", " ", "\t\n", "я" * 101, "имя\u0000"):
with self.subTest(display_name=display_name):
result = call("POST", "/api/v1/users", user_body(display_name=display_name))
self.assert_error(result, 400, "invalid_request")
def test_display_name_length_is_counted_in_code_points(self):
# 100 эмодзи это 400 байт в UTF-8, но ровно 100 code points
status, user, _ = call("POST", "/api/v1/users", user_body(display_name="🙂" * 100))
self.assertEqual(status, 201)
self.assertEqual(user["display_name"], "🙂" * 100)
def test_bad_passwords(self):
for password in ("short-11-ch", "x" * 129, " " * 12, "", 123456789012, None):
with self.subTest(password=password):
self.assert_error(call("POST", "/api/v1/users", user_body(password=password)), 400, "invalid_request")
def test_password_length_limits(self):
for password in ("exactly-12-c", "x" * 128, "пароль🙂пароль"):
with self.subTest(length=len(password)):
body = user_body(password=password)
self.assertEqual(call("POST", "/api/v1/users", body)[0], 201)
self.assertEqual(login(body["username"], password)[0], 201)
def test_password_is_never_returned(self):
body = user_body()
status, user, _ = call("POST", "/api/v1/users", body)
self.assertEqual(status, 201)
self.assertEqual(set(user), {"id", "username", "display_name", "created_at"})
_, session, _ = login(body["username"], body["password"])
self.assertNotIn(body["password"], str(session))
def test_duplicate_username(self):
body = user_body(display_name="Первый")
self.assertEqual(call("POST", "/api/v1/users", body)[0], 201)
body["display_name"] = "Второй"
self.assert_error(call("POST", "/api/v1/users", body), 409, "conflict")
class MessageValidationTest(ErrorFormatTest):
@classmethod
def setUpClass(cls):
cls.user = new_user()
cls.chat = new_chat(cls.user)
def test_empty_message(self):
self.assert_error(send(self.user, self.chat, ""), 400, "invalid_request")
def test_whitespace_message(self):
for text in (" ", " \n\t ", "  "):
with self.subTest(text=repr(text)):
self.assert_error(send(self.user, self.chat, text), 400, "invalid_request")
def test_nul_character(self):
self.assert_error(send(self.user, self.chat, "до\u0000после"), 400, "invalid_request")
def test_text_limit(self):
self.assertEqual(send(self.user, self.chat, "ы" * 4000)[0], 201)
self.assert_error(send(self.user, self.chat, "ы" * 4001), 400, "invalid_request")
def test_sender_id_is_not_accepted(self):
path = "/api/v1/chats/" + self.chat["id"] + "/messages"
body = {"text": "привет", "sender_id": self.user["id"]}
headers = {"Idempotency-Key": new_key()}
self.assert_error(call("POST", path, body, user=self.user, headers=headers), 400, "invalid_request")
def test_unicode_is_kept(self):
text = "Привет 👋 你好 مرحبا\nвторая строка"
status, message, _ = send(self.user, self.chat, text)
self.assertEqual(status, 201)
self.assertEqual(message["text"], text)
self.assertEqual(message["sender_id"], self.user["id"])
if __name__ == "__main__":
unittest.main()