forked from mtusi2026/mtusi2026lab4
Добавить вложения: S3, метаданные в PostgreSQL и очистка незавершённых загрузок
This commit is contained in:
@@ -0,0 +1,102 @@
|
||||
"""Маленький HTTP-клиент для собственных тестов. Только стандартная библиотека."""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/")
|
||||
|
||||
|
||||
def call(method, path, body=None, user=None, raw=None, headers=None, base_url=None, timeout=15):
|
||||
"""Возвращает (код ответа, разобранный JSON или None, заголовки)."""
|
||||
request_headers = {"Accept": "application/json"}
|
||||
if body is not None:
|
||||
raw = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||||
if raw is not None:
|
||||
request_headers["Content-Type"] = "application/json"
|
||||
if user is not None:
|
||||
request_headers["Authorization"] = "Bearer " + user["token"]
|
||||
request_headers.update(headers or {})
|
||||
request = urllib.request.Request((base_url or BASE_URL) + path, data=raw, headers=request_headers, method=method)
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
data = response.read()
|
||||
return response.status, json.loads(data) if data else None, response.headers
|
||||
|
||||
|
||||
def basic(username, password):
|
||||
return {"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode("utf-8")).decode()}
|
||||
|
||||
|
||||
def login(username, password, base_url=None):
|
||||
return call("POST", "/api/v1/auth/sessions", headers=basic(username, password), base_url=base_url)
|
||||
|
||||
|
||||
def user_body(**fields):
|
||||
"""Корректное тело регистрации со случайным username; поля можно переопределить."""
|
||||
body = {"username": "u_" + uuid.uuid4().hex[:12], "display_name": "Тест", "password": "Pw-" + uuid.uuid4().hex}
|
||||
body.update(fields)
|
||||
return body
|
||||
|
||||
|
||||
def new_user(prefix="t", base_url=None):
|
||||
"""Регистрирует синтетического пользователя и сразу входит. В словаре id, username, password и token."""
|
||||
body = user_body(username=prefix + "_" + uuid.uuid4().hex[:12], display_name="Тест " + prefix)
|
||||
status, user, _ = call("POST", "/api/v1/users", body, base_url=base_url)
|
||||
assert status == 201, status
|
||||
status, session, _ = login(body["username"], body["password"], base_url=base_url)
|
||||
assert status == 201, status
|
||||
return {**user, "password": body["password"], "token": session["token"], "user": user}
|
||||
|
||||
|
||||
def new_chat(owner, *others, title="Тестовый чат"):
|
||||
member_ids = [owner["id"]] + [other["id"] for other in others]
|
||||
status, chat, _ = call("POST", "/api/v1/chats", {"title": title, "member_ids": member_ids}, user=owner)
|
||||
assert status == 201, status
|
||||
return chat
|
||||
|
||||
|
||||
def new_key():
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
def send(user, chat, text, key=None, attachment_ids=None, **options):
|
||||
headers = {"Idempotency-Key": key or new_key()}
|
||||
body = {"text": text}
|
||||
if attachment_ids is not None:
|
||||
body["attachment_ids"] = attachment_ids
|
||||
return call("POST", "/api/v1/chats/" + chat["id"] + "/messages", body, user=user, headers=headers, **options)
|
||||
|
||||
|
||||
def request_bytes(method, path, user=None, base_url=None, timeout=15):
|
||||
"""Как call, но ответ не разбирается как JSON: для скачивания файлов."""
|
||||
headers = {"Authorization": "Bearer " + user["token"]} if user else {}
|
||||
request = urllib.request.Request((base_url or BASE_URL) + path, headers=headers, method=method)
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=timeout)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
return response.status, response.read(), response.headers
|
||||
|
||||
|
||||
def history(user, chat, limit):
|
||||
"""Обходит всю историю чата страницами по limit, возвращает сообщения и размеры страниц."""
|
||||
items, sizes, cursor = [], [], None
|
||||
while True:
|
||||
path = "/api/v1/chats/%s/messages?limit=%d" % (chat["id"], limit)
|
||||
if cursor:
|
||||
path += "&cursor=" + urllib.parse.quote(cursor, safe="")
|
||||
status, page, _ = call("GET", path, user=user)
|
||||
assert status == 200, status
|
||||
items += page["items"]
|
||||
sizes.append(len(page["items"]))
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
return items, sizes
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Управление стендом для тестов отказов: останавливает и запускает сервисы через docker compose."""
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
|
||||
from api import BASE_URL, call # noqa: E402
|
||||
|
||||
|
||||
def compose(*args):
|
||||
command = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", "compose.test.yaml", *args]
|
||||
result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True)
|
||||
if result.returncode:
|
||||
raise AssertionError("compose %s: %s" % (" ".join(args[:4]), result.stderr[-600:]))
|
||||
return result.stdout
|
||||
|
||||
|
||||
def sql(query):
|
||||
"""Выполняет запрос через psql внутри контейнера postgres и возвращает вывод без оформления."""
|
||||
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", query).strip()
|
||||
|
||||
|
||||
def wait_ready(base_url=BASE_URL, seconds=60):
|
||||
"""Ждёт устойчивой готовности: балансировщик мог ещё не отметить отказ, поэтому одного ответа 200 мало."""
|
||||
deadline = time.monotonic() + seconds
|
||||
stable_since = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
ready = call("GET", "/health/ready", base_url=base_url)[0] == 200
|
||||
except OSError:
|
||||
ready = False
|
||||
if not ready:
|
||||
stable_since = None
|
||||
elif stable_since is None:
|
||||
stable_since = time.monotonic()
|
||||
elif time.monotonic() - stable_since >= 3:
|
||||
return
|
||||
time.sleep(0.3)
|
||||
raise AssertionError("стенд не стал готов за %d секунд" % seconds)
|
||||
|
||||
|
||||
def timed(*args, **kwargs):
|
||||
"""Как call, но первым элементом возвращает время ответа в секундах."""
|
||||
started = time.monotonic()
|
||||
result = call(*args, **kwargs)
|
||||
return (time.monotonic() - started, *result)
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Недоступность Redis и PostgreSQL через балансировщик.
|
||||
|
||||
Пока реплики не признаны неготовыми, 503 отвечает API, потом сам балансировщик. Формат ошибки у них один.
|
||||
"""
|
||||
import unittest
|
||||
|
||||
from stand import call, compose, timed, wait_ready
|
||||
from api import basic, history, new_chat, new_key, new_user, send, user_body
|
||||
|
||||
# таймауты в .env.example: 2 с на Redis, 3 с на ожидание соединения с БД; берём запас
|
||||
MAX_SECONDS = 6
|
||||
|
||||
|
||||
class DependencyDownTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
wait_ready()
|
||||
self.user = new_user("dep")
|
||||
self.chat = new_chat(self.user)
|
||||
send(self.user, self.chat, "сообщение до отказа")
|
||||
self.messages = "/api/v1/chats/" + self.chat["id"] + "/messages"
|
||||
|
||||
def assert_unavailable(self, result):
|
||||
elapsed, status, body, headers = result
|
||||
self.assertEqual(status, 503)
|
||||
self.assertEqual(body["error"]["code"], "unavailable")
|
||||
self.assertEqual(body["error"]["request_id"], headers["X-Request-Id"])
|
||||
self.assertLess(elapsed, MAX_SECONDS)
|
||||
|
||||
def test_redis_down(self):
|
||||
compose("stop", "redis")
|
||||
self.addCleanup(wait_ready)
|
||||
self.addCleanup(compose, "start", "redis")
|
||||
|
||||
# сессию проверить нечем: запрос с настоящим токеном не выполняется, но и чужим не становится
|
||||
self.assert_unavailable(timed("GET", "/api/v1/users/me", user=self.user))
|
||||
self.assert_unavailable(timed("GET", self.messages, user=self.user))
|
||||
key = {"Idempotency-Key": new_key()}
|
||||
self.assert_unavailable(timed("POST", self.messages, {"text": "во время отказа"}, user=self.user, headers=key))
|
||||
# поддельный токен тоже не принимается за действующий
|
||||
self.assert_unavailable(timed("GET", "/api/v1/users/me", user={"token": "x" * 43}))
|
||||
# без токена Redis не нужен: API ответит 401, балансировщик без готовых реплик 503, но не данные
|
||||
self.assertIn(call("GET", "/api/v1/users/me")[0], (401, 503))
|
||||
self.assertIn(call("GET", self.messages)[0], (401, 503))
|
||||
# новую сессию создать негде
|
||||
self.assert_unavailable(timed("POST", "/api/v1/auth/sessions",
|
||||
headers=basic(self.user["username"], self.user["password"])))
|
||||
self.assert_unavailable(timed("GET", "/health/ready"))
|
||||
|
||||
compose("start", "redis")
|
||||
wait_ready()
|
||||
# прежний токен снова работает: сессия пережила остановку Redis, а сообщение во время отказа не записалось
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user=self.user)[1], self.user["user"])
|
||||
self.assertEqual([item["text"] for item in history(self.user, self.chat, 50)[0]], ["сообщение до отказа"])
|
||||
|
||||
def test_uncertain_write_can_be_repeated_with_same_key(self):
|
||||
compose("stop", "postgres")
|
||||
self.addCleanup(wait_ready)
|
||||
self.addCleanup(compose, "start", "postgres")
|
||||
key = new_key()
|
||||
self.assertEqual(send(self.user, self.chat, "важное", key=key)[0], 503)
|
||||
compose("start", "postgres")
|
||||
wait_ready()
|
||||
first = send(self.user, self.chat, "важное", key=key)
|
||||
second = send(self.user, self.chat, "важное", key=key)
|
||||
self.assertEqual((first[0], second[0]), (201, 201))
|
||||
self.assertEqual(first[1], second[1])
|
||||
|
||||
def test_postgres_down(self):
|
||||
compose("stop", "postgres")
|
||||
self.addCleanup(wait_ready)
|
||||
self.addCleanup(compose, "start", "postgres")
|
||||
|
||||
self.assert_unavailable(timed("GET", "/api/v1/users/me", user=self.user))
|
||||
self.assert_unavailable(timed("GET", self.messages, user=self.user))
|
||||
key = {"Idempotency-Key": new_key()}
|
||||
self.assert_unavailable(timed("POST", self.messages, {"text": "во время отказа"}, user=self.user, headers=key))
|
||||
self.assert_unavailable(timed("POST", "/api/v1/users", user_body()))
|
||||
self.assert_unavailable(timed("POST", "/api/v1/auth/sessions",
|
||||
headers=basic(self.user["username"], self.user["password"])))
|
||||
self.assertIn(call("GET", "/api/v1/users/me")[0], (401, 503))
|
||||
self.assert_unavailable(timed("GET", "/health/ready"))
|
||||
|
||||
compose("start", "postgres")
|
||||
wait_ready()
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user=self.user)[1], self.user["user"])
|
||||
self.assertEqual([item["text"] for item in history(self.user, self.chat, 50)[0]], ["сообщение до отказа"])
|
||||
self.assertEqual(send(self.user, self.chat, "после восстановления")[0], 201)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Истечение сессии и ключа. Проверяется на отдельном API с TTL 2 секунды (порт 8081, только compose.test.yaml)."""
|
||||
import datetime
|
||||
import time
|
||||
import unittest
|
||||
|
||||
from stand import call, compose, wait_ready
|
||||
from api import login, new_key, new_user, send
|
||||
|
||||
SHORT_TTL_URL = "http://localhost:8081"
|
||||
|
||||
|
||||
class SessionExpiryTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
compose("up", "-d", "--no-deps", "--no-build", "api-short-ttl")
|
||||
cls.addClassCleanup(compose, "rm", "-sf", "api-short-ttl")
|
||||
wait_ready(SHORT_TTL_URL)
|
||||
|
||||
def test_session_expires(self):
|
||||
user = new_user("ttl", base_url=SHORT_TTL_URL)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user=user, base_url=SHORT_TTL_URL)[0], 200)
|
||||
time.sleep(3)
|
||||
status, body, headers = call("GET", "/api/v1/users/me", user=user, base_url=SHORT_TTL_URL)
|
||||
self.assertEqual(status, 401)
|
||||
self.assertEqual(body["error"]["code"], "unauthorized")
|
||||
self.assertTrue(headers["WWW-Authenticate"].startswith("Bearer "))
|
||||
# logout с истёкшим токеном тоже 401
|
||||
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", user=user, base_url=SHORT_TTL_URL)[0], 401)
|
||||
|
||||
def test_idempotency_key_is_free_after_ttl(self):
|
||||
user = new_user("idem", base_url=SHORT_TTL_URL)
|
||||
chat = new_chat_short(user)
|
||||
key = new_key()
|
||||
first = send(user, chat, "первое", key=key, base_url=SHORT_TTL_URL)
|
||||
repeat = send(user, chat, "первое", key=key, base_url=SHORT_TTL_URL)
|
||||
self.assertEqual((first[0], repeat[1]), (201, first[1]))
|
||||
time.sleep(2.2)
|
||||
# сессия живёт столько же, поэтому нужен новый вход
|
||||
_, session, _ = login(user["username"], user["password"], base_url=SHORT_TTL_URL)
|
||||
again = send({**user, "token": session["token"]}, chat, "уже другое", key=key, base_url=SHORT_TTL_URL)
|
||||
self.assertEqual(again[0], 201)
|
||||
self.assertNotEqual(again[1]["id"], first[1]["id"])
|
||||
|
||||
def test_expires_at_matches_ttl(self):
|
||||
user = new_user("ttl", base_url=SHORT_TTL_URL)
|
||||
_, session, _ = login(user["username"], user["password"], base_url=SHORT_TTL_URL)
|
||||
expires_at = datetime.datetime.fromisoformat(session["expires_at"][:-1] + "+00:00")
|
||||
left = (expires_at - datetime.datetime.now(datetime.timezone.utc)).total_seconds()
|
||||
self.assertTrue(0 < left <= 2.5, left)
|
||||
|
||||
def test_requests_do_not_extend_session(self):
|
||||
user = new_user("ttl", base_url=SHORT_TTL_URL)
|
||||
deadline = time.monotonic() + 3
|
||||
while time.monotonic() < deadline:
|
||||
call("GET", "/api/v1/users/me", user=user, base_url=SHORT_TTL_URL)
|
||||
time.sleep(0.2)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user=user, base_url=SHORT_TTL_URL)[0], 401)
|
||||
|
||||
def test_login_after_expiry(self):
|
||||
user = new_user("ttl", base_url=SHORT_TTL_URL)
|
||||
time.sleep(3)
|
||||
status, session, _ = login(user["username"], user["password"], base_url=SHORT_TTL_URL)
|
||||
self.assertEqual(status, 201)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user=session, base_url=SHORT_TTL_URL)[0], 200)
|
||||
|
||||
|
||||
def new_chat_short(owner):
|
||||
body = {"title": "Чат", "member_ids": [owner["id"]]}
|
||||
status, chat, _ = call("POST", "/api/v1/chats", body, user=owner, base_url=SHORT_TTL_URL)
|
||||
assert status == 201, status
|
||||
return chat
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Согласованность S3 и PostgreSQL, отказ S3, приватность bucket. Состояние после сбоя создаётся настоящими сбоями."""
|
||||
import hashlib
|
||||
import socket
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from stand import BASE_URL, call, compose, sql, timed, wait_ready
|
||||
from api import new_chat, new_user, request_bytes, send
|
||||
from samples import noisy_png
|
||||
|
||||
FIXTURE = (Path(__file__).resolve().parents[2] / "tests/fixtures/sample.png").read_bytes()
|
||||
LIST_OBJECTS = """
|
||||
import config
|
||||
from objects import ObjectStore
|
||||
store = ObjectStore(config.read_secret("S3_JANITOR_ACCESS_KEY_ID"), config.read_secret("S3_JANITOR_SECRET_ACCESS_KEY"))
|
||||
for page in store.pages():
|
||||
for item in page:
|
||||
print(item["Key"])
|
||||
"""
|
||||
PROBE_API_KEY = """
|
||||
import urllib.error, urllib.request
|
||||
import config
|
||||
from botocore.exceptions import ClientError
|
||||
from objects import ObjectStore
|
||||
store = ObjectStore(config.read_secret("S3_ACCESS_KEY_ID"), config.read_secret("S3_SECRET_ACCESS_KEY"))
|
||||
def attempt(name, action):
|
||||
try:
|
||||
action()
|
||||
print(name, "allowed")
|
||||
except ClientError as error:
|
||||
print(name, error.response["Error"]["Code"])
|
||||
attempt("list", lambda: store.client.list_objects_v2(Bucket=store.bucket))
|
||||
attempt("create_bucket", lambda: store.client.create_bucket(Bucket="other"))
|
||||
attempt("delete_bucket", lambda: store.client.delete_bucket(Bucket=store.bucket))
|
||||
try:
|
||||
urllib.request.urlopen(config.S3_ENDPOINT + "/" + store.bucket + "/KEY", timeout=3)
|
||||
print("anonymous allowed")
|
||||
except urllib.error.HTTPError as error:
|
||||
print("anonymous", error.code)
|
||||
"""
|
||||
|
||||
|
||||
def upload(user, chat, data=FIXTURE):
|
||||
path = "/api/v1/chats/%s/attachments?filename=a.png" % chat["id"]
|
||||
return call("POST", path, raw=data, user=user, headers={"Content-Type": "image/png"}, timeout=30)
|
||||
|
||||
|
||||
def bucket_keys():
|
||||
return set(compose("exec", "-T", "janitor", "python", "-c", LIST_OBJECTS).split())
|
||||
|
||||
|
||||
def janitor(*args):
|
||||
# логи идут в stdout контейнера; сервисный проход на проверки не влияет: его период 300 секунд
|
||||
return compose("run", "--rm", "--no-deps", "janitor", "python", "janitor.py", "--once", *args)
|
||||
|
||||
|
||||
def in_api_replica(*command):
|
||||
# по id контейнера: после поочерёдной замены номера реплик уже не 1 и 2
|
||||
container = compose("ps", "-q", "api").split()[0]
|
||||
return subprocess.run(["docker", "exec", container, *command], check=True, capture_output=True, text=True).stdout
|
||||
|
||||
|
||||
def wait_s3(seconds=90):
|
||||
"""S3 готов, когда контейнер healthy: том данных зарегистрирован, запросы к объектам не зависают."""
|
||||
deadline = time.monotonic() + seconds
|
||||
while compose("ps", "--format", "{{.Health}}", "s3").strip() != "healthy":
|
||||
if time.monotonic() > deadline:
|
||||
raise AssertionError("S3 не стал готов за %d секунд" % seconds)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def rows(condition="true"):
|
||||
output = sql("SELECT object_key FROM attachments WHERE " + condition)
|
||||
return set(output.split())
|
||||
|
||||
|
||||
class StorageBase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
wait_ready()
|
||||
self.user = new_user("files")
|
||||
self.chat = new_chat(self.user)
|
||||
|
||||
def tearDown(self):
|
||||
# следы теста не должны влиять на сверку в следующем; S3 к этому моменту может ещё запускаться
|
||||
compose("start", "s3")
|
||||
wait_s3()
|
||||
janitor("--grace-seconds", "0")
|
||||
|
||||
|
||||
class ConsistencyTest(StorageBase):
|
||||
def test_commit_lost_after_s3_put(self):
|
||||
"""Состояние как после падения между PUT и подтверждением: объект есть, строка pending, клиент не получил 201."""
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
key = "originals/" + attachment["id"]
|
||||
self.assertIn(key, bucket_keys())
|
||||
sql("UPDATE attachments SET state = 'pending', created_at = now() - interval '2 hours' WHERE id = '%s'" % attachment["id"])
|
||||
|
||||
janitor()
|
||||
|
||||
self.assertNotIn(key, bucket_keys())
|
||||
self.assertEqual(rows("id = '%s'" % attachment["id"]), set())
|
||||
|
||||
def test_grace_period_protects_fresh_upload(self):
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
key = "originals/" + attachment["id"]
|
||||
sql("UPDATE attachments SET state = 'pending' WHERE id = '%s'" % attachment["id"])
|
||||
|
||||
janitor() # строка свежая, загрузка могла ещё идти
|
||||
|
||||
self.assertIn(key, bucket_keys())
|
||||
self.assertEqual(rows("id = '%s'" % attachment["id"]), {key})
|
||||
# строку не видно снаружи, пока она pending
|
||||
self.assertEqual(call("GET", "/api/v1/attachments/" + attachment["id"], user=self.user)[0], 404)
|
||||
|
||||
def test_ready_attachments_are_never_removed(self):
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
janitor("--grace-seconds", "0")
|
||||
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
|
||||
self.assertEqual((status, body), (200, FIXTURE))
|
||||
|
||||
def test_orphan_object_is_removed_only_after_grace(self):
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
key = "originals/" + attachment["id"]
|
||||
sql("DELETE FROM attachments WHERE id = '%s'" % attachment["id"])
|
||||
|
||||
janitor()
|
||||
self.assertIn(key, bucket_keys())
|
||||
|
||||
janitor("--grace-seconds", "0", "--dry-run")
|
||||
self.assertIn(key, bucket_keys())
|
||||
|
||||
janitor("--grace-seconds", "0")
|
||||
self.assertNotIn(key, bucket_keys())
|
||||
|
||||
def test_second_pass_changes_nothing(self):
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
sql("UPDATE attachments SET state = 'pending', created_at = now() - interval '2 hours' WHERE id = '%s'" % attachment["id"])
|
||||
janitor()
|
||||
before = bucket_keys(), rows()
|
||||
janitor()
|
||||
self.assertEqual((bucket_keys(), rows()), before)
|
||||
|
||||
def test_interrupted_upload_leaves_nothing(self):
|
||||
before = bucket_keys(), rows()
|
||||
address = urllib.parse.urlsplit(BASE_URL)
|
||||
path = "/api/v1/chats/%s/attachments?filename=a.png" % self.chat["id"]
|
||||
head = ("POST %s HTTP/1.1\r\nHost: x\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n"
|
||||
"Authorization: Bearer %s\r\n\r\n" % (path, self.user["token"])).encode()
|
||||
for _ in range(4):
|
||||
with socket.create_connection((address.hostname, address.port), timeout=5) as connection:
|
||||
connection.sendall(head + b"\x89PNG" + b"x" * 200000)
|
||||
time.sleep(0.2)
|
||||
time.sleep(1)
|
||||
self.assertEqual((bucket_keys(), rows()), before)
|
||||
|
||||
def test_postgres_lost_during_parallel_uploads(self):
|
||||
"""Настоящий сбой: база пропадает, пока идут загрузки. Часть объектов попадает в S3 без подтверждения."""
|
||||
data = noisy_png(3 * 1024 * 1024)
|
||||
# подтверждённые до сбоя файлы: они обязаны уцелеть при любом исходе гонки ниже
|
||||
confirmed_before = [upload(self.user, self.chat, data)[1] for _ in range(3)]
|
||||
before_keys = bucket_keys()
|
||||
barrier = threading.Barrier(13)
|
||||
|
||||
def post(_):
|
||||
barrier.wait()
|
||||
confirmed = []
|
||||
for _ in range(6):
|
||||
try:
|
||||
status, attachment, _ = upload(self.user, self.chat, data)
|
||||
except OSError:
|
||||
continue
|
||||
if status == 201:
|
||||
confirmed.append(attachment)
|
||||
return confirmed
|
||||
|
||||
with ThreadPoolExecutor(max_workers=12) as pool:
|
||||
jobs = [pool.submit(post, number) for number in range(12)]
|
||||
barrier.wait()
|
||||
time.sleep(0.25)
|
||||
compose("kill", "postgres")
|
||||
confirmed = confirmed_before + [item for job in jobs for item in job.result()]
|
||||
compose("start", "postgres")
|
||||
self.addCleanup(wait_ready)
|
||||
wait_ready()
|
||||
|
||||
leftovers = bucket_keys() - before_keys - {"originals/" + item["id"] for item in confirmed}
|
||||
janitor("--grace-seconds", "0")
|
||||
|
||||
self.assertEqual(rows("state = 'pending'"), set())
|
||||
self.assertEqual(bucket_keys(), rows("state = 'ready'"))
|
||||
self.assertGreaterEqual(len(confirmed), 3)
|
||||
for attachment in confirmed:
|
||||
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
|
||||
self.assertEqual((status, hashlib.sha256(body).hexdigest()), (200, attachment["sha256"]))
|
||||
print("\nподтверждено %d, в S3 без подтверждения %d, после очистки расхождений нет" % (len(confirmed), len(leftovers)))
|
||||
|
||||
|
||||
class S3DownTest(StorageBase):
|
||||
def test_s3_down(self):
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
compose("stop", "s3")
|
||||
self.addCleanup(wait_ready)
|
||||
self.addCleanup(compose, "start", "s3")
|
||||
|
||||
path = "/api/v1/chats/%s/attachments?filename=a.png" % self.chat["id"]
|
||||
elapsed, status, body, headers = timed("POST", path, raw=FIXTURE, user=self.user, headers={"Content-Type": "image/png"})
|
||||
self.assertEqual((status, body["error"]["code"]), (503, "unavailable"))
|
||||
self.assertEqual(body["error"]["request_id"], headers["X-Request-Id"])
|
||||
self.assertLess(elapsed, 10)
|
||||
self.assertEqual(request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)[0], 503)
|
||||
|
||||
# без S3 мессенджер продолжает работать: текст, метаданные и готовность реплик от него не зависят
|
||||
self.assertEqual(send(self.user, self.chat, "текст без S3")[0], 201)
|
||||
self.assertEqual(call("GET", "/api/v1/attachments/" + attachment["id"], user=self.user)[0], 200)
|
||||
self.assertEqual(call("GET", "/health/ready")[0], 200)
|
||||
|
||||
compose("start", "s3")
|
||||
wait_s3()
|
||||
status, body, _ = request_bytes("GET", "/api/v1/attachments/%s/content" % attachment["id"], user=self.user)
|
||||
self.assertEqual((status, body), (200, FIXTURE))
|
||||
|
||||
def test_failed_upload_is_cleaned(self):
|
||||
compose("stop", "s3")
|
||||
self.addCleanup(wait_ready)
|
||||
self.addCleanup(compose, "start", "s3")
|
||||
self.assertEqual(upload(self.user, self.chat)[0], 503)
|
||||
self.assertTrue(rows("state = 'pending'")) # строка осталась, клиент получил ошибку
|
||||
compose("start", "s3")
|
||||
wait_s3()
|
||||
janitor("--grace-seconds", "0")
|
||||
self.assertEqual(rows("state = 'pending'"), set())
|
||||
|
||||
|
||||
class PrivateBucketTest(StorageBase):
|
||||
def test_bucket_is_private_and_api_key_is_limited(self):
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
script = PROBE_API_KEY.replace("/KEY", "/originals/" + attachment["id"])
|
||||
output = in_api_replica("python", "-c", script)
|
||||
self.assertIn("list AccessDenied", output)
|
||||
self.assertIn("create_bucket AccessDenied", output)
|
||||
self.assertIn("delete_bucket AccessDenied", output)
|
||||
self.assertIn("anonymous 403", output)
|
||||
|
||||
def test_bucket_bootstrap_can_be_repeated(self):
|
||||
before = bucket_keys()
|
||||
output = compose("run", "--rm", "--no-deps", "s3-init")
|
||||
self.assertIn("bucket уже есть", output)
|
||||
self.assertIn("пробная запись прошла", output)
|
||||
self.assertEqual(bucket_keys(), before)
|
||||
|
||||
def test_no_file_on_api_replica(self):
|
||||
attachment = upload(self.user, self.chat)[1]
|
||||
output = in_api_replica("sh", "-c", "find / -xdev -name '*%s*' 2>/dev/null | wc -l" % attachment["id"])
|
||||
self.assertEqual(output.split()[-1], "0")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Проверки на уровне БД: откат составной операции, гонка регистрации, повторный запуск миграций."""
|
||||
import threading
|
||||
import unittest
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from stand import call, compose, sql, wait_ready
|
||||
from api import new_chat, new_key, new_user, send, user_body
|
||||
|
||||
|
||||
class TransactionTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
wait_ready()
|
||||
cls.alice, cls.bob = new_user("alice"), new_user("bob")
|
||||
|
||||
def create_chat(self, title, member_ids):
|
||||
return call("POST", "/api/v1/chats", {"title": title, "member_ids": member_ids}, user=self.alice)
|
||||
|
||||
def test_chat_with_unknown_member_is_rolled_back(self):
|
||||
title = "rollback-" + uuid.uuid4().hex
|
||||
# Алиса и Боб существуют, третьего участника нет: вставка чата и двух участников уже выполнена,
|
||||
# на третьем срабатывает внешний ключ, и транзакция должна откатить всё
|
||||
status, body, _ = self.create_chat(title, [self.alice["id"], self.bob["id"], str(uuid.uuid4())])
|
||||
self.assertEqual(status, 400)
|
||||
self.assertEqual(body["error"]["code"], "invalid_request")
|
||||
self.assertEqual(sql("SELECT count(*) FROM chats WHERE title = '%s'" % title), "0")
|
||||
self.assertEqual(sql("SELECT count(*) FROM chat_members m LEFT JOIN chats c ON c.id = m.chat_id WHERE c.id IS NULL"), "0")
|
||||
|
||||
def test_valid_chat_is_committed_whole(self):
|
||||
title = "commit-" + uuid.uuid4().hex
|
||||
status, chat, _ = self.create_chat(title, [self.alice["id"], self.bob["id"]])
|
||||
self.assertEqual(status, 201)
|
||||
self.assertEqual(sql("SELECT count(*) FROM chats WHERE title = '%s'" % title), "1")
|
||||
self.assertEqual(sql("SELECT count(*) FROM chat_members WHERE chat_id = '%s'" % chat["id"]), "2")
|
||||
|
||||
def test_username_race_creates_one_row(self):
|
||||
body = user_body(username="race_" + uuid.uuid4().hex[:8])
|
||||
barrier = threading.Barrier(10)
|
||||
|
||||
def register(_):
|
||||
barrier.wait()
|
||||
return call("POST", "/api/v1/users", body)[0]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
statuses = sorted(pool.map(register, range(10)))
|
||||
self.assertEqual(statuses, [201] + [409] * 9)
|
||||
self.assertEqual(sql("SELECT count(*) FROM users WHERE username = '%s'" % body["username"]), "1")
|
||||
|
||||
def test_same_key_in_parallel_creates_one_row(self):
|
||||
chat = new_chat(self.alice)
|
||||
key = new_key()
|
||||
barrier = threading.Barrier(20)
|
||||
|
||||
def post(_):
|
||||
barrier.wait()
|
||||
return send(self.alice, chat, "один раз", key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=20) as pool:
|
||||
results = list(pool.map(post, range(20)))
|
||||
self.assertEqual({status for status, _, _ in results}, {201})
|
||||
self.assertEqual(len({message["id"] for _, message, _ in results}), 1)
|
||||
self.assertEqual(sql("SELECT count(*) FROM messages WHERE chat_id = '%s'" % chat["id"]), "1")
|
||||
self.assertEqual(sql("SELECT count(*) FROM idempotency_keys WHERE chat_id = '%s'" % chat["id"]), "1")
|
||||
|
||||
def test_failed_message_leaves_no_key(self):
|
||||
# чужой чат: сообщение не создано, и ключ тоже не занят
|
||||
eve = new_user("eve")
|
||||
chat = new_chat(self.alice)
|
||||
key = new_key()
|
||||
self.assertEqual(send(eve, chat, "чужое", key=key)[0], 404)
|
||||
self.assertEqual(sql("SELECT count(*) FROM idempotency_keys WHERE chat_id = '%s'" % chat["id"]), "0")
|
||||
|
||||
def test_password_is_stored_as_argon2id_hash(self):
|
||||
stored = sql("SELECT password_hash FROM users WHERE id = '%s'" % self.alice["id"])
|
||||
self.assertTrue(stored.startswith("$argon2id$"), stored[:12])
|
||||
self.assertNotIn(self.alice["password"], stored)
|
||||
|
||||
def test_migrations_can_be_repeated(self):
|
||||
before = sql("SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM chats), (SELECT count(*) FROM messages)")
|
||||
output = compose("run", "--rm", "migrate")
|
||||
self.assertIn("уже применена: 0001_init.sql", output)
|
||||
self.assertIn("уже применена: 0003_attachments.sql", output)
|
||||
self.assertEqual(sql("SELECT count(*) FROM schema_migrations"), "3")
|
||||
after = sql("SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM chats), (SELECT count(*) FROM messages)")
|
||||
self.assertEqual(before, after)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Настоящие маленькие изображения, созданы Pillow, и сборка синтетических PNG."""
|
||||
import base64
|
||||
import struct
|
||||
import zlib
|
||||
from random import Random
|
||||
|
||||
REAL_IMAGES = {
|
||||
"jpeg_baseline": ("image/jpeg", base64.b64decode(
|
||||
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAoHBwgHBgoICAgLCgoLDhgQDg0NDh0VFhEYIx8lJCIfIiEmKzcvJik0KSEiMEExNDk7"
|
||||
"Pj4+JS5ESUM8SDc9Pjv/2wBDAQoLCw4NDhwQEBw7KCIoOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7"
|
||||
"Ozs7Ozs7Ozv/wAARCAAQABgDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUF"
|
||||
"BAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVW"
|
||||
"V1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi"
|
||||
"4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAEC"
|
||||
"AxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVm"
|
||||
"Z2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq"
|
||||
"8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDz220fp8ta9to/T5a6W20fp8ta9to/T5a5YYg4sBm22pzNto/T5aK7y20fp8tFdKxGh9nQ"
|
||||
"zb3Nz//Z"
|
||||
)),
|
||||
"jpeg_progressive": ("image/jpeg", base64.b64decode(
|
||||
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAoHBwgHBgoICAgLCgoLDhgQDg0NDh0VFhEYIx8lJCIfIiEmKzcvJik0KSEiMEExNDk7"
|
||||
"Pj4+JS5ESUM8SDc9Pjv/2wBDAQoLCw4NDhwQEBw7KCIoOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7"
|
||||
"Ozs7Ozs7Ozv/wgARCAAQABgDASIAAhEBAxEB/8QAFwAAAwEAAAAAAAAAAAAAAAAAAAQFBv/EABUBAQEAAAAAAAAAAAAAAAAAAAMF"
|
||||
"/9oADAMBAAIQAxAAAAHPN02yCYXhLX//xAAWEAADAAAAAAAAAAAAAAAAAAAAAgP/2gAIAQEAAQUCWIsRYixFiLE//8QAGBEAAgMA"
|
||||
"AAAAAAAAAAAAAAAAAAQCAyH/2gAIAQMBAT8BobIN4f/EABcRAQADAAAAAAAAAAAAAAAAAAABAxP/2gAIAQIBAT8Bixo//8QAFBAB"
|
||||
"AAAAAAAAAAAAAAAAAAAAIP/aAAgBAQAGPwJf/8QAFRABAQAAAAAAAAAAAAAAAAAAAGH/2gAIAQEAAT8hgggggg//2gAMAwEAAgAD"
|
||||
"AAAAEFfv/8QAFxEBAAMAAAAAAAAAAAAAAAAAACExUf/aAAgBAwEBPxCqW1//xAAWEQEBAQAAAAAAAAAAAAAAAAAAASH/2gAIAQIB"
|
||||
"AT8QUw//xAAVEAEBAAAAAAAAAAAAAAAAAAAAMf/aAAgBAQABPxCRIkSJEj//2Q=="
|
||||
)),
|
||||
"jpeg_gray": ("image/jpeg", base64.b64decode(
|
||||
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAx"
|
||||
"NDQ0Hyc5PTgyPC4zNDL/wAALCAAQABgBAREA/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUF"
|
||||
"BAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVW"
|
||||
"V1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi"
|
||||
"4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APLLTQ+nyVu2mh9PkrctND6fJWlaaH0+St200Pp8lblpofT5K//Z"
|
||||
)),
|
||||
"jpeg_cmyk": ("image/jpeg", base64.b64decode(
|
||||
"/9j/7gAOQWRvYmUAZAAAAAAA/9sAQwAIBgYHBgUIBwcHCQkICgwUDQwLCwwZEhMPFB0aHx4dGhwcICQuJyAiLCMcHCg3KSwwMTQ0"
|
||||
"NB8nOT04MjwuMzQy/8AAFAgAEAAYBEMRAE0RAFkRAEsRAP/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIB"
|
||||
"AwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZH"
|
||||
"SElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV"
|
||||
"1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/aAA4EQwBNAFkASwAAPwDyLQf4a8z/AOER/wCmf6V5FYWHTivf69Q0H+Gj/hEf+mf6"
|
||||
"V1FhYdOKK9Q0H+Gj/hEf+mf6V1FhYdOKK+X9B/hr3n/hEf8Apn+lcvYWHTiivUNB/ho/4RH/AKZ/pXUWFh04or1DQf4aP+ER/wCm"
|
||||
"f6V1FhYdOKK//9k="
|
||||
)),
|
||||
"png_rgb": ("image/png", base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAABgAAAAQCAIAAACDRijCAAAAIklEQVR4nGNkYGDgYmClHLEw8LMyMFABjRo0atCoQSPVIACIwAmA"
|
||||
"4cyUgAAAAABJRU5ErkJggg=="
|
||||
)),
|
||||
"png_rgba": ("image/png", base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAABgAAAAQCAYAAAAMJL+VAAAAKUlEQVR4nGNkYGD4z8XAykArzMLAz8rAwEA7PGrBqAWjFoxaMGoB"
|
||||
"NTAAUr4Kf23u9hgAAAAASUVORK5CYII="
|
||||
)),
|
||||
"png_palette": ("image/png", base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAABgAAAAQCAMAAAA7+k+nAAACplBMVEUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAz"
|
||||
"AABmAACZAADMAAD/AAAAMwAzMwBmMwCZMwDMMwD/MwAAZgAzZgBmZgCZZgDMZgD/ZgAAmQAzmQBmmQCZmQDMmQD/mQAAzAAzzABm"
|
||||
"zACZzADMzAD/zAAA/wAz/wBm/wCZ/wDM/wD//wAAADMzADNmADOZADPMADP/ADMAMzMzMzNmMzOZMzPMMzP/MzMAZjMzZjNmZjOZ"
|
||||
"ZjPMZjP/ZjMAmTMzmTNmmTOZmTPMmTP/mTMAzDMzzDNmzDOZzDPMzDP/zDMA/zMz/zNm/zOZ/zPM/zP//zMAAGYzAGZmAGaZAGbM"
|
||||
"AGb/AGYAM2YzM2ZmM2aZM2bMM2b/M2YAZmYzZmZmZmaZZmbMZmb/ZmYAmWYzmWZmmWaZmWbMmWb/mWYAzGYzzGZmzGaZzGbMzGb/"
|
||||
"zGYA/2Yz/2Zm/2aZ/2bM/2b//2YAAJkzAJlmAJmZAJnMAJn/AJkAM5kzM5lmM5mZM5nMM5n/M5kAZpkzZplmZpmZZpnMZpn/ZpkA"
|
||||
"mZkzmZlmmZmZmZnMmZn/mZkAzJkzzJlmzJmZzJnMzJn/zJkA/5kz/5lm/5mZ/5nM/5n//5kAAMwzAMxmAMyZAMzMAMz/AMwAM8wz"
|
||||
"M8xmM8yZM8zMM8z/M8wAZswzZsxmZsyZZszMZsz/ZswAmcwzmcxmmcyZmczMmcz/mcwAzMwzzMxmzMyZzMzMzMz/zMwA/8wz/8xm"
|
||||
"/8yZ/8zM/8z//8wAAP8zAP9mAP+ZAP/MAP//AP8AM/8zM/9mM/+ZM//MM///M/8AZv8zZv9mZv+ZZv/MZv//Zv8Amf8zmf9mmf+Z"
|
||||
"mf/Mmf//mf8AzP8zzP9mzP+ZzP/MzP//zP8A//8z//9m//+Z///M//////91g91EAAABH0lEQVR4nC3LMW6DMBTGce8ssOQaBAkT"
|
||||
"KVAJOsAFSiIZqmJUoMJGart2IUOcwVTC7pBzdMnU3q3P0G/8/95DCCHLtVx3a+Z5uZcfzI7QXQRgbYG8By/PvYOhD2TbNnYwDnAQ"
|
||||
"BGRHdkVRwsYjsn3Hx7C1F9BBxvIE4DsrkMDcGxnL8RH5G+yHGEeYRIRQUtCiFOVYn9Bmvw/NoqeouqsopfW5FrW4IOhGIMOomaBC"
|
||||
"XATA8lGFVVRRkDPkWohPFMdJnDRJ05p1nezki5ymyUCcGLgH6lro3TT9Q/KcNE27vEiQSa6Qxn3WZKzhvOWqVYNUWmr9rVHaZz1j"
|
||||
"jDPOZ64GpZSG3b4AUpYZgT4PM3Slbwu8sv4N4J3N/Ar9CvKjb79/rgyKjTEBaRcAAAAASUVORK5CYII="
|
||||
)),
|
||||
"png_1bit": ("image/png", base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAABgAAAAQAQAAAAAkX4I4AAAAS0lEQVR4nAFAAL//AAAAAgAAEqkBAkLgARABRAEFRQAES9UEAQVM"
|
||||
"BQJN/P8BCRwwAaoLuQKo9UcC2AMmAaoMAQKr9bcAqt37AV8Y6K/aEe5QdGjgAAAAAElFTkSuQmCC"
|
||||
)),
|
||||
}
|
||||
|
||||
|
||||
def png_chunk(kind, data):
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
|
||||
|
||||
|
||||
def make_png(width, height, raw=None):
|
||||
"""PNG RGB с верными CRC. Без raw пиксели настоящие, с raw можно подложить любые данные."""
|
||||
if raw is None:
|
||||
row = b"\x00" + bytes(i % 251 for i in range(width * 3))
|
||||
raw = row * height
|
||||
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", header) + png_chunk(b"IDAT", zlib.compress(raw)) + png_chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def noisy_png(size):
|
||||
"""Настоящий PNG ровно из size байт. Случайные пиксели не сжимаются, поэтому файл действительно большой."""
|
||||
width = 1000
|
||||
row = 1 + width * 3
|
||||
height = (size - size // 500 - 200) // row
|
||||
random = Random(size)
|
||||
raw = b"".join(b"\x00" + random.randbytes(width * 3) for _ in range(height))
|
||||
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
start = b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", header) + png_chunk(b"IDAT", zlib.compress(raw, 1))
|
||||
end = png_chunk(b"IEND", b"")
|
||||
# остаток добирается служебным блоком: он не меняет изображение
|
||||
padding = size - len(start) - len(end) - 12
|
||||
assert padding >= 0, "образец больше запрошенного размера"
|
||||
return start + png_chunk(b"tEXt", b"x" * padding) + end
|
||||
@@ -0,0 +1,85 @@
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from api import call, new_chat, new_user, send
|
||||
|
||||
|
||||
class ForeignChatTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.alice, cls.bob, cls.eve = new_user("alice"), new_user("bob"), new_user("eve")
|
||||
cls.chat = new_chat(cls.alice, cls.bob)
|
||||
cls.path = "/api/v1/chats/" + cls.chat["id"]
|
||||
send(cls.alice, cls.chat, "секрет")
|
||||
|
||||
def test_outsider_gets_404_everywhere(self):
|
||||
self.assertEqual(call("GET", self.path, user=self.eve)[0], 404)
|
||||
self.assertEqual(call("GET", self.path + "/messages", user=self.eve)[0], 404)
|
||||
status, body, _ = send(self.eve, self.chat, "я тут")
|
||||
self.assertEqual(status, 404)
|
||||
self.assertEqual(body["error"]["code"], "not_found")
|
||||
|
||||
def test_foreign_chat_looks_like_missing_chat(self):
|
||||
_, foreign, _ = call("GET", self.path, user=self.eve)
|
||||
_, missing, _ = call("GET", "/api/v1/chats/" + str(uuid.uuid4()), user=self.eve)
|
||||
self.assertEqual(foreign["error"]["message"], missing["error"]["message"])
|
||||
|
||||
def test_outsider_message_is_not_saved(self):
|
||||
send(self.eve, self.chat, "не должно появиться")
|
||||
_, page, _ = call("GET", self.path + "/messages", user=self.alice)
|
||||
self.assertNotIn("не должно появиться", [item["text"] for item in page["items"]])
|
||||
|
||||
def test_chat_is_not_listed_for_outsider(self):
|
||||
_, page, _ = call("GET", "/api/v1/chats?limit=100", user=self.eve)
|
||||
self.assertNotIn(self.chat["id"], [item["id"] for item in page["items"]])
|
||||
_, page, _ = call("GET", "/api/v1/chats?limit=100", user=self.bob)
|
||||
self.assertIn(self.chat["id"], [item["id"] for item in page["items"]])
|
||||
|
||||
def test_members_can_read_and_write(self):
|
||||
self.assertEqual(send(self.bob, self.chat, "ответ")[0], 201)
|
||||
self.assertEqual(call("GET", self.path, user=self.bob)[0], 200)
|
||||
|
||||
def test_bad_chat_id(self):
|
||||
chat_id = self.chat["id"]
|
||||
for bad in ("123", chat_id.replace("-", ""), "urn:uuid:" + chat_id):
|
||||
with self.subTest(chat_id=bad):
|
||||
self.assertEqual(call("GET", "/api/v1/chats/" + bad, user=self.alice)[0], 400)
|
||||
self.assertEqual(call("GET", "/api/v1/chats/" + bad + "/messages", user=self.alice)[0], 400)
|
||||
|
||||
def test_member_id_must_be_canonical_uuid(self):
|
||||
body = {"title": "Чат", "member_ids": ["urn:uuid:" + self.alice["id"]]}
|
||||
self.assertEqual(call("POST", "/api/v1/chats", body, user=self.alice)[0], 400)
|
||||
|
||||
|
||||
class CreateChatTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.alice, cls.bob = new_user("alice"), new_user("bob")
|
||||
|
||||
def create(self, member_ids, title="Чат"):
|
||||
return call("POST", "/api/v1/chats", {"title": title, "member_ids": member_ids}, user=self.alice)[0]
|
||||
|
||||
def test_creator_must_be_member(self):
|
||||
self.assertEqual(self.create([self.bob["id"]]), 400)
|
||||
|
||||
def test_duplicate_members(self):
|
||||
self.assertEqual(self.create([self.alice["id"], self.bob["id"], self.bob["id"]]), 400)
|
||||
|
||||
def test_unknown_member(self):
|
||||
self.assertEqual(self.create([self.alice["id"], str(uuid.uuid4())]), 400)
|
||||
|
||||
def test_empty_members(self):
|
||||
self.assertEqual(self.create([]), 400)
|
||||
|
||||
def test_blank_title(self):
|
||||
self.assertEqual(self.create([self.alice["id"]], title=" "), 400)
|
||||
|
||||
def test_failed_create_leaves_no_chat(self):
|
||||
_, before, _ = call("GET", "/api/v1/chats?limit=100", user=self.bob)
|
||||
self.create([self.alice["id"], self.bob["id"], str(uuid.uuid4())])
|
||||
_, after, _ = call("GET", "/api/v1/chats?limit=100", user=self.bob)
|
||||
self.assertEqual(before, after)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,314 @@
|
||||
import hashlib
|
||||
import socket
|
||||
import unittest
|
||||
import urllib.parse
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from api import BASE_URL, call, history, new_chat, new_key, new_user, request_bytes, send
|
||||
from samples import REAL_IMAGES, make_png, noisy_png
|
||||
|
||||
FIXTURE = (Path(__file__).resolve().parents[1] / "tests/fixtures/sample.png").read_bytes()
|
||||
MIB = 1024 * 1024
|
||||
|
||||
|
||||
def upload(user, chat, data, content_type="image/png", filename="photo.png", headers=None):
|
||||
path = "/api/v1/chats/%s/attachments" % chat["id"]
|
||||
if filename is not None:
|
||||
path += "?filename=" + urllib.parse.quote(filename, safe="")
|
||||
return call("POST", path, raw=data, user=user, headers={"Content-Type": content_type, **(headers or {})})
|
||||
|
||||
|
||||
def content(user, attachment_id, query=""):
|
||||
return request_bytes("GET", "/api/v1/attachments/%s/content%s" % (attachment_id, query), user=user)
|
||||
|
||||
|
||||
class Base(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.alice, cls.bob, cls.eve = new_user("alice"), new_user("bob"), new_user("eve")
|
||||
cls.chat = new_chat(cls.alice, cls.bob)
|
||||
|
||||
def assert_error(self, result, status, code):
|
||||
got, body, headers = result
|
||||
self.assertEqual(got, status)
|
||||
self.assertEqual(body["error"]["code"], code)
|
||||
self.assertEqual(body["error"]["request_id"], headers["X-Request-Id"])
|
||||
|
||||
|
||||
class UploadTest(Base):
|
||||
def test_upload_and_download_real_images(self):
|
||||
for name, (content_type, data) in REAL_IMAGES.items():
|
||||
with self.subTest(name):
|
||||
status, attachment, _ = upload(self.alice, self.chat, data, content_type, name)
|
||||
self.assertEqual(status, 201)
|
||||
self.assertEqual(attachment["content_type"], content_type)
|
||||
self.assertEqual(attachment["sha256"], hashlib.sha256(data).hexdigest())
|
||||
got, body, headers = content(self.bob, attachment["id"])
|
||||
self.assertEqual(got, 200)
|
||||
self.assertEqual(body, data)
|
||||
self.assertEqual(headers.get_content_type(), content_type)
|
||||
|
||||
def test_attachment_fields(self):
|
||||
status, attachment, _ = upload(self.alice, self.chat, FIXTURE, filename="отчёт 👋.png")
|
||||
self.assertEqual(status, 201)
|
||||
self.assertEqual(attachment["status"], "ready")
|
||||
self.assertEqual(attachment["variants"], [])
|
||||
self.assertIsNone(attachment["error_code"])
|
||||
self.assertEqual(attachment["filename"], "отчёт 👋.png")
|
||||
self.assertEqual(attachment["owner_id"], self.alice["id"])
|
||||
self.assertEqual(attachment["size_bytes"], len(FIXTURE))
|
||||
self.assertEqual(call("GET", "/api/v1/attachments/" + attachment["id"], user=self.bob)[1], attachment)
|
||||
|
||||
def test_download_headers(self):
|
||||
_, attachment, _ = upload(self.alice, self.chat, FIXTURE, filename="отчёт.png")
|
||||
_, _, headers = content(self.alice, attachment["id"])
|
||||
self.assertEqual(headers["X-Content-Type-Options"], "nosniff")
|
||||
self.assertIn("filename*=UTF-8''", headers["Content-Disposition"])
|
||||
self.assertEqual(headers["Content-Length"], str(len(FIXTURE)))
|
||||
|
||||
def test_file_of_exactly_10_mib(self):
|
||||
data = noisy_png(10 * MIB)
|
||||
self.assertEqual(len(data), 10 * MIB)
|
||||
status, attachment, _ = upload(self.alice, self.chat, data)
|
||||
self.assertEqual(status, 201)
|
||||
got, body, _ = content(self.bob, attachment["id"])
|
||||
self.assertEqual((got, hashlib.sha256(body).hexdigest()), (200, attachment["sha256"]))
|
||||
|
||||
def test_one_byte_over_10_mib(self):
|
||||
self.assert_error(upload(self.alice, self.chat, noisy_png(10 * MIB + 1)), 413, "payload_too_large")
|
||||
|
||||
|
||||
class LimitsTest(Base):
|
||||
def test_too_large_with_length(self):
|
||||
self.assert_error(upload(self.alice, self.chat, b"\x89PNG" + b"x" * (10 * MIB)), 413, "payload_too_large")
|
||||
|
||||
def test_too_large_for_balancer(self):
|
||||
self.assert_error(upload(self.alice, self.chat, b"x" * (12 * MIB)), 413, "payload_too_large")
|
||||
|
||||
def test_too_large_chunked(self):
|
||||
# без Content-Length размер известен только по ходу чтения
|
||||
address = urllib.parse.urlsplit(BASE_URL)
|
||||
path = "/api/v1/chats/%s/attachments?filename=a.png" % self.chat["id"]
|
||||
head = ("POST %s HTTP/1.1\r\nHost: x\r\nContent-Type: image/png\r\nAuthorization: Bearer %s\r\n"
|
||||
"Transfer-Encoding: chunked\r\n\r\n" % (path, self.alice["token"])).encode()
|
||||
with socket.create_connection((address.hostname, address.port), timeout=15) as connection:
|
||||
connection.sendall(head)
|
||||
block = b"x" * MIB
|
||||
try:
|
||||
for _ in range(12):
|
||||
connection.sendall(b"100000\r\n" + block + b"\r\n")
|
||||
connection.sendall(b"0\r\n\r\n")
|
||||
except OSError:
|
||||
pass # сервер закрыл соединение, не дочитав: это и нужно
|
||||
self.assertIn(b" 413 ", connection.recv(4096).split(b"\r\n")[0])
|
||||
|
||||
def test_decompression_bomb(self):
|
||||
data = make_png(50000, 50000, b"\x00" * 1000)
|
||||
self.assert_error(upload(self.alice, self.chat, data), 413, "payload_too_large")
|
||||
|
||||
def test_pixels_over_limit_with_small_file(self):
|
||||
self.assert_error(upload(self.alice, self.chat, make_png(8000, 8000, b"\x00" * 100)), 413, "payload_too_large")
|
||||
|
||||
def test_empty_file(self):
|
||||
self.assert_error(upload(self.alice, self.chat, b""), 415, "unsupported_media_type")
|
||||
|
||||
|
||||
class ContentCheckTest(Base):
|
||||
def test_broken_images(self):
|
||||
png, jpeg = FIXTURE, REAL_IMAGES["jpeg_baseline"][1]
|
||||
flipped = bytearray(png)
|
||||
flipped[len(png) // 2] ^= 0xFF
|
||||
cases = {
|
||||
"текст": (b"not an image at all", "image/png"),
|
||||
"обрезанный PNG": (png[: len(png) // 2], "image/png"),
|
||||
"обрезанный JPEG": (jpeg[: len(jpeg) // 2], "image/jpeg"),
|
||||
"только сигнатура PNG": (png[:8], "image/png"),
|
||||
"только сигнатура JPEG": (jpeg[:3], "image/jpeg"),
|
||||
"испорченный байт в PNG": (bytes(flipped), "image/png"),
|
||||
"GIF": (b"GIF89a" + b"\x00" * 100, "image/png"),
|
||||
"SVG": (b"<svg xmlns='http://www.w3.org/2000/svg'/>", "image/png"),
|
||||
}
|
||||
for name, (data, content_type) in cases.items():
|
||||
with self.subTest(name):
|
||||
self.assert_error(upload(self.alice, self.chat, data, content_type), 415, "unsupported_media_type")
|
||||
|
||||
def test_mime_spoofing(self):
|
||||
png, jpeg = FIXTURE, REAL_IMAGES["jpeg_baseline"][1]
|
||||
self.assert_error(upload(self.alice, self.chat, png, "image/jpeg"), 415, "unsupported_media_type")
|
||||
self.assert_error(upload(self.alice, self.chat, jpeg, "image/png"), 415, "unsupported_media_type")
|
||||
|
||||
def test_unsupported_content_type(self):
|
||||
for content_type in ("image/gif", "text/html", "application/octet-stream", "multipart/form-data"):
|
||||
with self.subTest(content_type):
|
||||
self.assert_error(upload(self.alice, self.chat, FIXTURE, content_type), 415, "unsupported_media_type")
|
||||
|
||||
def test_extension_is_not_trusted(self):
|
||||
status, attachment, _ = upload(self.alice, self.chat, FIXTURE, filename="evil.exe")
|
||||
self.assertEqual((status, attachment["content_type"]), (201, "image/png"))
|
||||
|
||||
|
||||
class FilenameTest(Base):
|
||||
def test_bad_filenames(self):
|
||||
names = ("", " ", "a/b.png", "../../etc/passwd", "a\\b.png", "a\x00b.png", "a\nb.png", "a\tb.png", "a\x7fb.png",
|
||||
".", "..", "x" * 256, "photognp.exe")
|
||||
for name in names:
|
||||
with self.subTest(name=ascii(name[:20])):
|
||||
self.assert_error(upload(self.alice, self.chat, FIXTURE, filename=name), 400, "invalid_request")
|
||||
|
||||
def test_missing_filename(self):
|
||||
self.assert_error(upload(self.alice, self.chat, FIXTURE, filename=None), 400, "invalid_request")
|
||||
|
||||
def test_good_filenames(self):
|
||||
for name in ("a", "x" * 255, "я" * 255, "two words.png", "семья 👨👩👧.jpg", ".hidden.png", "'; DROP TABLE users; --.png"):
|
||||
with self.subTest(name=name[:20]):
|
||||
status, attachment, _ = upload(self.alice, self.chat, FIXTURE, filename=name)
|
||||
self.assertEqual((status, attachment["filename"]), (201, name))
|
||||
|
||||
|
||||
class AccessTest(Base):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
_, cls.attachment, _ = upload(cls.alice, cls.chat, FIXTURE)
|
||||
cls.other_chat = new_chat(cls.eve)
|
||||
|
||||
def test_outsider_gets_404(self):
|
||||
attachment_id = self.attachment["id"]
|
||||
self.assert_error(call("GET", "/api/v1/attachments/" + attachment_id, user=self.eve), 404, "not_found")
|
||||
self.assertEqual(content(self.eve, attachment_id)[0], 404)
|
||||
|
||||
def test_outsider_looks_the_same_as_missing(self):
|
||||
_, foreign, _ = call("GET", "/api/v1/attachments/" + self.attachment["id"], user=self.eve)
|
||||
_, missing, _ = call("GET", "/api/v1/attachments/" + str(uuid.uuid4()), user=self.eve)
|
||||
self.assertEqual(foreign["error"]["message"], missing["error"]["message"])
|
||||
|
||||
def test_no_token(self):
|
||||
self.assertEqual(call("GET", "/api/v1/attachments/" + self.attachment["id"])[0], 401)
|
||||
self.assertEqual(content({"token": "x" * 43}, self.attachment["id"])[0], 401)
|
||||
self.assertEqual(upload({"token": "x" * 43}, self.chat, FIXTURE)[0], 401)
|
||||
|
||||
def test_outsider_cannot_upload(self):
|
||||
self.assert_error(upload(self.eve, self.chat, FIXTURE), 404, "not_found")
|
||||
|
||||
def test_bad_ids(self):
|
||||
self.assertEqual(call("GET", "/api/v1/attachments/123", user=self.alice)[0], 400)
|
||||
self.assertEqual(call("GET", "/api/v1/attachments/urn:uuid:" + self.attachment["id"], user=self.alice)[0], 400)
|
||||
|
||||
def test_download_never_redirects(self):
|
||||
status, body, headers = content(self.bob, self.attachment["id"])
|
||||
self.assertEqual(status, 200)
|
||||
self.assertIsNone(headers.get("Location"))
|
||||
self.assertNotIn(b"X-Amz", body[:200])
|
||||
|
||||
def test_variants(self):
|
||||
attachment_id = self.attachment["id"]
|
||||
self.assertEqual(content(self.bob, attachment_id, "?variant=original")[0], 200)
|
||||
self.assertEqual(content(self.bob, attachment_id, "?variant=thumbnail")[0], 404)
|
||||
self.assertEqual(content(self.bob, attachment_id, "?variant=nonsense")[0], 400)
|
||||
self.assertEqual(content(self.eve, attachment_id, "?variant=nonsense")[0], 404)
|
||||
|
||||
|
||||
class MessageLinkTest(Base):
|
||||
def test_message_with_attachments(self):
|
||||
ids = [upload(self.alice, self.chat, FIXTURE)[1]["id"] for _ in range(2)]
|
||||
status, message, _ = send(self.alice, self.chat, "смотри", attachment_ids=ids)
|
||||
self.assertEqual((status, message["attachment_ids"]), (201, ids))
|
||||
items, _ = history(self.bob, self.chat, 100)
|
||||
self.assertIn(message, items)
|
||||
|
||||
def test_attachment_only_message(self):
|
||||
attachment_id = upload(self.alice, self.chat, FIXTURE)[1]["id"]
|
||||
status, message, _ = send(self.bob, self.chat, "", attachment_ids=[attachment_id])
|
||||
self.assertEqual((status, message["text"]), (201, ""))
|
||||
|
||||
def test_text_message_has_empty_list(self):
|
||||
self.assertEqual(send(self.alice, self.chat, "текст")[1]["attachment_ids"], [])
|
||||
|
||||
def test_empty_message_without_attachments(self):
|
||||
self.assert_error(send(self.alice, self.chat, "", attachment_ids=[]), 400, "invalid_request")
|
||||
self.assert_error(send(self.alice, self.chat, " ", attachment_ids=[]), 400, "invalid_request")
|
||||
|
||||
def test_foreign_attachment(self):
|
||||
foreign_chat = new_chat(self.eve)
|
||||
foreign = upload(self.eve, foreign_chat, FIXTURE)[1]["id"]
|
||||
self.assert_error(send(self.alice, self.chat, "чужое", attachment_ids=[foreign]), 404, "not_found")
|
||||
|
||||
def test_attachment_of_my_other_chat(self):
|
||||
other = new_chat(self.alice)
|
||||
attachment_id = upload(self.alice, other, FIXTURE)[1]["id"]
|
||||
self.assert_error(send(self.alice, self.chat, "не тот чат", attachment_ids=[attachment_id]), 404, "not_found")
|
||||
|
||||
def test_unknown_attachment(self):
|
||||
self.assert_error(send(self.alice, self.chat, "нет", attachment_ids=[str(uuid.uuid4())]), 404, "not_found")
|
||||
|
||||
def test_failed_link_saves_nothing_and_frees_key(self):
|
||||
key = new_key()
|
||||
attachment_id = upload(self.alice, self.chat, FIXTURE)[1]["id"]
|
||||
self.assertEqual(send(self.alice, self.chat, "повтор", key=key, attachment_ids=[str(uuid.uuid4())])[0], 404)
|
||||
self.assertEqual(send(self.alice, self.chat, "повтор", key=key, attachment_ids=[attachment_id])[0], 201)
|
||||
|
||||
def test_limits(self):
|
||||
one = str(uuid.uuid4())
|
||||
self.assert_error(send(self.alice, self.chat, "x", attachment_ids=[one, one]), 400, "invalid_request")
|
||||
many = [str(uuid.uuid4()) for _ in range(11)]
|
||||
self.assert_error(send(self.alice, self.chat, "x", attachment_ids=many), 400, "invalid_request")
|
||||
self.assert_error(send(self.alice, self.chat, "x", attachment_ids=["abc"]), 400, "invalid_request")
|
||||
|
||||
def test_ten_attachments(self):
|
||||
ids = [upload(self.alice, self.chat, FIXTURE)[1]["id"] for _ in range(10)]
|
||||
self.assertEqual(send(self.alice, self.chat, "десять", attachment_ids=ids)[1]["attachment_ids"], ids)
|
||||
|
||||
def test_idempotent_repeat(self):
|
||||
attachment_id = upload(self.alice, self.chat, FIXTURE)[1]["id"]
|
||||
key = new_key()
|
||||
first = send(self.alice, self.chat, "с файлом", key=key, attachment_ids=[attachment_id])
|
||||
second = send(self.alice, self.chat, "с файлом", key=key, attachment_ids=[attachment_id])
|
||||
self.assertEqual(first[1], second[1])
|
||||
self.assertEqual(send(self.alice, self.chat, "с файлом", key=key)[0], 409)
|
||||
|
||||
def test_missing_list_equals_empty_list(self):
|
||||
key = new_key()
|
||||
first = send(self.alice, self.chat, "без файла", key=key)
|
||||
second = send(self.alice, self.chat, "без файла", key=key, attachment_ids=[])
|
||||
self.assertEqual(first[1], second[1])
|
||||
|
||||
def test_one_file_in_two_messages(self):
|
||||
attachment_id = upload(self.alice, self.chat, FIXTURE)[1]["id"]
|
||||
self.assertEqual(send(self.alice, self.chat, "первое", attachment_ids=[attachment_id])[0], 201)
|
||||
self.assertEqual(send(self.bob, self.chat, "второе", attachment_ids=[attachment_id])[0], 201)
|
||||
|
||||
|
||||
class ParallelLimitTest(Base):
|
||||
def test_too_many_parallel_uploads_get_429(self):
|
||||
address = urllib.parse.urlsplit(BASE_URL)
|
||||
path = "/api/v1/chats/%s/attachments?filename=a.png" % self.chat["id"]
|
||||
head = ("POST %s HTTP/1.1\r\nHost: x\r\nContent-Type: image/png\r\nContent-Length: 1000\r\n"
|
||||
"Authorization: Bearer %s\r\n\r\nabc" % (path, self.alice["token"])).encode()
|
||||
# начатые загрузки, которые ничего не досылают; на каждую реплику лимит 32
|
||||
connections = []
|
||||
answers = []
|
||||
try:
|
||||
for _ in range(80):
|
||||
connection = socket.create_connection((address.hostname, address.port), timeout=5)
|
||||
connection.sendall(head)
|
||||
connections.append(connection)
|
||||
import time
|
||||
|
||||
time.sleep(1.5)
|
||||
for connection in connections:
|
||||
connection.setblocking(False)
|
||||
try:
|
||||
answers.append(connection.recv(4096).split(b"\r\n")[0])
|
||||
except BlockingIOError:
|
||||
pass
|
||||
finally:
|
||||
for connection in connections:
|
||||
connection.close()
|
||||
self.assertTrue(answers, "ни одна лишняя загрузка не получила ответ")
|
||||
self.assertTrue(all(b" 429 " in answer for answer in answers), answers[:3])
|
||||
self.assertEqual(upload(self.alice, self.chat, FIXTURE)[0], 201)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,144 @@
|
||||
import datetime
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from api import basic, call, login, new_user, user_body
|
||||
|
||||
|
||||
class LoginTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.body = user_body()
|
||||
_, cls.user, _ = call("POST", "/api/v1/users", cls.body)
|
||||
|
||||
def assert_basic_challenge(self, result):
|
||||
status, body, headers = result
|
||||
self.assertEqual(status, 401)
|
||||
self.assertEqual(body["error"]["code"], "unauthorized")
|
||||
self.assertTrue(headers["WWW-Authenticate"].startswith("Basic "))
|
||||
return body
|
||||
|
||||
def test_login(self):
|
||||
status, session, _ = login(self.body["username"], self.body["password"])
|
||||
self.assertEqual(status, 201)
|
||||
self.assertEqual(set(session), {"token", "token_type", "expires_at", "user"})
|
||||
self.assertEqual(session["token_type"], "Bearer")
|
||||
self.assertEqual(session["user"], self.user)
|
||||
self.assertGreaterEqual(len(session["token"]), 32)
|
||||
expires_at = datetime.datetime.fromisoformat(session["expires_at"][:-1] + "+00:00")
|
||||
left = (expires_at - datetime.datetime.now(datetime.timezone.utc)).total_seconds()
|
||||
self.assertTrue(300 <= left <= 3600 + 5, left)
|
||||
|
||||
def test_token_is_opaque_and_unique(self):
|
||||
first = login(self.body["username"], self.body["password"])[1]["token"]
|
||||
second = login(self.body["username"], self.body["password"])[1]["token"]
|
||||
self.assertNotEqual(first, second)
|
||||
for token in (first, second):
|
||||
self.assertNotIn(self.user["id"], token)
|
||||
self.assertNotIn(self.body["username"], token)
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user={"token": token})[0], 200)
|
||||
|
||||
def test_wrong_password_and_unknown_user_look_the_same(self):
|
||||
wrong = self.assert_basic_challenge(login(self.body["username"], "wrong-password-123"))
|
||||
unknown = self.assert_basic_challenge(login("nobody_" + uuid.uuid4().hex[:8], self.body["password"]))
|
||||
self.assertEqual(wrong["error"]["message"], unknown["error"]["message"])
|
||||
|
||||
def test_malformed_credentials(self):
|
||||
token = login(self.body["username"], self.body["password"])[1]["token"]
|
||||
for header in (None, "Basic", "Basic !!!not-base64!!!", "Basic bm8tY29sb24=", "Bearer " + token, "Basic 0KU6"):
|
||||
with self.subTest(header=header):
|
||||
headers = {"Authorization": header} if header else {}
|
||||
self.assert_basic_challenge(call("POST", "/api/v1/auth/sessions", headers=headers))
|
||||
|
||||
def test_garbage_after_basic_is_401_not_500(self):
|
||||
# значение заголовка приходит на сервер как latin-1, поэтому после Basic может оказаться не ASCII
|
||||
values = ("\u00e9", "\u00e9\u00e9\u00e9=", "\u00ff\u00fe", "//46cHc=", "=", "====", "a", " ", "A" * 5000, "Og==",
|
||||
"dXNlcjpwYXNz dXNlcjpwYXNz")
|
||||
for value in values:
|
||||
with self.subTest(value=ascii(value[:16])):
|
||||
result = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + value})
|
||||
self.assert_basic_challenge(result)
|
||||
|
||||
def test_username_that_cannot_exist(self):
|
||||
for username in ("", "Has Space", "ab\u0000cd", "имя_кириллицей"):
|
||||
with self.subTest(username=username):
|
||||
self.assert_basic_challenge(login(username, self.body["password"]))
|
||||
|
||||
def test_colon_and_unicode_in_password(self):
|
||||
body = user_body(password="пароль:с:двоеточиями")
|
||||
self.assertEqual(call("POST", "/api/v1/users", body)[0], 201)
|
||||
self.assertEqual(login(body["username"], body["password"])[0], 201)
|
||||
self.assert_basic_challenge(login(body["username"], "пароль"))
|
||||
|
||||
|
||||
class BearerTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.alice, cls.eve = new_user("alice"), new_user("eve")
|
||||
|
||||
def assert_bearer_challenge(self, result):
|
||||
status, body, headers = result
|
||||
self.assertEqual(status, 401)
|
||||
self.assertEqual(body["error"]["code"], "unauthorized")
|
||||
self.assertTrue(headers["WWW-Authenticate"].startswith("Bearer "))
|
||||
|
||||
def test_no_token(self):
|
||||
self.assert_bearer_challenge(call("GET", "/api/v1/users/me"))
|
||||
|
||||
def test_unknown_tokens(self):
|
||||
for token in ("x" * 43, self.alice["id"], self.alice["username"], self.alice["token"][:-1]):
|
||||
with self.subTest(token=token[:8]):
|
||||
self.assert_bearer_challenge(call("GET", "/api/v1/users/me", user={"token": token}))
|
||||
|
||||
def test_garbage_token_is_401_not_500(self):
|
||||
for token in ("\u00e9", "\u00ff\u00fe", "a b", "x" * 10000):
|
||||
with self.subTest(token=ascii(token[:16])):
|
||||
self.assert_bearer_challenge(call("GET", "/api/v1/users/me", user={"token": token}))
|
||||
|
||||
def test_basic_is_not_accepted_on_business_endpoints(self):
|
||||
headers = basic(self.alice["username"], self.alice["password"])
|
||||
self.assert_bearer_challenge(call("GET", "/api/v1/users/me", headers=headers))
|
||||
self.assert_bearer_challenge(call("GET", "/api/v1/chats", headers=headers))
|
||||
|
||||
def test_x_user_id_gives_no_access(self):
|
||||
self.assert_bearer_challenge(call("GET", "/api/v1/users/me", headers={"X-User-Id": self.alice["id"]}))
|
||||
|
||||
def test_x_user_id_does_not_change_identity(self):
|
||||
status, me, _ = call("GET", "/api/v1/users/me", user=self.alice, headers={"X-User-Id": self.eve["id"]})
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(me, self.alice["user"])
|
||||
|
||||
def test_token_is_checked_before_anything_else(self):
|
||||
self.assert_bearer_challenge(call("GET", "/api/v1/chats/not-a-uuid"))
|
||||
self.assert_bearer_challenge(call("GET", "/api/v1/chats?limit=0"))
|
||||
self.assert_bearer_challenge(call("POST", "/api/v1/chats", {"title": 5}))
|
||||
|
||||
|
||||
class LogoutTest(unittest.TestCase):
|
||||
def test_logout_revokes_only_current_session(self):
|
||||
user = new_user()
|
||||
other_token = login(user["username"], user["password"])[1]["token"]
|
||||
|
||||
status, body, _ = call("DELETE", "/api/v1/auth/sessions/current", user=user)
|
||||
self.assertEqual(status, 204)
|
||||
self.assertIsNone(body)
|
||||
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user=user)[0], 401)
|
||||
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", user=user)[0], 401)
|
||||
# вторая сессия того же пользователя продолжает работать
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user={"token": other_token})[0], 200)
|
||||
|
||||
def test_login_again_after_logout(self):
|
||||
user = new_user()
|
||||
call("DELETE", "/api/v1/auth/sessions/current", user=user)
|
||||
status, session, _ = login(user["username"], user["password"])
|
||||
self.assertEqual(status, 201)
|
||||
self.assertNotEqual(session["token"], user["token"])
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user=session)[0], 200)
|
||||
|
||||
def test_logout_requires_token(self):
|
||||
self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current")[0], 401)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,94 @@
|
||||
import json
|
||||
import socket
|
||||
import unittest
|
||||
import urllib.parse
|
||||
import uuid
|
||||
|
||||
from api import BASE_URL, basic, call, history, new_chat, new_user, send
|
||||
|
||||
|
||||
class ErrorsFromBalancerTest(unittest.TestCase):
|
||||
"""Запросы, которые до API не доходят: ответ всё равно в формате контракта."""
|
||||
|
||||
def assert_contract_error(self, raw_request):
|
||||
address = urllib.parse.urlsplit(BASE_URL)
|
||||
with socket.create_connection((address.hostname, address.port), timeout=5) as connection:
|
||||
connection.sendall(raw_request)
|
||||
data = b""
|
||||
while chunk := connection.recv(65536):
|
||||
data += chunk
|
||||
head, _, body = data.partition(b"\r\n\r\n")
|
||||
lines = head.decode("latin-1").split("\r\n")
|
||||
headers = {line.split(":")[0].lower(): line.split(":", 1)[1].strip() for line in lines[1:]}
|
||||
self.assertEqual(headers["content-type"], "application/json")
|
||||
error = json.loads(body)["error"]
|
||||
self.assertEqual(error["request_id"], headers["x-request-id"])
|
||||
self.assertEqual(error["code"], "invalid_request")
|
||||
return int(lines[0].split()[1])
|
||||
|
||||
def test_invalid_http(self):
|
||||
self.assertEqual(self.assert_contract_error(b"GET /a b HTTP/1.1\r\nHost: x\r\n\r\n"), 400)
|
||||
|
||||
def test_too_long_uri(self):
|
||||
self.assertEqual(self.assert_contract_error(b"GET /" + b"a" * 20000 + b" HTTP/1.1\r\nHost: x\r\n\r\n"), 414)
|
||||
|
||||
def test_too_large_headers(self):
|
||||
raw = b"GET /x HTTP/1.1\r\nHost: x\r\nX-Big: " + b"a" * 20000 + b"\r\n\r\n"
|
||||
self.assertEqual(self.assert_contract_error(raw), 431)
|
||||
|
||||
|
||||
class ReplicasTest(unittest.TestCase):
|
||||
def test_every_answer_has_instance_id(self):
|
||||
user = new_user()
|
||||
answers = (
|
||||
call("GET", "/health/live"),
|
||||
call("GET", "/health/ready"),
|
||||
call("GET", "/api/v1/users/me"),
|
||||
call("GET", "/api/v1/users/me", user=user),
|
||||
call("GET", "/api/v1/chats/" + str(uuid.uuid4()), user=user),
|
||||
call("GET", "/nowhere"),
|
||||
call("POST", "/api/v1/users", raw=b"{"),
|
||||
)
|
||||
for status, _, headers in answers:
|
||||
with self.subTest(status=status):
|
||||
self.assertTrue(headers["X-Instance-Id"])
|
||||
self.assertTrue(headers["X-Request-Id"])
|
||||
|
||||
def test_one_session_is_served_by_different_replicas(self):
|
||||
user = new_user()
|
||||
instances = set()
|
||||
for _ in range(20):
|
||||
status, me, headers = call("GET", "/api/v1/users/me", user=user)
|
||||
self.assertEqual((status, me), (200, user["user"]))
|
||||
instances.add(headers["X-Instance-Id"])
|
||||
self.assertGreaterEqual(len(instances), 2)
|
||||
|
||||
def test_session_created_on_one_replica_works_on_all(self):
|
||||
user = new_user()
|
||||
for _ in range(6):
|
||||
status, session, _ = call("POST", "/api/v1/auth/sessions", headers=basic(user["username"], user["password"]))
|
||||
self.assertEqual(status, 201)
|
||||
seen = {call("GET", "/api/v1/users/me", user=session)[2]["X-Instance-Id"] for _ in range(4)}
|
||||
self.assertGreaterEqual(len(seen), 2)
|
||||
|
||||
def test_read_after_write_on_any_replica(self):
|
||||
user = new_user()
|
||||
chat = new_chat(user)
|
||||
for number in range(20):
|
||||
_, message, _ = send(user, chat, "запись %d" % number)
|
||||
items, _ = history(user, chat, 100)
|
||||
self.assertEqual(items[-1], message)
|
||||
|
||||
def test_logout_works_on_all_replicas(self):
|
||||
user = new_user()
|
||||
call("DELETE", "/api/v1/auth/sessions/current", user=user)
|
||||
for _ in range(4):
|
||||
self.assertEqual(call("GET", "/api/v1/users/me", user=user)[0], 401)
|
||||
|
||||
def test_no_sticky_cookie(self):
|
||||
_, _, headers = call("GET", "/health/live")
|
||||
self.assertIsNone(headers.get("Set-Cookie"), BASE_URL)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Проверка гонок: 10 клиентов параллельно отправляют 100 сообщений в один чат."""
|
||||
import threading
|
||||
import unittest
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from api import call, history, new_chat, new_user, send, user_body
|
||||
|
||||
CLIENTS = 10
|
||||
MESSAGES_PER_CLIENT = 10
|
||||
|
||||
|
||||
class ConcurrencyTest(unittest.TestCase):
|
||||
def test_100_messages_from_10_clients(self):
|
||||
users = [new_user("c%d" % number) for number in range(CLIENTS)]
|
||||
chat = new_chat(users[0], *users[1:])
|
||||
barrier = threading.Barrier(CLIENTS)
|
||||
|
||||
def client(number):
|
||||
barrier.wait() # все потоки стартуют одновременно
|
||||
results = []
|
||||
for index in range(MESSAGES_PER_CLIENT):
|
||||
results.append(send(users[number], chat, "клиент %d сообщение %d" % (number, index)))
|
||||
return results
|
||||
|
||||
with ThreadPoolExecutor(max_workers=CLIENTS) as pool:
|
||||
results = [result for chunk in pool.map(client, range(CLIENTS)) for result in chunk]
|
||||
|
||||
total = CLIENTS * MESSAGES_PER_CLIENT
|
||||
self.assertEqual([status for status, _, _ in results], [201] * total)
|
||||
confirmed = {message["id"]: message for _, message, _ in results}
|
||||
self.assertEqual(len(confirmed), total)
|
||||
|
||||
items, _ = history(users[0], chat, limit=7)
|
||||
self.assertEqual(len(items), total)
|
||||
self.assertEqual(len({item["id"] for item in items}), total)
|
||||
self.assertEqual(len({item["text"] for item in items}), total)
|
||||
# каждое подтверждённое сообщение есть в истории без изменений
|
||||
self.assertEqual({item["id"]: item for item in items}, confirmed)
|
||||
keys = [(item["created_at"], item["id"]) for item in items]
|
||||
self.assertEqual(keys, sorted(keys))
|
||||
# сообщения одного клиента идут в том порядке, в котором он их отправлял
|
||||
for number in range(CLIENTS):
|
||||
own = [item["text"] for item in items if item["sender_id"] == users[number]["id"]]
|
||||
self.assertEqual(own, ["клиент %d сообщение %d" % (number, index) for index in range(MESSAGES_PER_CLIENT)])
|
||||
print("\nотправлено %d, подтверждено %d, в истории %d, уникальных id %d"
|
||||
% (total, len(confirmed), len(items), len({item["id"] for item in items})))
|
||||
|
||||
def test_same_username_registered_once(self):
|
||||
body = user_body(username="race_" + uuid.uuid4().hex[:8])
|
||||
barrier = threading.Barrier(CLIENTS)
|
||||
|
||||
def register(_):
|
||||
barrier.wait()
|
||||
return call("POST", "/api/v1/users", body)[0]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=CLIENTS) as pool:
|
||||
statuses = sorted(pool.map(register, range(CLIENTS)))
|
||||
self.assertEqual(statuses, [201] + [409] * (CLIENTS - 1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,98 @@
|
||||
import threading
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from api import call, history, new_chat, new_key, new_user, send
|
||||
|
||||
|
||||
class IdempotencyTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.alice, cls.bob, cls.eve = new_user("alice"), new_user("bob"), new_user("eve")
|
||||
cls.chat = new_chat(cls.alice, cls.bob)
|
||||
cls.path = "/api/v1/chats/" + cls.chat["id"] + "/messages"
|
||||
|
||||
def test_key_is_required(self):
|
||||
status, body, _ = call("POST", self.path, {"text": "без ключа"}, user=self.alice)
|
||||
self.assertEqual(status, 400)
|
||||
self.assertEqual(body["error"]["code"], "invalid_request")
|
||||
|
||||
def test_bad_keys(self):
|
||||
for key in ("", "a" * 129, "с пробелом", "кириллица", "a/b", "a,b"):
|
||||
with self.subTest(key=key[:12]):
|
||||
headers = {"Idempotency-Key": key.encode("utf-8").decode("latin-1")}
|
||||
self.assertEqual(call("POST", self.path, {"text": "x"}, user=self.alice, headers=headers)[0], 400)
|
||||
|
||||
def test_good_key_alphabet(self):
|
||||
for key in ("a", "A.b_c:d-1", "x" * 128):
|
||||
with self.subTest(key=key[:12]):
|
||||
self.assertEqual(send(self.alice, self.chat, "ключ " + key, key=key)[0], 201)
|
||||
|
||||
def test_repeat_returns_same_message(self):
|
||||
key = new_key()
|
||||
first = send(self.alice, self.chat, "Привет 👋", key=key)
|
||||
second = send(self.alice, self.chat, "Привет 👋", key=key)
|
||||
self.assertEqual((first[0], second[0]), (201, 201))
|
||||
self.assertEqual(first[1], second[1])
|
||||
|
||||
def test_key_order_in_json_does_not_matter(self):
|
||||
key = new_key()
|
||||
headers = {"Idempotency-Key": key}
|
||||
first = call("POST", self.path, {"text": "порядок"}, user=self.alice, headers=headers)
|
||||
second = call("POST", self.path, raw=' { "text" : "порядок" } '.encode(), user=self.alice, headers=headers)
|
||||
self.assertEqual(first[1], second[1])
|
||||
|
||||
def test_other_text_with_same_key_is_409(self):
|
||||
key = new_key()
|
||||
first = send(self.alice, self.chat, "первый текст", key=key)
|
||||
status, body, _ = send(self.alice, self.chat, "другой текст", key=key)
|
||||
self.assertEqual(status, 409)
|
||||
self.assertEqual(body["error"]["code"], "conflict")
|
||||
# конфликт ничего не меняет: исходный текст по-прежнему возвращается
|
||||
self.assertEqual(send(self.alice, self.chat, "первый текст", key=key)[1], first[1])
|
||||
|
||||
def test_same_key_for_other_user_and_other_chat(self):
|
||||
key = new_key()
|
||||
other_chat = new_chat(self.alice)
|
||||
first = send(self.alice, self.chat, "общий ключ", key=key)[1]
|
||||
by_bob = send(self.bob, self.chat, "общий ключ", key=key)[1]
|
||||
in_other_chat = send(self.alice, other_chat, "общий ключ", key=key)[1]
|
||||
self.assertEqual(len({first["id"], by_bob["id"], in_other_chat["id"]}), 3)
|
||||
self.assertEqual(by_bob["sender_id"], self.bob["id"])
|
||||
|
||||
def test_repeat_checks_access_again(self):
|
||||
key = new_key()
|
||||
send(self.alice, self.chat, "только участникам", key=key)
|
||||
self.assertEqual(send(self.eve, self.chat, "только участникам", key=key)[0], 404)
|
||||
self.assertEqual(send({"token": "x" * 43}, self.chat, "только участникам", key=key)[0], 401)
|
||||
|
||||
def test_invalid_body_does_not_take_key(self):
|
||||
key = new_key()
|
||||
self.assertEqual(send(self.alice, self.chat, "", key=key)[0], 400)
|
||||
self.assertEqual(send(self.alice, self.chat, "теперь верно", key=key)[0], 201)
|
||||
|
||||
def test_parallel_repeat_through_different_replicas(self):
|
||||
user = new_user("idem")
|
||||
chat = new_chat(user)
|
||||
instances = set()
|
||||
for round_number in range(10):
|
||||
key = new_key()
|
||||
barrier = threading.Barrier(8)
|
||||
|
||||
def post(_):
|
||||
barrier.wait()
|
||||
return send(user, chat, "сообщение %d" % round_number, key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(post, range(8)))
|
||||
self.assertEqual({status for status, _, _ in results}, {201})
|
||||
self.assertEqual(len({message["id"] for _, message, _ in results}), 1)
|
||||
instances |= {headers["X-Instance-Id"] for _, _, headers in results}
|
||||
self.assertGreaterEqual(len(instances), 2, "запросы не дошли до разных реплик")
|
||||
items, _ = history(user, chat, 100)
|
||||
self.assertEqual(len(items), 10)
|
||||
print("\n10 ключей по 8 параллельных запросов: реплик %d, сообщений в истории %d" % (len(instances), len(items)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Проверка разбора изображений без сервера: файлы настоящие, а испорченные версии не должны ронять разбор."""
|
||||
import io
|
||||
import random
|
||||
import struct
|
||||
import sys
|
||||
import unittest
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
||||
import images # noqa: E402
|
||||
|
||||
from samples import REAL_IMAGES, make_png, png_chunk # noqa: E402
|
||||
|
||||
LIMIT = 40_000_000
|
||||
|
||||
|
||||
def inspect(data, limit=LIMIT):
|
||||
return images.inspect(io.BytesIO(data), limit)
|
||||
|
||||
|
||||
class RealImagesTest(unittest.TestCase):
|
||||
def test_real_images_are_accepted(self):
|
||||
for name, (content_type, data) in REAL_IMAGES.items():
|
||||
with self.subTest(name):
|
||||
self.assertEqual(inspect(data), content_type)
|
||||
|
||||
def test_course_fixture(self):
|
||||
path = Path(__file__).resolve().parents[1] / "tests/fixtures/sample.png"
|
||||
self.assertEqual(inspect(path.read_bytes()), "image/png")
|
||||
|
||||
def test_jpeg_with_data_after_end(self):
|
||||
# камеры дописывают после конца изображения превью
|
||||
self.assertEqual(inspect(REAL_IMAGES["jpeg_baseline"][1] + b"\xff\xd8" + b"x" * 50), "image/jpeg")
|
||||
|
||||
def test_marker_on_block_border(self):
|
||||
data = bytearray(REAL_IMAGES["jpeg_baseline"][1])
|
||||
start = data.index(b"\xff\xda")
|
||||
length = struct.unpack(">H", data[start + 2:start + 4])[0]
|
||||
padded = data[:start + 2 + length] + b"\x12" * (images.BLOCK - 3) + b"\xff\x00" * 40 + data[start + 2 + length:]
|
||||
self.assertEqual(inspect(bytes(padded)), "image/jpeg")
|
||||
|
||||
|
||||
class BrokenImagesTest(unittest.TestCase):
|
||||
def assert_not_image(self, data):
|
||||
with self.assertRaises(images.NotAnImage):
|
||||
inspect(data)
|
||||
|
||||
def test_truncated(self):
|
||||
for name, (_, data) in REAL_IMAGES.items():
|
||||
for size in (0, 1, 2, 3, 8, 12, len(data) // 2, len(data) - 1):
|
||||
with self.subTest(name=name, size=size):
|
||||
self.assert_not_image(data[:size])
|
||||
|
||||
def test_not_images(self):
|
||||
for data in (b"", b"GIF89a" + b"\x00" * 20, b"%PDF-1.7", b"<svg></svg>", b"\xff\xd8", b"\x89PNG\r\n\x1a\n"):
|
||||
with self.subTest(start=data[:8]):
|
||||
self.assert_not_image(data)
|
||||
|
||||
def test_png_bad_crc(self):
|
||||
data = bytearray(make_png(8, 8))
|
||||
data[-20] ^= 0xFF
|
||||
self.assert_not_image(bytes(data))
|
||||
|
||||
def test_png_data_after_end(self):
|
||||
self.assert_not_image(make_png(8, 8) + b"<script>")
|
||||
|
||||
def test_png_with_wrong_amount_of_pixels(self):
|
||||
self.assert_not_image(make_png(8, 8, b"\x00" * 10))
|
||||
self.assert_not_image(make_png(8, 8, b"\x00" * 10000))
|
||||
|
||||
def test_png_data_is_not_zlib(self):
|
||||
header = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)
|
||||
data = b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", header) + png_chunk(b"IDAT", b"not zlib") + png_chunk(b"IEND", b"")
|
||||
self.assert_not_image(data)
|
||||
|
||||
def test_png_zero_size(self):
|
||||
self.assert_not_image(make_png(0, 8, b""))
|
||||
|
||||
def test_jpeg_without_frame_or_scan(self):
|
||||
self.assert_not_image(b"\xff\xd8\xff\xd9")
|
||||
self.assert_not_image(b"\xff\xd8\xff\xe0\x00\x04ab\xff\xd9")
|
||||
|
||||
def test_random_corruption_never_crashes(self):
|
||||
random.seed(20261030)
|
||||
for name, (_, original) in REAL_IMAGES.items():
|
||||
for _ in range(1500):
|
||||
data = bytearray(original)
|
||||
for _ in range(random.randint(1, 8)):
|
||||
data[random.randrange(len(data))] = random.randrange(256)
|
||||
try:
|
||||
inspect(bytes(data))
|
||||
except (images.NotAnImage, images.TooManyPixels):
|
||||
pass
|
||||
|
||||
|
||||
class PixelLimitTest(unittest.TestCase):
|
||||
def test_png_bomb_is_rejected_before_unpacking(self):
|
||||
header = struct.pack(">IIBBBBB", 50000, 50000, 8, 2, 0, 0, 0)
|
||||
# данные заголовка верны, а распаковать пришлось бы 7,5 ГБ
|
||||
data = b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", header) + png_chunk(b"IDAT", zlib.compress(b"\x00" * 1000))
|
||||
with self.assertRaises(images.TooManyPixels):
|
||||
inspect(data)
|
||||
|
||||
def test_jpeg_over_limit(self):
|
||||
data = bytearray(REAL_IMAGES["jpeg_baseline"][1])
|
||||
start = data.index(b"\xff\xc0")
|
||||
data[start + 5:start + 9] = struct.pack(">HH", 20000, 20000)
|
||||
with self.assertRaises(images.TooManyPixels):
|
||||
inspect(bytes(data))
|
||||
|
||||
def test_limit_is_exact(self):
|
||||
self.assertEqual(inspect(make_png(10, 10), limit=100), "image/png")
|
||||
with self.assertRaises(images.TooManyPixels):
|
||||
inspect(make_png(10, 10), limit=99)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,145 @@
|
||||
import base64
|
||||
import json
|
||||
import unittest
|
||||
import urllib.parse
|
||||
|
||||
from api import call, history, new_chat, new_user, send
|
||||
|
||||
|
||||
class MessagePaginationTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.user = new_user()
|
||||
cls.chat = new_chat(cls.user)
|
||||
cls.sent = [send(cls.user, cls.chat, "сообщение %d" % number)[1] for number in range(25)]
|
||||
cls.path = "/api/v1/chats/" + cls.chat["id"] + "/messages"
|
||||
|
||||
def test_walk_several_pages(self):
|
||||
items, sizes = history(self.user, self.chat, limit=10)
|
||||
self.assertEqual(sizes, [10, 10, 5])
|
||||
self.assertEqual(items, self.sent)
|
||||
|
||||
def test_no_duplicates_and_order(self):
|
||||
items, _ = history(self.user, self.chat, limit=7)
|
||||
ids = [item["id"] for item in items]
|
||||
self.assertEqual(len(ids), len(set(ids)))
|
||||
keys = [(item["created_at"], item["id"]) for item in items]
|
||||
self.assertEqual(keys, sorted(keys))
|
||||
|
||||
def test_exact_page_has_no_cursor(self):
|
||||
_, page, _ = call("GET", self.path + "?limit=25", user=self.user)
|
||||
self.assertEqual(len(page["items"]), 25)
|
||||
self.assertIsNone(page["next_cursor"])
|
||||
|
||||
def test_default_limit(self):
|
||||
_, page, _ = call("GET", self.path, user=self.user)
|
||||
self.assertEqual(len(page["items"]), 25)
|
||||
|
||||
def test_limit_can_change_between_pages(self):
|
||||
_, first, _ = call("GET", self.path + "?limit=3", user=self.user)
|
||||
cursor = urllib.parse.quote(first["next_cursor"], safe="")
|
||||
_, second, _ = call("GET", self.path + "?limit=100&cursor=" + cursor, user=self.user)
|
||||
self.assertEqual(first["items"] + second["items"], self.sent)
|
||||
|
||||
def test_bad_limit(self):
|
||||
for limit in ("0", "101", "-1", "abc", "1.5", ""):
|
||||
with self.subTest(limit=limit):
|
||||
self.assertEqual(call("GET", self.path + "?limit=" + limit, user=self.user)[0], 400)
|
||||
|
||||
def test_bad_cursor(self):
|
||||
for cursor in ("garbage", "", "e30=", "%D0%B9"):
|
||||
with self.subTest(cursor=cursor):
|
||||
self.assertEqual(call("GET", self.path + "?cursor=" + cursor, user=self.user)[0], 400)
|
||||
|
||||
def test_cursor_format_is_checked(self):
|
||||
_, first, _ = call("GET", self.path + "?limit=3", user=self.user)
|
||||
good = first["next_cursor"]
|
||||
scope, created_at, item_id = json.loads(base64.urlsafe_b64decode(good))
|
||||
|
||||
def encode(value, **options):
|
||||
return base64.urlsafe_b64encode(json.dumps(value, **options).encode()).decode()
|
||||
|
||||
# тест знает внутренний формат cursor: собранный заново правильный cursor совпадает с выданным
|
||||
self.assertEqual(encode([scope, created_at, item_id]), good)
|
||||
bad = {
|
||||
"посторонний символ внутри base64": good[:5] + "!" + good[5:],
|
||||
"пробел внутри base64": good[:5] + " " + good[5:],
|
||||
"перевод строки в конце": good + "\n",
|
||||
"лишний знак =": good + "=",
|
||||
"обрезанный": good[:-4],
|
||||
"время не в формате сервера": encode([scope, "вчера", item_id]),
|
||||
"время без микросекунд": encode([scope, created_at[:19] + "Z", item_id]),
|
||||
"время с неполными микросекундами": encode([scope, created_at[:21] + "Z", item_id]),
|
||||
"время со смещением вместо Z": encode([scope, created_at[:-1] + "+00:00", item_id]),
|
||||
"несуществующая дата": encode([scope, "2026-13-45T25:61:61.000000Z", item_id]),
|
||||
"число вместо времени": encode([scope, 1234567890, item_id]),
|
||||
"id не UUID": encode([scope, created_at, "abc"]),
|
||||
"UUID в верхнем регистре": encode([scope, created_at, item_id.upper()]),
|
||||
"UUID без дефисов": encode([scope, created_at, item_id.replace("-", "")]),
|
||||
"null вместо id": encode([scope, created_at, None]),
|
||||
"лишний элемент": encode([scope, created_at, item_id, "x"]),
|
||||
"не хватает элемента": encode([scope, created_at]),
|
||||
"объект вместо списка": encode({"scope": scope, "created_at": created_at, "id": item_id}),
|
||||
"тот же JSON без пробелов": encode([scope, created_at, item_id], separators=(",", ":")),
|
||||
"пустой список": encode([]),
|
||||
}
|
||||
for name, cursor in bad.items():
|
||||
with self.subTest(name):
|
||||
path = self.path + "?cursor=" + urllib.parse.quote(cursor, safe="")
|
||||
status, body, _ = call("GET", path, user=self.user)
|
||||
self.assertEqual(status, 400)
|
||||
self.assertEqual(body["error"]["code"], "invalid_request")
|
||||
# правильный cursor при этом продолжает работать
|
||||
self.assertEqual(call("GET", self.path + "?cursor=" + urllib.parse.quote(good, safe=""), user=self.user)[0], 200)
|
||||
|
||||
def test_cursor_from_another_chat(self):
|
||||
other = new_chat(self.user, title="Другой чат")
|
||||
send(self.user, other, "раз")
|
||||
send(self.user, other, "два")
|
||||
_, page, _ = call("GET", "/api/v1/chats/" + other["id"] + "/messages?limit=1", user=self.user)
|
||||
cursor = urllib.parse.quote(page["next_cursor"], safe="")
|
||||
self.assertEqual(call("GET", self.path + "?cursor=" + cursor, user=self.user)[0], 400)
|
||||
|
||||
def test_new_message_appears_after_last_page(self):
|
||||
user = new_user()
|
||||
chat = new_chat(user)
|
||||
send(user, chat, "раз")
|
||||
send(user, chat, "два")
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
_, first, _ = call("GET", path + "?limit=1", user=user)
|
||||
_, created, _ = send(user, chat, "три")
|
||||
cursor = urllib.parse.quote(first["next_cursor"], safe="")
|
||||
_, rest, _ = call("GET", path + "?limit=100&cursor=" + cursor, user=user)
|
||||
self.assertEqual([item["text"] for item in rest["items"]], ["два", "три"])
|
||||
self.assertEqual(rest["items"][-1], created)
|
||||
|
||||
|
||||
class ChatPaginationTest(unittest.TestCase):
|
||||
def test_walk_chat_list(self):
|
||||
user = new_user()
|
||||
created = [new_chat(user, title="Чат %d" % number) for number in range(5)]
|
||||
items, cursor = [], None
|
||||
while True:
|
||||
path = "/api/v1/chats?limit=2"
|
||||
if cursor:
|
||||
path += "&cursor=" + urllib.parse.quote(cursor, safe="")
|
||||
status, page, _ = call("GET", path, user=user)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertLessEqual(len(page["items"]), 2)
|
||||
items += page["items"]
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
self.assertEqual(items, created)
|
||||
|
||||
def test_chat_cursor_belongs_to_user(self):
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
for number in range(2):
|
||||
new_chat(alice, bob, title="Общий %d" % number)
|
||||
_, page, _ = call("GET", "/api/v1/chats?limit=1", user=alice)
|
||||
cursor = urllib.parse.quote(page["next_cursor"], safe="")
|
||||
self.assertEqual(call("GET", "/api/v1/chats?cursor=" + cursor, user=bob)[0], 400)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,153 @@
|
||||
import unittest
|
||||
|
||||
from api import call, login, new_chat, new_key, new_user, send, user_body
|
||||
|
||||
|
||||
class ErrorFormatTest(unittest.TestCase):
|
||||
def assert_error(self, result, status, code):
|
||||
got_status, body, headers = result
|
||||
self.assertEqual(got_status, status)
|
||||
self.assertEqual(set(body), {"error"})
|
||||
self.assertEqual(set(body["error"]), {"code", "message", "request_id"})
|
||||
self.assertEqual(body["error"]["code"], code)
|
||||
self.assertTrue(body["error"]["message"])
|
||||
self.assertEqual(body["error"]["request_id"], headers["X-Request-Id"])
|
||||
|
||||
|
||||
class InvalidJsonTest(ErrorFormatTest):
|
||||
def test_malformed_json(self):
|
||||
self.assert_error(call("POST", "/api/v1/users", raw=b'{"username": "abc", '), 400, "invalid_request")
|
||||
|
||||
def test_not_an_object(self):
|
||||
self.assert_error(call("POST", "/api/v1/users", raw=b'["abc"]'), 400, "invalid_request")
|
||||
|
||||
def test_extra_field(self):
|
||||
self.assert_error(call("POST", "/api/v1/users", user_body(is_admin=True)), 400, "invalid_request")
|
||||
|
||||
def test_wrong_type(self):
|
||||
self.assert_error(call("POST", "/api/v1/users", user_body(username=12345)), 400, "invalid_request")
|
||||
|
||||
def test_missing_field(self):
|
||||
body = user_body()
|
||||
del body["password"]
|
||||
self.assert_error(call("POST", "/api/v1/users", body), 400, "invalid_request")
|
||||
|
||||
def test_malformed_json_in_message(self):
|
||||
user = new_user()
|
||||
chat = new_chat(user)
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
headers = {"Idempotency-Key": new_key()}
|
||||
self.assert_error(call("POST", path, raw=b"{text: no quotes}", user=user, headers=headers), 400, "invalid_request")
|
||||
|
||||
def test_only_utf8_is_accepted(self):
|
||||
user = new_user()
|
||||
chat = new_chat(user)
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
text = '{"text": "привет"}'
|
||||
for raw in (text.encode("utf-16"), text.encode("utf-32"), b"\xef\xbb\xbf" + text.encode(), text.encode("cp1251")):
|
||||
with self.subTest(start=raw[:4]):
|
||||
headers = {"Idempotency-Key": new_key()}
|
||||
self.assert_error(call("POST", path, raw=raw, user=user, headers=headers), 400, "invalid_request")
|
||||
|
||||
def test_content_type_must_be_json(self):
|
||||
body = str(user_body()).replace("'", '"').encode()
|
||||
for content_type in ("text/plain", "application/xml"):
|
||||
with self.subTest(content_type=content_type):
|
||||
headers = {"Content-Type": content_type}
|
||||
self.assert_error(call("POST", "/api/v1/users", raw=body, headers=headers), 400, "invalid_request")
|
||||
|
||||
def test_too_large_json(self):
|
||||
body = b'{"username": "' + b"x" * (2 * 1024 * 1024) + b'"}'
|
||||
self.assert_error(call("POST", "/api/v1/users", raw=body), 413, "payload_too_large")
|
||||
|
||||
def test_deeply_nested_json(self):
|
||||
self.assert_error(call("POST", "/api/v1/users", raw=b"[" * 100000), 400, "invalid_request")
|
||||
|
||||
def test_broken_json_without_token_is_401(self):
|
||||
# сначала проверяется токен, и только потом разбирается тело
|
||||
status, _, _ = call("POST", "/api/v1/chats", raw=b"{broken")
|
||||
self.assertEqual(status, 401)
|
||||
|
||||
|
||||
class UserValidationTest(ErrorFormatTest):
|
||||
def test_bad_usernames(self):
|
||||
for username in ("ab", "a" * 33, "Alice", "with space", "имя", "abc\n", ""):
|
||||
with self.subTest(username=username):
|
||||
self.assert_error(call("POST", "/api/v1/users", user_body(username=username)), 400, "invalid_request")
|
||||
|
||||
def test_bad_display_names(self):
|
||||
for display_name in ("", " ", "\t\n", "я" * 101, "имя\u0000"):
|
||||
with self.subTest(display_name=display_name):
|
||||
result = call("POST", "/api/v1/users", user_body(display_name=display_name))
|
||||
self.assert_error(result, 400, "invalid_request")
|
||||
|
||||
def test_display_name_length_is_counted_in_code_points(self):
|
||||
# 100 эмодзи это 400 байт в UTF-8, но ровно 100 code points
|
||||
status, user, _ = call("POST", "/api/v1/users", user_body(display_name="🙂" * 100))
|
||||
self.assertEqual(status, 201)
|
||||
self.assertEqual(user["display_name"], "🙂" * 100)
|
||||
|
||||
def test_bad_passwords(self):
|
||||
for password in ("short-11-ch", "x" * 129, " " * 12, "", 123456789012, None):
|
||||
with self.subTest(password=password):
|
||||
self.assert_error(call("POST", "/api/v1/users", user_body(password=password)), 400, "invalid_request")
|
||||
|
||||
def test_password_length_limits(self):
|
||||
for password in ("exactly-12-c", "x" * 128, "пароль🙂пароль"):
|
||||
with self.subTest(length=len(password)):
|
||||
body = user_body(password=password)
|
||||
self.assertEqual(call("POST", "/api/v1/users", body)[0], 201)
|
||||
self.assertEqual(login(body["username"], password)[0], 201)
|
||||
|
||||
def test_password_is_never_returned(self):
|
||||
body = user_body()
|
||||
status, user, _ = call("POST", "/api/v1/users", body)
|
||||
self.assertEqual(status, 201)
|
||||
self.assertEqual(set(user), {"id", "username", "display_name", "created_at"})
|
||||
_, session, _ = login(body["username"], body["password"])
|
||||
self.assertNotIn(body["password"], str(session))
|
||||
|
||||
def test_duplicate_username(self):
|
||||
body = user_body(display_name="Первый")
|
||||
self.assertEqual(call("POST", "/api/v1/users", body)[0], 201)
|
||||
body["display_name"] = "Второй"
|
||||
self.assert_error(call("POST", "/api/v1/users", body), 409, "conflict")
|
||||
|
||||
|
||||
class MessageValidationTest(ErrorFormatTest):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.user = new_user()
|
||||
cls.chat = new_chat(cls.user)
|
||||
|
||||
def test_empty_message(self):
|
||||
self.assert_error(send(self.user, self.chat, ""), 400, "invalid_request")
|
||||
|
||||
def test_whitespace_message(self):
|
||||
for text in (" ", " \n\t ", " "):
|
||||
with self.subTest(text=repr(text)):
|
||||
self.assert_error(send(self.user, self.chat, text), 400, "invalid_request")
|
||||
|
||||
def test_nul_character(self):
|
||||
self.assert_error(send(self.user, self.chat, "до\u0000после"), 400, "invalid_request")
|
||||
|
||||
def test_text_limit(self):
|
||||
self.assertEqual(send(self.user, self.chat, "ы" * 4000)[0], 201)
|
||||
self.assert_error(send(self.user, self.chat, "ы" * 4001), 400, "invalid_request")
|
||||
|
||||
def test_sender_id_is_not_accepted(self):
|
||||
path = "/api/v1/chats/" + self.chat["id"] + "/messages"
|
||||
body = {"text": "привет", "sender_id": self.user["id"]}
|
||||
headers = {"Idempotency-Key": new_key()}
|
||||
self.assert_error(call("POST", path, body, user=self.user, headers=headers), 400, "invalid_request")
|
||||
|
||||
def test_unicode_is_kept(self):
|
||||
text = "Привет 👋 你好 مرحبا\nвторая строка"
|
||||
status, message, _ = send(self.user, self.chat, text)
|
||||
self.assertEqual(status, 201)
|
||||
self.assertEqual(message["text"], text)
|
||||
self.assertEqual(message["sender_id"], self.user["id"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user