Добавил загрузку картинок в S3 и проверку доступа к вложениям

This commit is contained in:
2026-11-17 13:19:43 +03:00
parent ec540a1a72
commit 372471ae58
35 changed files with 2004 additions and 19 deletions
+44
View File
@@ -0,0 +1,44 @@
"""Stop the API that actually accepted the binary upload, then download via LB."""
import hashlib
import json
from pathlib import Path
import sys
import time
ROOT=Path(__file__).resolve().parents[1]
sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
from client import Client, Failure
from ops import containers, command, compose
def main():
c=Client(timeout=5)
alice,bob,eve,chat=c.bootstrap()
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=failover.png',201,
actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')
instance=response['headers']['X-Instance-Id']
matches=[r for r in containers() if instance.startswith(r['Config']['Hostname']+'-')]
assert len(matches)==1
victim=matches[0]['Id']; started=time.monotonic()
command(['docker','stop','--time','6',victim])
try:
deadline=time.monotonic()+10
while True:
try:
result=c.call('GET','/api/v1/attachments/'+response['json']['id']+'/content',actor=bob)
break
except Failure:
if time.monotonic()>deadline: raise
time.sleep(.2)
assert result['headers']['X-Instance-Id']!=instance
assert result['bytes']==raw
assert command(['docker','inspect','--format','{{.State.Running}}',victim])=='false'
report={'upload_instance':instance,'stopped_container':victim,
'download_instance':result['headers']['X-Instance-Id'],
'sha256':hashlib.sha256(raw).hexdigest(),'size_bytes':len(raw),
'stop_and_download_seconds':round(time.monotonic()-started,3),'status':'PASS'}
(ROOT/'evidence/uploader-failover.json').write_text(json.dumps(report,indent=2))
finally:
compose('start','api')
print('PASS: original downloaded through another replica after uploader stopped')
if __name__=='__main__': main()
+35
View File
@@ -0,0 +1,35 @@
"""Create local synthetic credentials once without printing secret values."""
from pathlib import Path
import secrets
ROOT = Path(__file__).resolve().parents[1]
def main():
directory = ROOT / "secrets"
directory.mkdir(exist_ok=True)
for name in ("postgres_password", "redis_password", "cursor_key", "s3_root_password", "s3_secret_key"):
path = directory / name
if not path.exists():
path.write_text(secrets.token_hex(32) + "\n", encoding="utf-8", newline="\n")
else:
# Shell command substitution on Linux must not retain a Windows CR.
value = path.read_text(encoding="utf-8").strip()
path.write_text(value + "\n", encoding="utf-8", newline="\n")
pg_password = (directory / "postgres_password").read_text().strip()
redis_password = (directory / "redis_password").read_text().strip()
for name, value in {
"database_url": f"postgresql://messenger:{pg_password}@postgres:5432/messenger",
"cache_url": f"redis://:{redis_password}@redis:6379/0",
}.items():
path = directory / name
if not path.exists():
path.write_text(value + "\n", encoding="utf-8", newline="\n")
env = ROOT / ".env"
if not env.exists():
env.write_bytes((ROOT / ".env.example").read_bytes())
print("PASS: local .env and secret files exist; existing values preserved")
if __name__ == "__main__":
main()
+94
View File
@@ -0,0 +1,94 @@
"""Docker operations scoped to the configured local Compose project."""
import csv
import io
import json
import os
from pathlib import Path
import socket
import subprocess
import time
ROOT = Path(__file__).resolve().parents[1]
def command(args, timeout=120, env=None):
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=timeout, env=env)
if result.returncode:
# Commands and stderr can contain secrets. Keep detailed errors local.
raise RuntimeError("Local command failed: " + " ".join(args[:3]) + "\n" + result.stderr[-1500:])
return result.stdout.strip()
def compose(*args, timeout=120, env=None):
return command(["docker", "compose", "--env-file", ".env", *args], timeout, env)
def project():
return os.getenv("COMPOSE_PROJECT_NAME") or json.loads(compose("config", "--format", "json"))["name"]
def containers(service="api", running=True):
args = ["docker", "ps"] + ([] if running else ["-a"])
ids = command(args + ["--filter", "label=com.docker.compose.project=" + project(),
"--filter", "label=com.docker.compose.service=" + service,
"--filter", "label=com.docker.compose.oneoff=False", "--format", "{{.ID}}"])
return json.loads(command(["docker", "inspect", *ids.split()])) if ids else []
def summary():
result = []
for service in ("api", "postgres", "redis", "minio", "lb"):
for row in containers(service, running=False):
result.append({"service": service, "id": row["Id"], "name": row["Name"].lstrip("/"),
"started_at": row["State"]["StartedAt"], "status": row["State"]["Status"],
"image": row["Image"], "user": row["Config"]["User"],
"memory_bytes": row["HostConfig"]["Memory"], "nano_cpus": row["HostConfig"]["NanoCpus"],
"published_ports": row["HostConfig"]["PortBindings"],
"volumes": [m["Name"] for m in row["Mounts"] if m["Type"] == "volume"]})
return result
def runtime(line):
with socket.create_connection(("lb", 9999), timeout=3) as sock:
sock.sendall((line + "\n").encode())
sock.shutdown(socket.SHUT_WR)
chunks = []
while True:
data = sock.recv(65536)
if not data:
return b"".join(chunks).decode()
chunks.append(data)
def drain(address):
lines = runtime("show servers state messenger").splitlines()
fields = next(line[2:].split() for line in lines if line.startswith("# "))
for line in lines:
values = line.split()
if not values or line.startswith("#") or len(values) != len(fields):
continue
row = dict(zip(fields, values))
if row["srv_addr"] == address:
slot = "messenger/" + row["srv_name"]
runtime("set server " + slot + " state drain")
# Let requests already assigned to this replica finish (HTTP budget 5 s).
deadline = time.monotonic() + 7
while time.monotonic() < deadline:
stats = list(csv.DictReader(io.StringIO(runtime("show stat").lstrip("# "))))
active = [s for s in stats if s["pxname"] == "messenger" and s["svname"] == row["srv_name"]]
if active and int(active[0]["scur"]) == 0:
return slot
time.sleep(.1)
raise RuntimeError("Replica did not drain within 7 seconds")
raise RuntimeError("Backend address was not found in HAProxy")
if __name__ == "__main__":
import sys
if sys.argv[1] == "drain":
print(drain(sys.argv[2]))
elif sys.argv[1] == "ready":
runtime("set server " + sys.argv[2] + " state ready")
elif sys.argv[1] == "summary":
print(json.dumps(summary(), indent=2))
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose --env-file .env restart --timeout 6 api postgres redis minio lb
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python scripts/stop_one.py
+24
View File
@@ -0,0 +1,24 @@
"""Stop exactly the replica selected by the public failover scenario."""
import json
import os
import time
from ops import ROOT, command, containers
def main():
target = os.environ["TARGET_INSTANCE_ID"]
matches = [row for row in containers() if target.startswith(row["Config"]["Hostname"] + "-")]
if len(matches) != 1:
raise RuntimeError("Target does not identify exactly one running API replica")
row = matches[0]
started = time.monotonic()
command(["docker", "stop", "--time", "6", row["Id"]], timeout=9)
state = json.loads(command(["docker", "inspect", row["Id"]]))[0]["State"]
assert not state["Running"] and state["ExitCode"] == 0, "Graceful shutdown failed"
report = {"target_instance": target, "container": row["Id"], "exit_code": state["ExitCode"],
"stop_seconds": round(time.monotonic() - started, 3), "remains_stopped": True}
(ROOT / "evidence" / "last-stop.json").write_text(json.dumps(report, indent=2) + "\n")
if __name__ == "__main__":
main()
+42
View File
@@ -0,0 +1,42 @@
"""Build once per image and run all required local acceptance checks."""
import json
from pathlib import Path
import subprocess
import sys
import time
from ops import ROOT, compose, summary
def run(name, args):
started=time.monotonic()
with (ROOT/'evidence'/name).open('w',encoding='utf-8') as output:
result=subprocess.run(args,cwd=ROOT,stdout=output,stderr=subprocess.STDOUT)
if result.returncode: raise RuntimeError('Check failed; see evidence/'+name)
return round(time.monotonic()-started,3)
def main():
run('init.txt',[sys.executable,'scripts/init_local.py'])
run('build.txt',['docker','compose','build','api','checks','minio'])
run('startup.txt',['docker','compose','up','-d','--no-build','--wait','--wait-timeout','180'])
checks={}
checks['smoke_seconds']=run('smoke.txt',['docker','compose','run','--rm','--no-deps','checks','make','check','test'])
checks['student_seconds']=run('student-tests.txt',['docker','compose','run','--rm','--no-deps','checks','make','test-student'])
before=summary()
checks['persistence_seconds']=run('persistence.txt',['docker','compose','run','--rm','--no-deps','checks','sh','-ec','chmod +x scripts/restart.sh; make test-persistence ACTION_SCRIPT=scripts/restart.sh'])
after=summary()
for row in before:
matching=next(x for x in after if x['id']==row['id'])
assert matching['started_at']!=row['started_at']
assert matching['volumes']==row['volumes']
(ROOT/'evidence/restart.json').write_text(json.dumps({'before':before,'after':after},indent=2))
for i in (1,2):
checks[f'failover_{i}_seconds']=run(f'failover-{i}.txt',['docker','compose','run','--rm','--no-deps','checks','sh','-ec','chmod +x scripts/stop-one.sh; make test-failover ACTION_SCRIPT=scripts/stop-one.sh'])
(ROOT/f'evidence/stop-{i}.json').write_bytes((ROOT/'evidence/last-stop.json').read_bytes())
compose('start','api'); time.sleep(7)
checks['uploader_failover_seconds']=run('uploader-failover.txt',['docker','compose','run','--rm','--no-deps','checks','python','scripts/check_uploader_failover.py'])
time.sleep(7)
checks['final_smoke_seconds']=run('smoke-final.txt',['docker','compose','run','--rm','--no-deps','checks','make','test'])
checks['status']='PASS'; checks['containers']=summary()
(ROOT/'evidence/verification.json').write_text(json.dumps(checks,indent=2))
print('PASS: lab4 acceptance checks completed')
if __name__=='__main__': main()