forked from mtusi2026/mtusi2026lab4
Добавил загрузку картинок в S3 и проверку доступа к вложениям
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
"""Stop the API that actually accepted the binary upload, then download via LB."""
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path[:0]=[str(ROOT/'tests'),str(ROOT/'scripts')]
|
||||
from client import Client, Failure
|
||||
from ops import containers, command, compose
|
||||
|
||||
def main():
|
||||
c=Client(timeout=5)
|
||||
alice,bob,eve,chat=c.bootstrap()
|
||||
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=failover.png',201,
|
||||
actor=alice,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')
|
||||
instance=response['headers']['X-Instance-Id']
|
||||
matches=[r for r in containers() if instance.startswith(r['Config']['Hostname']+'-')]
|
||||
assert len(matches)==1
|
||||
victim=matches[0]['Id']; started=time.monotonic()
|
||||
command(['docker','stop','--time','6',victim])
|
||||
try:
|
||||
deadline=time.monotonic()+10
|
||||
while True:
|
||||
try:
|
||||
result=c.call('GET','/api/v1/attachments/'+response['json']['id']+'/content',actor=bob)
|
||||
break
|
||||
except Failure:
|
||||
if time.monotonic()>deadline: raise
|
||||
time.sleep(.2)
|
||||
assert result['headers']['X-Instance-Id']!=instance
|
||||
assert result['bytes']==raw
|
||||
assert command(['docker','inspect','--format','{{.State.Running}}',victim])=='false'
|
||||
report={'upload_instance':instance,'stopped_container':victim,
|
||||
'download_instance':result['headers']['X-Instance-Id'],
|
||||
'sha256':hashlib.sha256(raw).hexdigest(),'size_bytes':len(raw),
|
||||
'stop_and_download_seconds':round(time.monotonic()-started,3),'status':'PASS'}
|
||||
(ROOT/'evidence/uploader-failover.json').write_text(json.dumps(report,indent=2))
|
||||
finally:
|
||||
compose('start','api')
|
||||
print('PASS: original downloaded through another replica after uploader stopped')
|
||||
|
||||
if __name__=='__main__': main()
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Create local synthetic credentials once without printing secret values."""
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
directory = ROOT / "secrets"
|
||||
directory.mkdir(exist_ok=True)
|
||||
for name in ("postgres_password", "redis_password", "cursor_key", "s3_root_password", "s3_secret_key"):
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(secrets.token_hex(32) + "\n", encoding="utf-8", newline="\n")
|
||||
else:
|
||||
# Shell command substitution on Linux must not retain a Windows CR.
|
||||
value = path.read_text(encoding="utf-8").strip()
|
||||
path.write_text(value + "\n", encoding="utf-8", newline="\n")
|
||||
pg_password = (directory / "postgres_password").read_text().strip()
|
||||
redis_password = (directory / "redis_password").read_text().strip()
|
||||
for name, value in {
|
||||
"database_url": f"postgresql://messenger:{pg_password}@postgres:5432/messenger",
|
||||
"cache_url": f"redis://:{redis_password}@redis:6379/0",
|
||||
}.items():
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(value + "\n", encoding="utf-8", newline="\n")
|
||||
env = ROOT / ".env"
|
||||
if not env.exists():
|
||||
env.write_bytes((ROOT / ".env.example").read_bytes())
|
||||
print("PASS: local .env and secret files exist; existing values preserved")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Docker operations scoped to the configured local Compose project."""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(args, timeout=120, env=None):
|
||||
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding="utf-8", errors="replace", timeout=timeout, env=env)
|
||||
if result.returncode:
|
||||
# Commands and stderr can contain secrets. Keep detailed errors local.
|
||||
raise RuntimeError("Local command failed: " + " ".join(args[:3]) + "\n" + result.stderr[-1500:])
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def compose(*args, timeout=120, env=None):
|
||||
return command(["docker", "compose", "--env-file", ".env", *args], timeout, env)
|
||||
|
||||
|
||||
def project():
|
||||
return os.getenv("COMPOSE_PROJECT_NAME") or json.loads(compose("config", "--format", "json"))["name"]
|
||||
|
||||
|
||||
def containers(service="api", running=True):
|
||||
args = ["docker", "ps"] + ([] if running else ["-a"])
|
||||
ids = command(args + ["--filter", "label=com.docker.compose.project=" + project(),
|
||||
"--filter", "label=com.docker.compose.service=" + service,
|
||||
"--filter", "label=com.docker.compose.oneoff=False", "--format", "{{.ID}}"])
|
||||
return json.loads(command(["docker", "inspect", *ids.split()])) if ids else []
|
||||
|
||||
|
||||
def summary():
|
||||
result = []
|
||||
for service in ("api", "postgres", "redis", "minio", "lb"):
|
||||
for row in containers(service, running=False):
|
||||
result.append({"service": service, "id": row["Id"], "name": row["Name"].lstrip("/"),
|
||||
"started_at": row["State"]["StartedAt"], "status": row["State"]["Status"],
|
||||
"image": row["Image"], "user": row["Config"]["User"],
|
||||
"memory_bytes": row["HostConfig"]["Memory"], "nano_cpus": row["HostConfig"]["NanoCpus"],
|
||||
"published_ports": row["HostConfig"]["PortBindings"],
|
||||
"volumes": [m["Name"] for m in row["Mounts"] if m["Type"] == "volume"]})
|
||||
return result
|
||||
|
||||
|
||||
def runtime(line):
|
||||
with socket.create_connection(("lb", 9999), timeout=3) as sock:
|
||||
sock.sendall((line + "\n").encode())
|
||||
sock.shutdown(socket.SHUT_WR)
|
||||
chunks = []
|
||||
while True:
|
||||
data = sock.recv(65536)
|
||||
if not data:
|
||||
return b"".join(chunks).decode()
|
||||
chunks.append(data)
|
||||
|
||||
|
||||
def drain(address):
|
||||
lines = runtime("show servers state messenger").splitlines()
|
||||
fields = next(line[2:].split() for line in lines if line.startswith("# "))
|
||||
for line in lines:
|
||||
values = line.split()
|
||||
if not values or line.startswith("#") or len(values) != len(fields):
|
||||
continue
|
||||
row = dict(zip(fields, values))
|
||||
if row["srv_addr"] == address:
|
||||
slot = "messenger/" + row["srv_name"]
|
||||
runtime("set server " + slot + " state drain")
|
||||
# Let requests already assigned to this replica finish (HTTP budget 5 s).
|
||||
deadline = time.monotonic() + 7
|
||||
while time.monotonic() < deadline:
|
||||
stats = list(csv.DictReader(io.StringIO(runtime("show stat").lstrip("# "))))
|
||||
active = [s for s in stats if s["pxname"] == "messenger" and s["svname"] == row["srv_name"]]
|
||||
if active and int(active[0]["scur"]) == 0:
|
||||
return slot
|
||||
time.sleep(.1)
|
||||
raise RuntimeError("Replica did not drain within 7 seconds")
|
||||
raise RuntimeError("Backend address was not found in HAProxy")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
if sys.argv[1] == "drain":
|
||||
print(drain(sys.argv[2]))
|
||||
elif sys.argv[1] == "ready":
|
||||
runtime("set server " + sys.argv[2] + " state ready")
|
||||
elif sys.argv[1] == "summary":
|
||||
print(json.dumps(summary(), indent=2))
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env restart --timeout 6 api postgres redis minio lb
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python scripts/stop_one.py
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Stop exactly the replica selected by the public failover scenario."""
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from ops import ROOT, command, containers
|
||||
|
||||
|
||||
def main():
|
||||
target = os.environ["TARGET_INSTANCE_ID"]
|
||||
matches = [row for row in containers() if target.startswith(row["Config"]["Hostname"] + "-")]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError("Target does not identify exactly one running API replica")
|
||||
row = matches[0]
|
||||
started = time.monotonic()
|
||||
command(["docker", "stop", "--time", "6", row["Id"]], timeout=9)
|
||||
state = json.loads(command(["docker", "inspect", row["Id"]]))[0]["State"]
|
||||
assert not state["Running"] and state["ExitCode"] == 0, "Graceful shutdown failed"
|
||||
report = {"target_instance": target, "container": row["Id"], "exit_code": state["ExitCode"],
|
||||
"stop_seconds": round(time.monotonic() - started, 3), "remains_stopped": True}
|
||||
(ROOT / "evidence" / "last-stop.json").write_text(json.dumps(report, indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Build once per image and run all required local acceptance checks."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from ops import ROOT, compose, summary
|
||||
|
||||
def run(name, args):
|
||||
started=time.monotonic()
|
||||
with (ROOT/'evidence'/name).open('w',encoding='utf-8') as output:
|
||||
result=subprocess.run(args,cwd=ROOT,stdout=output,stderr=subprocess.STDOUT)
|
||||
if result.returncode: raise RuntimeError('Check failed; see evidence/'+name)
|
||||
return round(time.monotonic()-started,3)
|
||||
|
||||
def main():
|
||||
run('init.txt',[sys.executable,'scripts/init_local.py'])
|
||||
run('build.txt',['docker','compose','build','api','checks','minio'])
|
||||
run('startup.txt',['docker','compose','up','-d','--no-build','--wait','--wait-timeout','180'])
|
||||
checks={}
|
||||
checks['smoke_seconds']=run('smoke.txt',['docker','compose','run','--rm','--no-deps','checks','make','check','test'])
|
||||
checks['student_seconds']=run('student-tests.txt',['docker','compose','run','--rm','--no-deps','checks','make','test-student'])
|
||||
before=summary()
|
||||
checks['persistence_seconds']=run('persistence.txt',['docker','compose','run','--rm','--no-deps','checks','sh','-ec','chmod +x scripts/restart.sh; make test-persistence ACTION_SCRIPT=scripts/restart.sh'])
|
||||
after=summary()
|
||||
for row in before:
|
||||
matching=next(x for x in after if x['id']==row['id'])
|
||||
assert matching['started_at']!=row['started_at']
|
||||
assert matching['volumes']==row['volumes']
|
||||
(ROOT/'evidence/restart.json').write_text(json.dumps({'before':before,'after':after},indent=2))
|
||||
for i in (1,2):
|
||||
checks[f'failover_{i}_seconds']=run(f'failover-{i}.txt',['docker','compose','run','--rm','--no-deps','checks','sh','-ec','chmod +x scripts/stop-one.sh; make test-failover ACTION_SCRIPT=scripts/stop-one.sh'])
|
||||
(ROOT/f'evidence/stop-{i}.json').write_bytes((ROOT/'evidence/last-stop.json').read_bytes())
|
||||
compose('start','api'); time.sleep(7)
|
||||
checks['uploader_failover_seconds']=run('uploader-failover.txt',['docker','compose','run','--rm','--no-deps','checks','python','scripts/check_uploader_failover.py'])
|
||||
time.sleep(7)
|
||||
checks['final_smoke_seconds']=run('smoke-final.txt',['docker','compose','run','--rm','--no-deps','checks','make','test'])
|
||||
checks['status']='PASS'; checks['containers']=summary()
|
||||
(ROOT/'evidence/verification.json').write_text(json.dumps(checks,indent=2))
|
||||
print('PASS: lab4 acceptance checks completed')
|
||||
|
||||
if __name__=='__main__': main()
|
||||
Reference in New Issue
Block a user