forked from mtusi2026/mtusi2026lab4
feat(app) implemented lab 4
This commit is contained in:
Executable
+104
@@ -0,0 +1,104 @@
|
||||
#!/bin/sh
|
||||
# Prove the bucket is private outside the API and that the API credential is
|
||||
# scoped. The probe object is created with the RustFS root credential, fetched
|
||||
# anonymously (must be 403), accessed with the scoped credential inside the
|
||||
# allowed prefix, and removed again.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
COMPOSE="docker compose --env-file .env"
|
||||
TMP_DIR="$PWD/evidence/.tmp-private-bucket.$$"
|
||||
mkdir -p "$TMP_DIR"
|
||||
chmod 700 "$TMP_DIR"
|
||||
TMP_ENV="$TMP_DIR/env"
|
||||
umask 077
|
||||
# AGENT: only the S3 settings go into the temporary env file. The scoped
|
||||
# credential is intentionally used for most checks; the root pair is needed
|
||||
# only to create and remove the probe object.
|
||||
cat > "$TMP_ENV" <<EOF
|
||||
S3_ENDPOINT=${S3_ENDPOINT:-http://rustfs:9000}
|
||||
S3_BUCKET=${S3_BUCKET:-messenger-attachments}
|
||||
S3_ROOT_ACCESS_KEY=$(sed -n 's/^S3_ROOT_ACCESS_KEY=//p' .env)
|
||||
S3_ROOT_SECRET_KEY=$(sed -n 's/^S3_ROOT_SECRET_KEY=//p' .env)
|
||||
S3_ACCESS_KEY_ID=$(sed -n 's/^S3_ACCESS_KEY_ID=//p' .env)
|
||||
S3_SECRET_ACCESS_KEY=$(sed -n 's/^S3_SECRET_ACCESS_KEY=//p' .env)
|
||||
EOF
|
||||
|
||||
cleanup() {
|
||||
docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \
|
||||
-e PROBE="$PROBE" \
|
||||
--entrypoint /bin/sh rustfs/rc:v0.1.36 -c \
|
||||
'rc alias set root "$S3_ENDPOINT" "$S3_ROOT_ACCESS_KEY" "$S3_ROOT_SECRET_KEY" >/dev/null 2>&1 && rc rm -f --quiet "root/$S3_BUCKET/$PROBE"' \
|
||||
>/dev/null 2>&1 || true
|
||||
rm -rf "$TMP_DIR"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
network=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$($COMPOSE ps -q haproxy)")
|
||||
[ -n "$network" ] || {
|
||||
echo "cannot determine the Compose network; is the stack running?" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
# shellcheck disable=SC1090
|
||||
. "$TMP_ENV"
|
||||
PROBE="attachments/_privacy_probe_$$"
|
||||
URL="http://rustfs:9000/$S3_BUCKET/$PROBE"
|
||||
|
||||
echo "1/4 create a probe object with the root credential"
|
||||
docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \
|
||||
-v "$PWD/tests/fixtures:/fixtures:ro" -e PROBE="$PROBE" \
|
||||
--entrypoint /bin/sh rustfs/rc:v0.1.36 -c \
|
||||
'rc alias set root "$S3_ENDPOINT" "$S3_ROOT_ACCESS_KEY" "$S3_ROOT_SECRET_KEY" >/dev/null &&
|
||||
rc cp --quiet /fixtures/sample.png "root/$S3_BUCKET/$PROBE"'
|
||||
|
||||
echo "2/4 anonymous object GET must be denied"
|
||||
anon_code=$(docker run --rm --network "$network" -e URL="$URL" python:3.12-alpine \
|
||||
python3 -c '
|
||||
import os, sys, urllib.error, urllib.request
|
||||
try:
|
||||
with urllib.request.urlopen(os.environ["URL"], timeout=5) as response:
|
||||
print(response.status)
|
||||
except urllib.error.HTTPError as error:
|
||||
print(error.code)
|
||||
except Exception as error:
|
||||
print(type(error).__name__, file=sys.stderr)
|
||||
print(0)
|
||||
')
|
||||
[ "$anon_code" = "403" ] || {
|
||||
echo "anonymous object GET returned $anon_code, expected 403" >&2
|
||||
exit 1
|
||||
}
|
||||
echo " anonymous GET -> 403"
|
||||
|
||||
echo "3/4 scoped credential works inside attachments/ and is denied elsewhere"
|
||||
docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \
|
||||
-v "$PWD/tests/fixtures:/fixtures:ro" -e PROBE="$PROBE" \
|
||||
--entrypoint /bin/sh rustfs/rc:v0.1.36 -c \
|
||||
'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null &&
|
||||
rc cp --quiet /fixtures/sample.png "app/$S3_BUCKET/$PROBE" &&
|
||||
rc head --quiet "app/$S3_BUCKET/$PROBE" >/dev/null'
|
||||
echo " scoped PUT/HEAD inside attachments/ -> allowed"
|
||||
|
||||
if docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \
|
||||
-v "$PWD/tests/fixtures:/fixtures:ro" -e PROBE="$PROBE" \
|
||||
--entrypoint /bin/sh rustfs/rc:v0.1.36 -c \
|
||||
'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null &&
|
||||
rc cp --quiet /fixtures/sample.png "app/$S3_BUCKET/outside-$PROBE"' >/dev/null 2>&1; then
|
||||
echo "scoped credential wrote outside attachments/, expected denial" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo " scoped PUT outside attachments/ -> denied"
|
||||
|
||||
if docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \
|
||||
--entrypoint /bin/sh rustfs/rc:v0.1.36 -c \
|
||||
'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null &&
|
||||
rc bucket create -p "app/forbidden-'"$$"'"' >/dev/null 2>&1; then
|
||||
echo "scoped credential created a bucket, expected denial" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo " scoped CreateBucket -> denied"
|
||||
|
||||
echo "4/4 cleanup the probe object on exit"
|
||||
chmod 700 "$TMP_DIR" 2>/dev/null || true
|
||||
echo "PASS: bucket is private and the API credential is limited to attachments/*"
|
||||
Executable
+174
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure API container memory while concurrent uploads run.
|
||||
|
||||
The HTTP workload runs in a throwaway Python container attached to the Compose
|
||||
network. This host-side script samples the Docker Engine stats API at a short
|
||||
interval, because the repository shell cannot always reach published ports and
|
||||
`docker stats --no-stream` is too slow to catch a short upload burst.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DOCKER_SOCKET = "/var/run/docker.sock"
|
||||
|
||||
|
||||
def docker_json(path):
|
||||
request = f"GET {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
|
||||
sock.connect(DOCKER_SOCKET)
|
||||
sock.sendall(request.encode())
|
||||
data = bytearray()
|
||||
while True:
|
||||
part = sock.recv(65536)
|
||||
if not part:
|
||||
break
|
||||
data.extend(part)
|
||||
head, _, body = bytes(data).partition(b"\r\n\r\n")
|
||||
if b"transfer-encoding: chunked" not in head.lower():
|
||||
return json.loads(body)
|
||||
|
||||
# The Engine API uses chunked transfer encoding for stats and inspect.
|
||||
decoded = bytearray()
|
||||
position = 0
|
||||
while position < len(body):
|
||||
line_end = body.find(b"\r\n", position)
|
||||
if line_end < 0:
|
||||
break
|
||||
size = int(body[position:line_end].split(b";", 1)[0], 16)
|
||||
if size == 0:
|
||||
break
|
||||
start = line_end + 2
|
||||
decoded.extend(body[start : start + size])
|
||||
position = start + size + 2
|
||||
return json.loads(bytes(decoded))
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.check_output(
|
||||
["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, text=True
|
||||
).strip()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--uploads", type=int, default=10)
|
||||
parser.add_argument("--width", type=int, default=1800)
|
||||
parser.add_argument("--height", type=int, default=1400)
|
||||
parser.add_argument("--interval", type=float, default=0.05)
|
||||
parser.add_argument("--output", type=Path, default=ROOT / "evidence/upload-memory.json")
|
||||
args = parser.parse_args()
|
||||
|
||||
api_ids = [line for line in compose("ps", "-q", "api").splitlines() if line]
|
||||
if len(api_ids) < 2:
|
||||
raise SystemExit("at least two running API replicas are required")
|
||||
names = {
|
||||
container_id: docker_json(f"/containers/{container_id}/json")["Name"].lstrip("/")
|
||||
for container_id in api_ids
|
||||
}
|
||||
network = docker_json(
|
||||
f"/containers/{compose('ps', '-q', 'haproxy')}/json"
|
||||
)["NetworkSettings"]["Networks"]
|
||||
network_name = next(iter(network))
|
||||
|
||||
stop = threading.Event()
|
||||
samples = []
|
||||
|
||||
def sample():
|
||||
while not stop.is_set():
|
||||
per_container = {}
|
||||
for container_id in api_ids:
|
||||
stats = docker_json(
|
||||
f"/containers/{container_id}/stats?stream=false&one-shot=true"
|
||||
)
|
||||
per_container[names[container_id]] = stats.get("memory_stats", {}).get(
|
||||
"usage", 0
|
||||
)
|
||||
samples.append(
|
||||
{
|
||||
"at": time.time(),
|
||||
"total": sum(per_container.values()),
|
||||
"per_container": per_container,
|
||||
}
|
||||
)
|
||||
time.sleep(args.interval)
|
||||
|
||||
sampler = threading.Thread(target=sample, daemon=True)
|
||||
sampler.start()
|
||||
time.sleep(0.3) # capture an idle baseline before the burst
|
||||
|
||||
command = [
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--network",
|
||||
network_name,
|
||||
"-v",
|
||||
f"{ROOT}:/work:ro",
|
||||
"-w",
|
||||
"/work",
|
||||
"python:3.12-alpine",
|
||||
"python3",
|
||||
"tests/custom/concurrent_uploads.py",
|
||||
"--base-url",
|
||||
"http://haproxy:8080",
|
||||
"--uploads",
|
||||
str(args.uploads),
|
||||
"--width",
|
||||
str(args.width),
|
||||
"--height",
|
||||
str(args.height),
|
||||
]
|
||||
started = time.monotonic()
|
||||
try:
|
||||
output = subprocess.check_output(command, text=True)
|
||||
finally:
|
||||
stop.set()
|
||||
sampler.join(timeout=2)
|
||||
|
||||
workload = json.loads(output.strip().splitlines()[-1])
|
||||
baseline = samples[0]["total"] if samples else 0
|
||||
peak = max(samples, key=lambda item: item["total"]) if samples else {"total": 0}
|
||||
report = {
|
||||
"workload": workload,
|
||||
"sampling_interval_seconds": args.interval,
|
||||
"samples": len(samples),
|
||||
"wall_seconds": round(time.monotonic() - started, 3),
|
||||
"baseline_total_mib": round(baseline / (1024 * 1024), 1),
|
||||
"peak_total_mib": round(peak["total"] / (1024 * 1024), 1),
|
||||
"peak_per_container_mib": {
|
||||
name: round(value / (1024 * 1024), 1)
|
||||
for name, value in peak["per_container"].items()
|
||||
},
|
||||
"api_memory_limit_mib": None,
|
||||
}
|
||||
# The limit is configured in .env; keep the evidence honest by reading the
|
||||
# effective value from Compose rather than repeating a default.
|
||||
configured = subprocess.check_output(
|
||||
[
|
||||
"docker",
|
||||
"inspect",
|
||||
"-f",
|
||||
"{{.HostConfig.Memory}}",
|
||||
api_ids[0],
|
||||
],
|
||||
text=True,
|
||||
).strip()
|
||||
limit_bytes = int(configured or 0)
|
||||
report["api_memory_limit_mib"] = (
|
||||
round(limit_bytes / (1024 * 1024), 1) if limit_bytes else None
|
||||
)
|
||||
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, indent=2) + "\n")
|
||||
print(json.dumps(report, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env restart postgres redis haproxy api
|
||||
docker compose --env-file .env restart postgres redis rustfs haproxy api
|
||||
|
||||
Executable
+54
@@ -0,0 +1,54 @@
|
||||
#!/bin/sh
|
||||
# Verify the lab4 durability claim: upload through the load balancer, stop the
|
||||
# API replica that served the upload, then download the same attachment through
|
||||
# another replica and compare the SHA-256. The stopped replica is restored on
|
||||
# exit so the stand is left as it was found.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
COMPOSE="docker compose --env-file .env"
|
||||
STATE_DIR="$PWD/evidence/.tmp-attachment-failover.$$"
|
||||
mkdir -p "$STATE_DIR"
|
||||
chmod 700 "$STATE_DIR"
|
||||
|
||||
restore() {
|
||||
rm -rf "$STATE_DIR"
|
||||
$COMPOSE up -d --scale api=2 api >/dev/null 2>&1 || true
|
||||
attempts=0
|
||||
while [ "$attempts" -lt 60 ]; do
|
||||
ids=$($COMPOSE ps -q api)
|
||||
count=$(printf '%s\n' "$ids" | sed '/^$/d' | wc -l)
|
||||
healthy=$(for id in $ids; do docker inspect -f '{{.State.Health.Status}}' "$id"; done | grep -c '^healthy$' || true)
|
||||
if [ "$count" -eq 2 ] && [ "$healthy" -eq 2 ]; then
|
||||
return 0
|
||||
fi
|
||||
attempts=$((attempts + 1))
|
||||
sleep 1
|
||||
done
|
||||
echo "warning: API replicas did not return to two healthy instances" >&2
|
||||
}
|
||||
trap restore EXIT INT TERM
|
||||
|
||||
network=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$($COMPOSE ps -q haproxy)")
|
||||
[ -n "$network" ] || {
|
||||
echo "cannot determine the Compose network; is the stack running?" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
run_python() {
|
||||
docker run --rm --network "$network" \
|
||||
-v "$PWD:/work:ro" -v "$STATE_DIR:/state" -w /work \
|
||||
python:3.12-alpine \
|
||||
python3 tests/custom/attachment_flow.py --base-url http://haproxy:8080 "$@"
|
||||
}
|
||||
|
||||
upload_json=$(run_python --action upload --state /state/upload.json)
|
||||
printf '%s\n' "$upload_json"
|
||||
instance=$(printf '%s' "$upload_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["uploaded_by"])')
|
||||
|
||||
echo "stopping API replica $instance"
|
||||
TARGET_INSTANCE_ID="$instance" scripts/stop-one.sh
|
||||
|
||||
run_python --action download --state /state/upload.json --timeout 30
|
||||
|
||||
echo "PASS: attachment survived the stop of the replica that accepted it"
|
||||
Executable
+92
@@ -0,0 +1,92 @@
|
||||
#!/bin/sh
|
||||
# A client that declares a large body and disconnects halfway must not leave a
|
||||
# pending metadata row, an S3 object or a temp file behind. The handler creates
|
||||
# the pending row only after the whole bounded body has been copied, so this is
|
||||
# the expected clean failure.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
COMPOSE="docker compose --env-file .env"
|
||||
network=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$($COMPOSE ps -q haproxy)")
|
||||
[ -n "$network" ] || {
|
||||
echo "cannot determine the Compose network; is the stack running?" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
POSTGRES_USER=$(sed -n 's/^POSTGRES_USER=//p' .env)
|
||||
POSTGRES_DB=$(sed -n 's/^POSTGRES_DB=//p' .env)
|
||||
POSTGRES_USER=${POSTGRES_USER:-messenger}
|
||||
POSTGRES_DB=${POSTGRES_DB:-messenger}
|
||||
|
||||
count_rows() {
|
||||
$COMPOSE exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc \
|
||||
"SELECT count(*) FROM attachments" | tr -d '\r'
|
||||
}
|
||||
|
||||
count_objects() {
|
||||
docker run --rm --network "$network" --env-file .env -e HOME=/tmp \
|
||||
--entrypoint /bin/sh rustfs/rc:v0.1.36 -c \
|
||||
'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null &&
|
||||
rc bucket list -r --no-color "app/$S3_BUCKET/attachments/" | wc -l' | tr -d '\r '
|
||||
}
|
||||
|
||||
rows_before=$(count_rows)
|
||||
objects_before=$(count_objects)
|
||||
|
||||
# The helper prints the chat id and token for this one short-lived probe. The
|
||||
# token is passed through the environment, not written to disk or evidence.
|
||||
state=$(docker run --rm --network "$network" -v "$PWD:/work:ro" -w /work \
|
||||
python:3.12-alpine python3 -c '
|
||||
import sys
|
||||
sys.path.insert(0, "tests")
|
||||
from client import Client
|
||||
alice, _, _, chat = Client("http://haproxy:8080").bootstrap()
|
||||
print(chat["id"] + " " + alice["token"])
|
||||
')
|
||||
chat_id=$(printf '%s' "$state" | cut -d' ' -f1)
|
||||
token=$(printf '%s' "$state" | cut -d' ' -f2)
|
||||
|
||||
docker run --rm --network "$network" -e CHAT_ID="$chat_id" -e TOKEN="$token" \
|
||||
python:3.12-alpine python3 -c '
|
||||
import os, socket, time
|
||||
host, port = "haproxy", 8080
|
||||
request = (
|
||||
"POST /api/v1/chats/" + os.environ["CHAT_ID"] + "/attachments?filename=cut.png HTTP/1.1\r\n"
|
||||
"Host: haproxy\r\n"
|
||||
"Authorization: Bearer " + os.environ["TOKEN"] + "\r\n"
|
||||
"Content-Type: image/png\r\n"
|
||||
"Content-Length: 5000000\r\n"
|
||||
"Connection: close\r\n\r\n"
|
||||
).encode()
|
||||
with socket.create_connection((host, port), timeout=5) as sock:
|
||||
sock.sendall(request)
|
||||
sock.sendall(b"\x89PNG\r\n\x1a\n" + b"\x00" * 1024)
|
||||
time.sleep(0.2)
|
||||
# close without sending the declared body
|
||||
'
|
||||
sleep 2
|
||||
|
||||
rows_after=$(count_rows)
|
||||
objects_after=$(count_objects)
|
||||
temp_files=0
|
||||
for id in $($COMPOSE ps -q api); do
|
||||
count=$(docker exec "$id" sh -c 'find /tmp/messenger-uploads -type f 2>/dev/null | wc -l' | tr -d '\r ')
|
||||
temp_files=$((temp_files + count))
|
||||
done
|
||||
|
||||
echo "attachments rows: $rows_before -> $rows_after"
|
||||
echo "stored objects: $objects_before -> $objects_after"
|
||||
echo "temp files left: $temp_files"
|
||||
[ "$rows_before" = "$rows_after" ] || {
|
||||
echo "interrupted upload left a metadata row" >&2
|
||||
exit 1
|
||||
}
|
||||
[ "$objects_before" = "$objects_after" ] || {
|
||||
echo "interrupted upload left an S3 object" >&2
|
||||
exit 1
|
||||
}
|
||||
[ "$temp_files" = "0" ] || {
|
||||
echo "interrupted upload left a temporary file" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "PASS: interrupted upload left no row, object or temp file"
|
||||
Executable
+199
@@ -0,0 +1,199 @@
|
||||
#!/bin/sh
|
||||
# Exercise the two-store reconciler without breaking real data.
|
||||
#
|
||||
# 1. An old pending row plus its object must be removed; a dry run must not.
|
||||
# 2. A fresh pending row must survive the grace period.
|
||||
# 3. An object with no metadata row (orphan) must be removed.
|
||||
# 4. A ready attachment referenced by PostgreSQL must never be removed.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
COMPOSE="docker compose --env-file .env"
|
||||
TMP_DIR="$PWD/evidence/.tmp-reconcile.$$"
|
||||
mkdir -p "$TMP_DIR"
|
||||
chmod 700 "$TMP_DIR"
|
||||
TMP_ENV="$TMP_DIR/env"
|
||||
umask 077
|
||||
cat > "$TMP_ENV" <<EOF
|
||||
S3_ENDPOINT=$(sed -n 's/^S3_ENDPOINT=//p' .env)
|
||||
S3_BUCKET=$(sed -n 's/^S3_BUCKET=//p' .env)
|
||||
S3_ACCESS_KEY_ID=$(sed -n 's/^S3_ACCESS_KEY_ID=//p' .env)
|
||||
S3_SECRET_ACCESS_KEY=$(sed -n 's/^S3_SECRET_ACCESS_KEY=//p' .env)
|
||||
EOF
|
||||
[ -s "$TMP_ENV" ] || {
|
||||
echo ".env is missing S3 settings" >&2
|
||||
exit 2
|
||||
}
|
||||
# shellcheck disable=SC1090
|
||||
. "$TMP_ENV"
|
||||
|
||||
POSTGRES_USER=$(sed -n 's/^POSTGRES_USER=//p' .env)
|
||||
POSTGRES_DB=$(sed -n 's/^POSTGRES_DB=//p' .env)
|
||||
POSTGRES_USER=${POSTGRES_USER:-messenger}
|
||||
POSTGRES_DB=${POSTGRES_DB:-messenger}
|
||||
|
||||
network=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$($COMPOSE ps -q haproxy)")
|
||||
[ -n "$network" ] || {
|
||||
echo "cannot determine the Compose network; is the stack running?" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
# The long-running reconciler has a 900 s grace period. Pause it so the probe
|
||||
# rows cannot be removed by a background pass while they are being asserted.
|
||||
$COMPOSE stop reconciler >/dev/null
|
||||
|
||||
psql_value() {
|
||||
$COMPOSE exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "$1" | tr -d '\r'
|
||||
}
|
||||
|
||||
rc_shell() {
|
||||
docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \
|
||||
--entrypoint /bin/sh rustfs/rc:v0.1.36 -c \
|
||||
'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null && '"$1"
|
||||
}
|
||||
|
||||
object_exists() {
|
||||
rc_shell "rc head --quiet \"app/\$S3_BUCKET/$1\" >/dev/null" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
put_object() {
|
||||
docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \
|
||||
-v "$PWD/tests/fixtures:/fixtures:ro" -e KEY="$1" \
|
||||
--entrypoint /bin/sh rustfs/rc:v0.1.36 -c \
|
||||
'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null &&
|
||||
rc cp --quiet /fixtures/sample.png "app/$S3_BUCKET/$KEY"'
|
||||
}
|
||||
|
||||
remove_object() {
|
||||
rc_shell "rc rm -f --quiet \"app/\$S3_BUCKET/$1\"" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
insert_pending() {
|
||||
id=$1
|
||||
key=$2
|
||||
age_sql=$3
|
||||
psql_value "INSERT INTO attachments
|
||||
(id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
object_key, status, created_at, updated_at)
|
||||
VALUES ('$id', '$CHAT_ID', '$OWNER_ID', 'probe.png', 'image/png',
|
||||
$(wc -c < tests/fixtures/sample.png), decode('$(sha256sum tests/fixtures/sample.png | cut -d' ' -f1)', 'hex'),
|
||||
'$key', 'pending', now() - $age_sql, now() - $age_sql)" >/dev/null
|
||||
}
|
||||
|
||||
run_reconciler() {
|
||||
$COMPOSE run --rm --no-deps -T reconciler --once "$@"
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
for id in ${STALE_ID:-} ${FRESH_ID:-} ${READY_ID:-}; do
|
||||
[ -n "$id" ] || continue
|
||||
psql_value "DELETE FROM attachments WHERE id = '$id'" >/dev/null 2>&1 || true
|
||||
done
|
||||
for key in ${STALE_KEY:-} ${FRESH_KEY:-} ${READY_KEY:-}; do
|
||||
[ -n "$key" ] || continue
|
||||
remove_object "$key"
|
||||
done
|
||||
$COMPOSE start reconciler >/dev/null 2>&1 || true
|
||||
rm -rf "$TMP_DIR"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# A baseline chat/user pair makes the inserted rows valid without depending on
|
||||
# data left by an earlier test run.
|
||||
ids=$(docker run --rm --network "$network" -v "$PWD:/work:ro" -w /work \
|
||||
python:3.12-alpine python3 -c '
|
||||
import sys
|
||||
sys.path.insert(0, "tests")
|
||||
from client import Client
|
||||
alice, _, _, chat = Client("http://haproxy:8080").bootstrap()
|
||||
print(chat["id"], alice["id"])
|
||||
')
|
||||
CHAT_ID=$(printf '%s' "$ids" | cut -d' ' -f1)
|
||||
OWNER_ID=$(printf '%s' "$ids" | cut -d' ' -f2)
|
||||
|
||||
echo "1/4 old pending row is visible to dry-run but unchanged"
|
||||
STALE_ID=$(python3 -c 'import uuid; print(uuid.uuid4())')
|
||||
STALE_KEY="attachments/$CHAT_ID/$STALE_ID"
|
||||
put_object "$STALE_KEY"
|
||||
insert_pending "$STALE_ID" "$STALE_KEY" "interval '1 hour'"
|
||||
dry=$(run_reconciler --dry-run --grace=1s 2>&1)
|
||||
printf '%s\n' "$dry" | grep -q "would remove stale pending attachment" || {
|
||||
echo "dry run did not report the stale pending attachment" >&2
|
||||
exit 1
|
||||
}
|
||||
[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$STALE_ID'")" = "1" ] || {
|
||||
echo "dry run deleted the pending row" >&2
|
||||
exit 1
|
||||
}
|
||||
object_exists "$STALE_KEY" || {
|
||||
echo "dry run deleted the pending object" >&2
|
||||
exit 1
|
||||
}
|
||||
echo " dry-run reported it and changed neither store"
|
||||
|
||||
echo "2/4 old pending row and object are removed by a real pass"
|
||||
run_reconciler --grace=1s 2>&1 | tee "$TMP_DIR/stale.log"
|
||||
[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$STALE_ID'")" = "0" ] || {
|
||||
echo "stale pending row survived" >&2
|
||||
exit 1
|
||||
}
|
||||
if object_exists "$STALE_KEY"; then
|
||||
echo "stale pending object survived" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo " row and object removed"
|
||||
|
||||
echo "3/4 fresh pending row survives the grace period"
|
||||
FRESH_ID=$(python3 -c 'import uuid; print(uuid.uuid4())')
|
||||
FRESH_KEY="attachments/$CHAT_ID/$FRESH_ID"
|
||||
put_object "$FRESH_KEY"
|
||||
insert_pending "$FRESH_ID" "$FRESH_KEY" "interval '0 seconds'"
|
||||
run_reconciler --grace=1s 2>&1 | tee "$TMP_DIR/fresh.log"
|
||||
[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$FRESH_ID'")" = "1" ] || {
|
||||
echo "fresh pending row was removed" >&2
|
||||
exit 1
|
||||
}
|
||||
object_exists "$FRESH_KEY" || {
|
||||
echo "fresh pending object was removed" >&2
|
||||
exit 1
|
||||
}
|
||||
echo " fresh row and object kept"
|
||||
|
||||
# Remove the row only; the object now has no metadata row and becomes an
|
||||
# orphan. A very small grace period makes the old object eligible immediately.
|
||||
psql_value "DELETE FROM attachments WHERE id = '$FRESH_ID'" >/dev/null
|
||||
run_reconciler --grace=1ns 2>&1 | tee "$TMP_DIR/orphan.log"
|
||||
if object_exists "$FRESH_KEY"; then
|
||||
echo "orphan object survived" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo " row removed, orphan object cleaned up"
|
||||
|
||||
echo "4/4 ready attachment is not touched"
|
||||
READY_ID=$(docker run --rm --network "$network" -v "$PWD:/work:ro" -w /work \
|
||||
python:3.12-alpine python3 -c '
|
||||
import sys
|
||||
sys.path.insert(0, "tests")
|
||||
from client import Client
|
||||
c = Client("http://haproxy:8080")
|
||||
alice, _, _, chat = c.bootstrap()
|
||||
att, _ = c.attachment(alice, chat)
|
||||
print(att["id"])
|
||||
')
|
||||
READY_KEY=$(psql_value "SELECT object_key FROM attachments WHERE id = '$READY_ID'")
|
||||
[ -n "$READY_KEY" ] || {
|
||||
echo "could not read the ready object key" >&2
|
||||
exit 1
|
||||
}
|
||||
run_reconciler --grace=1ns 2>&1 | tee "$TMP_DIR/ready.log"
|
||||
[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$READY_ID'")" = "1" ] || {
|
||||
echo "ready attachment row was removed" >&2
|
||||
exit 1
|
||||
}
|
||||
object_exists "$READY_KEY" || {
|
||||
echo "ready attachment object was removed" >&2
|
||||
exit 1
|
||||
}
|
||||
echo " ready metadata and bytes kept"
|
||||
|
||||
echo "PASS: reconciler removed only abandoned data and respected the grace period"
|
||||
Reference in New Issue
Block a user