forked from mtusi2026/mtusi2026lab4
200 lines
6.7 KiB
Bash
Executable File
200 lines
6.7 KiB
Bash
Executable File
#!/bin/sh
|
|
# Exercise the two-store reconciler without breaking real data.
|
|
#
|
|
# 1. An old pending row plus its object must be removed; a dry run must not.
|
|
# 2. A fresh pending row must survive the grace period.
|
|
# 3. An object with no metadata row (orphan) must be removed.
|
|
# 4. A ready attachment referenced by PostgreSQL must never be removed.
|
|
set -eu
|
|
cd "$(dirname "$0")/.."
|
|
|
|
COMPOSE="docker compose --env-file .env"
|
|
TMP_DIR="$PWD/evidence/.tmp-reconcile.$$"
|
|
mkdir -p "$TMP_DIR"
|
|
chmod 700 "$TMP_DIR"
|
|
TMP_ENV="$TMP_DIR/env"
|
|
umask 077
|
|
cat > "$TMP_ENV" <<EOF
|
|
S3_ENDPOINT=$(sed -n 's/^S3_ENDPOINT=//p' .env)
|
|
S3_BUCKET=$(sed -n 's/^S3_BUCKET=//p' .env)
|
|
S3_ACCESS_KEY_ID=$(sed -n 's/^S3_ACCESS_KEY_ID=//p' .env)
|
|
S3_SECRET_ACCESS_KEY=$(sed -n 's/^S3_SECRET_ACCESS_KEY=//p' .env)
|
|
EOF
|
|
[ -s "$TMP_ENV" ] || {
|
|
echo ".env is missing S3 settings" >&2
|
|
exit 2
|
|
}
|
|
# shellcheck disable=SC1090
|
|
. "$TMP_ENV"
|
|
|
|
POSTGRES_USER=$(sed -n 's/^POSTGRES_USER=//p' .env)
|
|
POSTGRES_DB=$(sed -n 's/^POSTGRES_DB=//p' .env)
|
|
POSTGRES_USER=${POSTGRES_USER:-messenger}
|
|
POSTGRES_DB=${POSTGRES_DB:-messenger}
|
|
|
|
network=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$($COMPOSE ps -q haproxy)")
|
|
[ -n "$network" ] || {
|
|
echo "cannot determine the Compose network; is the stack running?" >&2
|
|
exit 2
|
|
}
|
|
|
|
# The long-running reconciler has a 900 s grace period. Pause it so the probe
|
|
# rows cannot be removed by a background pass while they are being asserted.
|
|
$COMPOSE stop reconciler >/dev/null
|
|
|
|
psql_value() {
|
|
$COMPOSE exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "$1" | tr -d '\r'
|
|
}
|
|
|
|
rc_shell() {
|
|
docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \
|
|
--entrypoint /bin/sh rustfs/rc:v0.1.36 -c \
|
|
'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null && '"$1"
|
|
}
|
|
|
|
object_exists() {
|
|
rc_shell "rc head --quiet \"app/\$S3_BUCKET/$1\" >/dev/null" >/dev/null 2>&1
|
|
}
|
|
|
|
put_object() {
|
|
docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \
|
|
-v "$PWD/tests/fixtures:/fixtures:ro" -e KEY="$1" \
|
|
--entrypoint /bin/sh rustfs/rc:v0.1.36 -c \
|
|
'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null &&
|
|
rc cp --quiet /fixtures/sample.png "app/$S3_BUCKET/$KEY"'
|
|
}
|
|
|
|
remove_object() {
|
|
rc_shell "rc rm -f --quiet \"app/\$S3_BUCKET/$1\"" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
insert_pending() {
|
|
id=$1
|
|
key=$2
|
|
age_sql=$3
|
|
psql_value "INSERT INTO attachments
|
|
(id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
|
object_key, status, created_at, updated_at)
|
|
VALUES ('$id', '$CHAT_ID', '$OWNER_ID', 'probe.png', 'image/png',
|
|
$(wc -c < tests/fixtures/sample.png), decode('$(sha256sum tests/fixtures/sample.png | cut -d' ' -f1)', 'hex'),
|
|
'$key', 'pending', now() - $age_sql, now() - $age_sql)" >/dev/null
|
|
}
|
|
|
|
run_reconciler() {
|
|
$COMPOSE run --rm --no-deps -T reconciler --once "$@"
|
|
}
|
|
|
|
cleanup() {
|
|
for id in ${STALE_ID:-} ${FRESH_ID:-} ${READY_ID:-}; do
|
|
[ -n "$id" ] || continue
|
|
psql_value "DELETE FROM attachments WHERE id = '$id'" >/dev/null 2>&1 || true
|
|
done
|
|
for key in ${STALE_KEY:-} ${FRESH_KEY:-} ${READY_KEY:-}; do
|
|
[ -n "$key" ] || continue
|
|
remove_object "$key"
|
|
done
|
|
$COMPOSE start reconciler >/dev/null 2>&1 || true
|
|
rm -rf "$TMP_DIR"
|
|
}
|
|
trap cleanup EXIT INT TERM
|
|
|
|
# A baseline chat/user pair makes the inserted rows valid without depending on
|
|
# data left by an earlier test run.
|
|
ids=$(docker run --rm --network "$network" -v "$PWD:/work:ro" -w /work \
|
|
python:3.12-alpine python3 -c '
|
|
import sys
|
|
sys.path.insert(0, "tests")
|
|
from client import Client
|
|
alice, _, _, chat = Client("http://haproxy:8080").bootstrap()
|
|
print(chat["id"], alice["id"])
|
|
')
|
|
CHAT_ID=$(printf '%s' "$ids" | cut -d' ' -f1)
|
|
OWNER_ID=$(printf '%s' "$ids" | cut -d' ' -f2)
|
|
|
|
echo "1/4 old pending row is visible to dry-run but unchanged"
|
|
STALE_ID=$(python3 -c 'import uuid; print(uuid.uuid4())')
|
|
STALE_KEY="attachments/$CHAT_ID/$STALE_ID"
|
|
put_object "$STALE_KEY"
|
|
insert_pending "$STALE_ID" "$STALE_KEY" "interval '1 hour'"
|
|
dry=$(run_reconciler --dry-run --grace=1s 2>&1)
|
|
printf '%s\n' "$dry" | grep -q "would remove stale pending attachment" || {
|
|
echo "dry run did not report the stale pending attachment" >&2
|
|
exit 1
|
|
}
|
|
[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$STALE_ID'")" = "1" ] || {
|
|
echo "dry run deleted the pending row" >&2
|
|
exit 1
|
|
}
|
|
object_exists "$STALE_KEY" || {
|
|
echo "dry run deleted the pending object" >&2
|
|
exit 1
|
|
}
|
|
echo " dry-run reported it and changed neither store"
|
|
|
|
echo "2/4 old pending row and object are removed by a real pass"
|
|
run_reconciler --grace=1s 2>&1 | tee "$TMP_DIR/stale.log"
|
|
[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$STALE_ID'")" = "0" ] || {
|
|
echo "stale pending row survived" >&2
|
|
exit 1
|
|
}
|
|
if object_exists "$STALE_KEY"; then
|
|
echo "stale pending object survived" >&2
|
|
exit 1
|
|
fi
|
|
echo " row and object removed"
|
|
|
|
echo "3/4 fresh pending row survives the grace period"
|
|
FRESH_ID=$(python3 -c 'import uuid; print(uuid.uuid4())')
|
|
FRESH_KEY="attachments/$CHAT_ID/$FRESH_ID"
|
|
put_object "$FRESH_KEY"
|
|
insert_pending "$FRESH_ID" "$FRESH_KEY" "interval '0 seconds'"
|
|
run_reconciler --grace=1s 2>&1 | tee "$TMP_DIR/fresh.log"
|
|
[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$FRESH_ID'")" = "1" ] || {
|
|
echo "fresh pending row was removed" >&2
|
|
exit 1
|
|
}
|
|
object_exists "$FRESH_KEY" || {
|
|
echo "fresh pending object was removed" >&2
|
|
exit 1
|
|
}
|
|
echo " fresh row and object kept"
|
|
|
|
# Remove the row only; the object now has no metadata row and becomes an
|
|
# orphan. A very small grace period makes the old object eligible immediately.
|
|
psql_value "DELETE FROM attachments WHERE id = '$FRESH_ID'" >/dev/null
|
|
run_reconciler --grace=1ns 2>&1 | tee "$TMP_DIR/orphan.log"
|
|
if object_exists "$FRESH_KEY"; then
|
|
echo "orphan object survived" >&2
|
|
exit 1
|
|
fi
|
|
echo " row removed, orphan object cleaned up"
|
|
|
|
echo "4/4 ready attachment is not touched"
|
|
READY_ID=$(docker run --rm --network "$network" -v "$PWD:/work:ro" -w /work \
|
|
python:3.12-alpine python3 -c '
|
|
import sys
|
|
sys.path.insert(0, "tests")
|
|
from client import Client
|
|
c = Client("http://haproxy:8080")
|
|
alice, _, _, chat = c.bootstrap()
|
|
att, _ = c.attachment(alice, chat)
|
|
print(att["id"])
|
|
')
|
|
READY_KEY=$(psql_value "SELECT object_key FROM attachments WHERE id = '$READY_ID'")
|
|
[ -n "$READY_KEY" ] || {
|
|
echo "could not read the ready object key" >&2
|
|
exit 1
|
|
}
|
|
run_reconciler --grace=1ns 2>&1 | tee "$TMP_DIR/ready.log"
|
|
[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$READY_ID'")" = "1" ] || {
|
|
echo "ready attachment row was removed" >&2
|
|
exit 1
|
|
}
|
|
object_exists "$READY_KEY" || {
|
|
echo "ready attachment object was removed" >&2
|
|
exit 1
|
|
}
|
|
echo " ready metadata and bytes kept"
|
|
|
|
echo "PASS: reconciler removed only abandoned data and respected the grace period"
|