Files
mtusi2026lab5/scripts/verify-reconcile.sh
T

200 lines
6.7 KiB
Bash
Executable File

#!/bin/sh
# Exercise the two-store reconciler without breaking real data.
#
# 1. An old pending row plus its object must be removed; a dry run must not.
# 2. A fresh pending row must survive the grace period.
# 3. An object with no metadata row (orphan) must be removed.
# 4. A ready attachment referenced by PostgreSQL must never be removed.
set -eu
cd "$(dirname "$0")/.."
COMPOSE="docker compose --env-file .env"
TMP_DIR="$PWD/evidence/.tmp-reconcile.$$"
mkdir -p "$TMP_DIR"
chmod 700 "$TMP_DIR"
TMP_ENV="$TMP_DIR/env"
umask 077
cat > "$TMP_ENV" <<EOF
S3_ENDPOINT=$(sed -n 's/^S3_ENDPOINT=//p' .env)
S3_BUCKET=$(sed -n 's/^S3_BUCKET=//p' .env)
S3_ACCESS_KEY_ID=$(sed -n 's/^S3_ACCESS_KEY_ID=//p' .env)
S3_SECRET_ACCESS_KEY=$(sed -n 's/^S3_SECRET_ACCESS_KEY=//p' .env)
EOF
[ -s "$TMP_ENV" ] || {
echo ".env is missing S3 settings" >&2
exit 2
}
# shellcheck disable=SC1090
. "$TMP_ENV"
POSTGRES_USER=$(sed -n 's/^POSTGRES_USER=//p' .env)
POSTGRES_DB=$(sed -n 's/^POSTGRES_DB=//p' .env)
POSTGRES_USER=${POSTGRES_USER:-messenger}
POSTGRES_DB=${POSTGRES_DB:-messenger}
network=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$($COMPOSE ps -q haproxy)")
[ -n "$network" ] || {
echo "cannot determine the Compose network; is the stack running?" >&2
exit 2
}
# The long-running reconciler has a 900 s grace period. Pause it so the probe
# rows cannot be removed by a background pass while they are being asserted.
$COMPOSE stop reconciler >/dev/null
psql_value() {
$COMPOSE exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "$1" | tr -d '\r'
}
rc_shell() {
docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \
--entrypoint /bin/sh rustfs/rc:v0.1.36 -c \
'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null && '"$1"
}
object_exists() {
rc_shell "rc head --quiet \"app/\$S3_BUCKET/$1\" >/dev/null" >/dev/null 2>&1
}
put_object() {
docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \
-v "$PWD/tests/fixtures:/fixtures:ro" -e KEY="$1" \
--entrypoint /bin/sh rustfs/rc:v0.1.36 -c \
'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null &&
rc cp --quiet /fixtures/sample.png "app/$S3_BUCKET/$KEY"'
}
remove_object() {
rc_shell "rc rm -f --quiet \"app/\$S3_BUCKET/$1\"" >/dev/null 2>&1 || true
}
insert_pending() {
id=$1
key=$2
age_sql=$3
psql_value "INSERT INTO attachments
(id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, status, created_at, updated_at)
VALUES ('$id', '$CHAT_ID', '$OWNER_ID', 'probe.png', 'image/png',
$(wc -c < tests/fixtures/sample.png), decode('$(sha256sum tests/fixtures/sample.png | cut -d' ' -f1)', 'hex'),
'$key', 'pending', now() - $age_sql, now() - $age_sql)" >/dev/null
}
run_reconciler() {
$COMPOSE run --rm --no-deps -T reconciler --once "$@"
}
cleanup() {
for id in ${STALE_ID:-} ${FRESH_ID:-} ${READY_ID:-}; do
[ -n "$id" ] || continue
psql_value "DELETE FROM attachments WHERE id = '$id'" >/dev/null 2>&1 || true
done
for key in ${STALE_KEY:-} ${FRESH_KEY:-} ${READY_KEY:-}; do
[ -n "$key" ] || continue
remove_object "$key"
done
$COMPOSE start reconciler >/dev/null 2>&1 || true
rm -rf "$TMP_DIR"
}
trap cleanup EXIT INT TERM
# A baseline chat/user pair makes the inserted rows valid without depending on
# data left by an earlier test run.
ids=$(docker run --rm --network "$network" -v "$PWD:/work:ro" -w /work \
python:3.12-alpine python3 -c '
import sys
sys.path.insert(0, "tests")
from client import Client
alice, _, _, chat = Client("http://haproxy:8080").bootstrap()
print(chat["id"], alice["id"])
')
CHAT_ID=$(printf '%s' "$ids" | cut -d' ' -f1)
OWNER_ID=$(printf '%s' "$ids" | cut -d' ' -f2)
echo "1/4 old pending row is visible to dry-run but unchanged"
STALE_ID=$(python3 -c 'import uuid; print(uuid.uuid4())')
STALE_KEY="attachments/$CHAT_ID/$STALE_ID"
put_object "$STALE_KEY"
insert_pending "$STALE_ID" "$STALE_KEY" "interval '1 hour'"
dry=$(run_reconciler --dry-run --grace=1s 2>&1)
printf '%s\n' "$dry" | grep -q "would remove stale pending attachment" || {
echo "dry run did not report the stale pending attachment" >&2
exit 1
}
[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$STALE_ID'")" = "1" ] || {
echo "dry run deleted the pending row" >&2
exit 1
}
object_exists "$STALE_KEY" || {
echo "dry run deleted the pending object" >&2
exit 1
}
echo " dry-run reported it and changed neither store"
echo "2/4 old pending row and object are removed by a real pass"
run_reconciler --grace=1s 2>&1 | tee "$TMP_DIR/stale.log"
[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$STALE_ID'")" = "0" ] || {
echo "stale pending row survived" >&2
exit 1
}
if object_exists "$STALE_KEY"; then
echo "stale pending object survived" >&2
exit 1
fi
echo " row and object removed"
echo "3/4 fresh pending row survives the grace period"
FRESH_ID=$(python3 -c 'import uuid; print(uuid.uuid4())')
FRESH_KEY="attachments/$CHAT_ID/$FRESH_ID"
put_object "$FRESH_KEY"
insert_pending "$FRESH_ID" "$FRESH_KEY" "interval '0 seconds'"
run_reconciler --grace=1s 2>&1 | tee "$TMP_DIR/fresh.log"
[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$FRESH_ID'")" = "1" ] || {
echo "fresh pending row was removed" >&2
exit 1
}
object_exists "$FRESH_KEY" || {
echo "fresh pending object was removed" >&2
exit 1
}
echo " fresh row and object kept"
# Remove the row only; the object now has no metadata row and becomes an
# orphan. A very small grace period makes the old object eligible immediately.
psql_value "DELETE FROM attachments WHERE id = '$FRESH_ID'" >/dev/null
run_reconciler --grace=1ns 2>&1 | tee "$TMP_DIR/orphan.log"
if object_exists "$FRESH_KEY"; then
echo "orphan object survived" >&2
exit 1
fi
echo " row removed, orphan object cleaned up"
echo "4/4 ready attachment is not touched"
READY_ID=$(docker run --rm --network "$network" -v "$PWD:/work:ro" -w /work \
python:3.12-alpine python3 -c '
import sys
sys.path.insert(0, "tests")
from client import Client
c = Client("http://haproxy:8080")
alice, _, _, chat = c.bootstrap()
att, _ = c.attachment(alice, chat)
print(att["id"])
')
READY_KEY=$(psql_value "SELECT object_key FROM attachments WHERE id = '$READY_ID'")
[ -n "$READY_KEY" ] || {
echo "could not read the ready object key" >&2
exit 1
}
run_reconciler --grace=1ns 2>&1 | tee "$TMP_DIR/ready.log"
[ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$READY_ID'")" = "1" ] || {
echo "ready attachment row was removed" >&2
exit 1
}
object_exists "$READY_KEY" || {
echo "ready attachment object was removed" >&2
exit 1
}
echo " ready metadata and bytes kept"
echo "PASS: reconciler removed only abandoned data and respected the grace period"