#!/bin/sh # Exercise the two-store reconciler without breaking real data. # # 1. An old pending row plus its object must be removed; a dry run must not. # 2. A fresh pending row must survive the grace period. # 3. An object with no metadata row (orphan) must be removed. # 4. A ready attachment referenced by PostgreSQL must never be removed. set -eu cd "$(dirname "$0")/.." COMPOSE="docker compose --env-file .env" TMP_DIR="$PWD/evidence/.tmp-reconcile.$$" mkdir -p "$TMP_DIR" chmod 700 "$TMP_DIR" TMP_ENV="$TMP_DIR/env" umask 077 cat > "$TMP_ENV" <&2 exit 2 } # shellcheck disable=SC1090 . "$TMP_ENV" POSTGRES_USER=$(sed -n 's/^POSTGRES_USER=//p' .env) POSTGRES_DB=$(sed -n 's/^POSTGRES_DB=//p' .env) POSTGRES_USER=${POSTGRES_USER:-messenger} POSTGRES_DB=${POSTGRES_DB:-messenger} network=$(docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}' "$($COMPOSE ps -q haproxy)") [ -n "$network" ] || { echo "cannot determine the Compose network; is the stack running?" >&2 exit 2 } # The long-running reconciler has a 900 s grace period. Pause it so the probe # rows cannot be removed by a background pass while they are being asserted. $COMPOSE stop reconciler >/dev/null psql_value() { $COMPOSE exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "$1" | tr -d '\r' } rc_shell() { docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \ --entrypoint /bin/sh rustfs/rc:v0.1.36 -c \ 'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null && '"$1" } object_exists() { rc_shell "rc head --quiet \"app/\$S3_BUCKET/$1\" >/dev/null" >/dev/null 2>&1 } put_object() { docker run --rm --network "$network" --env-file "$TMP_ENV" -e HOME=/tmp \ -v "$PWD/tests/fixtures:/fixtures:ro" -e KEY="$1" \ --entrypoint /bin/sh rustfs/rc:v0.1.36 -c \ 'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null && rc cp --quiet /fixtures/sample.png "app/$S3_BUCKET/$KEY"' } remove_object() { rc_shell "rc rm -f --quiet \"app/\$S3_BUCKET/$1\"" >/dev/null 2>&1 || true } insert_pending() { id=$1 key=$2 age_sql=$3 psql_value "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, created_at, updated_at) VALUES ('$id', '$CHAT_ID', '$OWNER_ID', 'probe.png', 'image/png', $(wc -c < tests/fixtures/sample.png), decode('$(sha256sum tests/fixtures/sample.png | cut -d' ' -f1)', 'hex'), '$key', 'pending', now() - $age_sql, now() - $age_sql)" >/dev/null } run_reconciler() { $COMPOSE run --rm --no-deps -T reconciler --once "$@" } cleanup() { for id in ${STALE_ID:-} ${FRESH_ID:-} ${READY_ID:-}; do [ -n "$id" ] || continue psql_value "DELETE FROM attachments WHERE id = '$id'" >/dev/null 2>&1 || true done for key in ${STALE_KEY:-} ${FRESH_KEY:-} ${READY_KEY:-}; do [ -n "$key" ] || continue remove_object "$key" done $COMPOSE start reconciler >/dev/null 2>&1 || true rm -rf "$TMP_DIR" } trap cleanup EXIT INT TERM # A baseline chat/user pair makes the inserted rows valid without depending on # data left by an earlier test run. ids=$(docker run --rm --network "$network" -v "$PWD:/work:ro" -w /work \ python:3.12-alpine python3 -c ' import sys sys.path.insert(0, "tests") from client import Client alice, _, _, chat = Client("http://haproxy:8080").bootstrap() print(chat["id"], alice["id"]) ') CHAT_ID=$(printf '%s' "$ids" | cut -d' ' -f1) OWNER_ID=$(printf '%s' "$ids" | cut -d' ' -f2) echo "1/4 old pending row is visible to dry-run but unchanged" STALE_ID=$(python3 -c 'import uuid; print(uuid.uuid4())') STALE_KEY="attachments/$CHAT_ID/$STALE_ID" put_object "$STALE_KEY" insert_pending "$STALE_ID" "$STALE_KEY" "interval '1 hour'" dry=$(run_reconciler --dry-run --grace=1s 2>&1) printf '%s\n' "$dry" | grep -q "would remove stale pending attachment" || { echo "dry run did not report the stale pending attachment" >&2 exit 1 } [ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$STALE_ID'")" = "1" ] || { echo "dry run deleted the pending row" >&2 exit 1 } object_exists "$STALE_KEY" || { echo "dry run deleted the pending object" >&2 exit 1 } echo " dry-run reported it and changed neither store" echo "2/4 old pending row and object are removed by a real pass" run_reconciler --grace=1s 2>&1 | tee "$TMP_DIR/stale.log" [ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$STALE_ID'")" = "0" ] || { echo "stale pending row survived" >&2 exit 1 } if object_exists "$STALE_KEY"; then echo "stale pending object survived" >&2 exit 1 fi echo " row and object removed" echo "3/4 fresh pending row survives the grace period" FRESH_ID=$(python3 -c 'import uuid; print(uuid.uuid4())') FRESH_KEY="attachments/$CHAT_ID/$FRESH_ID" put_object "$FRESH_KEY" insert_pending "$FRESH_ID" "$FRESH_KEY" "interval '0 seconds'" run_reconciler --grace=1s 2>&1 | tee "$TMP_DIR/fresh.log" [ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$FRESH_ID'")" = "1" ] || { echo "fresh pending row was removed" >&2 exit 1 } object_exists "$FRESH_KEY" || { echo "fresh pending object was removed" >&2 exit 1 } echo " fresh row and object kept" # Remove the row only; the object now has no metadata row and becomes an # orphan. A very small grace period makes the old object eligible immediately. psql_value "DELETE FROM attachments WHERE id = '$FRESH_ID'" >/dev/null run_reconciler --grace=1ns 2>&1 | tee "$TMP_DIR/orphan.log" if object_exists "$FRESH_KEY"; then echo "orphan object survived" >&2 exit 1 fi echo " row removed, orphan object cleaned up" echo "4/4 ready attachment is not touched" READY_ID=$(docker run --rm --network "$network" -v "$PWD:/work:ro" -w /work \ python:3.12-alpine python3 -c ' import sys sys.path.insert(0, "tests") from client import Client c = Client("http://haproxy:8080") alice, _, _, chat = c.bootstrap() att, _ = c.attachment(alice, chat) print(att["id"]) ') READY_KEY=$(psql_value "SELECT object_key FROM attachments WHERE id = '$READY_ID'") [ -n "$READY_KEY" ] || { echo "could not read the ready object key" >&2 exit 1 } run_reconciler --grace=1ns 2>&1 | tee "$TMP_DIR/ready.log" [ "$(psql_value "SELECT count(*) FROM attachments WHERE id = '$READY_ID'")" = "1" ] || { echo "ready attachment row was removed" >&2 exit 1 } object_exists "$READY_KEY" || { echo "ready attachment object was removed" >&2 exit 1 } echo " ready metadata and bytes kept" echo "PASS: reconciler removed only abandoned data and respected the grace period"