forked from mtusi2026/mtusi2026lab2
reconcile-inject.sh wrote an orphan under a key whose uuid differed from the id it printed, so the demo could only compare counts; it now uses one uuid for both, and the dry-run check compares the found and the injected sets per class. The heal step returns the original bytes under one quarantined key and random bytes of the same length under another, and checks that only the first leaves the quarantine. The load phase sums rechecked over its passes, and a new step stops a pass hanging on a paused S3 with docker stop and checks exit code 143 within seconds and a free lock afterwards. reconcile_track.py cost clears the run files of the previous series and measures 1k, 10k, 30k and 100k with --deep everywhere. The scenarios record date, commit and docker inspect of the replicas or of S3 at the moment they are down. make load-evidence and make clean-clone-evidence rerun the message load and the clean clone block with raw output, and make upload-memory passes ARGS to reproduce the run directories. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
38 lines
2.9 KiB
Bash
Executable File
38 lines
2.9 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
cd "$(dirname "$0")/.."
|
|
kind="${1:?usage: reconcile-inject.sh orphan|missing|corrupt|bitflip|stale-pending|pending-object|multipart [count]}"
|
|
count="${2:-1}"
|
|
age="${INJECT_AGE_SECONDS:-7200}"
|
|
prefix="${S3_PREFIX:-attachments/}"
|
|
sql() { docker compose exec -T postgres psql -v ON_ERROR_STOP=1 -qAt -F ' ' -U "${PGUSER:-messenger}" -d "${PGDATABASE:-messenger}" -c "$1"; }
|
|
s3() { docker compose run --rm --no-deps -T -e ROWS="$1" s3-cli sh -c "$2"; }
|
|
ready() { sql "SELECT id, object_key, size_bytes FROM attachments WHERE status = 'ready' ${INJECT_IDS:+AND id = ANY('{$INJECT_IDS}'::uuid[])} ORDER BY random() LIMIT $count"; }
|
|
pending() {
|
|
sql "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, status, created_at, updated_at)
|
|
SELECT g.id, s.chat_id, s.owner_id, 'injected.png', 'image/png', 2048, repeat('0', 64), 1, 1, '$prefix' || g.id, 'pending',
|
|
clock_timestamp() - make_interval(secs => $age), clock_timestamp() - make_interval(secs => $age)
|
|
FROM (SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1) s,
|
|
(SELECT gen_random_uuid() AS id FROM generate_series(1, $count)) g
|
|
RETURNING id, object_key"
|
|
}
|
|
fresh() { sql "SELECT g, '$prefix' || g FROM (SELECT gen_random_uuid() AS g FROM generate_series(1, $count)) s"; }
|
|
put_random='printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; head -c "${size:-2048}" /dev/urandom | rc pipe "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done'
|
|
case "$kind" in
|
|
orphan) rows=$(fresh); s3 "$rows" "$put_random" | sed "s/^/orphan /" ;;
|
|
missing) rows=$(ready); s3 "$rows" 'printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; rc object remove --force "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done' | sed "s/^/missing /" ;;
|
|
corrupt) rows=$(ready | awk '{print $1, $2, int($3 / 2) + 1}'); s3 "$rows" "$put_random" | sed "s/^/corrupt /" ;;
|
|
bitflip) rows=$(ready); s3 "$rows" "$put_random" | sed "s/^/bitflip /" ;;
|
|
stale-pending) pending | sed "s/^/stale-pending /" ;;
|
|
pending-object) rows=$(pending); s3 "$rows" "$put_random" | sed "s/^/pending-object /" ;;
|
|
multipart)
|
|
fresh | while read -r _ key; do
|
|
docker compose exec -T -e KEY="$key" -e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" s3 sh -c 'u="http://127.0.0.1:9000/$S3_BUCKET/$KEY"; a="$(cat /run/secrets/s3_root_user):$(cat /run/secrets/s3_root_password)"; s="aws:amz:${RUSTFS_REGION:-us-east-1}:s3"
|
|
upload=$(curl -fsS --aws-sigv4 "$s" --user "$a" -X POST "$u?uploads" | sed -n "s:.*<UploadId>\(.*\)</UploadId>.*:\1:p")
|
|
head -c 4096 /dev/urandom | curl -fsS --aws-sigv4 "$s" --user "$a" -X PUT --data-binary @- "$u?partNumber=1&uploadId=$upload" -o /dev/null' </dev/null
|
|
echo "multipart - $key"
|
|
done
|
|
;;
|
|
*) echo "unknown kind: $kind" >&2; exit 2 ;;
|
|
esac
|