forked from mtusi2026/mtusi2026lab4
reconcile-inject.sh and reconcile_check.py come from the prepared kit: the first breaks either side on purpose (an object without a row, a removed object, another size, the same size with other bytes, an old intent with and without its object), the second is an oracle outside the API that classifies every row and object from psql and rc, hashes objects with --deep and reports ready attachments lost against a saved set. reconcile_track.py demo uploads through the balancer, links some attachments to messages, injects each class on both sides with old and fresh variants, and checks that a deep dry-run finds exactly what was injected and changes nothing, that apply fixes exactly that and a second apply does nothing, that broken rows and their message links survive in quarantine with metadata 200 and content 404, that a restored object heals, that a second pass is skipped while the lock is held, and that passes in a loop during 75 s of uploads lose nothing. Intents are injected 300 s old, between the reconciler grace and the API sweep grace, so only the reconciler removes them. reconcile_track.py cost snapshots docker stats of the whole host, pins S3, PostgreSQL and the reconciler to their own CPUs, fills the stand to each N with seed_storage.py run in the API image (parallel PutObject and COPY), and records three dry runs per N and mode with a median table. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
38 lines
2.9 KiB
Bash
Executable File
38 lines
2.9 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
cd "$(dirname "$0")/.."
|
|
kind="${1:?usage: reconcile-inject.sh orphan|missing|corrupt|bitflip|stale-pending|pending-object|multipart [count]}"
|
|
count="${2:-1}"
|
|
age="${INJECT_AGE_SECONDS:-7200}"
|
|
prefix="${S3_PREFIX:-attachments/}"
|
|
sql() { docker compose exec -T postgres psql -v ON_ERROR_STOP=1 -qAt -F ' ' -U "${PGUSER:-messenger}" -d "${PGDATABASE:-messenger}" -c "$1"; }
|
|
s3() { docker compose run --rm --no-deps -T -e ROWS="$1" s3-cli sh -c "$2"; }
|
|
ready() { sql "SELECT id, object_key, size_bytes FROM attachments WHERE status = 'ready' ${INJECT_IDS:+AND id = ANY('{$INJECT_IDS}'::uuid[])} ORDER BY random() LIMIT $count"; }
|
|
pending() {
|
|
sql "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, status, created_at, updated_at)
|
|
SELECT g.id, s.chat_id, s.owner_id, 'injected.png', 'image/png', 2048, repeat('0', 64), 1, 1, '$prefix' || g.id, 'pending',
|
|
clock_timestamp() - make_interval(secs => $age), clock_timestamp() - make_interval(secs => $age)
|
|
FROM (SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1) s,
|
|
(SELECT gen_random_uuid() AS id FROM generate_series(1, $count)) g
|
|
RETURNING id, object_key"
|
|
}
|
|
fresh() { sql "SELECT gen_random_uuid(), '$prefix' || gen_random_uuid() FROM generate_series(1, $count)"; }
|
|
put_random='printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; head -c "${size:-2048}" /dev/urandom | rc pipe "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done'
|
|
case "$kind" in
|
|
orphan) rows=$(fresh); s3 "$rows" "$put_random" | sed "s/^/orphan /" ;;
|
|
missing) rows=$(ready); s3 "$rows" 'printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; rc object remove --force "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done' | sed "s/^/missing /" ;;
|
|
corrupt) rows=$(ready | awk '{print $1, $2, int($3 / 2) + 1}'); s3 "$rows" "$put_random" | sed "s/^/corrupt /" ;;
|
|
bitflip) rows=$(ready); s3 "$rows" "$put_random" | sed "s/^/bitflip /" ;;
|
|
stale-pending) pending | sed "s/^/stale-pending /" ;;
|
|
pending-object) rows=$(pending); s3 "$rows" "$put_random" | sed "s/^/pending-object /" ;;
|
|
multipart)
|
|
fresh | while read -r _ key; do
|
|
docker compose exec -T -e KEY="$key" -e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" s3 sh -c 'u="http://127.0.0.1:9000/$S3_BUCKET/$KEY"; a="$(cat /run/secrets/s3_root_user):$(cat /run/secrets/s3_root_password)"; s="aws:amz:${RUSTFS_REGION:-us-east-1}:s3"
|
|
upload=$(curl -fsS --aws-sigv4 "$s" --user "$a" -X POST "$u?uploads" | sed -n "s:.*<UploadId>\(.*\)</UploadId>.*:\1:p")
|
|
head -c 4096 /dev/urandom | curl -fsS --aws-sigv4 "$s" --user "$a" -X PUT --data-binary @- "$u?partNumber=1&uploadId=$upload" -o /dev/null' </dev/null
|
|
echo "multipart - $key"
|
|
done
|
|
;;
|
|
*) echo "unknown kind: $kind" >&2; exit 2 ;;
|
|
esac
|