-- 0004 (lab 4): attachments. Metadata live here, the bytes in the private S3 bucket. -- Additive (new tables, a relaxed CHECK, a NEW function post_message_v2), so replicas of the -- lab 3 version keep working while it is applied (they never send attachment_ids). -- -- Consistency between the two stores (docs/CONSISTENCY.md): -- 1. INSERT the row as status='pending' (intent: the server-generated object_key is known -- BEFORE anything is written to S3) and COMMIT; -- 2. PUT the object into S3; -- 3. UPDATE status='ready' WHERE status='pending'; only then 201. -- Invariant: an object never exists without a row that names its key (row first, object -- second; on cleanup object first, row second). A crash/outage between the steps leaves a -- 'pending' row; the janitor deletes its object (idempotent) and then the row once it is older -- than PENDING_GRACE_SECONDS. The reconciler (python -m messenger.reconcile) checks the -- invariant over the whole bucket. CREATE TABLE attachments ( id uuid PRIMARY KEY, chat_id uuid NOT NULL, owner_id uuid NOT NULL, filename text NOT NULL, -- display name only, never a path or a key content_type text NOT NULL, -- detected from the bytes, not from the header size_bytes integer NOT NULL, sha256 bytea NOT NULL, width integer NOT NULL, height integer NOT NULL, -- Random server-generated key ("o/" + 128 random bits in hex), unrelated to id/filename. -- COLLATE "C": byte order = S3 ListObjectsV2 order, the reconciler merge-joins both lists. object_key text COLLATE "C" NOT NULL, status text NOT NULL DEFAULT 'pending', created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), -- Set by the reconciler when the object of a READY row is missing or has the wrong size: -- the row is quarantined (content -> 404, cannot be attached), never deleted, and the mark -- is removed again if the object comes back (restore from backup). lost_at timestamptz, lost_reason text, -- The uploader must be a member of the chat (enforced by the database too). CONSTRAINT attachments_owner_is_member FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members (chat_id, user_id), CONSTRAINT attachments_object_key_key UNIQUE (object_key), CONSTRAINT attachments_object_key_format CHECK (object_key ~ '^o/[0-9a-f]{32}$'), CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready')), CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), CONSTRAINT attachments_sha256_len CHECK (octet_length(sha256) = 32), CONSTRAINT attachments_pixels CHECK (width >= 1 AND height >= 1 AND width::bigint * height <= 40000000), CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), CONSTRAINT attachments_lost CHECK ((lost_at IS NULL) = (lost_reason IS NULL) AND (lost_at IS NULL OR status = 'ready')) ); -- Janitor: stale intents, oldest first; tiny partial index (only rows of unfinished uploads). CREATE INDEX attachments_pending_created ON attachments (created_at) WHERE status = 'pending'; -- Which attachments a message references, in the order of attachment_ids. CREATE TABLE message_attachments ( message_id uuid NOT NULL REFERENCES messages (id), position smallint NOT NULL, attachment_id uuid NOT NULL REFERENCES attachments (id), PRIMARY KEY (message_id, position), CONSTRAINT message_attachments_unique UNIQUE (message_id, attachment_id), CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9) ); -- "Is this attachment used by any message?" (janitor/reconciler never touch used ones). CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id); -- Lab 4: text may be "" when the message has attachments (the API enforces "text or files"). -- DROP CONSTRAINT takes a short exclusive lock on messages and the new CHECK scans the table -- once, inside this migration's transaction (seconds for a lab-sized table; for a big table -- this would be ADD ... NOT VALID + VALIDATE in a separate migration). ALTER TABLE messages DROP CONSTRAINT messages_text_len; ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (octet_length(text) <= 16000); -- post_message_v2: post_message (0003) + attachment_ids, still ONE statement and ONE transaction. -- The attachments are checked BEFORE the key is claimed and the key is claimed only if they are -- fine, so a failed check never leaves a key behind. A replay (same key, same body) returns the -- stored message even if an attachment was quarantined since: the result of a key is final. -- -- o_status: created | replayed | conflict | not_found | retry | attachment_not_found | attachment_conflict CREATE FUNCTION post_message_v2(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid, p_text bytea, p_ttl_seconds integer, p_attachments uuid[]) RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz, o_attachments uuid[]) LANGUAGE plpgsql VOLATILE AS $$ DECLARE v_created timestamptz; v_found integer; v_usable integer; v_check text := 'ok'; BEGIN PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user; IF NOT FOUND THEN RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[]; RETURN; END IF; IF cardinality(p_attachments) > 0 THEN -- Visible = ready attachment of THIS chat (pending, foreign or unknown -> 404). -- Usable = visible and not quarantined (lost -> 409, like a failed attachment). -- FOR SHARE: a concurrent quarantine waits until this message is committed. SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL) INTO v_found, v_usable FROM (SELECT a.lost_at FROM attachments a WHERE a.id = ANY (p_attachments) AND a.chat_id = p_chat AND a.status = 'ready' FOR SHARE) a; IF v_found <> cardinality(p_attachments) THEN v_check := 'attachment_not_found'; ELSIF v_usable <> v_found THEN v_check := 'attachment_conflict'; END IF; END IF; IF v_check = 'ok' THEN INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id) VALUES (p_user, p_chat, p_key, p_hash, p_message) ON CONFLICT (user_id, chat_id, key) DO UPDATE SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds); IF FOUND THEN UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond') WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created; INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (p_message, p_chat, p_user, p_text, v_created); INSERT INTO message_attachments (message_id, position, attachment_id) SELECT p_message, (a.ord - 1)::smallint, a.id FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord); RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created, p_attachments; RETURN; END IF; END IF; RETURN QUERY SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at, ARRAY(SELECT ma.attachment_id FROM message_attachments ma WHERE ma.message_id = m.id ORDER BY ma.position) FROM idempotency_keys k JOIN messages m ON m.id = k.message_id WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key; IF NOT FOUND THEN RETURN QUERY SELECT CASE WHEN v_check = 'ok' THEN 'retry' ELSE v_check END, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[]; END IF; END $$;