[dialog] Fix backdrop clicks inside shadow roots and iframes

[atomiks]

Fixes #2867

This solution doesn't seem to work in CodeSandbox iframe.

I can't reproduce the issue locally with a local iframe in beta.3 to begin with though:

import * as React from "react";
import * as ReactDOM from "react-dom/client";
import { createPortal } from "react-dom";
import { Dialog } from "@base-ui-components/react/dialog";

/**
 * Dialog rendered entirely inside an iframe (similar to CodeSandbox behavior).
 * The whole React subtree (trigger + popup) is portaled into the iframe's body.
 */
export function ExampleDialog({
  portalContainer,
}: {
  portalContainer: HTMLElement | ShadowRoot; // generalized beyond ShadowRoot
}) {
  return (
    <Dialog.Root>
      <Dialog.Trigger>Open Dialog</Dialog.Trigger>
      <Dialog.Portal container={portalContainer}>
        <Dialog.Backdrop
          style={{ position: "fixed", inset: 0, background: "#00000022" }}
        />
        <Dialog.Popup
          style={{
            position: "fixed",
            top: "30%",
            border: "1px solid",
            background: "white",
            padding: 10,
          }}
        >
          <Dialog.Title>Title</Dialog.Title>
          <Dialog.Description>Dialog Description</Dialog.Description>

          <Dialog.Close>Close</Dialog.Close>
        </Dialog.Popup>
      </Dialog.Portal>
    </Dialog.Root>
  );
}

/**
 * Hosts the React dialog UI inside an iframe for isolation.
 */
function IFrameHost() {
  const [iframeEl, setIframeEl] = React.useState<HTMLIFrameElement | null>(
    null
  );
  const mountNode = iframeEl?.contentWindow?.document.body ?? null;

  const ensureDefinition = React.useCallback((doc: Document) => {
    const win = doc.defaultView;
    if (!win) return;
    if (win.customElements.get("webcomponent-example")) return;

    const OuterReactDOM = ReactDOM;

    class WebcomponentExample extends win.HTMLElement {
      private _shadow: ShadowRoot;
      private _mountEl?: HTMLDivElement;
      private _reactRoot?: ReturnType<typeof OuterReactDOM.createRoot>;
      constructor() {
        super();
        this._shadow = this.attachShadow({ mode: "open" });
      }
      connectedCallback() {
        if (this._reactRoot) return;
        this._mountEl = (this.ownerDocument || doc).createElement("div");
        this._shadow.appendChild(this._mountEl);
        this._reactRoot = OuterReactDOM.createRoot(this._mountEl);
        this._reactRoot.render(
          <ExampleDialog portalContainer={this._shadow} />
        );
      }
      disconnectedCallback() {
        if (this._reactRoot) {
          this._reactRoot.unmount();
          this._reactRoot = undefined;
        }
        if (this._mountEl && this._mountEl.parentNode) {
          this._mountEl.parentNode.removeChild(this._mountEl);
          this._mountEl = undefined;
        }
      }
    }

    win.customElements.define("webcomponent-example", WebcomponentExample);
  }, []);

  React.useLayoutEffect(() => {
    if (mountNode) {
      mountNode.style.margin = "0";
      ensureDefinition(mountNode.ownerDocument);
    }
  }, [mountNode, ensureDefinition]);

  return (
    <iframe
      ref={setIframeEl}
      // about:blank same-origin content; no srcDoc so load event is implicit
      width={"100%"}
      height={400}
      style={{ border: "1px solid #ccc" }}
      title="Dialog IFrame Sandbox"
    >
      {mountNode &&
        createPortal(
          // Render custom element after it's defined in that document
          React.createElement("webcomponent-example"),
          mountNode
        )}
    </iframe>
  );
}

export default function Example() {
  return <IFrameHost />;
}

[pkg-pr-new]

vite-css-base-ui-example

pnpm add https://pkg.pr.new/mui/base-ui/@base-ui-components/react@2871

pnpm add https://pkg.pr.new/mui/base-ui/@base-ui-components/utils@2871

commit: 46e1dcc

[netlify]

✅ Deploy Preview for base-ui ready!

Name	Link
🔨 Latest commit	46e1dcc
🔍 Latest deploy log	https://app.netlify.com/projects/base-ui/deploys/68dca707bbf6550008b190f6
😎 Deploy Preview	https://deploy-preview-2871--base-ui.netlify.app
📱 Preview on mobile	Toggle QR Code... Use your smartphone camera to open QR code link.

---

To edit notification comments on pull requests, go to your Netlify project configuration.

[mui-bot]

Bundle size report

Bundle	Parsed size	Gzip size
@base-ui-components/react	🔺+2B(0.00%)	🔺+9B(+0.01%)

Details of bundle changes