feat(nextjs): Wait for the resumed part of a document before starting the pageload

CHANGELOG.md

@@ -6,7 +6,7 @@
 
 - **feat(nextjs): Never emit trace meta tags from a prerender ([#25141](https://github.com/getsentry/sentry-javascript/pull/25141))**
 
-  Pageloads of apps with `cacheComponents` enabled are connected to their server request again. Pages that Next.js prerenders no longer carry trace information, so a cached page can no longer connect visitors to a request that was not theirs. This also applies to ISR pages.
+  Pageloads of apps with `cacheComponents` enabled are connected to their server request again. Pages that Next.js prerenders no longer carry trace information, so a cached page can no longer connect visitors to a request that was not theirs. This also applies to ISR pages. The pageload waits for the trace information when it arrives with the part of the page that streams in after a cached shell.
 
 ## 11.7.0
 

dev-packages/e2e-tests/test-applications/nextjs-16-cacheComponents/app/blocking/page.tsx

@@ -0,0 +1,10 @@
+import { headers } from 'next/headers';
+
+// Opts out of the static shell: the whole page renders on the server for every request, so the
+// trace meta tags land in <head> like in a classic dynamic render.
+export const instant = false;
+
+export default async function Page() {
+  const requestHeaders = await headers();
+  return <h1 id="blocking">Blocking render for {requestHeaders.get('user-agent') ? 'a browser' : 'nobody'}</h1>;
+}

dev-packages/e2e-tests/test-applications/nextjs-16-cacheComponents/app/static-shell/page.tsx

@@ -0,0 +1,4 @@
+// No request data anywhere, so the whole page is a static shell and nothing renders per request.
+export default function Page() {
+  return <h1 id="static-shell">Static shell</h1>;
+}

dev-packages/e2e-tests/test-applications/nextjs-16-cacheComponents/tests/cacheOrigin-pageloadConnection.spec.ts

@@ -1,21 +1,24 @@
+import type { Page } from '@playwright/test';
 import { expect, test } from '@playwright/test';
 import { waitForTransaction } from '@sentry-internal/test-utils';
 
-test('connects the pageload trace to the runtime server request', async ({ page }) => {
-  // The resumed part of the document, which carries the tag, arrives after the SDK started. Enabled
-  // once the pageload instrumentation waits for it.
-  test.skip(true, 'Needs the deferred pageload start');
+const APP = 'nextjs-16-cacheComponents';
+
+// The server only writes `sentry-trace` meta tags for a render that happened for the current request,
+// never while it prerenders a shell. So a tag in the document can always be continued; the only
+// question is when it arrives. The resumed part of a prerendered shell arrives after the SDK started,
+// which the pageload instrumentation waits for.
 
+test('connects the pageload trace to the runtime server request', async ({ page }) => {
   // The shell is prerendered, but the page's dynamic part (`DynamicContent` awaits `headers()`)
   // runs on the server on every request and produces the `GET /pageload-tracing` transaction.
-  // A fully prerendered page would produce no server transaction.
-  const serverTxPromise = waitForTransaction('nextjs-16-cacheComponents', transactionEvent => {
+  const serverTxPromise = waitForTransaction(APP, transactionEvent => {
     return (
       transactionEvent.contexts?.trace?.op === 'http.server' && transactionEvent.transaction === 'GET /pageload-tracing'
     );
   });
 
-  const pageloadTxPromise = waitForTransaction('nextjs-16-cacheComponents', transactionEvent => {
+  const pageloadTxPromise = waitForTransaction(APP, transactionEvent => {
     return transactionEvent.contexts?.trace?.op === 'pageload' && transactionEvent.transaction === '/pageload-tracing';
   });
 
@@ -27,4 +30,83 @@ test('connects the pageload trace to the runtime server request', async ({ page
 
   expect(serverTx.contexts?.trace?.trace_id).toBeTruthy();
   expect(pageloadTx.contexts?.trace?.trace_id).toBe(serverTx.contexts?.trace?.trace_id);
+  expect(pageloadTx.contexts?.trace?.links).toBeUndefined();
+});
+
+test('connects the pageload trace to a blocking render', async ({ page }) => {
+  const serverTxPromise = waitForTransaction(APP, transactionEvent => {
+    return transactionEvent.contexts?.trace?.op === 'http.server' && transactionEvent.transaction === 'GET /blocking';
+  });
+
+  const pageloadTxPromise = waitForTransaction(APP, transactionEvent => {
+    return transactionEvent.contexts?.trace?.op === 'pageload' && transactionEvent.transaction === '/blocking';
+  });
+
+  await page.goto('/blocking');
+
+  await expect(page.locator('#blocking')).toHaveText('Blocking render for a browser');
+
+  const [serverTx, pageloadTx] = await Promise.all([serverTxPromise, pageloadTxPromise]);
+
+  // The whole page renders per request, so the tag in <head> is this request's and is continued.
+  expect(serverTx.contexts?.trace?.trace_id).toBeTruthy();
+  expect(pageloadTx.contexts?.trace?.trace_id).toBe(serverTx.contexts?.trace?.trace_id);
+  expect(pageloadTx.contexts?.trace?.links).toBeUndefined();
+});
+
+test('does not share a trace between visitors of a shell generated at request time', async ({ browser }) => {
+  test.skip(process.env.TEST_ENV !== 'production', 'Shells are only cached in production builds');
+
+  // `[id]` only lists `test` in `generateStaticParams`, so the first visit prerenders and caches the
+  // shell for this id. The prerender writes no trace tag, so every visitor starts a fresh trace.
+  const id = `shell-${Date.now()}`;
+
+  const visit = async (): Promise<{ traceId: string | undefined; links: unknown }> => {
+    const context = await browser.newContext();
+    const page: Page = await context.newPage();
+
+    const pageloadTxPromise = waitForTransaction(APP, transactionEvent => {
+      return (
+        transactionEvent.contexts?.trace?.op === 'pageload' &&
+        transactionEvent.contexts?.trace?.data?.['url.path'] === `/${id}`
+      );
+    });
+
+    await page.goto(`/${id}`);
+    await expect(page.locator('#result')).toHaveText(`Hello, ${id}!`);
+    expect(await page.locator('meta[name="sentry-trace"]').count()).toBe(0);
+
+    const pageloadTx = await pageloadTxPromise;
+    await context.close();
+
+    return { traceId: pageloadTx.contexts?.trace?.trace_id, links: pageloadTx.contexts?.trace?.links };
+  };
+
+  const first = await visit();
+  const second = await visit();
+
+  expect(first.traceId).toBeTruthy();
+  expect(second.traceId).toBeTruthy();
+  expect(second.traceId).not.toBe(first.traceId);
+  expect(first.links).toBeUndefined();
+  expect(second.links).toBeUndefined();
+});
+
+test('starts a fresh trace on a fully static page', async ({ page }) => {
+  test.skip(process.env.TEST_ENV !== 'production', 'Pages are only prerendered in production builds');
+
+  const pageloadTxPromise = waitForTransaction(APP, transactionEvent => {
+    return transactionEvent.contexts?.trace?.op === 'pageload' && transactionEvent.transaction === '/static-shell';
+  });
+
+  await page.goto('/static-shell');
+
+  await expect(page.locator('#static-shell')).toHaveText('Static shell');
+
+  const pageloadTx = await pageloadTxPromise;
+
+  // A build-time shell carries no trace context at all.
+  expect(await page.locator('meta[name="sentry-trace"]').count()).toBe(0);
+  expect(pageloadTx.contexts?.trace?.trace_id).toBeTruthy();
+  expect(pageloadTx.contexts?.trace?.links).toBeUndefined();
 });

dev-packages/e2e-tests/test-applications/nextjs-16-streaming-cacheComponents/app/blocking/page.tsx

@@ -0,0 +1,10 @@
+import { headers } from 'next/headers';
+
+// Opts out of the static shell: the whole page renders on the server for every request, so the
+// trace meta tags land in <head> like in a classic dynamic render.
+export const instant = false;
+
+export default async function Page() {
+  const requestHeaders = await headers();
+  return <h1 id="blocking">Blocking render for {requestHeaders.get('user-agent') ? 'a browser' : 'nobody'}</h1>;
+}

dev-packages/e2e-tests/test-applications/nextjs-16-streaming-cacheComponents/app/static-shell/page.tsx

@@ -0,0 +1,4 @@
+// No request data anywhere, so the whole page is a static shell and nothing renders per request.
+export default function Page() {
+  return <h1 id="static-shell">Static shell</h1>;
+}

dev-packages/e2e-tests/test-applications/nextjs-16-streaming-cacheComponents/tests/cacheOrigin-pageloadConnection.spec.ts

@@ -1,19 +1,22 @@
+import type { Page } from '@playwright/test';
 import { expect, test } from '@playwright/test';
 import { getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils';
 
-test('connects the pageload trace to the runtime server request', async ({ page }) => {
-  // The resumed part of the document, which carries the tag, arrives after the SDK started. Enabled
-  // once the pageload instrumentation waits for it.
-  test.skip(true, 'Needs the deferred pageload start');
+const APP = 'nextjs-16-streaming-cacheComponents';
+
+// The server only writes `sentry-trace` meta tags for a render that happened for the current request,
+// never while it prerenders a shell. So a tag in the document can always be continued; the only
+// question is when it arrives. The resumed part of a prerendered shell arrives after the SDK started,
+// which the pageload instrumentation waits for.
 
+test('connects the pageload trace to the runtime server request', async ({ page }) => {
   // The shell is prerendered, but the page's dynamic part (`DynamicContent` awaits `headers()`)
   // runs on the server on every request and produces the `GET /pageload-tracing` root span.
-  // A fully prerendered page would produce no server span.
-  const serverSpanPromise = waitForStreamedSpan('nextjs-16-streaming-cacheComponents', span => {
+  const serverSpanPromise = waitForStreamedSpan(APP, span => {
     return span.name === 'GET /pageload-tracing' && getSpanOp(span) === 'http.server' && span.is_segment;
   });
 
-  const pageloadSpanPromise = waitForStreamedSpan('nextjs-16-streaming-cacheComponents', span => {
+  const pageloadSpanPromise = waitForStreamedSpan(APP, span => {
     return span.name === '/pageload-tracing' && getSpanOp(span) === 'pageload' && span.is_segment;
   });
 
@@ -25,4 +28,80 @@ test('connects the pageload trace to the runtime server request', async ({ page
 
   expect(serverSpan.trace_id).toBeTruthy();
   expect(pageloadSpan.trace_id).toBe(serverSpan.trace_id);
+  expect(pageloadSpan.links).toBeUndefined();
+});
+
+test('connects the pageload trace to a blocking render', async ({ page }) => {
+  const serverSpanPromise = waitForStreamedSpan(APP, span => {
+    return span.name === 'GET /blocking' && getSpanOp(span) === 'http.server' && span.is_segment;
+  });
+
+  const pageloadSpanPromise = waitForStreamedSpan(APP, span => {
+    return span.name === '/blocking' && getSpanOp(span) === 'pageload' && span.is_segment;
+  });
+
+  await page.goto('/blocking');
+
+  await expect(page.locator('#blocking')).toHaveText('Blocking render for a browser');
+
+  const [serverSpan, pageloadSpan] = await Promise.all([serverSpanPromise, pageloadSpanPromise]);
+
+  // The whole page renders per request, so the tag in <head> is this request's and is continued.
+  expect(serverSpan.trace_id).toBeTruthy();
+  expect(pageloadSpan.trace_id).toBe(serverSpan.trace_id);
+  expect(pageloadSpan.links).toBeUndefined();
+});
+
+test('does not share a trace between visitors of a shell generated at request time', async ({ browser }) => {
+  test.skip(process.env.TEST_ENV !== 'production', 'Shells are only cached in production builds');
+
+  // `[id]` only lists `test` in `generateStaticParams`, so the first visit prerenders and caches the
+  // shell for this id. The prerender writes no trace tag, so every visitor starts a fresh trace.
+  const id = `shell-${Date.now()}`;
+
+  const visit = async (): Promise<{ traceId: string | undefined; links: unknown }> => {
+    const context = await browser.newContext();
+    const page: Page = await context.newPage();
+
+    const pageloadSpanPromise = waitForStreamedSpan(APP, span => {
+      return getSpanOp(span) === 'pageload' && span.is_segment && span.attributes['url.path']?.value === `/${id}`;
+    });
+
+    await page.goto(`/${id}`);
+    await expect(page.locator('#result')).toHaveText(`Hello, ${id}!`);
+    expect(await page.locator('meta[name="sentry-trace"]').count()).toBe(0);
+
+    const pageloadSpan = await pageloadSpanPromise;
+    await context.close();
+
+    return { traceId: pageloadSpan.trace_id, links: pageloadSpan.links };
+  };
+
+  const first = await visit();
+  const second = await visit();
+
+  expect(first.traceId).toBeTruthy();
+  expect(second.traceId).toBeTruthy();
+  expect(second.traceId).not.toBe(first.traceId);
+  expect(first.links).toBeUndefined();
+  expect(second.links).toBeUndefined();
+});
+
+test('starts a fresh trace on a fully static page', async ({ page }) => {
+  test.skip(process.env.TEST_ENV !== 'production', 'Pages are only prerendered in production builds');
+
+  const pageloadSpanPromise = waitForStreamedSpan(APP, span => {
+    return span.name === '/static-shell' && getSpanOp(span) === 'pageload' && span.is_segment;
+  });
+
+  await page.goto('/static-shell');
+
+  await expect(page.locator('#static-shell')).toHaveText('Static shell');
+
+  const pageloadSpan = await pageloadSpanPromise;
+
+  // A build-time shell carries no trace context at all.
+  expect(await page.locator('meta[name="sentry-trace"]').count()).toBe(0);
+  expect(pageloadSpan.trace_id).toBeTruthy();
+  expect(pageloadSpan.links).toBeUndefined();
 });

packages/nextjs/src/client/browserTracingIntegration.ts

@@ -1,5 +1,6 @@
 import type { Integration } from '@sentry/core';
 import { browserTracingIntegration as originalBrowserTracingIntegration, isBotUserAgent } from '@sentry/react';
+import { settlePendingPageloadWait } from './routing/appRouterRoutingInstrumentation';
 import { nextRouterInstrumentNavigation, nextRouterInstrumentPageLoad } from './routing/nextRoutingInstrumentation';
 
 /**
@@ -41,6 +42,14 @@ export function browserTracingIntegration(
         nextRouterInstrumentNavigation(client);
       }
 
+      // Registered before the browser tracing integration's own handler, which creates the navigation
+      // span: a pageload that is still waiting for its trace meta tag has to exist by then, so the
+      // handler ends it like any pageload a navigation interrupts. Started later, the pageload would
+      // end the navigation span instead.
+      if (instrumentPageLoad) {
+        client.on('startNavigationSpan', settlePendingPageloadWait);
+      }
+
       browserTracingIntegrationInstance.afterAllSetup(client);
 
       if (instrumentPageLoad) {

packages/nextjs/src/client/routing/appRouterRoutingInstrumentation.ts

@@ -1,5 +1,6 @@
-import type { Client, Span } from '@sentry/core';
+import type { Client, Span, StartSpanOptions } from '@sentry/core';
 import {
+  getCurrentScope,
   GLOBAL_OBJ,
   hasSpanStreamingEnabled,
   isObjectLike,
@@ -17,6 +18,8 @@ import {
   resolveRoute,
 } from '@sentry/react';
 import { stripTrailingSlash } from './parameterization';
+import type { TraceMetaTagValues, TraceMetaTagWait } from './traceMetaTags';
+import { addTraceMetaTagLink, canWaitForTraceMetaTag, readTraceMetaTags, waitForTraceMetaTag } from './traceMetaTags';
 import {
   SENTRY_OP,
   SENTRY_SEGMENT_NAME_SOURCE,
@@ -95,21 +98,90 @@ let navigationRoutingMode: 'router-patch' | 'transition-start-hook' = 'router-pa
 
 const currentRouterPatchingNavigationSpanRef: NavigationSpanRef = { current: undefined };
 
+/**
+ * How long the pageload start may wait for the resumed part of a Cache Components document. The wait
+ * normally ends with `DOMContentLoaded`; the cap only matters for a response the server keeps open.
+ */
+const PAGELOAD_TRACE_META_TAG_TIMEOUT_MS = 10_000;
+
+let pendingPageloadWait: TraceMetaTagWait | undefined;
+
+/**
+ * Starts a pageload that is still waiting for its trace meta tag with a fresh trace. Called when a
+ * navigation starts, before the navigation span is created, so the navigation ends the pageload like
+ * any pageload it interrupts, and the pageload is still recorded.
+ */
+export function settlePendingPageloadWait(): void {
+  pendingPageloadWait?.giveUp();
+}
+
+/**
+ * Starts the pageload span with the request data of the document. The span may start after a
+ * navigation already changed the location, so the browser SDK would otherwise read the new one.
+ */
+function startPageloadSpan(
+  client: Client,
+  spanOptions: StartSpanOptions,
+  documentUrl: string,
+  traceMetaTags?: TraceMetaTagValues,
+): Span | undefined {
+  const span = startBrowserTracingPageLoadSpan(client, spanOptions, traceMetaTags);
+  const scope = getCurrentScope();
+  const { normalizedRequest } = scope.getScopeData().sdkProcessingMetadata;
+  if (normalizedRequest) {
+    scope.setSDKProcessingMetadata({ normalizedRequest: { ...normalizedRequest, url: documentUrl } });
+  }
+  return span;
+}
+
 /** Instruments the Next.js app router for pageloads. */
 export function appRouterInstrumentPageLoad(client: Client): void {
+  const documentUrl = WINDOW.location.href;
   const pathname = stripTrailingSlash(WINDOW.location.pathname);
   const parameterizedPathname = resolveCurrentRoute(client);
-  startBrowserTracingPageLoadSpan(client, {
+  const spanOptions: StartSpanOptions = {
     // With span streaming, span names have to be low cardinality, so we can't fall back to the URL.
     name: parameterizedPathname ?? (hasSpanStreamingEnabled(client) ? PAGELOAD_SPAN_NAME_FALLBACK : pathname),
     // pageload should always start at timeOrigin (and needs to be in s, not ms)
     attributes: {
       [SENTRY_OP]: PAGELOAD,
       [SENTRY_ORIGIN]: 'auto.pageload.nextjs.app_router_instrumentation',
       [SENTRY_SEGMENT_NAME_SOURCE]: parameterizedPathname ? 'route' : 'url',
+      [URL_PATH]: WINDOW.location.pathname,
+      [URL_FULL]: filterCollectedUrl(documentUrl, client),
       ...(parameterizedPathname && { [URL_TEMPLATE]: parameterizedPathname }),
     },
-  });
+  };
+
+  // A tag that is already in the document was rendered for this request and can be continued. A
+  // Cache Components document that resumes a prerendered shell gets its tag with the resumed part,
+  // which usually arrives after the SDK started. Waiting for it lets the pageload continue the server
+  // trace instead of linking it; the span's start is backdated to the time origin either way.
+  const traceMetaTags = readTraceMetaTags();
+  if (traceMetaTags || !canWaitForTraceMetaTag()) {
+    startPageloadSpan(client, spanOptions, documentUrl, traceMetaTags);
+    return;
+  }
+
+  pendingPageloadWait = waitForTraceMetaTag(
+    values => {
+      pendingPageloadWait = undefined;
+      startPageloadSpan(client, spanOptions, documentUrl, values);
+    },
+    () => {
+      pendingPageloadWait = undefined;
+      const span = startPageloadSpan(client, spanOptions, documentUrl);
+      // Only reachable through the cap or a navigation while the document still streams: a tag that
+      // shows up after this can no longer be continued, but the server request can still be linked.
+      if (span) {
+        waitForTraceMetaTag(
+          values => span.isRecording() && addTraceMetaTagLink(span, values),
+          () => undefined,
+        );
+      }
+    },
+    PAGELOAD_TRACE_META_TAG_TIMEOUT_MS,
+  );
 }
 
 interface NavigationSpanRef {