Merge pull request #3931 from plotly/feat/streaming

Streaming callbacks with multiplexed transport

Author: T4rk1n

.ai/ARCHITECTURE.md

@@ -20,6 +20,7 @@ jobs:
       dcc_paths_changed: ${{ steps.filter.outputs.dcc_related_paths }}
       html_paths_changed: ${{ steps.filter.outputs.html_related_paths }}
       websocket_changed: ${{ steps.filter.outputs.websocket_paths }}
+      streaming_changed: ${{ steps.filter.outputs.streaming_paths }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -79,6 +80,14 @@ jobs:
               - '@dash-websocket-worker/**'
               - 'dash/dash-renderer/src/**'
               - 'tests/websocket/**'
+            streaming_paths:
+              - *shared_paths
+              - 'dash/_callback.py'
+              - 'dash/_streaming.py'
+              - 'dash/_callback_context.py'
+              - 'dash/backends/**'
+              - 'dash/dash-renderer/src/**'
+              - 'tests/streaming/**'
 
   lint-unit:
     name: Lint & Unit Tests (Python ${{ matrix.python-version }})
@@ -256,6 +265,77 @@ jobs:
             tests/compliance/test_typing.py \
             tests/compliance/test_callback_typing.py
 
+  shared-storage:
+    name: Run Shared Storage Tests (Python ${{ matrix.python-version }})
+    needs: build
+    timeout-minutes: 20
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.12"]
+
+    # Redis service for the RedisSharedStorage backend.
+    services:
+      redis:
+        image: redis:6
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    env:
+      REDIS_URL: redis://localhost:6379
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24'
+          cache: 'npm'
+
+      - name: Install Node.js dependencies
+        run: npm ci
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: requirements/*.txt
+
+      - name: Download built Dash packages
+        uses: actions/download-artifact@v4
+        with:
+          name: dash-packages
+          path: packages/
+
+      - name: Install Dash packages
+        run: |
+          python -m pip install --upgrade pip wheel
+          python -m pip install "setuptools<80.0.0"
+          find packages -name dash-*.whl -print -exec sh -c 'pip install "{}[ci,testing,dev,diskcache,redis]"' \;
+
+      - name: Setup Chrome and ChromeDriver
+        uses: browser-actions/setup-chrome@v1
+        with:
+          chrome-version: stable
+
+      - name: Verify Redis connection
+        run: python -c "import redis; redis.Redis(host='localhost', port=6379).ping(); print('Redis OK')"
+
+      - name: Build/Setup test components
+        run: npm run setup-tests.py
+
+      - name: Run shared-storage tests
+        run: pytest --headless tests/shared_storage -v
+
   background-callbacks:
     name: Run Background & Async Callback Tests (Python ${{ matrix.python-version }})
     needs: [build, changes_filter]
@@ -625,6 +705,69 @@ jobs:
           touch __init__.py
           pytest --headless --nopercyfinalize tests/websocket -v -s
 
+  streaming-tests:
+    name: Streaming Callback Tests (Python ${{ matrix.python-version }})
+    needs: [build, changes_filter]
+    if: |
+      (github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev')) ||
+      needs.changes_filter.outputs.streaming_changed == 'true'
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.12"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24'
+          cache: 'npm'
+
+      - name: Install Node.js dependencies
+        run: npm ci
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: requirements/*.txt
+
+      - name: Download built Dash packages
+        uses: actions/download-artifact@v4
+        with:
+          name: dash-packages
+          path: packages/
+
+      - name: Install Dash packages
+        # Streaming callbacks are async generators; the async extra pulls in
+        # flask[async], which they require on the Flask backend.
+        run: |
+          python -m pip install --upgrade pip wheel
+          python -m pip install "setuptools<80.0.0"
+          find packages -name dash-*.whl -print -exec sh -c 'pip install "{}[async,ci,testing,dev,fastapi,quart]"' \;
+
+      - name: Setup Chrome and ChromeDriver
+        uses: browser-actions/setup-chrome@v1
+        with:
+          chrome-version: stable
+
+      - name: Build/Setup test components
+        run: npm run setup-tests.py
+
+      - name: Run streaming tests
+        run: |
+          mkdir streamtests
+          cp -r tests streamtests/tests
+          cd streamtests
+          touch __init__.py
+          pytest --headless --nopercyfinalize tests/streaming -v -s
+
   test-main:
     name: Main Dash Tests (Python ${{ matrix.python-version }}, React ${{ matrix.react-version }}, Group ${{ matrix.test-group }})
     needs: build

.github/workflows/testing.yml

@@ -5,6 +5,14 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 ## [Unreleased]
 
 ### Added
+- [#3930](https://github.com/plotly/dash/pull/3930) Shared storage: a backend-agnostic state manager available on every app via `dash.ctx.shared_storage` (and `app.shared_storage`), for sharing state between callbacks or across worker processes without an external service. Started lazily on first use, so it costs nothing until touched; pass `shared_storage=None` to `Dash(...)` to disable.
+  - Cross-process key/value store (`get`/`set`/`delete`, with optional TTL) and ordered, replayable publish/subscribe (`publish`/`subscribe`). Subscriptions resume from the caller's last-seen sequence out of a bounded buffer, and a buffer overrun surfaces as an explicit gap rather than a silent loss. Values must be JSON-compatible (like `dcc.Store`).
+  - Three backends ship, selected by passing a `BaseSharedStorage` instance to `shared_storage=`:
+    - `LocalSharedStorage` (default): in-memory, elects one owner process per machine over an `AF_UNIX`/loopback socket, correct within a single container, including across multiple gunicorn workers.
+    - `DiskcacheSharedStorage`: on a `diskcache.Cache`, shared by every process on one host, for single-machine deployments (not for multi-pod ones with ephemeral disks).
+    - `RedisSharedStorage`: on Redis, using Redis Streams for the ordered pub/sub: the backend for horizontally-scaled deployments behind a load balancer, e.g. apps scaled across pods.
+  - `LocalSharedStorage` additionally accepts `mode=` to make its key/value store durable: `"memory"` (default, in-memory only), `"persist"` (write-through to disk on every change), or `"persist-reset"` (in-memory speed with a periodic flush every `flush_interval` seconds and on clean exit). Persistent modes recover on start and on owner re-election, so state survives a process restart or a crashed owner. Data is stored in a chunked, atomically-written msgpack store (a per-namespace folder under the user cache directory by default, overridable via `path=`). TTLs are preserved across restarts; pub/sub remains transient.
+- [#3931](https://github.com/plotly/dash/pull/3931) Streaming callbacks: a callback defined as an `async def` generator streams its yields to the browser as they are produced (`dash.Patch` yields give incremental updates). All of a browser's streams share one connection hosted in a SharedWorker, so they don't count against the per-host connection limit; closing a tab cancels its streams.
 - [#3977](https://github.com/plotly/dash/pull/3977) Add partial WebSocket prop reads with `get_prop(..., path=...)`. Closes [#3975](https://github.com/plotly/dash/issues/3975).
 - [#3765](https://github.com/plotly/dash/pull/3765) Add opt-in partial pattern matching for callback `Input`, `Output`, and `State` dependencies via `partial_pattern=True`. Dictionary ID patterns can now match component IDs containing additional keys, and partial patterns can be combined with `ALL` and `MATCH` wildcards. Fixes [#3764](https://github.com/plotly/dash/issues/3764).
 - [#3646](https://github.com/plotly/dash/pull/3646) Experimental support for React 19. The default is still React 18.3.1; to use React 19 set the environment variable `REACT_VERSION=19.2.4` before running your app, or call `dash._dash_renderer._set_react_version("19.2.4")` inside the app. React 19 has no official UMD builds, so Dash serves the [`umd-react`](https://www.npmjs.com/package/umd-react) package, together with a compatibility shim loaded after react-dom and before any component package. The shim keeps component libraries built against React <=18 (e.g. dash-bootstrap-components, dash-mantine-components) working under React 19: it stubs the removed `ReactCurrentOwner` internals, redirects the legacy element `$$typeof` symbol so pre-bundled React 18 jsx-runtimes produce elements React 19 accepts (error #525), and exposes a global `react/jsx-runtime` (`window.ReactJSXRuntime`) that Dash's own component bundles externalize to. Component library authors adopting this convention should copy the defensive `jsxRuntimeExternal` webpack external from `components/dash-core-components/webpack.config.js` rather than a bare `'ReactJSXRuntime'` string: it falls back to a `React.createElement`-based runtime when the global is missing, so the same build also works on Dash versions older than this release.

CHANGELOG.md

@@ -44,6 +44,16 @@
 from ._remount import remount  # noqa: F401,E402
 from ._jupyter import jupyter_dash  # noqa: F401,E402
 
+from ._shared_storage import (  # noqa: F401,E402
+    BaseSharedStorage,
+    DiskcacheSharedStorage,
+    LocalSharedStorage,
+    RedisSharedStorage,
+    SharedStorageError,
+    SharedStorageGap,
+    Subscription,
+)
+
 from ._hooks import hooks  # noqa: F401,E402
 
 ctx = callback_context
@@ -96,4 +106,11 @@ def _jupyter_nbextension_paths():
     "ctx",
     "hooks",
     "stringify_id",
+    "BaseSharedStorage",
+    "DiskcacheSharedStorage",
+    "LocalSharedStorage",
+    "RedisSharedStorage",
+    "SharedStorageError",
+    "SharedStorageGap",
+    "Subscription",
 ]