diff --git a/cookbooks/cosmos3/generator/audiovisual/README.md b/cookbooks/cosmos3/generator/audiovisual/README.md index d9cbf1e1..c6751250 100644 --- a/cookbooks/cosmos3/generator/audiovisual/README.md +++ b/cookbooks/cosmos3/generator/audiovisual/README.md @@ -145,12 +145,17 @@ export_to_video(result.video, "/tmp/cosmos3_t2v_diffusers.mp4", fps=24) To run **Cosmos3-Super** instead, load the larger checkpoint: `Cosmos3OmniPipeline.from_pretrained("nvidia/Cosmos3-Super", ...)`. +To run **Cosmos3-Edge** instead, load `nvidia/Cosmos3-Edge` and use its +single-GPU, no-audio settings: `height=480`, `width=832`, `num_frames=121`, +`num_inference_steps=50`, `guidance_scale=5.0`, and `flow_shift=3.0`. + ### Notebook walkthrough [`run_with_diffusers.ipynb`](./run_with_diffusers.ipynb) is the full tutorial for the Diffusers backend: it provisions a dedicated venv, then walks through text-to-image, text-to-video, and image-to-video generation (with and without -audio) using `Cosmos3OmniPipeline`, including how to preview the generated media. +audio) using `Cosmos3OmniPipeline`, including a dedicated Cosmos3-Edge section +with 480p text-to-image, text-to-video, and image-to-video examples and previews. ## Run with vLLM-Omni diff --git a/cookbooks/cosmos3/generator/audiovisual/run_with_diffusers.ipynb b/cookbooks/cosmos3/generator/audiovisual/run_with_diffusers.ipynb index dfb1af16..56a92ce9 100644 --- a/cookbooks/cosmos3/generator/audiovisual/run_with_diffusers.ipynb +++ b/cookbooks/cosmos3/generator/audiovisual/run_with_diffusers.ipynb @@ -16,7 +16,7 @@ "\n", "This notebook runs Cosmos3 audiovisual generation directly with `Cosmos3OmniPipeline`.\n", "\n", - "Run all Cosmos3-Nano examples first, then run the Cosmos3-Super T2V/I2V examples without audio. Each section loads the matching model explicitly.\n", + "Run the Cosmos3-Nano, Cosmos3-Super, or Cosmos3-Edge examples independently. Cosmos3-Edge has no audio modules and uses its 480p generation settings. Run Cosmos3-Super T2V/I2V examples without audio. Each section loads the matching model explicitly.\n", "\n", "Note: if you have already completed steps 1-3 and installed the `Cosmos3 Diffusers (Python 3.13)` kernel, switch to that kernel and jump directly to step 4. Run the restore cell there, then continue with verification and the examples.\n" ] @@ -379,12 +379,44 @@ " \"prompt\": \"assets/prompts/text2image/robot_draping.json\",\n", " \"enable_sound\": False,\n", " },\n", + " \"t2i_edge\": {\n", + " \"model\": \"Cosmos3-Edge\",\n", + " \"mode\": \"text2image\",\n", + " \"prompt\": \"assets/prompts/text2image/robot_draping.json\",\n", + " \"enable_sound\": False,\n", + " \"sampling\": {\n", + " \"num_steps\": 50,\n", + " \"guidance\": 5.0,\n", + " \"shift\": 3.0,\n", + " \"fps\": 24,\n", + " \"num_frames\": 1,\n", + " \"resolution\": \"480\",\n", + " \"aspect_ratio\": \"16,9\",\n", + " \"seed\": 0,\n", + " },\n", + " },\n", " \"t2v_nano_noaudio\": {\n", " \"model\": \"Cosmos3-Nano\",\n", " \"mode\": \"text2video\",\n", " \"prompt\": \"assets/prompts/text2video/robot_kitchen.json\",\n", " \"enable_sound\": False,\n", " },\n", + " \"t2v_edge_noaudio\": {\n", + " \"model\": \"Cosmos3-Edge\",\n", + " \"mode\": \"text2video\",\n", + " \"prompt\": \"assets/prompts/text2video/robot_kitchen.json\",\n", + " \"enable_sound\": False,\n", + " \"sampling\": {\n", + " \"num_steps\": 50,\n", + " \"guidance\": 5.0,\n", + " \"shift\": 3.0,\n", + " \"fps\": 24,\n", + " \"num_frames\": 121,\n", + " \"resolution\": \"480\",\n", + " \"aspect_ratio\": \"16,9\",\n", + " \"seed\": 0,\n", + " },\n", + " },\n", " \"t2vs\": {\n", " \"model\": \"Cosmos3-Nano\",\n", " \"mode\": \"text2video\",\n", @@ -398,6 +430,23 @@ " \"image\": \"assets/images/image2video/car_driving.jpg\",\n", " \"enable_sound\": False,\n", " },\n", + " \"i2v_edge_noaudio\": {\n", + " \"model\": \"Cosmos3-Edge\",\n", + " \"mode\": \"image2video\",\n", + " \"prompt\": \"assets/prompts/image2video/car_driving.json\",\n", + " \"image\": \"assets/images/image2video/car_driving.jpg\",\n", + " \"enable_sound\": False,\n", + " \"sampling\": {\n", + " \"num_steps\": 50,\n", + " \"guidance\": 5.0,\n", + " \"shift\": 3.0,\n", + " \"fps\": 24,\n", + " \"num_frames\": 121,\n", + " \"resolution\": \"480\",\n", + " \"aspect_ratio\": \"16,9\",\n", + " \"seed\": 0,\n", + " },\n", + " },\n", " \"i2vs\": {\n", " \"model\": \"Cosmos3-Nano\",\n", " \"mode\": \"image2video\",\n", @@ -437,6 +486,8 @@ " return 720, 1280\n", " if payload.get(\"resolution\") == \"256\" and payload.get(\"aspect_ratio\") == \"16,9\":\n", " return 192, 320\n", + " if payload.get(\"resolution\") == \"480\" and payload.get(\"aspect_ratio\") == \"16,9\":\n", + " return 480, 832\n", " raise ValueError(f\"Unsupported payload resolution/aspect ratio: {payload.get('resolution')} {payload.get('aspect_ratio')}\")\n", "\n", "\n", @@ -467,6 +518,7 @@ " \"negative_prompt\": negative_prompt,\n", " \"enable_sound\": spec[\"enable_sound\"],\n", " **FIXED_SAMPLING,\n", + " **spec.get(\"sampling\", {}),\n", " }\n", " if spec[\"mode\"] == \"image2video\":\n", " image_path = asset_path(spec[\"image\"])\n", @@ -512,6 +564,7 @@ "MODEL_IDS = {\n", " \"Cosmos3-Nano\": \"nvidia/Cosmos3-Nano\",\n", " \"Cosmos3-Super\": \"nvidia/Cosmos3-Super\",\n", + " \"Cosmos3-Edge\": \"nvidia/Cosmos3-Edge\",\n", "}\n", "\n", "_pipe = None\n", @@ -648,8 +701,7 @@ " display_video(src)\n", " for src in images:\n", " print(f\"source: {src} ({src.stat().st_size // 1024} KB)\")\n", - " display(Image(filename=str(src), width=720))\n", - "" + " display(Image(filename=str(src), width=720))\n" ], "execution_count": null, "outputs": [], @@ -1084,6 +1136,190 @@ ], "execution_count": null, "outputs": [] + }, + { + "cell_type": "markdown", + "id": "482d68de", + "metadata": {}, + "source": [ + "# Cosmos3-Edge Examples\n", + "\n", + "Cosmos3-Edge runs on a single GPU and has no audio tokenizer, so all Edge video examples keep audio disabled. The Edge checkpoint is trained for 480p generation; these examples use 832x480, 121 frames, 50 denoising steps, guidance scale 5.0, and flow shift 3.0.\n" + ] + }, + { + "cell_type": "markdown", + "id": "c5e29c2a", + "metadata": {}, + "source": [ + "## Edge: Text to Image\n", + "\n", + "Generate a single 832x480 image from the structured JSON prompt.\n", + "\n", + "### Create Payload\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "edd2c2ed", + "metadata": {}, + "outputs": [], + "source": [ + "t2i_edge_payload, t2i_edge_output, t2i_edge_model = create_payload(\"t2i_edge\", backend=\"diffusers\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "444115f2", + "metadata": {}, + "source": [ + "### Run\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "65857776", + "metadata": {}, + "outputs": [], + "source": [ + "run_diffusers_payload(t2i_edge_payload, t2i_edge_output, model=\"Cosmos3-Edge\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "9c1d7b62", + "metadata": {}, + "source": [ + "### View Results\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4cd4817", + "metadata": {}, + "outputs": [], + "source": [ + "view_run(t2i_edge_output)\n" + ] + }, + { + "cell_type": "markdown", + "id": "2fb4e265", + "metadata": {}, + "source": [ + "## Edge: Text to Video Without Audio\n", + "\n", + "Generate a 121-frame 832x480 video from the structured JSON prompt.\n", + "\n", + "### Create Payload\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c133f313", + "metadata": {}, + "outputs": [], + "source": [ + "t2v_edge_noaudio_payload, t2v_edge_noaudio_output, t2v_edge_noaudio_model = create_payload(\"t2v_edge_noaudio\", backend=\"diffusers\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "490c2c93", + "metadata": {}, + "source": [ + "### Run\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "557831a9", + "metadata": {}, + "outputs": [], + "source": [ + "run_diffusers_payload(t2v_edge_noaudio_payload, t2v_edge_noaudio_output, model=\"Cosmos3-Edge\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "6fdd64ed", + "metadata": {}, + "source": [ + "### View Results\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44f6f2e8", + "metadata": {}, + "outputs": [], + "source": [ + "view_run(t2v_edge_noaudio_output)\n" + ] + }, + { + "cell_type": "markdown", + "id": "33d5e6ba", + "metadata": {}, + "source": [ + "## Edge: Image to Video Without Audio\n", + "\n", + "Animate the supplied driving image for 121 frames at 832x480.\n", + "\n", + "### Create Payload\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4dcd641", + "metadata": {}, + "outputs": [], + "source": [ + "i2v_edge_noaudio_payload, i2v_edge_noaudio_output, i2v_edge_noaudio_model = create_payload(\"i2v_edge_noaudio\", backend=\"diffusers\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "2043536b", + "metadata": {}, + "source": [ + "### Run\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa8a12d3", + "metadata": {}, + "outputs": [], + "source": [ + "run_diffusers_payload(i2v_edge_noaudio_payload, i2v_edge_noaudio_output, model=\"Cosmos3-Edge\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "a022c344", + "metadata": {}, + "source": [ + "### View Results\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c40acf67", + "metadata": {}, + "outputs": [], + "source": [ + "view_run(i2v_edge_noaudio_output)\n" + ] } ], "metadata": {