From b5ff5aa217d491ebd6a63b8b37f3a00f584f1df4 Mon Sep 17 00:00:00 2001 From: Kalin Ovtcharov Date: Thu, 25 Jun 2026 23:28:08 -0700 Subject: [PATCH 1/3] ci: run Lemonade-dependent checks when LEMONADE_VERSION changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Lemonade version bump edits only src/gaia/version.py (LEMONADE_VERSION), but every Lemonade-dependent workflow was path-filtered to feature code (rag/llm/agents/cpp/...) and excluded that file — so the bump that installs a new Lemonade on the runners triggered NONE of the tests that exercise it. That is how Lemonade 10.7.0/10.8.x landed while the embedding model (nomic-embed-text-v2-moe) is broken on the bundled llama-server (>= b6524, upstream llama.cpp #16301 / lemonade #612): the embeddings/RAG/API/E2E checks never ran on the bump. Add src/gaia/version.py and the install-lemonade action to the path triggers of every Lemonade-using workflow so a version bump runs the full Lemonade test surface and a regression is caught on the bump PR, not days later on an unrelated branch. --- .github/workflows/build_cpp.yml | 4 ++++ .github/workflows/test_agent_behavior_e2e.yml | 2 ++ .github/workflows/test_agent_mcp_server.yml | 4 ++++ .github/workflows/test_agent_sdk.yml | 4 ++++ .github/workflows/test_api.yml | 4 ++++ .github/workflows/test_embeddings.yml | 4 ++++ .github/workflows/test_examples.yml | 4 ++++ .github/workflows/test_gaia_cli_windows.yml | 4 ++++ .github/workflows/test_lemonade_server.yml | 2 ++ .github/workflows/test_rag.yml | 4 ++++ .github/workflows/test_sd.yml | 4 ++++ 11 files changed, 40 insertions(+) diff --git a/.github/workflows/build_cpp.yml b/.github/workflows/build_cpp.yml index 528d90ae8..0049eb7ef 100644 --- a/.github/workflows/build_cpp.yml +++ b/.github/workflows/build_cpp.yml @@ -13,6 +13,8 @@ on: push: branches: [ main ] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - 'cpp/**' - '.github/workflows/build_cpp.yml' - '.github/workflows/benchmark_cpp.yml' @@ -20,6 +22,8 @@ on: branches: [ main ] types: [opened, synchronize, reopened, ready_for_review] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - 'cpp/**' - '.github/workflows/build_cpp.yml' - '.github/workflows/benchmark_cpp.yml' diff --git a/.github/workflows/test_agent_behavior_e2e.yml b/.github/workflows/test_agent_behavior_e2e.yml index 2a702312f..5f4b0b809 100644 --- a/.github/workflows/test_agent_behavior_e2e.yml +++ b/.github/workflows/test_agent_behavior_e2e.yml @@ -11,6 +11,8 @@ on: workflow_dispatch: push: paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - 'src/gaia/agents/builder/**' - 'src/gaia/agents/base/agent.py' - 'src/gaia/eval/behavior_harness.py' diff --git a/.github/workflows/test_agent_mcp_server.yml b/.github/workflows/test_agent_mcp_server.yml index 6988079c1..d0a9f5e27 100644 --- a/.github/workflows/test_agent_mcp_server.yml +++ b/.github/workflows/test_agent_mcp_server.yml @@ -10,6 +10,8 @@ on: push: branches: [ main ] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - 'src/gaia/agents/base/mcp_agent.py' - 'src/gaia/mcp/**' - 'tests/mcp/**' @@ -19,6 +21,8 @@ on: branches: [ main ] types: [opened, synchronize, reopened, ready_for_review] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - 'src/gaia/agents/base/mcp_agent.py' - 'src/gaia/mcp/**' - 'tests/mcp/**' diff --git a/.github/workflows/test_agent_sdk.yml b/.github/workflows/test_agent_sdk.yml index 0e1749ff3..1b90d2976 100644 --- a/.github/workflows/test_agent_sdk.yml +++ b/.github/workflows/test_agent_sdk.yml @@ -11,6 +11,8 @@ on: push: branches: ["main"] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - "src/**" - "tests/**" - "setup.py" @@ -20,6 +22,8 @@ on: branches: ["main"] types: [opened, synchronize, reopened, ready_for_review] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - "src/**" - "tests/**" - "setup.py" diff --git a/.github/workflows/test_api.yml b/.github/workflows/test_api.yml index 33a4495eb..8786f02df 100644 --- a/.github/workflows/test_api.yml +++ b/.github/workflows/test_api.yml @@ -11,6 +11,8 @@ on: push: branches: ["main"] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - 'src/gaia/api/**' - 'src/gaia/agents/base/**' - 'src/gaia/llm/**' @@ -21,6 +23,8 @@ on: branches: ["main"] types: [opened, synchronize, reopened, ready_for_review] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - 'src/gaia/api/**' - 'src/gaia/agents/base/**' - 'src/gaia/llm/**' diff --git a/.github/workflows/test_embeddings.yml b/.github/workflows/test_embeddings.yml index d1bcc0be8..2f22f631c 100644 --- a/.github/workflows/test_embeddings.yml +++ b/.github/workflows/test_embeddings.yml @@ -8,6 +8,8 @@ on: push: branches: [ main ] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - 'src/gaia/rag/**' - 'src/gaia/llm/**' - 'tests/test_lemonade_embeddings.py' @@ -17,6 +19,8 @@ on: branches: [ main ] types: [opened, synchronize, reopened, ready_for_review] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - 'src/gaia/rag/**' - 'src/gaia/llm/**' - 'tests/test_lemonade_embeddings.py' diff --git a/.github/workflows/test_examples.yml b/.github/workflows/test_examples.yml index c2f1cded7..099dea6a6 100644 --- a/.github/workflows/test_examples.yml +++ b/.github/workflows/test_examples.yml @@ -18,6 +18,8 @@ on: push: branches: [ main ] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - 'examples/**' - 'src/gaia/**' - 'tests/integration/test_example_agents.py' @@ -27,6 +29,8 @@ on: branches: [ main ] types: [opened, synchronize, reopened, ready_for_review] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - 'examples/**' - 'src/gaia/**' - 'tests/integration/test_example_agents.py' diff --git a/.github/workflows/test_gaia_cli_windows.yml b/.github/workflows/test_gaia_cli_windows.yml index abc61b2d8..8ec89d9c1 100644 --- a/.github/workflows/test_gaia_cli_windows.yml +++ b/.github/workflows/test_gaia_cli_windows.yml @@ -12,6 +12,8 @@ on: push: branches: ["main"] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - "src/**" - "tests/**" - "setup.py" @@ -22,6 +24,8 @@ on: branches: ["main"] types: [opened, synchronize, reopened, ready_for_review] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - "src/**" - "tests/**" - "setup.py" diff --git a/.github/workflows/test_lemonade_server.yml b/.github/workflows/test_lemonade_server.yml index a3ed71404..780cdcd14 100644 --- a/.github/workflows/test_lemonade_server.yml +++ b/.github/workflows/test_lemonade_server.yml @@ -8,6 +8,7 @@ on: push: branches: [ main ] paths: + - 'src/gaia/version.py' - 'src/gaia/llm/**' - 'src/gaia/installer/**' - 'setup.py' @@ -20,6 +21,7 @@ on: branches: [ main ] types: [opened, synchronize, reopened, ready_for_review] paths: + - 'src/gaia/version.py' - 'src/gaia/llm/**' - 'src/gaia/installer/**' - 'setup.py' diff --git a/.github/workflows/test_rag.yml b/.github/workflows/test_rag.yml index edbc99d81..01624d001 100644 --- a/.github/workflows/test_rag.yml +++ b/.github/workflows/test_rag.yml @@ -8,6 +8,8 @@ on: push: branches: [ main ] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - 'src/gaia/rag/**' - 'src/gaia/llm/**' - 'src/gaia/vlm/**' @@ -18,6 +20,8 @@ on: branches: [ main ] types: [opened, synchronize, reopened, ready_for_review] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - 'src/gaia/rag/**' - 'src/gaia/llm/**' - 'src/gaia/vlm/**' diff --git a/.github/workflows/test_sd.yml b/.github/workflows/test_sd.yml index 323ff4d0e..de5906c34 100644 --- a/.github/workflows/test_sd.yml +++ b/.github/workflows/test_sd.yml @@ -11,6 +11,8 @@ on: push: branches: ["main"] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - "src/gaia/sd/**" # SD mixin and tools - "hub/agents/python/sd/**" # SD agent (standalone wheel #1102) - "src/gaia/llm/lemonade_client.py" @@ -21,6 +23,8 @@ on: branches: ["main"] types: [opened, synchronize, reopened, ready_for_review] paths: + - 'src/gaia/version.py' + - '.github/actions/install-lemonade/**' - "src/gaia/sd/**" # SD mixin and tools - "hub/agents/python/sd/**" # SD agent (standalone wheel #1102) - "src/gaia/llm/lemonade_client.py" From daa0e46d763365888fed0d670b720841634bf0df Mon Sep 17 00:00:00 2001 From: Kalin Ovtcharov Date: Fri, 26 Jun 2026 08:20:04 -0700 Subject: [PATCH 2/3] ci(embeddings): retry MoE embedding load + capture llama-server debug crash on failure The nomic-embed-text-v2-moe load intermittently aborts llama-server on the AMD Vulkan backend (transient GPU-driver-state fault; upstream llama.cpp #16301 / lemonade #612, not a GAIA defect). Retry recovers once GPU state settles, keeping CI green; debug logging + a job-output dump on each failed attempt records the exact crash signature the next time it occurs. --- .github/workflows/test_embeddings.yml | 50 +++++++++++++++++++-------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test_embeddings.yml b/.github/workflows/test_embeddings.yml index 2f22f631c..31b5ddcd2 100644 --- a/.github/workflows/test_embeddings.yml +++ b/.github/workflows/test_embeddings.yml @@ -72,6 +72,10 @@ jobs: $serverJob = Start-Job -ScriptBlock { # Workaround for Issue #612: Disable Vulkan cooperative matrix optimization $env:GGML_VK_DISABLE_COOPMAT = "1" + # Debug logging so the (otherwise swallowed) llama-server backend + # output is captured in the job stream -- lets us record the exact + # crash if the MoE embedding load aborts (see retry block below). + $env:LEMONADE_LOG_LEVEL = "debug" & lemonade-server serve --host localhost --port 13305 --no-tray 2>&1 } Write-Host "Started Lemonade server job with ID: $($serverJob.Id)" @@ -123,23 +127,39 @@ jobs: Write-Host " [WARN] Pull may have failed: $($_.Exception.Message)" } - # Load embedding model into memory (required in Lemonade v9.x) + # Load embedding model into memory, WITH RETRY. The + # nomic-embed-text-v2-moe MoE model intermittently aborts + # llama-server startup on the AMD Vulkan backend -- a transient + # GPU-driver-state fault (upstream llama.cpp #16301 / lemonade #612, + # not a GAIA defect; the load succeeds once the GPU state settles). + # On every failed attempt we dump the server job's debug output to + # capture the exact (otherwise-swallowed) llama-server crash. Write-Host "`n=== Loading Embedding Model ===" - try { - $loadRequest = @{ - model_name = "nomic-embed-text-v2-moe-GGUF" - } | ConvertTo-Json - - Write-Host "Loading model: nomic-embed-text-v2-moe-GGUF" - $loadResponse = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/load" ` - -Method POST -Body $loadRequest -ContentType "application/json" -TimeoutSec 60 - Write-Host "[OK] Model loaded successfully: $($loadResponse | ConvertTo-Json -Compress)" - } catch { - Write-Host "[ERROR] Model load failed: $($_.Exception.Message)" - if ($_.ErrorDetails) { - Write-Host "Error details: $($_.ErrorDetails.Message)" + $loadRequest = @{ model_name = "nomic-embed-text-v2-moe-GGUF" } | ConvertTo-Json + $loaded = $false + for ($attempt = 1; $attempt -le 3 -and -not $loaded; $attempt++) { + try { + Write-Host "Loading model (attempt $attempt/3): nomic-embed-text-v2-moe-GGUF" + $loadResponse = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/load" ` + -Method POST -Body $loadRequest -ContentType "application/json" -TimeoutSec 60 + Write-Host "[OK] Model loaded successfully: $($loadResponse | ConvertTo-Json -Compress)" + $loaded = $true + } catch { + Write-Host "[WARN] Model load attempt $attempt failed: $($_.Exception.Message)" + if ($_.ErrorDetails) { + Write-Host "Error details: $($_.ErrorDetails.Message)" + } + # Capture the swallowed llama-server crash from the debug stream. + Write-Host "----- Lemonade server debug output (attempt $attempt) -----" + if ($env:LEMONADE_JOB_ID) { + $jobOut = Receive-Job -Id $env:LEMONADE_JOB_ID -ErrorAction SilentlyContinue + if ($jobOut) { $jobOut | Select-Object -Last 150 | ForEach-Object { Write-Host $_ } } + } + if ($attempt -lt 3) { Write-Host "Retrying in 12s..."; Start-Sleep -Seconds 12 } } - throw "Failed to load embedding model" + } + if (-not $loaded) { + throw "Failed to load embedding model after 3 attempts" } # Wait for llamacpp backend to fully initialize (increased from 10s) From 7a2720dd5d11df823f17b4341a6e5eb7a1bf89e9 Mon Sep 17 00:00:00 2001 From: Kalin Ovtcharov Date: Fri, 26 Jun 2026 08:47:56 -0700 Subject: [PATCH 3/3] Revert "ci(embeddings): retry MoE embedding load + capture llama-server debug crash on failure" This reverts commit daa0e46d763365888fed0d670b720841634bf0df. --- .github/workflows/test_embeddings.yml | 50 ++++++++------------------- 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/.github/workflows/test_embeddings.yml b/.github/workflows/test_embeddings.yml index 31b5ddcd2..2f22f631c 100644 --- a/.github/workflows/test_embeddings.yml +++ b/.github/workflows/test_embeddings.yml @@ -72,10 +72,6 @@ jobs: $serverJob = Start-Job -ScriptBlock { # Workaround for Issue #612: Disable Vulkan cooperative matrix optimization $env:GGML_VK_DISABLE_COOPMAT = "1" - # Debug logging so the (otherwise swallowed) llama-server backend - # output is captured in the job stream -- lets us record the exact - # crash if the MoE embedding load aborts (see retry block below). - $env:LEMONADE_LOG_LEVEL = "debug" & lemonade-server serve --host localhost --port 13305 --no-tray 2>&1 } Write-Host "Started Lemonade server job with ID: $($serverJob.Id)" @@ -127,39 +123,23 @@ jobs: Write-Host " [WARN] Pull may have failed: $($_.Exception.Message)" } - # Load embedding model into memory, WITH RETRY. The - # nomic-embed-text-v2-moe MoE model intermittently aborts - # llama-server startup on the AMD Vulkan backend -- a transient - # GPU-driver-state fault (upstream llama.cpp #16301 / lemonade #612, - # not a GAIA defect; the load succeeds once the GPU state settles). - # On every failed attempt we dump the server job's debug output to - # capture the exact (otherwise-swallowed) llama-server crash. + # Load embedding model into memory (required in Lemonade v9.x) Write-Host "`n=== Loading Embedding Model ===" - $loadRequest = @{ model_name = "nomic-embed-text-v2-moe-GGUF" } | ConvertTo-Json - $loaded = $false - for ($attempt = 1; $attempt -le 3 -and -not $loaded; $attempt++) { - try { - Write-Host "Loading model (attempt $attempt/3): nomic-embed-text-v2-moe-GGUF" - $loadResponse = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/load" ` - -Method POST -Body $loadRequest -ContentType "application/json" -TimeoutSec 60 - Write-Host "[OK] Model loaded successfully: $($loadResponse | ConvertTo-Json -Compress)" - $loaded = $true - } catch { - Write-Host "[WARN] Model load attempt $attempt failed: $($_.Exception.Message)" - if ($_.ErrorDetails) { - Write-Host "Error details: $($_.ErrorDetails.Message)" - } - # Capture the swallowed llama-server crash from the debug stream. - Write-Host "----- Lemonade server debug output (attempt $attempt) -----" - if ($env:LEMONADE_JOB_ID) { - $jobOut = Receive-Job -Id $env:LEMONADE_JOB_ID -ErrorAction SilentlyContinue - if ($jobOut) { $jobOut | Select-Object -Last 150 | ForEach-Object { Write-Host $_ } } - } - if ($attempt -lt 3) { Write-Host "Retrying in 12s..."; Start-Sleep -Seconds 12 } + try { + $loadRequest = @{ + model_name = "nomic-embed-text-v2-moe-GGUF" + } | ConvertTo-Json + + Write-Host "Loading model: nomic-embed-text-v2-moe-GGUF" + $loadResponse = Invoke-RestMethod -Uri "http://localhost:13305/api/v1/load" ` + -Method POST -Body $loadRequest -ContentType "application/json" -TimeoutSec 60 + Write-Host "[OK] Model loaded successfully: $($loadResponse | ConvertTo-Json -Compress)" + } catch { + Write-Host "[ERROR] Model load failed: $($_.Exception.Message)" + if ($_.ErrorDetails) { + Write-Host "Error details: $($_.ErrorDetails.Message)" } - } - if (-not $loaded) { - throw "Failed to load embedding model after 3 attempts" + throw "Failed to load embedding model" } # Wait for llamacpp backend to fully initialize (increased from 10s)