From 2a16f63a090f4776a274d3d9c5541e5aeadbb555 Mon Sep 17 00:00:00 2001 From: alnavone Date: Fri, 7 Aug 2026 13:51:21 -0700 Subject: [PATCH] fix(local-ai): repair Windows CUDA inference and engine packaging Four bugs in the sd.cpp local-inference path, found while setting the feature up on Windows with an RTX 3090. 1. Windows CUDA builds silently fell back to CPU. The cuda12 asset ships ggml-cuda.dll but not the CUDA runtime it links against; those DLLs are a separate asset in the same release. ggml skips the backend it cannot load and reports no error, so the app looked like it was working while leaving the GPU idle. Add ensureCudaRuntime() to fetch and stage them. 2. --sd-version and --flux no longer exist upstream. sd-cli aborts with "unknown argument" before doing any work, so every SDXL model in the catalog failed 100% of the time. The architecture is detected from the checkpoint now, so just stop passing them. 3. SDXL's sampler was spelled dpmpp2m; the accepted name is dpm++2m. This was masked by #2, which failed argument parsing first. 4. stage-local-ai-binary.js copied an allowlist of filenames, which on Windows meant sd-cli.exe alone out of the ~24 files the runtime needs. Packaged builds shipped an engine that could not start at all (0xC0000135 STATUS_DLL_NOT_FOUND). Stage the whole source directory and keep REQUIRED_FILES as a validation gate instead of a copy manifest. Adds tests/stageLocalAiBinary.test.js covering the staging fix. Co-Authored-By: Claude Opus 5 --- electron/lib/localInference.js | 61 ++++++++++++++++-- electron/lib/modelCatalog.js | 2 +- scripts/stage-local-ai-binary.js | 43 ++++++++----- tests/stageLocalAiBinary.test.js | 102 +++++++++++++++++++++++++++++++ 4 files changed, 187 insertions(+), 21 deletions(-) create mode 100644 tests/stageLocalAiBinary.test.js diff --git a/electron/lib/localInference.js b/electron/lib/localInference.js index d26ea6f6d..3abe6e783 100644 --- a/electron/lib/localInference.js +++ b/electron/lib/localInference.js @@ -230,6 +230,52 @@ const CUSTOM_BINARIES = { 'darwin-arm64': 'https://github.com/Anil-matcha/Open-Generative-AI/releases/download/v1.0.3-binaries/sd-cli-metal-macos-arm64.zip', }; +// The Windows CUDA build ships ggml-cuda.dll but not the CUDA runtime it links +// against. Without these DLLs next to the binary, ggml silently skips the CUDA +// backend and falls back to CPU — orders of magnitude slower, with no error +// surfaced anywhere. leejet publishes them as a separate asset in the same +// release, so pull it whenever we installed a CUDA build that lacks them. +const CUDA_RUNTIME_FILES = ['cudart64_12.dll', 'cublas64_12.dll', 'cublasLt64_12.dll']; + +function cudaRuntimeInstalled() { + return CUDA_RUNTIME_FILES.every((name) => fs.existsSync(path.join(BIN_DIR, name))); +} + +async function ensureCudaRuntime(release, send) { + if (process.platform !== 'win32') return; + // Only CUDA builds ship ggml-cuda.dll; CPU/Vulkan/ROCm builds need nothing extra. + if (!fs.existsSync(path.join(BIN_DIR, 'ggml-cuda.dll'))) return; + if (cudaRuntimeInstalled()) return; + + const asset = (release?.assets || []).find((a) => /^cudart-sd-bin-win-cu\d+-x64\.zip$/.test(a.name)); + if (!asset) { + console.warn('[local-ai] Installed a CUDA build but found no cudart asset in the release — generation will fall back to CPU.'); + return; + } + + send({ phase: 'downloading', progress: 0 }); + const zipPath = path.join(BIN_DIR, asset.name); + await downloadFile(asset.browser_download_url, zipPath, (p) => { + send({ phase: 'downloading', progress: p }); + }); + + send({ phase: 'extracting', progress: 0.98 }); + await extractZip(zipPath, BIN_DIR); + fs.unlinkSync(zipPath); + + // The archive may extract into a subdirectory — flatten to BIN_DIR so the + // DLLs sit next to sd-cli.exe, which is where the loader looks for them. + for (const name of CUDA_RUNTIME_FILES) { + if (fs.existsSync(path.join(BIN_DIR, name))) continue; + const found = findFile(BIN_DIR, name); + if (found) fs.renameSync(found, path.join(BIN_DIR, name)); + } + + if (!cudaRuntimeInstalled()) { + console.warn('[local-ai] CUDA runtime install incomplete — generation may fall back to CPU.'); + } +} + async function downloadBinary(mainWindow) { const send = (data) => mainWindow?.webContents.send('local-ai:download-progress', { id: '__binary__', ...data }); @@ -245,6 +291,7 @@ async function downloadBinary(mainWindow) { const customUrl = CUSTOM_BINARIES[platformKey]; let downloadUrl, zipName; + let chosenRelease = null; if (customUrl) { downloadUrl = customUrl; @@ -271,6 +318,7 @@ async function downloadBinary(mainWindow) { }); if (pickedName) { chosen = zips.find(a => a.name === pickedName); + chosenRelease = release; break; } } @@ -310,6 +358,9 @@ async function downloadBinary(mainWindow) { ensureBinaryPermissions(); + // Windows CUDA builds are useless without the CUDA runtime beside them + await ensureCudaRuntime(chosenRelease, send); + // macOS: strip Gatekeeper quarantine so the downloaded binary can run if (process.platform === 'darwin') { await new Promise((res) => execFile('xattr', ['-cr', BIN_DIR], () => res())); @@ -480,13 +531,11 @@ async function generate(params, mainWindow) { args.push('--llm', llmPath); args.push('--vae', vaePath); if (model.scheduler) args.push('--scheduler', model.scheduler); - } else if (model.type === 'sdxl') { - args.push('--sd-version', 'sdxl'); - } else if (model.type === 'sd2') { - args.push('--sd-version', 'sd2'); - } else if (model.type === 'flux') { - args.push('--flux'); } + // sdxl/sd2/flux used to need an explicit architecture flag (--sd-version, + // --flux). Those flags were removed upstream — sd.cpp now detects the + // architecture from the checkpoint itself, and passing them aborts with + // "unknown argument" before any work starts. Nothing to add here. return new Promise((resolve, reject) => { const startupStartedAt = Date.now(); diff --git a/electron/lib/modelCatalog.js b/electron/lib/modelCatalog.js index d2e5bdf41..a9248952d 100644 --- a/electron/lib/modelCatalog.js +++ b/electron/lib/modelCatalog.js @@ -123,7 +123,7 @@ const LOCAL_MODEL_CATALOG = [ defaultHeight: 1024, defaultSteps: 30, defaultGuidance: 7.5, - sampler: 'dpmpp2m', + sampler: 'dpm++2m', tags: ['sdxl', 'high-quality', 'versatile'], }, ]; diff --git a/scripts/stage-local-ai-binary.js b/scripts/stage-local-ai-binary.js index 8be96387a..f0e72eff3 100644 --- a/scripts/stage-local-ai-binary.js +++ b/scripts/stage-local-ai-binary.js @@ -1,14 +1,20 @@ const fs = require('fs'); const path = require('path'); +// Entry points that must exist in the source directory for a staged build to +// be usable. Everything else found alongside them is copied verbatim. +// +// sd.cpp splits its runtime across many files — ggml backend libraries, per-ISA +// CPU variants, codec libraries, and (for CUDA builds) the CUDA runtime — and +// the exact set differs per platform and per build flavour (CPU / CUDA / Vulkan +// / ROCm). Copying only an allowlist of known names silently drops the rest and +// produces a packaged app whose bundled engine cannot load at all. const REQUIRED_FILES = { darwin: ['sd-cli', 'libstable-diffusion.dylib'], linux: ['sd-cli', 'libstable-diffusion.so'], - win32: ['sd-cli.exe'], + win32: ['sd-cli.exe', 'stable-diffusion.dll'], }; -const OPTIONAL_FILES = ['sd-server']; - function resolveSourceBinDir(sourcePath) { const absoluteSourcePath = path.resolve(sourcePath); const nestedBinDir = path.join(absoluteSourcePath, 'bin'); @@ -20,6 +26,25 @@ function resolveSourceBinDir(sourcePath) { return absoluteSourcePath; } +function copyTree(sourceDir, targetDir, platform) { + for (const entry of fs.readdirSync(sourceDir, { withFileTypes: true })) { + const sourceFile = path.join(sourceDir, entry.name); + const targetFile = path.join(targetDir, entry.name); + + if (entry.isDirectory()) { + fs.mkdirSync(targetFile, { recursive: true }); + copyTree(sourceFile, targetFile, platform); + continue; + } + + fs.copyFileSync(sourceFile, targetFile); + + if (platform !== 'win32') { + fs.chmodSync(targetFile, 0o755); + } + } +} + function stageLocalAiBinary({ platform, arch, sourcePath }) { const sourceBinDir = resolveSourceBinDir(sourcePath); const requiredFiles = REQUIRED_FILES[platform]; @@ -39,17 +64,7 @@ function stageLocalAiBinary({ platform, arch, sourcePath }) { fs.rmSync(stageDir, { recursive: true, force: true }); fs.mkdirSync(stageDir, { recursive: true }); - for (const fileName of [...requiredFiles, ...OPTIONAL_FILES]) { - const sourceFile = path.join(sourceBinDir, fileName); - if (!fs.existsSync(sourceFile)) continue; - - const targetFile = path.join(stageDir, fileName); - fs.copyFileSync(sourceFile, targetFile); - - if (platform !== 'win32') { - fs.chmodSync(targetFile, 0o755); - } - } + copyTree(sourceBinDir, stageDir, platform); return stageDir; } diff --git a/tests/stageLocalAiBinary.test.js b/tests/stageLocalAiBinary.test.js new file mode 100644 index 000000000..73f93fde1 --- /dev/null +++ b/tests/stageLocalAiBinary.test.js @@ -0,0 +1,102 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +const { resolveSourceBinDir, stageLocalAiBinary } = require('../scripts/stage-local-ai-binary'); + +const repoRoot = path.resolve(__dirname, '..'); + +function makeSourceDir(files) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sd-stage-')); + for (const [name, contents] of Object.entries(files)) { + const full = path.join(dir, name); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, contents); + } + return dir; +} + +function cleanup(sourceDir, platform, arch) { + fs.rmSync(sourceDir, { recursive: true, force: true }); + fs.rmSync(path.join(repoRoot, 'build', 'local-ai', `${platform}-${arch}`), { + recursive: true, + force: true, + }); +} + +test('stages every runtime file beside the entry point, not just known names', () => { + // A Windows CUDA build: the engine is useless without its ggml backends and + // the CUDA runtime, none of which are named in REQUIRED_FILES. + const sourceDir = makeSourceDir({ + 'sd-cli.exe': 'exe', + 'sd-server.exe': 'exe', + 'stable-diffusion.dll': 'dll', + 'ggml.dll': 'dll', + 'ggml-base.dll': 'dll', + 'ggml-cuda.dll': 'dll', + 'ggml-cpu-haswell.dll': 'dll', + 'cudart64_12.dll': 'dll', + 'cublas64_12.dll': 'dll', + 'cublasLt64_12.dll': 'dll', + 'libwebp.dll': 'dll', + }); + + try { + const stageDir = stageLocalAiBinary({ platform: 'win32', arch: 'x64', sourcePath: sourceDir }); + const staged = fs.readdirSync(stageDir).sort(); + + assert.deepEqual(staged, [ + 'cublas64_12.dll', + 'cublasLt64_12.dll', + 'cudart64_12.dll', + 'ggml-base.dll', + 'ggml-cpu-haswell.dll', + 'ggml-cuda.dll', + 'ggml.dll', + 'libwebp.dll', + 'sd-cli.exe', + 'sd-server.exe', + 'stable-diffusion.dll', + ]); + } finally { + cleanup(sourceDir, 'win32', 'x64'); + } +}); + +test('rejects a source directory missing the core shared library', () => { + const sourceDir = makeSourceDir({ 'sd-cli.exe': 'exe' }); + + try { + assert.throws( + () => stageLocalAiBinary({ platform: 'win32', arch: 'x64', sourcePath: sourceDir }), + /Missing required files.*stable-diffusion\.dll/s + ); + } finally { + cleanup(sourceDir, 'win32', 'x64'); + } +}); + +test('rejects an unsupported platform', () => { + const sourceDir = makeSourceDir({ 'sd-cli': 'bin' }); + + try { + assert.throws( + () => stageLocalAiBinary({ platform: 'sunos', arch: 'x64', sourcePath: sourceDir }), + /Unsupported platform "sunos"/ + ); + } finally { + cleanup(sourceDir, 'sunos', 'x64'); + } +}); + +test('resolveSourceBinDir prefers a nested bin directory when present', () => { + const sourceDir = makeSourceDir({ [path.join('bin', 'sd-cli')]: 'bin' }); + + try { + assert.equal(resolveSourceBinDir(sourceDir), path.join(path.resolve(sourceDir), 'bin')); + } finally { + fs.rmSync(sourceDir, { recursive: true, force: true }); + } +});