Fix local inference on Windows: silent CPU fallback, dead CLI flags, and broken engine packaging

electron/lib/localInference.js

@@ -230,6 +230,52 @@ const CUSTOM_BINARIES = {
     'darwin-arm64': 'https://github.com/Anil-matcha/Open-Generative-AI/releases/download/v1.0.3-binaries/sd-cli-metal-macos-arm64.zip',
 };
 
+// The Windows CUDA build ships ggml-cuda.dll but not the CUDA runtime it links
+// against. Without these DLLs next to the binary, ggml silently skips the CUDA
+// backend and falls back to CPU — orders of magnitude slower, with no error
+// surfaced anywhere. leejet publishes them as a separate asset in the same
+// release, so pull it whenever we installed a CUDA build that lacks them.
+const CUDA_RUNTIME_FILES = ['cudart64_12.dll', 'cublas64_12.dll', 'cublasLt64_12.dll'];
+
+function cudaRuntimeInstalled() {
+    return CUDA_RUNTIME_FILES.every((name) => fs.existsSync(path.join(BIN_DIR, name)));
+}
+
+async function ensureCudaRuntime(release, send) {
+    if (process.platform !== 'win32') return;
+    // Only CUDA builds ship ggml-cuda.dll; CPU/Vulkan/ROCm builds need nothing extra.
+    if (!fs.existsSync(path.join(BIN_DIR, 'ggml-cuda.dll'))) return;
+    if (cudaRuntimeInstalled()) return;
+
+    const asset = (release?.assets || []).find((a) => /^cudart-sd-bin-win-cu\d+-x64\.zip$/.test(a.name));
+    if (!asset) {
+        console.warn('[local-ai] Installed a CUDA build but found no cudart asset in the release — generation will fall back to CPU.');
+        return;
+    }
+
+    send({ phase: 'downloading', progress: 0 });
+    const zipPath = path.join(BIN_DIR, asset.name);
+    await downloadFile(asset.browser_download_url, zipPath, (p) => {
+        send({ phase: 'downloading', progress: p });
+    });
+
+    send({ phase: 'extracting', progress: 0.98 });
+    await extractZip(zipPath, BIN_DIR);
+    fs.unlinkSync(zipPath);
+
+    // The archive may extract into a subdirectory — flatten to BIN_DIR so the
+    // DLLs sit next to sd-cli.exe, which is where the loader looks for them.
+    for (const name of CUDA_RUNTIME_FILES) {
+        if (fs.existsSync(path.join(BIN_DIR, name))) continue;
+        const found = findFile(BIN_DIR, name);
+        if (found) fs.renameSync(found, path.join(BIN_DIR, name));
+    }
+
+    if (!cudaRuntimeInstalled()) {
+        console.warn('[local-ai] CUDA runtime install incomplete — generation may fall back to CPU.');
+    }
+}
+
 async function downloadBinary(mainWindow) {
     const send = (data) => mainWindow?.webContents.send('local-ai:download-progress', { id: '__binary__', ...data });
 
@@ -245,6 +291,7 @@ async function downloadBinary(mainWindow) {
         const customUrl = CUSTOM_BINARIES[platformKey];
 
         let downloadUrl, zipName;
+        let chosenRelease = null;
 
         if (customUrl) {
             downloadUrl = customUrl;
@@ -271,6 +318,7 @@ async function downloadBinary(mainWindow) {
                 });
                 if (pickedName) {
                     chosen = zips.find(a => a.name === pickedName);
+                    chosenRelease = release;
                     break;
                 }
             }
@@ -310,6 +358,9 @@ async function downloadBinary(mainWindow) {
 
         ensureBinaryPermissions();
 
+        // Windows CUDA builds are useless without the CUDA runtime beside them
+        await ensureCudaRuntime(chosenRelease, send);
+
         // macOS: strip Gatekeeper quarantine so the downloaded binary can run
         if (process.platform === 'darwin') {
             await new Promise((res) => execFile('xattr', ['-cr', BIN_DIR], () => res()));
@@ -480,13 +531,11 @@ async function generate(params, mainWindow) {
         args.push('--llm', llmPath);
         args.push('--vae', vaePath);
         if (model.scheduler) args.push('--scheduler', model.scheduler);
-    } else if (model.type === 'sdxl') {
-        args.push('--sd-version', 'sdxl');
-    } else if (model.type === 'sd2') {
-        args.push('--sd-version', 'sd2');
-    } else if (model.type === 'flux') {
-        args.push('--flux');
     }
+    // sdxl/sd2/flux used to need an explicit architecture flag (--sd-version,
+    // --flux). Those flags were removed upstream — sd.cpp now detects the
+    // architecture from the checkpoint itself, and passing them aborts with
+    // "unknown argument" before any work starts. Nothing to add here.
 
     return new Promise((resolve, reject) => {
         const startupStartedAt = Date.now();

electron/lib/modelCatalog.js

@@ -123,7 +123,7 @@ const LOCAL_MODEL_CATALOG = [
         defaultHeight: 1024,
         defaultSteps: 30,
         defaultGuidance: 7.5,
-        sampler: 'dpmpp2m',
+        sampler: 'dpm++2m',
         tags: ['sdxl', 'high-quality', 'versatile'],
     },
 ];

scripts/stage-local-ai-binary.js

@@ -1,14 +1,20 @@
 const fs = require('fs');
 const path = require('path');
 
+// Entry points that must exist in the source directory for a staged build to
+// be usable. Everything else found alongside them is copied verbatim.
+//
+// sd.cpp splits its runtime across many files — ggml backend libraries, per-ISA
+// CPU variants, codec libraries, and (for CUDA builds) the CUDA runtime — and
+// the exact set differs per platform and per build flavour (CPU / CUDA / Vulkan
+// / ROCm). Copying only an allowlist of known names silently drops the rest and
+// produces a packaged app whose bundled engine cannot load at all.
 const REQUIRED_FILES = {
     darwin: ['sd-cli', 'libstable-diffusion.dylib'],
     linux: ['sd-cli', 'libstable-diffusion.so'],
-    win32: ['sd-cli.exe'],
+    win32: ['sd-cli.exe', 'stable-diffusion.dll'],
 };
 
-const OPTIONAL_FILES = ['sd-server'];
-
 function resolveSourceBinDir(sourcePath) {
     const absoluteSourcePath = path.resolve(sourcePath);
     const nestedBinDir = path.join(absoluteSourcePath, 'bin');
@@ -20,6 +26,25 @@ function resolveSourceBinDir(sourcePath) {
     return absoluteSourcePath;
 }
 
+function copyTree(sourceDir, targetDir, platform) {
+    for (const entry of fs.readdirSync(sourceDir, { withFileTypes: true })) {
+        const sourceFile = path.join(sourceDir, entry.name);
+        const targetFile = path.join(targetDir, entry.name);
+
+        if (entry.isDirectory()) {
+            fs.mkdirSync(targetFile, { recursive: true });
+            copyTree(sourceFile, targetFile, platform);
+            continue;
+        }
+
+        fs.copyFileSync(sourceFile, targetFile);
+
+        if (platform !== 'win32') {
+            fs.chmodSync(targetFile, 0o755);
+        }
+    }
+}
+
 function stageLocalAiBinary({ platform, arch, sourcePath }) {
     const sourceBinDir = resolveSourceBinDir(sourcePath);
     const requiredFiles = REQUIRED_FILES[platform];
@@ -39,17 +64,7 @@ function stageLocalAiBinary({ platform, arch, sourcePath }) {
     fs.rmSync(stageDir, { recursive: true, force: true });
     fs.mkdirSync(stageDir, { recursive: true });
 
-    for (const fileName of [...requiredFiles, ...OPTIONAL_FILES]) {
-        const sourceFile = path.join(sourceBinDir, fileName);
-        if (!fs.existsSync(sourceFile)) continue;
-
-        const targetFile = path.join(stageDir, fileName);
-        fs.copyFileSync(sourceFile, targetFile);
-
-        if (platform !== 'win32') {
-            fs.chmodSync(targetFile, 0o755);
-        }
-    }
+    copyTree(sourceBinDir, stageDir, platform);
 
     return stageDir;
 }

tests/stageLocalAiBinary.test.js

@@ -0,0 +1,102 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+
+const { resolveSourceBinDir, stageLocalAiBinary } = require('../scripts/stage-local-ai-binary');
+
+const repoRoot = path.resolve(__dirname, '..');
+
+function makeSourceDir(files) {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sd-stage-'));
+    for (const [name, contents] of Object.entries(files)) {
+        const full = path.join(dir, name);
+        fs.mkdirSync(path.dirname(full), { recursive: true });
+        fs.writeFileSync(full, contents);
+    }
+    return dir;
+}
+
+function cleanup(sourceDir, platform, arch) {
+    fs.rmSync(sourceDir, { recursive: true, force: true });
+    fs.rmSync(path.join(repoRoot, 'build', 'local-ai', `${platform}-${arch}`), {
+        recursive: true,
+        force: true,
+    });
+}
+
+test('stages every runtime file beside the entry point, not just known names', () => {
+    // A Windows CUDA build: the engine is useless without its ggml backends and
+    // the CUDA runtime, none of which are named in REQUIRED_FILES.
+    const sourceDir = makeSourceDir({
+        'sd-cli.exe': 'exe',
+        'sd-server.exe': 'exe',
+        'stable-diffusion.dll': 'dll',
+        'ggml.dll': 'dll',
+        'ggml-base.dll': 'dll',
+        'ggml-cuda.dll': 'dll',
+        'ggml-cpu-haswell.dll': 'dll',
+        'cudart64_12.dll': 'dll',
+        'cublas64_12.dll': 'dll',
+        'cublasLt64_12.dll': 'dll',
+        'libwebp.dll': 'dll',
+    });
+
+    try {
+        const stageDir = stageLocalAiBinary({ platform: 'win32', arch: 'x64', sourcePath: sourceDir });
+        const staged = fs.readdirSync(stageDir).sort();
+
+        assert.deepEqual(staged, [
+            'cublas64_12.dll',
+            'cublasLt64_12.dll',
+            'cudart64_12.dll',
+            'ggml-base.dll',
+            'ggml-cpu-haswell.dll',
+            'ggml-cuda.dll',
+            'ggml.dll',
+            'libwebp.dll',
+            'sd-cli.exe',
+            'sd-server.exe',
+            'stable-diffusion.dll',
+        ]);
+    } finally {
+        cleanup(sourceDir, 'win32', 'x64');
+    }
+});
+
+test('rejects a source directory missing the core shared library', () => {
+    const sourceDir = makeSourceDir({ 'sd-cli.exe': 'exe' });
+
+    try {
+        assert.throws(
+            () => stageLocalAiBinary({ platform: 'win32', arch: 'x64', sourcePath: sourceDir }),
+            /Missing required files.*stable-diffusion\.dll/s
+        );
+    } finally {
+        cleanup(sourceDir, 'win32', 'x64');
+    }
+});
+
+test('rejects an unsupported platform', () => {
+    const sourceDir = makeSourceDir({ 'sd-cli': 'bin' });
+
+    try {
+        assert.throws(
+            () => stageLocalAiBinary({ platform: 'sunos', arch: 'x64', sourcePath: sourceDir }),
+            /Unsupported platform "sunos"/
+        );
+    } finally {
+        cleanup(sourceDir, 'sunos', 'x64');
+    }
+});
+
+test('resolveSourceBinDir prefers a nested bin directory when present', () => {
+    const sourceDir = makeSourceDir({ [path.join('bin', 'sd-cli')]: 'bin' });
+
+    try {
+        assert.equal(resolveSourceBinDir(sourceDir), path.join(path.resolve(sourceDir), 'bin'));
+    } finally {
+        fs.rmSync(sourceDir, { recursive: true, force: true });
+    }
+});