auto: whole-batch GPU bias so matmul-heavy blocks don't thrash devices

The per-op auto threshold is greedy: it places each op in isolation, so a
transformer block's small projection gemms (seq*d*d, below the matmul
threshold) stay on the CPU while its one large FFN gemm goes to the GPU. The
block then thrashes CPU<->GPU and every hand-off is a blocking flush.

graph::run_ now sums M*N*K over every dot in an evaluation batch and, when the
block as a whole earns the GPU (batch_matmul_bias_threshold_, types.h), pins
the whole batch to the GPU via a thread_local flag read by gpu_mode_. Ops with
no GPU kernel still fall back per-op, so results are unchanged; only device
placement shifts. The flag is scoped to the run_ call by an RAII guard.

Calibrated on the pipelined transformer bench (position-rotated, per-config
min, M1 Pro): the Metal threshold sits between d768's ~7e8 attention batch
(pinning it is a net loss) and d1024's ~1.2e9 attention batch (pinning it is a
large win). auto d_model=1024 drops 8.73 -> 5.51 ms/iter (gpu 4.90); d512 stays
on the CPU. env TL_BATCH_MATMUL_BIAS re-calibrates per target.

Author: yhirose

include/array.h

@@ -188,6 +188,12 @@ inline void (*run_noflush_hook)(const std::vector<node_ptr>&) = nullptr;
 // nodes are single-threaded like the rest of evaluation).
 inline uint64_t visit_counter = 0;
 
+// Whole-batch GPU bias for auto_ mode: set by graph::run_ for the span of one
+// evaluation batch when its total matmul work crosses batch_matmul_bias_
+// threshold_ (types.h), read by gpu_mode_. thread_local because each eval
+// thread runs its own batch.
+inline thread_local bool batch_gpu_bias_ = false;
+
 }  // namespace detail
 
 class array {
@@ -1381,6 +1387,7 @@ struct graph {
     if (device_ == device_type::gpu) return true;
     if (device_ != device_type::auto_) return false;
     if (gpu::pending()) return true;
+    if (batch_gpu_bias_) return true;  // batch pinned to GPU (see run_)
     return n >= auto_threshold_(kc);
   }
 
@@ -1756,6 +1763,24 @@ struct graph {
         }
       }
     }
+    // Auto-mode whole-batch device bias: sum this batch's matmul work and, if
+    // the block as a whole earns the GPU, pin every op in it there so the
+    // sub-threshold projection gemms don't strand on the CPU and thrash the
+    // pipeline (types.h batch_matmul_bias_threshold_). cpu/gpu modes and a
+    // pipeline already in flight short-circuit gpu_mode_, so skip the scan.
+    struct BiasGuard {
+      ~BiasGuard() { batch_gpu_bias_ = false; }
+    } bias_guard;
+    if (device_ == device_type::auto_ && gpu::available() && !gpu::pending()) {
+      int64_t work = 0;
+      for (const node* n : order) {
+        if (n->op != node::op_t::dot || n->inputs.size() != 2) continue;
+        const auto& sa = n->inputs[0]->shape;
+        const auto& sb = n->inputs[1]->shape;
+        if (sa.size() == 2 && sb.size() == 2) work += sa[0] * sa[1] * sb[1];
+      }
+      if (work >= batch_matmul_bias_threshold_()) batch_gpu_bias_ = true;
+    }
     for (auto* n : order) eval_one(*n);
     if (do_flush) gpu::flush();  // blocking eval: batch done when run() returns
   }

include/types.h

@@ -7,6 +7,7 @@
 // backends, no kernel changes). Activations, results and scalars stay F32.
 
 #include <cstdint>
+#include <cstdlib>
 #include <cstring>
 
 namespace tl {
@@ -120,4 +121,36 @@ inline int64_t auto_threshold_(kernel_class kc) {
   return 1'000'000'000'000;  // unreachable
 }
 
+// Batch device bias (auto_ mode). The per-op auto_threshold_ is greedy: it
+// decides each op in isolation, so a transformer block's small projection
+// gemms (e.g. seq·d·d = 256·768·768 = 1.5e8, below the matmul threshold)
+// strand on the CPU while its one big FFN gemm goes to the GPU — the batch
+// then thrashes CPU⇄GPU (each hand-off is a blocking flush). This threshold
+// is compared against the *sum* of M·N·K over every dot in one evaluation
+// batch (graph::run_); above it the whole batch is pinned to the GPU so the
+// projections ride along and the pipeline stays on one device. Env override
+// TL_BATCH_MATMUL_BIAS (element-product units) for host calibration, mirroring
+// the misc/census.cpp re-measure note above.
+inline int64_t batch_matmul_bias_threshold_() {
+  static const int64_t v = []() -> int64_t {
+    if (const char* e = std::getenv("TL_BATCH_MATMUL_BIAS")) {
+      char* end = nullptr;
+      long long x = std::strtoll(e, &end, 10);
+      if (end != e && x > 0) return static_cast<int64_t>(x);
+    }
+#if defined(TENSORLIB_CUDA)
+    return 4'000'000;             // GPU wins early; a couple of small gemms
+#else
+    // Metal / M1 Pro, calibrated 2026-07-18 on the pipelined transformer bench
+    // (position-rotated, per-config min). The batch bias helps only when the
+    // whole block is GPU-favourable: pinning d768's ~7e8 attention batch to
+    // the GPU is a net loss (encode/flush > the small gemms' savings), while
+    // pinning d1024's ~1.2e9 attention batch is a large win (8.73→5.6 ms/iter).
+    // Sit the threshold between them; env TL_BATCH_MATMUL_BIAS re-calibrates.
+    return 800'000'000;
+#endif
+  }();
+  return v;
+}
+
 }  // namespace tl