diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index 2ba71e93c4d..33683e37383 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -164,7 +164,7 @@ The tables below list every Spark built-in expression with its current status. | `array_repeat` | ✅ | Native | | | `array_union` | ✅ | Native | NaN/signed-zero handling may differ ([details](compatibility/floating-point.md)) | | `arrays_overlap` | ✅ | Native | | -| `arrays_zip` | ✅ | Native | | +| `arrays_zip` | ✅ | Hybrid | Arrays of maps route through the JVM codegen dispatcher; scalar/array/struct elements run natively. NullType elements fall back to Spark | | `element_at` | ✅ | Native | | | `flatten` | ✅ | Native | Binary/struct/map elements fall back | | `get` | ✅ | — | | diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index 63860c629cc..69638346b8b 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -814,10 +814,14 @@ object CometArrayPosition extends CometExpressionSerde[ArrayPosition] with Array } } -object CometArraysZip extends CometExpressionSerde[ArraysZip] { +object CometArraysZip extends CometExpressionSerde[ArraysZip] with CodegenDispatchFallback { + // MapType elements have no native impl; they route through the JVM codegen dispatcher. + // This reason surfaces only when the dispatcher is disabled or refuses the tree. override def getUnsupportedReasons(): Seq[String] = Seq( - "Not all input data types are supported; falls back to Spark for unsupported types") + "MapType element arrays are not supported natively and are handled via JVM codegen dispatch; " + + "this fallback only applies when the dispatcher is disabled. NullType elements remain " + + "unsupported (the dispatcher's isSupportedDataType does not admit NullType).") private def isTypeSupported(dt: DataType): Boolean = { import DataTypes._ diff --git a/spark/src/test/resources/sql-tests/expressions/array/arrays_zip.sql b/spark/src/test/resources/sql-tests/expressions/array/arrays_zip.sql index e62e3842e8f..265f5231223 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/arrays_zip.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/arrays_zip.sql @@ -218,12 +218,18 @@ SELECT arrays_zip(array(struct(1, 2, 3), struct(2, 3, 4))); -- query -- SELECT arrays_zip(array(struct(1, 2, 3), struct(2, 3, 4), struct(null, null, null))); --- Arrays of maps --- FIXME: COMET: map is not supported, unsupported arguments for CreateArray, unsupported arguments for ArraysZip --- +------------------------------------------------------------------+ --- |arrays_zip(array(map(1.0, 2, 3.0, 4)), array(map(1.0, 2, 3.0, 4)))| --- +------------------------------------------------------------------+ --- |[{{1.0 -> 2, 3.0 -> 4}, {1.0 -> 2, 3.0 -> 4}}] | --- +------------------------------------------------------------------+ --- query --- SELECT arrays_zip(array(map(1.0, '2', 3.0, '4')), array(map(1.0, '2', 3.0, '4'))); +-- arrays_zip over arrays of maps: MapType elements have no native impl; the projection stays +-- in Comet via the JVM codegen dispatcher. Input is built from a table column so the +-- expression is not constant-folded before reaching Comet. +statement +CREATE TABLE arrays_zip_map_tbl using parquet AS SELECT 1 AS k1, 2 AS v1, 3 AS k2, 4 AS v2; + +-- positive: arrays of maps must execute via dispatch, not whole-projection Spark fallback +query +SELECT arrays_zip(array(map(k1, v1), map(k2, v2)), array(map('x', v2), map('y', v1))) +FROM arrays_zip_map_tbl + +-- mixed projection: map-element dispatch + scalar native path in the same projection +query +SELECT arrays_zip(array(map(k1, v1)), array(k1, k2)), v1 + v2 +FROM arrays_zip_map_tbl