diff --git a/backends/vulkan/patterns/BUCK b/backends/vulkan/patterns/BUCK index 7803ba64f60..93d8a0c684d 100644 --- a/backends/vulkan/patterns/BUCK +++ b/backends/vulkan/patterns/BUCK @@ -18,6 +18,7 @@ fbcode_target(_kind = runtime.python_library, "quantized_pixel_shuffle.py", "quantized_unary.py", "rms_norm.py", + "weight_packing_utils.py", "sdpa.py", "select_as_symint.py", ], diff --git a/backends/vulkan/patterns/quantized_convolution.py b/backends/vulkan/patterns/quantized_convolution.py index 61546b108ae..0f6f13c27bc 100644 --- a/backends/vulkan/patterns/quantized_convolution.py +++ b/backends/vulkan/patterns/quantized_convolution.py @@ -183,7 +183,7 @@ def find_quantized_convolution_patterns( @register_pattern_replacement("quantized_convolution") -def make_q8ta_conv2d_custom_op( +def make_q8ta_conv2d_custom_op( # noqa: C901 ep: ExportedProgram, graph_module: torch.fx.GraphModule, match: QuantizedConvolutionMatch, @@ -249,9 +249,10 @@ def make_q8ta_conv2d_custom_op( # Need to make sure that OC dim is a multiple of 4 so that data load/stores are well # aligned with texel boundaries. Add padding to align to the next multiple of 4 if # needed. - utils.align_width_and_update_state_dict( - ep, match.weight_node, weight_tensor, force_update=True - ) + if utils.register_param_mutation(ep, match.weight_node, "8 bit conv2d weight"): + utils.align_width_and_update_state_dict( + ep, match.weight_node, weight_tensor, force_update=True + ) utils.align_width_and_update_state_dict( ep, match.weight_scales_node, weight_scales_tensor ) diff --git a/backends/vulkan/patterns/quantized_embedding.py b/backends/vulkan/patterns/quantized_embedding.py index d260c8feb71..dbd0d6fbf98 100644 --- a/backends/vulkan/patterns/quantized_embedding.py +++ b/backends/vulkan/patterns/quantized_embedding.py @@ -14,10 +14,18 @@ register_pattern_detector, register_pattern_replacement, ) +from executorch.backends.vulkan.patterns.weight_packing_utils import ( + pack_4bit_weight_tensor, +) from executorch.exir import ExportedProgram from executorch.exir.dialects._ops import ops as exir_ops +embedding_4bit_target = exir_ops.edge.quantized_decomposed.embedding_4bit.dtype +embedding_target = exir_ops.edge.aten.embedding.default +torchao_dequantize_affine_target = exir_ops.edge.torchao.dequantize_affine.default + + class QuantizedEmbeddingMatch(PatternMatch): def __init__(self, node: torch.fx.Node) -> None: self.anchor_node = node @@ -65,51 +73,22 @@ def __init__(self, node: torch.fx.Node) -> None: self.scales_node = scales_node self.all_nodes.extend(arg_chain) - self.match_found = True - - -embedding_4bit_target = exir_ops.edge.quantized_decomposed.embedding_4bit.dtype - - -def _detect_tied_linear_weight( - ep: ExportedProgram, - weight_node: torch.fx.Node, - weight_tensor: torch.Tensor, -) -> bool: - """Check if this embedding weight is tied to a linear weight. - - The embedding weight is packed uint8 [vocab_size, embed_dim/2]. The linear - output weight may be stored as unpacked int8 [vocab_size, embed_dim]. If we - find a placeholder whose int8 values match our unpacked embedding values, - the weights are tied and we should use the linear packing to enable dedup. - """ - vocab_size = weight_tensor.shape[0] - embed_dim = weight_tensor.shape[1] * 2 - - # Unpack embedding weight using embedding convention (high nibble first) - emb_high = (weight_tensor >> 4).to(torch.int8) - 8 - emb_low = (weight_tensor & 0xF).to(torch.int8) - 8 - emb_unpacked = torch.stack([emb_high, emb_low], dim=-1).reshape( - vocab_size, embed_dim - ) - - for node in ep.graph_module.graph.nodes: - if node.op != "placeholder" or node == weight_node: - continue - - try: - candidate = get_param_tensor(ep, node) - except RuntimeError: - continue - if candidate is None: - continue - if candidate.shape != (vocab_size, embed_dim) or candidate.dtype != torch.int8: - continue - - if torch.equal(emb_unpacked, candidate): - return True + # The weight placeholder stores values PACKED as uint8 [vocab, + # embed_dim / 2], so embed_dim is twice the inner dim. The op + # implementation requires that embed dim % 32 == 0 due to load/store + # granularity for the weight tensor; enforce that check now. + weight_val = ( + self.weight_node.meta.get("val", None) + if isinstance(self.weight_node, torch.fx.Node) + else None + ) + if not isinstance(weight_val, torch.Tensor) or weight_val.ndim != 2: + return + embed_dim = int(weight_val.shape[-1]) * 2 # packed, 2 values per byte + if embed_dim % 32 != 0: + return - return False + self.match_found = True @register_pattern_detector("quantized_embedding") @@ -137,23 +116,25 @@ def replace_quantized_embedding_patterns( scales_tensor = get_param_tensor(ep, match.scales_node) assert scales_tensor is not None - is_linear_weight = _detect_tied_linear_weight(ep, match.weight_node, weight_tensor) - - if is_linear_weight: + # The quantized_decomposed.embedding_4bit op (which is being replaced) + # already stores weights as packed uint8 [vocab, embed_dim / 2] (low nibble = odd, + # high nibble = even). However, in the Vulkan runtime 4-bit linear layers + # expect the reverse nibble packing (low nibble = even, high nibble = odd). + # In LLMs, where quantized embeddings are most frequently used, the embedding + # layer will share weights with the final LM head linear layer. For simplicity, + # always repack the weight tensor in the format expected by 4 bit linear layers; + # the runtime shader supports both the original and repacked packing formats + # for weights. + if utils.register_param_mutation(ep, match.weight_node, "4 bit linear weight"): # Repack using linear convention (low nibble = even, high nibble = odd) vocab_size = weight_tensor.shape[0] high = (weight_tensor >> 4).to(torch.int8) - 8 low = (weight_tensor & 0xF).to(torch.int8) - 8 unpacked = torch.stack([high, low], dim=-1).reshape(vocab_size, -1) - repacked = unpacked.to(torch.uint8) + 8 - weight_tensor = repacked[:, 1::2] << 4 | repacked[:, ::2] - # Update the state dict with repacked tensor - original_weight = get_param_tensor(ep, match.weight_node) - if original_weight is not None: - for key, value in ep.state_dict.items(): - if value.data_ptr() == original_weight.data_ptr(): - ep.state_dict[key] = weight_tensor - break + weight_tensor = pack_4bit_weight_tensor(unpacked) + utils.align_width_and_update_state_dict( + ep, match.weight_node, weight_tensor, align_to=1, force_update=True + ) # Compute group_size from weight and scales shapes embed_dim = weight_tensor.shape[1] * 2 # packed, 2 values per byte @@ -169,7 +150,191 @@ def replace_quantized_embedding_patterns( match.scales_node, group_size, match.indices_node, - is_linear_weight, + True, # is_linear_weight + ), + ) + + embedding_q4gsw_node.meta["val"] = match.anchor_node.meta["val"] + match.anchor_node.replace_all_uses_with(embedding_q4gsw_node) + + +class TorchAOQuantizedEmbeddingMatch(PatternMatch): + """Matches a torchao 4-bit weight-only quantized embedding and rewrites it + as a single et_vk.embedding_q4gsw.default node. + + The recognized graph shape is a split torchao.dequantize_affine -> + aten.embedding, whose weight is unpacked int8 [vocab, embed_dim] with values + in [-8, 7]. This requires symmetric 4-bit signed quantization (quant_min=-8, + quant_max=7, zero_point=0) and per-row groupwise blocks (block_size=[1, G]), + which the runtime shader assumes via a fixed subtract-8 offset. + """ + + def __init__(self, node: torch.fx.Node) -> None: # noqa: C901 + self.anchor_node = node + self.match_found = False + self.all_nodes = [node] + + # aten.embedding.default args: (weight, indices, *) + dequant_node = node.args[0] + self.indices_node = node.args[1] + + if not isinstance(dequant_node, torch.fx.Node): + return + if dequant_node.target != torchao_dequantize_affine_target: + return + + self.all_nodes.append(dequant_node) + + # torchao.dequantize_affine args: + # (input, block_size, scale, zero_point, input_dtype, quant_min, + # quant_max, ...) + block_size = dequant_node.args[1] + input_dtype = dequant_node.args[4] if len(dequant_node.args) > 4 else None + quant_min = dequant_node.args[5] if len(dequant_node.args) > 5 else None + quant_max = dequant_node.args[6] if len(dequant_node.args) > 6 else None + + # The shader hardcodes the 4-bit signed offset (subtract 8), which + # corresponds to quant_min=-8, quant_max=7, zero_point=0. + if quant_min != -8 or quant_max != 7: + return + + # Key off the dequant node's declared input_dtype, not the weight + # placeholder's live meta: a sibling match sharing the same (tied) weight + # may have repacked that placeholder in place (flipping it to packed + # uint8 [vocab, embed_dim / 2]), which would spuriously reject us here. + if input_dtype != torch.int8: + return + + # block_size must be per-row groupwise: [1, group_size] + if not isinstance(block_size, (list, tuple)) or len(block_size) != 2: + return + if block_size[0] != 1: + return + self.group_size = int(block_size[1]) + + # Trace weight (args[0]) and scales (args[2]) to their placeholders. A + # placeholder-backed zero_point's symmetric (zero_point == 0) + # requirement is verified on the real tensor in the replacement + # function, where the ExportedProgram is available; checking the fake + # meta tensor here would trigger a data-dependent guard error. + weight_node, arg_chain = utils.trace_args_until_placeholder( + dequant_node.args[0] + ) + if weight_node is None: + return + self.weight_node = weight_node + self.all_nodes.extend(arg_chain) + + # Read embed_dim from the dequant node's float output meta, not the + # weight placeholder's meta: a tied weight may have been repacked in + # place by a sibling match (halving its inner dim), but this output meta + # is stable. Runtime shader requires embed_dim % 32 == 0 and the groups + # to tile the row exactly; reject otherwise rather than emit an op the + # runtime would abort on. + dequant_val = dequant_node.meta.get("val", None) + if not isinstance(dequant_val, torch.Tensor) or dequant_val.ndim != 2: + return + embed_dim = int(dequant_val.shape[-1]) + if self.group_size <= 0 or embed_dim % self.group_size != 0: + return + if embed_dim % 32 != 0: + return + + scales_node, arg_chain = utils.trace_args_until_placeholder( + dequant_node.args[2] + ) + if scales_node is None: + return + self.scales_node = scales_node + self.all_nodes.extend(arg_chain) + + # zero_point (args[3]) must be provably zero, since the shader hardcodes + # a subtract-8 offset that assumes symmetric quantization. Reject the + # match otherwise so the op falls back cleanly rather than miscomputing. + zero_point = dequant_node.args[3] + self.zero_point_node = None + if zero_point is None: + # Symmetric quant; zero_point == 0 is implied. + pass + elif isinstance(zero_point, torch.fx.Node): + zero_point_node, arg_chain = utils.trace_args_until_placeholder(zero_point) + if zero_point_node is None: + # Untraceable to a placeholder; cannot verify it is zero. + return + self.zero_point_node = zero_point_node + self.all_nodes.extend(arg_chain) + else: + # Inline scalar / list / tuple; verify the literal value(s) are zero. + values = ( + zero_point if isinstance(zero_point, (list, tuple)) else [zero_point] + ) + if any(v != 0 for v in values): + return + + self.match_found = True + + +@register_pattern_detector("torchao_quantized_embedding") +def find_torchao_quantized_embedding_patterns( + node: torch.fx.Node, +) -> Optional[TorchAOQuantizedEmbeddingMatch]: + if node.target != embedding_target: + return None + + matched_pattern = TorchAOQuantizedEmbeddingMatch(node) + if matched_pattern.match_found: + return matched_pattern + return None + + +@register_pattern_replacement("torchao_quantized_embedding") +def replace_torchao_quantized_embedding_patterns( + ep: ExportedProgram, + graph_module: torch.fx.GraphModule, + match: TorchAOQuantizedEmbeddingMatch, +): + # Always repack with the packing expected by 4 bit linear layers for + # simplicity. See replace_quantized_embedding_patterns() for more details + if utils.register_param_mutation(ep, match.weight_node, "4 bit linear weight"): + weight_tensor = get_param_tensor(ep, match.weight_node) + assert weight_tensor is not None + + # The shader applies a fixed signed-4-bit offset (subtract 8), which + # assumes symmetric quantization (zero_point == 0). The None / inline + # literal cases were already proven zero in the matcher; a placeholder + # was committed to during matching, so its backing tensor must be + # fetchable and verifiable here. + if match.zero_point_node is not None: + zero_point_tensor = get_param_tensor(ep, match.zero_point_node) + if zero_point_tensor is None: + raise RuntimeError( + "embedding_q4gsw: zero_point traced to placeholder " + f"{match.zero_point_node.name!r} but its backing tensor " + "could not be fetched to verify symmetric quantization " + "(zero_point == 0)." + ) + assert torch.all( + zero_point_tensor == 0 + ), "embedding_q4gsw requires symmetric quantization (zero_point == 0)" + + packed_weight = pack_4bit_weight_tensor(weight_tensor) + + utils.align_width_and_update_state_dict( + ep, match.weight_node, packed_weight, align_to=1, force_update=True + ) + + group_size = match.group_size + + with graph_module.graph.inserting_before(match.anchor_node): + embedding_q4gsw_node = graph_module.graph.create_node( + "call_function", + exir_ops.edge.et_vk.embedding_q4gsw.default, + args=( + match.weight_node, + match.scales_node, + group_size, + match.indices_node, + True, # is_linear_weight ), ) diff --git a/backends/vulkan/patterns/quantized_linear.py b/backends/vulkan/patterns/quantized_linear.py index c6524102ac6..86a35298fa4 100644 --- a/backends/vulkan/patterns/quantized_linear.py +++ b/backends/vulkan/patterns/quantized_linear.py @@ -19,6 +19,9 @@ register_pattern_detector, register_pattern_replacement, ) +from executorch.backends.vulkan.patterns.weight_packing_utils import ( + pack_4bit_weight_tensor, +) from executorch.exir import ExportedProgram from executorch.exir.dialects._ops import ops as exir_ops from torch.export.graph_signature import InputKind @@ -290,45 +293,6 @@ def find_quantized_linear_patterns( ## -def pack_4bit_weight_tensor(weight_tensor: torch.Tensor) -> torch.Tensor: - """ - Given a 8-bit weight tensor containing values quantized to 4 bits, create a packed - weight tensor by transposing the weight tensor, then packing 2 4-bit values in one - 8-bit value. - - An input weight tensor of shape (N, K) will produce a packed weight tensor of shape - (K, N / 2). - """ - - # Assert we got a properly quantized tensor. - min_val, max_val = weight_tensor.min().item(), weight_tensor.max().item() - assert ( - max_val <= 7 and min_val >= -8 - ), f"pack_4bit_weight_tensor: [min_val,max_val] out of [-8, 7] range, got [{min_val}, {max_val}]" - - # Assuming we have a 2d tensor - if weight_tensor.ndim != 2: - weight_tensor = weight_tensor.squeeze() - assert ( - weight_tensor.ndim == 2 - ), f"pack_4bit_weight_tensor: expecting input tensor to be 2d, got {weight_tensor.ndim}" - - # Need to pad innermost dim to be a multiple of 8, since the minimum load granularity - # is int32 (4 bytes), which contains 8 4-bit values. - if weight_tensor.shape[-1] % 8 != 0: - num_pad = 8 - (weight_tensor.shape[-1] % 8) - weight_tensor = F.pad(input=weight_tensor, pad=(0, num_pad)) - - # Shape after padding - _, in_channels = weight_tensor.shape - assert in_channels % 8 == 0, "convert_to_qc4w: expecting ic to be divisible by 8" - - # Adjust weight_tensor tensor for zp - weight_tensor = weight_tensor.to(dtype=torch.uint8) + 8 - # Pack each 4-bit value into a single 8-bit value - return weight_tensor[::, 1::2] << 4 | weight_tensor[::, ::2] - - def compute_per_group_sums(weight_tensor: torch.Tensor, group_size: int): """ Compute the sum of weights per quantization group. @@ -373,24 +337,25 @@ def make_linear_q4gsw_op( in_channels = weight_tensor.shape[-1] group_size = in_channels // num_groups - weight_tensor = pack_4bit_weight_tensor(weight_tensor) - # Use this function for convenience to update the state dict with the packed - # weight tensor. Alignment will already have been done in the above function. - weight_tensor = utils.align_width_and_update_state_dict( - ep, match.weight_node, weight_tensor, align_to=1, force_update=True - ) + if utils.register_param_mutation(ep, match.weight_node, "4 bit linear weight"): + weight_tensor = pack_4bit_weight_tensor(weight_tensor) + # Use this function for convenience to update the state dict with the packed + # weight tensor. Alignment will already have been done in the above function. + weight_tensor = utils.align_width_and_update_state_dict( + ep, match.weight_node, weight_tensor, align_to=1, force_update=True + ) # Also transpose the weight scales tensor to shape [num_groups, N] - weight_scales_tensor = weight_scales_tensor.transpose(0, 1).contiguous() - # Align to multiple of 8 to ensure that data loads from the weight scales - # tensor do not go out of bounds. Each thread computes 8 output channels. - utils.align_width_and_update_state_dict( - ep, - match.weight_scales_node, - weight_scales_tensor, - align_to=8, - force_update=True, - ) + if utils.register_param_mutation( + ep, match.weight_scales_node, "4 bit linear scales" + ): + weight_scales_tensor = weight_scales_tensor.transpose(0, 1).contiguous() + utils.align_width_and_update_state_dict( + ep, + match.weight_scales_node, + weight_scales_tensor, + force_update=True, + ) # Pad bias to multiple of 4 if present if match.bias_node is not None: @@ -429,22 +394,25 @@ def make_linear_dq8ca_q4gsw_op( # Compute per quant group sums before packing the weight tensor sum_per_quant_group = compute_per_group_sums(weight_tensor, group_size) - weight_tensor = pack_4bit_weight_tensor(weight_tensor) - # Use this function for convenience to update the state dict with the packed - # weight tensor. Alignment will already have been done in the above function. - weight_tensor = utils.align_width_and_update_state_dict( - ep, match.weight_node, weight_tensor, align_to=1, force_update=True - ) + if utils.register_param_mutation(ep, match.weight_node, "4 bit linear weight"): + weight_tensor = pack_4bit_weight_tensor(weight_tensor) + # Use this function for convenience to update the state dict with the packed + # weight tensor. Alignment will already have been done in the above function. + weight_tensor = utils.align_width_and_update_state_dict( + ep, match.weight_node, weight_tensor, align_to=1, force_update=True + ) # Also transpose the weight scales tensor to shape [num_groups, N] - weight_scales_tensor = weight_scales_tensor.transpose(0, 1).contiguous() - utils.align_width_and_update_state_dict( - ep, - match.weight_scales_node, - weight_scales_tensor, - align_to=1, - force_update=True, - ) + if utils.register_param_mutation( + ep, match.weight_scales_node, "4 bit linear scales" + ): + weight_scales_tensor = weight_scales_tensor.transpose(0, 1).contiguous() + utils.align_width_and_update_state_dict( + ep, + match.weight_scales_node, + weight_scales_tensor, + force_update=True, + ) # Pad bias to multiple of 4 if present if match.bias_node is not None: diff --git a/backends/vulkan/patterns/weight_packing_utils.py b/backends/vulkan/patterns/weight_packing_utils.py new file mode 100644 index 00000000000..a859c1ab586 --- /dev/null +++ b/backends/vulkan/patterns/weight_packing_utils.py @@ -0,0 +1,51 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Optional + +import torch +import torch.nn.functional as F + + +def pack_4bit_weight_tensor( + weight_tensor: torch.Tensor, + *, + inner_dim_padding: Optional[int] = 8, +) -> torch.Tensor: + """Pack signed 4-bit values stored in int8 into uint8 byte pairs. + + The input tensor stores one quantized value per byte in the range [-8, 7]. + The returned tensor stores two 4-bit values per byte along the innermost dim. + + This preserves the legacy linear q4gsw packing convention: + packed_byte = (odd_val + 8) << 4 | (even_val + 8) + """ + min_val, max_val = weight_tensor.min().item(), weight_tensor.max().item() + assert ( + max_val <= 7 and min_val >= -8 + ), f"pack_4bit_weight_tensor: [min_val,max_val] out of [-8, 7] range, got [{min_val}, {max_val}]" + + if weight_tensor.ndim != 2: + weight_tensor = weight_tensor.squeeze() + assert ( + weight_tensor.ndim == 2 + ), f"pack_4bit_weight_tensor: expecting input tensor to be 2d, got {weight_tensor.ndim}" + + if ( + inner_dim_padding is not None + and weight_tensor.shape[-1] % inner_dim_padding != 0 + ): + num_pad = inner_dim_padding - (weight_tensor.shape[-1] % inner_dim_padding) + weight_tensor = F.pad(input=weight_tensor, pad=(0, num_pad)) + + assert ( + weight_tensor.shape[-1] % 2 == 0 + ), "pack_4bit_weight_tensor: expecting innermost dim to be divisible by 2" + + shifted_weight_tensor = weight_tensor.to(dtype=torch.uint8) + 8 + even_values = shifted_weight_tensor[:, ::2] + odd_values = shifted_weight_tensor[:, 1::2] + return odd_values << 4 | even_values diff --git a/backends/vulkan/test/test_vulkan_passes.py b/backends/vulkan/test/test_vulkan_passes.py index fa448102b8e..f030b9268a1 100644 --- a/backends/vulkan/test/test_vulkan_passes.py +++ b/backends/vulkan/test/test_vulkan_passes.py @@ -87,6 +87,285 @@ def op_node_count(graph_module: torch.fx.GraphModule, canonical_op_name: str) -> class TestVulkanPasses(unittest.TestCase): + def test_fuse_torchao_quantized_embedding(self): + """A torchao-dialect 4-bit weight-only quantized embedding + (torchao.dequantize_affine -> aten.embedding) should fuse into a single + et_vk.embedding_q4gsw.default node, with the dequant_affine and embedding + nodes removed. + """ + import executorch.backends.vulkan.custom_ops_lib # noqa: registers et_vk ops + from torchao.quantization.granularity import PerGroup + from torchao.quantization.quant_api import IntxWeightOnlyConfig, quantize_ + from torchao.utils import unwrap_tensor_subclass + + vocab_size = 64 + embed_dim = 128 + group_size = 32 + + class EmbModule(torch.nn.Module): + def __init__(self): + super().__init__() + self.emb = torch.nn.Embedding(vocab_size, embed_dim) + + def forward(self, x): + return self.emb(x) + + model = EmbModule() + quantize_( + model, + IntxWeightOnlyConfig( + weight_dtype=torch.int4, granularity=PerGroup(group_size) + ), + filter_fn=lambda mod, fqn: isinstance(mod, torch.nn.Embedding), + ) + unwrap_tensor_subclass(model) + + sample_inputs = (torch.tensor([0, 1, 2, 3, 4], dtype=torch.int64),) + # Eager reference output of the quantized embedding, before any fusion. + eager_ref = model(*sample_inputs) + + program = torch.export.export(model, sample_inputs, strict=True) + edge_program = to_edge( + program, + compile_config=EdgeCompileConfig(_check_ir_validity=False), + ) + + ep = edge_program._edge_programs["forward"] + fuse_pass = FusePatternsPass() + fuse_pass._exported_program = ep + result = fuse_pass.call(ep.graph_module) + + self.assertTrue(result.modified) + + gm = ep.graph_module + + self.assertEqual(op_node_count(gm, "embedding_q4gsw.default"), 1) + self.assertEqual(op_node_count(gm, "dequantize_affine.default"), 0) + self.assertEqual(op_node_count(gm, "embedding.default"), 0) + + # Verify the fused op carries the expected args: + # (weight, scales, group_size, indices, is_linear_weight) + fused_node = next( + n + for n in gm.graph.nodes + if get_target_canonical_name(n) == "embedding_q4gsw.default" + ) + self.assertEqual(fused_node.args[2], group_size) + # The weight is always packed in the LINEAR-weight q4gsw layout so a tied + # embedding/LM-head weight is supported, so is_linear_weight is True. + self.assertTrue(fused_node.args[4]) + + # The weight placeholder is repacked from unpacked int8 [vocab, embed_dim] + # to linear-convention 4-bit packed uint8. embed_dim % 32 == 0 means + # embed_dim / 2 is a multiple of 16, so the linear packing's mult-of-8 + # inner-dim padding is inert and the packed inner dim stays embed_dim / 2. + weight_node = fused_node.args[0] + self.assertEqual(weight_node.meta["val"].dtype, torch.uint8) + self.assertEqual( + tuple(weight_node.meta["val"].shape), (vocab_size, embed_dim // 2) + ) + + # Numerically verify the fused op (via its CompositeExplicitAutograd + # reference impl) reproduces the eager quantized embedding output. This + # exercises the repacked weight + scale layout end-to-end against an + # independently-computed reference. + from executorch.backends.transforms.utils import get_param_tensor + + packed_weight = get_param_tensor(ep, weight_node) + scales_tensor = get_param_tensor(ep, fused_node.args[1]) + fused_out = torch.ops.et_vk.embedding_q4gsw.default( + packed_weight, + scales_tensor, + group_size, + sample_inputs[0], + True, + ) + self.assertTrue(torch.allclose(fused_out, eager_ref, atol=1e-3, rtol=1e-3)) + + def test_torchao_quantized_embedding_rejects_bad_embed_dim(self): + """A torchao 4-bit quantized embedding whose embed_dim is not a multiple + of 32 must NOT fuse: the runtime shader asserts embed_dim % 32 == 0 + (VK_CHECK in EmbeddingQ4gsw.cpp), so the matcher's input-validation guard + rejects it and the op falls back to CPU rather than producing an op the + runtime would abort on. embed_dim=48 is divisible by group_size=16 (so the + group-size, zero_point, and qmin/qmax guards all pass) but 48 % 32 != 0. + """ + import executorch.backends.vulkan.custom_ops_lib # noqa: registers et_vk ops + from torchao.quantization.granularity import PerGroup + from torchao.quantization.quant_api import IntxWeightOnlyConfig, quantize_ + from torchao.utils import unwrap_tensor_subclass + + vocab_size = 64 + embed_dim = 48 # not a multiple of 32 + group_size = 16 + + class EmbModule(torch.nn.Module): + def __init__(self): + super().__init__() + self.emb = torch.nn.Embedding(vocab_size, embed_dim) + + def forward(self, x): + return self.emb(x) + + model = EmbModule() + quantize_( + model, + IntxWeightOnlyConfig( + weight_dtype=torch.int4, granularity=PerGroup(group_size) + ), + filter_fn=lambda mod, fqn: isinstance(mod, torch.nn.Embedding), + ) + unwrap_tensor_subclass(model) + + sample_inputs = (torch.tensor([0, 1, 2, 3, 4], dtype=torch.int64),) + + program = torch.export.export(model, sample_inputs, strict=True) + edge_program = to_edge( + program, + compile_config=EdgeCompileConfig(_check_ir_validity=False), + ) + + ep = edge_program._edge_programs["forward"] + fuse_pass = FusePatternsPass() + fuse_pass._exported_program = ep + fuse_pass.call(ep.graph_module) + + gm = ep.graph_module + + # The guard rejected the match: no fused op, and the original + # aten.embedding lookup remains for the CPU fallback path. + self.assertEqual(op_node_count(gm, "embedding_q4gsw.default"), 0) + self.assertEqual(op_node_count(gm, "embedding.default"), 1) + + def test_fuse_torchao_quantized_embedding_shared_weight(self): + """A single torchao-quantized embedding weight shared by multiple + aten.embedding call sites (two dequantize_affine -> embedding chains over + the same weight placeholder) must fuse into two et_vk.embedding_q4gsw + nodes that reference the SAME repacked weight, and the weight must only be + repacked once (regression test: repacking the shared state-dict entry + twice would corrupt it, halving its width on the second pass). + """ + import executorch.backends.vulkan.custom_ops_lib # noqa: registers et_vk ops + from executorch.backends.transforms.utils import get_param_tensor + from torchao.quantization.granularity import PerGroup + from torchao.quantization.quant_api import IntxWeightOnlyConfig, quantize_ + from torchao.utils import unwrap_tensor_subclass + + vocab_size = 64 + embed_dim = 128 + group_size = 32 + + class TwoLookupEmbModule(torch.nn.Module): + def __init__(self): + super().__init__() + self.emb = torch.nn.Embedding(vocab_size, embed_dim) + + def forward(self, x, y): + # Two lookups into the same embedding table. + return self.emb(x) + self.emb(y) + + model = TwoLookupEmbModule() + quantize_( + model, + IntxWeightOnlyConfig( + weight_dtype=torch.int4, granularity=PerGroup(group_size) + ), + filter_fn=lambda mod, fqn: isinstance(mod, torch.nn.Embedding), + ) + unwrap_tensor_subclass(model) + + sample_inputs = ( + torch.tensor([0, 1, 2, 3, 4], dtype=torch.int64), + torch.tensor([5, 6, 7, 8, 9], dtype=torch.int64), + ) + eager_ref = model(*sample_inputs) + + program = torch.export.export(model, sample_inputs, strict=True) + edge_program = to_edge( + program, + compile_config=EdgeCompileConfig(_check_ir_validity=False), + ) + + ep = edge_program._edge_programs["forward"] + fuse_pass = FusePatternsPass() + fuse_pass._exported_program = ep + result = fuse_pass.call(ep.graph_module) + + self.assertTrue(result.modified) + + gm = ep.graph_module + + # Both embedding call sites should fuse; neither dequant_affine nor + # embedding nodes should remain. + self.assertEqual(op_node_count(gm, "embedding_q4gsw.default"), 2) + self.assertEqual(op_node_count(gm, "dequantize_affine.default"), 0) + self.assertEqual(op_node_count(gm, "embedding.default"), 0) + + fused_nodes = [ + n + for n in gm.graph.nodes + if get_target_canonical_name(n) == "embedding_q4gsw.default" + ] + # Both fused nodes must reference the same (single) repacked weight. + self.assertEqual(fused_nodes[0].args[0], fused_nodes[1].args[0]) + + # Both fused nodes use the linear-weight q4gsw layout (is_linear_weight). + self.assertTrue(fused_nodes[0].args[4]) + self.assertTrue(fused_nodes[1].args[4]) + + # The shared weight must be repacked exactly once: linear-convention + # 4-bit packed uint8. Since embed_dim % 32 == 0, embed_dim / 2 is a + # multiple of 16 so the linear packing's inner-dim padding is inert and + # the packed inner dim is embed_dim / 2. A double-pack would yield + # [vocab, embed_dim / 4]. + weight_node = fused_nodes[0].args[0] + packed_weight = get_param_tensor(ep, weight_node) + self.assertEqual(packed_weight.dtype, torch.uint8) + self.assertEqual(tuple(packed_weight.shape), (vocab_size, embed_dim // 2)) + + # End-to-end numerical check against the eager reference. + scales_tensor = get_param_tensor(ep, fused_nodes[0].args[1]) + emb_x = torch.ops.et_vk.embedding_q4gsw.default( + packed_weight, scales_tensor, group_size, sample_inputs[0], True + ) + emb_y = torch.ops.et_vk.embedding_q4gsw.default( + packed_weight, scales_tensor, group_size, sample_inputs[1], True + ) + self.assertTrue(torch.allclose(emb_x + emb_y, eager_ref, atol=1e-3, rtol=1e-3)) + + def test_register_param_mutation(self): + """utils.register_param_mutation is a storage-keyed idempotency guard: + the first call for a param returns True (proceed and record the tag), a + repeat with the same tag returns False (skip), and a call with a + conflicting tag raises. + """ + import executorch.backends.vulkan.utils as vk_utils + + model = SingleLinearModule() + program = torch.export.export(model, model.get_sample_inputs(), strict=True) + edge_program = to_edge( + program, compile_config=EdgeCompileConfig(_check_ir_validity=False) + ) + ep = edge_program._edge_programs["forward"] + gm = ep.graph_module + + # Grab the linear weight parameter placeholder. The fused linear node + # consumes it as a constant tensor arg. + weight_node = next( + n + for n in gm.graph.nodes + if n.op == "placeholder" and vk_utils.is_param(ep, n) + ) + + # First call for this param: records the tag, returns True (proceed). + self.assertTrue(vk_utils.register_param_mutation(ep, weight_node, "fmt_a")) + # Repeat with the same tag: already mutated this way, returns False (skip). + self.assertFalse(vk_utils.register_param_mutation(ep, weight_node, "fmt_a")) + self.assertFalse(vk_utils.register_param_mutation(ep, weight_node, "fmt_a")) + # Conflicting tag on the same param: an incompatible re-mutation, raises. + with self.assertRaises(RuntimeError): + vk_utils.register_param_mutation(ep, weight_node, "fmt_b") + def test_fuse_rotary_emb(self): """Test conversion of rotary embedding pattern to et_vk.apply_rotary_emb custom op.""" diff --git a/backends/vulkan/utils.py b/backends/vulkan/utils.py index b349fb51001..84b901b6b6e 100644 --- a/backends/vulkan/utils.py +++ b/backends/vulkan/utils.py @@ -546,6 +546,80 @@ def get_tensor_name(exp_prog: ExportedProgram, node: torch.fx.Node) -> str: return "" +def register_param_mutation(ep: ExportedProgram, node: torch.fx.Node, tag: str) -> bool: + """Register an in-place mutation of a param/buffer's state-dict storage, + keyed on the stable identity of that storage, and report whether the caller + should perform the mutation. + + A graph pass that repacks (or otherwise mutates in place) the constant tensor + backing a placeholder must run the mutation exactly once per backing storage; + repeating it on the already-mutated tensor corrupts it. Keying the guard on + node identity is unsafe, because two distinct placeholder nodes can resolve to + the same state-dict entry (e.g. aliased weights), and each would miss a + per-node flag and re-run the mutation. This guard instead keys on the + state-dict FQN that backs `node` -- the same identity the in-place mutation + targets (see `update_program_state_dict`, which overwrites + `state_dict[input_spec.target]`). The FQN is resolved via `get_tensor_name` + (`graph_signature.inputs_to_{parameters,buffers,lifted_tensor_constants}`) and + is stable across the mutation, unlike `data_ptr()` which changes once the + state-dict tensor object is replaced. + + `tag` identifies the mutation/packing FORMAT, not the op. Two callers that + produce the same packed bytes for a weight should pass the same `tag` and + share this guard; observing two different tags on one weight means it would be + mutated two incompatible ways, which is corruption, so that case raises. + + A per-ExportedProgram registry mapping `param_key -> tag` is lazily + initialized on `ep` (`ep._et_vk_param_modification_tags`) without + module-scope mutable state. Because it is stored on `ep`, it persists for the + lifetime of that ExportedProgram -- across all pass runs on it, not just one + -- and there is no reset hook, so a pass that legitimately needs to re-mutate + an already-tagged param must clear `ep._et_vk_param_modification_tags` + itself. + + Returns True on the first call for a given param (and records the tag), + signalling the caller to proceed with the mutation. Returns False on a + subsequent call with the SAME tag, signalling the caller to skip (already + mutated this way). Raises RuntimeError on a call whose tag differs from the + recorded one. + """ + registry: Dict[str, str] = getattr(ep, "_et_vk_param_modification_tags", None) + if registry is None: + registry = {} + ep._et_vk_param_modification_tags = registry + + # The guard keys on the state-dict FQN that get_tensor_name resolves, which + # only exists for parameter / buffer / lifted-constant placeholders (via + # inputs_to_{parameters,buffers,lifted_tensor_constants}). For any other node + # kind get_tensor_name falls through to node.target, which is not a stable + # storage key, so the guard would silently key on the wrong identity. Reject + # such nodes up front. + if not ( + is_param(ep, node) or is_buffer(ep, node) or is_lifted_tensor_constant(ep, node) + ): + raise RuntimeError( + "register_param_mutation expects a parameter / buffer / " + f"lifted-constant placeholder, but got node {node.name!r} " + f"(op={node.op!r}, target={node.target!r})." + ) + + param_key = get_tensor_name(ep, node) + + recorded_tag = registry.get(param_key) + if recorded_tag is None: + registry[param_key] = tag + return True + + if recorded_tag != tag: + raise RuntimeError( + f"Param tensor {param_key!r} was already modified with tag " + f"{recorded_tag!r}, but a conflicting modification with tag {tag!r} " + "was attempted; a weight modified two different ways is corrupt." + ) + + return False + + def find_dequant_user(node: torch.fx.Node) -> Optional[torch.fx.Node]: """ Search the direct users of the given node and return the first one that is a