diff --git a/backends/qualcomm/debugger/utils.py b/backends/qualcomm/debugger/utils.py index 58f8f91f587..01cb267e917 100644 --- a/backends/qualcomm/debugger/utils.py +++ b/backends/qualcomm/debugger/utils.py @@ -4,7 +4,6 @@ import shutil import subprocess import tempfile -from pathlib import Path from typing import Sequence, Tuple import executorch.backends.qualcomm.python.PyQnnManagerAdaptor as PyQnnManager @@ -211,7 +210,7 @@ def __init__( self.adb = adb self.sample_input = sample_input self.build_folder = build_folder - self.root = str(Path(__file__).resolve().parents[3]) + self.root = os.getcwd() self.config = { "backend_extension_config": { "backend_extensions": { diff --git a/backends/qualcomm/tests/models.py b/backends/qualcomm/tests/models.py index 6dbf1183659..6decea516c0 100644 --- a/backends/qualcomm/tests/models.py +++ b/backends/qualcomm/tests/models.py @@ -1532,7 +1532,7 @@ def forward(self, x): class LargeTensorLinear(torch.nn.Module): def __init__(self): super().__init__() - hidden_dim = 8192 + hidden_dim = 16384 self.linear1_1 = torch.nn.Linear(512, hidden_dim) self.linear1_2 = torch.nn.Linear(512, hidden_dim) self.linear1_3 = torch.nn.Linear(512, hidden_dim) diff --git a/backends/qualcomm/tests/test_qnn_delegate.py b/backends/qualcomm/tests/test_qnn_delegate.py index 98c5839c515..7fb67fe334d 100644 --- a/backends/qualcomm/tests/test_qnn_delegate.py +++ b/backends/qualcomm/tests/test_qnn_delegate.py @@ -10470,7 +10470,7 @@ def test_cli(self): cmds = [ "python", "-m", - "examples.qualcomm.util_scripts.cli", + "executorch.examples.qualcomm.util_scripts.cli", "quantize", "--artifact", f"{tmp_dir}/relu.pt2", @@ -10489,7 +10489,7 @@ def test_cli(self): cmds = [ "python", "-m", - "examples.qualcomm.util_scripts.cli", + "executorch.examples.qualcomm.util_scripts.cli", "compile", "--artifact", f"{tmp_dir}/q_out/relu_quantized.pt2", @@ -10507,7 +10507,7 @@ def test_cli(self): cmds = [ "python", "-m", - "examples.qualcomm.util_scripts.cli", + "executorch.examples.qualcomm.util_scripts.cli", "execute", "--artifact", f"{tmp_dir}/c_out/relu_quantized.pte", @@ -10551,7 +10551,7 @@ def test_cli_with_input_list_assignment(self): cmds = [ "python", "-m", - "examples.qualcomm.util_scripts.cli", + "executorch.examples.qualcomm.util_scripts.cli", "quantize", "--artifact", f"{tmp_dir}/sub.pt2", @@ -10570,7 +10570,7 @@ def test_cli_with_input_list_assignment(self): cmds = [ "python", "-m", - "examples.qualcomm.util_scripts.cli", + "executorch.examples.qualcomm.util_scripts.cli", "compile", "--artifact", f"{tmp_dir}/q_out/sub_quantized.pt2", @@ -10588,7 +10588,7 @@ def test_cli_with_input_list_assignment(self): cmds = [ "python", "-m", - "examples.qualcomm.util_scripts.cli", + "executorch.examples.qualcomm.util_scripts.cli", "execute", "--artifact", f"{tmp_dir}/c_out/sub_quantized.pte", diff --git a/examples/qualcomm/oss_scripts/llama/__init__.py b/examples/qualcomm/oss_scripts/llama/__init__.py index 2e0b2278909..14954cef3b9 100644 --- a/examples/qualcomm/oss_scripts/llama/__init__.py +++ b/examples/qualcomm/oss_scripts/llama/__init__.py @@ -281,7 +281,7 @@ class Llama3_2_3B_Instruct(LLMModelConfig): transform_weight = True # The Llama3_2 enabled should be instruct, however, Llama's tokenizer does not provide utility to apply chat template. instruct_model = False - num_sharding = 4 + num_sharding = 3 masked_softmax = False seq_mse_candidates = 0 r1 = False diff --git a/examples/qualcomm/oss_scripts/llama/model/static_llama.py b/examples/qualcomm/oss_scripts/llama/model/static_llama.py index d57ada7fef6..262b78749b8 100755 --- a/examples/qualcomm/oss_scripts/llama/model/static_llama.py +++ b/examples/qualcomm/oss_scripts/llama/model/static_llama.py @@ -730,7 +730,7 @@ def forward( self.freqs_sin[input_pos][0] if self.use_kv_cache else self.freqs_sin ) - hidden_states = self.embedding_scale_factor * self.tok_embeddings(tokens) + hidden_states = self.tok_embeddings(tokens) for ind, decoder_layer in enumerate(self.layers): k_caches = None @@ -865,7 +865,11 @@ def forward( self.freqs_sin[input_pos][0] if self.use_kv_cache else self.freqs_sin ) - hidden_states = self.embedding_scale_factor * hidden_states + hidden_states = ( + self.embedding_scale_factor * hidden_states + if self.embedding_scale_factor != 1.0 + else hidden_states + ) for ind, decoder_layer in enumerate(self.layers): k_caches = None @@ -1024,7 +1028,8 @@ def forward( else self.local_freqs_sin ) - hidden_states = self.embedding_scale_factor * self.tok_embeddings(tokens) + hidden_states = self.tok_embeddings(tokens) + for ind, decoder_layer in enumerate(self.layers): k_caches = None v_caches = None diff --git a/examples/qualcomm/oss_scripts/llama/wrappers/llm_wrappers.py b/examples/qualcomm/oss_scripts/llama/wrappers/llm_wrappers.py index 9a1ad50f854..46615f446d0 100644 --- a/examples/qualcomm/oss_scripts/llama/wrappers/llm_wrappers.py +++ b/examples/qualcomm/oss_scripts/llama/wrappers/llm_wrappers.py @@ -241,6 +241,9 @@ def _prepare_model(self): # noqa: C901 # Llama does x.to(float16) * w whilst Gemma3 is (x * w).to(float16) # See https://github.com/huggingface/transformers/pull/29402 state_dict[k] = v.float() + torch.ones(v.shape, dtype=torch.float32) + for k, v in state_dict.items(): + if "tok_embeddings.weight" == k: + state_dict[k] = v.float() * self.model_args.embedding_scale_factor else: state_dict = torch.load( self.control_args.checkpoint, @@ -466,9 +469,7 @@ def _tag_ios(self, node, fixed_point_type): (self.meta["get_ar_len"], self.meta["get_head_dim"] // 2), } - freq_op = { - exir_ops.edge.aten.select.int, - } + freq_op = {exir_ops.edge.aten.select.int, exir_ops.edge.aten.select_copy.int} quant_io_type = None if node.op == "placeholder": diff --git a/examples/qualcomm/util_scripts/cli.py b/examples/qualcomm/util_scripts/cli.py index 95a7f0f73a3..76f2b24ca8d 100644 --- a/examples/qualcomm/util_scripts/cli.py +++ b/examples/qualcomm/util_scripts/cli.py @@ -166,6 +166,8 @@ def quantize(args): quant_dtype=quant_dtype, per_channel_conv=args.per_channel, per_channel_linear=args.per_row, + per_channel_embedding=args.per_channel_embedding, + act_symmetric=args.act_symmetric, act_observer=act_observer, backend=get_backend_type(args.backend), soc_model=args.soc_model, @@ -499,6 +501,16 @@ def main(): action="store_true", help="Use per_row encoding for operator linear.", ) + sub_quantize.add_argument( + "--per_channel_embedding", + action="store_true", + help="Use per_channel encoding for operator embedding.", + ) + sub_quantize.add_argument( + "--act_symmetric", + action="store_true", + help="Use symmetric quantization for activations.", + ) sub_quantize.add_argument( "--activation_observer", type=str,