Skip to content

An error when running fine-tuning-embeddinggemma-with-sentence-transformers.ipynb on kaggle #623

Description

@linsong8208

Description of the bug:

Everything goes on well until in the "training" part, instead of training starts, the error is like:
TypeError Traceback (most recent call last)
/tmp/ipykernel_55/2596833136.py in <cell line: 0>()
39 callbacks=[MyCallback(evaluate)]
40 )
---> 41 trainer.train()

/usr/local/lib/python3.12/dist-packages/transformers/trainer.py in train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
2326 hf_hub_utils.enable_progress_bars()
2327 else:
-> 2328 return inner_training_loop(
2329 args=args,
2330 resume_from_checkpoint=resume_from_checkpoint,

/usr/local/lib/python3.12/dist-packages/transformers/trainer.py in _inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
2670 )
2671 with context():
-> 2672 tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
2673
2674 if (

/usr/local/lib/python3.12/dist-packages/transformers/trainer.py in training_step(self, model, inputs, num_items_in_batch)
4018
4019 with self.compute_loss_context_manager():
-> 4020 loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
4021
4022 del inputs

/usr/local/lib/python3.12/dist-packages/sentence_transformers/base/trainer.py in compute_loss(self, model, inputs, return_outputs, num_items_in_batch)
471 ):
472 loss_fn = self.override_model_in_loss(loss_fn, model)
--> 473 loss = loss_fn(features, labels)
474 if isinstance(loss, dict):
475 self.track_loss_components(loss)

/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)
1774 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
1775 else:
-> 1776 return self._call_impl(*args, **kwargs)
1777
1778 # torchrec tests the code consistency with the following code

/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)
1785 or _global_backward_pre_hooks or _global_backward_hooks
1786 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1787 return forward_call(*args, **kwargs)
1788
1789 result = None

/usr/local/lib/python3.12/dist-packages/sentence_transformers/sentence_transformer/losses/multiple_negatives_ranking.py in forward(self, sentence_features, labels)
230 def forward(self, sentence_features: Iterable[dict[str, Tensor]], labels: Tensor) -> Tensor:
231 # Compute the embeddings and distribute them to anchor and candidates (positive and optionally negatives)
--> 232 embeddings = [self.model(sentence_feature)["sentence_embedding"] for sentence_feature in sentence_features]
233 return self.compute_loss_from_embeddings(embeddings, labels)
234

/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)
1774 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
1775 else:
-> 1776 return self._call_impl(*args, **kwargs)
1777
1778 # torchrec tests the code consistency with the following code

/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)
1785 or _global_backward_pre_hooks or _global_backward_hooks
1786 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1787 return forward_call(*args, **kwargs)
1788
1789 result = None

/usr/local/lib/python3.12/dist-packages/torch/nn/parallel/data_parallel.py in forward(self, *inputs, **kwargs)
196 replicas = self.replicate(self.module, self.device_ids[: len(inputs)])
197 outputs = self.parallel_apply(replicas, inputs, module_kwargs)
--> 198 return self.gather(outputs, self.output_device)
199
200 def replicate(self, module: T, device_ids: Sequence[int | torch.device]) -> list[T]:

/usr/local/lib/python3.12/dist-packages/torch/nn/parallel/data_parallel.py in gather(self, outputs, output_device)
217
218 def gather(self, outputs: Any, output_device: int | torch.device) -> Any:
--> 219 return gather(outputs, output_device, dim=self.dim)
220
221

/usr/local/lib/python3.12/dist-packages/torch/nn/parallel/scatter_gather.py in gather(outputs, target_device, dim)
149 # Setting the function to None clears the refcycle.
150 try:
--> 151 res = gather_map(outputs)
152 finally:
153 gather_map = None # type: ignore[assignment]

/usr/local/lib/python3.12/dist-packages/torch/nn/parallel/scatter_gather.py in gather_map(outputs)
139 raise ValueError("All dicts must have the same number of keys")
140 # pyrefly: ignore [not-callable]
--> 141 return type(out)((k, gather_map([d[k] for d in outputs])) for k in out)
142 if _is_namedtuple(out):
143 # pyrefly: ignore [no-matching-overload]

/usr/local/lib/python3.12/dist-packages/torch/nn/parallel/scatter_gather.py in (.0)
139 raise ValueError("All dicts must have the same number of keys")
140 # pyrefly: ignore [not-callable]
--> 141 return type(out)((k, gather_map([d[k] for d in outputs])) for k in out)
142 if _is_namedtuple(out):
143 # pyrefly: ignore [no-matching-overload]

/usr/local/lib/python3.12/dist-packages/torch/nn/parallel/scatter_gather.py in gather_map(outputs)
144 return type(out)._make(map(gather_map, zip(*outputs, strict=True)))
145 # pyrefly: ignore [no-matching-overload]
--> 146 return type(out)(map(gather_map, zip(*outputs, strict=True)))
147
148 # Recursive function calls like this create reference cycles.

TypeError: 'int' object is not iterable

Actual vs expected behavior:

No response

Any other information you'd like to share?

No response

Activity

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    type:bugSomething isn't working

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions