diff --git a/src/indobenchmark/tokenization_indonlg.py b/src/indobenchmark/tokenization_indonlg.py index f64447f..65601c8 100644 --- a/src/indobenchmark/tokenization_indonlg.py +++ b/src/indobenchmark/tokenization_indonlg.py @@ -15,37 +15,19 @@ """ Tokenization classes for IndoNLG model.""" import os -from shutil import copyfile from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Sequence, Tuple, Union from transformers import PreTrainedTokenizer, BatchEncoding from collections.abc import Mapping from transformers.utils import ( - EntryNotFoundError, - ExplicitEnum, PaddingStrategy, - PushToHubMixin, - RepositoryNotFoundError, - RevisionNotFoundError, TensorType, - add_end_docstrings, - cached_path, - copy_func, - get_file_from_repo, - hf_bucket_url, - is_flax_available, - is_offline_mode, - is_remote_url, is_tf_available, - is_tokenizers_available, is_torch_available, logging, to_py_obj, - torch_required, ) import sentencepiece as spm - -from transformers.utils import logging from transformers.utils.generic import _is_jax, _is_numpy, _is_tensorflow, _is_torch, _is_torch_device logger = logging.get_logger(__name__)