From 956cc3e90579472e97e20122c000e9d2987f693a Mon Sep 17 00:00:00 2001 From: Sharon Hart Date: Mon, 27 Jul 2026 12:09:11 +0300 Subject: [PATCH] Replace stale Hugging Face spaCy adapter Remove spacy-huggingface-pipelines so the analyzer can use secure Transformers 5 releases. Add an internal scored-span component with spaCy plugin discovery, preserve failure behavior, and cover it with focused regression tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e86159d-80a9-4926-9378-c5dabe07a39f --- docs/analyzer/nlp_engines/transformers.md | 4 +- .../nlp_engine/huggingface_token_pipe.py | 220 ++++++++++++ .../nlp_engine/transformers_nlp_engine.py | 22 +- .../transformers_recognizer.py | 9 +- presidio-analyzer/pyproject.toml | 10 +- .../tests/test_huggingface_token_pipe.py | 312 ++++++++++++++++++ .../tests/test_transformers_nlp_engine.py | 39 ++- presidio-analyzer/uv.lock | 130 ++++---- 8 files changed, 648 insertions(+), 98 deletions(-) create mode 100644 presidio-analyzer/presidio_analyzer/nlp_engine/huggingface_token_pipe.py create mode 100644 presidio-analyzer/tests/test_huggingface_token_pipe.py diff --git a/docs/analyzer/nlp_engines/transformers.md b/docs/analyzer/nlp_engines/transformers.md index fa98e5b756..65c9ce04bd 100644 --- a/docs/analyzer/nlp_engines/transformers.md +++ b/docs/analyzer/nlp_engines/transformers.md @@ -215,7 +215,9 @@ The `ner_model_configuration` section contains the following parameters: For example, for `bert-base-NER-uncased`, it can be found here: . Note that most NER models add a prefix to the class (e.g. `B-PER` for class `PER`). When creating the mapping, do not add the prefix. -See more information on parameters on the [spacy-huggingface-pipelines Github repo](https://github.com/explosion/spacy-huggingface-pipelines#token-classification). +See the Hugging Face +[token-classification pipeline documentation](https://huggingface.co/docs/transformers/main_classes/pipelines#transformers.TokenClassificationPipeline) +for more information on aggregation and stride behavior. Once created, see [the NLP configuration documentation](../customizing_nlp_models.md#Configure-Presidio-to-use-the-new-model) for more information. diff --git a/presidio-analyzer/presidio_analyzer/nlp_engine/huggingface_token_pipe.py b/presidio-analyzer/presidio_analyzer/nlp_engine/huggingface_token_pipe.py new file mode 100644 index 0000000000..5574c9678e --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/nlp_engine/huggingface_token_pipe.py @@ -0,0 +1,220 @@ +"""spaCy component for Hugging Face token-classification pipelines.""" + +import logging +from collections.abc import Iterable, Iterator, Mapping, Sequence +from pathlib import Path +from typing import Any, Optional + +from spacy import util +from spacy.language import Language +from spacy.pipeline import Pipe +from spacy.tokens import Doc, SpanGroup +from thinc.api import get_torch_default_device + +try: + from transformers import pipeline as hf_pipeline +except ImportError: + hf_pipeline = None + +logger = logging.getLogger("presidio-analyzer") + +FACTORY_NAME = "presidio_hf_token_pipe" + + +def is_available() -> bool: + """Return whether the optional Transformers dependency is installed.""" + return hf_pipeline is not None + + +def _pipeline_device() -> int: + """Return a Transformers-compatible CPU or CUDA device index.""" + device = get_torch_default_device() + if device.type != "cuda": + return -1 + return device.index if device.index is not None else 0 + + +@Language.factory( + FACTORY_NAME, + assigns=[], + default_config={ + "model": "", + "revision": "main", + "stride": 14, + "aggregation_strategy": "max", + "alignment_mode": "expand", + "spans_key": "bert-base-ner", + "pipeline_kwargs": {}, + }, + default_score_weights={}, +) +def create_huggingface_token_pipe( + nlp: Language, + name: str, + model: str, + revision: str, + stride: Optional[int], + aggregation_strategy: str, + alignment_mode: str, + spans_key: str, + pipeline_kwargs: dict[str, Any], +) -> "HuggingFaceTokenPipe": + """Create a spaCy component backed by a Hugging Face pipeline.""" + if hf_pipeline is None: + raise ImportError( + "transformers is not installed. Install presidio-analyzer[transformers] " + "to use TransformersNlpEngine." + ) + if not model: + raise ValueError("A Hugging Face token-classification model is required") + + kwargs = { + "task": "token-classification", + "model": model, + "revision": revision, + "aggregation_strategy": aggregation_strategy, + "device": _pipeline_device(), + } + if stride is not None: + kwargs["stride"] = stride + + pipeline = hf_pipeline(**kwargs, **pipeline_kwargs) + return HuggingFaceTokenPipe( + name=name, + pipeline=pipeline, + alignment_mode=alignment_mode, + spans_key=spans_key, + ) + + +class HuggingFaceTokenPipe(Pipe): + """Store token-classification predictions as scored spaCy spans.""" + + def __init__( + self, + name: str, + pipeline: Any, + alignment_mode: str, + spans_key: str, + ) -> None: + self.name = name + self.pipeline = pipeline + self.alignment_mode = alignment_mode + self.spans_key = spans_key + + def __call__(self, doc: Doc) -> Doc: + """Annotate one document.""" + return self._add_predictions(doc, self._predict_one(doc)) + + def pipe(self, stream: Iterable[Doc], *, batch_size: int = 128) -> Iterator[Doc]: + """Annotate documents in batches, retrying batch failures individually.""" + for batch in util.minibatch(stream, size=batch_size): + docs = list(batch) + predictions = self._predict_batch(docs) + for doc, doc_predictions in zip(docs, predictions, strict=True): + yield self._add_predictions(doc, doc_predictions) + + def _predict_batch(self, docs: Sequence[Doc]) -> list[Sequence[Mapping[str, Any]]]: + texts = [doc.text for doc in docs] + try: + output = self.pipeline(texts) + except Exception as error: + logger.warning( + "Hugging Face batch inference failed with %s; retrying each document", + type(error).__name__, + ) + return [self._predict_one(doc) for doc in docs] + + if not isinstance(output, list) or len(output) != len(docs): + raise ValueError( + "Hugging Face batch output count does not match the document count" + ) + return [self._validate_predictions(item) for item in output] + + def _predict_one(self, doc: Doc) -> Sequence[Mapping[str, Any]]: + try: + output = self.pipeline(doc.text) + except Exception as error: + logger.warning( + "Hugging Face inference failed with %s; skipping document", + type(error).__name__, + ) + return [] + return self._validate_predictions(output) + + @staticmethod + def _validate_predictions(output: Any) -> Sequence[Mapping[str, Any]]: + if not isinstance(output, list) or any( + not isinstance(item, Mapping) for item in output + ): + raise TypeError( + "Hugging Face token-classification output must be a list of mappings" + ) + return output + + def _add_predictions( + self, doc: Doc, predictions: Sequence[Mapping[str, Any]] + ) -> Doc: + spans = SpanGroup(doc, attrs={"scores": []}) + previous_end = 0 + + for prediction in predictions: + label = prediction.get("entity_group") or prediction.get("entity") + start = prediction.get("start") + end = prediction.get("end") + if ( + not isinstance(label, str) + or not isinstance(start, int) + or not isinstance(end, int) + ): + raise ValueError( + "Hugging Face prediction is missing a string label or integer " + f"offsets; keys={sorted(prediction.keys())}" + ) + try: + score = float(prediction["score"]) + except (KeyError, TypeError, ValueError): + raise ValueError( + "Hugging Face prediction is missing a numeric score" + ) from None + + span = None + if start >= previous_end: + span = doc.char_span( + start, + end, + label=label, + alignment_mode=self.alignment_mode, + ) + if span is None or span.start_char < previous_end: + logger.warning( + "Skipping unaligned or overlapping prediction " + "label=%s start=%d end=%d", + label, + start, + end, + ) + continue + + spans.append(span) + spans.attrs["scores"].append(score) + previous_end = end + + doc.spans[self.spans_key] = spans + return doc + + def to_bytes(self, **kwargs: Any) -> bytes: + """Return no model data because the pipeline is rebuilt from config.""" + return b"" + + def from_bytes(self, bytes_data: bytes, **kwargs: Any) -> "HuggingFaceTokenPipe": + """Keep the pipeline created from the current configuration.""" + return self + + def to_disk(self, path: Path, **kwargs: Any) -> None: + """Persist no model data because the pipeline is rebuilt from config.""" + return None + + def from_disk(self, path: Path, **kwargs: Any) -> "HuggingFaceTokenPipe": + """Keep the pipeline created from the current configuration.""" + return self diff --git a/presidio-analyzer/presidio_analyzer/nlp_engine/transformers_nlp_engine.py b/presidio-analyzer/presidio_analyzer/nlp_engine/transformers_nlp_engine.py index f3a12198bf..608da2b98a 100644 --- a/presidio-analyzer/presidio_analyzer/nlp_engine/transformers_nlp_engine.py +++ b/presidio-analyzer/presidio_analyzer/nlp_engine/transformers_nlp_engine.py @@ -4,17 +4,14 @@ import spacy from spacy.tokens import Doc, Span -try: - import spacy_huggingface_pipelines - import transformers -except ImportError: - spacy_huggingface_pipelines = None - transformers = None - from presidio_analyzer.nlp_engine import ( NerModelConfiguration, SpacyNlpEngine, ) +from presidio_analyzer.nlp_engine.huggingface_token_pipe import ( + FACTORY_NAME, + is_available, +) logger = logging.getLogger("presidio-analyzer") @@ -50,7 +47,7 @@ class TransformersNlpEngine(SpacyNlpEngine): """ engine_name = "transformers" - is_available = bool(spacy_huggingface_pipelines) + is_available = is_available() def __init__( self, @@ -89,14 +86,13 @@ def load(self) -> None: pipe_config = { "model": transformers_model, - "annotate": "spans", "stride": self.ner_model_configuration.stride, "alignment_mode": self.ner_model_configuration.alignment_mode, "aggregation_strategy": self.ner_model_configuration.aggregation_strategy, # noqa: E501 - "annotate_spans_key": self.entity_key, + "spans_key": self.entity_key, } - nlp.add_pipe("hf_token_pipe", config=pipe_config) + nlp.add_pipe(FACTORY_NAME, config=pipe_config) self.nlp[model["lang_code"]] = nlp @staticmethod @@ -118,7 +114,7 @@ def _get_entities(self, doc: Doc) -> List[Span]: """ Extract entities out of a spaCy pipeline, depending on the type of pipeline. - For spacy-huggingface-pipeline, this would be doc.spans[key] + For the Hugging Face token pipe, this is doc.spans[key]. :param doc: the output spaCy doc. :return: List of entities """ @@ -129,7 +125,7 @@ def _get_scores_for_entities(self, doc: Doc) -> List[float]: """Extract scores for entities from the doc. While spaCy does not provide confidence scores, - the spacy-huggingface-pipeline flow adds confidence scores + the Hugging Face token pipe adds confidence scores as SpanGroup attributes. :param doc: SpaCy doc """ diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/transformers_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/transformers_recognizer.py index 643d7b4ad6..915e123049 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/transformers_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/transformers_recognizer.py @@ -10,13 +10,10 @@ class TransformersRecognizer(SpacyRecognizer): """ - Recognize entities using the spacy-huggingface-pipeline package. + Recognize entities produced by the Transformers NLP engine. - The recognizer doesn't run transformers models, - but loads the output from the NlpArtifacts - See: - - https://huggingface.co/docs/transformers/main/en/index for transformer models - - https://github.com/explosion/spacy-huggingface-pipelines on the spaCy wrapper to transformers + The recognizer does not run Transformers models directly. It reads the + entities and confidence scores exposed through NlpArtifacts. """ # noqa: E501 ENTITIES = [ diff --git a/presidio-analyzer/pyproject.toml b/presidio-analyzer/pyproject.toml index 116203466c..7a5214ed5f 100644 --- a/presidio-analyzer/pyproject.toml +++ b/presidio-analyzer/pyproject.toml @@ -35,6 +35,9 @@ dependencies = [ "pydantic (>=2.12.5,<3.0.0)", ] +[project.entry-points."spacy_factories"] +presidio_hf_token_pipe = "presidio_analyzer.nlp_engine.huggingface_token_pipe:create_huggingface_token_pipe" + [project.optional-dependencies] server = [ "flask (>=1.1,<4.0.0)", @@ -42,10 +45,9 @@ server = [ "waitress (>=2.0.0,<4.0.0); platform_system == 'Windows'" ] transformers = [ - "transformers (>=4.0.0,<6.0.0)", + "transformers (>=5.5.0,<6.0.0)", "accelerate (>=0.20.0,<2.0.0)", - "huggingface_hub (>=0.20.0,<2.0.0)", - "spacy_huggingface_pipelines (>=0.0.4,<1.0.0)"] + "huggingface_hub (>=0.20.0,<2.0.0)"] stanza = [ "stanza (>=1.11.1,<2.0.0)", @@ -59,7 +61,7 @@ ahds = [ "azure-health-deidentification (>=1.1.0b1,<2.0.0)" ] gliner = [ - "transformers", + "transformers (>=5.5.0,<5.7.0)", "huggingface_hub", "gliner (>=0.2.26,<1.0.0)", "onnxruntime (>=1.19, <1.24.1) ; python_version == '3.10'", diff --git a/presidio-analyzer/tests/test_huggingface_token_pipe.py b/presidio-analyzer/tests/test_huggingface_token_pipe.py new file mode 100644 index 0000000000..a96937229f --- /dev/null +++ b/presidio-analyzer/tests/test_huggingface_token_pipe.py @@ -0,0 +1,312 @@ +import logging +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import spacy +from presidio_analyzer.nlp_engine import huggingface_token_pipe as token_pipe_module +from presidio_analyzer.nlp_engine.huggingface_token_pipe import ( + FACTORY_NAME, + HuggingFaceTokenPipe, +) + + +@pytest.mark.parametrize( + ("device_type", "device_index", "expected"), + [("cpu", None, -1), ("cuda", None, 0), ("cuda", 2, 2)], +) +def test_pipeline_device(mocker, device_type, device_index, expected): + mocker.patch.object( + token_pipe_module, + "get_torch_default_device", + return_value=SimpleNamespace(type=device_type, index=device_index), + ) + + assert token_pipe_module._pipeline_device() == expected + + +def test_factory_builds_token_classification_pipeline(mocker): + inference = Mock(return_value=[]) + creator = mocker.patch.object( + token_pipe_module, "hf_pipeline", return_value=inference + ) + mocker.patch.object( + token_pipe_module, + "get_torch_default_device", + return_value=SimpleNamespace(type="cuda", index=1), + ) + nlp = spacy.blank("en") + + nlp.add_pipe( + FACTORY_NAME, + config={ + "model": "test-model", + "revision": "model-revision", + "stride": 8, + "aggregation_strategy": "simple", + "alignment_mode": "strict", + "spans_key": "entities", + "pipeline_kwargs": {"trust_remote_code": False}, + }, + ) + + creator.assert_called_once_with( + task="token-classification", + model="test-model", + revision="model-revision", + stride=8, + aggregation_strategy="simple", + device=1, + trust_remote_code=False, + ) + + +def test_factory_omits_none_stride(mocker): + creator = mocker.patch.object(token_pipe_module, "hf_pipeline", return_value=Mock()) + mocker.patch.object( + token_pipe_module, + "get_torch_default_device", + return_value=SimpleNamespace(type="cpu", index=None), + ) + + spacy.blank("en").add_pipe( + FACTORY_NAME, config={"model": "test-model", "stride": None} + ) + + assert "stride" not in creator.call_args.kwargs + + +def test_factory_requires_transformers(mocker): + mocker.patch.object(token_pipe_module, "hf_pipeline", None) + + with pytest.raises(ImportError, match=r"presidio-analyzer\[transformers\]"): + spacy.blank("en").add_pipe(FACTORY_NAME, config={"model": "test-model"}) + + +def test_factory_requires_model(): + with pytest.raises(ValueError, match="model is required"): + spacy.blank("en").add_pipe(FACTORY_NAME) + + +def test_single_document_predictions_are_scored_spans(): + inference = Mock( + return_value=[ + { + "entity_group": "PER", + "start": 11, + "end": 14, + "score": 0.98, + } + ] + ) + pipe = HuggingFaceTokenPipe( + name=FACTORY_NAME, + pipeline=inference, + alignment_mode="strict", + spans_key="entities", + ) + + doc = pipe(spacy.blank("en").make_doc("my name is Dan")) + + assert [(span.text, span.label_) for span in doc.spans["entities"]] == [ + ("Dan", "PER") + ] + assert doc.spans["entities"].attrs["scores"] == [0.98] + + +def test_batch_predictions_remain_with_their_documents(): + inference = Mock( + return_value=[ + [{"entity_group": "PER", "start": 0, "end": 3, "score": 0.98}], + [{"entity_group": "ORG", "start": 0, "end": 6, "score": 0.92}], + ] + ) + pipe = HuggingFaceTokenPipe( + name=FACTORY_NAME, + pipeline=inference, + alignment_mode="strict", + spans_key="entities", + ) + nlp = spacy.blank("en") + + docs = list(pipe.pipe(map(nlp.make_doc, ["Dan", "GitHub"]), batch_size=2)) + + assert [doc.spans["entities"][0].text for doc in docs] == ["Dan", "GitHub"] + assert [doc.spans["entities"].attrs["scores"] for doc in docs] == [ + [0.98], + [0.92], + ] + inference.assert_called_once_with(["Dan", "GitHub"]) + + +def test_batch_failure_retries_individually_without_logging_text(caplog): + secret = "Sharon secret" + inference = Mock( + side_effect=[ + RuntimeError(secret), + [{"entity_group": "PER", "start": 0, "end": 6, "score": 0.98}], + [], + ] + ) + pipe = HuggingFaceTokenPipe( + name=FACTORY_NAME, + pipeline=inference, + alignment_mode="strict", + spans_key="entities", + ) + nlp = spacy.blank("en") + caplog.set_level(logging.WARNING, logger="presidio-analyzer") + + docs = list(pipe.pipe(map(nlp.make_doc, ["Sharon", "public"]), batch_size=2)) + + assert docs[0].spans["entities"][0].text == "Sharon" + assert not docs[1].spans["entities"] + assert "RuntimeError" in caplog.text + assert secret not in caplog.text + + +def test_individual_failures_are_isolated_without_logging_text(caplog): + batch_secret = "batch secret" + document_secret = "document secret" + inference = Mock( + side_effect=[ + RuntimeError(batch_secret), + RuntimeError(document_secret), + [], + ] + ) + pipe = HuggingFaceTokenPipe( + name=FACTORY_NAME, + pipeline=inference, + alignment_mode="strict", + spans_key="entities", + ) + nlp = spacy.blank("en") + caplog.set_level(logging.WARNING, logger="presidio-analyzer") + + docs = list(pipe.pipe(map(nlp.make_doc, ["private", "public"]), batch_size=2)) + + assert not docs[0].spans["entities"] + assert not docs[1].spans["entities"] + assert "RuntimeError" in caplog.text + assert batch_secret not in caplog.text + assert document_secret not in caplog.text + + +def test_single_document_failure_adds_empty_spans(caplog): + secret = "private text" + pipe = HuggingFaceTokenPipe( + name=FACTORY_NAME, + pipeline=Mock(side_effect=RuntimeError(secret)), + alignment_mode="strict", + spans_key="entities", + ) + caplog.set_level(logging.WARNING, logger="presidio-analyzer") + + doc = pipe(spacy.blank("en").make_doc(secret)) + + assert not doc.spans["entities"] + assert "RuntimeError" in caplog.text + assert secret not in caplog.text + + +def test_batch_output_count_must_match_documents(): + pipe = HuggingFaceTokenPipe( + name=FACTORY_NAME, + pipeline=Mock(return_value=[]), + alignment_mode="strict", + spans_key="entities", + ) + nlp = spacy.blank("en") + + with pytest.raises(ValueError, match="output count"): + list(pipe.pipe(map(nlp.make_doc, ["one", "two"]), batch_size=2)) + + +@pytest.mark.parametrize("output", [None, {}, ["invalid"]]) +def test_pipeline_output_must_be_prediction_mappings(output): + pipe = HuggingFaceTokenPipe( + name=FACTORY_NAME, + pipeline=Mock(return_value=output), + alignment_mode="strict", + spans_key="entities", + ) + + with pytest.raises(TypeError, match="list of mappings"): + pipe(spacy.blank("en").make_doc("text")) + + +def test_unaligned_and_overlapping_predictions_are_skipped(caplog): + pipe = HuggingFaceTokenPipe( + name=FACTORY_NAME, + pipeline=Mock( + return_value=[ + {"entity_group": "PER", "start": 0, "end": 5, "score": 0.9}, + {"entity_group": "PER", "start": 1, "end": 5, "score": 0.8}, + {"entity_group": "ORG", "start": 6, "end": 8, "score": 0.7}, + ] + ), + alignment_mode="strict", + spans_key="entities", + ) + caplog.set_level(logging.WARNING, logger="presidio-analyzer") + + doc = pipe(spacy.blank("en").make_doc("Alice works")) + + assert [span.text for span in doc.spans["entities"]] == ["Alice"] + assert doc.spans["entities"].attrs["scores"] == [0.9] + assert caplog.text.count("Skipping unaligned or overlapping prediction") == 2 + assert "Alice" not in caplog.text + + +def test_malformed_prediction_error_does_not_include_values(): + secret = "private text" + pipe = HuggingFaceTokenPipe( + name=FACTORY_NAME, + pipeline=Mock(return_value=[{"word": secret}]), + alignment_mode="strict", + spans_key="entities", + ) + + with pytest.raises(ValueError, match=r"keys=\['word'\]") as error: + pipe(spacy.blank("en").make_doc(secret)) + + assert secret not in str(error.value) + + +@pytest.mark.parametrize("score", [None, "private score"]) +def test_prediction_requires_numeric_score(score): + pipe = HuggingFaceTokenPipe( + name=FACTORY_NAME, + pipeline=Mock( + return_value=[ + { + "entity_group": "PER", + "start": 0, + "end": 5, + "score": score, + } + ] + ), + alignment_mode="strict", + spans_key="entities", + ) + + with pytest.raises(ValueError, match="numeric score") as error: + pipe(spacy.blank("en").make_doc("Alice")) + + assert str(score) not in str(error.value) + + +def test_serialization_rebuilds_pipeline_from_config(tmp_path): + pipe = HuggingFaceTokenPipe( + name=FACTORY_NAME, + pipeline=Mock(), + alignment_mode="strict", + spans_key="entities", + ) + + assert pipe.to_bytes() == b"" + assert pipe.from_bytes(b"serialized") is pipe + assert pipe.to_disk(tmp_path) is None + assert pipe.from_disk(tmp_path) is pipe diff --git a/presidio-analyzer/tests/test_transformers_nlp_engine.py b/presidio-analyzer/tests/test_transformers_nlp_engine.py index cd0274e0de..5ce6dccb79 100644 --- a/presidio-analyzer/tests/test_transformers_nlp_engine.py +++ b/presidio-analyzer/tests/test_transformers_nlp_engine.py @@ -1,6 +1,8 @@ import pytest - +import spacy from presidio_analyzer.nlp_engine import TransformersNlpEngine +from presidio_analyzer.nlp_engine import huggingface_token_pipe as token_pipe_module +from presidio_analyzer.nlp_engine.huggingface_token_pipe import FACTORY_NAME def test_default_models(): @@ -22,6 +24,41 @@ def test_validate_model_params_happy_path(): TransformersNlpEngine._validate_model_params(model) +def test_load_adds_internal_hugging_face_pipe(mocker): + inference = mocker.Mock() + creator = mocker.patch.object( + token_pipe_module, "hf_pipeline", return_value=inference + ) + mocker.patch.object( + token_pipe_module, + "_pipeline_device", + return_value=-1, + ) + mocker.patch( + "presidio_analyzer.nlp_engine.transformers_nlp_engine.spacy.load", + return_value=spacy.blank("en"), + ) + mocker.patch.object(TransformersNlpEngine, "_enable_gpu") + mocker.patch.object(TransformersNlpEngine, "_download_spacy_model_if_needed") + engine = TransformersNlpEngine( + models=[ + { + "lang_code": "en", + "model_name": { + "spacy": "en_core_web_sm", + "transformers": "test-model", + }, + } + ] + ) + + engine.load() + + assert engine.nlp["en"].pipe_names == [FACTORY_NAME] + creator.assert_called_once() + assert creator.call_args.kwargs["model"] == "test-model" + + @pytest.mark.parametrize( "key", [("lang_code"), ("model_name"), ("model_name.spacy"), ("model_name.transformers")], diff --git a/presidio-analyzer/uv.lock b/presidio-analyzer/uv.lock index 05ee39ebca..3540f0f93c 100644 --- a/presidio-analyzer/uv.lock +++ b/presidio-analyzer/uv.lock @@ -655,7 +655,7 @@ name = "coloredlogs" version = "15.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "humanfriendly" }, + { name = "humanfriendly", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520, upload-time = "2021-06-11T10:22:45.202Z" } wheels = [ @@ -839,7 +839,7 @@ name = "cuda-bindings" version = "13.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a9/21/8464d133752951c154feafb3b65c297e7d80f301183d220bec4c830f1441/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86", size = 6073403, upload-time = "2026-05-29T23:11:36.22Z" }, @@ -874,43 +874,43 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-runtime", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cufft = [ - { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cufft", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cufile = [ - { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-cupti", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] curand = [ - { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-curand", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cusolver = [ - { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusolver", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvtx", marker = "(python_full_version < '3.11' and platform_machine == 'AMD64' and sys_platform == 'win32') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [[package]] @@ -1362,7 +1362,7 @@ name = "gunicorn" version = "25.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "packaging" }, + { name = "packaging", marker = "python_full_version < '3.11' or sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c4/f4/e78fa054248fab913e2eab0332c6c2cb07421fca1ce56d8fe43b6aef57a4/gunicorn-25.3.0.tar.gz", hash = "sha256:f74e1b2f9f76f6cd1ca01198968bd2dd65830edc24b6e8e4d78de8320e2fe889", size = 634883, upload-time = "2026-03-27T00:00:26.092Z" } wheels = [ @@ -1440,21 +1440,22 @@ wheels = [ [[package]] name = "huggingface-hub" -version = "0.36.2" +version = "1.24.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "click" }, { name = "filelock" }, { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, { name = "packaging" }, { name = "pyyaml" }, - { name = "requests" }, { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7c/b7/8cb61d2eece5fb05a83271da168186721c450eb74e3c31f7ef3169fa475b/huggingface_hub-0.36.2.tar.gz", hash = "sha256:1934304d2fb224f8afa3b87007d58501acfda9215b334eed53072dd5e815ff7a", size = 649782, upload-time = "2026-02-06T09:24:13.098Z" } +sdist = { url = "https://files.pythonhosted.org/packages/df/9b/d3bb4e7d792835daf34dd7091bbc7d7b4e0437d9388f1ea7239cce49f478/huggingface_hub-1.24.0.tar.gz", hash = "sha256:18431ff4daae0749aa9ba102fc952e314c98e1d30ebdec5319d85ca0a83e1ae5", size = 921848, upload-time = "2026-07-17T09:54:01.022Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/af/48ac8483240de756d2438c380746e7130d1c6f75802ef22f3c6d49982787/huggingface_hub-0.36.2-py3-none-any.whl", hash = "sha256:48f0c8eac16145dfce371e9d2d7772854a4f591bcb56c9cf548accf531d54270", size = 566395, upload-time = "2026-02-06T09:24:11.133Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c3/aeaaf3911d2529614be18d1c8b5496afc185560e76568063d517283318af/huggingface_hub-1.24.0-py3-none-any.whl", hash = "sha256:6ed4120a84a6beec900640aa7e346bd766a6b7341e41526fef5dc8bd81fb7d59", size = 771904, upload-time = "2026-07-17T09:53:59.106Z" }, ] [[package]] @@ -1462,7 +1463,7 @@ name = "humanfriendly" version = "10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyreadline3", marker = "sys_platform == 'win32'" }, + { name = "pyreadline3", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } wheels = [ @@ -2221,7 +2222,7 @@ name = "nvidia-cublas" version = "13.1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, @@ -2260,7 +2261,7 @@ name = "nvidia-cudnn-cu13" version = "9.20.0.48" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, @@ -2272,7 +2273,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -2302,9 +2303,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, + { name = "nvidia-cublas", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-cusparse", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -2316,7 +2317,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -2376,12 +2377,12 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "coloredlogs" }, - { name = "flatbuffers" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "sympy" }, + { name = "coloredlogs", marker = "python_full_version < '3.11'" }, + { name = "flatbuffers", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "packaging", marker = "python_full_version < '3.11'" }, + { name = "protobuf", marker = "python_full_version < '3.11'" }, + { name = "sympy", marker = "python_full_version < '3.11'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/35/d6/311b1afea060015b56c742f3531168c1644650767f27ef40062569960587/onnxruntime-1.23.2-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:a7730122afe186a784660f6ec5807138bf9d792fa1df76556b27307ea9ebcbe3", size = 17195934, upload-time = "2025-10-27T23:06:14.143Z" }, @@ -2424,10 +2425,10 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "flatbuffers" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, - { name = "packaging" }, - { name = "protobuf" }, + { name = "flatbuffers", marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "packaging", marker = "python_full_version >= '3.11'" }, + { name = "protobuf", marker = "python_full_version >= '3.11'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/d4/e4/5353d7e09ced4a8f473f843223fc75d726b2b5519dcefc12f22a6c92852d/onnxruntime-1.27.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:8ba14a38c570087f3cdb8cfba33f7a38a1e826c1e5b29e17c28ceda0cc910016", size = 18416484, upload-time = "2026-06-15T22:43:43.894Z" }, @@ -2492,10 +2493,10 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "python-dateutil" }, - { name = "pytz" }, - { name = "tzdata" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "python-dateutil", marker = "python_full_version < '3.11'" }, + { name = "pytz", marker = "python_full_version < '3.11'" }, + { name = "tzdata", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -2564,9 +2565,9 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, - { name = "python-dateutil" }, - { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, + { name = "tzdata", marker = "(python_full_version >= '3.11' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } wheels = [ @@ -2781,7 +2782,6 @@ stanza = [ transformers = [ { name = "accelerate" }, { name = "huggingface-hub" }, - { name = "spacy-huggingface-pipelines" }, { name = "transformers" }, ] @@ -2824,11 +2824,10 @@ requires-dist = [ { name = "regex", specifier = ">=2023.0.0" }, { name = "spacy", marker = "python_full_version < '3.14'", specifier = ">=3.4.4,!=3.7.0,<4.0.0" }, { name = "spacy", marker = "python_full_version >= '3.14'", specifier = ">=3.4.4,!=3.7.0,!=3.8.14,<4.0.0" }, - { name = "spacy-huggingface-pipelines", marker = "extra == 'transformers'", specifier = ">=0.0.4,<1.0.0" }, { name = "stanza", marker = "extra == 'stanza'", specifier = ">=1.11.1,<2.0.0" }, { name = "tldextract", specifier = ">=5.3.1,<6.0.0" }, - { name = "transformers", marker = "extra == 'gliner'" }, - { name = "transformers", marker = "extra == 'transformers'", specifier = ">=4.0.0,<6.0.0" }, + { name = "transformers", marker = "extra == 'gliner'", specifier = ">=5.5.0,<5.7.0" }, + { name = "transformers", marker = "extra == 'transformers'", specifier = ">=5.5.0,<6.0.0" }, { name = "waitress", marker = "sys_platform == 'win32' and extra == 'server'", specifier = ">=2.0.0,<4.0.0" }, ] provides-extras = ["server", "transformers", "stanza", "azure-ai-language", "ahds", "gliner", "langextract"] @@ -3759,20 +3758,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/58/0001fd8124b62a2a9984278d793e3abfa1b70e969fc26a8668755178db84/spacy-3.8.13-cp314-cp314-win_arm64.whl", hash = "sha256:b2a402f229fcb5dba5454c346468757bd3a5215809e784b85d739ca84916f05b", size = 13834663, upload-time = "2026-03-23T17:44:29.43Z" }, ] -[[package]] -name = "spacy-huggingface-pipelines" -version = "0.0.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "spacy" }, - { name = "torch" }, - { name = "transformers" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/38/ca/07667af54b4efb3ee204db6db6ba9a3e7d7baf59219e5c86f7888121be06/spacy_huggingface_pipelines-0.0.4.tar.gz", hash = "sha256:35b409ed7d20c5b36d788912570e3444ec1b0c344255e847bf722b3286279e95", size = 11685, upload-time = "2023-06-02T18:18:03.859Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/69/1cf6333eebaadf8517f59b9dec676f42f5fef8b13a29eaf2cd2922470868/spacy_huggingface_pipelines-0.0.4-py2.py3-none-any.whl", hash = "sha256:9e38ee6eba7a11fca32b7d14f38259f7805eec211e8959105a90c95915168b00", size = 11236, upload-time = "2023-06-02T18:18:02.363Z" }, -] - [[package]] name = "spacy-legacy" version = "3.0.12" @@ -4127,24 +4112,23 @@ wheels = [ [[package]] name = "transformers" -version = "4.57.6" +version = "5.6.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock" }, { name = "huggingface-hub" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "packaging" }, { name = "pyyaml" }, { name = "regex" }, - { name = "requests" }, { name = "safetensors" }, { name = "tokenizers" }, { name = "tqdm" }, + { name = "typer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c4/35/67252acc1b929dc88b6602e8c4a982e64f31e733b804c14bc24b47da35e6/transformers-4.57.6.tar.gz", hash = "sha256:55e44126ece9dc0a291521b7e5492b572e6ef2766338a610b9ab5afbb70689d3", size = 10134912, upload-time = "2026-01-16T10:38:39.284Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/e9/c6c80a07690142a7d05444271f47b9f3c8aac7dea01d52e1137ee480ad78/transformers-5.6.2.tar.gz", hash = "sha256:e657134c3e5a6bc00a3c35f4e2674bb51adfcd89898495b788a18552bac2b91a", size = 8311867, upload-time = "2026-04-23T18:33:29.332Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/03/b8/e484ef633af3887baeeb4b6ad12743363af7cce68ae51e938e00aaa0529d/transformers-4.57.6-py3-none-any.whl", hash = "sha256:4c9e9de11333ddfe5114bc872c9f370509198acf0b87a832a0ab9458e2bd0550", size = 11993498, upload-time = "2026-01-16T10:38:31.289Z" }, + { url = "https://files.pythonhosted.org/packages/5d/95/0b0218149b0d6f14df35f5b8f676fa83df4f19ed253c3cc447107ef86eca/transformers-5.6.2-py3-none-any.whl", hash = "sha256:f8d3a1bb96778fed9b8aabfd0dd6e19843e4b0f2bb6b59f32b8a92051b0f348f", size = 10364898, upload-time = "2026-04-23T18:33:26.081Z" }, ] [[package]]