Replace stale Hugging Face spaCy adapter

docs/analyzer/nlp_engines/transformers.md

@@ -215,7 +215,9 @@ The `ner_model_configuration` section contains the following parameters:
     For example, for `bert-base-NER-uncased`, it can be found here: <https://huggingface.co/dslim/bert-base-NER-uncased/blob/main/config.json>.
     Note that most NER models add a prefix to the class (e.g. `B-PER` for class `PER`). When creating the mapping, do not add the prefix.
 
-See more information on parameters on the [spacy-huggingface-pipelines Github repo](https://github.com/explosion/spacy-huggingface-pipelines#token-classification).
+See the Hugging Face
+[token-classification pipeline documentation](https://huggingface.co/docs/transformers/main_classes/pipelines#transformers.TokenClassificationPipeline)
+for more information on aggregation and stride behavior.
 
 Once created, see [the NLP configuration documentation](../customizing_nlp_models.md#Configure-Presidio-to-use-the-new-model) for more information.
 

presidio-analyzer/presidio_analyzer/nlp_engine/huggingface_token_pipe.py

@@ -0,0 +1,220 @@
+"""spaCy component for Hugging Face token-classification pipelines."""
+
+import logging
+from collections.abc import Iterable, Iterator, Mapping, Sequence
+from pathlib import Path
+from typing import Any, Optional
+
+from spacy import util
+from spacy.language import Language
+from spacy.pipeline import Pipe
+from spacy.tokens import Doc, SpanGroup
+from thinc.api import get_torch_default_device
+
+try:
+    from transformers import pipeline as hf_pipeline
+except ImportError:
+    hf_pipeline = None
+
+logger = logging.getLogger("presidio-analyzer")
+
+FACTORY_NAME = "presidio_hf_token_pipe"
+
+
+def is_available() -> bool:
+    """Return whether the optional Transformers dependency is installed."""
+    return hf_pipeline is not None
+
+
+def _pipeline_device() -> int:
+    """Return a Transformers-compatible CPU or CUDA device index."""
+    device = get_torch_default_device()
+    if device.type != "cuda":
+        return -1
+    return device.index if device.index is not None else 0
+
+
+@Language.factory(
+    FACTORY_NAME,
+    assigns=[],
+    default_config={
+        "model": "",
+        "revision": "main",
+        "stride": 14,
+        "aggregation_strategy": "max",
+        "alignment_mode": "expand",
+        "spans_key": "bert-base-ner",
+        "pipeline_kwargs": {},
+    },
+    default_score_weights={},
+)
+def create_huggingface_token_pipe(
+    nlp: Language,
+    name: str,
+    model: str,
+    revision: str,
+    stride: Optional[int],
+    aggregation_strategy: str,
+    alignment_mode: str,
+    spans_key: str,
+    pipeline_kwargs: dict[str, Any],
+) -> "HuggingFaceTokenPipe":
+    """Create a spaCy component backed by a Hugging Face pipeline."""
+    if hf_pipeline is None:
+        raise ImportError(
+            "transformers is not installed. Install presidio-analyzer[transformers] "
+            "to use TransformersNlpEngine."
+        )
+    if not model:
+        raise ValueError("A Hugging Face token-classification model is required")
+
+    kwargs = {
+        "task": "token-classification",
+        "model": model,
+        "revision": revision,
+        "aggregation_strategy": aggregation_strategy,
+        "device": _pipeline_device(),
+    }
+    if stride is not None:
+        kwargs["stride"] = stride
+
+    pipeline = hf_pipeline(**kwargs, **pipeline_kwargs)
+    return HuggingFaceTokenPipe(
+        name=name,
+        pipeline=pipeline,
+        alignment_mode=alignment_mode,
+        spans_key=spans_key,
+    )
+
+
+class HuggingFaceTokenPipe(Pipe):
+    """Store token-classification predictions as scored spaCy spans."""
+
+    def __init__(
+        self,
+        name: str,
+        pipeline: Any,
+        alignment_mode: str,
+        spans_key: str,
+    ) -> None:
+        self.name = name
+        self.pipeline = pipeline
+        self.alignment_mode = alignment_mode
+        self.spans_key = spans_key
+
+    def __call__(self, doc: Doc) -> Doc:
+        """Annotate one document."""
+        return self._add_predictions(doc, self._predict_one(doc))
+
+    def pipe(self, stream: Iterable[Doc], *, batch_size: int = 128) -> Iterator[Doc]:
+        """Annotate documents in batches, retrying batch failures individually."""
+        for batch in util.minibatch(stream, size=batch_size):
+            docs = list(batch)
+            predictions = self._predict_batch(docs)
+            for doc, doc_predictions in zip(docs, predictions, strict=True):
+                yield self._add_predictions(doc, doc_predictions)
+
+    def _predict_batch(self, docs: Sequence[Doc]) -> list[Sequence[Mapping[str, Any]]]:
+        texts = [doc.text for doc in docs]
+        try:
+            output = self.pipeline(texts)
+        except Exception as error:
+            logger.warning(
+                "Hugging Face batch inference failed with %s; retrying each document",
+                type(error).__name__,
+            )
+            return [self._predict_one(doc) for doc in docs]
+
+        if not isinstance(output, list) or len(output) != len(docs):
+            raise ValueError(
+                "Hugging Face batch output count does not match the document count"
+            )
+        return [self._validate_predictions(item) for item in output]
+
+    def _predict_one(self, doc: Doc) -> Sequence[Mapping[str, Any]]:
+        try:
+            output = self.pipeline(doc.text)
+        except Exception as error:
+            logger.warning(
+                "Hugging Face inference failed with %s; skipping document",
+                type(error).__name__,
+            )
+            return []
+        return self._validate_predictions(output)
+
+    @staticmethod
+    def _validate_predictions(output: Any) -> Sequence[Mapping[str, Any]]:
+        if not isinstance(output, list) or any(
+            not isinstance(item, Mapping) for item in output
+        ):
+            raise TypeError(
+                "Hugging Face token-classification output must be a list of mappings"
+            )
+        return output
+
+    def _add_predictions(
+        self, doc: Doc, predictions: Sequence[Mapping[str, Any]]
+    ) -> Doc:
+        spans = SpanGroup(doc, attrs={"scores": []})
+        previous_end = 0
+
+        for prediction in predictions:
+            label = prediction.get("entity_group") or prediction.get("entity")
+            start = prediction.get("start")
+            end = prediction.get("end")
+            if (
+                not isinstance(label, str)
+                or not isinstance(start, int)
+                or not isinstance(end, int)
+            ):
+                raise ValueError(
+                    "Hugging Face prediction is missing a string label or integer "
+                    f"offsets; keys={sorted(prediction.keys())}"
+                )
+            try:
+                score = float(prediction["score"])
+            except (KeyError, TypeError, ValueError):
+                raise ValueError(
+                    "Hugging Face prediction is missing a numeric score"
+                ) from None
+
+            span = None
+            if start >= previous_end:
+                span = doc.char_span(
+                    start,
+                    end,
+                    label=label,
+                    alignment_mode=self.alignment_mode,
+                )
+            if span is None or span.start_char < previous_end:
+                logger.warning(
+                    "Skipping unaligned or overlapping prediction "
+                    "label=%s start=%d end=%d",
+                    label,
+                    start,
+                    end,
+                )
+                continue
+
+            spans.append(span)
+            spans.attrs["scores"].append(score)
+            previous_end = end
+
+        doc.spans[self.spans_key] = spans
+        return doc
+
+    def to_bytes(self, **kwargs: Any) -> bytes:
+        """Return no model data because the pipeline is rebuilt from config."""
+        return b""
+
+    def from_bytes(self, bytes_data: bytes, **kwargs: Any) -> "HuggingFaceTokenPipe":
+        """Keep the pipeline created from the current configuration."""
+        return self
+
+    def to_disk(self, path: Path, **kwargs: Any) -> None:
+        """Persist no model data because the pipeline is rebuilt from config."""
+        return None
+
+    def from_disk(self, path: Path, **kwargs: Any) -> "HuggingFaceTokenPipe":
+        """Keep the pipeline created from the current configuration."""
+        return self

presidio-analyzer/presidio_analyzer/nlp_engine/transformers_nlp_engine.py

@@ -4,17 +4,14 @@
 import spacy
 from spacy.tokens import Doc, Span
 
-try:
-    import spacy_huggingface_pipelines
-    import transformers
-except ImportError:
-    spacy_huggingface_pipelines = None
-    transformers = None
-
 from presidio_analyzer.nlp_engine import (
     NerModelConfiguration,
     SpacyNlpEngine,
 )
+from presidio_analyzer.nlp_engine.huggingface_token_pipe import (
+    FACTORY_NAME,
+    is_available,
+)
 
 logger = logging.getLogger("presidio-analyzer")
 
@@ -50,7 +47,7 @@ class TransformersNlpEngine(SpacyNlpEngine):
     """
 
     engine_name = "transformers"
-    is_available = bool(spacy_huggingface_pipelines)
+    is_available = is_available()
 
     def __init__(
         self,
@@ -89,14 +86,13 @@ def load(self) -> None:
 
             pipe_config = {
                 "model": transformers_model,
-                "annotate": "spans",
                 "stride": self.ner_model_configuration.stride,
                 "alignment_mode": self.ner_model_configuration.alignment_mode,
                 "aggregation_strategy": self.ner_model_configuration.aggregation_strategy,  # noqa: E501
-                "annotate_spans_key": self.entity_key,
+                "spans_key": self.entity_key,
             }
 
-            nlp.add_pipe("hf_token_pipe", config=pipe_config)
+            nlp.add_pipe(FACTORY_NAME, config=pipe_config)
             self.nlp[model["lang_code"]] = nlp
 
     @staticmethod
@@ -118,7 +114,7 @@ def _get_entities(self, doc: Doc) -> List[Span]:
         """
         Extract entities out of a spaCy pipeline, depending on the type of pipeline.
 
-        For spacy-huggingface-pipeline, this would be doc.spans[key]
+        For the Hugging Face token pipe, this is doc.spans[key].
         :param doc: the output spaCy doc.
         :return: List of entities
         """
@@ -129,7 +125,7 @@ def _get_scores_for_entities(self, doc: Doc) -> List[float]:
         """Extract scores for entities from the doc.
 
         While spaCy does not provide confidence scores,
-        the spacy-huggingface-pipeline flow adds confidence scores
+        the Hugging Face token pipe adds confidence scores
         as SpanGroup attributes.
         :param doc: SpaCy doc
         """

presidio-analyzer/presidio_analyzer/predefined_recognizers/nlp_engine_recognizers/transformers_recognizer.py

@@ -10,13 +10,10 @@
 
 class TransformersRecognizer(SpacyRecognizer):
     """
-    Recognize entities using the spacy-huggingface-pipeline package.
+    Recognize entities produced by the Transformers NLP engine.
 
-    The recognizer doesn't run transformers models,
-    but loads the output from the NlpArtifacts
-    See:
-     - https://huggingface.co/docs/transformers/main/en/index for transformer models
-     - https://github.com/explosion/spacy-huggingface-pipelines on the spaCy wrapper to transformers
+    The recognizer does not run Transformers models directly. It reads the
+    entities and confidence scores exposed through NlpArtifacts.
     """  # noqa: E501
 
     ENTITIES = [

presidio-analyzer/pyproject.toml

@@ -35,17 +35,19 @@ dependencies = [
     "pydantic (>=2.12.5,<3.0.0)",
 ]
 
+[project.entry-points."spacy_factories"]
+presidio_hf_token_pipe = "presidio_analyzer.nlp_engine.huggingface_token_pipe:create_huggingface_token_pipe"
+
 [project.optional-dependencies]
 server = [
     "flask (>=1.1,<4.0.0)",
     "gunicorn (>=20.0.0,<26.0.0); platform_system != 'Windows'",
     "waitress (>=2.0.0,<4.0.0); platform_system == 'Windows'"
 ]
 transformers = [
-    "transformers (>=4.0.0,<6.0.0)",
+    "transformers (>=5.5.0,<6.0.0)",
     "accelerate (>=0.20.0,<2.0.0)",
-    "huggingface_hub (>=0.20.0,<2.0.0)",
-    "spacy_huggingface_pipelines (>=0.0.4,<1.0.0)"]
+    "huggingface_hub (>=0.20.0,<2.0.0)"]
 
 stanza = [
     "stanza (>=1.11.1,<2.0.0)",
@@ -59,7 +61,7 @@ ahds = [
     "azure-health-deidentification (>=1.1.0b1,<2.0.0)"
 ]
 gliner = [
-    "transformers",
+    "transformers (>=5.5.0,<5.7.0)",
     "huggingface_hub",
     "gliner (>=0.2.26,<1.0.0)",
     "onnxruntime (>=1.19, <1.24.1) ; python_version == '3.10'",