Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions fenn/agents/rag/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
".txt",
".md",
".pdf",
".docx",
".json",
".py",
".js",
Expand Down Expand Up @@ -95,6 +96,8 @@ def _read_file(path):
try:
if path.suffix == ".pdf":
return _read_pdf(path)
if path.suffix == ".docx":
return _read_docx(path)
return path.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError, ValueError, RuntimeError) as e:
logger.error(f"[cofone] read error {path.name}: {e}")
Expand Down Expand Up @@ -124,6 +127,33 @@ def _read_pdf(path):
)


def _read_docx(path):
"""
Extract text from a Word (.docx) file using python-docx.
Reads both paragraphs and table cells.
Requires: pip install "cofone[docx]" or pip install python-docx
"""
try:
import docx

document = docx.Document(str(path))
parts = [p.text for p in document.paragraphs]
for table in document.tables:
for row in table.rows:
parts.append("\t".join(cell.text for cell in row.cells))
text = "\n".join(parts).strip()
if not text:
logger.warning(
f"[cofone] warning: DOCX '{path.name}' returned no text (empty document)"
)
return text or None
except ImportError:
raise ImportError(
"[cofone] python-docx not installed.\n"
'Run: pip install "cofone[docx]" or pip install python-docx'
)


def _load_url(url):
"""
Fetch a web page and return its visible text content.
Expand Down
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -107,11 +107,13 @@ llms = [
]

rag-pdf = ["pypdf>=4.0.0"]
rag-docx = ["python-docx>=1.0.0"]
rag-faiss = ["faiss-cpu>=1.7.0", "sentence-transformers>=2.0.0"]
rag-web = ["wikipedia>=1.4.0", "youtube-transcript-api>=0.6.0"]

rag-all = [
"pypdf>=4.0.0",
"python-docx>=1.0.0",
"faiss-cpu>=1.7.0",
"sentence-transformers>=2.0.0",
"wikipedia>=1.4.0",
Expand Down
72 changes: 72 additions & 0 deletions tests/unit/agents/test_docx_loader.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""Tests for the .docx loader in fenn/agents/rag/loader.py

Place at: tests/unit/agents/test_docx_loader.py
(or merge these cases into tests/unit/agents/test_rag_components.py).
"""

from unittest.mock import patch

import pytest

from fenn.agents.rag.loader import (
_read_docx,
_read_file,
load_documents,
)


def _make_docx(path, paragraphs=("Hello from docx", "second line"), table=None):
docx = pytest.importorskip("docx")
d = docx.Document()
for p in paragraphs:
d.add_paragraph(p)
if table:
t = d.add_table(rows=len(table), cols=len(table[0]))
for r, row in enumerate(table):
for c, val in enumerate(row):
t.rows[r].cells[c].text = val
d.save(str(path))
return path


class TestReadDocx:
def test_reads_paragraphs(self, tmp_path):
f = _make_docx(tmp_path / "doc.docx")
assert _read_docx(f) == "Hello from docx\nsecond line"

def test_reads_table_cells(self, tmp_path):
f = _make_docx(
tmp_path / "t.docx",
paragraphs=("Prices:",),
table=[["Item", "Price"], ["Coffee", "3.00"]],
)
text = _read_docx(f)
assert "Item\tPrice" in text
assert "Coffee\t3.00" in text

def test_empty_docx_returns_none(self, tmp_path):
docx = pytest.importorskip("docx")
d = docx.Document()
d.save(str(tmp_path / "empty.docx"))
assert _read_docx(tmp_path / "empty.docx") is None

def test_read_file_delegates_docx(self, tmp_path):
f = _make_docx(tmp_path / "d.docx")
assert _read_file(f) == "Hello from docx\nsecond line"

def test_load_documents_reads_docx(self, tmp_path):
f = _make_docx(tmp_path / "d.docx")
assert load_documents(str(f)) == ["Hello from docx\nsecond line"]

def test_missing_python_docx_raises_importerror(self, tmp_path):
f = _make_docx(tmp_path / "d.docx")
real_import = __import__

def fake_import(name, *args, **kwargs):
if name == "docx":
raise ImportError("no docx")
return real_import(name, *args, **kwargs)

with patch("builtins.__import__", side_effect=fake_import):
with pytest.raises(ImportError, match="python-docx"):
_read_docx(f)
Loading