From 83f392ae907a922e2b92e3087c1b37bb2cadecb7 Mon Sep 17 00:00:00 2001 From: Nick Franck <46548427+CyMule@users.noreply.github.com> Date: Sun, 19 Jul 2026 21:18:50 -0400 Subject: [PATCH 1/5] fix(csv): preserve rows from every uploaded file --- CHANGELOG.md | 6 ++++++ prepline_general/api/__version__.py | 2 +- prepline_general/api/general.py | 17 ++++++----------- test_general/api/test_app.py | 20 ++++++++++++++++++++ 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b75a37ce..dbad57de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.1.9 + +### Fixes + +- **Preserve rows in multi-file CSV responses**: CSV output now concatenates each file's partition results in request order without using filenames as dictionary keys, so duplicate filenames cannot overwrite earlier rows. + ## 0.1.8 ### Features diff --git a/prepline_general/api/__version__.py b/prepline_general/api/__version__.py index 93b52aec..4eeb8fc3 100644 --- a/prepline_general/api/__version__.py +++ b/prepline_general/api/__version__.py @@ -1 +1 @@ -__version__ = "0.1.8" # pragma: no cover +__version__ = "0.1.9" # pragma: no cover diff --git a/prepline_general/api/general.py b/prepline_general/api/general.py index 8a7b3cb2..6f616f90 100644 --- a/prepline_general/api/general.py +++ b/prepline_general/api/general.py @@ -742,18 +742,13 @@ def join_responses( if form_params.output_format != "text/csv": return cast(List[Union[str, List[Dict[str, Any]]]], responses) responses = cast(List[PlainTextResponse], responses) - data = pd.read_csv( # pyright: ignore[reportUnknownMemberType] - io.BytesIO(responses[0].body) + data = pd.concat( # pyright: ignore[reportUnknownMemberType] + [ + pd.read_csv(io.BytesIO(response.body)) # pyright: ignore[reportUnknownMemberType] + for response in responses + ] ) - if len(responses) > 1: - for resp in responses[1:]: - resp_data = pd.read_csv( # pyright: ignore[reportUnknownMemberType] - io.BytesIO(resp.body) - ) - data = data.merge( # pyright: ignore[reportUnknownMemberType] - resp_data, how="outer" - ) - return PlainTextResponse(data.to_csv()) + return PlainTextResponse(data.to_csv(index=False)) return ( MultipartMixedResponse( diff --git a/test_general/api/test_app.py b/test_general/api/test_app.py index ff5342d7..1baaf4a0 100644 --- a/test_general/api/test_app.py +++ b/test_general/api/test_app.py @@ -1154,6 +1154,26 @@ def test_output_format_csv_ignore_specified_accept_header(): assert df["text"][3] == "Make sure to RSVP!" +def test_output_format_csv_concatenates_multiple_files_without_index_column(): + client = TestClient(app) + test_file = Path("sample-docs") / "family-day.eml" + + with open(test_file, "rb") as first, open(test_file, "rb") as second: + response = client.post( + MAIN_API_ROUTE, + files=[ + ("files", ("same-name.eml", first, "message/rfc822")), + ("files", ("same-name.eml", second, "message/rfc822")), + ], + data={"output_format": "text/csv"}, + ) + + assert response.status_code == 200 + df = pd.read_csv(io.StringIO(response.text)) + assert len(df) == 18 + assert "Unnamed: 0" not in df.columns + + @pytest.mark.parametrize( "pdf_infer_table_structure, strategy, skip_infer_table_types, expected", [ From 9f0e804e3651a8dc466f8e80af62cc566e060d77 Mon Sep 17 00:00:00 2001 From: Nick Franck <46548427+CyMule@users.noreply.github.com> Date: Sun, 19 Jul 2026 22:07:15 -0400 Subject: [PATCH 2/5] fix(csv): tolerate documents with no elements in multi-file CSV output A file whose partition produces zero elements serializes to a bodiless CSV that pandas cannot parse, which failed the entire multi-file request. Such partitions now contribute no rows while every row from the remaining files is preserved. --- CHANGELOG.md | 2 +- prepline_general/api/general.py | 16 ++++++++++------ test_general/api/test_app.py | 20 ++++++++++++++++++++ 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbad57de..b6fbb0dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Fixes -- **Preserve rows in multi-file CSV responses**: CSV output now concatenates each file's partition results in request order without using filenames as dictionary keys, so duplicate filenames cannot overwrite earlier rows. +- **Preserve rows in multi-file CSV responses**: CSV output for a multi-file request was produced by outer-merging each file's DataFrame into the previous one, which collapsed identical rows across files into a single row and dropped legitimate duplicates. Results are now concatenated in request order, so every row from every file is retained, differing columns are unioned, and the row index is no longer emitted as a leading unnamed column. A file that produces no elements contributes no rows instead of failing the whole request. ## 0.1.8 diff --git a/prepline_general/api/general.py b/prepline_general/api/general.py index 6f616f90..9538db40 100644 --- a/prepline_general/api/general.py +++ b/prepline_general/api/general.py @@ -742,12 +742,16 @@ def join_responses( if form_params.output_format != "text/csv": return cast(List[Union[str, List[Dict[str, Any]]]], responses) responses = cast(List[PlainTextResponse], responses) - data = pd.concat( # pyright: ignore[reportUnknownMemberType] - [ - pd.read_csv(io.BytesIO(response.body)) # pyright: ignore[reportUnknownMemberType] - for response in responses - ] - ) + # -- a document that produced no elements serializes to a bodiless CSV, which pandas + # -- cannot parse; it contributes no rows -- + frames = [ + pd.read_csv(io.BytesIO(response.body)) # pyright: ignore[reportUnknownMemberType] + for response in responses + if response.body.strip() + ] + if not frames: + return PlainTextResponse(responses[0].body) + data = pd.concat(frames) # pyright: ignore[reportUnknownMemberType] return PlainTextResponse(data.to_csv(index=False)) return ( diff --git a/test_general/api/test_app.py b/test_general/api/test_app.py index 1baaf4a0..4aa55d23 100644 --- a/test_general/api/test_app.py +++ b/test_general/api/test_app.py @@ -1174,6 +1174,26 @@ def test_output_format_csv_concatenates_multiple_files_without_index_column(): assert "Unnamed: 0" not in df.columns +def test_output_format_csv_keeps_rows_when_one_file_has_no_elements(): + client = TestClient(app) + test_file = Path("sample-docs") / "family-day.eml" + + with open(test_file, "rb") as doc: + response = client.post( + MAIN_API_ROUTE, + files=[ + ("files", ("empty.txt", io.BytesIO(b""), "text/plain")), + ("files", (str(test_file), doc, "message/rfc822")), + ], + data={"output_format": "text/csv"}, + ) + + assert response.status_code == 200 + df = pd.read_csv(io.StringIO(response.text)) + assert len(df) == 9 + assert df["text"][3] == "Make sure to RSVP!" + + @pytest.mark.parametrize( "pdf_infer_table_structure, strategy, skip_infer_table_types, expected", [ From 56096ec0d3c07cba4790ed77f83341792700cf65 Mon Sep 17 00:00:00 2001 From: Nick Franck <46548427+CyMule@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:06:56 -0400 Subject: [PATCH 3/5] test: isolate non-retryable API call behavior --- test_general/api/test_app.py | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/test_general/api/test_app.py b/test_general/api/test_app.py index 4aa55d23..bd4bbed0 100644 --- a/test_general/api/test_app.py +++ b/test_general/api/test_app.py @@ -876,33 +876,24 @@ def mock_response(*args, **kwargs): assert response.status_code == 200 -def test_partition_file_via_api_not_retryable_error_code(monkeypatch, mocker): +def test_call_api_not_retryable_error_code(monkeypatch): """ Verify we didn't retry if the error code is not retryable """ - monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_ENABLED", "true") - monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_URL", "unused") - monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_THREADS", "1") - monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_RETRY_ATTEMPTS", "3") - - remote_partition = Mock(side_effect=HTTPException(status_code=401)) - - monkeypatch.setattr( - requests, - "post", - remote_partition, - ) - client = TestClient(app) - test_file = Path("sample-docs") / "list-item-example.pdf" - - response = client.post( - MAIN_API_ROUTE, - files=[("files", (str(test_file), open(test_file, "rb"), "application/pdf"))], - ) - - assert response.status_code == 401 + remote_partition = Mock(return_value=MockResponse(status_code=401)) + monkeypatch.setattr(requests, "post", remote_partition) + + with pytest.raises(HTTPException) as exc_info: + general.call_api( + request_url="unused", + api_key="", + filename="test.pdf", + file=io.BytesIO(b"test"), + content_type="application/pdf", + ) # no retries for non-retryable status codes + assert exc_info.value.status_code == 401 assert remote_partition.call_count == 1 From 5ac5505f920af2216a76c49d0213459139015871 Mon Sep 17 00:00:00 2001 From: Nick Franck <46548427+CyMule@users.noreply.github.com> Date: Mon, 20 Jul 2026 20:45:45 -0400 Subject: [PATCH 4/5] Revert test refactor unrelated to CSV fix --- test_general/api/test_app.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/test_general/api/test_app.py b/test_general/api/test_app.py index bd4bbed0..4aa55d23 100644 --- a/test_general/api/test_app.py +++ b/test_general/api/test_app.py @@ -876,24 +876,33 @@ def mock_response(*args, **kwargs): assert response.status_code == 200 -def test_call_api_not_retryable_error_code(monkeypatch): +def test_partition_file_via_api_not_retryable_error_code(monkeypatch, mocker): """ Verify we didn't retry if the error code is not retryable """ - remote_partition = Mock(return_value=MockResponse(status_code=401)) - monkeypatch.setattr(requests, "post", remote_partition) - - with pytest.raises(HTTPException) as exc_info: - general.call_api( - request_url="unused", - api_key="", - filename="test.pdf", - file=io.BytesIO(b"test"), - content_type="application/pdf", - ) + monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_ENABLED", "true") + monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_URL", "unused") + monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_THREADS", "1") + monkeypatch.setenv("UNSTRUCTURED_PARALLEL_MODE_RETRY_ATTEMPTS", "3") + + remote_partition = Mock(side_effect=HTTPException(status_code=401)) + + monkeypatch.setattr( + requests, + "post", + remote_partition, + ) + client = TestClient(app) + test_file = Path("sample-docs") / "list-item-example.pdf" + + response = client.post( + MAIN_API_ROUTE, + files=[("files", (str(test_file), open(test_file, "rb"), "application/pdf"))], + ) + + assert response.status_code == 401 # no retries for non-retryable status codes - assert exc_info.value.status_code == 401 assert remote_partition.call_count == 1 From 99ff7c436059a7c0358391df524aad3d5ed2a1d4 Mon Sep 17 00:00:00 2001 From: Nick Franck <46548427+CyMule@users.noreply.github.com> Date: Wed, 22 Jul 2026 22:40:22 -0400 Subject: [PATCH 5/5] test(csv): cover differing columns across files --- test_general/api/test_app.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test_general/api/test_app.py b/test_general/api/test_app.py index 4aa55d23..69dc7d04 100644 --- a/test_general/api/test_app.py +++ b/test_general/api/test_app.py @@ -1174,6 +1174,29 @@ def test_output_format_csv_concatenates_multiple_files_without_index_column(): assert "Unnamed: 0" not in df.columns +def test_output_format_csv_unions_columns_from_multiple_files(): + client = TestClient(app) + email_file = Path("sample-docs") / "family-day.eml" + text_file = Path("sample-docs") / "fake-text.txt" + + with open(email_file, "rb") as email, open(text_file, "rb") as text: + response = client.post( + MAIN_API_ROUTE, + files=[ + ("files", (email_file.name, email, "message/rfc822")), + ("files", (text_file.name, text, "text/plain")), + ], + data={"output_format": "text/csv"}, + ) + + assert response.status_code == 200 + df = pd.read_csv(io.StringIO(response.text)) + assert len(df) == 14 + assert {"sent_from", "parent_id"}.issubset(df.columns) + assert df.loc[df["filename"] == email_file.name, "parent_id"].isna().all() + assert df.loc[df["filename"] == text_file.name, "sent_from"].isna().all() + + def test_output_format_csv_keeps_rows_when_one_file_has_no_elements(): client = TestClient(app) test_file = Path("sample-docs") / "family-day.eml"