diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ebc78bf..e45c1498 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.1.10 + +### Fixes + +- **Preserve rows in multi-file CSV responses**: CSV output for a multi-file request was produced by outer-merging each file's DataFrame into the previous one, which collapsed identical rows across files into a single row and dropped legitimate duplicates. Results are now concatenated in request order, so every row from every file is retained, differing columns are unioned, and the row index is no longer emitted as a leading unnamed column. A file that produces no elements contributes no rows instead of failing the whole request. + ## 0.1.9 ### Security diff --git a/prepline_general/api/__version__.py b/prepline_general/api/__version__.py index 4eeb8fc3..f15b0cd2 100644 --- a/prepline_general/api/__version__.py +++ b/prepline_general/api/__version__.py @@ -1 +1 @@ -__version__ = "0.1.9" # pragma: no cover +__version__ = "0.1.10" # pragma: no cover diff --git a/prepline_general/api/general.py b/prepline_general/api/general.py index 2b4a46e8..df1b67ec 100644 --- a/prepline_general/api/general.py +++ b/prepline_general/api/general.py @@ -742,18 +742,17 @@ def join_responses( if form_params.output_format != "text/csv": return cast(List[Union[str, List[Dict[str, Any]]]], responses) responses = cast(List[PlainTextResponse], responses) - data = pd.read_csv( # pyright: ignore[reportUnknownMemberType] - io.BytesIO(responses[0].body) - ) - if len(responses) > 1: - for resp in responses[1:]: - resp_data = pd.read_csv( # pyright: ignore[reportUnknownMemberType] - io.BytesIO(resp.body) - ) - data = data.merge( # pyright: ignore[reportUnknownMemberType] - resp_data, how="outer" - ) - return PlainTextResponse(data.to_csv()) + # -- a document that produced no elements serializes to a bodiless CSV, which pandas + # -- cannot parse; it contributes no rows -- + frames = [ + pd.read_csv(io.BytesIO(response.body)) # pyright: ignore[reportUnknownMemberType] + for response in responses + if response.body.strip() + ] + if not frames: + return PlainTextResponse(responses[0].body) + data = pd.concat(frames) # pyright: ignore[reportUnknownMemberType] + return PlainTextResponse(data.to_csv(index=False)) return ( MultipartMixedResponse( diff --git a/test_general/api/test_app.py b/test_general/api/test_app.py index 9f76207d..6a1f8bc0 100644 --- a/test_general/api/test_app.py +++ b/test_general/api/test_app.py @@ -1156,6 +1156,69 @@ def test_output_format_csv_ignore_specified_accept_header(): assert df["text"][3] == "Make sure to RSVP!" +def test_output_format_csv_concatenates_multiple_files_without_index_column(): + client = TestClient(app) + test_file = Path("sample-docs") / "family-day.eml" + + with open(test_file, "rb") as first, open(test_file, "rb") as second: + response = client.post( + MAIN_API_ROUTE, + files=[ + ("files", ("same-name.eml", first, "message/rfc822")), + ("files", ("same-name.eml", second, "message/rfc822")), + ], + data={"output_format": "text/csv"}, + ) + + assert response.status_code == 200 + df = pd.read_csv(io.StringIO(response.text)) + assert len(df) == 18 + assert "Unnamed: 0" not in df.columns + + +def test_output_format_csv_unions_columns_from_multiple_files(): + client = TestClient(app) + email_file = Path("sample-docs") / "family-day.eml" + text_file = Path("sample-docs") / "fake-text.txt" + + with open(email_file, "rb") as email, open(text_file, "rb") as text: + response = client.post( + MAIN_API_ROUTE, + files=[ + ("files", (email_file.name, email, "message/rfc822")), + ("files", (text_file.name, text, "text/plain")), + ], + data={"output_format": "text/csv"}, + ) + + assert response.status_code == 200 + df = pd.read_csv(io.StringIO(response.text)) + assert len(df) == 14 + assert {"sent_from", "parent_id"}.issubset(df.columns) + assert df.loc[df["filename"] == email_file.name, "parent_id"].isna().all() + assert df.loc[df["filename"] == text_file.name, "sent_from"].isna().all() + + +def test_output_format_csv_keeps_rows_when_one_file_has_no_elements(): + client = TestClient(app) + test_file = Path("sample-docs") / "family-day.eml" + + with open(test_file, "rb") as doc: + response = client.post( + MAIN_API_ROUTE, + files=[ + ("files", ("empty.txt", io.BytesIO(b""), "text/plain")), + ("files", (str(test_file), doc, "message/rfc822")), + ], + data={"output_format": "text/csv"}, + ) + + assert response.status_code == 200 + df = pd.read_csv(io.StringIO(response.text)) + assert len(df) == 9 + assert df["text"][3] == "Make sure to RSVP!" + + @pytest.mark.parametrize( "pdf_infer_table_structure, strategy, skip_infer_table_types, expected", [