Preserve every row in multi-file CSV responses

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.1.10
+
+### Fixes
+
+- **Preserve rows in multi-file CSV responses**: CSV output for a multi-file request was produced by outer-merging each file's DataFrame into the previous one, which collapsed identical rows across files into a single row and dropped legitimate duplicates. Results are now concatenated in request order, so every row from every file is retained, differing columns are unioned, and the row index is no longer emitted as a leading unnamed column. A file that produces no elements contributes no rows instead of failing the whole request.
+
 ## 0.1.9
 
 ### Security

prepline_general/api/__version__.py

@@ -1 +1 @@
-__version__ = "0.1.9"  # pragma: no cover
+__version__ = "0.1.10"  # pragma: no cover

prepline_general/api/general.py

@@ -742,18 +742,17 @@ def join_responses(
         if form_params.output_format != "text/csv":
             return cast(List[Union[str, List[Dict[str, Any]]]], responses)
         responses = cast(List[PlainTextResponse], responses)
-        data = pd.read_csv(  # pyright: ignore[reportUnknownMemberType]
-            io.BytesIO(responses[0].body)
-        )
-        if len(responses) > 1:
-            for resp in responses[1:]:
-                resp_data = pd.read_csv(  # pyright: ignore[reportUnknownMemberType]
-                    io.BytesIO(resp.body)
-                )
-                data = data.merge(  # pyright: ignore[reportUnknownMemberType]
-                    resp_data, how="outer"
-                )
-        return PlainTextResponse(data.to_csv())
+        # -- a document that produced no elements serializes to a bodiless CSV, which pandas
+        # -- cannot parse; it contributes no rows --
+        frames = [
+            pd.read_csv(io.BytesIO(response.body))  # pyright: ignore[reportUnknownMemberType]
+            for response in responses
+            if response.body.strip()
+        ]
+        if not frames:
+            return PlainTextResponse(responses[0].body)
+        data = pd.concat(frames)  # pyright: ignore[reportUnknownMemberType]
+        return PlainTextResponse(data.to_csv(index=False))
 
     return (
         MultipartMixedResponse(

test_general/api/test_app.py

@@ -1156,6 +1156,69 @@ def test_output_format_csv_ignore_specified_accept_header():
     assert df["text"][3] == "Make sure to RSVP!"
 
 
+def test_output_format_csv_concatenates_multiple_files_without_index_column():
+    client = TestClient(app)
+    test_file = Path("sample-docs") / "family-day.eml"
+
+    with open(test_file, "rb") as first, open(test_file, "rb") as second:
+        response = client.post(
+            MAIN_API_ROUTE,
+            files=[
+                ("files", ("same-name.eml", first, "message/rfc822")),
+                ("files", ("same-name.eml", second, "message/rfc822")),
+            ],
+            data={"output_format": "text/csv"},
+        )
+
+    assert response.status_code == 200
+    df = pd.read_csv(io.StringIO(response.text))
+    assert len(df) == 18
+    assert "Unnamed: 0" not in df.columns
+
+
+def test_output_format_csv_unions_columns_from_multiple_files():
+    client = TestClient(app)
+    email_file = Path("sample-docs") / "family-day.eml"
+    text_file = Path("sample-docs") / "fake-text.txt"
+
+    with open(email_file, "rb") as email, open(text_file, "rb") as text:
+        response = client.post(
+            MAIN_API_ROUTE,
+            files=[
+                ("files", (email_file.name, email, "message/rfc822")),
+                ("files", (text_file.name, text, "text/plain")),
+            ],
+            data={"output_format": "text/csv"},
+        )
+
+    assert response.status_code == 200
+    df = pd.read_csv(io.StringIO(response.text))
+    assert len(df) == 14
+    assert {"sent_from", "parent_id"}.issubset(df.columns)
+    assert df.loc[df["filename"] == email_file.name, "parent_id"].isna().all()
+    assert df.loc[df["filename"] == text_file.name, "sent_from"].isna().all()
+
+
+def test_output_format_csv_keeps_rows_when_one_file_has_no_elements():
+    client = TestClient(app)
+    test_file = Path("sample-docs") / "family-day.eml"
+
+    with open(test_file, "rb") as doc:
+        response = client.post(
+            MAIN_API_ROUTE,
+            files=[
+                ("files", ("empty.txt", io.BytesIO(b""), "text/plain")),
+                ("files", (str(test_file), doc, "message/rfc822")),
+            ],
+            data={"output_format": "text/csv"},
+        )
+
+    assert response.status_code == 200
+    df = pd.read_csv(io.StringIO(response.text))
+    assert len(df) == 9
+    assert df["text"][3] == "Make sure to RSVP!"
+
+
 @pytest.mark.parametrize(
     "pdf_infer_table_structure, strategy, skip_infer_table_types, expected",
     [