diff --git a/flexeval/core/language_model/hf_lm.py b/flexeval/core/language_model/hf_lm.py index 46c7d636..e7ca2260 100644 --- a/flexeval/core/language_model/hf_lm.py +++ b/flexeval/core/language_model/hf_lm.py @@ -436,13 +436,17 @@ def _batch_generate_chat_response( if lm_output.raw_text is None: lm_output.raw_text = lm_output.text reasoning = self.reasoning_parser(lm_output.text) - lm_output.text = reasoning.text + # `reasoning.text` is None when the reasoning pattern fails to match. + # LMOutput.text must never be None (see LMOutput.__post_init__), so fall back to "". + lm_output.text = reasoning.text if reasoning.text is not None else "" lm_output.reasoning_text = reasoning.reasoning_text if self.tool_parser and tools is not None: parsed_tool_calling_message = self.tool_parser(lm_output.text) lm_output.tool_calls = parsed_tool_calling_message.tool_call_dicts - lm_output.text = parsed_tool_calling_message.text + lm_output.text = ( + parsed_tool_calling_message.text if parsed_tool_calling_message.text is not None else "" + ) if lm_output.raw_text is None: lm_output.raw_text = parsed_tool_calling_message.raw_text lm_output.tool_call_validation_result = parsed_tool_calling_message.validation_result diff --git a/flexeval/core/language_model/vllm_model.py b/flexeval/core/language_model/vllm_model.py index 7ac36da9..22c7abb8 100644 --- a/flexeval/core/language_model/vllm_model.py +++ b/flexeval/core/language_model/vllm_model.py @@ -284,13 +284,17 @@ def _batch_generate_chat_response( if lm_output.raw_text is None: lm_output.raw_text = lm_output.text reasoning = self.reasoning_parser(lm_output.text) - lm_output.text = reasoning.text + # `reasoning.text` is None when the reasoning pattern fails to match. + # LMOutput.text must never be None (see LMOutput.__post_init__), so fall back to "". + lm_output.text = reasoning.text if reasoning.text is not None else "" lm_output.reasoning_text = reasoning.reasoning_text if self.tool_parser and tools is not None: parsed_tool_calling_message = self.tool_parser(lm_output.text) lm_output.tool_calls = parsed_tool_calling_message.tool_call_dicts - lm_output.text = parsed_tool_calling_message.text + lm_output.text = ( + parsed_tool_calling_message.text if parsed_tool_calling_message.text is not None else "" + ) if lm_output.raw_text is None: lm_output.raw_text = parsed_tool_calling_message.raw_text lm_output.tool_call_validation_result = parsed_tool_calling_message.validation_result diff --git a/tests/core/language_model/test_hf_lm.py b/tests/core/language_model/test_hf_lm.py index 1ed6547c..e5ebb2d1 100644 --- a/tests/core/language_model/test_hf_lm.py +++ b/tests/core/language_model/test_hf_lm.py @@ -565,7 +565,7 @@ def test_reasoning_parser_handles_batch_chat_response(chat_lm: HuggingFaceLM) -> chat_lm.reasoning_parser = original_parser -def test_reasoning_parser_unmatched_pattern_sets_none(chat_lm: HuggingFaceLM) -> None: +def test_reasoning_parser_unmatched_pattern_sets_empty_text(chat_lm: HuggingFaceLM) -> None: reasoning_parser = UnifiedRegexReasoningParser(pattern=r"(?P.*?)(?P.*)") raw_output = "no think tags here" original_parser = chat_lm.reasoning_parser @@ -576,7 +576,28 @@ def test_reasoning_parser_unmatched_pattern_sets_none(chat_lm: HuggingFaceLM) -> ): response = chat_lm.generate_chat_response([{"role": "user", "content": "test"}], max_new_tokens=1) assert response.raw_text == raw_output - assert response.text is None + # LMOutput.text must never be None (see LMOutput.__post_init__), so a failed match falls back to "". + assert response.text == "" + assert response.reasoning_text is None + finally: + chat_lm.reasoning_parser = original_parser + + +def test_reasoning_parser_unclosed_think_tag_sets_empty_text(chat_lm: HuggingFaceLM) -> None: + reasoning_parser = UnifiedRegexReasoningParser(pattern=r"(?P.*?)(?P.*)") + # Simulates generation truncated (e.g. by max_new_tokens) before the closing tag. + raw_output = "reasoning that got cut off" + original_parser = chat_lm.reasoning_parser + chat_lm.reasoning_parser = reasoning_parser + try: + with patch.object( + chat_lm, "_batch_complete_text", return_value=[LMOutput(text=raw_output, finish_reason="length")] + ): + response = chat_lm.generate_chat_response([{"role": "user", "content": "test"}], max_new_tokens=1) + assert response.raw_text == raw_output + # The pattern requires a closing , so an unclosed tag fails to match just like no tags at all, + # and the unfinished reasoning must not leak into `text` (see LMOutput.__post_init__ for the None -> "" rule). + assert response.text == "" assert response.reasoning_text is None finally: chat_lm.reasoning_parser = original_parser diff --git a/tests/core/language_model/vllm/test_vllm_specific.py b/tests/core/language_model/vllm/test_vllm_specific.py index 26384ecf..0b104e20 100644 --- a/tests/core/language_model/vllm/test_vllm_specific.py +++ b/tests/core/language_model/vllm/test_vllm_specific.py @@ -314,6 +314,46 @@ def test_reasoning_parser_handles_batch_chat_response(chat_lm: VLLM) -> None: chat_lm.reasoning_parser = original_parser +@pytest.mark.skipif(not is_vllm_enabled(), reason="vllm library is not installed") +def test_reasoning_parser_unmatched_pattern_sets_empty_text(chat_lm: VLLM) -> None: + reasoning_parser = UnifiedRegexReasoningParser(pattern=r"(?P.*?)(?P.*)") + raw_output = "no think tags here" + original_parser = chat_lm.reasoning_parser + chat_lm.reasoning_parser = reasoning_parser + try: + with patch.object( + chat_lm, "_batch_complete_text", return_value=[LMOutput(text=raw_output, finish_reason="stop")] + ): + response = chat_lm.generate_chat_response([{"role": "user", "content": "test"}], max_new_tokens=1) + assert response.raw_text == raw_output + # LMOutput.text must never be None (see LMOutput.__post_init__), so a failed match falls back to "". + assert response.text == "" + assert response.reasoning_text is None + finally: + chat_lm.reasoning_parser = original_parser + + +@pytest.mark.skipif(not is_vllm_enabled(), reason="vllm library is not installed") +def test_reasoning_parser_unclosed_think_tag_sets_empty_text(chat_lm: VLLM) -> None: + reasoning_parser = UnifiedRegexReasoningParser(pattern=r"(?P.*?)(?P.*)") + # Simulates generation truncated (e.g. by max_new_tokens) before the closing tag. + raw_output = "reasoning that got cut off" + original_parser = chat_lm.reasoning_parser + chat_lm.reasoning_parser = reasoning_parser + try: + with patch.object( + chat_lm, "_batch_complete_text", return_value=[LMOutput(text=raw_output, finish_reason="length")] + ): + response = chat_lm.generate_chat_response([{"role": "user", "content": "test"}], max_new_tokens=1) + assert response.raw_text == raw_output + # The pattern requires a closing , so an unclosed tag fails to match just like no tags at all, + # and the unfinished reasoning must not leak into `text` (see LMOutput.__post_init__ for the None -> "" rule). + assert response.text == "" + assert response.reasoning_text is None + finally: + chat_lm.reasoning_parser = original_parser + + @pytest.mark.skipif(not is_vllm_enabled(), reason="vllm library is not installed") def test_no_reasoning_parser_leaves_chat_output_unchanged(chat_lm: VLLM) -> None: assert chat_lm.reasoning_parser is None