diff --git a/corrective-rag/workflow.py b/corrective-rag/workflow.py index a167e4930..ca4821d69 100644 --- a/corrective-rag/workflow.py +++ b/corrective-rag/workflow.py @@ -127,13 +127,21 @@ async def eval_relevance( relevancy = self.llm.complete(prompt) relevancy_results.append(relevancy.text.lower().strip()) + # The grader is asked for a binary 'yes'/'no', but LLMs often add + # punctuation or a short justification (e.g. "No, ..."). Match on the + # leading token so the decision is robust instead of requiring the result + # to be exactly "yes"/"no". relevant_texts = [ retrieved_nodes[i].text for i, result in enumerate(relevancy_results) - if result == "yes" + if result.startswith("yes") ] relevant_text = "\n".join(relevant_texts) - if "no" in relevancy_results: + # Trigger a corrective web search whenever at least one retrieved document + # was judged not relevant (anything that isn't a clear "yes"). The previous + # `"no" in relevancy_results` only matched a result that was exactly "no", + # so answers like "No, ..." or "no." silently skipped the web search. + if any(not result.startswith("yes") for result in relevancy_results): return WebSearchEvent(relevant_text=relevant_text) else: return QueryEvent(relevant_text=relevant_text, search_text="") diff --git a/firecrawl-agent/workflow.py b/firecrawl-agent/workflow.py index 5c85ae6ac..7d0a93d4e 100644 --- a/firecrawl-agent/workflow.py +++ b/firecrawl-agent/workflow.py @@ -180,7 +180,12 @@ async def eval_relevance( print(f"DEBUG: Relevant texts count: {len(relevant_texts)}") print(f"DEBUG: Relevant text preview: {relevant_text[:200]}...") - if "no" in relevancy_results_striped: + # A document is treated as relevant when its grade contains "yes" (see the + # filter above), so trigger a corrective web search whenever any document is + # *not* relevant. The previous `"no" in relevancy_results_striped` only + # matched a grade equal to exactly "no", so answers like "No, ..." or "no." + # silently skipped the web search. + if any("yes" not in result.lower() for result in relevancy_results_striped): print("DEBUG: Some documents irrelevant, returning WebSearchEvent") return WebSearchEvent(relevant_text=relevant_text) else: