diff --git a/sdk/ai/azure-ai-projects/.env.template b/sdk/ai/azure-ai-projects/.env.template index 2a61f80e8ef6..b2a361e3f92e 100644 --- a/sdk/ai/azure-ai-projects/.env.template +++ b/sdk/ai/azure-ai-projects/.env.template @@ -17,7 +17,7 @@ # `https://.services.ai.azure.com/api/projects/` AZURE_AI_PROJECT_ENDPOINT= AZURE_AI_MODEL_DEPLOYMENT_NAME= -AGENT_NAME= +AZURE_AI_AGENT_NAME= CONVERSATION_ID= CONNECTION_NAME= AZURE_AI_PROJECTS_AZURE_SUBSCRIPTION_ID= diff --git a/sdk/ai/azure-ai-projects/README.md b/sdk/ai/azure-ai-projects/README.md index e546e9d64c67..b623eae7e8d1 100644 --- a/sdk/ai/azure-ai-projects/README.md +++ b/sdk/ai/azure-ai-projects/README.md @@ -632,7 +632,7 @@ with ( agent = project_client.agents.create_version( agent_name=os.environ["AZURE_AI_AGENT_NAME"], definition=PromptAgentDefinition( - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=model_deployment_name, instructions="You are a helpful assistant that answers general questions", ), ) @@ -643,13 +643,30 @@ with ( item_schema={"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}, include_sample_schema=True, ) + # Notes: for data_mapping: + # sample.output_text is the string output of the agent + # sample.output_items is the structured JSON output of the agent, including tool calls information testing_criteria = [ { "type": "azure_ai_evaluator", "name": "violence_detection", "evaluator_name": "builtin.violence", - "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"}, - } + "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_text}}"}, + }, + { + "type": "azure_ai_evaluator", + "name": "fluency", + "evaluator_name": "builtin.fluency", + "initialization_parameters": {"deployment_name": f"{model_deployment_name}"}, + "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_text}}"}, + }, + { + "type": "azure_ai_evaluator", + "name": "task_adherence", + "evaluator_name": "builtin.task_adherence", + "initialization_parameters": {"deployment_name": f"{model_deployment_name}"}, + "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_items}}"}, + }, ] eval_object = openai_client.evals.create( name="Agent Evaluation", diff --git a/sdk/ai/azure-ai-projects/samples/agents/sample_agent_retrieve_basic.py b/sdk/ai/azure-ai-projects/samples/agents/sample_agent_retrieve_basic.py index a04bc7c64fe8..1102d326b2f3 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/sample_agent_retrieve_basic.py +++ b/sdk/ai/azure-ai-projects/samples/agents/sample_agent_retrieve_basic.py @@ -23,7 +23,7 @@ Set these environment variables with your own values: 1) AZURE_AI_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview page of your Microsoft Foundry portal. - 2) AGENT_NAME - The name of an existing Agent in your Microsoft Foundry project. + 2) AZURE_AI_AGENT_NAME - The name of an existing Agent in your Microsoft Foundry project. 3) CONVERSATION_ID - The ID of an existing Conversation associated with the Agent """ @@ -34,7 +34,7 @@ load_dotenv() -agent_name = os.environ["AGENT_NAME"] +agent_name = os.environ["AZURE_AI_AGENT_NAME"] conversation_id = os.environ["CONVERSATION_ID"] endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] diff --git a/sdk/ai/azure-ai-projects/samples/agents/sample_agent_retrieve_basic_async.py b/sdk/ai/azure-ai-projects/samples/agents/sample_agent_retrieve_basic_async.py index 16988819741c..5cae332aa01f 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/sample_agent_retrieve_basic_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/sample_agent_retrieve_basic_async.py @@ -23,7 +23,7 @@ Set these environment variables with your own values: 1) AZURE_AI_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview page of your Microsoft Foundry portal. - 2) AGENT_NAME - The name of an existing Agent in your Microsoft Foundry project. + 2) AZURE_AI_AGENT_NAME - The name of an existing Agent in your Microsoft Foundry project. 3) CONVERSATION_ID - The ID of an existing Conversation associated with the Agent """ @@ -36,7 +36,7 @@ load_dotenv() endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] -agent_name = os.environ["AGENT_NAME"] +agent_name = os.environ["AZURE_AI_AGENT_NAME"] conversation_id = os.environ["CONVERSATION_ID"] diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_evaluation.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_evaluation.py index 8dc67f31fa1d..af587d75b6b7 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_evaluation.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_evaluation.py @@ -40,6 +40,7 @@ load_dotenv() endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] +model_deployment_name = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "") # Sample : gpt-4o-mini # [START agent_evaluation_basic] with ( @@ -50,7 +51,7 @@ agent = project_client.agents.create_version( agent_name=os.environ["AZURE_AI_AGENT_NAME"], definition=PromptAgentDefinition( - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=model_deployment_name, instructions="You are a helpful assistant that answers general questions", ), ) @@ -61,13 +62,30 @@ item_schema={"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}, include_sample_schema=True, ) + # Notes: for data_mapping: + # sample.output_text is the string output of the agent + # sample.output_items is the structured JSON output of the agent, including tool calls information testing_criteria = [ { "type": "azure_ai_evaluator", "name": "violence_detection", "evaluator_name": "builtin.violence", - "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"}, - } + "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_text}}"}, + }, + { + "type": "azure_ai_evaluator", + "name": "fluency", + "evaluator_name": "builtin.fluency", + "initialization_parameters": {"deployment_name": f"{model_deployment_name}"}, + "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_text}}"}, + }, + { + "type": "azure_ai_evaluator", + "name": "task_adherence", + "evaluator_name": "builtin.task_adherence", + "initialization_parameters": {"deployment_name": f"{model_deployment_name}"}, + "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_items}}"}, + }, ] eval_object = openai_client.evals.create( name="Agent Evaluation", diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_evaluation.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_evaluation.py index 5fb80829e4db..91e45ecf3355 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_evaluation.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_model_evaluation.py @@ -51,16 +51,18 @@ item_schema={"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}, include_sample_schema=True, ) + # Notes: for data_mapping: + # {{sample.output_text}} is the string output of the provide model target for the given input in {{item.query}} testing_criteria = [ { "type": "azure_ai_evaluator", "name": "violence_detection", "evaluator_name": "builtin.violence", - "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"}, + "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_text}}"}, } ] eval_object = openai_client.evals.create( - name="Agent Evaluation", + name="Model Evaluation", data_source_config=data_source_config, testing_criteria=testing_criteria, # type: ignore )