Overview

# evals/weather_agent.eval.py
from agents import Agent, Runner, function_tool
from promptlayer import evaluate, contains_scorer, trajectory_scorer
from promptlayer.integrations.openai_agents import instrument_openai_agents

instrument_openai_agents()

@function_tool
def get_weather(city: str) -> str:
    """Return demo weather for a city."""
    return f"{city} is 72F and sunny."

agent = Agent(
    name="Weather agent",
    instructions="Use get_weather, then answer in one short sentence.",
    model="gpt-5.6",
    tools=[get_weather],
)

def run_agent(user_message: str) -> str:
    result = Runner.run_sync(agent, user_message)
    return str(result.final_output)

evaluate(
    "weather-agent-eval",
    dataset=[{"input": "What is the weather in Tokyo?"}],
    runner=run_agent,
    scorers=[
        trajectory_scorer(
            accepted_scenarios=[["get_weather"]],
            mode="non_strict",
        ),
        contains_scorer(source="Output", value="72"),
    ],
    passing_score=1.0,
)

// evals/weather_agent.eval.ts
import { Agent, run, tool } from "@openai/agents";
import { instrumentOpenAIAgents } from "promptlayer/openai-agents";
import { evaluate, containsScorer, trajectoryScorer } from "promptlayer";
import { z } from "zod";

await instrumentOpenAIAgents();

const getWeather = tool({
  name: "get_weather",
  description: "Return demo weather for a city.",
  parameters: z.object({
    city: z.string(),
  }),
  execute: async ({ city }) => `${city} is 72F and sunny.`,
});

const agent = new Agent({
  name: "Weather agent",
  instructions: "Use get_weather, then answer in one short sentence.",
  model: "gpt-5.6",
  tools: [getWeather],
});

async function runAgent(userMessage) {
  const result = await run(agent, userMessage);
  return String(result.finalOutput);
}

await evaluate("weather-agent-eval", {
  dataset: [{ input: "What is the weather in Tokyo?" }],
  runner: runAgent,
  scorers: [
    trajectoryScorer({
      acceptedScenarios: [["get_weather"]],
      mode: "non_strict",
    }),
    containsScorer({ source: "Output", value: "72" }),
  ],
  passingScore: 1.0,
});