Datasets

dataset = [
    {"input": "What is 2+2?", "expected": "4"},
    {
        "input": "What is the weather in Tokyo?",
        "expected": "72",
        "expected_trace": {
            "accepted_scenarios": [
                {"required_tools": ["get_weather"]}
            ]
        },
    },
]

const dataset = [
  { input: "What is 2+2?", expected: "4" },
  {
    input: "What is the weather in Tokyo?",
    expected: "72",
    expectedTrace: {
      accepted_scenarios: [{ required_tools: ["get_weather"] }],
    },
  },
];

from openai import OpenAI
from promptlayer import evaluate, contains_scorer

client = OpenAI()

def run_llm(user_message: str) -> str:
    completion = client.chat.completions.create(
        model="gpt-5.6",
        messages=[
            {"role": "system", "content": "Answer in one short sentence."},
            {"role": "user", "content": str(user_message)},
        ],
    )
    return completion.choices[0].message.content or ""

evaluate(
    "from-dashboard",
    dataset={"table_id": "00000000-0000-0000-0000-000000000000"},
    runner=run_llm,
    scorers=[contains_scorer(source="Output", value="reset")],
)

import OpenAI from "openai";
import { evaluate, containsScorer } from "promptlayer";

const client = new OpenAI();

async function runLlm(userMessage) {
  const completion = await client.chat.completions.create({
    model: "gpt-5.6",
    messages: [
      { role: "system", content: "Answer in one short sentence." },
      { role: "user", content: String(userMessage) },
    ],
  });
  return completion.choices[0].message.content ?? "";
}

await evaluate("from-dashboard", {
  dataset: { tableId: "00000000-0000-0000-0000-000000000000" },
  runner: runLlm,
  scorers: [containsScorer({ source: "Output", value: "reset" })],
});