createHarness

createHarness<TInput, TOutput>(options): Harness<TInput, TOutput>

Creates a harness from the common “run app code and return output” shape.

Type Parameters

TInput

TInput = unknown

TOutput

TOutput extends JsonValue | undefined = JsonValue | undefined

Parameters

options

CreateHarnessOptions<TInput, TOutput>

Harness name plus the callback that executes app code.

Returns

Harness<TInput, TOutput>

Example

import { createHarness } from "vitest-evals";

export const refundHarness = createHarness<
  string,
  { status: "approved" | "denied" }
>({
  name: "refund-agent",
  run: async ({ input, setArtifact }) => {
    const result = await runRefundFlow(input);
    const output = { status: result.status };

    setArtifact("case", { invoiceId: result.invoiceId });

    return {
      output,
      events: [
        { type: "message", role: "user", content: input },
        {
          type: "tool_call",
          id: "call_lookup",
          name: "lookupInvoice",
          arguments: { invoiceId: result.invoiceId },
        },
        {
          type: "tool_result",
          toolCallId: "call_lookup",
          name: "lookupInvoice",
          content: { refundable: result.refundable },
        },
        { type: "message", role: "assistant", content: output },
      ],
      usage: { provider: "openai", model: "gpt-4o-mini" },
    };
  },
});