createHarness

createHarness<TInput, TOutput, TMetadata>(options): Harness<TInput, TOutput, TMetadata>

Creates a harness from the common “run app code and return output” shape.

Type Parameters

TInput

TInput = unknown

TOutput

TOutput extends JsonValue | undefined = JsonValue | undefined

TMetadata

TMetadata extends HarnessMetadata = HarnessMetadata

Parameters

options

CreateHarnessOptions<TInput, TOutput, TMetadata>

Harness name plus the callback that executes app code.

Returns

Harness<TInput, TOutput, TMetadata>

Example

import { createHarness } from "vitest-evals";

export const refundHarness = createHarness<
  string,
  { status: "approved" | "denied" },
  { expected: { status: "approved" | "denied" } }
>({
  name: "refund-agent",
  run: async ({ input, metadata, setArtifact }) => {
    const result = await runRefundFlow(input, metadata);
    const output = { status: result.status };

    setArtifact("case", { expected: metadata.expected.status });

    return {
      output,
      toolCalls: result.toolCalls,
      usage: { provider: "openai", model: "gpt-4o-mini" },
    };
  },
});