The harness

import whileai as wai

careful = wai.Harness("openai:gpt-4.1-mini", instructions=POLICY, tools=TOOLS, label="careful@mini")
print(careful.kind, careful.model_name, careful.tool_names)

prompted gpt-4.1-mini ['issue_refund', 'lookup_order']

from whileai.harness import Disclosure

capped = wai.Harness(
    "openai:gpt-4.1-mini",
    instructions=POLICY,
    tools=TOOLS,
    disclosure=Disclosure(max_turns=6),
)
print(capped.fingerprint == careful.fingerprint, capped.version.startswith("h-"))

False True

coder = wai.Harness.claude_code("sonnet", cwd=".", max_turns=8, tools=["Read", "Edit", "Bash"])
print(coder.kind, coder.tool_names, coder.disclosure.max_turns)

codex = wai.Harness.codex("gpt-5-codex", cwd=".", sandbox="workspace-write")
pi = wai.Harness.pi("claude-sonnet-4-5", provider="anthropic", cwd=".", extensions=False)

command ['Bash', 'Edit', 'Read'] 8

out = coder("Make the failing test in tests/ pass. Do not touch the test.")
print(out["final_text"], len(out["steps"]), "tool calls")

careful = wai.Harness(
    agent=careful_agent,
    instructions=POLICY,
    tools=TOOLS,
    label="careful@scripted",
    model="scripted",
)
data = wai.simulate(
    careful,
    seeds=REFUND_ASKS,
    situations=4,
    budget=8,
    simulator=False,
    mode="rl",
    repeats=2,
    repeat_policy="fixed",
    fault_rate=0.0,
    avg_turns=1,
)
row = data.rows()[0]
print(row["harness"]["label"], row["harness"]["kind"])

careful@scripted callable

rows = []
for harness in ("careful", "eager"):
    for model in ("gpt-4.1-mini", "claude-haiku-4-5"):
        for t in range(200):
            solved = (t % 4 != 0) if harness == "careful" else (t % 2 == 0)
            lift = model == "claude-haiku-4-5" and t % 10 == 1
            rows.append(
                {
                    "task_id": f"t{t}",
                    "reward": 1.0 if (solved or lift) else 0.0,
                    "harness": {"label": harness, "model": model},
                }
            )
print(wai.harness.attribute(rows))

attribution on pass_at_1: 2 harnesses x 2 models, 200 tasks each cell
  harness     claude-haiku-4-5      gpt-4.1-mini
  careful                 75.0              75.0
  eager                   60.0              50.0
  spread explained: harness 89% [61..97], model 6% [2..19], interaction 6%
  harness moves the score by up to 20.0 points, the model by up to 5.0
  the same model leads under every harness
  the harness moved the score more than the model did

print(wai.harness.attribute([r for r in rows if int(r["task_id"][1:]) < 40]))

attribution on pass_at_1: 2 harnesses x 2 models, 40 tasks each cell
  harness     claude-haiku-4-5      gpt-4.1-mini
  careful                 75.0              75.0
  eager                   60.0              50.0
  spread explained: harness 89% [2..100], model 6% [0..49], interaction 6%
  harness moves the score by up to 20.0 points, the model by up to 5.0
  the same model leads under every harness
  which lever moved the score more could be chance: the interval on the difference in shares covers zero

import json
import tempfile
from pathlib import Path

eager = wai.Harness(
    agent=eager_agent,
    instructions="Refund the order the customer names. Do not look it up first.",
    tools=TOOLS[1:],
    label="eager@scripted",
    model="scripted",
)
out = Path(tempfile.mkdtemp()) / "refunds"
wai.export_environment(
    data, out, tools=TOOLS, system_prompt=POLICY, band=None, harnesses=[careful, eager]
)
spec = json.loads((out / "refunds" / "spec.json").read_text())
by_label = {"careful@scripted": careful, "eager@scripted": eager}
for h in spec["harnesses"]:
    print(h["label"], len(h["tools"]), "tools", h["hash"] == by_label[h["label"]].fingerprint)
print("### Harnesses" in (out / "README.md").read_text())

careful@scripted 2 tools True
eager@scripted 1 tools True
True

from whileai.platform import track

tracked = track("refund-bot", model="gpt-4.1-mini", harness=careful)
run = tracked.run("careful@scripted", method="eval", targets=["refund_policy"], harness=careful)