Reference overview

pip install whileai   # or: uv add whileai

import whileai.simulations as wai

# 1. Your tools, in OpenAI function-calling shape. This is all `tools=` wants.
TOOLS = [
    {
        "type": "function",
        "function": {
            "name": "get_order",
            "description": "Look up an order by id.",
            "parameters": {
                "type": "object",
                "properties": {"order_id": {"type": "string"}},
                "required": ["order_id"],
            },
        },
    }
]


# 2. Your agent: one call per rollout, in with the situation text,
#    out with the steps it took and what it finally said.
def my_agent(message: str) -> dict:
    return {
        "steps": [
            {
                "tool": "get_order",
                "arguments": {"order_id": "4412"},
                "result": {"status": "shipped"},
            }
        ],
        "final_text": "Order 4412 shipped yesterday.",
    }


# 3. simulator=False uses the built-in template writer: no model, no key.
data = wai.simulate(
    my_agent,
    tools=TOOLS,
    system_prompt="Help customers with orders.",
    simulator=False,
    budget=20,
)

# 4. Your grader. Any callable row -> {"reward": 0 or 1, ...}.
scored = data.grade(judge=lambda row: {"reward": int("4412" in row["final_text"])})
print(scored.pass_at)

pass@1 1.00 [1.00..1.00] | pass^2 (pass_pow_k) n/a | pass@2 n/a | headroom n/a (18 groups, k=2; set repeats>=4 for pass^k and pass@k)

WORLD = wai.world(TOOLS)


def my_agent(message: str) -> dict:
    result = WORLD.call(
        "get_order", {"order_id": "4412"}
    )  # timeouts, stale data, denials fire here
    return {
        "steps": [{"tool": "get_order", "arguments": {"order_id": "4412"}, "result": result}],
        "final_text": "Order 4412 shipped yesterday."
        if result.get("status") == "ok"
        else "The lookup did not go through, so I cannot confirm 4412 yet.",
    }

data = wai.simulate(
    wai.seeded_agent(TOOLS),
    tools=TOOLS,
    system_prompt="Help customers with orders.",
    simulator=False,
    budget=60,
)
rows = data.trajectories  # export_row scrubs privileged; the run keeps it
print(wai.style_report(rows)["markers"]["no_hedging"]["hits"])  # > 0, only on seeded rows
print(wai.format_leak_report(wai.leak_report(rows)))

4
checked 58 of 60 rows: 3 quoted privileged context (5%)
  sc-2f784fb823 r0: hidden_state.world_state = 'duplicate entity'
  sc-9ec7194fb6 r0: reference = 'unrelated ask: look if you like, change nothing'
  sc-4428cc9c78 r0: reference = 'two records match: look into it or ask which one before writing'

data = wai.simulate(
    agent,
    tools=TOOLS,
    system_prompt=POLICY,
    seeds=SEEDS,  # opening asks to start from, a list of strings
    simulator=False,
    mode="rl",
    repeats=4,
    repeat_policy="fixed",
)
scored = wai.evaluate(data, judge)  # eval lineage: never the reward
print(wai.pass_at(scored.rows), *scored.warnings)  # a hollow run says so here

export OPENAI_API_KEY=...
export OPENAI_BASE_URL=...   # only for a non-OpenAI endpoint

import whileai.simulations as wai

data = wai.simulate(
    agent="openai:gpt-4.1-mini",
    tools=my_tools,
    system_prompt=my_system_prompt,
    output="rollout.jsonl",
)

export ANTHROPIC_API_KEY=...

data = wai.simulate(
    agent="anthropic:claude-haiku-4-5",
    tools=my_tools,
    system_prompt=my_system_prompt,
    simulator="anthropic:claude-sonnet-5",  # the writer, on the same key
    output="rollout.jsonl",
)

export TYPESAFE_API_KEY=...

report = data.grade(spec="typesafe:jev-latest")
report["unsure"]  # rows whose verdict probability sat within 0.1 of even
data.trajectories[0]["judge_meta"]["confidence"]  # the probability of the verdict given
data.trajectories[0]["failure_class"]  # on a failing row: the judge's own choice

agent = wai.local_model(endpoint, name, tools=TOOLS, system=POLICY, thinking=False)
data = wai.simulate(tasks=pinned, agent=agent)

whileai login              # or: export WHILEAI_API_KEY=zp_...
export VLLM_API_KEY=...      # optional: the shared pool instead