Evaluation

from tenxgraph.qa.evaluation import (
    AgentEvaluator,
    EvalConfig,
    CriteriaConfig,
    CriterionConfig,
    MatchType,
    Rubric,
    EvalSet,
    EvalSetBuilder,
    EvalCase,
    Invocation,
    MessageContent,
    SessionInput,
    ToolCall,
    TrajectoryStep,
    StepType,
    EvalReport,
    EvalCaseResult,
    CriterionResult,
    TrajectoryCollector,
    make_trajectory_callback,
)

from tenxgraph.qa.evaluation import AgentEvaluator, create_eval_app

app, collector = create_eval_app(build_my_graph())   # uncompiled StateGraph in
evaluator = AgentEvaluator(app, collector)

from tenxgraph.qa.evaluation import TrajectoryCollector, make_trajectory_callback

collector = TrajectoryCollector(capture_all_events=True)
_, callback_mgr = make_trajectory_callback(collector, config={"thread_id": "eval-1"})

app = my_state_graph.compile(callback_manager=callback_mgr)

from tenxgraph.qa.evaluation import EvalCase, ToolCall

case = EvalCase.single_turn(
    eval_id="weather-paris-001",
    user_query="What is the weather in Paris today?",
    expected_response="It is sunny in Paris.",
    expected_tools=[ToolCall(name="get_weather", args={"location": "Paris"})],
    expected_node_order=["MAIN", "TOOL", "MAIN"],
    name="Paris weather",
)

case = EvalCase.multi_turn(
    eval_id="multi-turn-001",
    conversation=[
        ("Hello", "Hi"),
        ("What can you do?", "I can answer questions"),
        ("Tell me about Python", "Python is a programming language"),
    ],
    expected_tools=[ToolCall(name="search_docs")],
)

from tenxgraph.qa.evaluation import ToolCall

tc = ToolCall(
    name="get_weather",
    args={"location": "Paris", "units": "celsius"},
    call_id=None,   # optional
    result=None,    # populated on actual calls
)

tc.matches(other, check_args=True, check_call_id=False)  # -> bool

from tenxgraph.qa.evaluation import TrajectoryStep

node_step = TrajectoryStep.node("RESEARCH_NODE")
tool_step = TrajectoryStep.tool("search", args={"query": "AI trends"})

from tenxgraph.qa.evaluation import EvalSet

eval_set = EvalSet(eval_set_id="capitals", name="Capitals", eval_cases=[case1, case2])

from tenxgraph.qa.evaluation import (
    CriteriaConfig,
    CriterionConfig,
    EvalConfig,
    MatchType,
)

config = EvalConfig(
    criteria=CriteriaConfig(
        tool_name_match=CriterionConfig.tool_name_match(threshold=1.0),
        trajectory=CriterionConfig.trajectory(match_type=MatchType.IN_ORDER),
        llm_judge=CriterionConfig.llm_judge(),
    ),
    parallel=True,
    max_concurrency=4,
)

from tenxgraph.qa.evaluation import Rubric

rubric = Rubric(
    rubric_id="accuracy",
    content="The response must contain the correct capital city.",
    weight=1.0,
)

from tenxgraph.qa.evaluation import TrajectoryCollector, make_trajectory_callback

collector = TrajectoryCollector(capture_all_events=True)
_, callback_mgr = make_trajectory_callback(collector)

app = graph.compile(callback_manager=callback_mgr)
await app.ainvoke({"messages": [...]})

for step in collector.trajectory:
    print(step.step_type, step.name)

def make_trajectory_callback(
    collector: TrajectoryCollector,
    config: dict | None = None,
) -> tuple[TrajectoryCollector, CallbackManager]

from tenxgraph.qa.evaluation import AgentEvaluator, EvalConfig

evaluator = AgentEvaluator(app, collector, config=EvalConfig.default())
report = await evaluator.evaluate(eval_set)
print(report.format_summary())

async def evaluate(
    eval_set: EvalSet | str,
    parallel: bool = False,
    max_concurrency: int = 4,
    verbose: bool = False,
    output_dir: str | None = None,
) -> EvalReport

from tenxgraph.qa.evaluation import (
    ConsoleReporter,
    HTMLReporter,
    JSONReporter,
    JUnitXMLReporter,
    print_report,
)

print_report(report, verbose=False, use_color=True)   # console shortcut
ConsoleReporter(verbose=True).report(report)          # console, full control

JSONReporter(indent=2).save(report, "eval-report.json")
HTMLReporter().save(report, "eval-report.html")
JUnitXMLReporter().save(report, "eval-report_junit.xml")

from tenxgraph.qa.evaluation.config.types import DEFAULT_JUDGE_MODEL
# "gemini-2.5-flash"

import asyncio

from tenxgraph.qa.evaluation import (
    AgentEvaluator,
    CriteriaConfig,
    CriterionConfig,
    EvalCase,
    EvalConfig,
    EvalSet,
    MatchType,
    ToolCall,
    create_eval_app,
)

from my_project.graph import build_graph   # returns an uncompiled StateGraph

async def main():
    # 1. Compile with a collector attached
    app, collector = create_eval_app(build_graph())

    # 2. Build the eval set
    eval_set = EvalSet(
        eval_set_id="capitals",
        name="Capital cities",
        eval_cases=[
            EvalCase.single_turn(
                eval_id="capitals-001",
                user_query="What is the capital of France?",
                expected_response="The capital of France is Paris.",
                expected_tools=[ToolCall(name="lookup_capital")],
            ),
            EvalCase.single_turn(
                eval_id="capitals-002",
                user_query="What is the capital of Germany?",
                expected_response="The capital of Germany is Berlin.",
                expected_tools=[ToolCall(name="lookup_capital")],
            ),
        ],
    )

    # 3. Configure criteria
    config = EvalConfig(
        criteria=CriteriaConfig(
            trajectory=CriterionConfig.trajectory(
                threshold=1.0,
                match_type=MatchType.IN_ORDER,
            ),
            response_match=CriterionConfig.response_match(threshold=0.8),
        ),
    )

    # 4. Run
    evaluator = AgentEvaluator(app, collector, config=config)
    report = await evaluator.evaluate(eval_set, verbose=True)

    # 5. Inspect
    print(report.format_summary())
    print(f"Pass rate: {report.summary.pass_rate:.0%}")
    for case in report.failed_cases:
        for cr in case.failed_criteria:
            print(f"{case.eval_id} · {cr.criterion}: {cr.score:.2f} < {cr.threshold}")

asyncio.run(main())