tbench APItbench.ai ↗

"""
Terminal-Bench Leaderboard API Client
Get your API key from: https://parse.bot/settings
"""

import os
import requests
from typing import Optional, List, Dict, Any


class ParseClient:
    """Client for interacting with the Terminal-Bench Leaderboard API via Parse."""

    def __init__(self, api_key: Optional[str] = None):
        """
        Initialize the Parse API client.

        Args:
            api_key: API key for authentication. If not provided, reads from PARSE_API_KEY env var.
        """
        self.base_url = "https://api.parse.bot"
        self.scraper_id = "8c0968f8-4e4a-4ec7-97f3-48511c29d583"
        self.api_key = api_key or os.getenv("PARSE_API_KEY")

        if not self.api_key:
            raise ValueError(
                "API key not provided. Set PARSE_API_KEY environment variable or pass api_key parameter."
            )

    def _call(
        self, endpoint: str, method: str = "POST", **params
    ) -> Dict[str, Any]:
        """
        Make a request to the Parse API.

        Args:
            endpoint: The endpoint name to call
            method: HTTP method (GET or POST)
            **params: Parameters to pass to the endpoint

        Returns:
            Response data as dictionary

        Raises:
            requests.RequestException: If the API request fails
        """
        url = f"{self.base_url}/scraper/{self.scraper_id}/{endpoint}"
        headers = {"X-API-Key": self.api_key, "Content-Type": "application/json"}

        if method == "GET":
            response = requests.get(url, headers=headers, params=params)
        elif method == "POST":
            response = requests.post(url, headers=headers, json=params)
        else:
            raise ValueError(f"Unsupported HTTP method: {method}")

        response.raise_for_status()
        return response.json()

    def list_leaderboards(self) -> List[Dict[str, Any]]:
        """
        List all available Terminal-Bench leaderboards.

        Returns:
            List of leaderboard objects with benchmark name, version, path, and status
        """
        result = self._call("list_leaderboards", method="GET")
        return result.get("leaderboards", [])

    def get_leaderboard(
        self,
        benchmark: str = "terminal-bench",
        version: str = "2.0",
        search: Optional[str] = None,
        agent: Optional[str] = None,
        model: Optional[str] = None,
        verified_only: str = "false",
        limit: int = 50,
    ) -> Dict[str, Any]:
        """
        Get leaderboard entries for a specific Terminal-Bench benchmark version.

        Args:
            benchmark: Benchmark name (e.g., 'terminal-bench', 'terminal-bench-science')
            version: Benchmark version (e.g., '2.0', '3.0')
            search: Free-text search filter matching agent name, organization, or model names
            agent: Filter by agent name (case-insensitive substring match)
            model: Filter by model name (case-insensitive substring match)
            verified_only: Return only verified entries when set to 'true'
            limit: Maximum number of entries to return

        Returns:
            Dictionary containing benchmark info, total entries, filtered entries count, and entries list
        """
        params = {
            "benchmark": benchmark,
            "version": version,
            "verified_only": verified_only,
            "limit": limit,
        }

        # Only add optional parameters if provided
        if search:
            params["search"] = search
        if agent:
            params["agent"] = agent
        if model:
            params["model"] = model

        return self._call("get_leaderboard", method="GET", **params)


def main():
    """Demonstrate a practical workflow with the Terminal-Bench Leaderboard API."""
    # Initialize the client
    client = ParseClient()

    print("=" * 70)
    print("Terminal-Bench Leaderboard API - Practical Workflow")
    print("=" * 70)

    # Step 1: List all available leaderboards
    print("\n1. Fetching all available leaderboards...")
    leaderboards = client.list_leaderboards()
    print(f"   Found {len(leaderboards)} leaderboards:")
    for lb in leaderboards:
        status = lb.get("status", "unknown")
        print(
            f"   - {lb['benchmark']} v{lb['version']} (Status: {status}) - {lb['path']}"
        )

    # Step 2: Get the main leaderboard (terminal-bench v2.0)
    print("\n2. Fetching terminal-bench v2.0 leaderboard (top 10)...")
    leaderboard_data = client.get_leaderboard(
        benchmark="terminal-bench", version="2.0", limit=10
    )
    print(
        f"   Benchmark: {leaderboard_data['benchmark']} v{leaderboard_data['version']}"
    )
    print(f"   Total entries: {leaderboard_data['total_entries']}")
    print(f"   Showing: {leaderboard_data['filtered_entries']} entries\n")

    # Display top entries
    print("   Top 10 Leaderboard Entries:")
    print("   " + "-" * 66)
    print(
        "   Rank | Agent            | Model           | Accuracy | Verified | Org"
    )
    print("   " + "-" * 66)
    for entry in leaderboard_data["entries"]:
        rank = entry["rank"]
        agent = entry["agent"][:15].ljust(15)
        model = entry["model"][0][:15].ljust(15) if entry["model"] else "N/A".ljust(15)
        accuracy = f"{entry['accuracy']:.1%}".rjust(8)
        verified = "✓" if entry["verified"] else " "
        org = entry.get("agent_organization", "unknown")[:20]
        print(
            f"   {rank:4d} | {agent} | {model} | {accuracy} |    {verified}    | {org}"
        )

    # Step 3: Search for GPT models
    print("\n3. Searching for entries using GPT models (top 5)...")
    gpt_results = client.get_leaderboard(
        benchmark="terminal-bench", version="2.0", model="gpt", limit=5
    )
    print(f"   Found {gpt_results['filtered_entries']} entries with GPT models:")
    for entry in gpt_results["entries"]:
        print(f"   - Rank {entry['rank']}: {entry['agent']} with {entry['model']}")

    # Step 4: Filter by verified entries only
    print("\n4. Fetching only verified entries (top 5)...")
    verified_results = client.get_leaderboard(
        benchmark="terminal-bench", version="2.0", verified_only="true", limit=5
    )
    print(f"   Found {verified_results['filtered_entries']} verified entries:")
    for entry in verified_results["entries"]:
        print(
            f"   - Rank {entry['rank']}: {entry['agent']} - Accuracy: {entry['accuracy']:.1%}"
        )

    # Step 5: Analyze the results
    print("\n5. Analysis of the leaderboard:")
    all_entries = leaderboard_data["entries"]
    if all_entries:
        # Calculate average accuracy
        accuracies = [e["accuracy"] for e in all_entries]
        avg_accuracy = sum(accuracies) / len(accuracies)
        max_accuracy = max(accuracies)
        min_accuracy = min(accuracies)

        # Count unique models
        all_models = set()
        for entry in all_entries:
            for model in entry.get("model", []):
                all_models.add(model.lower())

        # Count verified entries
        verified_count = sum(1 for e in all_entries if e.get("verified", False))

        print(f"   - Average accuracy: {avg_accuracy:.1%}")
        print(f"   - Top accuracy: {max_accuracy:.1%}")
        print(f"   - Lowest accuracy (in top 10): {min_accuracy:.1%}")
        print(f"   - Unique models used: {len(all_models)}")
        print(f"   - Sample models: {', '.join(sorted(all_models)[:5])}")
        print(f"   - Verified entries in top 10: {verified_count}")

    # Step 6: Try the science benchmark if available
    print("\n6. Checking terminal-bench-science v1.0...")
    try:
        science_results = client.get_leaderboard(
            benchmark="terminal-bench-science", version="1.0", limit=5
        )
        if science_results["filtered_entries"] > 0:
            print(f"   Found {science_results['filtered_entries']} entries:")
            for entry in science_results["entries"]:
                print(
                    f"   - {entry['agent']}: {entry['accuracy']:.1%} accuracy"
                )
        else:
            print("   No entries found in terminal-bench-science v1.0")
    except Exception as e:
        print(f"   Could not fetch terminal-bench-science: {e}")

    print("\n" + "=" * 70)
    print("Workflow completed successfully!")
    print("=" * 70)


if __name__ == "__main__":
    main()