Arena APIarena.ai ↗

"""
Arena AI Leaderboard API Client

Fetch AI model leaderboard rankings from arena.ai across different modalities.
Get your API key from: https://parse.bot/settings
"""

import os
import requests
from typing import Optional
import json


class ParseClient:
    """Client for interacting with the Parse API for Arena AI Leaderboard data."""

    def __init__(self, api_key: Optional[str] = None):
        """Initialize the ParseClient with API credentials.
        
        Args:
            api_key: API key for Parse API. If not provided, reads from PARSE_API_KEY env var.
        """
        self.base_url = "https://api.parse.bot"
        self.scraper_id = "dee580cd-1286-483f-818e-02b6459a0d69"
        self.api_key = api_key or os.getenv("PARSE_API_KEY")
        
        if not self.api_key:
            raise ValueError("API key must be provided or set in PARSE_API_KEY environment variable")

    def _call(self, endpoint: str, method: str = "POST", **params) -> dict:
        """Make an API call to the Parse API.
        
        Args:
            endpoint: The endpoint name to call
            method: HTTP method (GET or POST)
            **params: Parameters to pass to the endpoint
            
        Returns:
            The JSON response from the API
            
        Raises:
            requests.exceptions.RequestException: If the API call fails
        """
        url = f"{self.base_url}/scraper/{self.scraper_id}/{endpoint}"
        headers = {
            "X-API-Key": self.api_key,
            "Content-Type": "application/json"
        }
        
        try:
            if method == "GET":
                response = requests.get(url, headers=headers, params=params)
            elif method == "POST":
                response = requests.post(url, headers=headers, json=params)
            else:
                raise ValueError(f"Unsupported HTTP method: {method}")
            
            response.raise_for_status()
            return response.json()
        except requests.exceptions.RequestException as e:
            print(f"Error calling {endpoint}: {e}")
            raise

    def get_leaderboard(self, arena: str = "agent") -> dict:
        """Get the full leaderboard rankings for a specified arena.
        
        Returns all ranked models with their scores, confidence intervals, and metadata.
        The agent arena returns signal-based scores (task outcome, steerability, etc.),
        while other arenas return ELO-style ratings with vote counts.
        
        Args:
            arena: The arena/modality to fetch rankings for. Options:
                - agent (default)
                - text
                - text-to-image
                - image-edit
                - text-to-video
                - image-to-video
                - video-edit
                - code/webdev
                - code/image-to-webdev
                
        Returns:
            Dictionary containing leaderboard data with models list
        """
        return self._call("get_leaderboard", method="GET", arena=arena)


def format_model_info(model: dict) -> str:
    """Format model information for display.
    
    Args:
        model: Model ranking object from leaderboard
        
    Returns:
        Formatted string with model information
    """
    rank = model.get("rank", "N/A")
    name = model.get("model", "Unknown")
    org = model.get("organization", "Unknown")
    avg_score = model.get("avg_score", {})
    score_value = avg_score.get("value", 0) if isinstance(avg_score, dict) else 0
    ci = avg_score.get("ci", 0) if isinstance(avg_score, dict) else 0
    sessions = model.get("sessions", 0)
    
    return (f"#{rank} {name} (by {org}) | "
            f"Score: {score_value:.4f} ±{ci:.4f} | "
            f"Sessions: {sessions:,}")


def main():
    """Main function demonstrating practical usage of the Arena AI Leaderboard API."""
    
    # Initialize the client
    client = ParseClient()
    
    print("=" * 80)
    print("Arena AI Leaderboard Analysis")
    print("=" * 80)
    
    # Define the arenas to analyze
    arenas_to_check = ["agent", "text", "code/webdev"]
    
    # Collect leaderboard data from multiple arenas
    leaderboard_data = {}
    
    for arena in arenas_to_check:
        print(f"\nFetching leaderboard for '{arena}' arena...")
        
        try:
            response = client.get_leaderboard(arena=arena)
            
            if response.get("status") == "success" or "data" in response:
                data = response.get("data", response)
                leaderboard_data[arena] = data
                
                print(f"✓ Successfully retrieved {arena} arena data")
                print(f"  Total models: {data.get('model_count', 0)}")
                
                if arena == "agent":
                    print(f"  Last updated: {data.get('last_updated', 'N/A')}")
                    print(f"  Total sessions: {data.get('total_sessions', 0):,}")
                else:
                    print(f"  Leaderboard: {data.get('leaderboard_slug', 'N/A')}")
                    
            else:
                print(f"✗ Failed to retrieve data for {arena}")
                
        except Exception as e:
            print(f"✗ Error fetching {arena}: {e}")
            continue
    
    # Analyze and display results
    print("\n" + "=" * 80)
    print("Leaderboard Rankings Summary")
    print("=" * 80)
    
    for arena, data in leaderboard_data.items():
        models = data.get("models", [])
        
        if not models:
            print(f"\nNo models found for {arena} arena")
            continue
        
        print(f"\n📊 {arena.upper()} ARENA - Top 5 Models:")
        print("-" * 80)
        
        # Show top 5 models
        for i, model in enumerate(models[:5], 1):
            print(f"  {format_model_info(model)}")
        
        # Calculate and display arena statistics
        if len(models) > 5:
            print(f"\n  ... and {len(models) - 5} more models")
        
        # For agent arena, show signal score analysis
        if arena == "agent" and models:
            top_model = models[0]
            signal_scores = top_model.get("signal_scores", {})
            
            if signal_scores:
                print(f"\n  🎯 Top Model ({top_model.get('model')}) Signal Scores:")
                for signal_name, score in signal_scores.items():
                    print(f"     - {signal_name}: {score:.4f}")
    
    # Comparative analysis
    print("\n" + "=" * 80)
    print("Comparative Analysis")
    print("=" * 80)
    
    if len(leaderboard_data) > 1:
        print("\nArena Comparison:")
        print("-" * 80)
        
        for arena, data in leaderboard_data.items():
            model_count = data.get("model_count", 0)
            top_model = data.get("models", [{}])[0] if data.get("models") else {}
            top_name = top_model.get("model", "N/A")
            
            print(f"  {arena:20} → {model_count:3} models | Top: {top_name}")
    
    print("\n" + "=" * 80)
    print("Analysis Complete")
    print("=" * 80)


if __name__ == "__main__":
    main()