NBER APInber.org ↗

"""
NBER Papers API Client
Practical example for searching and retrieving NBER research papers.

Get your API key from: https://parse.bot/settings
"""

import os
import requests
from typing import Optional, List, Dict, Any


class ParseClient:
    """Client for interacting with the NBER Papers API via Parse."""

    def __init__(self, api_key: Optional[str] = None):
        """Initialize the Parse client.
        
        Args:
            api_key: API key for Parse. If not provided, reads from PARSE_API_KEY env var.
        """
        self.base_url = "https://api.parse.bot"
        self.scraper_id = "1f177605-b040-467e-91e7-65b6a1d40fc4"
        self.api_key = api_key or os.getenv("PARSE_API_KEY")
        
        if not self.api_key:
            raise ValueError("API key not provided and PARSE_API_KEY environment variable not set")

    def _call(self, endpoint: str, method: str = "POST", **params) -> Dict[str, Any]:
        """Make a request to the Parse API.
        
        Args:
            endpoint: The endpoint name (e.g., 'search_papers', 'get_paper')
            method: HTTP method ('GET' or 'POST')
            **params: Query/body parameters for the request
            
        Returns:
            JSON response from the API
            
        Raises:
            requests.HTTPError: If the request fails
        """
        url = f"{self.base_url}/scraper/{self.scraper_id}/{endpoint}"
        headers = {
            "X-API-Key": self.api_key,
            "Content-Type": "application/json"
        }
        
        if method.upper() == "GET":
            response = requests.get(url, headers=headers, params=params)
        elif method.upper() == "POST":
            response = requests.post(url, headers=headers, json=params)
        else:
            raise ValueError(f"Unsupported HTTP method: {method}")
        
        response.raise_for_status()
        return response.json()

    def search_papers(
        self,
        query: str = "",
        page: int = 1,
        per_page: int = 20,
        content_type: str = "working_paper",
        topic: Optional[str] = None,
        program: Optional[str] = None,
        sort_by: str = "date"
    ) -> Dict[str, Any]:
        """Search NBER papers with filtering and pagination.
        
        Args:
            query: Search query for relevance ranking
            page: Page number (1-based)
            per_page: Results per page (20, 50, or 100)
            content_type: Content type filter (working_paper, chapter, article, etc.)
            topic: Topic filter (e.g., 'Macroeconomics', 'Labor Economics')
            program: Program filter (e.g., 'Monetary Economics', 'Labor Studies')
            sort_by: Sort order ('date' or 'relevance')
            
        Returns:
            Search results with papers list and pagination info
        """
        params = {
            "query": query,
            "page": page,
            "per_page": per_page,
            "content_type": content_type,
            "sort_by": sort_by
        }
        
        if topic:
            params["topic"] = topic
        if program:
            params["program"] = program
        
        return self._call("search_papers", method="GET", **params)

    def get_paper(self, paper_id: str) -> Dict[str, Any]:
        """Get detailed information for a specific NBER paper.
        
        Args:
            paper_id: Working paper ID (e.g., 'w10567' or '10567')
            
        Returns:
            Detailed paper information including abstract, authors, and programs
        """
        return self._call("get_paper", method="GET", paper_id=paper_id)


def print_paper_summary(paper: Dict[str, Any]) -> None:
    """Print a formatted summary of a paper."""
    title = paper.get('title', 'N/A')
    paper_id = paper.get('paper_number', 'N/A')
    date = paper.get('date', paper.get('issue_date', 'N/A'))
    
    print(f"\n  📄 {title}")
    print(f"     ID: {paper_id} | Date: {date}")
    
    if paper.get('authors'):
        authors = ", ".join([a['name'] for a in paper['authors']])
        print(f"     Authors: {authors}")
    
    if paper.get('abstract'):
        abstract = paper['abstract']
        if len(abstract) > 120:
            abstract = abstract[:120] + "..."
        print(f"     Abstract: {abstract}")


def print_detailed_paper(paper: Dict[str, Any]) -> None:
    """Print detailed information about a paper."""
    print(f"\n{'='*80}")
    print(f"📋 DETAILED PAPER INFORMATION")
    print(f"{'='*80}")
    print(f"Title: {paper['title']}")
    print(f"Paper Number: {paper['paper_number']}")
    print(f"Issue Date: {paper['issue_date']}")
    print(f"DOI: {paper['doi']}")
    print(f"URL: {paper['url']}")
    
    if paper.get('authors'):
        print(f"\nAuthors ({len(paper['authors'])}):")
        for author in paper['authors']:
            print(f"  • {author['name']}")
    
    if paper.get('programs'):
        print(f"\nAssociated Programs:")
        for program in paper['programs']:
            print(f"  • {program}")
    
    if paper.get('abstract'):
        print(f"\nAbstract:")
        print(f"{paper['abstract']}")


def main():
    """Demonstrate a practical workflow for the NBER Papers API."""
    
    client = ParseClient()
    
    print("\n" + "="*80)
    print("NBER PAPERS API - RESEARCH DISCOVERY WORKFLOW")
    print("="*80)
    
    # Workflow: Search for papers on a topic, then get details on papers of interest
    print("\n📚 STEP 1: Searching for papers on 'machine learning'...")
    
    search_results = client.search_papers(
        query="machine learning",
        per_page=5,
        sort_by="relevance"
    )
    
    total_found = search_results.get('total_results', 0)
    current_page = search_results.get('page', 1)
    total_pages = search_results.get('total_pages', 1)
    
    print(f"\n✓ Found {total_found:,} papers matching 'machine learning'")
    print(f"  Showing page {current_page} of {total_pages}")
    print(f"  ({len(search_results.get('papers', []))} papers on this page)")
    
    papers = search_results.get('papers', [])
    
    if not papers:
        print("\n❌ No papers found. Exiting.")
        return
    
    print(f"\n📖 Recent papers from search results:")
    for i, paper in enumerate(papers, 1):
        print(f"\n  [{i}] {paper.get('title', 'N/A')[:70]}...")
        print(f"      Type: {paper.get('type', 'N/A')} | Date: {paper.get('date', 'N/A')}")
    
    # Drill down into the first paper for detailed info
    first_paper = papers[0]
    paper_id = first_paper.get('paper_number')
    
    if paper_id:
        print(f"\n" + "="*80)
        print(f"📌 STEP 2: Fetching detailed information for paper {paper_id}...")
        print("="*80)
        
        detailed_paper = client.get_paper(paper_id)
        print_detailed_paper(detailed_paper)
    
    # Secondary workflow: Search by topic and program
    print(f"\n" + "="*80)
    print("📚 STEP 3: Searching papers by specific topic and program...")
    print("="*80)
    
    print("\n  Searching for 'Labor Economics' papers in 'Labor Studies' program...")
    
    topic_results = client.search_papers(
        topic="Labor Economics",
        program="Labor Studies",
        per_page=3,
        sort_by="date"
    )
    
    topic_papers = topic_results.get('papers', [])
    print(f"\n✓ Found {topic_results.get('total_results', 0)} papers in this category")
    print(f"  Showing {len(topic_papers)} most recent papers:\n")
    
    for i, paper in enumerate(topic_papers, 1):
        print(f"  [{i}]", end="")
        print_paper_summary(paper)
    
    # Demonstrate pagination
    if topic_results.get('total_pages', 1) > 1:
        print(f"\n" + "="*80)
        print("📄 STEP 4: Demonstrating pagination...")
        print("="*80)
        
        print("\n  Fetching page 2 of Labor Economics results...")
        page_2_results = client.search_papers(
            topic="Labor Economics",
            program="Labor Studies",
            page=2,
            per_page=3,
            sort_by="date"
        )
        
        page_2_papers = page_2_results.get('papers', [])
        print(f"✓ Retrieved page {page_2_results.get('page')} with {len(page_2_papers)} papers:\n")
        
        for i, paper in enumerate(page_2_papers, 1):
            print(f"  [{i}]", end="")
            print_paper_summary(paper)
    
    print(f"\n" + "="*80)
    print("✅ WORKFLOW COMPLETED SUCCESSFULLY!")
    print("="*80)
    print("\n💡 You can now:")
    print("  • Search papers by keyword, topic, or program")
    print("  • Filter by content type (working papers, articles, etc.)")
    print("  • Navigate through paginated results")
    print("  • Get detailed information for papers of interest")
    print("  • Access author profiles, DOIs, and abstracts")
    print()


if __name__ == "__main__":
    main()