test_pubmed.py

import requests
import json
from bs4 import BeautifulSoup
import logging

# Setup logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def test_pubmed_search():
    """Test PubMed search API"""
    base_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
    search_url = f"{base_url}esearch.fcgi"
    
    # Test query
    query = "clinical notes"
    
    search_params = {
        "db": "pubmed",
        "term": query,
        "retmax": 10,  # Just get 10 results for testing
        "retmode": "json",
        "sort": "relevance"
    }
    
    logger.info(f"Testing PubMed search with query: {query}")
    logger.info(f"Search URL: {search_url}")
    logger.info(f"Search params: {search_params}")
    
    try:
        response = requests.get(search_url, params=search_params)
        response.raise_for_status()
        search_results = response.json()
        
        logger.info(f"Response status code: {response.status_code}")
        logger.info(f"Response headers: {dict(response.headers)}")
        logger.info(f"Search results: {json.dumps(search_results, indent=2)}")
        
        if "esearchresult" in search_results:
            id_list = search_results["esearchresult"]["idlist"]
            logger.info(f"Found {len(id_list)} article IDs")
            
            # Test fetching one article
            if id_list:
                test_id = id_list[0]
                fetch_url = f"{base_url}efetch.fcgi"
                fetch_params = {
                    "db": "pubmed",
                    "id": test_id,
                    "retmode": "xml"
                }
                
                logger.info(f"\nTesting article fetch for ID: {test_id}")
                logger.info(f"Fetch URL: {fetch_url}")
                logger.info(f"Fetch params: {fetch_params}")
                
                response = requests.get(fetch_url, params=fetch_params)
                response.raise_for_status()
                
                logger.info(f"Fetch response status code: {response.status_code}")
                logger.info(f"Fetch response headers: {dict(response.headers)}")
                logger.info(f"First 500 chars of response: {response.text[:500]}")
                
                soup = BeautifulSoup(response.text, 'lxml')
                article = soup.find('PubmedArticle')
                
                if article:
                    logger.info("\nArticle structure:")
                    logger.info(f"Title: {article.find('ArticleTitle').get_text() if article.find('ArticleTitle') else 'Not found'}")
                    logger.info(f"Abstract: {article.find('Abstract').get_text()[:200] + '...' if article.find('Abstract') else 'Not found'}")
                else:
                    logger.error("No PubmedArticle found in response")
        
    except Exception as e:
        logger.error(f"Error during test: {str(e)}", exc_info=True)

if __name__ == "__main__":
    test_pubmed_search()