Spaces:

theaniketgiri
/

syn

Sleeping

App Files Files Community

theaniketgiri commited on Jun 17

Commit

7271401

1 Parent(s): 7ca254c

Completed

Browse files

Files changed (5) hide show

.gitignore +1 -0
analysis_api.py +195 -0
api.py +54 -6
data/reports/plots/sample_distribution.png +0 -0
src/web/index.html +193 -32

.gitignore CHANGED Viewed

@@ -58,6 +58,7 @@ ENV/
 **/*.model
 **/*.json
 **/*.txt
 **/*.vocab
 **/*.merges
 **/*.model

 **/*.model
 **/*.json
 **/*.txt
+!requirements.txt
 **/*.vocab
 **/*.merges
 **/*.model

analysis_api.py ADDED Viewed

	@@ -0,0 +1,195 @@

+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse, HTMLResponse
+from pydantic import BaseModel
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+import uvicorn
+import os
+from pathlib import Path
+import json
+import logging
+from contextlib import asynccontextmanager
+# Import the analyzer
+from analyze_data_quality import DataQualityAnalyzer
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+# Global state
+analyzer = None
+last_analysis_time = None
+analysis_results = None
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup
+    global analyzer
+    try:
+        analyzer = DataQualityAnalyzer()
+        logger.info("DataQualityAnalyzer initialized successfully")
+    except Exception as e:
+        logger.error(f"Error initializing analyzer: {str(e)}")
+    yield
+    # Shutdown
+    logger.info("Shutting down analysis API")
+app = FastAPI(
+    title="Synthex Medical Text Analysis API",
+    description="""
+    API for analyzing medical text data quality. This API provides endpoints for:
+    - Running data quality analysis
+    - Checking analysis status
+    - Accessing generated plots
+    - Listing available datasets
+    """,
+    version="1.0.0",
+    lifespan=lifespan
+)
+# Add CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+class AnalysisResponse(BaseModel):
+    summary: Dict[str, Any]
+    datasets: Dict[str, Dict[str, Any]]
+    plots_available: List[str]
+    timestamp: str
+class StatusResponse(BaseModel):
+    last_analysis: Optional[str]
+    summary: Optional[Dict[str, Any]]
+    is_analyzed: bool
+@app.get("/", response_class=HTMLResponse)
+async def root():
+    """API documentation and status page"""
+    return """
+    <html>
+        <head>
+            <title>Synthex Medical Text Analysis API</title>
+            <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet">
+            <style>
+                body { padding: 20px; }
+                .container { max-width: 800px; }
+            </style>
+        </head>
+        <body>
+            <div class="container">
+                <h1>Synthex Medical Text Analysis API</h1>
+                <p class="lead">API for analyzing medical text data quality</p>
+                <h2>Available Endpoints</h2>
+                <ul>
+                    <li><strong>POST /analyze</strong> - Run full data analysis</li>
+                    <li><strong>GET /analysis/status</strong> - Check analysis status</li>
+                    <li><strong>GET /plots/{plot_name}</strong> - Get specific plot</li>
+                    <li><strong>GET /datasets</strong> - List available datasets</li>
+                </ul>
+                <h2>API Documentation</h2>
+                <p>For detailed API documentation, visit:</p>
+                <ul>
+                    <li><a href="/docs">Swagger UI</a></li>
+                    <li><a href="/redoc">ReDoc</a></li>
+                </ul>
+                <h2>Status</h2>
+                <p>API is running and ready to process requests.</p>
+            </div>
+        </body>
+    </html>
+    """
+@app.post("/analyze", response_model=AnalysisResponse)
+async def analyze_data():
+    """Run full data analysis and generate reports"""
+    global analyzer, last_analysis_time, analysis_results
+    if analyzer is None:
+        try:
+            analyzer = DataQualityAnalyzer()
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Failed to initialize analyzer: {str(e)}")
+    try:
+        # Run analysis
+        analyzer.analyze_all_datasets()
+        report = analyzer.generate_report()
+        analyzer.plot_metrics()
+        # Get list of generated plots
+        plots_dir = analyzer.data_dir.parent / "reports" / "plots"
+        plots_available = [f.name for f in plots_dir.glob("*.png")]
+        # Update global state
+        last_analysis_time = datetime.now().isoformat()
+        analysis_results = report
+        return AnalysisResponse(
+            summary=report["summary"],
+            datasets=report["datasets"],
+            plots_available=plots_available,
+            timestamp=last_analysis_time
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Analysis failed: {str(e)}")
+@app.get("/analysis/status", response_model=StatusResponse)
+async def get_analysis_status():
+    """Get the status of the last analysis run"""
+    if last_analysis_time is None:
+        return StatusResponse(
+            last_analysis=None,
+            summary=None,
+            is_analyzed=False
+        )
+    return StatusResponse(
+        last_analysis=last_analysis_time,
+        summary=analysis_results["summary"] if analysis_results else None,
+        is_analyzed=True
+    )
+@app.get("/plots/{plot_name}")
+async def get_plot(plot_name: str):
+    """Get a specific plot by name"""
+    plots_dir = Path("data/reports/plots")
+    plot_path = plots_dir / plot_name
+    if not plot_path.exists():
+        raise HTTPException(status_code=404, detail=f"Plot {plot_name} not found")
+    return FileResponse(plot_path)
+@app.get("/datasets")
+async def get_datasets():
+    """List all available datasets for analysis"""
+    if analyzer is None:
+        try:
+            analyzer = DataQualityAnalyzer()
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Failed to initialize analyzer: {str(e)}")
+    try:
+        datasets = []
+        for file_path in analyzer.data_dir.glob("*.json"):
+            datasets.append({
+                "name": file_path.stem,
+                "path": str(file_path),
+                "size": file_path.stat().st_size
+            })
+        return {"datasets": datasets}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to list datasets: {str(e)}")
+if __name__ == "__main__":
+    uvicorn.run("analysis_api:app", host="0.0.0.0", port=8001, reload=True)

api.py CHANGED Viewed

@@ -1,20 +1,23 @@
 from fastapi import FastAPI, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
-from typing import List, Optional
 import uvicorn
 import sys
 import os
 # Add src directory to Python path
 sys.path.append(os.path.join(os.path.dirname(__file__), 'src'))
-# Import the medical generator
 from src.generation.medical_generator import MedicalTextGenerator, DEFAULT_GEMINI_API_KEY
 app = FastAPI(
     title="Synthex Medical Text Generator API",
-    description="API for generating synthetic medical records",
     version="1.0.0"
 )
@@ -27,8 +30,9 @@ app.add_middleware(
     allow_headers=["*"],  # Allows all headers
 )
-# Initialize the generator
 generator = None
 class GenerationRequest(BaseModel):
     record_type: str
@@ -41,13 +45,19 @@ class GenerationResponse(BaseModel):
     records: List[dict]
     total_generated: int
 @app.on_event("startup")
 async def startup_event():
-    global generator
     try:
         generator = MedicalTextGenerator()
     except Exception as e:
-        print(f"Error initializing generator: {str(e)}")
 @app.get("/")
 async def root():
@@ -91,5 +101,43 @@ async def get_record_types():
         ]
     }
 if __name__ == "__main__":
     uvicorn.run("api:app", host="0.0.0.0", port=8000, reload=True)

 from fastapi import FastAPI, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
+from typing import List, Optional, Dict, Any
 import uvicorn
 import sys
 import os
+from pathlib import Path
+from fastapi.responses import FileResponse
 # Add src directory to Python path
 sys.path.append(os.path.join(os.path.dirname(__file__), 'src'))
+# Import the medical generator and data analyzer
 from src.generation.medical_generator import MedicalTextGenerator, DEFAULT_GEMINI_API_KEY
+from analyze_data_quality import DataQualityAnalyzer
 app = FastAPI(
     title="Synthex Medical Text Generator API",
+    description="API for generating synthetic medical records and analyzing data quality",
     version="1.0.0"
 )
     allow_headers=["*"],  # Allows all headers
 )
+# Initialize the generator and analyzer
 generator = None
+analyzer = None
 class GenerationRequest(BaseModel):
     record_type: str
     records: List[dict]
     total_generated: int
+class AnalysisResponse(BaseModel):
+    summary: Dict[str, Any]
+    datasets: Dict[str, Dict[str, Any]]
+    plots_available: List[str]
 @app.on_event("startup")
 async def startup_event():
+    global generator, analyzer
     try:
         generator = MedicalTextGenerator()
+        analyzer = DataQualityAnalyzer()
     except Exception as e:
+        print(f"Error initializing services: {str(e)}")
 @app.get("/")
 async def root():
         ]
     }
+@app.post("/analyze", response_model=AnalysisResponse)
+async def analyze_data():
+    global analyzer
+    if analyzer is None:
+        try:
+            analyzer = DataQualityAnalyzer()
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Failed to initialize analyzer: {str(e)}")
+    try:
+        # Run analysis
+        analyzer.analyze_all_datasets()
+        report = analyzer.generate_report()
+        analyzer.plot_metrics()
+        # Get list of generated plots
+        plots_dir = analyzer.data_dir.parent / "reports" / "plots"
+        plots_available = [f.name for f in plots_dir.glob("*.png")]
+        return AnalysisResponse(
+            summary=report["summary"],
+            datasets=report["datasets"],
+            plots_available=plots_available
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Analysis failed: {str(e)}")
+@app.get("/analysis/plots/{plot_name}")
+async def get_plot(plot_name: str):
+    plots_dir = Path("data/reports/plots")
+    plot_path = plots_dir / plot_name
+    if not plot_path.exists():
+        raise HTTPException(status_code=404, detail=f"Plot {plot_name} not found")
+    return FileResponse(plot_path)
 if __name__ == "__main__":
     uvicorn.run("api:app", host="0.0.0.0", port=8000, reload=True)

data/reports/plots/sample_distribution.png CHANGED Viewed

src/web/index.html CHANGED Viewed

@@ -11,7 +11,7 @@
             background-color: #f8f9fa;
         }
         .container {
-            max-width: 800px;
             background-color: white;
             padding: 30px;
             border-radius: 10px;
@@ -29,58 +29,126 @@
             text-align: center;
             margin: 20px 0;
         }
     </style>
 </head>
 <body>
     <div class="container">
         <h1 class="mb-4">Synthex Medical Text Generator</h1>
-        <div class="mb-3">
-            <label for="recordType" class="form-label">Record Type</label>
-            <select class="form-select" id="recordType">
-                <option value="clinical_note">Clinical Note</option>
-                <option value="discharge_summary">Discharge Summary</option>
-                <option value="lab_report">Lab Report</option>
-                <option value="prescription">Prescription</option>
-                <option value="patient_intake">Patient Intake</option>
-            </select>
-        </div>
-        <div class="mb-3">
-            <label for="quantity" class="form-label">Quantity</label>
-            <input type="number" class="form-control" id="quantity" value="1" min="1" max="10">
-        </div>
-        <div class="mb-3 form-check">
-            <input type="checkbox" class="form-check-input" id="useGemini">
-            <label class="form-check-label" for="useGemini">Use Gemini (if available)</label>
-        </div>
-        <div class="mb-3 form-check">
-            <input type="checkbox" class="form-check-input" id="includeMetadata" checked>
-            <label class="form-check-label" for="includeMetadata">Include Metadata</label>
-        </div>
-        <button class="btn btn-primary" onclick="generateRecords()">Generate Records</button>
-        <div class="loading" id="loading">
-            <div class="spinner-border text-primary" role="status">
-                <span class="visually-hidden">Loading...</span>
             </div>
-            <p class="mt-2">Generating records...</p>
-        </div>
-        <div id="result" class="result-box"></div>
     </div>
     <script>
         async function generateRecords() {
             const recordType = document.getElementById('recordType').value;
             const quantity = parseInt(document.getElementById('quantity').value);
             const useGemini = document.getElementById('useGemini').checked;
             const includeMetadata = document.getElementById('includeMetadata').checked;
-            // Show loading
             document.getElementById('loading').style.display = 'block';
             document.getElementById('result').innerHTML = '';
@@ -101,7 +169,6 @@
                 const data = await response.json();
-                // Format and display results
                 let resultHtml = '<h3>Generated Records:</h3>';
                 data.records.forEach(record => {
                     resultHtml += `
@@ -122,6 +189,100 @@
                 document.getElementById('loading').style.display = 'none';
             }
         }
     </script>
 </body>
 </html>

             background-color: #f8f9fa;
         }
         .container {
+            max-width: 1200px;
             background-color: white;
             padding: 30px;
             border-radius: 10px;
             text-align: center;
             margin: 20px 0;
         }
+        .nav-tabs {
+            margin-bottom: 20px;
+        }
+        .plot-container {
+            margin-top: 20px;
+            text-align: center;
+        }
+        .plot-container img {
+            max-width: 100%;
+            height: auto;
+            border-radius: 5px;
+            box-shadow: 0 0 5px rgba(0,0,0,0.1);
+        }
+        .stats-card {
+            margin-bottom: 15px;
+        }
     </style>
 </head>
 <body>
     <div class="container">
         <h1 class="mb-4">Synthex Medical Text Generator</h1>
+        <!-- Navigation Tabs -->
+        <ul class="nav nav-tabs" id="myTab" role="tablist">
+            <li class="nav-item" role="presentation">
+                <button class="nav-link active" id="generation-tab" data-bs-toggle="tab" data-bs-target="#generation" type="button" role="tab">Generation</button>
+            </li>
+            <li class="nav-item" role="presentation">
+                <button class="nav-link" id="analysis-tab" data-bs-toggle="tab" data-bs-target="#analysis" type="button" role="tab">Analysis</button>
+            </li>
+        </ul>
+        <!-- Tab Content -->
+        <div class="tab-content" id="myTabContent">
+            <!-- Generation Tab -->
+            <div class="tab-pane fade show active" id="generation" role="tabpanel">
+                <div class="mb-3">
+                    <label for="recordType" class="form-label">Record Type</label>
+                    <select class="form-select" id="recordType">
+                        <option value="clinical_note">Clinical Note</option>
+                        <option value="discharge_summary">Discharge Summary</option>
+                        <option value="lab_report">Lab Report</option>
+                        <option value="prescription">Prescription</option>
+                        <option value="patient_intake">Patient Intake</option>
+                    </select>
+                </div>
+                <div class="mb-3">
+                    <label for="quantity" class="form-label">Quantity</label>
+                    <input type="number" class="form-control" id="quantity" value="1" min="1" max="10">
+                </div>
+                <div class="mb-3 form-check">
+                    <input type="checkbox" class="form-check-input" id="useGemini">
+                    <label class="form-check-label" for="useGemini">Use Gemini (if available)</label>
+                </div>
+                <div class="mb-3 form-check">
+                    <input type="checkbox" class="form-check-input" id="includeMetadata" checked>
+                    <label class="form-check-label" for="includeMetadata">Include Metadata</label>
+                </div>
+                <button class="btn btn-primary" onclick="generateRecords()">Generate Records</button>
+                <div class="loading" id="loading">
+                    <div class="spinner-border text-primary" role="status">
+                        <span class="visually-hidden">Loading...</span>
+                    </div>
+                    <p class="mt-2">Generating records...</p>
+                </div>
+                <div id="result" class="result-box"></div>
             </div>
+            <!-- Analysis Tab -->
+            <div class="tab-pane fade" id="analysis" role="tabpanel">
+                <div class="row mb-4">
+                    <div class="col">
+                        <button class="btn btn-primary" onclick="runAnalysis()">Run Analysis</button>
+                        <button class="btn btn-secondary" onclick="checkStatus()">Check Status</button>
+                    </div>
+                </div>
+                <div class="loading" id="analysisLoading">
+                    <div class="spinner-border text-primary" role="status">
+                        <span class="visually-hidden">Loading...</span>
+                    </div>
+                    <p class="mt-2">Running analysis...</p>
+                </div>
+                <!-- Analysis Results -->
+                <div id="analysisResult">
+                    <!-- Summary Stats -->
+                    <div class="row mb-4" id="summaryStats"></div>
+                    <!-- Dataset List -->
+                    <div class="mb-4">
+                        <h3>Available Datasets</h3>
+                        <div id="datasetList" class="list-group"></div>
+                    </div>
+                    <!-- Plots -->
+                    <div class="mb-4">
+                        <h3>Analysis Plots</h3>
+                        <div id="plotGallery" class="row"></div>
+                    </div>
+                </div>
+            </div>
+        </div>
     </div>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
     <script>
+        // Generation Functions
         async function generateRecords() {
             const recordType = document.getElementById('recordType').value;
             const quantity = parseInt(document.getElementById('quantity').value);
             const useGemini = document.getElementById('useGemini').checked;
             const includeMetadata = document.getElementById('includeMetadata').checked;
             document.getElementById('loading').style.display = 'block';
             document.getElementById('result').innerHTML = '';
                 const data = await response.json();
                 let resultHtml = '<h3>Generated Records:</h3>';
                 data.records.forEach(record => {
                     resultHtml += `
                 document.getElementById('loading').style.display = 'none';
             }
         }
+        // Analysis Functions
+        async function runAnalysis() {
+            document.getElementById('analysisLoading').style.display = 'block';
+            document.getElementById('analysisResult').style.display = 'none';
+            try {
+                const response = await fetch('http://localhost:8001/analyze', {
+                    method: 'POST'
+                });
+                const data = await response.json();
+                displayAnalysisResults(data);
+            } catch (error) {
+                alert(`Error running analysis: ${error.message}`);
+            } finally {
+                document.getElementById('analysisLoading').style.display = 'none';
+                document.getElementById('analysisResult').style.display = 'block';
+            }
+        }
+        async function checkStatus() {
+            try {
+                const response = await fetch('http://localhost:8001/analysis/status');
+                const data = await response.json();
+                if (data.is_analyzed) {
+                    displayAnalysisResults(data);
+                } else {
+                    alert('No analysis has been run yet.');
+                }
+            } catch (error) {
+                alert(`Error checking status: ${error.message}`);
+            }
+        }
+        function displayAnalysisResults(data) {
+            // Display summary stats
+            const summaryHtml = `
+                <div class="col-md-4">
+                    <div class="card stats-card">
+                        <div class="card-body">
+                            <h5 class="card-title">Total Samples</h5>
+                            <p class="card-text">${data.summary.total_samples}</p>
+                        </div>
+                    </div>
+                </div>
+                <div class="col-md-4">
+                    <div class="card stats-card">
+                        <div class="card-body">
+                            <h5 class="card-title">Total Datasets</h5>
+                            <p class="card-text">${data.summary.total_datasets}</p>
+                        </div>
+                    </div>
+                </div>
+                <div class="col-md-4">
+                    <div class="card stats-card">
+                        <div class="card-body">
+                            <h5 class="card-title">Last Updated</h5>
+                            <p class="card-text">${new Date(data.timestamp).toLocaleString()}</p>
+                        </div>
+                    </div>
+                </div>
+            `;
+            document.getElementById('summaryStats').innerHTML = summaryHtml;
+            // Display datasets
+            const datasetList = document.getElementById('datasetList');
+            datasetList.innerHTML = '';
+            Object.entries(data.datasets).forEach(([name, stats]) => {
+                datasetList.innerHTML += `
+                    <div class="list-group-item">
+                        <h5>${name}</h5>
+                        <p>Samples: ${stats.total_samples}</p>
+                        ${stats.abstract ? `<p>Avg Abstract Length: ${stats.abstract.avg_length.toFixed(1)}</p>` : ''}
+                    </div>
+                `;
+            });
+            // Display plots
+            const plotGallery = document.getElementById('plotGallery');
+            plotGallery.innerHTML = '';
+            data.plots_available.forEach(plotName => {
+                plotGallery.innerHTML += `
+                    <div class="col-md-6 mb-4">
+                        <div class="plot-container">
+                            <img src="http://localhost:8001/plots/${plotName}" alt="${plotName}" class="img-fluid">
+                            <p class="mt-2">${plotName.replace(/_/g, ' ').replace('.png', '')}</p>
+                        </div>
+                    </div>
+                `;
+            });
+        }
+        // Initial status check
+        checkStatus();
     </script>
 </body>
 </html>