Add jsons

src/app.py

@@ -10,27 +10,54 @@ from pathlib import Path
 load_dotenv()
 _FILE_PATH = Path(__file__).parents[1]
 
-# Mock data for demo purposes
-MOCK_FOMC_MEETINGS = [
-    {
-        "date": "2024-07-31",
-        "title": "Minutes of the Federal Open Market Committee July 30-31, 2024",
-        "rate_decision": "5.25-5.50%",
-        "summary": "The Committee maintained the federal funds rate target range at 5.25-5.50%. Inflation has eased over the past year but remains elevated."
-    },
-    {
-        "date": "2024-06-12",
-        "title": "Minutes of the Federal Open Market Committee June 11-12, 2024",
-        "rate_decision": "5.25-5.50%",
-        "summary": "The Committee held rates steady as inflation showed signs of cooling but labor market remained robust."
-    },
-    {
-        "date": "2024-05-01",
-        "title": "Minutes of the Federal Open Market Committee April 30-May 1, 2024",
-        "rate_decision": "5.25-5.50%",
-        "summary": "Officials maintained current rates while monitoring inflation progress and employment data."
-    }
-]
+# Load processed FOMC meetings data
+def load_processed_meetings():
+    """Load processed FOMC meetings from JSON file"""
+    try:
+        processed_file = _FILE_PATH / "data" / "fed_processed_meetings.json"
+        with open(processed_file, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        
+        # Transform to match expected format for the frontend
+        meetings = []
+        for meeting in data:
+            meetings.append({
+                "date": meeting.get("date", ""),
+                "title": meeting.get("title", ""),
+                "rate_decision": meeting.get("rate", ""),
+                "summary": meeting.get("forward_guidance", ""),  # Show full text
+                "action": meeting.get("action", ""),
+                "magnitude": meeting.get("magnitude", ""),
+                "key_economic_factors": meeting.get("key_economic_factors", []),
+                "economic_outlook": meeting.get("economic_outlook", ""),
+                "market_impact": meeting.get("market_impact", ""),
+                "full_text": meeting.get("full_text", "")[:500] + "..." if meeting.get("full_text") else "",
+                "url": meeting.get("url", "")
+            })
+        return meetings
+    except FileNotFoundError:
+        print("Fed processed meetings file not found. Using fallback data.")
+        return [
+            {
+                "date": "2025-06-18",
+                "title": "FOMC Meeting 2025-06-18",
+                "rate_decision": "4.25%-4.50%",
+                "summary": "No processed data available. Please run the data pipeline first.",
+                "action": "Unknown",
+                "magnitude": "Unknown",
+                "key_economic_factors": [],
+                "economic_outlook": "Data not available",
+                "market_impact": "Data not available",
+                "full_text": "No data available",
+                "url": ""
+            }
+        ]
+    except Exception as e:
+        print(f"Error loading processed meetings: {e}")
+        return []
+
+# Load the processed meetings
+FOMC_MEETINGS = load_processed_meetings()
 
 def simulate_llm_function_call(user_message: str, selected_model: str) -> Dict[str, Any]:
     """Simulate LLM function calls based on user intent"""
@@ -110,12 +137,12 @@ def format_response_with_reasoning(function_result: Dict[str, Any], model_name:
 """
     return response
 
-def respond(
+def respond_for_chat_interface(
     message: str,
     history: list[tuple[str, str]],
     api_key: str,
 ):
-    """Enhanced response function with Fed AI Savant capabilities using OAI OSS 120B"""
+    """Enhanced response function for gr.ChatInterface with Fed AI Savant capabilities"""
     
     if not message.strip():
         yield "Please enter a question about Federal Reserve policy or FOMC meetings."
@@ -147,7 +174,7 @@ def get_fomc_meetings_sidebar():
     """Generate sidebar content with FOMC meeting details"""
     sidebar_content = "## 📋 Recent FOMC Meetings\n\n"
     
-    for meeting in MOCK_FOMC_MEETINGS:
+    for meeting in FOMC_MEETINGS:
         sidebar_content += f"""
 **{meeting['date']}**
 *{meeting['title'][:50]}...*
@@ -224,7 +251,7 @@ MODEL_OPTIONS = [
 def create_fomc_meetings_accordion():
     """Create searchable accordion for FOMC meetings"""
     accordions = []
-    for meeting in MOCK_FOMC_MEETINGS:
+    for meeting in FOMC_MEETINGS:
         title = f"{meeting['date']} - Rate: {meeting['rate_decision']}"
         content = f"""
 **Meeting Title:** {meeting['title']}
@@ -246,7 +273,7 @@ with gr.Blocks(css=custom_css, title="Fed AI Savant", theme=gr.themes.Soft()) as
     with gr.Row():
         with gr.Column():
             gr.Markdown("""
-            # 🏛️ Fed AI Savant
+            # 🏛️ Fed AI Savant 🏛️ 
             **Intelligent Analysis of Federal Reserve Policy and FOMC Meetings**
             
             Ask questions about interest rate decisions, monetary policy changes, and economic analysis based on Federal Reserve meeting minutes.
@@ -254,16 +281,27 @@ with gr.Blocks(css=custom_css, title="Fed AI Savant", theme=gr.themes.Soft()) as
     
     # Row 2: API Key Configuration
     with gr.Row():
-        with gr.Column(scale=2):
-            gr.Markdown("### 🔐 Configuration")
+        with gr.Column(scale=1):
+            gr.Markdown("### Powered by")
+            gr.Image(
+                value=str(_FILE_PATH / "assets" / "fireworks_logo.png"),
+                height=60,
+                width=200,
+                show_label=False,
+                show_download_button=False,
+                container=False,
+                show_fullscreen_button=False,
+                show_share_button=False,
+            )
+        with gr.Column(scale=1):
+            gr.Markdown("###  🔑 Configuration")
             api_key = gr.Textbox(
                 label="AI API Key",
                 type="password",
-                placeholder="Enter your OpenAI, Anthropic, or other AI API key",
-                value=os.getenv("OPENAI_API_KEY", ""),
-                info="💡 Your API key is required to analyze Fed policy using AI"
+                placeholder="Please enter your FireworksAI API key",
+                value=os.getenv("FIREWORKS_API_KEY", ""),
             )
-        with gr.Column(scale=3):
+        with gr.Column(scale=2):
             gr.Markdown("### 📋 How to Use")
             gr.Markdown("""
             1. **Enter your AI API key** (OpenAI, Anthropic, etc.)
@@ -284,205 +322,97 @@ with gr.Blocks(css=custom_css, title="Fed AI Savant", theme=gr.themes.Soft()) as
                 lines=1
             )
             
-            # Meetings accordion
-            with gr.Accordion("Recent FOMC Meetings", open=False):
-                meetings_accordion = gr.HTML("""
-                <div style="space-y: 8px;">
-                    <details style="border: 1px solid #e5e7eb; border-radius: 6px; padding: 12px; margin-bottom: 8px;">
-                        <summary style="font-weight: 600; cursor: pointer; color: #1f2937;">
-                            📅 2024-07-31 - Rate: 5.25-5.50%
-                        </summary>
-                        <div style="margin-top: 12px; padding-top: 12px; border-top: 1px solid #e5e7eb;">
-                            <p><strong>Meeting:</strong> Minutes of the Federal Open Market Committee July 30-31, 2024</p>
-                            <p><strong>Decision:</strong> 5.25-5.50%</p>
-                            <p><strong>Summary:</strong> The Committee maintained the federal funds rate target range at 5.25-5.50%. Inflation has eased over the past year but remains elevated.</p>
-                        </div>
-                    </details>
-                    
-                    <details style="border: 1px solid #e5e7eb; border-radius: 6px; padding: 12px; margin-bottom: 8px;">
-                        <summary style="font-weight: 600; cursor: pointer; color: #1f2937;">
-                            📅 2024-06-12 - Rate: 5.25-5.50%
-                        </summary>
-                        <div style="margin-top: 12px; padding-top: 12px; border-top: 1px solid #e5e7eb;">
-                            <p><strong>Meeting:</strong> Minutes of the Federal Open Market Committee June 11-12, 2024</p>
-                            <p><strong>Decision:</strong> 5.25-5.50%</p>
-                            <p><strong>Summary:</strong> The Committee held rates steady as inflation showed signs of cooling but labor market remained robust.</p>
-                        </div>
-                    </details>
+            with gr.Accordion("FOMC Meetings", open=False):
+                # Dynamic HTML generation for meetings
+                def generate_meetings_html(meetings_list):
+                    """Generate HTML for meetings list"""
+                    if not meetings_list:
+                        return '<p style="color: #6b7280; text-align: center; padding: 20px;">No meetings available</p>'
                     
-                    <details style="border: 1px solid #e5e7eb; border-radius: 6px; padding: 12px; margin-bottom: 8px;">
-                        <summary style="font-weight: 600; cursor: pointer; color: #1f2937;">
-                            📅 2024-05-01 - Rate: 5.25-5.50%
-                        </summary>
-                        <div style="margin-top: 12px; padding-top: 12px; border-top: 1px solid #e5e7eb;">
-                            <p><strong>Meeting:</strong> Minutes of the Federal Open Market Committee April 30-May 1, 2024</p>
-                            <p><strong>Decision:</strong> 5.25-5.50%</p>
-                            <p><strong>Summary:</strong> Officials maintained current rates while monitoring inflation progress and employment data.</p>
-                        </div>
-                    </details>
-                </div>
-                """)
+                    html_content = '<div style="space-y: 8px;">'
+                    for meeting in meetings_list:
+                        # Format key economic factors for display (show all factors)
+                        factors_html = ""
+                        if meeting.get('key_economic_factors') and len(meeting['key_economic_factors']) > 0:
+                            factors_html = "<p><strong>Key Factors:</strong></p><ul>"
+                            for factor in meeting['key_economic_factors']:  # Show all factors
+                                factors_html += f"<li>{factor}</li>"
+                            factors_html += "</ul>"
+                        
+                        html_content += f"""
+                        <details style="border: 1px solid #e5e7eb; border-radius: 6px; padding: 12px; margin-bottom: 8px;">
+                            <summary style="font-weight: 600; cursor: pointer; color: #1f2937;">
+                                📅 {meeting['date']} - Rate: {meeting['rate_decision']}
+                            </summary>
+                            <div style="margin-top: 12px; padding-top: 12px; border-top: 1px solid #e5e7eb;">
+                                <p><strong>Meeting:</strong> {meeting['title']}</p>
+                                <p><strong>Action:</strong> {meeting.get('action', 'N/A')}</p>
+                                <p><strong>Rate:</strong> {meeting['rate_decision']}</p>
+                                <p><strong>Magnitude:</strong> {meeting.get('magnitude', 'N/A')}</p>
+                                <p><strong>Forward Guidance:</strong> {meeting['summary']}</p>
+                                {factors_html}
+                                <p><strong>Economic Outlook:</strong> {meeting.get('economic_outlook', 'N/A')}</p>
+                                <p><strong>Market Impact:</strong> {meeting.get('market_impact', 'N/A')}</p>
+                                {f'<p><strong>Source:</strong> <a href="{meeting["url"]}" target="_blank">Fed Minutes PDF</a></p>' if meeting.get('url') else ''}
+                            </div>
+                        </details>
+                        """
+                    html_content += '</div>'
+                    return html_content
+
+                meetings_accordion = gr.HTML(generate_meetings_html(FOMC_MEETINGS))
     
-    # Row 4: Chat Interface with Audio
+    # Row 4: Chat Interface using gr.ChatInterface
     with gr.Row():
         with gr.Column():
             gr.Markdown("### 💬 Fed AI Assistant")
-            
-            # Example questions (moved above chat)
-            gr.Markdown("### 💡 Example Questions")
-            with gr.Row():
-                example1 = gr.Button("What was the rate decision in the last FOMC meeting?", variant="secondary", size="sm")
-                example2 = gr.Button("Compare June 2024 vs July 2024 FOMC meetings", variant="secondary", size="sm")
-            with gr.Row():
-                example3 = gr.Button("Tell me about inflation expectations", variant="secondary", size="sm")
-                example4 = gr.Button("What factors influenced recent policy decisions?", variant="secondary", size="sm")
-            
-            # Chat interface
-            chatbot = gr.Chatbot(
-                label="Fed AI Assistant",
-                height=400,
-                show_label=False,
-                avatar_images=(None, None)
+
+            chat_interface = gr.ChatInterface(
+                fn=respond_for_chat_interface,
+                chatbot=gr.Chatbot(height=200, show_label=False),
+                textbox=gr.Textbox(placeholder="Ask about Fed policy, rate decisions, or FOMC meetings...", scale=10),
+                examples=[
+                    "What was the rate decision in the last FOMC meeting?"
+                    "Compare June 2024 vs July 2024 FOMC meetings",
+                    "Tell me about inflation expectations",
+                    "Has the Fed's employment stance changed?",
+                    "What was the rate decision in the last FOMC meeting?",
+                ],
+                submit_btn="Send",
             )
-            
-            # Input row with text and audio button (larger text, smaller button)
-            with gr.Row():
-                with gr.Column(scale=10):
-                    msg = gr.Textbox(
-                        placeholder="Ask about Fed policy, rate decisions, or FOMC meetings...",
-                        label="Your Question",
-                        lines=2,
-                        show_label=False
-                    )
-                with gr.Column(scale=1):
-                    # Audio components
-                    audio_input = gr.Audio(
-                        label="🎙️ Voice Input",
-                        sources=["microphone"],
-                        type="filepath",
-                        visible=False
-                    )
-                    voice_btn = gr.Button("🎙️", variant="secondary", size="sm", min_width=40)
-    
-    # Chat functionality
-    def user_message(message, history):
-        return "", history + [[message, None]]
     
-    def bot_response(history, api_key_val):
-        if history and history[-1][1] is None:
-            user_msg = history[-1][0]
-            bot_msg = ""
-            
-            # Get the response generator with simplified parameters
-            response_gen = respond(user_msg, history[:-1], api_key_val)
-            
-            for partial_response in response_gen:
-                bot_msg = partial_response
-                history[-1][1] = bot_msg
-                yield history
-    
-    msg.submit(
-        user_message, 
-        [msg, chatbot], 
-        [msg, chatbot], 
-        queue=False
-    ).then(
-        bot_response,
-        [chatbot, api_key],
-        chatbot
-    )
-    
-    # Voice functionality
-    def toggle_audio_input():
-        return gr.Audio(visible=True)
-    
-    def handle_audio_input(audio_file):
-        if audio_file:
-            transcribed_text = process_audio_input(audio_file)
-            return transcribed_text, gr.Audio(visible=False)
-        return "", gr.Audio(visible=False)
-    
-    def handle_tts(history):
-        if history and len(history) > 0 and history[-1][1]:
-            last_response = history[-1][1]
-            tts_message = text_to_speech(last_response)
-            return tts_message
-        return "No response to read aloud."
     
     # Search functionality for FOMC meetings
     def search_meetings(search_term):
         """Filter FOMC meetings based on search term"""
         if not search_term.strip():
             # Return all meetings if no search term
-            html_content = """
-            <div style="space-y: 8px;">
-                <details style="border: 1px solid #e5e7eb; border-radius: 6px; padding: 12px; margin-bottom: 8px;">
-                    <summary style="font-weight: 600; cursor: pointer; color: #1f2937;">
-                        📅 2024-07-31 - Rate: 5.25-5.50%
-                    </summary>
-                    <div style="margin-top: 12px; padding-top: 12px; border-top: 1px solid #e5e7eb;">
-                        <p><strong>Meeting:</strong> Minutes of the Federal Open Market Committee July 30-31, 2024</p>
-                        <p><strong>Decision:</strong> 5.25-5.50%</p>
-                        <p><strong>Summary:</strong> The Committee maintained the federal funds rate target range at 5.25-5.50%. Inflation has eased over the past year but remains elevated.</p>
-                    </div>
-                </details>
-                
-                <details style="border: 1px solid #e5e7eb; border-radius: 6px; padding: 12px; margin-bottom: 8px;">
-                    <summary style="font-weight: 600; cursor: pointer; color: #1f2937;">
-                        📅 2024-06-12 - Rate: 5.25-5.50%
-                    </summary>
-                    <div style="margin-top: 12px; padding-top: 12px; border-top: 1px solid #e5e7eb;">
-                        <p><strong>Meeting:</strong> Minutes of the Federal Open Market Committee June 11-12, 2024</p>
-                        <p><strong>Decision:</strong> 5.25-5.50%</p>
-                        <p><strong>Summary:</strong> The Committee held rates steady as inflation showed signs of cooling but labor market remained robust.</p>
-                    </div>
-                </details>
-                
-                <details style="border: 1px solid #e5e7eb; border-radius: 6px; padding: 12px; margin-bottom: 8px;">
-                    <summary style="font-weight: 600; cursor: pointer; color: #1f2937;">
-                        📅 2024-05-01 - Rate: 5.25-5.50%
-                    </summary>
-                    <div style="margin-top: 12px; padding-top: 12px; border-top: 1px solid #e5e7eb;">
-                        <p><strong>Meeting:</strong> Minutes of the Federal Open Market Committee April 30-May 1, 2024</p>
-                        <p><strong>Decision:</strong> 5.25-5.50%</p>
-                        <p><strong>Summary:</strong> Officials maintained current rates while monitoring inflation progress and employment data.</p>
-                    </div>
-                </details>
-            </div>
-            """
+            return generate_meetings_html(FOMC_MEETINGS)
         else:
             # Filter meetings based on search term
             filtered_meetings = []
             search_lower = search_term.lower()
             
-            for meeting in MOCK_FOMC_MEETINGS:
-                # Search in date, title, or summary
-                if (search_lower in meeting['date'].lower() or
-                    search_lower in meeting['title'].lower() or 
-                    search_lower in meeting['summary'].lower() or
-                    search_lower in meeting['rate_decision'].lower()):
+            for meeting in FOMC_MEETINGS:
+                # Search in date, title, summary, economic factors, etc.
+                search_fields = [
+                    meeting.get('date', ''),
+                    meeting.get('title', ''),
+                    meeting.get('summary', ''),
+                    meeting.get('rate_decision', ''),
+                    meeting.get('action', ''),
+                    meeting.get('economic_outlook', ''),
+                    meeting.get('market_impact', ''),
+                    ' '.join(meeting.get('key_economic_factors', []))
+                ]
+                
+                if any(search_lower in field.lower() for field in search_fields):
                     filtered_meetings.append(meeting)
             
             if filtered_meetings:
-                html_content = '<div style="space-y: 8px;">'
-                for meeting in filtered_meetings:
-                    html_content += f"""
-                    <details style="border: 1px solid #e5e7eb; border-radius: 6px; padding: 12px; margin-bottom: 8px;">
-                        <summary style="font-weight: 600; cursor: pointer; color: #1f2937;">
-                            📅 {meeting['date']} - Rate: {meeting['rate_decision']}
-                        </summary>
-                        <div style="margin-top: 12px; padding-top: 12px; border-top: 1px solid #e5e7eb;">
-                            <p><strong>Meeting:</strong> {meeting['title']}</p>
-                            <p><strong>Decision:</strong> {meeting['rate_decision']}</p>
-                            <p><strong>Summary:</strong> {meeting['summary']}</p>
-                        </div>
-                    </details>
-                    """
-                html_content += '</div>'
+                return generate_meetings_html(filtered_meetings)
             else:
-                html_content = f'<p style="color: #6b7280; text-align: center; padding: 20px;">No meetings found matching "{search_term}"</p>'
-        
-        return html_content
+                return f'<p style="color: #6b7280; text-align: center; padding: 20px;">No meetings found matching "{search_term}"</p>'
     
     # Wire up search functionality
     date_search.change(
@@ -491,23 +421,7 @@ with gr.Blocks(css=custom_css, title="Fed AI Savant", theme=gr.themes.Soft()) as
         outputs=meetings_accordion
     )
     
-    # Wire up voice buttons
-    voice_btn.click(
-        toggle_audio_input,
-        outputs=audio_input
-    )
-    
-    audio_input.change(
-        handle_audio_input,
-        inputs=audio_input,
-        outputs=[msg, audio_input]
-    )
-    
-    # Wire up example question buttons
-    example1.click(lambda: "What was the rate decision in the last FOMC meeting?", outputs=msg)
-    example2.click(lambda: "Compare June 2024 vs July 2024 FOMC meetings", outputs=msg)
-    example3.click(lambda: "Tell me about inflation expectations", outputs=msg)
-    example4.click(lambda: "What factors influenced recent policy decisions?", outputs=msg)
+    # Example buttons are now handled by ChatInterface examples parameter
 
 if __name__ == "__main__":
     demo.launch()

src/modules/data_pipeline.py

@@ -16,11 +16,13 @@ from dotenv import load_dotenv
 from pydantic import BaseModel
 import pdfplumber
 import tempfile
+import argparse
 
 from src.modules.llm_completions import get_llm, run_multi_llm_completions
 from src.modules.constants import PROMPT_LIBRARY
 
 DATA_DIR = Path(__file__).parents[2] / "data"
+PROCESSED_MEETINGS = "fed_processed_meetings.json"
 
 class RateDecision(BaseModel):
     """Enhanced Pydantic model for comprehensive Fed decision analysis"""
@@ -41,8 +43,14 @@ class Meeting:
         self.title = title
         self.full_text = full_text
         self.url = url
-        self.rate_decision = None
+        self.action = None
         self.summary = None
+        self.rate = None
+        self.magnitude = None
+        self.forward_guidance = None
+        self.key_economic_factors = None
+        self.economic_outlook = None
+        self.market_impact = None
         
     def to_dict(self) -> Dict:
         return {
@@ -50,8 +58,13 @@ class Meeting:
             "title": self.title,
             "full_text": self.full_text,
             "url": self.url,
-            "rate_decision": self.rate_decision,
-            "summary": self.summary
+            "action": self.action,
+            "rate": self.rate,
+            "magnitude": self.magnitude,
+            "forward_guidance": self.forward_guidance,
+            "key_economic_factors": self.key_economic_factors,
+            "economic_outlook": self.economic_outlook,
+            "market_impact": self.market_impact
         }
     
     @classmethod
@@ -59,6 +72,13 @@ class Meeting:
         meeting = cls(data["date"], data["title"], data["full_text"], data.get("url", ""))
         meeting.rate_decision = data.get("rate_decision")
         meeting.summary = data.get("summary")
+        meeting.action = data.get("action")
+        meeting.rate = data.get("rate")
+        meeting.magnitude = data.get("magnitude")
+        meeting.forward_guidance = data.get("forward_guidance")
+        meeting.key_economic_factors = data.get("key_economic_factors")
+        meeting.economic_outlook = data.get("economic_outlook")
+        meeting.market_impact = data.get("market_impact")
         return meeting
 
 
@@ -114,8 +134,9 @@ class FedScraper:
             response.raise_for_status()
             return BeautifulSoup(response.content, 'html.parser')
 
-    async def scrape_meetings(self, max_meetings: int = 20, year_range: Tuple[int, int] = (2022, 2024)) -> List[
-        Meeting]:
+    async def scrape_meetings(
+            self, max_meetings: int = 20, year_range: Tuple[int, int] = (2022, 2024)
+    ) -> List[Meeting]:
         """Scrape multiple meetings"""
         print("Fetching FOMC calendar page...")
         soup = self.get_calendar_page()
@@ -123,7 +144,7 @@ class FedScraper:
         print(f"Extracting meeting links for years {year_range[0]}-{year_range[1]}...")
         meeting_links = self.extract_meeting_links(soup, year_range)
 
-        pdf_links = [
+        meeting_links = [
             (date, f"FOMC Meeting {date}", link)
             for date, _, link in meeting_links if link.lower().endswith('.pdf')
         ]
@@ -141,8 +162,8 @@ class FedScraper:
 
         meetings = []
 
-        async with self:  # This will call __aenter__ and __aexit__
-            for i, (date, title, url) in enumerate(pdf_links, 1):
+        async with self:
+            for i, (date, title, url) in enumerate(meeting_links, 1):
                 try:
                     print(f"\n[{i}/{len(meeting_links)}] Scraping: {date}")
                     print(f"  URL: {url}")
@@ -157,8 +178,8 @@ class FedScraper:
 
                     # Rate limiting - be respectful to Fed servers
                     if i < len(meeting_links):
-                        print("  Waiting 2 seconds before next request...")
-                        await asyncio.sleep(2)
+                        print("  Waiting 1 seconds before next request...")
+                        await asyncio.sleep(1)
 
                 except Exception as e:
                     print(f"  Error scraping meeting {date}: {e}")
@@ -328,8 +349,8 @@ class DataProcessor:
         self.llm = get_llm(model, api_key)
 
 
-    async def process_meetings(self, meetings: List[Meeting]) -> List[str]:
-        """Process all meetings with LLM analysis"""
+    async def process_meetings(self, meetings: List[Meeting]) -> List[Meeting]:
+        """Process all meetings with LLM analysis and update meeting objects"""
         print(f"Processing {len(meetings)} meetings with LLM analysis...")
 
         prompts = [
@@ -347,7 +368,25 @@ class DataProcessor:
             output_class=RateDecision
         )
 
-        return meetings_extracted
+        final_results = [
+            RateDecision.model_validate_json(
+                response.choices[0].message.content
+            )
+            for response in meetings_extracted
+        ]
+
+        # Update meetings with processed results
+        if len(final_results) == len(meetings):
+            for i, result in enumerate(final_results):
+                meetings[i].action = result.action
+                meetings[i].rate = result.rate
+                meetings[i].magnitude = result.magnitude
+                meetings[i].forward_guidance = result.forward_guidance
+                meetings[i].key_economic_factors = result.key_economic_factors
+                meetings[i].economic_outlook = result.economic_outlook
+                meetings[i].market_impact = result.market_impact
+        
+        return meetings
 
 
 class FedDataPipeline:
@@ -400,16 +439,8 @@ class FedDataPipeline:
         
         print(f"\nProcessing {len(meetings)} meetings with LLM analysis...")
         processed_results = await self.processor.process_meetings(meetings)
-        
-        # Update meetings with processed results
-        if len(processed_results) == len(meetings):
-            for i, result in enumerate(processed_results):
-                meetings[i].rate_decision = result.dict() if hasattr(result, 'dict') else result
-        
-        # Save final processed data
-        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-        processed_filename = f"fed_meetings_processed_{timestamp}.json"
-        output_file = self.save_meetings(meetings, processed_filename)
+
+        output_file = self.save_meetings(processed_results, PROCESSED_MEETINGS)
         
         print("\nProcessing completed successfully!")
         print(f"Processed data: {output_file}")
@@ -438,16 +469,11 @@ class FedDataPipeline:
         # Step 2: Process with LLM analysis
         print("\n2. Processing meetings with LLM analysis...")
         processed_results = await self.processor.process_meetings(meetings)
-        
-        # Update meetings with processed results
-        if len(processed_results) == len(meetings):
-            for i, result in enumerate(processed_results):
-                meetings[i].rate_decision = result.dict() if hasattr(result, 'dict') else result
+
         
         # Step 3: Save final processed data
         print("\n3. Saving final processed data...")
-        processed_filename = f"fed_meetings_processed_{timestamp}.json"
-        output_file = self.save_meetings(meetings, processed_filename)
+        output_file = self.save_meetings(processed_results, PROCESSED_MEETINGS)
         
         print("\nPipeline completed successfully!")
         print(f"Scraped data: {scraped_filepath}")
@@ -457,16 +483,13 @@ class FedDataPipeline:
 
 async def main():
     """Main function for running the pipeline as a script"""
-    import argparse
-    
-    # Load environment variables
+
     load_dotenv()
     
     parser = argparse.ArgumentParser(description="Fed AI Savant Data Pipeline")
-    parser.add_argument("--max-meetings", type=int, default=3, help="Maximum number of meetings to scrape")
+    parser.add_argument("--max-meetings", type=int, default=25, help="Maximum number of meetings to scrape")
     parser.add_argument("--start-year", type=int, default=2022, help="Start year for meeting range")
     parser.add_argument("--end-year", type=int, default=2025, help="End year for meeting range")
-    parser.add_argument("--data-dir", default="data", help="Directory to save data files")
     parser.add_argument("--from-scraped", type=str, help="Process from already scraped data file (skips scraping)")
     
     args = parser.parse_args()
@@ -482,7 +505,6 @@ async def main():
     pipeline = FedDataPipeline(
         api_key=api_key,
         model="small",
-        data_dir=args.data_dir
     )
     
     # Check if processing from already scraped data

src/modules/llm_completions.py

@@ -3,18 +3,18 @@ from pydantic import BaseModel
 import asyncio
 
 MODELS = {
-    "small": "accounts/fireworks/models/gpt-oss-20b",
-    "large": "accounts/fireworks/models/gpt-oss-120b"
+    "small": "accounts/fireworks/models/qwen3-235b-a22b-instruct-2507",
+    "large": "accounts/fireworks/models/kimi-k2-instruct"
 }
 
-semaphore = asyncio.Semaphore(100)
+semaphore = asyncio.Semaphore(10)
 
 def get_llm(model: str, api_key: str) -> LLM:
     return LLM(model=MODELS[model], api_key=api_key, deployment_type="serverless")
 
 
 async def get_llm_completion(llm: LLM, prompt_text: str, output_class: BaseModel = None) -> str:
-    if isinstance(output_class, BaseModel):
+    if output_class:
         return llm.chat.completions.create(
             messages=[
                 {
@@ -23,7 +23,10 @@ async def get_llm_completion(llm: LLM, prompt_text: str, output_class: BaseModel
                 },
             ],
             temperature=0.1,
-            output_class=output_class
+            response_format={
+                "type": "json_object",
+                "schema": output_class.model_json_schema(),
+            },
         )
     return llm.chat.completions.create(
         messages=[
@@ -45,14 +48,16 @@ async def run_multi_llm_completions(llm: LLM, prompts: list[str], output_class:
     :param output_class:
     :return:
     """
-    with semaphore:
-        if isinstance(output_class, BaseModel):
+    async with semaphore:
+        if output_class:
+            print(f"Running LLM with structured outputs")
             tasks = [
                 asyncio.create_task(
                     get_llm_completion(llm=llm, prompt_text=prompt, output_class=output_class)
                 ) for prompt in prompts
             ]
         else:
+            print(f"Running LLM with non-structured outputs")
             tasks = [
                 asyncio.create_task(
                     get_llm_completion(llm=llm, prompt_text=prompt)