Update advanced_analysis.py

advanced_analysis.py

@@ -11,11 +11,13 @@ from sklearn.feature_extraction import text
 import gensim
 from gensim import corpora
 from gensim.models import LdaModel
-from gensim.summarization import summarize as gensim_summarize
+# Hapus impor yang bermasalah
+# from gensim.summarization import summarize as gensim_summarize
 from transformers import pipeline
 import torch
-from keybert import KeyBERT
-from yake import KeywordExtractor
+# Hapus impor yang membutuhkan OMP
+# from keybert import KeyBERT
+# from yake import KeywordExtractor
 import spacy
 from collections import defaultdict
 import matplotlib.pyplot as plt
@@ -24,6 +26,7 @@ import plotly.graph_objects as go
 from wordcloud import WordCloud
 import io
 import base64
+import os
 
 # Download NLTK data
 try:
@@ -40,8 +43,9 @@ class AdvancedTextAnalysis:
     def __init__(self):
         self.sentiment_analyzer = None
         self.summarizer = None
-        self.keybert_model = None
-        self.nlp = None
+        # Hapus model yang bermasalah
+        # self.keybert_model = None
+        # self.nlp = None
         self.stop_words_id = None
         
         # Load stopwords Indonesia
@@ -138,32 +142,34 @@ class AdvancedTextAnalysis:
     
     def extract_keywords_yake(self, texts, num_keywords=10):
         """
-        Extract keywords menggunakan YAKE
+        Extract keywords menggunakan YAKE - Fallback version
         """
         try:
-            keyword_extractor = KeywordExtractor(
-                lan="id",
-                n=2,  # n-gram size
-                dedupLim=0.8,
-                dedupFunc='seqm',
-                windowsSize=1,
-                top=num_keywords
-            )
-            
             all_keywords = []
             for i, text in enumerate(texts):
                 if not text or len(text.strip()) < 50:
                     continue
                 
                 processed_text = self.preprocess_text(text)
-                keywords = keyword_extractor.extract_keywords(processed_text)
                 
-                for score, keyword in keywords:
+                # Simple keyword extraction based on TF-IDF as fallback
+                words = processed_text.split()
+                word_freq = Counter(words)
+                
+                # Remove stopwords and short words
+                filtered_words = {word: freq for word, freq in word_freq.items() 
+                                if word not in self.stop_words_id and len(word) > 2}
+                
+                # Get top keywords
+                top_keywords = sorted(filtered_words.items(), key=lambda x: x[1], reverse=True)[:num_keywords]
+                
+                for keyword, freq in top_keywords:
+                    score = freq / len(words)  # Simple frequency-based score
                     all_keywords.append({
                         'doc_id': i,
                         'keyword': keyword,
                         'score': round(score, 4),
-                        'type': 'YAKE'
+                        'type': 'FREQUENCY'
                     })
             
             return all_keywords
@@ -210,7 +216,7 @@ class AdvancedTextAnalysis:
     
     def text_summarization(self, texts, ratio=0.3):
         """
-        Text summarization menggunakan extractive methods
+        Text summarization menggunakan extractive methods - Simplified version
         """
         try:
             summaries = []
@@ -227,8 +233,10 @@ class AdvancedTextAnalysis:
                     continue
                 
                 try:
-                    # Gunakan gensim untuk summarization
-                    summary = gensim_summarize(text, ratio=ratio)
+                    # Simple extractive summarization: take first few sentences
+                    sentences = sent_tokenize(text)
+                    num_sentences = max(1, int(len(sentences) * ratio))
+                    summary = ' '.join(sentences[:num_sentences])
                     
                     summaries.append({
                         'doc_id': i,
@@ -516,4 +524,27 @@ def save_advanced_analysis_results(results):
         print("✅ Hasil analisis lanjutan disimpan ke folder 'analisis'")
         
     except Exception as e:
-        print(f"❌ Error menyimpan hasil analisis lanjutan: {e}")
+        print(f"❌ Error menyimpan hasil analisis lanjutan: {e}")
+
+# Tambahkan fungsi dummy untuk menghindari error di main app
+def perform_advanced_analysis_wrapper():
+    """Wrapper function untuk analisis lanjutan"""
+    try:
+        # Load metadata
+        metadata_df = pd.read_csv('scrapper_result/article_metadata.csv')
+        
+        if metadata_df.empty:
+            return "❌ Tidak ada data untuk dianalisis", None, None, None, None, None, None, None
+        
+        # Perform analysis
+        result_msg, results, topic_viz, keyword_viz, concept_viz = perform_advanced_analysis(metadata_df)
+        
+        # Prepare dataframes
+        topic_df = pd.DataFrame(results['topics']['topics']) if results and 'topics' in results else pd.DataFrame()
+        keyword_df = pd.DataFrame(results['keywords']) if results and 'keywords' in results else pd.DataFrame()
+        summary_df = pd.DataFrame(results['summaries']) if results and 'summaries' in results else pd.DataFrame()
+        concept_df = pd.DataFrame(results['concepts']) if results and 'concepts' in results else pd.DataFrame()
+        
+        return result_msg, topic_viz, keyword_viz, concept_viz, topic_df, keyword_df, summary_df, concept_df
+    except Exception as e:
+        return f"❌ Error: {str(e)}", None, None, None, pd.DataFrame(), pd.DataFrame(), pd.DataFrame(), pd.DataFrame()