# metadata 
print(drug_reviews_druglib_com.metadata) 
  
# variable information 
print(drug_reviews_druglib_com.variables) 

# print the first few rows of the dataset
print(X.head())

import pandas as pd
import spacy
from textblob import TextBlob
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
import nltk
import seaborn as sns
import matplotlib.pyplot as plt

# Download necessary NLTK data
nltk.download('punkt')
nltk.download('stopwords')

# Load spaCy model
nlp = spacy.load("en_core_web_sm")

# Data Preprocessing
print("Starting text preprocessing...")

try:
    # Load the dataset
    df = X.copy()  # Create a copy of the original data
    
    # Combine different review fields into a single text field
    print("Combining review fields...")
    df['full_review'] = df['benefitsReview'].fillna('') + ' ' + \
                        df['sideEffectsReview'].fillna('') + ' ' + \
                        df['commentsReview'].fillna('')
    
    # Text Cleaning
    print("Cleaning text...")
    stop_words = set(stopwords.words('english'))
    
    def clean_text(text):
        """
        Clean and preprocess text data:
        1. Convert to lowercase
        2. Tokenize
        3. Remove stopwords and non-alphabetic tokens
        
        Args:
            text (str): Input text to clean
            
        Returns:
            str: Cleaned text
        """
        if not isinstance(text, str):
            return ""
        try:
            tokens = word_tokenize(text.lower())
            tokens = [t for t in tokens if t.isalpha() and t not in stop_words]
            return ' '.join(tokens)
        except Exception as e:
            print(f"Error processing text: {e}")
            return ""
    
    df['clean_review'] = df['full_review'].apply(clean_text)
    
    # Sentiment Analysis
    print("Performing sentiment analysis...")
    def analyze_sentiment(text):
        """
        Calculate sentiment polarity score using TextBlob.
        
        Args:
            text (str): Input text to analyze
            
        Returns:
            float: Sentiment score between -1 (negative) and 1 (positive)
        """
        try:
            blob = TextBlob(text)
            return blob.sentiment.polarity
        except Exception as e:
            print(f"Error analyzing sentiment: {e}")
            return 0.0
    
    df['sentiment_score'] = df['clean_review'].apply(analyze_sentiment)
    print("Preprocessing complete!")
    
except Exception as e:
    print(f"Error during preprocessing: {e}")
    raise

# Create visualization for top drugs by sentiment
plt.figure(figsize=(12, 8))
top_sentiment = sentiment_by_drug.head(10)  # Top 10 drugs

# Create bar plot
ax = sns.barplot(x=top_sentiment.values, y=top_sentiment.index, palette='viridis')

# Customize the plot
plt.title("Top 10 Drugs by Average Sentiment Score", fontsize=14, pad=20)
plt.xlabel("Average Sentiment Score", fontsize=12)
plt.ylabel("Drug Name", fontsize=12)

# Add value labels on the bars
for i, v in enumerate(top_sentiment.values):
    ax.text(v, i, f'{v:.3f}', va='center', fontsize=10)

plt.tight_layout()
plt.show()

# Heatmap of themes by condition
# plt.figure(figsize=(10, 6))
# sns.heatmap(themes_by_condition, cmap="YlGnBu", annot=True, fmt=".0f")
# plt.title("Themes by Condition")
# plt.xlabel("Theme")
# plt.ylabel("Condition")
# plt.tight_layout()
# plt.show()

# ===========================================================================================
# Heatmap of themes by condition for top N conditions
# ===========================================================================================


# Set N for top conditions
N = 15  # You can change this value

# Find top N conditions by number of reviews
top_conditions = df['condition'].value_counts().head(N).index

# Filter theme data to only those top conditions
filtered_themes = themes_by_condition.loc[themes_by_condition.index.isin(top_conditions)]

plt.figure(figsize=(10, 6))
sns.heatmap(filtered_themes, cmap="YlGnBu", annot=True, fmt=".0f")
plt.title(f"Themes by Top {N} Conditions")
plt.xlabel("Theme")
plt.ylabel("Condition")
plt.tight_layout()
plt.show()

# Group low-frequency conditions
condition_counts = df['condition'].value_counts()
threshold = 15  # you can adjust this
df['condition_grouped'] = df['condition'].where(condition_counts > threshold, 'Other')

# Recalculate theme matrix
grouped_themes = df.groupby('condition_grouped')['theme'].value_counts().unstack().fillna(0)

# Plot
plt.figure(figsize=(10, 6))
sns.heatmap(grouped_themes, cmap="coolwarm", annot=True, fmt=".0f")
plt.title("Themes by Grouped Condition")
plt.xlabel("Theme")
plt.ylabel("Grouped Condition")
plt.tight_layout()
plt.show()

theme_counts = df['theme'].value_counts()

sns.barplot(x=theme_counts.index, y=theme_counts.values)
plt.title("Distribution of Themes Across All Conditions")
plt.xlabel("Theme")
plt.ylabel("Count")
plt.tight_layout()
plt.show()

from wordcloud import WordCloud

# Combine all cleaned text into one string
all_text = ' '.join(df['clean_review'].dropna())

# Generate Word Cloud
wordcloud = WordCloud(
    width=1000,
    height=500,
    background_color='white',
    colormap='viridis',
    max_words=25
).generate(all_text)

# Display the image
plt.figure(figsize=(15, 7))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis('off')
plt.title("Most Common Words in Drug Reviews", fontsize=18)
plt.tight_layout()
plt.show()

# Save for use on your website
wordcloud.to_file("project_wordcloud.png")

from wordcloud import WordCloud
import os

# Ensure output folder exists
os.makedirs("wordclouds/themes", exist_ok=True)

themes_list = df['theme'].unique()

for theme in themes_list:
    text = ' '.join(df[df['theme'] == theme]['clean_review'].dropna())
    if len(text.strip()) < 10:
        continue  # Skip short ones

    wc = WordCloud(
        width=1000,
        height=500,
        background_color='white',
        colormap='plasma',
        max_words=25  # 👈 LIMIT TO TOP 25 WORDS
    ).generate(text)

    plt.figure(figsize=(10, 5))
    plt.imshow(wc, interpolation='bilinear')
    plt.axis('off')
    plt.title(f"Top 25 Words – Theme: {theme}", fontsize=16)
    plt.tight_layout()
    plt.show()

    wc.to_file(f"wordclouds/themes/wordcloud_{theme}.png")

os.makedirs("wordclouds/drugs", exist_ok=True)

top_drugs = df['urlDrugName'].value_counts().head(5).index

for drug in top_drugs:
    text = ' '.join(df[df['urlDrugName'] == drug]['clean_review'].dropna())
    if len(text.strip()) < 10:
        continue

    wc = WordCloud(
        width=1000,
        height=500,
        background_color='white',
        colormap='cool',
        max_words=25  # 👈 LIMIT TO TOP 25 WORDS
    ).generate(text)

    plt.figure(figsize=(10, 5))
    plt.imshow(wc, interpolation='bilinear')
    plt.axis('off')
    plt.title(f"Top 25 Words – Drug: {drug}", fontsize=16)
    plt.tight_layout()
    plt.show()

    wc.to_file(f"wordclouds/drugs/wordcloud_{drug.replace('/', '_')}.png")