from sklearn.linear_model import Ridge
from sklearn.preprocessing import StandardScaler

# Always scale before regularization!
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Try different alpha values
ridge_low = Ridge(alpha=0.01)   # Less regularization
ridge_med = Ridge(alpha=1.0)    # Medium
ridge_high = Ridge(alpha=100)   # High regularization

# Train
ridge_med.fit(X_scaled, y)

# Coefficients
print("Coefficients:", ridge_med.coef_)
print("Intercept:", ridge_med.intercept_)

import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import Ridge

alphas = [0.001, 0.01, 0.1, 1, 10, 100, 1000]
coefficients = []

for alpha in alphas:
    model = Ridge(alpha=alpha)
    model.fit(X_scaled, y)
    coefficients.append(model.coef_)

# Plot
plt.figure(figsize=(10, 6))
plt.plot(alphas, coefficients)
plt.xscale('log')
plt.xlabel('Alpha (Regularization Strength)')
plt.ylabel('Coefficient Value')
plt.title('Ridge: Coefficients Shrink as Alpha Increases')
plt.legend(X.columns, loc='best')
plt.grid(True)
plt.show()

from sklearn.linear_model import Lasso

# Train Lasso
lasso = Lasso(alpha=1.0)
lasso.fit(X_scaled, y)

# Check which features were selected
n_kept = np.sum(lasso.coef_ != 0)
n_removed = np.sum(lasso.coef_ == 0)

print(f"Features kept: {n_kept}")
print(f"Features removed: {n_removed}")

# Show coefficients
for feature, coef in zip(X.columns, lasso.coef_):
    status = "KEPT" if coef != 0 else "REMOVED"
    print(f"  {feature}: {coef:.4f} - {status}")

# Use Lasso to find important features
from sklearn.linear_model import LassoCV

# LassoCV automatically finds best alpha
lasso_cv = LassoCV(cv=5, random_state=42)
lasso_cv.fit(X_scaled, y)

print(f"Best alpha: {lasso_cv.alpha_:.4f}")

# Selected features
selected = X.columns[lasso_cv.coef_ != 0]
print(f"Selected features: {selected.tolist()}")

from sklearn.linear_model import ElasticNet, ElasticNetCV

# Manual setup
elastic = ElasticNet(
    alpha=1.0,        # Total regularization
    l1_ratio=0.5      # 50% L1, 50% L2
)
elastic.fit(X_scaled, y)

# Auto-tune with cross-validation
elastic_cv = ElasticNetCV(
    l1_ratio=[0.1, 0.3, 0.5, 0.7, 0.9],  # Try different mixes
    cv=5,
    random_state=42
)
elastic_cv.fit(X_scaled, y)

print(f"Best alpha: {elastic_cv.alpha_:.4f}")
print(f"Best l1_ratio: {elastic_cv.l1_ratio_}")

from sklearn.linear_model import RidgeCV, LassoCV, ElasticNetCV

# Define alpha range
alphas = np.logspace(-4, 4, 50)  # 10^-4 to 10^4

# Ridge with CV
ridge_cv = RidgeCV(alphas=alphas, cv=5)
ridge_cv.fit(X_scaled, y)
print(f"Best Ridge alpha: {ridge_cv.alpha_}")

# Lasso with CV
lasso_cv = LassoCV(alphas=alphas, cv=5, random_state=42)
lasso_cv.fit(X_scaled, y)
print(f"Best Lasso alpha: {lasso_cv.alpha_}")

# Elastic Net with CV
elastic_cv = ElasticNetCV(
    l1_ratio=[0.1, 0.5, 0.7, 0.9],
    alphas=alphas,
    cv=5,
    random_state=42
)
elastic_cv.fit(X_scaled, y)
print(f"Best Elastic alpha: {elastic_cv.alpha_}")
print(f"Best Elastic l1_ratio: {elastic_cv.l1_ratio_}")

import matplotlib.pyplot as plt
from sklearn.linear_model import Ridge
from sklearn.model_selection import cross_val_score

alphas = np.logspace(-4, 4, 50)
scores = []

for alpha in alphas:
    model = Ridge(alpha=alpha)
    score = cross_val_score(model, X_scaled, y, cv=5, scoring='r2').mean()
    scores.append(score)

plt.figure(figsize=(10, 6))
plt.semilogx(alphas, scores)
plt.xlabel('Alpha (log scale)')
plt.ylabel('Cross-validated R²')
plt.title('Finding Best Alpha for Ridge')
plt.axvline(x=alphas[np.argmax(scores)], color='red', linestyle='--', 
            label=f'Best alpha: {alphas[np.argmax(scores)]:.4f}')
plt.legend()
plt.grid(True)
plt.show()

from sklearn.linear_model import LogisticRegression

# L2 (Ridge-like)
logreg_l2 = LogisticRegression(penalty='l2', C=1.0)

# L1 (Lasso-like)
logreg_l1 = LogisticRegression(penalty='l1', solver='liblinear', C=1.0)

# Elastic Net
logreg_elastic = LogisticRegression(
    penalty='elasticnet', 
    solver='saga', 
    l1_ratio=0.5,
    C=1.0
)

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score, mean_squared_error

# Create dataset with many features (some useful, some noise)
np.random.seed(42)
n_samples = 200
n_features = 50  # Many features!

X = np.random.randn(n_samples, n_features)
# Only first 5 features are useful
true_coefs = np.zeros(n_features)
true_coefs[:5] = [3, -2, 4, -1, 5]
y = X @ true_coefs + np.random.randn(n_samples) * 0.5

# Split
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# Scale (CRITICAL for regularization!)
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Train all models
models = {
    'Linear (no reg)': LinearRegression(),
    'Ridge (α=1)': Ridge(alpha=1),
    'Lasso (α=0.1)': Lasso(alpha=0.1),
    'Elastic Net': ElasticNet(alpha=0.1, l1_ratio=0.5)
}

results = {}
for name, model in models.items():
    model.fit(X_train_scaled, y_train)
    
    train_pred = model.predict(X_train_scaled)
    test_pred = model.predict(X_test_scaled)
    
    train_r2 = r2_score(y_train, train_pred)
    test_r2 = r2_score(y_test, test_pred)
    
    n_zero = np.sum(np.abs(model.coef_) < 1e-5)
    
    results[name] = {
        'Train R²': train_r2,
        'Test R²': test_r2,
        'Gap': train_r2 - test_r2,
        'Zero Coefs': n_zero
    }

# Print results
results_df = pd.DataFrame(results).T
print("\n📊 Model Comparison:")
print(results_df.round(3))

# Visualize coefficients
fig, axes = plt.subplots(1, 4, figsize=(20, 5), sharey=True)
for ax, (name, model) in zip(axes, models.items()):
    ax.bar(range(n_features), model.coef_)
    ax.set_title(name)
    ax.set_xlabel('Feature Index')
    ax.axhline(y=0, color='r', linestyle='--', alpha=0.3)
axes[0].set_ylabel('Coefficient Value')
plt.suptitle('Coefficients Across Models', fontsize=14)
plt.tight_layout()
plt.show()

# ❌ WRONG: Different scales
# size: 1000-4000
# bedrooms: 1-5
# Without scaling, "size" dominates the penalty!

# ✅ CORRECT: Always scale first
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
ridge.fit(X_scaled, y)

# ❌ Don't try just one alpha
ridge = Ridge(alpha=1.0)  # Maybe wrong!

# ✅ Try a range with cross-validation
alphas = np.logspace(-4, 4, 50)
ridge_cv = RidgeCV(alphas=alphas, cv=5)

# In Ridge/Lasso: alpha higher = MORE regularization
ridge = Ridge(alpha=10)  # Strong regularization

# In LogisticRegression: C higher = LESS regularization
logreg = LogisticRegression(C=10)  # WEAK regularization!

# To get strong regularization in LogReg:
logreg_strong = LogisticRegression(C=0.1)

# ❌ Can't directly compare with different alphas
ridge_low = Ridge(alpha=0.01)   # Bigger coefs
ridge_high = Ridge(alpha=10)    # Smaller coefs
# These coefs aren't directly comparable!

# ✅ Compare predictions instead
score_low = ridge_low.score(X_test, y_test)
score_high = ridge_high.score(X_test, y_test)

# After Lasso eliminates features, check which ones!
selected = X.columns[lasso.coef_ != 0]
print(f"Lasso selected: {selected.tolist()}")

# Don't assume which features matter

from sklearn.linear_model import RidgeCV, LassoCV, ElasticNetCV
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

# Universal pipeline
def train_with_regularization(X, y, method='ridge'):
    """Train regularized model with auto-tuning"""
    
    if method == 'ridge':
        model = RidgeCV(alphas=np.logspace(-4, 4, 50), cv=5)
    elif method == 'lasso':
        model = LassoCV(alphas=np.logspace(-4, 4, 50), cv=5, random_state=42)
    elif method == 'elastic':
        model = ElasticNetCV(
            l1_ratio=[0.1, 0.5, 0.7, 0.9],
            alphas=np.logspace(-4, 4, 50),
            cv=5, random_state=42
        )
    
    pipeline = Pipeline([
        ('scaler', StandardScaler()),
        ('model', model)
    ])
    
    pipeline.fit(X, y)
    return pipeline

# Use it!
ridge_model = train_with_regularization(X_train, y_train, 'ridge')
lasso_model = train_with_regularization(X_train, y_train, 'lasso')
elastic_model = train_with_regularization(X_train, y_train, 'elastic')