🧹 Data Cleaning and Preprocessing

# Check for missing values
df.isnull().sum()  # Count per column
df.isnull().sum().sum()  # Total missing

# Visualize missing data pattern
import missingno as msno
msno.matrix(df)  # Visual matrix
msno.bar(df)     # Bar chart

# Find percentage of missing
missing_pct = (df.isnull().sum() / len(df)) * 100
print(missing_pct[missing_pct > 0].sort_values(ascending=False))

# Strategy 1: Drop missing values
df.dropna()  # Drop rows with any NaN
df.dropna(how='all')  # Drop rows where ALL values are NaN
df.dropna(subset=['important_col'])  # Drop if specific column is NaN

# Strategy 2: Fill missing values
df.fillna(0)  # Fill with constant
df.fillna(df.mean())  # Fill with mean
df.fillna(method='ffill')  # Forward fill
df.fillna(method='bfill')  # Backward fill

# Strategy 3: Interpolation
df.interpolate(method='linear')  # Linear interpolation
df.interpolate(method='time')    # Time-based

# Strategy 4: Advanced imputation
from sklearn.impute import KNNImputer
imputer = KNNImputer(n_neighbors=5)
df_imputed = pd.DataFrame(imputer.fit_transform(df), columns=df.columns)

# Find duplicates
df.duplicated()  # Boolean mask
df.duplicated().sum()  # Count duplicates

# Show duplicate rows
df[df.duplicated(keep=False)]  # Show all duplicates

# Remove duplicates
df.drop_duplicates()  # Remove exact duplicates
df.drop_duplicates(subset=['id'])  # Based on specific columns
df.drop_duplicates(keep='first')  # Keep first occurrence
df.drop_duplicates(keep='last')   # Keep last occurrence

# Fuzzy matching for near-duplicates
from fuzzywuzzy import fuzz
def find_similar(name, names_list):
    return [(n, fuzz.ratio(name, n)) for n in names_list 
            if fuzz.ratio(name, n) > 80]

# Check current data types
df.dtypes
df.info()

# Convert data types
df['price'] = df['price'].astype(float)
df['quantity'] = df['quantity'].astype(int)
df['category'] = df['category'].astype('category')

# Parse dates
df['date'] = pd.to_datetime(df['date'], format='%Y-%m-%d')
df['date'] = pd.to_datetime(df['date'], errors='coerce')  # Handle errors

# Convert numeric with error handling
df['amount'] = pd.to_numeric(df['amount'], errors='coerce')

# Memory optimization
df['category'] = df['category'].astype('category')  # For repeated values
df['small_int'] = df['small_int'].astype('int8')   # Use smaller int types

# Statistical methods
# Z-score method
from scipy import stats
z_scores = stats.zscore(df['value'])
outliers = df[abs(z_scores) > 3]

# IQR method
Q1 = df['value'].quantile(0.25)
Q3 = df['value'].quantile(0.75)
IQR = Q3 - Q1
lower_bound = Q1 - 1.5 * IQR
upper_bound = Q3 + 1.5 * IQR
outliers = df[(df['value'] < lower_bound) | (df['value'] > upper_bound)]

# Visual detection
import matplotlib.pyplot as plt
df.boxplot(column='value')
plt.show()

# Handle outliers
# Option 1: Remove
df_clean = df[(df['value'] >= lower_bound) & (df['value'] <= upper_bound)]

# Option 2: Cap values
df['value'] = df['value'].clip(lower=lower_bound, upper=upper_bound)

# Option 3: Transform
df['value_log'] = np.log1p(df['value'])  # Log transformation

# Basic string cleaning
df['name'] = df['name'].str.strip()  # Remove whitespace
df['name'] = df['name'].str.lower()  # Lowercase
df['name'] = df['name'].str.title()  # Title case

# Remove special characters
df['phone'] = df['phone'].str.replace(r'[^0-9]', '', regex=True)

# Fix inconsistencies
df['category'] = df['category'].replace({
    'cat1': 'Category 1',
    'CAT1': 'Category 1',
    'category_1': 'Category 1'
})

# Extract information
df['domain'] = df['email'].str.extract(r'@([^.]+)')
df['year'] = df['date_string'].str.extract(r'(\d{4})')

# Split strings
df[['first_name', 'last_name']] = df['full_name'].str.split(' ', expand=True)

# Normalize text
import unicodedata
def normalize_text(text):
    return unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8')