示例import numpy as np import pandas as pd from sklearn.feature_selection import VarianceThreshold from sklearn.preprocessing import StandardScaler from sklearn.datasets import make_classification from sklearn.model_selection import train_test_split from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score # 定义函数 def ignore_low_variance(X_train, X_test, threshold0.01, use_stdFalse, scale_dataFalse, verboseTrue): 删除低方差列 Parameters: ----------- X_train, X_test : DataFrame 训练集和测试集 threshold : float 方差阈值 use_std : bool 是否使用标准差模式 scale_data : bool 是否先标准化数据推荐 verbose : bool 是否打印删除信息 # 1. 只选择数值列 numeric_cols X_train.select_dtypes(include[np.number]).columns if len(numeric_cols) X_train.shape[1]: print(f警告排除了 {X_train.shape[1] - len(numeric_cols)} 个非数值列) X_train_num X_train[numeric_cols].copy() X_test_num X_test[numeric_cols].copy() # 2. 可选标准化数据 if scale_data: from sklearn.preprocessing import StandardScaler scaler StandardScaler() X_train_num pd.DataFrame( scaler.fit_transform(X_train_num), columnsX_train_num.columns, indexX_train_num.index ) X_test_num pd.DataFrame( scaler.transform(X_test_num), columnsX_test_num.columns, indexX_test_num.index ) # 3. 方差过滤 if use_std: threshold threshold ** 2 selector VarianceThreshold(thresholdthreshold) X_train_transformed selector.fit_transform(X_train_num) X_test_transformed selector.transform(X_test_num) # 4. 获取保留的列 cols_kept X_train_num.columns[selector.get_support()].tolist() cols_removed X_train_num.columns[~selector.get_support()].tolist() if verbose and cols_removed: print(f删除了 {len(cols_removed)} 个低方差列: {cols_removed}) # 5. 返回DataFrame包含所有原始列包括非数值列 if len(numeric_cols) X_train.shape[1]: # 保留非数值列 non_numeric_cols X_train.columns.difference(numeric_cols) result_train pd.concat([ pd.DataFrame(X_train_transformed, columnscols_kept, indexX_train.index), X_train[non_numeric_cols] ], axis1) result_test pd.concat([ pd.DataFrame(X_test_transformed, columnscols_kept, indexX_test.index), X_test[non_numeric_cols] ], axis1) return result_train, result_test else: return (pd.DataFrame(X_train_transformed, columnscols_kept, indexX_train.index), pd.DataFrame(X_test_transformed, columnscols_kept, indexX_test.index))调用示例print( * 60) print(示例1模拟数据 - 比较不同策略的效果) print( * 60) # 创建包含不同方差特征的模拟数据 np.random.seed(42) n_samples 500 # 创建不同类型的特征 data { feature_high_var: np.random.randn(n_samples) * 10, # 高方差 feature_medium_var: np.random.randn(n_samples) * 2, # 中方差 feature_low_var: np.random.randn(n_samples) * 0.1, # 低方差 feature_near_constant: np.random.randn(n_samples) * 0.001 5, # 近常数 feature_binary: np.random.binomial(1, 0.5, n_samples), # 二值特征 feature_constant: np.ones(n_samples) * 3, # 常数特征 } X pd.DataFrame(data) y (X[feature_high_var] X[feature_binary] * 2 np.random.randn(n_samples) * 0.5 0).astype(int) # 划分数据集 X_train, X_test, y_train, y_test train_test_split(X, y, test_size0.3, random_state42) print(原始数据形状:, X_train.shape) print(\n各特征方差:) print(X_train.var().round(6)) # 策略1不标准化使用默认阈值 print(\n - * 40) print(策略1不标准化阈值0.01) X_train1, X_test1 ignore_low_variance(X_train, X_test, threshold0.01, scale_dataFalse) print(f处理后形状: {X_train1.shape}) # 策略2先标准化再删除推荐 print(\n - * 40) print(策略2先标准化阈值0.01推荐) X_train2, X_test2 ignore_low_variance(X_train, X_test, threshold0.01, scale_dataTrue) print(f处理后形状: {X_train2.shape}) # 策略3使用标准差模式 print(\n - * 40) print(策略3使用标准差模式阈值0.1) X_train3, X_test3 ignore_low_variance(X_train, X_test, threshold0.1, use_stdTrue, scale_dataTrue) print(f处理后形状: {X_train3.shape})输出原始数据形状: (350, 6) 各特征方差: feature_high_var 98.129830 feature_medium_var 3.586963 feature_low_var 0.010559 feature_near_constant 0.000001 feature_binary 0.250053 feature_constant 0.000000 dtype: float64 ---------------------------------------- 策略1不标准化阈值0.01 删除了 2 个低方差列: [feature_near_constant, feature_constant] 处理后形状: (350, 4) ---------------------------------------- 策略2先标准化阈值0.01推荐 删除了 1 个低方差列: [feature_constant] 处理后形状: (350, 5) ---------------------------------------- 策略3使用标准差模式阈值0.1 删除了 1 个低方差列: [feature_constant] 处理后形状: (350, 5)说明 方法针对的是数值列一般数据先标准化 再移除低方差列
