使用具有多输出回归链的 SKLearn RFECV 获取特征重要性，可能吗？

Question

我正在尝试使用 RFECV 获取最重要功能的列表，但尝试将它与 RegressionChain 一起用于多输出回归问题，并遇到了问题。我在下面创建了一个可重现的代码：

import pandas as pd
import numpy as np
import warnings
warnings.simplefilter("once")
from sklearn.multioutput import RegressorChain
from sklearn.utils import all_estimators
from sklearn.model_selection import *
from sklearn.feature_selection import RFECV
from sklearn.pipeline import Pipeline
from sklearn.datasets import make_regression

# Create multioutput regression dataset and split
X, y = make_regression(n_samples=100, n_features=100, n_informative=25, n_targets=6)
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.80, test_size=0.20, random_state=42)

# Get a list of all SKLearn regressors, and store in dictionary
estimators = all_estimators(type_filter="regressor")
my_estimators = {name[0]:name[1] for name in estimators}

# Iterate through all estimators
for key, value in my_estimators.items():
    
    # Don't use CV estimators, they hang too often
    if 'cv' in str(key).lower():
        continue

    print(key, value)
    
    # Get the estimator being used currently
    try:
        estimator = my_estimators[key]()
    except:
        continue

    # Use RFECV on the multiouput regression problem
    rfe = RFECV(estimator=RegressorChain(base_estimator=estimator, order=[0, 1, 3, 4, 2, 5])) # importance_getter=manual_feature_importance_getter(RegressorChain(base_estimator=estimator, order=[0, 1, 3, 4, 2, 5]), None, 1)
    model = RegressorChain(base_estimator=estimator, order=[0, 1, 3, 4, 2, 5])
    pipeline = Pipeline(steps=[('s',rfe),('m',model)])
    # evaluate model
    cv = KFold(n_splits=2) # n_repeats=2, for Repeated, , random_state=1
    n_scores = cross_val_score(pipeline, X_train, y_train, scoring='neg_root_mean_squared_error', cv=cv, n_jobs=-1, error_score='raise')
    # report performance
    print('neg_root_mean_squared_error: %.3f (%.3f)' % (mean(n_scores), std(n_scores)))

...产生以下错误：

    ValueError: when `importance_getter=='auto'`, the underlying estimator RegressorChain should
 have `coef_` or `feature_importances_` attribute. Either pass a fitted estimator to 
feature selector or call fit before calling transform.

我收到此错误是因为 RegressionChain 不是主估计器，它只是链，所以它没有关联 coef_ 或 feature_importance_ 属性，但基本估计器有。因此，我需要创建自己的特征重要性 getter 函数，并将其传递给我在这里尝试的 RFECV 函数：

def manual_feature_importance_getter(estimator, transform_func=None, norm_order=1):

    # Get the base estimator from the regression chain
    estimator = estimator.base_estimator

    # This is modified from the SKLearn > Feature Selection > _base.py file
    if hasattr(estimator, 'coef_'):
        getter = attrgetter('coef_')
        importances = getter(estimator)

    elif hasattr(estimator, 'feature_importances_'):
        getter = attrgetter('feature_importances_')
        importances = getter(estimator)

    if transform_func is None:
        return importances
    elif transform_func == "norm":
        if importances.ndim == 1:
            importances = np.abs(importances)
        else:
            importances = np.linalg.norm(importances, axis=0,
                                         ord=norm_order)
    elif transform_func == "square":
        if importances.ndim == 1:
            importances = safe_sqr(importances)
        else:
            importances = safe_sqr(importances).sum(axis=0)
    else:
        raise ValueError("Valid values for `transform_func` are " +
                         "None, 'norm' and 'square'. Those two " +
                         "transformation are only supported now")

    return importances

然后我将 RFECV 调用更改为：

rfe = RFECV(estimator=RegressorChain(base_estimator=estimator, order=[0, 1, 3, 4, 2, 5]), importance_getter=manual_feature_importance_getter(RegressorChain(base_estimator=estimator, order=[0, 1, 3, 4, 2, 5]), None, 1))

...但现在出现此错误：

UnboundLocalError: local variable 'importances' referenced before assignment

...这告诉我新函数的 getter 变量分配不起作用，因为尚未安装估计器，因此它还没有 coef_ 或 feature_importance_ 属性。

关于如何让这个东西运行的任何想法？谢谢！

更新

经过一番研究，我发现特征重要性 getter 需要是“可调用的”，因此我将 manual_feature_importance_getter 更改为可调用类，并打印了一些输出以查看它在做什么：

class manual_feature_importance_getter:

    def __init__ (self, estimator, transform_func=None, norm_order=1):
        self.estimator = estimator
        self.transform_func = transform_func
        self.norm_order = norm_order

    def __call__(self, estimator):

        # Get the base estimator from the regression chain
        estimator = estimator.base_estimator

        # This is modified from the SKLearn > Feature Selection > _base.py file
        if hasattr(estimator, 'coef_'):
            print('yay!')
            getter = attrgetter('coef_')
            importances = getter(estimator)

        elif hasattr(estimator, 'feature_importances_'):
            print('yay!')
            getter = attrgetter('feature_importances_')
            importances = getter(estimator)
        
        else:
            print('DOH!')
            importances = np.zeros(shape=(X_train.shape[1],1))
        

        if self.transform_func is None:
            return importances
        elif self.transform_func == "norm":
            if importances.ndim == 1:
                importances = np.abs(importances)
            else:
                importances = np.linalg.norm(importances, axis=0,
                                             ord=self.norm_order)
        elif self.transform_func == "square":
            if importances.ndim == 1:
                importances = safe_sqr(importances)
            else:
                importances = safe_sqr(importances).sum(axis=0)
        else:
            raise ValueError("Valid values for `self.transform_func` are " +
                             "None, 'norm' and 'square'. Those two " +
                             "transformation are only supported now")

        return importances

... 以 ARDRegression <class 'sklearn.linear_model._bayes.ARDRegression'> 回归量开始，但随后输出如下：

ARDRegression <class 'sklearn.linear_model._bayes.ARDRegression'>
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!
DOH!

...这意味着它在运行其 RFECV 时仍然没有从估计器的属性中正确获取特征重要性。就好像它根本不适合回归链的估计器，只是恢复到我在那里添加的零重要性。我制作了零重要性数组，希望它能够从那部分开始，并且重要性会在整个 RFECV 运行期间更新，但这似乎没有发生。有什么想法吗？

Answer 1

我已经找到解决办法了！

我需要在manual_feature_importance_getter 中做的是在链中逐个迭代 FITTED 回归，然后在最后总结重要性。所以我完成的可调用类看起来像这样：

class manual_feature_importance_getter:

    def __init__ (self, estimator, transform_func=None, norm_order=1):
        self.estimator = estimator
        self.transform_func = transform_func
        self.norm_order = norm_order

    def __call__(self, estimator):

        # Get the individual base estimators from the fitted regression chain
        all_estimators = estimator.estimators_

        # A place to store the feature importances for each estimator
        all_importances = []

        # Iterate through the fitted estimators
        for i in range(len(all_estimators)):

            estimator = all_estimators[i]

            # This is modified from the SKLearn > Feature Selection > _base.py file
            if hasattr(estimator, 'coef_'):
                print('yay!')
                getter = attrgetter('coef_')
                importances = getter(estimator)

            elif hasattr(estimator, 'feature_importances_'):
                print('yay!')
                getter = attrgetter('feature_importances_')
                importances = getter(estimator)
            
            else:
                print('DOH!')
                importances = np.zeros(shape=(X_train.shape[1],1))

            # Add the importances to the main list
            all_importances.append(importances)

        # Sum all importances together, and put them into an importances numpy array
        importances = np.array([sum(x) for x in zip(*all_importances)])

        if self.transform_func is None:
            return importances
        elif self.transform_func == "norm":
            if importances.ndim == 1:
                importances = np.abs(importances)
            else:
                importances = np.linalg.norm(importances, axis=0,
                                             ord=self.norm_order)
        elif self.transform_func == "square":
            if importances.ndim == 1:
                importances = safe_sqr(importances)
            else:
                importances = safe_sqr(importances).sum(axis=0)
        else:
            raise ValueError("Valid values for `self.transform_func` are " +
                             "None, 'norm' and 'square'. Those two " +
                             "transformation are only supported now")

        return importances

将其与脚本的其余部分结合起来，我现在可以获得多输出回归链 RFECV 的特征重要性。哇哦！