machine-learningscikit-learnfeature-extractionfeature-selectionrfe

How to use Recursive Feature elimination?


I am new to ML and have been trying Feature selection with RFE approach. My dataset has 5K records and its binary classification problem. This is the code that I am following based on a tutorial online

#no of features
nof_list=np.arange(1,13)            
high_score=0
#Variable to store the optimum features
nof=0           
score_list =[]
for n in range(len(nof_list)):
    X_train, X_test, y_train, y_test = train_test_split(X,y, test_size = 0.3, random_state = 0)
    model = RandomForestClassifier()
    rfe = RFE(model,nof_list[n])
    X_train_rfe = rfe.fit_transform(X_train,y_train)
    X_test_rfe = rfe.transform(X_test)
    model.fit(X_train_rfe,y_train)
    score = model.score(X_test_rfe,y_test)
    score_list.append(score)
    if(score>high_score):
        high_score = score
        nof = nof_list[n]
print("Optimum number of features: %d" %nof)
print("Score with %d features: %f" % (nof, high_score))

I encounter the below error. Can someone please help

TypeError                                 Traceback (most recent call last)
<ipython-input-332-a23dfb331001> in <module>
      9     model = RandomForestClassifier()
     10     rfe = RFE(model,nof_list[n])
---> 11     X_train_rfe = rfe.fit_transform(X_train,y_train)
     12     X_test_rfe = rfe.transform(X_test)
     13     model.fit(X_train_rfe,y_train)

~\AppData\Local\Continuum\anaconda3\lib\site-packages\sklearn\base.py in fit_transform(self, X, y, **fit_params)
    554             Training set.
    555 
--> 556         y : numpy array of shape [n_samples]
    557             Target values.
    558 

~\AppData\Local\Continuum\anaconda3\lib\site-packages\sklearn\feature_selection\_base.py in transform(self, X)
     75         X = check_array(X, dtype=None, accept_sparse='csr',
     76                         force_all_finite=not tags.get('allow_nan', True))
---> 77         mask = self.get_support()
     78         if not mask.any():
     79             warn("No features were selected: either the data is"

~\AppData\Local\Continuum\anaconda3\lib\site-packages\sklearn\feature_selection\_base.py in get_support(self, indices)
     44             values are indices into the input feature vector.
     45         """
---> 46         mask = self._get_support_mask()
     47         return mask if not indices else np.where(mask)[0]
     48 

~\AppData\Local\Continuum\anaconda3\lib\site-packages\sklearn\feature_selection\_rfe.py in _get_support_mask(self)
    269 
    270     def _get_support_mask(self):
--> 271         check_is_fitted(self)
    272         return self.support_
    273 

TypeError: check_is_fitted() missing 1 required positional argument: 'attributes'

Solution

  • What is your sklearn version ?

    The following (using artificial data) should work fine:

    from sklearn.model_selection import train_test_split
    import numpy as np
    from sklearn.feature_selection import RFE
    from sklearn.ensemble import RandomForestClassifier
    
    X = np.random.rand(100,20)
    y = np.ones((X.shape[0]))
    
    #no of features
    nof_list=np.arange(1,13)            
    high_score=0
    #Variable to store the optimum features
    nof=0           
    score_list =[]
    for n in range(len(nof_list)):
        X_train, X_test, y_train, y_test = train_test_split(X,y, test_size = 0.3, random_state = 0)
        model = RandomForestClassifier()
        rfe = RFE(model,nof_list[n])
        X_train_rfe = rfe.fit_transform(X_train,y_train)
        X_test_rfe = rfe.transform(X_test)
        model.fit(X_train_rfe,y_train)
        score = model.score(X_test_rfe,y_test)
        score_list.append(score)
        if(score>high_score):
            high_score = score
            nof = nof_list[n]
    print("Optimum number of features: %d" %nof)
    print("Score with %d features: %f" % (nof, high_score))
    

    Optimum number of features: 1

    Score with 1 features: 1.000000

    Versions tested:

    sklearn.__version__
    '0.20.4'
    
    sklearn.__version__
    '0.21.3'