Handson-ml: Invalid number of arguments when preparing full pipeline housing data

[jasrys]

Hi Aurélien -

Thanks for the informative book. Much appreciated. Following along in the Chapter 2 housing example I hit an error when calling full_pipeline.fit_transform(housing). I thought I must have missed something but I cloned the 02_end_to_end notebook and still get the same invalid number of arguments error. Any ideas?

Thanks!

TypeError                                 Traceback (most recent call last)
<ipython-input-277-020337ad7bee> in <module>()
----> 1 housing_prepared = full_pipeline.fit_transform(housing)
      2 housing_prepared

/usr/local/lib/python3.6/site-packages/sklearn/pipeline.py in fit_transform(self, X, y, **fit_params)
    744             delayed(_fit_transform_one)(trans, weight, X, y,
    745                                         **fit_params)
--> 746             for name, trans, weight in self._iter())
    747 
    748         if not result:

/usr/local/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py in __call__(self, iterable)
    777             # was dispatched. In particular this covers the edge
    778             # case of Parallel used with an exhausted iterator.
--> 779             while self.dispatch_one_batch(iterator):
    780                 self._iterating = True
    781             else:

/usr/local/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py in dispatch_one_batch(self, iterator)
    623                 return False
    624             else:
--> 625                 self._dispatch(tasks)
    626                 return True
    627 

/usr/local/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py in _dispatch(self, batch)
    586         dispatch_timestamp = time.time()
    587         cb = BatchCompletionCallBack(dispatch_timestamp, len(batch), self)
--> 588         job = self._backend.apply_async(batch, callback=cb)
    589         self._jobs.append(job)
    590 

/usr/local/lib/python3.6/site-packages/sklearn/externals/joblib/_parallel_backends.py in apply_async(self, func, callback)
    109     def apply_async(self, func, callback=None):
    110         """Schedule a func to be run"""
--> 111         result = ImmediateResult(func)
    112         if callback:
    113             callback(result)

/usr/local/lib/python3.6/site-packages/sklearn/externals/joblib/_parallel_backends.py in __init__(self, batch)
    330         # Don't delay the application, to avoid keeping the input
    331         # arguments in memory
--> 332         self.results = batch()
    333 
    334     def get(self):

/usr/local/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py in __call__(self)
    129 
    130     def __call__(self):
--> 131         return [func(*args, **kwargs) for func, args, kwargs in self.items]
    132 
    133     def __len__(self):

/usr/local/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py in <listcomp>(.0)
    129 
    130     def __call__(self):
--> 131         return [func(*args, **kwargs) for func, args, kwargs in self.items]
    132 
    133     def __len__(self):

/usr/local/lib/python3.6/site-packages/sklearn/pipeline.py in _fit_transform_one(transformer, weight, X, y, **fit_params)
    587                        **fit_params):
    588     if hasattr(transformer, 'fit_transform'):
--> 589         res = transformer.fit_transform(X, y, **fit_params)
    590     else:
    591         res = transformer.fit(X, y, **fit_params).transform(X)

/usr/local/lib/python3.6/site-packages/sklearn/pipeline.py in fit_transform(self, X, y, **fit_params)
    290         Xt, fit_params = self._fit(X, y, **fit_params)
    291         if hasattr(last_step, 'fit_transform'):
--> 292             return last_step.fit_transform(Xt, y, **fit_params)
    293         elif last_step is None:
    294             return Xt

TypeError: fit_transform() takes 2 positional arguments but 3 were given

[gr3ybr0w]

This is covered in the extra sections near the bottom of the notebook. Use

class SupervisionFriendlyLabelBinarizer(LabelBinarizer):
    def fit_transform(self, X, y=None):
        return super(SupervisionFriendlyLabelBinarizer, self).fit_transform(X)

before the pipeline

[Nirmal-Ramjee]

Hi

I am having the same issues. I have tried what was mentioned by gr3ybr0w but still am getting the same error. Any ideas what I am missing?

`from sklearn.pipeline import FeatureUnion

full_pipeline = FeatureUnion(transformer_list=[
("num_pipeline", num_pipeline),
("cat_pipeline", cat_pipeline),
])`

housing_prepared = full_pipeline.fit_transform(housing) housing_prepared

---

TypeError Traceback (most recent call last)
in ()
----> 1 housing_prepared = full_pipeline.fit_transform(housing)
2 housing_prepared

/usr/local/lib/python2.7/dist-packages/sklearn/pipeline.pyc in fit_transform(self, X, y, *fit_params)
744 delayed(_fit_transform_one)(trans, weight, X, y,
745 *fit_params)
--> 746 for name, trans, weight in self._iter())
747
748 if not result:

/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/parallel.pyc in __call__(self, iterable)
777 # was dispatched. In particular this covers the edge
778 # case of Parallel used with an exhausted iterator.
--> 779 while self.dispatch_one_batch(iterator):
780 self._iterating = True
781 else:

/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/parallel.pyc in dispatch_one_batch(self, iterator)
623 return False
624 else:
--> 625 self._dispatch(tasks)
626 return True
627

/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/parallel.pyc in _dispatch(self, batch)
586 dispatch_timestamp = time.time()
587 cb = BatchCompletionCallBack(dispatch_timestamp, len(batch), self)
--> 588 job = self._backend.apply_async(batch, callback=cb)
589 self._jobs.append(job)
590

/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/_parallel_backends.pyc in apply_async(self, func, callback)
109 def apply_async(self, func, callback=None):
110 """Schedule a func to be run"""
--> 111 result = ImmediateResult(func)
112 if callback:
113 callback(result)

/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/_parallel_backends.pyc in __init__(self, batch)
330 # Don't delay the application, to avoid keeping the input
331 # arguments in memory
--> 332 self.results = batch()
333
334 def get(self):

/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/parallel.pyc in __call__(self)
129
130 def __call__(self):
--> 131 return [func(args, *kwargs) for func, args, kwargs in self.items]
132
133 def __len__(self):

/usr/local/lib/python2.7/dist-packages/sklearn/pipeline.pyc in _fit_transform_one(transformer, weight, X, y, *fit_params)
587 *fit_params):
588 if hasattr(transformer, 'fit_transform'):
--> 589 res = transformer.fit_transform(X, y, *fit_params)
590 else:
591 res = transformer.fit(X, y, *fit_params).transform(X)

/usr/local/lib/python2.7/dist-packages/sklearn/pipeline.pyc in fit_transform(self, X, y, *fit_params)
290 Xt, fit_params = self._fit(X, y, *fit_params)
291 if hasattr(last_step, 'fit_transform'):
--> 292 return last_step.fit_transform(Xt, y, **fit_params)
293 elif last_step is None:
294 return Xt

TypeError: fit_transform() takes exactly 2 arguments (3 given)

[Rydez]

Hello there,

Same problem. Check out this link for a different method to perform one hot encoding:
https://github.com/ageron/handson-ml/issues/55

[nirmalramjee]

Hi

I tried following the steps from issue 55 but am getting the same error. Does not seem to work.

Please let me know if there is anything else I can do

Thanks!

Just to add to my comment above, I have the following code

num_attribs = list(housing_num)
cat_attribs = ["ocean_proximity"]

num_pipeline = Pipeline([
        ('selector', DataFrameSelector(num_attribs)),
        ('imputer', Imputer(strategy="median")),
        ('attribs_adder', CombinedAttributesAdder()),
        ('std_scaler', StandardScaler()),
    ])

class SupervisionFriendlyLabelBinarizer(LabelBinarizer):
    def fit_transform(self, X, y=None):
        return super(SupervisionFriendlyLabelBinarizer, self).fit_transform(X)

cat_pipeline = Pipeline([
        ('selector', DataFrameSelector(cat_attribs)),
        ('label_binarizer', SupervisionFriendlyLabelBinarizer()),
    ])

# Now you can create a full pipeline with a supervised predictor at the end.
full_pipeline_with_predictor = Pipeline([
        ("num_pipeline", num_pipeline),
        ("cat_pipeline", cat_pipeline)
    ])

housing_prepared = full_pipeline_with_predictor.fit_transform(housing)
housing_prepared

but I am getting the following errors now

---

IndexError Traceback (most recent call last)
in ()
----> 1 housing_prepared = full_pipeline_with_predictor.fit_transform(housing)
2 housing_prepared

/usr/local/lib/python2.7/dist-packages/sklearn/pipeline.pyc in fit_transform(self, X, y, *fit_params)
290 Xt, fit_params = self._fit(X, y, *fit_params)
291 if hasattr(last_step, 'fit_transform'):
--> 292 return last_step.fit_transform(Xt, y, **fit_params)
293 elif last_step is None:
294 return Xt

/usr/local/lib/python2.7/dist-packages/sklearn/pipeline.pyc in fit_transform(self, X, y, *fit_params)
288 """
289 last_step = self._final_estimator
--> 290 Xt, fit_params = self._fit(X, y, *fit_params)
291 if hasattr(last_step, 'fit_transform'):
292 return last_step.fit_transform(Xt, y, **fit_params)

/usr/local/lib/python2.7/dist-packages/sklearn/pipeline.pyc in _fit(self, X, y, *fit_params)
220 Xt, fitted_transformer = fit_transform_one_cached(
221 cloned_transformer, None, Xt, y,
--> 222 *fit_params_steps[name])
223 # Replace the transformer of the step with the fitted
224 # transformer. This is necessary when loading the transformer

/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/memory.pyc in __call__(self, args, *kwargs)
360
361 def __call__(self, args, *kwargs):
--> 362 return self.func(args, *kwargs)
363
364 def call_and_shelve(self, args, *kwargs):

/usr/local/lib/python2.7/dist-packages/sklearn/pipeline.pyc in _fit_transform_one(transformer, weight, X, y, *fit_params)
587 *fit_params):
588 if hasattr(transformer, 'fit_transform'):
--> 589 res = transformer.fit_transform(X, y, *fit_params)
590 else:
591 res = transformer.fit(X, y, *fit_params).transform(X)

/usr/local/lib/python2.7/dist-packages/sklearn/base.pyc in fit_transform(self, X, y, *fit_params)
516 if y is None:
517 # fit method of arity 1 (unsupervised transformation)
--> 518 return self.fit(X, *fit_params).transform(X)
519 else:
520 # fit method of arity 2 (supervised transformation)

in transform(self, X)
9 return self
10 def transform(self, X):
---> 11 return X[self.attribute_names].values

IndexError: only integers, slices (:), ellipsis (...), numpy.newaxis (None) and integer or boolean arrays are valid indices

[Freyert]

@jasrys I'm having the same issue as you and I think I've isolated it to the DataFrameSelector's implementation:

from sklearn.base import BaseEstimator, TransformerMixin

class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, attribute_names):
        self.attribute_names = attribute_names
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        return X[self.attribute_names].values

It all seems correct though, but when I remove the DataFrameSelector from the pipeline it proceeds and I get a different error.

This is what my pipeline looks like when I get your error @jasrys:

from sklearn.pipeline import FeatureUnion

num_attribs = list(housing_num)
cat_attribs = ["ocean_proximity"]

num_pipeline = Pipeline([
        ('selector', DataFrameSelector(num_attribs)),
        ('imputer', Imputer(strategy="median")),
        ('attribs_adder', CombinedAttributesAdder()),
        ('std_scaler', StandardScaler()),
    ])

cat_pipeline = Pipeline([
        ('selector', DataFrameSelector(cat_attribs)),
        ('label_binarizer', LabelBinarizer()),
    ])

full_pipeline = FeatureUnion(transformer_list=[
        ("num_pipeline", num_pipeline),
        ("cat_pipeline", cat_pipeline)
    ])

full_pipeline.fit_transform(housing)
TypeError: fit_transform() takes 2 positional arguments but 3 were given

@nirmalramjee I believe your issue is unrelated to @jasrys's, but perhaps not? Could you share your DataFrameSelector code with us?

[Freyert]

OK, got this working for myself:

Copy and paste this implementation of LabelBinarizerPipelineFriendly into your notebook:

Thanks @Kallin for pointing this out in issue #55 and @hesenp for the implementation. From reading #55 this seems like a workaround and not at all ideal.

Apparently it's an issue caused in the latest version of sklearn.

[nirmalramjee]

@Freyert, could you share your notebook with me please. Thanks.

[Freyert]

@nirmalramjee this is from CH. 2

from sklearn.base import BaseEstimator, TransformerMixin

#credit to @hesenp 
class LabelBinarizerPipelineFriendly(LabelBinarizer):
    def fit(self, X, y=None):
        """this would allow us to fit the model based on the X input."""
        super(LabelBinarizerPipelineFriendly, self).fit(X)
    def transform(self, X, y=None):
        return super(LabelBinarizerPipelineFriendly, self).transform(X)

    def fit_transform(self, X, y=None):
        return super(LabelBinarizerPipelineFriendly, self).fit(X).transform(X)


class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, attribute_names):
        self.attribute_names = attribute_names
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        return X[self.attribute_names].values

from sklearn.pipeline import FeatureUnion

num_attribs = list(housing_num)
cat_attribs = ["ocean_proximity"]

num_pipeline = Pipeline([
        ('selector', DataFrameSelector(num_attribs)),
        ('imputer', Imputer(strategy="median")),
        ('attribs_adder', CombinedAttributesAdder()),
        ('std_scaler', StandardScaler()),
    ])

cat_pipeline = Pipeline([
        ('selector', DataFrameSelector(cat_attribs)),
        ('label_binarizer', LabelBinarizerPipelineFriendly()),
    ])

full_pipeline = FeatureUnion(transformer_list=[
        ("num_pipeline", num_pipeline),
        ("cat_pipeline", cat_pipeline)
    ])

[jasrys]

Fantastic, that custom binarizer transformer works perfectly @Freyert. Thanks!

[dustbort]

@Freyert According to the documentation, LabelBinarizer does not work on the 2-dimensional input X the way you intend.
http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.LabelBinarizer.html

fit(y)
Parameters: | y : array of shape [n_samples,] or [n_samples, n_classes]  
Target values. The 2-d matrix should only contain 0 and 1, represents multilabel classification.

Similarly for transform(y) and fit_transform(y).

So, it seems to treat the columns of the 2-d matrix X as a single multiple-label classification feature with Boolean entries, not multiple single-label classification features. I guess it just treats zero as False and non-zero as True, and although it seems to run OK, the behavior should be quite different than you intend. Or have I misunderstood?

[NirajNair]

@Freyert thanks for the solution.