我只是想在sklearn上试试/试验一下。我在使用加州住房数据集，打算搭建一个管道来生成一些额外的特征，然后对某些特征取对数，最后对所有数值特征进行缩放。实现得既快又粗糙，但到目前为止就是这样。

import pandas as pd
import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.preprocessing import FunctionTransformer, StandardScaler, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline


def safe_divide(a, b):
    return np.where(b == 0, 0, a / b)


class AddCensoredFeatures(BaseEstimator, TransformerMixin):
    def __init__(self, add_censored_age=True):
        self.add_censored_age = add_censored_age

    def fit(self, X, y=None):
        if self.add_censored_age:
            if "housing_median_age" not in X.columns:
                raise ValueError("housing_median_age is not a column in this data.")
            self.age_cap_ = X["housing_median_age"].max()
        return self

    def transform(self, X, y=None):
        X = X.copy()

        if self.add_censored_age:
            X["censored_age"] = (X["housing_median_age"] == self.age_cap_).astype(int)

        return X

    def get_feature_names_out(self, input_features=None):
        if self.add_censored_age:
            return list(input_features) + ["censored_age"]
        else:
            return list(input_features)


class AddComboFeatures(BaseEstimator, TransformerMixin):
    def fit(self, X, y=None):
        return self

    def transform(self, X, y=None):
        X_copy = X.copy()

        required_columns = [
            "total_rooms",
            "total_bedrooms",
            "population",
            "median_income",
            "households",
        ]

        missing_columns = np.array(required_columns)[
            np.isin(required_columns, X_copy.columns, invert=True)
        ]

        if len(missing_columns) > 0:
            raise ValueError(f"The following columns are missing: {missing_columns}")

        X_copy["rooms_per_bedroom"] = safe_divide(
            X_copy["total_rooms"], X_copy["total_bedrooms"]
        )
        X_copy["population_per_room"] = safe_divide(
            X_copy["population"], X_copy["total_rooms"]
        )
        X_copy["income_per_house"] = safe_divide(
            X_copy["median_income"], X_copy["households"]
        )
        X_copy["income_per_population"] = safe_divide(
            X_copy["median_income"], X_copy["population"]
        )

        return X_copy

    def get_feature_names_out(self, input_features=None):
        return list(input_features) + [
            "rooms_per_bedroom",
            "population_per_room",
            "income_per_house",
            "income_per_population",
        ]


def log_transform(X):
    return np.log(X)


def inv_log_transform(X):
    return np.exp(X)


def get_columns():
    log_like_columns = [
        "total_rooms",
        "total_bedrooms",
        "population",
        "households",
        "median_income",
        "population_per_room",
        "income_per_house",
        "income_per_population",
    ]

    numerical_columns = [
        "longitude",
        "latitude",
        "housing_median_age",
        "total_rooms",
        "total_bedrooms",
        "population",
        "households",
        "median_income",
        "rooms_per_bedroom",
        "population_per_room",
        "income_per_house",
        "income_per_population",
    ]

    remaining_columns = np.array(numerical_columns)[
        np.isin(numerical_columns, log_like_columns, invert=True)
    ]

    categorical_columns = ["ocean_proximity"]
    return log_like_columns, remaining_columns, categorical_columns


def get_pipeline():
    log_like_columns, remaining_columns, categorical_columns = get_columns()
    log_transformer = FunctionTransformer(
        log_transform, inverse_func=inv_log_transform, validate=False
    )

    log_and_scale = Pipeline(
        [("log_transform", log_transformer), ("scale_transform", StandardScaler())]
    )

    full_pipeline = Pipeline(
        [
            (
                "add_features",
                Pipeline(
                    [
                        ("add_censored_cols", AddCensoredFeatures()),
                        ("add_combo_cols", AddComboFeatures()),
                    ]
                ),
            ),
            (
                "preprocessing",
                ColumnTransformer(
                    [
                        ("log_and_scale_lognormal", log_and_scale, log_like_columns),
                        ("scale_normal", StandardScaler(), remaining_columns),
                        (
                            "categorical_encoder",
                            OneHotEncoder(sparse_output=False),
                            categorical_columns,
                        ),
                    ],
                    remainder="passthrough",
                ),
            ),
        ]
    )
    return full_pipeline


def preprocess_data(df_X):
    pipeline = get_pipeline()
    pipeline = pipeline.set_output(transform="pandas")
    df_X_prepped = pipeline.fit_transform(df_X)
    df_X_prepped.columns = (
        pd.Series(df_X_prepped.columns).str.split("__", expand=True).iloc[:, -1]
    )
    return df_X_prepped


if __name__ == "__main__":
    import data

    housing_path = r"Paths\projects\housing_project_ex\data\housing.csv"
    df_housing = data.load_data(housing_path)
    df_housing = data.clean_data(df_housing)

    df_housing_y = df_housing["median_house_value"]
    df_housing_X = df_housing.drop(["median_house_value"], axis=1)

    df_housing_X_prepped = preprocess_data(df_housing_X)

    print(df_housing_X_prepped.info())
    print(df_housing_X_prepped.describe())

在我用训练数据拟合 pipeline.fit(train_X)（看起来没问题）后，我尝试对测试数据 pipeline.transform(test_X) 进行变换，但出现了一个错误：

import data
import features

housing_path = r"Path\projects\housing_project_ex\data\housing.csv"
df_housing = data.load_data(housing_path)
df_housing = data.clean_data(df_housing)

train_X_raw, train_labels, test_X_raw, test_labels = train_test_split(df_housing)

pipeline = features.get_pipeline()

train_X = pipeline.fit_transform(train_X_raw)
test_X = pipeline.transform(test_X_raw)

Traceback (most recent call last):
  File "c:\Users\caitl\projects\housing_project_ex\src\train.py", line 72, in <module>
    test_X = pipeline.transform(test_X_raw)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\caitl\anaconda3\envs\housing_project\Lib\site-packages\sklearn\pipeline.py", line 1043, in transform
    Xt = transform.transform(Xt, **routed_params[name].transform)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\caitl\anaconda3\envs\housing_project\Lib\site-packages\sklearn\pipeline.py", line 1035, in transform
    check_is_fitted(self)
  File "C:\Users\caitl\anaconda3\envs\housing_project\Lib\site-packages\sklearn\utils\validation.py", line 1705, in check_is_fitted
    raise NotFittedError(msg % {"name": type(estimator).__name__})
sklearn.exceptions.NotFittedError: This Pipeline instance is not fitted yet. Call 'fit' with appropriate arguments before using this estimator.

它无法拟合“add_combo_cols”。

# test_X = pipeline.transform(test_X_raw)
from sklearn.utils.validation import check_is_fitted
from sklearn.exceptions import NotFittedError


check_is_fitted(pipeline)
for name, step in pipeline.named_steps["add_features"].named_steps.items():
   try:
        check_is_fitted(step)
        print(f"Step '{name}' is fitted.")
   except NotFittedError:
        print(f"Step '{name}' is NOT fitted.")

这会打印出：

Step 'add_censored_cols' is fitted.

Step 'add_combo_cols' is NOT fitted.

dict_items([('add_censored_cols', AddCensoredFeatures()), ('add_combo_cols', AddComboFeatures())])

我知道这很可能和我把列硬编码的方式以及 ColumnTransfomer 的工作原理和信息传递方式有关，但我并不完全理解其中的细节，也不知道如何修复它，并防止将来再次发生。

编辑：以下是data.py的其余代码。

def load_data(filepath):
    df_housing = pd.read_csv(filepath)
    return df_housing


def clean_data(df):
    bedrooms_median = df.total_bedrooms.median()
    df["total_bedrooms"] = df["total_bedrooms"].fillna(bedrooms_median)
    return df

解决方案

这是因为sklearn检查估计量是否已经拟合的方式（在进行变换或预测时常会这么做）；你的 AddComboFeatures 在sklearn那里看起来像未拟合。

check_is_fitted 实用函数会查找以尾随下划线结尾的属性（“已拟合属性”），除非该类实现了一个 __sklearn_is_fitted__ 方法。请参阅 开发者指南。

因此，你可以要么创建一个带尾随下划线的属性（例如 self.is_fitted_ = True 在 fit 之内），要么添加一个拟合性方法，该方法不接收输入并且只返回 True（例如 这里的实现 为 FunctionTransformer）。