tian/MLPlatform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改--修改数据处理时对target列的错误处理

Browse Source
This commit is contained in:
haotian 2025-02-19 14:48:37 +08:00
parent e2fcde42f7
commit 5215666cea
1 changed files with 39 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
function/data_processor_date.py
View File

@ -211,19 +211,27 @@ class DataProcessor:
processor = self.preprocessing_methods[method_name](**params) processor = self.preprocessing_methods[method_name](**params)
# 分离特征和标签
features = df.drop('target', axis=1)
target = df['target']
# 根据不同类型的方法进行处理 # 根据不同类型的方法进行处理
if method_name in ['IsolationForest', 'OneClassSVM', 'LocalOutlierFactor', 'EllipticEnvelope']: if method_name in ['IsolationForest', 'OneClassSVM', 'LocalOutlierFactor', 'EllipticEnvelope']:
# 异常值检测方法 # 异常值检测方法
mask = processor.fit_predict(df) != -1 mask = processor.fit_predict(features) != -1
df = df[mask] features = features[mask]
target = target[mask]
else: else:
# 其他预处理方法 # 其他预处理方法
df = pd.DataFrame( features = pd.DataFrame(
processor.fit_transform(df), processor.fit_transform(features),
columns=df.columns, columns=features.columns,
index=df.index index=features.index
) )
# 重新组合特征和标签
df = pd.concat([features, target], axis=1)
self.logger.info(f"Applied preprocessing method {method_name}") self.logger.info(f"Applied preprocessing method {method_name}")
return df return df
@ -236,17 +244,21 @@ class DataProcessor:
try: try:
method_name = method['method_name'] method_name = method['method_name']
params = method.get('params', {}) params = method.get('params', {})
columns = method.get('columns', df.columns) columns = method.get('columns', df.drop('target', axis=1).columns) # 排除target列
if method_name not in self.feature_engineering_methods: if method_name not in self.feature_engineering_methods:
raise ValueError(f"Unknown feature engineering method: {method_name}") raise ValueError(f"Unknown feature engineering method: {method_name}")
processor = self.feature_engineering_methods[method_name](**params) processor = self.feature_engineering_methods[method_name](**params)
# 分离特征和标签
features = df.drop('target', axis=1)
target = df['target']
# 根据不同类型的特征工程方法进行处理 # 根据不同类型的特征工程方法进行处理
if method_name in ['LabelEncoder', 'OneHotEncoder']: if method_name in ['LabelEncoder', 'OneHotEncoder']:
# 编码方法 # 编码方法
df_temp = df[columns].copy() df_temp = features[columns].copy()
if method_name == 'LabelEncoder': if method_name == 'LabelEncoder':
for col in columns: for col in columns:
df_temp[col] = processor.fit_transform(df_temp[col]) df_temp[col] = processor.fit_transform(df_temp[col])
@ -256,17 +268,17 @@ class DataProcessor:
encoded = pd.DataFrame( encoded = pd.DataFrame(
encoded, encoded,
columns=[f"{col}_{i}" for col in columns for i in range(encoded.shape[1]//len(columns))], columns=[f"{col}_{i}" for col in columns for i in range(encoded.shape[1]//len(columns))],
index=df.index index=features.index
) )
df_temp = encoded df_temp = encoded
# 更新原始数据框 # 更新特征数据框
df = df.drop(columns=columns) features = features.drop(columns=columns)
df = pd.concat([df, df_temp], axis=1) features = pd.concat([features, df_temp], axis=1)
elif method_name in ['KBinsDiscretizer']: elif method_name in ['KBinsDiscretizer']:
# 离散化方法 # 离散化方法
transformed = processor.fit_transform(df[columns]) transformed = processor.fit_transform(features[columns])
if processor.encode == 'onehot': if processor.encode == 'onehot':
feature_names = [f"{col}_{i}" for col in columns for i in range(processor.n_bins_)] feature_names = [f"{col}_{i}" for col in columns for i in range(processor.n_bins_)]
else: else:
@ -275,33 +287,36 @@ class DataProcessor:
df_temp = pd.DataFrame( df_temp = pd.DataFrame(
transformed, transformed,
columns=feature_names, columns=feature_names,
index=df.index index=features.index
) )
df = df.drop(columns=columns) features = features.drop(columns=columns)
df = pd.concat([df, df_temp], axis=1) features = pd.concat([features, df_temp], axis=1)
elif method_name in ['PCA', 'SelectKBest', 'RFE']: elif method_name in ['PCA', 'SelectKBest', 'RFE']:
# 降维和特征选择方法 # 降维和特征选择方法
transformed = processor.fit_transform(df[columns]) transformed = processor.fit_transform(features[columns])
n_features = transformed.shape[1] n_features = transformed.shape[1]
df_temp = pd.DataFrame( df_temp = pd.DataFrame(
transformed, transformed,
columns=[f"feature_{i}" for i in range(n_features)], columns=[f"feature_{i}" for i in range(n_features)],
index=df.index index=features.index
) )
df = df.drop(columns=columns) features = features.drop(columns=columns)
df = pd.concat([df, df_temp], axis=1) features = pd.concat([features, df_temp], axis=1)
else: else:
# 其他特征工程方法 # 其他特征工程方法
transformed = processor.fit_transform(df[columns]) transformed = processor.fit_transform(features[columns])
df_temp = pd.DataFrame( df_temp = pd.DataFrame(
transformed, transformed,
columns=[f"{col}_transformed" for col in columns], columns=[f"{col}_transformed" for col in columns],
index=df.index index=features.index
) )
df = df.drop(columns=columns) features = features.drop(columns=columns)
df = pd.concat([df, df_temp], axis=1) features = pd.concat([features, df_temp], axis=1)
# 重新组合特征和标签
df = pd.concat([features, target], axis=1)
self.logger.info(f"Applied feature engineering method {method_name}") self.logger.info(f"Applied feature engineering method {method_name}")
return df return df