tian/MLPlatform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改--修改数据处理时对target列的错误处理

Browse Source
This commit is contained in:
haotian 2025-02-19 14:48:37 +08:00
parent e2fcde42f7
commit 5215666cea
1 changed files with 39 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
function/data_processor_date.py
View File

@ -211,19 +211,27 @@ class DataProcessor:
processor = self.preprocessing_methods[method_name](**params)
# 分离特征和标签
features = df.drop('target', axis=1)
target = df['target']
# 根据不同类型的方法进行处理
if method_name in ['IsolationForest', 'OneClassSVM', 'LocalOutlierFactor', 'EllipticEnvelope']:
# 异常值检测方法
mask = processor.fit_predict(df) != -1
df = df[mask]
mask = processor.fit_predict(features) != -1
features = features[mask]
target = target[mask]
else:
# 其他预处理方法
df = pd.DataFrame(
processor.fit_transform(df),
columns=df.columns,
index=df.index
features = pd.DataFrame(
processor.fit_transform(features),
columns=features.columns,
index=features.index
)
# 重新组合特征和标签
df = pd.concat([features, target], axis=1)
self.logger.info(f"Applied preprocessing method {method_name}")
return df
@ -236,17 +244,21 @@ class DataProcessor:
try:
method_name = method['method_name']
params = method.get('params', {})
columns = method.get('columns', df.columns)
columns = method.get('columns', df.drop('target', axis=1).columns) # 排除target列
if method_name not in self.feature_engineering_methods:
raise ValueError(f"Unknown feature engineering method: {method_name}")
processor = self.feature_engineering_methods[method_name](**params)
# 分离特征和标签
features = df.drop('target', axis=1)
target = df['target']
# 根据不同类型的特征工程方法进行处理
if method_name in ['LabelEncoder', 'OneHotEncoder']:
# 编码方法
df_temp = df[columns].copy()
df_temp = features[columns].copy()
if method_name == 'LabelEncoder':
for col in columns:
df_temp[col] = processor.fit_transform(df_temp[col])
@ -256,17 +268,17 @@ class DataProcessor:
encoded = pd.DataFrame(
encoded,
columns=[f"{col}_{i}" for col in columns for i in range(encoded.shape[1]//len(columns))],
index=df.index
index=features.index
)
df_temp = encoded
# 更新原始数据框
df = df.drop(columns=columns)
df = pd.concat([df, df_temp], axis=1)
# 更新特征数据框
features = features.drop(columns=columns)
features = pd.concat([features, df_temp], axis=1)
elif method_name in ['KBinsDiscretizer']:
# 离散化方法
transformed = processor.fit_transform(df[columns])
transformed = processor.fit_transform(features[columns])
if processor.encode == 'onehot':
feature_names = [f"{col}_{i}" for col in columns for i in range(processor.n_bins_)]
else:
@ -275,33 +287,36 @@ class DataProcessor:
df_temp = pd.DataFrame(
transformed,
columns=feature_names,
index=df.index
index=features.index
)
df = df.drop(columns=columns)
df = pd.concat([df, df_temp], axis=1)
features = features.drop(columns=columns)
features = pd.concat([features, df_temp], axis=1)
elif method_name in ['PCA', 'SelectKBest', 'RFE']:
# 降维和特征选择方法
transformed = processor.fit_transform(df[columns])
transformed = processor.fit_transform(features[columns])
n_features = transformed.shape[1]
df_temp = pd.DataFrame(
transformed,
columns=[f"feature_{i}" for i in range(n_features)],
index=df.index
index=features.index
)
df = df.drop(columns=columns)
df = pd.concat([df, df_temp], axis=1)
features = features.drop(columns=columns)
features = pd.concat([features, df_temp], axis=1)
else:
# 其他特征工程方法
transformed = processor.fit_transform(df[columns])
transformed = processor.fit_transform(features[columns])
df_temp = pd.DataFrame(
transformed,
columns=[f"{col}_transformed" for col in columns],
index=df.index
index=features.index
)
df = df.drop(columns=columns)
df = pd.concat([df, df_temp], axis=1)
features = features.drop(columns=columns)
features = pd.concat([features, df_temp], axis=1)
# 重新组合特征和标签
df = pd.concat([features, target], axis=1)
self.logger.info(f"Applied feature engineering method {method_name}")
return df