diff --git a/api/__pycache__/data_api.cpython-39.pyc b/api/__pycache__/data_api.cpython-39.pyc
index 4776ebd..006115b 100644
Binary files a/api/__pycache__/data_api.cpython-39.pyc and b/api/__pycache__/data_api.cpython-39.pyc differ
diff --git a/api/data_api.py b/api/data_api.py
index f70e1b3..6385779 100644
--- a/api/data_api.py
+++ b/api/data_api.py
@@ -75,6 +75,7 @@ async def get_datasets():
"""获取可用数据集列表"""
try:
datasets = data_manager.get_dataset()
+ print("datasets", datasets)
return {
"status": "success",
"datasets": datasets
diff --git a/doc/接口文档code.md b/doc/接口文档code.md
index 8daf658..16d1595 100644
--- a/doc/接口文档code.md
+++ b/doc/接口文档code.md
@@ -764,7 +764,7 @@ Error Response:
### 3.3 获取系统中训练状态 ---- 未完成, 等开发系统后台时再实现.
```http
-GET /api/train/status/{task_id}
+GET /model/train/status/{task_id}
Response:
{
diff --git a/frontend/index.html b/frontend/index.html
new file mode 100644
index 0000000..1f10fdf
--- /dev/null
+++ b/frontend/index.html
@@ -0,0 +1,13 @@
+
+
+
+
+
+
+ 机器学习平台
+
+
+
+
+
+
\ No newline at end of file
diff --git a/frontend/src/api/data.ts b/frontend/src/api/data.ts
index 795aa8a..39de53d 100644
--- a/frontend/src/api/data.ts
+++ b/frontend/src/api/data.ts
@@ -5,11 +5,15 @@ import type {
FeatureMethod,
ProcessRequest,
CSVRequest,
- DatasetInfo
+ DatasetInfo,
+ MethodDetail
} from '@/types/data'
// 获取预处理方法列表
-export const getPreprocessMethods = (): Promise> => {
+export const getPreprocessMethods = (): Promise<{
+ status: string;
+ methods: PreprocessMethod[];
+}> => {
return request({
url: '/data/preprocessing/methods',
method: 'get'
@@ -17,7 +21,10 @@ export const getPreprocessMethods = (): Promise => {
+export const getPreprocessMethodDetails = (methodName: string): Promise<{
+ status: string;
+ method: MethodDetail;
+}> => {
return request({
url: `/data/preprocessing/method/${methodName}`,
method: 'get'
@@ -25,7 +32,10 @@ export const getPreprocessMethodDetails = (methodName: string): Promise> => {
+export const getFeatureMethods = (): Promise<{
+ status: string;
+ methods: FeatureMethod[];
+}> => {
return request({
url: '/data/feature/methods',
method: 'get'
@@ -33,7 +43,10 @@ export const getFeatureMethods = (): Promise> =>
}
// 获取特征工程方法详情
-export const getFeatureMethodDetails = (methodName: string): Promise => {
+export const getFeatureMethodDetails = (methodName: string): Promise<{
+ status: string;
+ method: MethodDetail;
+}> => {
return request({
url: `/data/feature/method/${methodName}`,
method: 'get'
@@ -41,7 +54,10 @@ export const getFeatureMethodDetails = (methodName: string): Promise => {
+export const processDataset = (data: ProcessRequest): Promise<{
+ status: string;
+ data: any;
+}> => {
return request({
url: '/data/process',
method: 'post',
@@ -49,8 +65,11 @@ export const processDataset = (data: ProcessRequest): Promise =>
})
}
-// 获取可用数据集列表
-export const getDatasets = (): Promise> => {
+// 获取数据集列表
+export const getDatasets = (): Promise<{
+ status: string;
+ datasets: DatasetInfo[];
+}> => {
return request({
url: '/data/datasets',
method: 'get'
@@ -58,7 +77,10 @@ export const getDatasets = (): Promise> => {
}
// 读取CSV文件
-export const readCSV = (data: CSVRequest): Promise => {
+export const readCSV = (data: CSVRequest): Promise<{
+ status: string;
+ data: any;
+}> => {
return request({
url: '/data/csv',
method: 'post',
diff --git a/frontend/src/components/data/DataPreview.vue b/frontend/src/components/data/DataPreview.vue
index 5721ea1..919c650 100644
--- a/frontend/src/components/data/DataPreview.vue
+++ b/frontend/src/components/data/DataPreview.vue
@@ -4,74 +4,80 @@
title="数据预览"
width="80%"
>
-
-
-
-
-
-
- {{ dataInfo.train?.info?.rows || '-' }}
-
-
- {{ dataInfo.train?.info?.columns || '-' }}
-
-
- {{ dataInfo.train?.info?.memory_usage || '-' }}
-
-
-
-
-
数据预览
-
-
-
-
-
-
-
数据统计
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+ {{ dataInfo?.info?.rows || '-' }}
+
+
+ {{ dataInfo?.info?.columns || '-' }}
+
+
+ {{ dataInfo?.info?.memory_usage || '-' }}
+
+
+
+
+
+
数据预览
+
+
+
+
+
+
+
+
数据统计
+
+
+
+
+
+
+
+
+
缺失值统计
+
+
+
+
+
+ {{ row.percentage }}%
+
+
+
+
+
\ No newline at end of file
diff --git a/frontend/src/components/data/DataProcessDialog.vue b/frontend/src/components/data/DataProcessDialog.vue
new file mode 100644
index 0000000..be0b916
--- /dev/null
+++ b/frontend/src/components/data/DataProcessDialog.vue
@@ -0,0 +1,197 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/frontend/src/components/data/DatasetList.vue b/frontend/src/components/data/DatasetList.vue
index a06c34a..6edea69 100644
--- a/frontend/src/components/data/DatasetList.vue
+++ b/frontend/src/components/data/DatasetList.vue
@@ -1,22 +1,36 @@
-
-
+
+
+
+ 上传数据集
+
+
+
+
+
+
{{ getFileName(row.input_file) }}
-
+
{{ formatTime(row.timestamp) }}
-
+
@@ -28,7 +42,7 @@
-
+
-
-
- 训练集: {{ (1 - row.split_params.test_size - row.split_params.val_size) * 100 }}%
- 验证集: {{ row.split_params.val_size * 100 }}%
- 测试集: {{ row.split_params.test_size * 100 }}%
-
-
-
-
-
-
-
-
-
+
预览
-
+
+ 处理
+
+
下载
-
- 详情
+
+ 删除
-
-
+
+
-
-
-
-
-
- {{ getStepTitle(step) }}
- 数据形状: {{ step.shape[0] }} × {{ step.shape[1] }}
-
- 处理方法: {{ step.method }}
- 参数配置:
-
-
- {{ value }}
-
-
-
-
-
-
+
+
+
@@ -186,7 +181,12 @@ const handleDetails = (dataset: DatasetInfo) => {
padding: 20px;
}
+.upload-section {
+ margin-bottom: 20px;
+}
+
.el-tag {
margin-right: 5px;
+ margin-bottom: 5px;
}
\ No newline at end of file
diff --git a/frontend/src/router/index.ts b/frontend/src/router/index.ts
index 0494112..8ba15ba 100644
--- a/frontend/src/router/index.ts
+++ b/frontend/src/router/index.ts
@@ -9,11 +9,38 @@ const router = createRouter({
},
{
path: '/data',
- name: 'Data',
- component: () => import('@/views/data/DatasetView.vue'),
- meta: {
- title: '数据管理'
- }
+ component: () => import('@/views/data/DataView.vue'),
+ children: [
+ {
+ path: '',
+ name: 'Data',
+ redirect: '/data/datasets'
+ },
+ {
+ path: 'datasets',
+ name: 'Datasets',
+ component: () => import('@/views/data/DatasetView.vue'),
+ meta: {
+ title: '数据集管理'
+ }
+ },
+ {
+ path: 'preprocess',
+ name: 'Preprocess',
+ component: () => import('@/views/data/PreprocessView.vue'),
+ meta: {
+ title: '预处理方法'
+ }
+ },
+ {
+ path: 'feature',
+ name: 'Feature',
+ component: () => import('@/views/data/FeatureView.vue'),
+ meta: {
+ title: '特征工程方法'
+ }
+ }
+ ]
},
{
path: '/model',
diff --git a/frontend/src/types/data.ts b/frontend/src/types/data.ts
index c8e442f..c8f9d8d 100644
--- a/frontend/src/types/data.ts
+++ b/frontend/src/types/data.ts
@@ -16,7 +16,7 @@ export interface FeatureMethod {
export interface MethodParameter {
name: string
type: string
- default: any
+ default: string | number | boolean | null
description: string
}
@@ -86,4 +86,18 @@ export interface DatasetInfo {
validation: string
test: string
}
+}
+
+// 数据预览信息
+export interface DataPreview {
+ info: {
+ rows: number
+ columns: number
+ column_types: Record
+ memory_usage: string
+ missing_values: Record
+ }
+ head: any[]
+ tail: any[]
+ describe: Record
}
\ No newline at end of file
diff --git a/frontend/src/views/data/DataView.vue b/frontend/src/views/data/DataView.vue
new file mode 100644
index 0000000..fec9f0b
--- /dev/null
+++ b/frontend/src/views/data/DataView.vue
@@ -0,0 +1,51 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/frontend/src/views/data/DatasetView.vue b/frontend/src/views/data/DatasetView.vue
index fd5e985..bb63b66 100644
--- a/frontend/src/views/data/DatasetView.vue
+++ b/frontend/src/views/data/DatasetView.vue
@@ -1,25 +1,56 @@
-
数据集管理
-
+
+
+
+
+
+
+
@@ -27,4 +58,10 @@ onMounted(async () => {
.dataset-view {
min-height: 100%;
}
+
+.card-header {
+ display: flex;
+ justify-content: space-between;
+ align-items: center;
+}
\ No newline at end of file
diff --git a/frontend/src/views/data/FeatureView.vue b/frontend/src/views/data/FeatureView.vue
new file mode 100644
index 0000000..c8ef8d2
--- /dev/null
+++ b/frontend/src/views/data/FeatureView.vue
@@ -0,0 +1,202 @@
+
+
+
+
+
+
+
+
+
+
+ {{ method }}
+
+
+
+
+
+
+
+
+
+
+ {{ methodDetails?.name }}
+
+
+ {{ methodDetails?.description }}
+
+
+ {{ methodDetails?.principle }}
+
+
+
+
+
优点
+
+ {{ adv }}
+
+
+
+
+
缺点
+
+ {{ dis }}
+
+
+
+
+
适用场景
+
+ {{ scene }}
+
+
+
+
+
参数说明
+
+
+
+
+
+ {{ row.default === null ? 'null' : row.default }}
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/frontend/src/views/data/PreprocessView.vue b/frontend/src/views/data/PreprocessView.vue
new file mode 100644
index 0000000..5c2e518
--- /dev/null
+++ b/frontend/src/views/data/PreprocessView.vue
@@ -0,0 +1,253 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {{ method }}
+
+
+
+
+
+
+
+
+
+
+ {{ method }}
+
+
+
+
+
+
+
+
+
+
+ {{ method }}
+
+
+
+
+
+
+
+
+
+
+
+
+ {{ methodDetails?.name }}
+
+
+ {{ methodDetails?.description }}
+
+
+ {{ methodDetails?.principle }}
+
+
+
+
+
优点
+
+ {{ adv }}
+
+
+
+
+
缺点
+
+ {{ dis }}
+
+
+
+
+
适用场景
+
+ {{ scene }}
+
+
+
+
+
参数说明
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/function_old/data_process/__init__.py b/function_old/data_process/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/function_old/data_process/__pycache__/__init__.cpython-39.pyc b/function_old/data_process/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000..8fb5816
Binary files /dev/null and b/function_old/data_process/__pycache__/__init__.cpython-39.pyc differ
diff --git a/function_old/data_process/__pycache__/data_cleaner.cpython-39.pyc b/function_old/data_process/__pycache__/data_cleaner.cpython-39.pyc
new file mode 100644
index 0000000..553627f
Binary files /dev/null and b/function_old/data_process/__pycache__/data_cleaner.cpython-39.pyc differ
diff --git a/function_old/data_process/__pycache__/data_processor.cpython-39.pyc b/function_old/data_process/__pycache__/data_processor.cpython-39.pyc
new file mode 100644
index 0000000..8a8485f
Binary files /dev/null and b/function_old/data_process/__pycache__/data_processor.cpython-39.pyc differ
diff --git a/function_old/data_process/__pycache__/data_processor_date.cpython-39.pyc b/function_old/data_process/__pycache__/data_processor_date.cpython-39.pyc
new file mode 100644
index 0000000..0d8123d
Binary files /dev/null and b/function_old/data_process/__pycache__/data_processor_date.cpython-39.pyc differ
diff --git a/function_old/data_process/__pycache__/data_splitter.cpython-39.pyc b/function_old/data_process/__pycache__/data_splitter.cpython-39.pyc
new file mode 100644
index 0000000..f97444e
Binary files /dev/null and b/function_old/data_process/__pycache__/data_splitter.cpython-39.pyc differ
diff --git a/function_old/data_process/__pycache__/feature_engineer.cpython-39.pyc b/function_old/data_process/__pycache__/feature_engineer.cpython-39.pyc
new file mode 100644
index 0000000..98c7c74
Binary files /dev/null and b/function_old/data_process/__pycache__/feature_engineer.cpython-39.pyc differ
diff --git a/function_old/data_process/__pycache__/method_reader.cpython-39.pyc b/function_old/data_process/__pycache__/method_reader.cpython-39.pyc
new file mode 100644
index 0000000..20a9060
Binary files /dev/null and b/function_old/data_process/__pycache__/method_reader.cpython-39.pyc differ
diff --git a/function_old/data_process/__pycache__/method_reader_date_feature.cpython-39.pyc b/function_old/data_process/__pycache__/method_reader_date_feature.cpython-39.pyc
new file mode 100644
index 0000000..20dfd1f
Binary files /dev/null and b/function_old/data_process/__pycache__/method_reader_date_feature.cpython-39.pyc differ
diff --git a/function_old/data_process/__pycache__/method_reader_date_process.cpython-39.pyc b/function_old/data_process/__pycache__/method_reader_date_process.cpython-39.pyc
new file mode 100644
index 0000000..1e22f73
Binary files /dev/null and b/function_old/data_process/__pycache__/method_reader_date_process.cpython-39.pyc differ
diff --git a/function_old/data_process/data_cleaner.py b/function_old/data_process/data_cleaner.py
new file mode 100644
index 0000000..58532e1
--- /dev/null
+++ b/function_old/data_process/data_cleaner.py
@@ -0,0 +1,66 @@
+from .data_processor import DataProcessor
+import pandas as pd
+import numpy as np
+from typing import Dict, List
+from sklearn.impute import SimpleImputer
+from sklearn.ensemble import IsolationForest
+from scipy import stats
+
+class DataCleaner(DataProcessor):
+ """数据清洗类"""
+
+ def __init__(self, config: Dict = None):
+ super().__init__(config)
+ self.missing_value_methods = {
+ 'mean': SimpleImputer(strategy='mean'),
+ 'median': SimpleImputer(strategy='median'),
+ 'mode': SimpleImputer(strategy='most_frequent'),
+ 'constant': SimpleImputer(strategy='constant')
+ }
+
+ def handle_missing_values(self, df: pd.DataFrame, method: str = 'mean', columns: List[str] = None) -> pd.DataFrame:
+ """处理缺失值"""
+ try:
+ if columns is None:
+ columns = df.select_dtypes(include=[np.number]).columns
+
+ if method not in self.missing_value_methods:
+ raise ValueError(f"Unsupported method: {method}")
+
+ imputer = self.missing_value_methods[method]
+ df[columns] = imputer.fit_transform(df[columns])
+
+ self.logger.info(f"Successfully handled missing values using {method} method")
+ return df
+
+ except Exception as e:
+ self.logger.error(f"Error handling missing values: {str(e)}")
+ raise
+
+ def remove_duplicates(self, df: pd.DataFrame, subset: List[str] = None) -> pd.DataFrame:
+ """删除重复值"""
+ try:
+ original_shape = df.shape
+ df = df.drop_duplicates(subset=subset)
+ self.logger.info(f"Removed {original_shape[0] - df.shape[0]} duplicate rows")
+ return df
+ except Exception as e:
+ self.logger.error(f"Error removing duplicates: {str(e)}")
+ raise
+
+ def detect_outliers(self, df: pd.DataFrame, method: str = 'zscore', threshold: float = 3) -> pd.DataFrame:
+ """检测异常值"""
+ try:
+ if method == 'zscore':
+ z_scores = np.abs(stats.zscore(df.select_dtypes(include=[np.number])))
+ outliers = (z_scores > threshold).any(axis=1)
+ elif method == 'isolation_forest':
+ iso_forest = IsolationForest(contamination=0.1, random_state=42)
+ outliers = iso_forest.fit_predict(df.select_dtypes(include=[np.number])) == -1
+
+ self.logger.info(f"Detected {sum(outliers)} outliers using {method} method")
+ return df[~outliers]
+
+ except Exception as e:
+ self.logger.error(f"Error detecting outliers: {str(e)}")
+ raise
\ No newline at end of file
diff --git a/function_old/data_process/data_processor.py b/function_old/data_process/data_processor.py
new file mode 100644
index 0000000..f5ed18c
--- /dev/null
+++ b/function_old/data_process/data_processor.py
@@ -0,0 +1,63 @@
+import pandas as pd
+import numpy as np
+from typing import Dict, List, Union, Optional
+from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler
+from sklearn.impute import SimpleImputer
+from sklearn.model_selection import train_test_split
+import logging
+import hashlib
+from datetime import datetime
+import json
+import os
+
+class DataProcessor:
+ """数据处理基类"""
+
+ def __init__(self, config: Dict = None):
+ self.config = config or {}
+ self.logger = logging.getLogger(__name__)
+ self._setup_logging()
+
+ def _setup_logging(self):
+ """设置日志"""
+ logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+ )
+
+ def load_data(self, file_path: str) -> pd.DataFrame:
+ """加载数据"""
+ try:
+ file_type = file_path.split('.')[-1].lower()
+ if file_type == 'csv':
+ df = pd.read_csv(file_path, **self.config.get('csv_params', {}))
+ elif file_type == 'parquet':
+ df = pd.read_parquet(file_path)
+ elif file_type == 'hdf5':
+ df = pd.read_hdf(file_path)
+ else:
+ raise ValueError(f"Unsupported file type: {file_type}")
+
+ self.logger.info(f"Successfully loaded data from {file_path}")
+ return df
+
+ except Exception as e:
+ self.logger.error(f"Error loading data: {str(e)}")
+ raise
+
+ def save_data(self, df: pd.DataFrame, file_path: str):
+ """保存数据"""
+ try:
+ file_type = file_path.split('.')[-1].lower()
+ if file_type == 'csv':
+ df.to_csv(file_path, index=False)
+ elif file_type == 'parquet':
+ df.to_parquet(file_path)
+ elif file_type == 'hdf5':
+ df.to_hdf(file_path, key='data')
+
+ self.logger.info(f"Successfully saved data to {file_path}")
+
+ except Exception as e:
+ self.logger.error(f"Error saving data: {str(e)}")
+ raise
\ No newline at end of file
diff --git a/function_old/data_process/data_splitter.py b/function_old/data_process/data_splitter.py
new file mode 100644
index 0000000..0018422
--- /dev/null
+++ b/function_old/data_process/data_splitter.py
@@ -0,0 +1,49 @@
+from .data_processor import DataProcessor
+import pandas as pd
+from typing import Dict, Tuple
+from sklearn.model_selection import train_test_split
+
+class DataSplitter(DataProcessor):
+ """数据集划分类"""
+
+ def __init__(self, config: Dict = None):
+ super().__init__(config)
+
+ def train_val_test_split(
+ self,
+ df: pd.DataFrame,
+ target: str,
+ test_size: float = 0.2,
+ val_size: float = 0.2,
+ random_state: int = 42
+ ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+ """划分训练集、验证集和测试集"""
+ try:
+ # 首先划分训练集和测试集
+ train_val, test = train_test_split(
+ df,
+ test_size=test_size,
+ random_state=random_state,
+ stratify=df[target] if df[target].dtype == 'object' else None
+ )
+
+ # 再划分训练集和验证集
+ train, val = train_test_split(
+ train_val,
+ test_size=val_size,
+ random_state=random_state,
+ stratify=train_val[target] if train_val[target].dtype == 'object' else None
+ )
+
+ self.logger.info(f"""
+ Data split complete:
+ - Training set: {train.shape[0]} samples
+ - Validation set: {val.shape[0]} samples
+ - Test set: {test.shape[0]} samples
+ """)
+
+ return train, val, test
+
+ except Exception as e:
+ self.logger.error(f"Error splitting data: {str(e)}")
+ raise
\ No newline at end of file
diff --git a/function_old/data_process/feature_engineer.py b/function_old/data_process/feature_engineer.py
new file mode 100644
index 0000000..57105e8
--- /dev/null
+++ b/function_old/data_process/feature_engineer.py
@@ -0,0 +1,77 @@
+from .data_processor import DataProcessor
+import pandas as pd
+import numpy as np
+from typing import Dict, List
+from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler
+from sklearn.feature_selection import SelectKBest, chi2, f_classif
+
+class FeatureEngineer(DataProcessor):
+ """特征工程类"""
+
+ def __init__(self, config: Dict = None):
+ super().__init__(config)
+ self.scalers = {
+ 'standard': StandardScaler(),
+ 'minmax': MinMaxScaler(),
+ 'robust': RobustScaler()
+ }
+
+ def scale_features(self, df: pd.DataFrame, method: str = 'standard', columns: List[str] = None) -> pd.DataFrame:
+ """特征缩放"""
+ try:
+ if columns is None:
+ columns = df.select_dtypes(include=[np.number]).columns
+
+ if method not in self.scalers:
+ raise ValueError(f"Unsupported scaling method: {method}")
+
+ scaler = self.scalers[method]
+ df[columns] = scaler.fit_transform(df[columns])
+
+ self.logger.info(f"Successfully scaled features using {method} method")
+ return df
+
+ except Exception as e:
+ self.logger.error(f"Error scaling features: {str(e)}")
+ raise
+
+ def select_features(self, df: pd.DataFrame, target: str, method: str = 'chi2', k: int = 10) -> pd.DataFrame:
+ """特征选择"""
+ try:
+ X = df.drop(columns=[target])
+ y = df[target]
+
+ if method == 'chi2':
+ # 要求输入x不能为负的
+ selector = SelectKBest(chi2, k=k)
+ elif method == 'f_classif':
+ selector = SelectKBest(f_classif, k=k)
+ else:
+ raise ValueError(f"Unsupported feature selection method: {method}")
+
+ X_selected = selector.fit_transform(X, y)
+ selected_features = X.columns[selector.get_support()].tolist()
+
+ self.logger.info(f"Selected {len(selected_features)} features")
+ return df[selected_features + [target]]
+
+ except Exception as e:
+ self.logger.error(f"Error selecting features: {str(e)}")
+ raise
+
+ def create_datetime_features(self, df: pd.DataFrame, datetime_column: str) -> pd.DataFrame:
+ """创建时间特征"""
+ try:
+ df[datetime_column] = pd.to_datetime(df[datetime_column])
+ df[f'{datetime_column}_year'] = df[datetime_column].dt.year
+ df[f'{datetime_column}_month'] = df[datetime_column].dt.month
+ df[f'{datetime_column}_day'] = df[datetime_column].dt.day
+ df[f'{datetime_column}_weekday'] = df[datetime_column].dt.weekday
+ df[f'{datetime_column}_is_weekend'] = df[datetime_column].dt.weekday.isin([5, 6])
+
+ self.logger.info(f"Created datetime features from {datetime_column}")
+ return df
+
+ except Exception as e:
+ self.logger.error(f"Error creating datetime features: {str(e)}")
+ raise
\ No newline at end of file
diff --git a/function_old/data_process/test_data_processor.py b/function_old/data_process/test_data_processor.py
new file mode 100644
index 0000000..317c35b
--- /dev/null
+++ b/function_old/data_process/test_data_processor.py
@@ -0,0 +1,34 @@
+import unittest
+import pandas as pd
+import numpy as np
+from data_processor import DataProcessor
+from data_cleaner import DataCleaner
+from feature_engineer import FeatureEngineer
+from data_splitter import DataSplitter
+
+class TestDataProcessor(unittest.TestCase):
+ def setUp(self):
+ # 创建测试数据
+ self.test_data = pd.DataFrame({
+ 'feature1': [1, 2, np.nan, 4, 5],
+ 'feature2': ['A', 'B', 'A', 'B', 'C'],
+ 'target': [0, 1, 0, 1, 0]
+ })
+
+ def test_data_cleaner(self):
+ cleaner = DataCleaner()
+ cleaned_data = cleaner.handle_missing_values(self.test_data.copy())
+ self.assertFalse(cleaned_data.isnull().any().any())
+
+ def test_feature_engineer(self):
+ engineer = FeatureEngineer()
+ scaled_data = engineer.scale_features(self.test_data.copy())
+ self.assertTrue('feature1' in scaled_data.columns)
+
+ def test_data_splitter(self):
+ splitter = DataSplitter()
+ train, val, test = splitter.train_val_test_split(self.test_data.copy(), 'target')
+ self.assertEqual(len(train) + len(val) + len(test), len(self.test_data))
+
+if __name__ == '__main__':
+ unittest.main()
\ No newline at end of file
diff --git a/function_old/data_process/ttt.py b/function_old/data_process/ttt.py
new file mode 100644
index 0000000..3d0bf95
--- /dev/null
+++ b/function_old/data_process/ttt.py
@@ -0,0 +1,2 @@
+from data_processor import DataProcessor
+
diff --git a/function_old/test_data_processor.py b/function_old/test_data_processor.py
new file mode 100644
index 0000000..cdde864
--- /dev/null
+++ b/function_old/test_data_processor.py
@@ -0,0 +1,93 @@
+import unittest
+import pandas as pd
+import numpy as np
+from pathlib import Path
+from function.data_processor_date import DataProcessor
+
+class TestDataProcessor(unittest.TestCase):
+ def setUp(self):
+ self.processor = DataProcessor()
+
+ # 创建测试数据
+ self.test_data = pd.DataFrame({
+ 'feature1': [1, 2, np.nan, 4, 5],
+ 'feature2': [10, 20, 30, 40, 50],
+ 'target': [0, 1, 0, 1, 0]
+ })
+
+ # 保存测试数据
+ self.input_path = 'dataset/dataset_raw/test_data.csv'
+ Path(self.input_path).parent.mkdir(parents=True, exist_ok=True)
+ self.test_data.to_csv(self.input_path, index=False)
+
+ # 设置输出目录
+ self.output_dir = 'dataset/dataset_processed'
+
+ def test_process_dataset(self):
+ # 定义处理方法
+ cleaning_methods = [
+ {
+ 'method_name': 'SimpleImputer',
+ 'params': {'strategy': 'mean'}
+ }
+ ]
+
+ feature_methods = [
+ {
+ 'method_name': 'StandardScaler',
+ 'params': {}
+ }
+ ]
+
+ split_params = {
+ 'test_size': 0.2,
+ 'val_size': 0.2
+ }
+
+ # 处理数据集
+ result = self.processor.process_dataset(
+ self.input_path,
+ self.output_dir,
+ cleaning_methods,
+ feature_methods,
+ split_params
+ )
+
+ # 验证结果
+ self.assertEqual(result['status'], 'success')
+ self.assertIn('process_record', result)
+
+ # 验证输出文件
+ record = result['process_record']
+ self.assertTrue(Path(record['output_files']['train']).exists())
+ self.assertTrue(Path(record['output_files']['validation']).exists())
+ self.assertTrue(Path(record['output_files']['test']).exists())
+
+ def test_invalid_method(self):
+ # 测试无效的方法名
+ cleaning_methods = [
+ {
+ 'method_name': 'InvalidMethod',
+ 'params': {}
+ }
+ ]
+
+ result = self.processor.process_dataset(
+ self.input_path,
+ self.output_dir,
+ cleaning_methods,
+ [],
+ {'test_size': 0.2, 'val_size': 0.2}
+ )
+
+ self.assertEqual(result['status'], 'error')
+
+ def tearDown(self):
+ # 清理测试文件
+ try:
+ Path(self.input_path).unlink()
+ except:
+ pass
+
+if __name__ == '__main__':
+ unittest.main()
\ No newline at end of file
diff --git a/function_old/test_method_reader.py b/function_old/test_method_reader.py
new file mode 100644
index 0000000..9f7140e
--- /dev/null
+++ b/function_old/test_method_reader.py
@@ -0,0 +1,49 @@
+import unittest
+from function.method_reader_date_process import MethodReader
+
+class TestMethodReader(unittest.TestCase):
+ def setUp(self):
+ self.reader = MethodReader()
+
+ def test_get_preprocessing_methods(self):
+ result = self.reader.get_preprocessing_methods()
+ self.assertEqual(result['status'], 'success')
+ self.assertIsInstance(result['methods'], list)
+
+ # 检查返回的方法列表
+ methods = result['methods']
+ self.assertTrue(any(m['name'] == 'data_scaler' for m in methods))
+ self.assertTrue(any(m['name'] == 'missing_value_handler' for m in methods))
+ self.assertTrue(any(m['name'] == 'outlier_detector' for m in methods))
+
+ def test_get_method_details(self):
+ # 测试获取StandardScaler的详细信息
+ result = self.reader.get_method_details('StandardScaler')
+ self.assertEqual(result['status'], 'success')
+ self.assertEqual(result['method']['name'], 'StandardScaler')
+
+ # 检查返回的详细信息字段
+ method = result['method']
+ self.assertIn('description', method)
+ self.assertIn('principle', method)
+ self.assertIn('advantages', method)
+ self.assertIn('disadvantages', method)
+ self.assertIn('applicable_scenarios', method)
+ self.assertIn('parameters', method)
+
+ # 检查参数信息
+ parameters = method['parameters']
+ self.assertIsInstance(parameters, list)
+ if parameters:
+ param = parameters[0]
+ self.assertIn('name', param)
+ self.assertIn('type', param)
+ self.assertIn('default', param)
+ self.assertIn('description', param)
+
+ # 测试获取不存在的方法
+ result = self.reader.get_method_details('NonExistentMethod')
+ self.assertEqual(result['status'], 'error')
+
+if __name__ == '__main__':
+ unittest.main()
\ No newline at end of file
diff --git a/function_old/test_model_manager.py b/function_old/test_model_manager.py
new file mode 100644
index 0000000..47b6471
--- /dev/null
+++ b/function_old/test_model_manager.py
@@ -0,0 +1,99 @@
+import pytest
+import pandas as pd
+import numpy as np
+import mlflow
+from pathlib import Path
+from function.model_manager import ModelManager
+
+class TestModelManager:
+ @pytest.fixture
+ def model_manager(self):
+ return ModelManager()
+
+ @pytest.fixture
+ def sample_data(self):
+ # 创建测试数据
+ np.random.seed(42)
+ n_samples = 100
+ X = np.random.randn(n_samples, 4)
+ y = (X[:, 0] + X[:, 1] > 0).astype(int)
+
+ # 保存测试数据
+ data_dir = Path("dataset/dataset_processed/test_data")
+ data_dir.mkdir(parents=True, exist_ok=True)
+
+ df = pd.DataFrame(X, columns=[f'feature_{i}' for i in range(4)])
+ df['label'] = y
+
+ data_path = data_dir / "test_data.csv"
+ df.to_csv(data_path, index=False)
+
+ return str(data_path)
+
+ @pytest.fixture
+ def trained_model(self, sample_data):
+ # 训练一个简单的模型用于测试
+ from sklearn.ensemble import RandomForestClassifier
+
+ # 加载数据
+ data = pd.read_csv(sample_data)
+ X = data.drop('label', axis=1).values
+ y = data['label'].values
+
+ # 训练模型
+ model = RandomForestClassifier(n_estimators=10, random_state=42)
+ model.fit(X, y)
+
+ # 使用MLflow记录模型
+ with mlflow.start_run() as run:
+ mlflow.sklearn.log_model(model, "model")
+ mlflow.log_param("algorithm", "RandomForestClassifier")
+
+ return run.info.run_id
+
+ def test_predict(self, model_manager, sample_data, trained_model):
+ # 设置输出路径
+ output_dir = Path("predictions/test")
+ output_dir.mkdir(parents=True, exist_ok=True)
+ output_path = str(output_dir / "test_predictions.csv")
+
+ # 执行预测
+ result = model_manager.predict(
+ run_id=trained_model,
+ data_path=sample_data,
+ output_path=output_path,
+ metrics=['accuracy', 'f1']
+ )
+
+ # 验证结果
+ assert result['status'] == 'success'
+ assert 'prediction' in result
+ assert Path(result['prediction']['output_file']).exists()
+ assert result['prediction']['samples_count'] == 100
+ assert 'accuracy' in result['prediction']['metrics']
+ assert 'f1' in result['prediction']['metrics']
+
+ # 验证预测结果格式
+ predictions = pd.read_csv(output_path)
+ assert 'prediction' in predictions.columns
+ assert len(predictions) == 100
+
+ def test_predict_invalid_run_id(self, model_manager, sample_data):
+ result = model_manager.predict(
+ run_id="invalid_run_id",
+ data_path=sample_data,
+ output_path="predictions/test/invalid.csv"
+ )
+
+ assert result['status'] == 'error'
+ assert '未找到运行ID' in result['message']
+
+ def test_predict_invalid_data_path(self, model_manager, trained_model):
+ result = model_manager.predict(
+ run_id=trained_model,
+ data_path="invalid/path/data.csv",
+ output_path="predictions/test/invalid.csv"
+ )
+
+ assert result['status'] == 'error'
+ assert '数据加载失败' in result['message']
\ No newline at end of file
diff --git a/function_old/test_model_trainer.py b/function_old/test_model_trainer.py
new file mode 100644
index 0000000..efb19c2
--- /dev/null
+++ b/function_old/test_model_trainer.py
@@ -0,0 +1,85 @@
+import unittest
+import numpy as np
+from function.model_trainer import ModelTrainer
+
+class TestModelTrainer(unittest.TestCase):
+ def setUp(self):
+ self.trainer = ModelTrainer()
+
+ # 创建测试数据
+ np.random.seed(42)
+ self.X_train = np.random.randn(100, 5)
+ self.y_train = np.random.randint(0, 2, 100)
+ self.X_val = np.random.randn(30, 5)
+ self.y_val = np.random.randint(0, 2, 30)
+
+ def test_train_model(self):
+ # 准备训练数据
+ train_data = {
+ 'features': self.X_train,
+ 'labels': self.y_train
+ }
+
+ val_data = {
+ 'features': self.X_val,
+ 'labels': self.y_val
+ }
+
+ # 模型配置
+ model_config = {
+ 'algorithm': 'LogisticRegression',
+ 'task_type': 'classification',
+ 'params': {
+ 'random_state': 42
+ }
+ }
+
+ # 训练模型
+ result = self.trainer.train_model(
+ train_data,
+ val_data,
+ model_config,
+ 'test_experiment'
+ )
+
+ # 验证结果
+ self.assertEqual(result['status'], 'success')
+ self.assertIn('run_id', result)
+ self.assertIn('metrics', result)
+
+ # 验证指标
+ metrics = result['metrics']
+ self.assertIn('accuracy', metrics)
+ self.assertIn('precision', metrics)
+ self.assertIn('recall', metrics)
+ self.assertIn('f1', metrics)
+
+ def test_invalid_algorithm(self):
+ # 测试无效的算法名
+ train_data = {
+ 'features': self.X_train,
+ 'labels': self.y_train
+ }
+
+ val_data = {
+ 'features': self.X_val,
+ 'labels': self.y_val
+ }
+
+ model_config = {
+ 'algorithm': 'InvalidAlgorithm',
+ 'task_type': 'classification',
+ 'params': {}
+ }
+
+ result = self.trainer.train_model(
+ train_data,
+ val_data,
+ model_config,
+ 'test_experiment'
+ )
+
+ self.assertEqual(result['status'], 'error')
+
+if __name__ == '__main__':
+ unittest.main()
\ No newline at end of file
diff --git a/function_old/test_system_monitor.py b/function_old/test_system_monitor.py
new file mode 100644
index 0000000..b591fcb
--- /dev/null
+++ b/function_old/test_system_monitor.py
@@ -0,0 +1,86 @@
+import pytest
+from function.system_monitor import SystemMonitor
+from typing import Dict
+
+class TestSystemMonitor:
+ @pytest.fixture
+ def system_monitor(self):
+ return SystemMonitor()
+
+ def test_get_system_resources(self, system_monitor):
+ """测试获取系统资源信息"""
+ result = system_monitor.get_system_resources()
+
+ # 验证返回格式
+ assert isinstance(result, dict)
+ assert 'status' in result
+ assert result['status'] == 'success'
+ assert 'resources' in result
+ assert 'timestamp' in result
+
+ resources = result['resources']
+
+ # 验证GPU信息
+ assert 'gpu' in resources
+ if resources['gpu']: # 如果有GPU
+ gpu = resources['gpu'][0]
+ assert 'id' in gpu
+ assert 'name' in gpu
+ assert 'memory' in gpu
+ assert 'utilization' in gpu
+ assert 'temperature' in gpu
+
+ # 验证CPU信息
+ assert 'cpu' in resources
+ cpu = resources['cpu']
+ assert 'count' in cpu
+ assert 'utilization' in cpu
+ assert 'memory' in cpu
+ assert 'swap' in cpu
+
+ # 验证内存信息
+ memory = cpu['memory']
+ assert 'total' in memory
+ assert 'used' in memory
+ assert 'free' in memory
+ assert 'percent' in memory
+ assert memory['total'] > 0
+ assert 0 <= memory['percent'] <= 100
+
+ # 验证磁盘信息
+ assert 'disk' in resources
+ assert len(resources['disk']) > 0
+ for mount_point, disk_info in resources['disk'].items():
+ assert 'total' in disk_info
+ assert 'used' in disk_info
+ assert 'free' in disk_info
+ assert 'percent' in disk_info
+ assert disk_info['total'] > 0
+ assert 0 <= disk_info['percent'] <= 100
+
+ # 验证进程信息
+ assert 'processes' in resources
+ processes = resources['processes']
+ assert 'total' in processes
+ assert 'running' in processes
+ assert 'sleeping' in processes
+ assert processes['total'] > 0
+ assert processes['running'] >= 0
+ assert processes['sleeping'] >= 0
+
+ def test_error_handling(self, system_monitor, monkeypatch):
+ """测试错误处理"""
+ def mock_gpu_error(*args, **kwargs):
+ raise Exception("GPU query failed")
+
+ # 模拟GPU查询错误
+ monkeypatch.setattr(system_monitor, '_get_gpu_info', mock_gpu_error)
+
+ result = system_monitor.get_system_resources()
+ assert result['status'] == 'success' # 即使GPU查询失败,其他资源信息仍应返回
+ assert result['resources']['gpu'] == [] # GPU信息应为空列表
+
+ # 验证其他资源信息仍然可用
+ assert 'cpu' in result['resources']
+ assert 'disk' in result['resources']
+ assert 'processes' in result['resources']
\ No newline at end of file
diff --git a/function_old/处理乳腺癌数据集.py b/function_old/处理乳腺癌数据集.py
new file mode 100644
index 0000000..a84aa3b
--- /dev/null
+++ b/function_old/处理乳腺癌数据集.py
@@ -0,0 +1,6 @@
+import pandas as pd
+from sklearn.datasets import load_breast_cancer
+cancer = load_breast_cancer()
+df = pd.DataFrame(cancer.data, columns=cancer.feature_names)
+df['target'] = cancer.target
+df.to_csv('./dataset/breast_cancer.csv', index=False)