完成--完成加载模型预测方法
This commit is contained in:
parent
7b99da7251
commit
b0609e1ad1
@ -500,26 +500,49 @@ Response:
|
||||
}
|
||||
```
|
||||
|
||||
### 2.8 模型预测
|
||||
### 2.9 模型预测
|
||||
```http
|
||||
POST /api/predict
|
||||
POST /api/model/predict
|
||||
Content-Type: application/json
|
||||
|
||||
Request:
|
||||
{
|
||||
"model_id": "model_20230820_001",
|
||||
"run_id": "7970364d490f4e0aa0375c2db26215f3",
|
||||
"data": "dataset/dataset_processed/test.csv",
|
||||
"output_path": "predictions/pred_20230820_001.csv"
|
||||
"output_path": "predictions/pred_20250219_001.csv",
|
||||
"batch_size": 32,
|
||||
"device": "cuda",
|
||||
"return_proba": true,
|
||||
"metrics": ["accuracy", "f1", "precision", "recall"]
|
||||
}
|
||||
|
||||
Response:
|
||||
{
|
||||
"status": "success",
|
||||
"prediction_id": "pred_20230820_001",
|
||||
"output_file": "predictions/pred_20230820_001.csv",
|
||||
"metrics": {
|
||||
"accuracy": 0.95,
|
||||
"f1": 0.94
|
||||
"prediction": {
|
||||
"id": "pred_20250219_001",
|
||||
"run_id": "7970364d490f4e0aa0375c2db26215f3",
|
||||
"model_name": "XGBClassifier",
|
||||
"output_file": "predictions/pred_20250219_001.csv",
|
||||
"prediction_time": "2025-02-19 15:30:45",
|
||||
"samples_count": 1000,
|
||||
"metrics": {
|
||||
"accuracy": 0.956,
|
||||
"f1": 0.948,
|
||||
"precision": 0.962,
|
||||
"recall": 0.935
|
||||
},
|
||||
"execution_time": "5.23s"
|
||||
}
|
||||
}
|
||||
|
||||
Error Response:
|
||||
{
|
||||
"status": "error",
|
||||
"message": "模型预测失败",
|
||||
"details": {
|
||||
"error_type": "ValueError",
|
||||
"error_message": "输入数据格式不正确"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
6
example_model_delete.py
Normal file
6
example_model_delete.py
Normal file
@ -0,0 +1,6 @@
|
||||
from function.model_manager import ModelManager
|
||||
|
||||
# 创建模型管理器实例
|
||||
manager = ModelManager()
|
||||
back = manager.delete_model('7970364d490f4e0aa0375c2db26215f3')
|
||||
print(back)
|
||||
@ -7,7 +7,7 @@ manager = ModelManager()
|
||||
result = manager.get_finished_models(
|
||||
page=1,
|
||||
page_size=10,
|
||||
experiment_name='breast_cancer_classification_2'
|
||||
experiment_name='breast_cancer_classification_3'
|
||||
)
|
||||
|
||||
# 打印结果
|
||||
|
||||
8
example_model_predict.py
Normal file
8
example_model_predict.py
Normal file
@ -0,0 +1,8 @@
|
||||
from function.model_manager import ModelManager
|
||||
|
||||
model_manager = ModelManager()
|
||||
|
||||
print(model_manager.predict(run_id = "33939ea6d8ce4d43a268f23f7361651e",\
|
||||
data_path="/home/admin-root/haotian/MLPlatform/dataset/dataset_processed/breast_cancer_20250219_145614/test_breast_cancer_20250219_145614.csv",\
|
||||
output_path="predictions/pred_breast_cancer_20250219_145614.csv" ,\
|
||||
metrics= ["accuracy", "f1", "precision", "recall"] ))
|
||||
@ -30,7 +30,7 @@ model_config = {
|
||||
|
||||
# 训练模型, 删除训练实验时要删除 mlruns/.trash/ 回收站里的文件
|
||||
# 模型文件 直接在 mlruns/文件夹下
|
||||
for i in range(8, 20):
|
||||
for i in range(3, 4):
|
||||
result = trainer.train_model(
|
||||
{
|
||||
'features': X_train,
|
||||
|
||||
Binary file not shown.
@ -4,9 +4,19 @@ import pandas as pd
|
||||
from typing import Dict, List, Optional
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import datetime
|
||||
from datetime import datetime
|
||||
import yaml
|
||||
import json
|
||||
import time
|
||||
import os
|
||||
import numpy as np
|
||||
from sklearn.metrics import (
|
||||
accuracy_score, precision_score, recall_score, f1_score, roc_auc_score,
|
||||
mean_absolute_error, mean_squared_error, r2_score, explained_variance_score,
|
||||
adjusted_rand_score, homogeneity_score, completeness_score, silhouette_score
|
||||
)
|
||||
import torch
|
||||
from torch.utils.data import DataLoader, TensorDataset
|
||||
|
||||
class ModelManager:
|
||||
"""模型管理类"""
|
||||
@ -16,6 +26,7 @@ class ModelManager:
|
||||
self.config = config or {}
|
||||
self.logger = logging.getLogger(__name__)
|
||||
self._setup_logging()
|
||||
self._metrics_map()
|
||||
|
||||
# 初始化MLflow客户端
|
||||
self.mlflow_uri = self.config.get('mlflow_uri', 'http://10.0.0.202:5000')
|
||||
@ -28,13 +39,30 @@ class ModelManager:
|
||||
log_dir.mkdir(exist_ok=True)
|
||||
|
||||
file_handler = logging.FileHandler(
|
||||
log_dir / f'model_manager_{datetime.datetime.now():%Y%m%d_%H%M%S}.log'
|
||||
log_dir / f'model_manager_{datetime.now():%Y%m%d_%H%M%S}.log'
|
||||
)
|
||||
file_handler.setFormatter(
|
||||
logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
)
|
||||
self.logger.addHandler(file_handler)
|
||||
self.logger.setLevel(logging.INFO)
|
||||
|
||||
def _metrics_map(self):
|
||||
self.metrics_map={
|
||||
'accuracy' : accuracy_score,
|
||||
'precision' : precision_score,
|
||||
'recall' : recall_score,
|
||||
'f1' : f1_score,
|
||||
'mae' : mean_absolute_error,
|
||||
'mse' : mean_squared_error,
|
||||
# 'rmse' : np.sqrt(mean_absolute_error), # 这里要特殊处理一下
|
||||
'r2': r2_score,
|
||||
'explained_variance' : explained_variance_score,
|
||||
'adjusted_rand' : adjusted_rand_score,
|
||||
'homogeneity' : homogeneity_score,
|
||||
'completeness': completeness_score,
|
||||
'silhouette' : silhouette_score
|
||||
}
|
||||
|
||||
def get_finished_models(
|
||||
self,
|
||||
@ -269,4 +297,158 @@ class ModelManager:
|
||||
return {
|
||||
'status': 'error',
|
||||
'message': error_msg
|
||||
}
|
||||
}
|
||||
|
||||
def predict(
|
||||
self,
|
||||
run_id: str,
|
||||
data_path: str,
|
||||
output_path: str,
|
||||
batch_size: int = 32,
|
||||
device: str = 'cuda' if torch.cuda.is_available() else 'cpu',
|
||||
return_proba: bool = True,
|
||||
metrics: List[str] = None
|
||||
) -> Dict:
|
||||
"""
|
||||
使用指定的模型进行预测
|
||||
|
||||
Args:
|
||||
run_id: MLflow运行ID
|
||||
data_path: 输入数据路径
|
||||
output_path: 预测结果保存路径
|
||||
batch_size: 批处理大小
|
||||
device: 计算设备 ('cuda' or 'cpu')
|
||||
return_proba: 是否返回概率预测
|
||||
metrics: 评估指标列表
|
||||
|
||||
Returns:
|
||||
预测结果信息
|
||||
"""
|
||||
# try:
|
||||
start_time = time.time()
|
||||
|
||||
# 获取模型信息
|
||||
run = self.client.get_run(run_id)
|
||||
if not run:
|
||||
return {
|
||||
'status': 'error',
|
||||
'message': f'未找到运行ID为 {run_id} 的模型'
|
||||
}
|
||||
|
||||
# 加载模型
|
||||
model = mlflow.pyfunc.load_model(f"runs:/{run_id}/model")
|
||||
model_name = run.data.params.get('algorithm', 'Unknown')
|
||||
|
||||
# 加载数据
|
||||
try:
|
||||
data = pd.read_csv(data_path)
|
||||
if 'label' in data.columns:
|
||||
y_true = data.pop('label').values
|
||||
has_labels = True
|
||||
elif 'target' in data.columns:
|
||||
y_true = data.pop('target').values
|
||||
has_labels = True
|
||||
else:
|
||||
has_labels = False
|
||||
X = data.values
|
||||
except Exception as e:
|
||||
return {
|
||||
'status': 'error',
|
||||
'message': '数据加载失败',
|
||||
'details': {
|
||||
'error_type': type(e).__name__,
|
||||
'error_message': str(e)
|
||||
}
|
||||
}
|
||||
|
||||
# 创建预测ID
|
||||
pred_id = f"pred_{datetime.now():%Y%m%d_%H%M%S}"
|
||||
|
||||
# 进行预测
|
||||
if isinstance(model, torch.nn.Module):
|
||||
# PyTorch模型预测
|
||||
model.to(device)
|
||||
model.eval()
|
||||
|
||||
dataset = TensorDataset(torch.FloatTensor(X))
|
||||
dataloader = DataLoader(dataset, batch_size=batch_size)
|
||||
|
||||
predictions = []
|
||||
probas = []
|
||||
|
||||
with torch.no_grad():
|
||||
for batch in dataloader:
|
||||
batch = batch[0].to(device)
|
||||
outputs = model(batch)
|
||||
|
||||
if return_proba:
|
||||
proba = torch.softmax(outputs, dim=1)
|
||||
probas.append(proba.cpu().numpy())
|
||||
|
||||
preds = outputs.argmax(dim=1)
|
||||
predictions.append(preds.cpu().numpy())
|
||||
|
||||
predictions = np.concatenate(predictions)
|
||||
if return_proba:
|
||||
probas = np.concatenate(probas)
|
||||
else:
|
||||
# 其他模型预测
|
||||
predictions = model.predict(X)
|
||||
if return_proba and hasattr(model, 'predict_proba'):
|
||||
probas = model.predict_proba(X)
|
||||
else:
|
||||
probas = []
|
||||
|
||||
# 计算评估指标
|
||||
metrics_results = {}
|
||||
if has_labels and metrics:
|
||||
for metric in metrics:
|
||||
if metric in self.metrics_map.keys():
|
||||
metrics_results[metric] = float(self.metrics_map[metric](y_true, predictions))
|
||||
|
||||
# 保存预测结果
|
||||
results_df = pd.DataFrame({
|
||||
'prediction': predictions
|
||||
})
|
||||
if return_proba and len(probas) > 0:
|
||||
for i in range(probas.shape[1]):
|
||||
results_df[f'probability_{i}'] = probas[:, i]
|
||||
|
||||
# 确保输出目录存在
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
results_df.to_csv(output_path, index=False)
|
||||
|
||||
# 计算执行时间
|
||||
execution_time = time.time() - start_time
|
||||
|
||||
# 记录日志
|
||||
self.logger.info(
|
||||
f"预测完成 - Run ID: {run_id}, 模型: {model_name}, "
|
||||
f"样本数: {len(predictions)}, 耗时: {execution_time:.2f}s"
|
||||
)
|
||||
|
||||
return {
|
||||
'status': 'success',
|
||||
'prediction': {
|
||||
'id': pred_id,
|
||||
'run_id': run_id,
|
||||
'model_name': model_name,
|
||||
'output_file': output_path,
|
||||
'prediction_time': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
|
||||
'samples_count': len(predictions),
|
||||
'metrics': metrics_results,
|
||||
'execution_time': f"{execution_time:.2f}s"
|
||||
}
|
||||
}
|
||||
|
||||
# except Exception as e:
|
||||
# error_msg = f"预测过程发生错误: {str(e)}"
|
||||
# self.logger.error(error_msg)
|
||||
# return {
|
||||
# 'status': 'error',
|
||||
# 'message': '模型预测失败',
|
||||
# 'details': {
|
||||
# 'error_type': type(e).__name__,
|
||||
# 'error_message': str(e)
|
||||
# }
|
||||
# }
|
||||
@ -0,0 +1,20 @@
|
||||
artifact_path: model
|
||||
flavors:
|
||||
python_function:
|
||||
env:
|
||||
conda: conda.yaml
|
||||
virtualenv: python_env.yaml
|
||||
loader_module: mlflow.sklearn
|
||||
model_path: model.pkl
|
||||
predict_fn: predict
|
||||
python_version: 3.9.19
|
||||
sklearn:
|
||||
code: null
|
||||
pickled_model: model.pkl
|
||||
serialization_format: cloudpickle
|
||||
sklearn_version: 1.5.2
|
||||
mlflow_version: 2.20.1
|
||||
model_size_bytes: 96534
|
||||
model_uuid: 80ec24f7f34643b9be9da8be761430eb
|
||||
run_id: 725c551dce9c477391856e6ac41c75bf
|
||||
utc_time_created: '2025-02-20 01:43:39.758338'
|
||||
@ -0,0 +1,15 @@
|
||||
channels:
|
||||
- conda-forge
|
||||
dependencies:
|
||||
- python=3.9.19
|
||||
- pip<=24.0
|
||||
- pip:
|
||||
- mlflow==2.20.1
|
||||
- cloudpickle==3.1.0
|
||||
- numpy==1.26.4
|
||||
- pandas==2.2.2
|
||||
- psutil==6.0.0
|
||||
- scikit-learn==1.5.2
|
||||
- scipy==1.13.1
|
||||
- xgboost==2.1.4
|
||||
name: mlflow-env
|
||||
Binary file not shown.
@ -0,0 +1,7 @@
|
||||
python: 3.9.19
|
||||
build_dependencies:
|
||||
- pip==24.0
|
||||
- setuptools==60.2.0
|
||||
- wheel==0.43.0
|
||||
dependencies:
|
||||
- -r requirements.txt
|
||||
@ -0,0 +1,8 @@
|
||||
mlflow==2.20.1
|
||||
cloudpickle==3.1.0
|
||||
numpy==1.26.4
|
||||
pandas==2.2.2
|
||||
psutil==6.0.0
|
||||
scikit-learn==1.5.2
|
||||
scipy==1.13.1
|
||||
xgboost==2.1.4
|
||||
@ -0,0 +1,20 @@
|
||||
artifact_path: model
|
||||
flavors:
|
||||
python_function:
|
||||
env:
|
||||
conda: conda.yaml
|
||||
virtualenv: python_env.yaml
|
||||
loader_module: mlflow.sklearn
|
||||
model_path: model.pkl
|
||||
predict_fn: predict
|
||||
python_version: 3.9.19
|
||||
sklearn:
|
||||
code: null
|
||||
pickled_model: model.pkl
|
||||
serialization_format: cloudpickle
|
||||
sklearn_version: 1.5.2
|
||||
mlflow_version: 2.20.1
|
||||
model_size_bytes: 96534
|
||||
model_uuid: 80a9b0fb66324b0f982aebb166759911
|
||||
run_id: d16c356bb3324ede819f9998d427780a
|
||||
utc_time_created: '2025-02-20 01:44:24.622216'
|
||||
@ -0,0 +1,15 @@
|
||||
channels:
|
||||
- conda-forge
|
||||
dependencies:
|
||||
- python=3.9.19
|
||||
- pip<=24.0
|
||||
- pip:
|
||||
- mlflow==2.20.1
|
||||
- cloudpickle==3.1.0
|
||||
- numpy==1.26.4
|
||||
- pandas==2.2.2
|
||||
- psutil==6.0.0
|
||||
- scikit-learn==1.5.2
|
||||
- scipy==1.13.1
|
||||
- xgboost==2.1.4
|
||||
name: mlflow-env
|
||||
Binary file not shown.
@ -0,0 +1,7 @@
|
||||
python: 3.9.19
|
||||
build_dependencies:
|
||||
- pip==24.0
|
||||
- setuptools==60.2.0
|
||||
- wheel==0.43.0
|
||||
dependencies:
|
||||
- -r requirements.txt
|
||||
@ -0,0 +1,8 @@
|
||||
mlflow==2.20.1
|
||||
cloudpickle==3.1.0
|
||||
numpy==1.26.4
|
||||
pandas==2.2.2
|
||||
psutil==6.0.0
|
||||
scikit-learn==1.5.2
|
||||
scipy==1.13.1
|
||||
xgboost==2.1.4
|
||||
@ -0,0 +1,20 @@
|
||||
artifact_path: model
|
||||
flavors:
|
||||
python_function:
|
||||
env:
|
||||
conda: conda.yaml
|
||||
virtualenv: python_env.yaml
|
||||
loader_module: mlflow.sklearn
|
||||
model_path: model.pkl
|
||||
predict_fn: predict
|
||||
python_version: 3.9.19
|
||||
sklearn:
|
||||
code: null
|
||||
pickled_model: model.pkl
|
||||
serialization_format: cloudpickle
|
||||
sklearn_version: 1.5.2
|
||||
mlflow_version: 2.20.1
|
||||
model_size_bytes: 96534
|
||||
model_uuid: 770483f5da38405496fdedeef1cfe93f
|
||||
run_id: 81b3390084a146bb949fec21c80dba2c
|
||||
utc_time_created: '2025-02-20 01:43:43.110167'
|
||||
@ -0,0 +1,15 @@
|
||||
channels:
|
||||
- conda-forge
|
||||
dependencies:
|
||||
- python=3.9.19
|
||||
- pip<=24.0
|
||||
- pip:
|
||||
- mlflow==2.20.1
|
||||
- cloudpickle==3.1.0
|
||||
- numpy==1.26.4
|
||||
- pandas==2.2.2
|
||||
- psutil==6.0.0
|
||||
- scikit-learn==1.5.2
|
||||
- scipy==1.13.1
|
||||
- xgboost==2.1.4
|
||||
name: mlflow-env
|
||||
Binary file not shown.
@ -0,0 +1,7 @@
|
||||
python: 3.9.19
|
||||
build_dependencies:
|
||||
- pip==24.0
|
||||
- setuptools==60.2.0
|
||||
- wheel==0.43.0
|
||||
dependencies:
|
||||
- -r requirements.txt
|
||||
@ -0,0 +1,8 @@
|
||||
mlflow==2.20.1
|
||||
cloudpickle==3.1.0
|
||||
numpy==1.26.4
|
||||
pandas==2.2.2
|
||||
psutil==6.0.0
|
||||
scikit-learn==1.5.2
|
||||
scipy==1.13.1
|
||||
xgboost==2.1.4
|
||||
@ -0,0 +1,15 @@
|
||||
artifact_uri: mlflow-artifacts:/189649205577051698/725c551dce9c477391856e6ac41c75bf/artifacts
|
||||
end_time: 1740015822953
|
||||
entry_point_name: ''
|
||||
experiment_id: '189649205577051698'
|
||||
lifecycle_stage: active
|
||||
run_id: 725c551dce9c477391856e6ac41c75bf
|
||||
run_name: redolent-slug-558
|
||||
run_uuid: 725c551dce9c477391856e6ac41c75bf
|
||||
source_name: ''
|
||||
source_type: 4
|
||||
source_version: ''
|
||||
start_time: 1740015819406
|
||||
status: 3
|
||||
tags: []
|
||||
user_id: admin-root
|
||||
@ -0,0 +1 @@
|
||||
1740015819735 0.961038961038961 0
|
||||
@ -0,0 +1 @@
|
||||
1740015819747 0.9612318007520749 0
|
||||
@ -0,0 +1 @@
|
||||
1740015819739 0.9617833147244911 0
|
||||
@ -0,0 +1 @@
|
||||
1740015819743 0.961038961038961 0
|
||||
@ -0,0 +1 @@
|
||||
1740015819752 0.9607692307692308 0
|
||||
@ -0,0 +1 @@
|
||||
['计算效率高，支持并行计算。', '具有内置的缺失值处理能力。']
|
||||
@ -0,0 +1 @@
|
||||
XGBClassifier
|
||||
@ -0,0 +1 @@
|
||||
/home/admin-root/haotian/MLPlatform/dataset/dataset_processed/breast_cancer_20250219_144629
|
||||
@ -0,0 +1 @@
|
||||
['参数较多，调优较复杂。']
|
||||
@ -0,0 +1 @@
|
||||
0.1
|
||||
@ -0,0 +1 @@
|
||||
6
|
||||
@ -0,0 +1 @@
|
||||
100
|
||||
@ -0,0 +1 @@
|
||||
XGBoost（Extreme Gradient Boosting）是一种基于梯度提升树（GBDT）的改进算法，具有更强的正则化和并行处理能力。
|
||||
@ -0,0 +1 @@
|
||||
42
|
||||
@ -0,0 +1 @@
|
||||
classification
|
||||
@ -0,0 +1 @@
|
||||
[{"run_id": "725c551dce9c477391856e6ac41c75bf", "artifact_path": "model", "utc_time_created": "2025-02-20 01:43:39.758338", "model_uuid": "80ec24f7f34643b9be9da8be761430eb", "flavors": {"python_function": {"model_path": "model.pkl", "predict_fn": "predict", "loader_module": "mlflow.sklearn", "python_version": "3.9.19", "env": {"conda": "conda.yaml", "virtualenv": "python_env.yaml"}}, "sklearn": {"pickled_model": "model.pkl", "sklearn_version": "1.5.2", "serialization_format": "cloudpickle", "code": null}}}]
|
||||
@ -0,0 +1 @@
|
||||
redolent-slug-558
|
||||
@ -0,0 +1 @@
|
||||
7b99da7251d23ceb9b3dfbb599e96dc00054e961
|
||||
@ -0,0 +1 @@
|
||||
/home/admin-root/haotian/MLPlatform/example_model_trainer.py
|
||||
@ -0,0 +1 @@
|
||||
LOCAL
|
||||
@ -0,0 +1 @@
|
||||
admin-root
|
||||
6
mlruns/189649205577051698/meta.yaml
Normal file
6
mlruns/189649205577051698/meta.yaml
Normal file
@ -0,0 +1,6 @@
|
||||
artifact_location: mlflow-artifacts:/189649205577051698
|
||||
creation_time: 1740015819325
|
||||
experiment_id: '189649205577051698'
|
||||
last_update_time: 1740015819325
|
||||
lifecycle_stage: active
|
||||
name: breast_cancer_classification_1
|
||||
@ -0,0 +1,15 @@
|
||||
artifact_uri: mlflow-artifacts:/433321862082712659/d16c356bb3324ede819f9998d427780a/artifacts
|
||||
end_time: 1740015867827
|
||||
entry_point_name: ''
|
||||
experiment_id: '433321862082712659'
|
||||
lifecycle_stage: active
|
||||
run_id: d16c356bb3324ede819f9998d427780a
|
||||
run_name: skillful-swan-139
|
||||
run_uuid: d16c356bb3324ede819f9998d427780a
|
||||
source_name: ''
|
||||
source_type: 4
|
||||
source_version: ''
|
||||
start_time: 1740015864275
|
||||
status: 3
|
||||
tags: []
|
||||
user_id: admin-root
|
||||
@ -0,0 +1 @@
|
||||
1740015864600 0.961038961038961 0
|
||||
@ -0,0 +1 @@
|
||||
1740015864611 0.9612318007520749 0
|
||||
@ -0,0 +1 @@
|
||||
1740015864604 0.9617833147244911 0
|
||||
@ -0,0 +1 @@
|
||||
1740015864607 0.961038961038961 0
|
||||
@ -0,0 +1 @@
|
||||
1740015864616 0.9607692307692308 0
|
||||
@ -0,0 +1 @@
|
||||
['计算效率高，支持并行计算。', '具有内置的缺失值处理能力。']
|
||||
@ -0,0 +1 @@
|
||||
XGBClassifier
|
||||
@ -0,0 +1 @@
|
||||
/home/admin-root/haotian/MLPlatform/dataset/dataset_processed/breast_cancer_20250219_144629
|
||||
@ -0,0 +1 @@
|
||||
['参数较多，调优较复杂。']
|
||||
@ -0,0 +1 @@
|
||||
0.1
|
||||
@ -0,0 +1 @@
|
||||
6
|
||||
@ -0,0 +1 @@
|
||||
100
|
||||
@ -0,0 +1 @@
|
||||
XGBoost（Extreme Gradient Boosting）是一种基于梯度提升树（GBDT）的改进算法，具有更强的正则化和并行处理能力。
|
||||
@ -0,0 +1 @@
|
||||
42
|
||||
@ -0,0 +1 @@
|
||||
classification
|
||||
@ -0,0 +1 @@
|
||||
[{"run_id": "d16c356bb3324ede819f9998d427780a", "artifact_path": "model", "utc_time_created": "2025-02-20 01:44:24.622216", "model_uuid": "80a9b0fb66324b0f982aebb166759911", "flavors": {"python_function": {"model_path": "model.pkl", "predict_fn": "predict", "loader_module": "mlflow.sklearn", "python_version": "3.9.19", "env": {"conda": "conda.yaml", "virtualenv": "python_env.yaml"}}, "sklearn": {"pickled_model": "model.pkl", "sklearn_version": "1.5.2", "serialization_format": "cloudpickle", "code": null}}}]
|
||||
@ -0,0 +1 @@
|
||||
skillful-swan-139
|
||||
@ -0,0 +1 @@
|
||||
7b99da7251d23ceb9b3dfbb599e96dc00054e961
|
||||
@ -0,0 +1 @@
|
||||
/home/admin-root/haotian/MLPlatform/example_model_trainer.py
|
||||
@ -0,0 +1 @@
|
||||
LOCAL
|
||||
@ -0,0 +1 @@
|
||||
admin-root
|
||||
@ -0,0 +1,15 @@
|
||||
artifact_uri: mlflow-artifacts:/656341556838275234/81b3390084a146bb949fec21c80dba2c/artifacts
|
||||
end_time: 1740015825618
|
||||
entry_point_name: ''
|
||||
experiment_id: '656341556838275234'
|
||||
lifecycle_stage: active
|
||||
run_id: 81b3390084a146bb949fec21c80dba2c
|
||||
run_name: rebellious-fish-472
|
||||
run_uuid: 81b3390084a146bb949fec21c80dba2c
|
||||
source_name: ''
|
||||
source_type: 4
|
||||
source_version: ''
|
||||
start_time: 1740015822972
|
||||
status: 3
|
||||
tags: []
|
||||
user_id: admin-root
|
||||
@ -0,0 +1 @@
|
||||
1740015823091 0.961038961038961 0
|
||||
@ -0,0 +1 @@
|
||||
1740015823104 0.9612318007520749 0
|
||||
@ -0,0 +1 @@
|
||||
1740015823096 0.9617833147244911 0
|
||||
@ -0,0 +1 @@
|
||||
1740015823100 0.961038961038961 0
|
||||
@ -0,0 +1 @@
|
||||
1740015823106 0.9607692307692308 0
|
||||
@ -0,0 +1 @@
|
||||
['计算效率高，支持并行计算。', '具有内置的缺失值处理能力。']
|
||||
@ -0,0 +1 @@
|
||||
XGBClassifier
|
||||
@ -0,0 +1 @@
|
||||
/home/admin-root/haotian/MLPlatform/dataset/dataset_processed/breast_cancer_20250219_144629
|
||||
@ -0,0 +1 @@
|
||||
['参数较多，调优较复杂。']
|
||||
@ -0,0 +1 @@
|
||||
0.1
|
||||
@ -0,0 +1 @@
|
||||
6
|
||||
@ -0,0 +1 @@
|
||||
100
|
||||
@ -0,0 +1 @@
|
||||
XGBoost（Extreme Gradient Boosting）是一种基于梯度提升树（GBDT）的改进算法，具有更强的正则化和并行处理能力。
|
||||
@ -0,0 +1 @@
|
||||
42
|
||||
@ -0,0 +1 @@
|
||||
classification
|
||||
@ -0,0 +1 @@
|
||||
[{"run_id": "81b3390084a146bb949fec21c80dba2c", "artifact_path": "model", "utc_time_created": "2025-02-20 01:43:43.110167", "model_uuid": "770483f5da38405496fdedeef1cfe93f", "flavors": {"python_function": {"model_path": "model.pkl", "predict_fn": "predict", "loader_module": "mlflow.sklearn", "python_version": "3.9.19", "env": {"conda": "conda.yaml", "virtualenv": "python_env.yaml"}}, "sklearn": {"pickled_model": "model.pkl", "sklearn_version": "1.5.2", "serialization_format": "cloudpickle", "code": null}}}]
|
||||
@ -0,0 +1 @@
|
||||
rebellious-fish-472
|
||||
@ -0,0 +1 @@
|
||||
7b99da7251d23ceb9b3dfbb599e96dc00054e961
|
||||
@ -0,0 +1 @@
|
||||
/home/admin-root/haotian/MLPlatform/example_model_trainer.py
|
||||
@ -0,0 +1 @@
|
||||
LOCAL
|
||||
@ -0,0 +1 @@
|
||||
admin-root
|
||||
104
predictions/pred_breast_cancer_20250219_145614.csv
Normal file
104
predictions/pred_breast_cancer_20250219_145614.csv
Normal file
@ -0,0 +1,104 @@
|
||||
prediction
|
||||
0
|
||||
1
|
||||
0
|
||||
0
|
||||
1
|
||||
0
|
||||
1
|
||||
1
|
||||
0
|
||||
1
|
||||
1
|
||||
0
|
||||
1
|
||||
1
|
||||
0
|
||||
1
|
||||
1
|
||||
0
|
||||
0
|
||||
0
|
||||
1
|
||||
1
|
||||
1
|
||||
0
|
||||
0
|
||||
0
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
0
|
||||
1
|
||||
1
|
||||
1
|
||||
0
|
||||
0
|
||||
0
|
||||
0
|
||||
0
|
||||
1
|
||||
0
|
||||
1
|
||||
0
|
||||
0
|
||||
1
|
||||
0
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
0
|
||||
1
|
||||
1
|
||||
0
|
||||
1
|
||||
1
|
||||
0
|
||||
1
|
||||
1
|
||||
0
|
||||
0
|
||||
1
|
||||
1
|
||||
0
|
||||
0
|
||||
0
|
||||
0
|
||||
0
|
||||
1
|
||||
0
|
||||
1
|
||||
1
|
||||
0
|
||||
0
|
||||
1
|
||||
1
|
||||
0
|
||||
1
|
||||
0
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
0
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
0
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
|
99
test_model_manager.py
Normal file
99
test_model_manager.py
Normal file
@ -0,0 +1,99 @@
|
||||
import pytest
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import mlflow
|
||||
from pathlib import Path
|
||||
from function.model_manager import ModelManager
|
||||
|
||||
class TestModelManager:
|
||||
@pytest.fixture
|
||||
def model_manager(self):
|
||||
return ModelManager()
|
||||
|
||||
@pytest.fixture
|
||||
def sample_data(self):
|
||||
# 创建测试数据
|
||||
np.random.seed(42)
|
||||
n_samples = 100
|
||||
X = np.random.randn(n_samples, 4)
|
||||
y = (X[:, 0] + X[:, 1] > 0).astype(int)
|
||||
|
||||
# 保存测试数据
|
||||
data_dir = Path("dataset/dataset_processed/test_data")
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
df = pd.DataFrame(X, columns=[f'feature_{i}' for i in range(4)])
|
||||
df['label'] = y
|
||||
|
||||
data_path = data_dir / "test_data.csv"
|
||||
df.to_csv(data_path, index=False)
|
||||
|
||||
return str(data_path)
|
||||
|
||||
@pytest.fixture
|
||||
def trained_model(self, sample_data):
|
||||
# 训练一个简单的模型用于测试
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
|
||||
# 加载数据
|
||||
data = pd.read_csv(sample_data)
|
||||
X = data.drop('label', axis=1).values
|
||||
y = data['label'].values
|
||||
|
||||
# 训练模型
|
||||
model = RandomForestClassifier(n_estimators=10, random_state=42)
|
||||
model.fit(X, y)
|
||||
|
||||
# 使用MLflow记录模型
|
||||
with mlflow.start_run() as run:
|
||||
mlflow.sklearn.log_model(model, "model")
|
||||
mlflow.log_param("algorithm", "RandomForestClassifier")
|
||||
|
||||
return run.info.run_id
|
||||
|
||||
def test_predict(self, model_manager, sample_data, trained_model):
|
||||
# 设置输出路径
|
||||
output_dir = Path("predictions/test")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = str(output_dir / "test_predictions.csv")
|
||||
|
||||
# 执行预测
|
||||
result = model_manager.predict(
|
||||
run_id=trained_model,
|
||||
data_path=sample_data,
|
||||
output_path=output_path,
|
||||
metrics=['accuracy', 'f1']
|
||||
)
|
||||
|
||||
# 验证结果
|
||||
assert result['status'] == 'success'
|
||||
assert 'prediction' in result
|
||||
assert Path(result['prediction']['output_file']).exists()
|
||||
assert result['prediction']['samples_count'] == 100
|
||||
assert 'accuracy' in result['prediction']['metrics']
|
||||
assert 'f1' in result['prediction']['metrics']
|
||||
|
||||
# 验证预测结果格式
|
||||
predictions = pd.read_csv(output_path)
|
||||
assert 'prediction' in predictions.columns
|
||||
assert len(predictions) == 100
|
||||
|
||||
def test_predict_invalid_run_id(self, model_manager, sample_data):
|
||||
result = model_manager.predict(
|
||||
run_id="invalid_run_id",
|
||||
data_path=sample_data,
|
||||
output_path="predictions/test/invalid.csv"
|
||||
)
|
||||
|
||||
assert result['status'] == 'error'
|
||||
assert '未找到运行ID' in result['message']
|
||||
|
||||
def test_predict_invalid_data_path(self, model_manager, trained_model):
|
||||
result = model_manager.predict(
|
||||
run_id=trained_model,
|
||||
data_path="invalid/path/data.csv",
|
||||
output_path="predictions/test/invalid.csv"
|
||||
)
|
||||
|
||||
assert result['status'] == 'error'
|
||||
assert '数据加载失败' in result['message']
|
||||
Loading…
Reference in New Issue
Block a user