From e81b2e96d2fe22d0f243e76167b651431e5b7328 Mon Sep 17 00:00:00 2001 From: haotian <2421912570@qq.com> Date: Mon, 24 Feb 2025 11:55:35 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90--fastapi=E6=A1=86=E6=9E=B6?= =?UTF-8?q?=E5=88=9D=E6=AD=A5=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- __pycache__/main.cpython-39.pyc | Bin 0 -> 4599 bytes api/__pycache__/data_api.cpython-39.pyc | Bin 0 -> 2750 bytes api/__pycache__/model_api.cpython-39.pyc | Bin 0 -> 3713 bytes api/__pycache__/system_api.cpython-39.pyc | Bin 0 -> 1616 bytes api/data_api.py | 76 +++++++++ api/model_api.py | 114 +++++++++++++ api/system_api.py | 6 +- doc/安装文档.md | 63 +++++++ doc/接口文档code.md | 38 ++--- .../__pycache__/data_manager.cpython-39.pyc | Bin 13955 -> 14393 bytes function/data_manager.py | 17 +- main.py | 158 ++++++++++++++---- 12 files changed, 419 insertions(+), 53 deletions(-) create mode 100644 __pycache__/main.cpython-39.pyc create mode 100644 api/__pycache__/data_api.cpython-39.pyc create mode 100644 api/__pycache__/model_api.cpython-39.pyc create mode 100644 api/__pycache__/system_api.cpython-39.pyc create mode 100644 doc/安装文档.md diff --git a/__pycache__/main.cpython-39.pyc b/__pycache__/main.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..afaf6e2c37e4f89fe9f7f372825c74d5e31ad0dd GIT binary patch literal 4599 zcmZ`6YjYIGac6gL_wJ+H zNLM0dTR^3-6>#F%DCX-g+cmAK(O8@Fml&|n(=?|%UV1)lr6@@C4xQa&##8V5Zrxi3$FX&#P zkRV7?UBfdAri3*&>7@!O3F~g!YbmryIN`Q>nL3+>*@!b*uV-Ht*B z?3Hv^d7XvM2Cl2n1vt~~E_A~?<@R_zg`Ng&bzwE&((W4Xj>1}rOS|g;Uf+Q41o$qZ zFhW~sE6vb0x`MXPBnnx&l6IU?3LDslX{E4{b<$PSq_Bx?8rw`eY1a&aHC@4Ny-DD!Gmlq?W6e;q5X7# zZgYr3==K>MB>>ubdOzLq1GKy71N6b$(ZCK5)d+oh zj{viqJ|;0c*;cw|T)n9Bzp;+7N9p6zR(t8bnku0u=#vd-KkaNlKc)v7&_VhX+MGT; zgPv&)DY-*8(Zh2IVeYrX=<%ZmMdrZp@X;qHimV(tfiKJ_Sef}W6f5@H#qkmkDn1>y z!|_m<$Jj3_ER1qQwCy``Z0N8XJCW**_rzJ zS~amPKad}&Ci-bm468;zX8?ft5|I(1)^AhK@%wlX zME#?75IMHrfB4W**N#R4?)7`N*rpRM@ z_Jjo|IM0G_fPZCd*9}ft%{43I&my=J^T8Bu*yL#L)L78ikN1PpLLZHyGTfZW;05lkXhY{|8DN^Q?nsQp5B2$o^4uW4CP)=(xc1m5XoYo^U1-W)g8&fISrA%qy zH|k)u?b*JgJSc*N`i3z&HKC(X`JJ#Q&%=~U@CV=}Qbo|h^4UZr(qUxt$U;7mla>xbaL}gFW2W1|U+V z$QYpnW|gW?O*Lr{z0wZE*dSOo>VJ8${@G_zdl2?9pz%F8;hp3(DLvl@urOj(Z>vH# zdg*fEkHfk+m_c|Qgx9|X(GLK2ndnJCO40WK5Gg2l1a2HBKTN7KDuDz^b()}tOpYc^ z)>OVD=JHjVqG`DG7O7-w1)9NzlJ&EnT)*~OtY)?A89z2G%`TdHXLeYo#)+ck_#=Vy z??rEjOA{6mtWM|7-kiQ1dyGg>cA`?+Kgf5(IkBw_z7An@qd0Eji;vTq&`K=2t?Aa3 zn9sQFF1OwZ*eS%zQnb}@?eXPzt+>4S#kUvVB*ZAt0YZylKvlm7P(BB{VL^UCeVQ@O@IfW z0Vu)R9YG9ElQ9jt1MCmbnN^xthA|o#6Z?0$I*F@k3e1(Rptx~UQ@?n%e*W_O>z_cU zQ~%YQ_4i(?zx>+#Z)dK5Ia}=-lI1NcCB))hDd+R~>MC@XkVVpM;B9gH)G*X-vkZNn zABFsG*uA>u`&^B5%_!dtbl!_hYnPtDXf~`Av3879Tz7I1?gj%v=om~KQwhh#WfiD+=jUuJI&$kN_;(s%ZznEW-F z{2LZK`|-`!-;XUe|Jhk^@P@g1n##Zl5t<8+0fP-fI+PLpQ`q>nvbQybiiMvAzEFip z(YOV(NLm)QIF?mS;(I4JAk_09jH)Iyr1%KE9Pr4=Zh{k@!X+kj-s8|r+(b7LDQI{s z*^+P+B4=k?2Cj1=_7oFaOJrYzoEXRaL1b?QP4eiP7OI3v5&CxgjQ>PH zCk1qw$B;p^#$bckhQv6RNouA6$7GIGN_LnW6N5kzEOghO$x}|)p;O!^+iIa79Nd3I zwCs0Wb|CHy`7_9r8HzZwy#`LCAlDy?TV!wnY(Xt6j)^vGfW9-gOhO{P1c6?F%}4#u{{f*eK(zn> literal 0 HcmV?d00001 diff --git a/api/__pycache__/data_api.cpython-39.pyc b/api/__pycache__/data_api.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88acd5a5dd146b24cf8c271b91ef81f8401c86ea GIT binary patch literal 2750 zcmb_e-ES2|6rb6zyZ5Ut^aBba2#8)2xES9EK{WW4&{`2g+?UPf&a_>;yL*}4O>AvV zw3TQK;)CD=m{0=-UPPk_Att`~U(ACKntt%^lO)7*X1ABUwc-oAnX@xz&&y!$yAo8LDkYpQ5U0k=AnUIYW-@D$FpF0gbf=HkuJyCh5@j|kFOmi73@A-y ziSjM1dX=&ot1lX$->UR|VA;8sx zM%fOv!Zx<^ZPIw+9{5Nax-7pm@y4krO}LQt*H53Gc;)>T?<9T{O7pFhiy1Jn-&6{( zEsc|YoJiw@-%1)(R!?||H|B-j6c;cozwE_)EMmOf!nKQ+4uj*+;#Wa%;!s8$?H$5( zZZLW@dus4Bxiv~arx*GL^ewO~O={|$c@%I24@$CcLPRYd$EWzYl*7I%e%MJ9chYZj zSp}DH(D9OKS?>tm$!z^_N)AqPFG&S=c{t^VoC{P|<4)U8T;YN5;|*Kdt{Zv*cU@U= z-5_FV8~ZiaJ(qgz{EZ$bLiW3^7lu)S6T{ebg#|u}5(sHdwj(dOMZ^fIQ2hNub2joB@T&7~Q|&BF}mOuS^ZRw`8VgMkGrOAbe#J#`Z zi?=iYRcYN8M1`~%PdvX30aa7Z1zF_C(J2n+R6HR!_d#IX+@jJFJWkumLm0(2cp-L! z*a9uKK@c558K6o#yKhranjd5ftCO9_2ri6U2-5ncL8lAYb1*=_=ExGAr*pIm*#QL^ zEkH&UWIJI(2H6I%)^2~dvha!64)4V%iXAA>yrGLH;UV|@7Z`4aUYuc~iM==z&t@Yk z`#^_l=crUNR8TuR(99$Mu?Gq6J@a_=v%AZ`ezbh&#_G+_ixKh&r=8vC^d?E!69-}s zd}D*NaXq}uHV9d0u+8H2XgCyF_ zI3@FxlG(k9&U9uZfb0UjLcYm?Z zIiC@@Q;O3T7E|yP=}Fo-F0taMbQkD)s=)fh1zGO? zQz$IOAgl$SL0XlRZ%pZ0VQTy$*yOls)YderJ+O^F3Qt;lAP&OY@y2#pn)IO1bbNte zX5T>amR2&;f!Z!iRCmj=Gd!7wEv(juigid}6_IvlhIwJ)w?tX3Ak?psa!Qq3sl)bE zjZtn0aW&)c zV`hw~IY+LU>?!uNnzJ8so?u5Y$70X0XW4gFj2T+As*#U*J?hv{6!lAb7hS)_YIPBo(q81YsOE(~y8jfkMsMpN3a2ke(H-2pHHj)0;&fMS-P16UTzVQl-3(mNOm%@j5%3H>eXv(aAC?L$XzPQWh;PZxL=)lWT}_p0H=Fsg^=>--^v!8n?Q_Ai?J z9J*9n)z`E|Z2>$q4$k?NA&#a7Ga(}=nYCj4T3^tBRjxe-^Nru$>3s0}&Ih;e|L*$w z^|j6??{}{JVPoyOv=c;5JCQjqL{oJCix1r}WFgZAH3Tamm10fEt5OymjVytDZkqKv~M{|mi?Xe=8WTs3M|be5QcT}JT&wBwjiip&OS)-zKDGF zw!|7CvUwR8^wBv6fzVmd*7QYvLGR=9tinehz(-g3kb^UPGGsPxf4aW(8}Ss>;%VxR zQuhqHiYW+txvRhNAj2lxf5i~T$&^;G4FPcix+7@44SU&!NjvXWdwKsYczomb^1Zuv z)~~*|zWQF~@yG3}p!SuKkn*Ug?68#y=t%#t??^juS5qQ8xXKY3;XUFf$_*98lNhAh z1^DTLK|Z4Us>zHs(pYFYG@F??ORN=Bc`}E{wh`&Crw()ASr>9%MLK!XYZvj#o)2%` zynlB|=G1bec`-!zX3n9^%aYubo_ZW8&TwIqjfz>6)td24G`kHh;^-uy<3q)Ro5e#R zdykqYO89^@fux}2i2nDo6fs1Vv^#~!i+J!ow1iSdGc8bqXIMIZ53C1vUS$Z`%onTt z-4+*7gU4yor9=SW#w_m=Rd*T5hM;px8g-6rwpv)Q%u}xzutxc^c~Ro_bBCOm`GoRj z#aJ{J43#e?P(OzJu^=liDAWz0?zD*qv;3Kwe6`+u=iWbV-uwJdotq!s|9H7`>yz$h zSH;UP*@mQefpmjeRi@}0*qH(hZu=1#o}z9i)Ly{@YFRYeLs?>xX8zy+PSwj>Z&oMk z;tf-lx9PS%RU=q7FM=nBro#5S7;HiWFm!~bxqz@mC=y7=6-ed)NTvdbIHX6ycmB4a zqO)`B>iU%r0n#8Li_DY#;ZAJYpSkJ%ZRQ2Hfw7HN*98}xf% z=TwZ*)A@Y$a(1LHPP8e6rzU6M+KtaY+*rH%#u>z0f$^j!B2{QrbU}yABe_n%DbUDq zkL6?w@a6iMY`2eKLg&hA_m6)8cXe)k43v7H{q>)H#J&-xB911MRvAE9I2mfCJ4Vv0 z_~tLr9;Rs0f}&elz46Ev^LVVbNOqfa2I9g&yhUOdJ7_MV#Zg4I87-&NJW@&@h!!F; zXMo7Me_mey`}-9~E!)o%RjIN@?8o<1F|t*MWEIG^b89Qjyo{`z-c~Xo=n($;cbW9` z2h{Z|NkQx+UPH&w67oB7)kDAk_1KE6&Q=adXC_2J(TW5SPj;|SXb4IwX{VQ3Q9L_P zIZ?6c@11GKHL7VRGCJ-R9fAOtZtD^Y<20%XqC8X`>R*A{pQ=t(F(qEYGgVV$#Q{NDq{Rv>PhB`Tw?VB| zb?~g-Kj=OXj&q-k)6X1?H!!mY5Ymw>a$?qEF>mqMZMlqmJSxRr%gc2)60vBBTrWlC z*l+pFQEc03Rg@1|Rh3(F(%WNF$nuV}&06!h$KPYhl@*NYE{8S;ZC=)PUAa(<4WVin zo?2LQ{wiht8zC3v(vF9S^*rJkoH70Ok;rFHA|v8IM5B}BG9s?X)jZ;&T)5;k&W(YA z({OF&&GmOaO8Z7>TYLNR<@L9&bX3oTX=1$(`$`WQ%=#bXjvyjk21c1MR?usg2ANUu z#WV>`s<*05t+KSQJ1Xl?@RNr@DPjJGnY{>MoLg+qx#|o-!5Sd{gzNK$OBZ$twztba zV?zg*6#b{M{rLJXqZ?lz{d{nA@a^%Pox@+h`2Fs^qc6Wd-2QxY?T!_h2~0n;p3*wi z<3DhmEypU$g09-&qFj^UB|1y-{>Gr!lc`tqEX;q){oQfCrCU=;wchzaZTl+mq-vXg*Z#vGZ5s@ zRw0~fiSgN(2tP-j_H230DD7tBW$1v`i`1tonw?@roXUQrW;ghs0Ho`muJgm2-;M6w z)X&2GpU~A$l`bW7g4%QR@eHu%$($kicc`7mmKQMp4K>=+ShD4f09DruHK(xDCz^WP z3Nz?|LG6);imi2Sqn~u>hjQW3meW*zXs!I;Qxt)VO!9`XZkQPBW=7{sYC7~3#W~r; zv-Mvq2zs6RCVLlR8Eu8Z1ryw7)w0j4u&4A-iZ|A1Ocqvreo`3Nh1KZ_?VtP!HwA+@ literal 0 HcmV?d00001 diff --git a/api/data_api.py b/api/data_api.py index e69de29..fd334e1 100644 --- a/api/data_api.py +++ b/api/data_api.py @@ -0,0 +1,76 @@ +from fastapi import APIRouter, HTTPException, Query +from typing import Optional, List, Dict +from function.data_manager import DataManager +from pydantic import BaseModel + +router = APIRouter() +data_manager = DataManager() + +# 数据模型 +class ProcessRequest(BaseModel): + input_file: str + output_path: str + preprocessing: List[Dict] + feature_engineering: List[Dict] + split_ratio: Dict[str, float] + +@router.get("/preprocessing/methods") +async def get_preprocessing_methods(): + """获取数据预处理方法列表""" + result = data_manager.get_preprocessing_methods() + if result['status'] == 'error': + raise HTTPException(status_code=500, detail=result['error']) + return result + +@router.get("/preprocessing/method/{method_name}") +async def get_preprocessing_method_details(method_name: str): + """获取预处理方法详情""" + result = data_manager.get_preprocessing_method_details(method_name) + if result['status'] == 'error': + raise HTTPException(status_code=500, detail=result['error']) + return result + +@router.get("/feature/methods") +async def get_feature_methods(): + """获取特征工程方法列表""" + result = data_manager.get_feature_engineering_methods() + if result['status'] == 'error': + raise HTTPException(status_code=500, detail=result['error']) + return result + +@router.get("/feature/method/{method_name}") +async def get_feature_method_details(method_name: str): + """获取特征工程方法详情""" + result = data_manager.get_feature_engineering_method_details(method_name) + if result['status'] == 'error': + raise HTTPException(status_code=500, detail=result['error']) + return result + +@router.post("/process") +async def process_dataset(request: ProcessRequest): + """处理数据集""" + result = data_manager.process_dataset( + input_path=request.input_file, + output_dir=request.output_path, + process_methods=request.preprocessing, + feature_methods=request.feature_engineering, + split_params=request.split_ratio + ) + if result['status'] == 'error': + raise HTTPException(status_code=500, detail=result['message']) + return result + +@router.get("/datasets") +async def get_datasets(): + """获取可用数据集列表""" + try: + datasets = data_manager.get_dataset() + return { + "status": "success", + "datasets": datasets + } + except Exception as e: + raise HTTPException( + status_code=500, + detail=f"获取数据集列表失败: {str(e)}" + ) diff --git a/api/model_api.py b/api/model_api.py index e69de29..1fac347 100644 --- a/api/model_api.py +++ b/api/model_api.py @@ -0,0 +1,114 @@ +from fastapi import APIRouter, HTTPException, Query, Path +from typing import Optional, List, Dict +from function.model_manager import ModelManager +from pydantic import BaseModel + +router = APIRouter() +model_manager = ModelManager() + +# 数据模型 +class TrainRequest(BaseModel): + model: str + dataset: Dict[str, str] + parameters: Dict + metrics: List[str] + +class PredictRequest(BaseModel): + run_id: str + data: str + output_path: str + batch_size: int = 32 + device: str = "cuda" + return_proba: bool = True + metrics: Optional[List[str]] = None + +@router.get("/available") +async def get_available_models(): + """获取可用模型列表""" + result = model_manager.get_models() + if result['status'] == 'error': + raise HTTPException(status_code=500, detail=result['error']) + return result + +@router.get("/available/{model_name}") +async def get_model_details(model_name: str): + """获取模型详情""" + result = model_manager.get_model_details(model_name) + if result['status'] == 'error': + raise HTTPException(status_code=500, detail=result['error']) + return result + +@router.get("/metrics") +async def get_metrics(): + """获取评价指标列表""" + result = model_manager.get_metrics() + if result['status'] == 'error': + raise HTTPException(status_code=500, detail=result['error']) + return result + +@router.post("/train") +async def train_model(request: TrainRequest): + """模型训练""" + result = model_manager.train_model( + train_data=request.dataset['train'], + val_data=request.dataset.get('val'), + model_config={ + 'model_name': request.model, + 'parameters': request.parameters, + 'metrics': request.metrics + } + ) + if result['status'] == 'error': + raise HTTPException(status_code=500, detail=result['message']) + return result + +@router.get("/experiments") +async def get_experiments( + page: int = Query(1, ge=1), + page_size: int = Query(10, ge=1, le=100) +): + """获取MLFlow中保存的实验""" + result = model_manager.get_experiments(page=page, page_size=page_size) + if result['status'] == 'error': + raise HTTPException(status_code=500, detail=result['message']) + return result + +@router.get("/experiment/{experiment_name}") +async def get_finished_models( + experiment_name: str, + page: int = Query(1, ge=1), + page_size: int = Query(10, ge=1, le=100) +): + """获取已训练完成的模型列表""" + result = model_manager.get_finished_models( + experiment_name=experiment_name, + page=page, + page_size=page_size + ) + if result['status'] == 'error': + raise HTTPException(status_code=500, detail=result['message']) + return result + +@router.delete("/{run_id}") +async def delete_model(run_id: str = Path(..., description="MLflow运行ID")): + """删除指定的训练好的模型""" + result = model_manager.delete_model(run_id) + if result['status'] == 'error': + raise HTTPException(status_code=500, detail=result['message']) + return result + +@router.post("/predict") +async def predict(request: PredictRequest): + """模型预测""" + result = model_manager.predict( + run_id=request.run_id, + data_path=request.data, + output_path=request.output_path, + batch_size=request.batch_size, + device=request.device, + return_proba=request.return_proba, + metrics=request.metrics + ) + if result['status'] == 'error': + raise HTTPException(status_code=500, detail=result['message']) + return result diff --git a/api/system_api.py b/api/system_api.py index 340b0c5..4458f9a 100644 --- a/api/system_api.py +++ b/api/system_api.py @@ -6,7 +6,7 @@ from function.system_monitor import SystemMonitor router = APIRouter() system_monitor = SystemMonitor() -@router.get("/system/resources") +@router.get("/resources") async def get_system_resources(): """获取系统资源使用情况""" result = system_monitor.get_system_resources() @@ -14,7 +14,7 @@ async def get_system_resources(): raise HTTPException(status_code=500, detail=result['message']) return result -@router.get("/system/history") +@router.get("/history") async def get_training_history( page: int = Query(1, ge=1), page_size: int = Query(10, ge=1, le=100), @@ -36,7 +36,7 @@ async def get_training_history( raise HTTPException(status_code=500, detail=result['message']) return result -@router.get("/system/logs") +@router.get("/logs") async def get_system_logs( level: Optional[str] = None, start_time: Optional[str] = None, diff --git a/doc/安装文档.md b/doc/安装文档.md new file mode 100644 index 0000000..6d66e13 --- /dev/null +++ b/doc/安装文档.md @@ -0,0 +1,63 @@ +# 运行说明 + +## 1. 启动mlflow + - mlflow server --host 10.0.0.202 --port 5000 +## 2. 运行 main.py + + +## 3. 测试接口 + +GET http://10.0.0.202:8992/health + +### 3.1 获取数据预处理方法列表 +GET http://10.0.0.202:8992/data/preprocessing/methods + +### 3.2 获取预处理方法详情 + +GET http://10.0.0.202:8992/data/preprocessing/method/{method_name} + +### 3.3 获取特征工程方法列表 +GET http://10.0.0.202:8992/data/feature/methods + +### 3.4 获取特征工程方法详情 + +GET http://10.0.0.202:8992/data/feature/method/{method_name} + +### 3.5 处理数据集 +POST http://10.0.0.202:8992/data/process + +### 3.6 获取可用数据集列表 +GET http://10.0.0.202:8992/data/datasets + +### 3.7 获取可用模型列表 +GET http://10.0.0.202:8992/model/available + +### 3.8 获取模型详情 +GET http://10.0.0.202:8992/model/available/{model_name} + +### 3.9 获取评价指标列表 +GET http://10.0.0.202:8992/model/metrics + +### 3.10 模型训练 +POST + +### 3.11 获取实验列表 +GET http://10.0.0.202:8992/model/experiments + +### 3.12 获取具体实验内容 +GET http://10.0.0.202://8992/model/experiment/{experiment_name} + +### 3.13 删除模型指定实验模型 +DELETE + +### 3.14 模型预测 +POST + +### 3.15 查看系统资源使用情况 +GET http://10.0.0.202:8992/system/resources + +### 3.16 查看训练历史 +GET http://10.0.0.202:8992/system/history + +### 3.17 查看系统训练日志 +GET http://10.0.0.202:8992/system/logs \ No newline at end of file diff --git a/doc/接口文档code.md b/doc/接口文档code.md index 6124707..362a38e 100644 --- a/doc/接口文档code.md +++ b/doc/接口文档code.md @@ -3,7 +3,7 @@ ## 1. 数据处理模块 ### 1.1 获取数据预处理方法列表 ```http -GET /api/data/preprocessing/methods +GET /data/preprocessing/methods Response: { @@ -33,7 +33,7 @@ Response: ### 1.2 获取预处理方法详情 ```http -GET /api/data/preprocessing/methods/{method_name} +GET /data/preprocessing/method/{method_name} Response: { @@ -67,7 +67,7 @@ Response: ### 1.3 获取特征工程方法列表 ```http -GET /api/data/feature/methods +GET /data/feature/methods Response: { @@ -97,7 +97,7 @@ Response: ### 1.4 获取特征工程方法详情 ```http -GET /api/data/feature/methods/{method_name} +GET /data/feature/method/{method_name} Response: { @@ -128,7 +128,7 @@ Response: ### 1.5 处理数据集 ```http -POST /api/data/process +POST /data/process Content-Type: application/json Request: @@ -182,7 +182,7 @@ Response: ### 1.6 查看可用数据集 ```http -GET /api/data/datasets +GET /data/datasets Response: { @@ -275,7 +275,7 @@ Response: ## 2. 模型接口 ### 2.1 获取可用模型列表 ```http -GET /api/models/available +GET /model/available Response: { @@ -299,7 +299,7 @@ Response: ### 2.2 获取模型详情 ```http -GET /api/models/{model_name} +GET /model/available/{model_name} Response: { @@ -329,7 +329,7 @@ Response: ### 2.3 获取评价指标列表 ```http -GET /api/metrics +GET /model/metrics Response: { @@ -369,7 +369,7 @@ Response: ### 2.4 模型训练 ```http -POST /api/train +POST /model/train Content-Type: application/json Request: @@ -396,7 +396,7 @@ Response: ``` ### 2.5 获取MLFlow中保存的实验 ```http -GET /api/experiments +GET /model/experiments Response: { @@ -424,7 +424,7 @@ Response: ### 2.6 获取已经训练好的模型列表 ```http -GET /api/models/finished/{experiment_name} +GET /model/experiment/{experiment_name} Response: { @@ -458,7 +458,7 @@ Response: ``` ### 2.7 删除指定的训练好的模型 ```http -DELETE /api/models/{run_id} +DELETE /model/{run_id} Response: { @@ -478,7 +478,7 @@ Response: ``` ### 2.8 模型预测 ```http -POST /api/model/predict +POST /model/predict Content-Type: application/json Request: @@ -526,10 +526,11 @@ Error Response: ### 2.9 模型优化 -- 未实现 + ## 3. 系统监控 ### 3.1 获取资源使用情况 ```http -GET /api/system/resources +GET /system/resources Response: { @@ -623,7 +624,7 @@ Error Response: ### 3.2 获取训练历史 ```http -GET /api/system/history?page=1&page_size=10&start_time=2025-02-01&end_time=2025-02-19&status=completed&experiment_name=breast_cancer_classification +GET /system/history?page=1&page_size=10&start_time=2025-02-01&end_time=2025-02-19&status=completed&experiment_name=breast_cancer_classification Parameters: - page: 页码 (默认: 1) @@ -709,7 +710,7 @@ Response: ### 3.4 获取系统日志 ```http -GET /api/system/logs?level=error&start_time=2025-02-19T00:00:00&end_time=2025-02-19T23:59:59&module=training&page=1&page_size=20 +GET /system/logs?level=error&start_time=2025-02-19T00:00:00&end_time=2025-02-19T23:59:59&module=training&page=1&page_size=20 Parameters: - level: 日志级别过滤 (可选: debug, info, warning, error, critical) @@ -778,9 +779,8 @@ MLPlatform/ │ ├── model_api.py # 模型相关接口 │ └── system_api.py # 系统监控相关接口 ├── function/ # 功能实现层 -│ ├── data_processor.py # 数据处理类 +│ ├── data_manager.py # 数据处理类 │ ├── model_manager.py # 模型管理类 -│ ├── model_trainer.py # 模型训练类 │ ├── system_monitor.py # 系统监控类 │ └── utils/ # 工具函数 ├── config/ # 配置文件 diff --git a/function/__pycache__/data_manager.cpython-39.pyc b/function/__pycache__/data_manager.cpython-39.pyc index c8343e265a29e915ebb45d0a739586a25873f4c6..6bbf28896b148ddc21fbf4544ad5802a68054623 100644 GIT binary patch delta 3087 zcmZ`*U2q#$72dnM`nO%zv1P?BtrI)3H+B>|c76bprmgF?Nl^<VoznV1U_NX-Wn^9Tq1gB%n!*N= z!u8wQ*E-tA7j8RAMyk!t7u3!NkVO&)G}-m#b!M23&Bj5ed0qay*7j`&KxPLKK7mY+ z?HGBx(DE<@BV(q!z+Js7#jIQxdjk7;d)tR2Pom__8P2ptSrQsdN6k8ZDY}lA+9KW* z>eBp`wh`j;-?ojeuf+?rf0hBj7jMR-;ET&QgT)*wG6o!X+QTHI~0vX#% zi5Q8fu)F!+!W~{QW0?Q3Gt9f&cTC=k^iia%hi%JrRXUT)Ypo(k?Bm!ufv{$^ffQ^_ z!qj2DNgNxpW}N(tlgR7NG_z47(-fopa{EN{7>-}H#XJ(JHXlbNMHm4{D*V(?r24PO zHsYN^Ws>)GJaYSEATB}plU8;nXWDFRpfepG7iZ&+%~AZ2OV%L3J(Ay=My@Xv~Kv?5G(v2FuZNou9 z*Ouuo--14K!}^E#uQxPx3xi*}1^4M1_rDts_B{`jm=C{CQA7k{spd`$qPo57@xbD94rpuA0q2 z4cUS!+&))5V$9iW7TbFf#9CII_twJyb>61!Y+maDKctn*8f_)eHo?iBfNp|1B5F8H{7Lse4uAk`0+Zb{VOhu6k`K8VlTq` zJziS&c#2;b=)cM1>YoNuM8`y(X#u$Y?3H(~9)I_Z*REfD{@VE~*QzhFucAre_Yz9q zK&ZPBW8Z|frY;8s`$W>O4NUc7^lehU1pP5M_=56XBv%o>hp;L#*t}IZ*blMyO@tpI z)IDK8#^zN7acTu%F1wr4hLcDcy2CF=!#p_jOj!o?jR$2FUi={2eE>yLktm_^qFf?E zR;XH+J=9y1;cDQExk^lrxvDvn&AR@F%v8qAICF8&kTE>CyAarVLsRf>6)KImK4#>z zWj$#?AsU3qF#Aknt;fxYWY*Bl1b&zjNO&B)wEe)BFp)|Nhw7rgyDq0=pU2jNh#n2D*-$lqf`0mF5*9nQ+yx z&kagM{2m8{kTw*9i-br=$Bu@J1On-g6}(%bxLCFW{HxuS%DX^(-$HPwJn*iA*A4@u zrJ^+1R3Zyf1MWF~sMDfmQY@bQk(pM>K8l zL&Je`bTn%v^{l-I1gH{8E0^EHGHAGwVJKbMvRxl`(F|i-QKRBj0PYl4#uQf)S50TF zsoG7$uiJ^dWoKqt38@~uK-Nl)781_I)hRuBB;JAr5Uy|h>h=!uEB?;*zC&X$Al@j_ z!k{82Y&RhCLljG~s9yqyiIfjv$}byv5_2a4T@vxi4YHp!%YLOv_N%R=g+!Gmh3_BM zl=W;gKQ=s29!47)LO%i?2vM{drVslJ!piq+4x0r4*OP*^I1CRG!xGHCgYXlCpCbGW z0e6J`9%1E#uI$D}=xE3NwRiK4BfGn{`h5&_+OD`a58oKw7Bj~^CS26!ScT7y43mA; Z*GFbZ4~CO{1mP_N9Cp(j{QjML{s-Uo<%$3R delta 2671 zcmaJ@OK=ob6z$hNnNKDO!OD+BAdnB~WD)`ifs*k58gNJ%5NL^0o#~#WlbPz+X3 z#DJ*ahcebf1wT|GibN5CE~{|i)^cOHlCpYXl}k6STv)=wd*8q$BABV1zQ1?hy|?du zufI*7PL-&|#f1X=Mb_UQv|~3*jz~}316CIlq6vzq5k=DK*$HWiG_b#r%jTj1{_q}@Flcv31>LbjUFV@XTZXd`>Mu#T)|R|>n!+QA#;um-6nfGE3G+~C%P ze<0ybl&wSPVt*9PRBu3X8^S~Dn%%3#aoCd;Evhk;B3pNMTQsFQj!u{kj6Go5i_=*kN@MoPCGHI=6OK}(-Oh_bt7%L@(x zsj%hc8!Mj!%JZjmXUNj%QM5SBb))4G|MSQmW0%W2A=_WdyO-q31Jd6`09jQ0hN;2! z3Tv&n*s}-C_X2o9+o1{Eo%&I>8$byeww2(8;-#UR&`?ZkN|5TR7K__so?lI;byK5< zS!rc&|5jYO5v_c~wq<%gEt5*iWxQ&%2bKE}7Nmx@AznpOg}n~~uY|3PgB3}u&Jfkn zLevseVc%8um2SuN3*IiPt#ZSoC{z$S++VBq6aNvUkFxga1B-isoP(mrtz;%;+O(fd zSND=u_FHw`)DY;r&`#A+cT?EiD^3_r%%Q4jW7c#6z4W7vKW>?E)tRpX-Hf^|2wMT< z!d!~;f#f0pl^04=E3VtNMZI7`SDg&iZ90aU8edAaN9Ycq7p8>luzRVdTdK}`$;E=J z=y8M-?9XNIY~$p!z{`?X0AUa6yc0Yrshe|>C$_bF66Ea>Ll(IQFQA)oOVhR7erwqG z%X@?4=z!1(wyd^)9rq^ES3sr&%-Dco+A;cC&f*k2?k_~@O@KH7;w^Tgc1JV5lsS=} zLDgA=1#byiG}m+HOIIA?<`5M(fBiA$nFrpVVs}=Q)NzOB9%FklXB${~XzE2UiOLl; zg{G4T{7tqYl?Cv8aVtGer;sc(Y)H{j&A9G~W*TzVR7%xxioHQ!Lck87CjmTvu6n%4 zyhM*|h|aBGD;GbuBk0SZ=N+H~v_#C&Vc5^IMRk!C`A&d?hKKwHTAo7S$KMD$R`+!& zZ?fE+qo%{HsqZA^C{T)^2{bDegM=JCN{=(Gp>GOLH9orZRiyr>bMsD>5|{$RjE$+u zm4{K##=PrKZk( zI6cVVYNVrcAR!V3wKRtrx_uwv0z28#6UON1dk8PEpIf3W@8kh+5KIl@(hIe-V@ zD=ZqR@8)rRgyP_Q3cTZJ3$A!X26Oi_IKr++S{|7}Eo4yZmb7joq>c5q)$BTtu6U>y zk@^rJzjKs+47}{SUl-Kzwtu)b?v1t|$1xC+oOQ2Qmq zHH3U5^ebeqBmDay@EQkoxb5I1WcS)H5W;Hfs~WKCj&NF-Bo3Jov%;`6OD4svxKrq# zn3Nnc?3*dbvRiBGr+Ov>PB0so3C#+cs7addob-0}WY8&`31@?_D0EoR1RwlPOGGeB z*9i7RyFmEwC}5V%6lFuRgb1TMM=P=fK1d3p6aMBlyszkj4+;nfX8003Jkhef05&FE z`CDiNMLxWn;F7Zpli!z$Z!l>M@T*S=s&*`G*~STKa#Q|MgDs4=1ncRHkZ;)W&ZeI2 zAW}lOZ*ImODpMHCgu@5lAV)+&uq(h`Abzn(43bhYD3!poj5LWQg#FzqOI5UiHLiQKMe*^qT+<~qPsmBu0ZUh$rm;Gl0 Jc6a^ee*pE%aE|~0 diff --git a/function/data_manager.py b/function/data_manager.py index 4fd29b9..c858d08 100644 --- a/function/data_manager.py +++ b/function/data_manager.py @@ -597,6 +597,14 @@ class DataManager: "error": str(e) } + + def _clean_json_line(self,line): + # 替换掉不符合JSON标准的特殊浮点数值 + line = line.replace('NaN', 'null') + line = line.replace('Infinity', '1e308') # 或者选择一个合适的替代值 + line = line.replace('-Infinity', '-1e308') # 同上 + return line + def get_dataset(self): back = list() @@ -611,8 +619,15 @@ class DataManager: json_files = list(folder_path.glob('*.json')) for json_file in json_files: + + + # json是不能处理NaN这类的数值,需要单独处理他们 with open(json_file.as_posix(), 'r', encoding='utf-8') as f: - json_data = json.load(f) + cleaned_lines = [self._clean_json_line(line) for line in f] + json_data = json.loads(''.join(cleaned_lines)) + # json_data = json.load(f ,allow_nan=True) back.append(json_data) + + # print("可用数据集", back) return back \ No newline at end of file diff --git a/main.py b/main.py index 753e676..21c8136 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,15 @@ -from fastapi import FastAPI, HTTPException, Depends, BackgroundTasks +from fastapi import FastAPI, HTTPException, Depends, BackgroundTasks, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.security import OAuth2PasswordBearer +from fastapi.responses import JSONResponse +from fastapi.exceptions import RequestValidationError from typing import Optional, Dict, List import uvicorn from pathlib import Path import logging import yaml from datetime import datetime +import time # 导入API路由 from api.data_api import router as data_router @@ -17,30 +20,11 @@ from api.system_api import router as system_router app = FastAPI( title="机器学习平台API", description="提供数据处理、模型训练和系统监控功能的API服务", - version="1.0.0" + version="1.0.0", + docs_url="/docs", + redoc_url="/redoc" ) -# 配置CORS -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -# 设置日志 -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[ - logging.FileHandler(f'.log/server_{datetime.now():%Y%m%d_%H%M%S}.log'), - logging.StreamHandler() - ] -) - -logger = logging.getLogger(__name__) - # 加载配置 def load_config(): try: @@ -50,23 +34,137 @@ def load_config(): logger.error(f"Error loading config: {str(e)}") return {} +# 初始化配置 config = load_config() +# 设置日志 +log_dir = Path('.log') +log_dir.mkdir(exist_ok=True) + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler(log_dir / f'server_{datetime.now():%Y%m%d_%H%M%S}.log'), + logging.StreamHandler() + ] +) + +logger = logging.getLogger(__name__) + +# 配置CORS +app.add_middleware( + CORSMiddleware, + allow_origins=config.get('cors', {}).get('origins', ["*"]), + allow_credentials=True, + allow_methods=config.get('cors', {}).get('methods', ["*"]), + allow_headers=config.get('cors', {}).get('headers', ["*"]), + max_age=config.get('cors', {}).get('max_age', 600), +) + +# 请求计时中间件 +@app.middleware("http") +async def add_process_time_header(request: Request, call_next): + start_time = time.time() + response = await call_next(request) + process_time = time.time() - start_time + response.headers["X-Process-Time"] = str(process_time) + return response + +# 全局错误处理 +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + """处理请求参数验证错误""" + return JSONResponse( + status_code=422, + content={ + "status": "error", + "message": "请求参数验证失败", + "details": exc.errors() + } + ) + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + """处理HTTP异常""" + return JSONResponse( + status_code=exc.status_code, + content={ + "status": "error", + "message": exc.detail + } + ) + +@app.exception_handler(Exception) +async def general_exception_handler(request: Request, exc: Exception): + """处理其他异常""" + logger.error(f"Unhandled exception: {str(exc)}", exc_info=True) + return JSONResponse( + status_code=500, + content={ + "status": "error", + "message": "服务器内部错误", + "details": str(exc) if config.get('debug', False) else None + } + ) + # 注册路由 -app.include_router(data_router, prefix="/api", tags=["数据处理"]) -app.include_router(model_router, prefix="/api", tags=["模型管理"]) -app.include_router(system_router, prefix="/api", tags=["系统监控"]) +app.include_router( + data_router, + prefix="/data", + tags=["数据处理"], + responses={404: {"description": "Not found"}}, +) + +app.include_router( + model_router, + prefix="/model", + tags=["模型管理"], + responses={404: {"description": "Not found"}}, +) + +app.include_router( + system_router, + prefix="/system", + tags=["系统监控"], + responses={404: {"description": "Not found"}}, +) # 健康检查 @app.get("/health") async def health_check(): - return {"status": "healthy", "timestamp": datetime.now().isoformat()} + """系统健康检查""" + return { + "status": "healthy", + "timestamp": datetime.now().isoformat(), + "version": app.version, + "environment": config.get('environment', 'production') + } + +# 启动事件 +@app.on_event("startup") +async def startup_event(): + """服务启动时的初始化操作""" + logger.info("Server starting up...") + # 创建必要的目录 + Path("dataset/dataset_raw").mkdir(parents=True, exist_ok=True) + Path("dataset/dataset_processed").mkdir(parents=True, exist_ok=True) + Path(".log").mkdir(exist_ok=True) + logger.info("Server started successfully") + +# 关闭事件 +@app.on_event("shutdown") +async def shutdown_event(): + """服务关闭时的清理操作""" + logger.info("Server shutting down...") if __name__ == "__main__": uvicorn.run( "main:app", host=config.get('host', '0.0.0.0'), - port=config.get('port', 8000), - reload=True, - workers=config.get('workers', 4) + port=config.get('port', 8992), + reload=config.get('debug', True), + workers=config.get('workers', 4), + log_level=config.get('log_level', 'info'), + access_log=True ) \ No newline at end of file