Skip to content

Commit 54f8b63

Browse files
author
huangzhipeng
committed
init
0 parents  commit 54f8b63

37 files changed

+3196
-0
lines changed

Diff for: ‎.gitignore

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Created by .ignore support plugin (hsz.mobi)
2+
### Go template
3+
# Binaries for programs and plugins
4+
*.exe
5+
*.exe~
6+
*.dll
7+
*.so
8+
*.dylib
9+
10+
# Test binary, build with `go test -c`
11+
*.test
12+
13+
# Output of the go coverage tool, specifically when used with LiteIDE
14+
*.out
15+
### Example user template template
16+
### Example user template
17+
18+
# IntelliJ project files
19+
.idea
20+
21+
# VsCode project files
22+
.vscode/*
23+
24+
*.log
25+
*.zip
26+
27+
log/*
28+
*.swp
29+
*.swo
30+
31+
venv
32+
pandas
33+
site

Diff for: ‎README.md

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# 依赖
2+
python3.9
3+
# 安装/升级
4+
待正式对外发布
5+
# 配置
6+
1. 鉴权通过OAuth2.0, 所以这里要先申请用户的`appid`、`api_key`、`secret_key`
7+
2. 配置方式一：直接通过制定参数的方式实例化api_client， 参考快速开始部分
8+
3. 配置方式二: yaml配置文件， 配置的默认路径为`~./.config/deepwisdom/dwconfig.yaml`, 配置内容如下
9+
```yaml
10+
api_key: "xx"
11+
secret_key: "xxx"
12+
appid: 3
13+
domain: "xxx"
14+
```
15+
16+
# 快速开始
17+
以表格二分类为例
18+
```python
19+
import deepwisdom as dw
20+
21+
if __name__ == "__main__":
22+
23+
api_client = dw.Client(appid="your appid", api_key="your api key", secret_key="your secret key")
24+
dw.set_client(client=api_client)
25+
26+
train_dataset = dw.Dataset.create(file_path="path to train", stage="train")
27+
automl = dw.AutoML.create(dataset=train_dataset, name="sdk-demo", label_col="k", time_col="g")
28+
automl.train()
29+
30+
test_dataset = dw.Dataset.create(file_path="path to test", stage="test")
31+
eval_res = automl.evaluate(test_dataset)
32+
33+
service = dw.Deployment.create(project=automl, name="sdk-demo")
34+
ret = service.call_service()
35+
36+
```
37+
# 特性
38+
1. 数据集管理。 包括数据集的增删改查、数据集模糊搜索等
39+
2. 项目管理。 项目的增删改查、训练管理、离线预测、高级设置更新、方案/部署模型列表等
40+
3. 实验管理。 实验详情数据查询，包括耗时、性能和效果指标等
41+
4. 最佳方案。 实验的方案列表及对应的部署模型信息等
42+
5. 离线预测。 获取离线预测列表，进行离线预测等
43+
6. 推理部署。推理服务创建，获取列表，修改常驻状态，修改推理服务名称，调用服务等
44+
45+
# 详细文档
46+
1. API Reference。
47+
2. tutorials
48+

Diff for: ‎deepwisdom/__init__.py

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# flake8: noqa
2+
3+
from ._version import __version__
4+
from .client import Client, set_client
5+
6+
from .errors import AppPlatformError
7+
from .models import (
8+
Dataset,
9+
OfflinePrediction,
10+
Deployment,
11+
Project,
12+
AdvanceSetting,
13+
TrainSetting,
14+
SearchSpace,
15+
CreateDeploymentRequest
16+
)
17+
18+
from .models import Project as AutoML

Diff for: ‎deepwisdom/_compat.py

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
"""
2+
This module contains compatibility fixes to allow usage of both 1.x and 2.x Trafaret versions
3+
"""
4+
try:
5+
from trafaret import ToInt as Int
6+
except ImportError:
7+
from trafaret import Int # noqa
8+
9+
10+
try:
11+
from trafaret import AnyString as String
12+
except ImportError:
13+
from trafaret import String # noqa
14+
15+
try:
16+
from trafaret import Any as Any
17+
except ImportError:
18+
from trafaret import Any

Diff for: ‎deepwisdom/_version.py

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
__version__ = "0.2.4"

Diff for: ‎deepwisdom/client.py

+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
import os
2+
3+
import yaml
4+
from ._version import __version__
5+
6+
from .rest import DeepWisdomClientConfig, RESTClientObject
7+
from deepwisdom.enums import API_DOMAIN
8+
9+
__all__ = ("Client", "get_client", "set_client")
10+
11+
_global_client = None
12+
13+
14+
def Client(
15+
api_key=None,
16+
secret_key=None,
17+
appid=None,
18+
domain=None,
19+
admin_domain=API_DOMAIN.ADMIN
20+
):
21+
"""
22+
api请求客户端
23+
Args:
24+
admin_domain (str):
25+
api_key:
26+
secret_key:
27+
appid:
28+
domain:
29+
30+
Returns:
31+
32+
"""
33+
global _global_client
34+
35+
if api_key and appid and secret_key:
36+
dwconfig = DeepWisdomClientConfig(
37+
appid=appid,
38+
api_key=api_key,
39+
secret_key=secret_key,
40+
domain=domain,
41+
admin_domain=admin_domain
42+
)
43+
else:
44+
config_path = _get_default_config_file()
45+
if config_path is None:
46+
raise ValueError("Config No Found.")
47+
48+
dwconfig = _config_from_file(config_path)
49+
50+
if dwconfig.domain is None:
51+
dwconfig.domain = API_DOMAIN.API
52+
53+
_global_client = RESTClientObject.from_config(dwconfig)
54+
55+
return _global_client
56+
57+
58+
def _get_client_version():
59+
return __version__
60+
61+
62+
def get_client():
63+
return _global_client or Client()
64+
65+
66+
class staticproperty(property):
67+
def __get__(self, instance, owner):
68+
return self.fget()
69+
70+
71+
def set_client(client):
72+
"""
73+
Set the global HTTP client for sdk.
74+
Returns previous client.
75+
"""
76+
global _global_client
77+
previous = _global_client
78+
_global_client = client
79+
return previous
80+
81+
82+
def _get_config_dir():
83+
return os.path.expanduser(os.path.join("~", ".config", "deepwisdom"))
84+
85+
86+
def _get_default_config_file():
87+
first_choice_config_path = os.path.join(_get_config_dir(), "dwconfig.yaml")
88+
if _file_exists(first_choice_config_path):
89+
return first_choice_config_path
90+
else:
91+
return None
92+
93+
94+
_file_exists = os.path.isfile
95+
96+
97+
def _config_from_file(config_path):
98+
"""
99+
Create and return a DeepWisdomClientConfig from a config path. The file must be
100+
a yaml formatted file
101+
102+
Parameters
103+
----------
104+
config_path : str
105+
Path to the configuration file
106+
107+
Returns
108+
-------
109+
config : DeepWisdomClientConfig
110+
"""
111+
with open(config_path, "rb") as f:
112+
data = yaml.load(f, Loader=yaml.SafeLoader)
113+
return DeepWisdomClientConfig.from_data(data)
114+

Diff for: ‎deepwisdom/enums.py

+107
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import json
2+
3+
4+
def enum(*vals, **enums):
5+
"""
6+
Enum without third party libs and compatible with py2 and py3 versions.
7+
"""
8+
enums.update(dict(zip(vals, vals)))
9+
return type("Enum", (), enums)
10+
11+
12+
PROJECT_DEFAULT_ADVANCE_SETTING = json.loads('{"search_space":[{"hp_subspace":"feature_engineering","hp_values":{"KeyTimeBinSecond":{"hp_type":"bool","hp_values":[true,false]},"KeyTimeBinMsecond":{"hp_type":"bool","hp_values":[true,false]},"KeyTimeWeekday":{"hp_type":"bool","hp_values":[true,false]},"KeyTimeHour":{"hp_type":"bool","hp_values":[true,false]},"KeyTimeDay":{"hp_type":"bool","hp_values":[true,false]},"KeyTimeMonth":{"hp_type":"bool","hp_values":[true,false]},"KeyTimeYear":{"hp_type":"bool","hp_values":[true,false]},"KeyNumDiff":{"hp_type":"bool","hp_values":[true,false]},"KeyTimeDiff_BW_Window_1":{"hp_type":"bool","hp_values":[true,false]},"KeyTimeDiff_FW_Window_10":{"hp_type":"bool","hp_values":[true,false]},"McCatRank":{"hp_type":"bool","hp_values":[true,false]},"McMcInnerLen":{"hp_type":"bool","hp_values":[true,false]},"GroupCntDivNunique":{"hp_type":"bool","hp_values":[true,false]},"CatCnt":{"hp_type":"bool","hp_values":[true,false]},"GroupMean":{"hp_type":"bool","hp_values":[true,false]},"GroupMax":{"hp_type":"bool","hp_values":[true,false]},"GroupMin":{"hp_type":"bool","hp_values":[true,false]},"GroupStd":{"hp_type":"bool","hp_values":[true,false]},"GroupMeanMinusSelf":{"hp_type":"bool","hp_values":[true,false]},"GroupMaxMinusSelf":{"hp_type":"bool","hp_values":[true,false]},"GroupMinMinusSelf":{"hp_type":"bool","hp_values":[true,false]},"CatSegCtrOrigin":{"hp_type":"bool","hp_values":[true,false]}}},{"hp_subspace":"modeling","hp_values":{"model":{"hp_type":"choice","hp_values":[{"hp_name":"LIGHTGBM","learning_rate":{"hp_type":"loguniform","hp_values":[0.005,0.2]},"feature_fraction":{"hp_type":"uniform","hp_values":[0.75,1]},"min_data_in_leaf":{"hp_type":"randint","hp_values":[2,30]},"num_leaves":{"hp_type":"randint","hp_values":[16,96]}},{"hp_name":"RANDOMFOREST","n_estimators":{"hp_type":"randint","hp_values":[30,200]},"max_features":{"hp_type":"uniform","hp_values":[0,0.5]}},{"hp_name":"GBTREE","learning_rate":{"hp_type":"loguniform","hp_values":[0.01,1]},"n_estimators":{"hp_type":"randint","hp_values":[30,200]},"subsample":{"hp_type":"uniform","hp_values":[0.5,1]}},{"hp_name":"CATBOOST","depth":{"hp_type":"randint","hp_values":[5,8]},"l2_leaf_reg":{"hp_type":"uniform","hp_values":[1,5]}},{"hp_name":"LOGISTIC_REGRESSION","C":{"hp_type":"loguniform","hp_values":[0.1,10]},"fit_intercept":{"hp_type":"choice","hp_values":[true,false]}},{"hp_name":"RIDGE","alpha":{"hp_type":"uniform","hp_values":[0.1,5]}},{"hp_name":"DECISIONTREE","max_features":{"hp_type":"uniform","hp_values":[0.3,1]},"max_depth":{"hp_type":"randint","hp_values":[3,9]}},{"hp_name":"TABNET","max_epochs":{"hp_type":"randint","hp_values":[3,20]},"gamma":{"hp_type":"uniform","hp_values":[1.1,1.5]}}]}}}],"target_train":{"train_data_ratio":80,"training_program":"指标优先","call_limit":[5,20],"instance_num":2,"call_delay":50,"gpu_mem":0,"memory_limit":20,"program_num":5,"max_trials":30,"trial_concurrency":3,"random_seed":1647}}')
13+
14+
DEFAULT_DOMAIN = ""
15+
DEFAULT_ADMIN_DOMAIN = ""
16+
17+
# default time out values in seconds for waiting response from client
18+
DEFAULT_TIMEOUT = enum(
19+
CONNECT=6.05, # time in seconds for the connection to server to be established
20+
SOCKET=60,
21+
READ=60, # time in seconds after which to conclude the server isn't responding anymore
22+
UPLOAD=600, # time in seconds after which to conclude that project dataset cannot be uploaded
23+
)
24+
25+
API_DOMAIN = enum(
26+
ACCESS_TOKEN="",
27+
API="",
28+
ADMIN=""
29+
)
30+
31+
SVR_STATUS = enum(
32+
CREATE=1,
33+
SCHEDULE=2,
34+
SCHEDULE_SUCC=3,
35+
RUNNING=4,
36+
FAIL=5,
37+
HANDUP=6,
38+
CANCEL=7,
39+
DELETING=8,
40+
DELETED=9,
41+
)
42+
43+
MODAL_TYPE = enum(
44+
CSV=0,
45+
VIDEO=1,
46+
IMAGE=2,
47+
SPEECH=3,
48+
TEXT=4
49+
)
50+
51+
# 0二分类/分类,1多分类/检测/意图识别/匹配,2回归/定位/语音分离/序列,3时序/分隔
52+
TASK_TYPE = enum(
53+
54+
)
55+
56+
API_URL = enum(
57+
AUTH="appmng/token",
58+
DATASET_PREPARE="sdk/datasetprepare",
59+
FILE_UPLOAD="sdk/fileupload",
60+
DATASET_UPLOAD="sdk/datasetupload",
61+
DATASET_QUERY="sdksvr/querydataset", # 获取数据集id和状态
62+
DATASET_INFO="sdk/datasetinfo",
63+
DATASET_DELETE="sdk/datasetdelete", # 数据集删除 /datawarehouse/datasets DELETE
64+
DATASET_LIST="sdk/datasetlist", # 数据集搜索 /datawarehouse/datasets GET
65+
DATASET_MODIFY="sdk/datasetmodify", # 数据集修改 /datawarehouse/dataset Patch
66+
DATASET_SUMMIT="sdk/connectionsummit", # 从数据源导入生成数据集 /connection/submit POST
67+
DATASET_EDA="sdk/dataseteda", # 获取数据集的EDA /dataset/eda GET PATCH
68+
PROJECT_CREATE="sdk/createproj",
69+
PROJECT_DELETE="sdk/deleteproj", # 批量删除项目 /projects DELETE
70+
PROJECT_ADVANCESETTING_UPDATE="sdk/projsetting", # 高级设置修改 /project/advance_settings PATCH
71+
PROJECT_TRAIN="sdk/projtrain",
72+
PROJECT_TERMINATE_TRAIN="sdk/terminatetrain", # 终止训练 /project/terminate/train
73+
PROJECT_INFO="sdk/projinfo",
74+
PROJECT_DATASET_LIST="sdk/projdatasets", # 项目下的数据集列表 /project/dataset
75+
PROJECT_TRAIN_RESULT="/sdk/trainresult",
76+
PROJECT_EFFECT="sdk/projeffect", #迭代效果，实验列表： /project/iterative/effect
77+
PROJECT_SCHEME="sdk/projscheme", #有最终方案的实验/solution： /project/trial/scheme
78+
PROJECT_MODEL_LIST="sdk/modellist", #项目的模型列表：/project/models
79+
PROJECT_MODEL="sdk/model", #模型详情： /project/model
80+
PROJECT_MODEL_SELECT="sdk/modelselect", #需要部署的模型信息： /project/model/select
81+
PROJECT_MODEL_SS="/basic/searchspace/modal/task/ss", #搜索空间详情： 请求admin /basic/searchspace/modal/task/ss
82+
MODEL_DOWNLOAD="sdk/modeldownload", #模型文件下载： /project/model/download
83+
DATASET_PREDICT_UPLOAD="sdk/predictupload", #离线预测数据集上传： /predict/dataset/upload
84+
DATASET_PREDICT_LIST="sdk/predictlist", # 离线预测数据集列表 /predict/datasets
85+
PREDICTION_DETAIL="sdk/projeval/detail",
86+
PREDICTION_PREDICT="sdk/projeval/predict",
87+
PREDICTION_LIST="sdk/projeval/list",
88+
PREDICTION_DELETE="sdk/projeval/delete",
89+
PREDICTION_DATASET_DOWNLOAD="sdk/projeval/dataset_download",
90+
PREDICTION_RESULT_DOWNLOAD="sdk/projeval/result_download",
91+
92+
DEPLOY_CREATE_SERVICE='sdk/deploy/create',
93+
DEPLOY_GET_SERVICE_DETAIL='sdk/deploy/detail',
94+
DEPLOY_LIST_DEPLOYMENTS='sdk/deploy/list',
95+
DEPLOY_RESIDENT_DEPLOYMENT='sdk/deploy/resident', # 推理服务常驻
96+
DEPLOY_RENAME_DEPLOYMENT='sdk/deploy/rename',
97+
DEPLOY_DELETE_DEPLOYMENT='sdk/deploy/delete',
98+
DEPLOY_GET_DEPLOYMENT_LOG='sdk/deploy/logs',
99+
DEPLOY_GET_SERVICE_API='sdk/deploy/get_api',
100+
101+
# DATASET_DOWNLOAD_HOST="http://192.168.50.24:5000/proxy/static/"
102+
103+
)
104+
105+
106+
107+

0 commit comments

Comments
 (0)