Skip to content

Commit 8506419

Browse files
daylin-gaodaylin-gao
authored
Merge pull request #82 from WeBankFinTech/dev-0.3.3
Dev 0.3.3
2 parents e3ff04e + fc5e123 commit 8506419

File tree

57 files changed

+1067
-2142
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+1067
-2142
lines changed

‎appconns/dss-mlflow-appconn/pom.xml‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
<parent>
2222
<artifactId>dss</artifactId>
2323
<groupId>com.webank.wedatasphere.dss</groupId>
24-
<version>1.0.1</version>
24+
<version>1.1.0</version>
2525
</parent>
2626
<!-- <parent>-->
2727
<!-- <artifactId>dss</artifactId>-->
@@ -30,7 +30,7 @@
3030
<!-- <relativePath>../../../pom.xml</relativePath>-->
3131
<!-- </parent>-->
3232
<artifactId>dss-mlflow-appconn</artifactId>
33-
<version>1.0.0</version>
33+
<version>1.1.3</version>
3434
<dependencies>
3535
<dependency>
3636
<groupId>com.webank.wedatasphere.dss</groupId>
@@ -164,7 +164,7 @@
164164
<dependency>
165165
<groupId>com.webank.wedatasphere.dss</groupId>
166166
<artifactId>dss-development-process-standard</artifactId>
167-
<version>1.0.0</version>
167+
<version>${dss.version}</version>
168168
<scope>compile</scope>
169169
</dependency>
170170
<!-- <dependency>-->
@@ -179,6 +179,18 @@
179179
<version>${dss.version}</version>
180180
<scope>compile</scope>
181181
</dependency>
182+
<dependency>
183+
<groupId>org.appconn</groupId>
184+
<artifactId>appconn</artifactId>
185+
<version>1.1.3</version>
186+
<scope>compile</scope>
187+
</dependency>
188+
<dependency>
189+
<groupId>com.webank.wedatasphere.dss</groupId>
190+
<artifactId>dss-workflow-appconn</artifactId>
191+
<version>${dss.version}</version>
192+
<scope>compile</scope>
193+
</dependency>
182194
</dependencies>
183195

184196

‎appconns/dss-mlflow-appconn/src/main/assembly/distribution.xml‎

Lines changed: 20 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,19 @@
1-
<!--
2-
~ Copyright 2019 WeBank
3-
~ Licensed under the Apache License, Version 2.0 (the "License");
4-
~ you may not use this file except in compliance with the License.
5-
~ You may obtain a copy of the License at
6-
~
7-
~ http://www.apache.org/licenses/LICENSE-2.0
8-
~
9-
~ Unless required by applicable law or agreed to in writing, software
10-
~ distributed under the License is distributed on an "AS IS" BASIS,
11-
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12-
~ See the License for the specific language governing permissions and
13-
~ limitations under the License.
14-
~
15-
-->
16-
171
<assembly
182
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"
193
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
204
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
5+
<!-- 这里需保证 id 唯一 -->
216
<id>dss-mlflow-appconn</id>
227
<formats>
238
<format>dir</format>
249
</formats>
2510
<includeBaseDirectory>true</includeBaseDirectory>
11+
<!-- 具体的 AppConn 名 -->
2612
<baseDirectory>mlflow</baseDirectory>
2713

2814
<dependencySets>
15+
<!-- 如果有 dependency，则需 lib 目录 -->
2916
<dependencySet>
30-
<!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> -->
31-
<!-- Now, select which projects to include in this module-set. -->
3217
<outputDirectory>lib</outputDirectory>
3318
<useProjectArtifact>true</useProjectArtifact>
3419
<useTransitiveDependencies>true</useTransitiveDependencies>
@@ -39,28 +24,33 @@
3924
</dependencySets>
4025

4126
<fileSets>
27+
<!-- 如果有 appconn.properties，则需 conf 目录 -->
4228
<fileSet>
43-
<directory>${basedir}/src/main/resources</directory>
29+
<directory>${basedir}/conf</directory>
4430
<includes>
45-
<include>appconn.properties</include>
31+
<include>*</include>
4632
</includes>
4733
<fileMode>0777</fileMode>
48-
<outputDirectory>/</outputDirectory>
34+
<outputDirectory>conf</outputDirectory>
4935
<lineEnding>unix</lineEnding>
5036
</fileSet>
51-
37+
<!-- 存放 init.sql -->
5238
<fileSet>
5339
<directory>${basedir}/src/main/resources</directory>
5440
<includes>
55-
<include>log4j.properties</include>
56-
<include>log4j2.xml</include>
41+
<include>init.sql</include>
5742
</includes>
5843
<fileMode>0777</fileMode>
59-
<outputDirectory>conf</outputDirectory>
60-
<lineEnding>unix</lineEnding>
44+
<outputDirectory>db</outputDirectory>
45+
</fileSet>
46+
<!-- 存放工作流节点的 icons -->
47+
<fileSet>
48+
<directory>${basedir}/src/main/icons</directory>
49+
<includes>
50+
<include>*</include>
51+
</includes>
52+
<fileMode>0777</fileMode>
53+
<outputDirectory>icons</outputDirectory>
6154
</fileSet>
62-
6355
</fileSets>
64-
65-
</assembly>
66-
56+
</assembly>

‎appconns/dss-mlflow-appconn/src/main/icons/mlflow.icon‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1603264751694" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5768" xmlns:xlink="http://www.w3.org/1999/xlink" width="200" height="200"><defs><style type="text/css"></style></defs><path d="M826.993778 748.373333l-2.901334 1.763556-113.436444-114.915556 2.161778-3.072c24.974222-35.043556 38.115556-76.572444 38.115555-120.092444a207.075556 207.075556 0 0 0-37.888-119.921778l-2.218666-3.072 113.550222-115.029333 2.844444 1.706666c19.911111 11.719111 42.666667 17.92 65.877334 17.976889C965.290667 293.717333 1024 234.211556 1024 160.995556 1024 87.950222 965.290667 28.444444 893.098667 28.444444c-72.248889 0-130.958222 59.505778-130.958223 132.608 0 23.381333 6.087111 46.364444 17.692445 66.787556l1.706667 2.958222-113.550223 115.086222-3.072-2.161777a201.272889 201.272889 0 0 0-250.538666 12.458666A206.734222 206.734222 0 0 0 346.396444 480.711111l-0.568888 3.811556H258.958222l-0.796444-3.470223A130.958222 130.958222 0 0 0 130.844444 379.847111C58.709333 379.847111 0 439.352889 0 512.568889c0 73.102222 58.709333 132.551111 130.901333 132.551111A131.811556 131.811556 0 0 0 256.853333 548.977778l0.853334-3.185778h88.974222l0.625778 3.640889c17.749333 97.792 101.888 168.732444 200.192 168.732444a200.248889 200.248889 0 0 0 117.134222-37.546666l2.958222-2.161778 113.777778 115.143111-1.706667 3.015111c-11.605333 20.252444-17.635556 43.064889-17.635555 66.275556 0 73.159111 58.766222 132.664889 130.958222 132.664889 72.192 0 130.958222-59.505778 130.958222-132.664889 0-73.159111-58.766222-132.608-130.844444-132.608-23.324444 0-46.250667 6.257778-66.104889 18.090666z m-380.586667-133.916444A144.725333 144.725333 0 0 1 404.536889 512c0-38.684444 14.848-75.093333 41.870222-102.4a141.084444 141.084444 0 0 1 101.091556-42.439111c38.172444 0 74.695111 15.36 101.091555 42.439111 27.079111 27.420444 42.097778 64.170667 41.927111 102.4 0 38.684444-14.904889 75.036444-41.927111 102.4a140.913778 140.913778 0 0 1-101.091555 42.439111c-38.172444 0-74.695111-15.36-101.091556-42.439111z m517.12-453.404445c0 39.367111-31.630222 71.395556-70.428444 71.395556-14.222222 0-28.046222-4.380444-39.253334-12.003556l-18.773333-18.887111a71.452444 71.452444 0 0 1-12.458667-40.504889c0-39.367111 31.573333-71.338667 70.485334-71.338666 38.798222 0 70.428444 31.971556 70.428444 71.395555z m-70.428444 773.233778a70.997333 70.997333 0 0 1-70.485334-71.395555c0-39.310222 31.573333-71.338667 70.485334-71.338667 38.798222 0 70.428444 32.028444 70.428444 71.338667 0 39.367111-31.630222 71.395556-70.428444 71.395555zM201.329778 512.398222c0 39.310222-31.516444 71.338667-70.428445 71.338667a70.940444 70.940444 0 0 1-70.428444-71.338667c0-39.367111 31.630222-71.395556 70.428444-71.395555 38.912 0 70.485333 32.028444 70.485334 71.395555z" fill="#8470FF" p-id="5769"></path></svg>

‎appconns/dss-mlflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/mlflow/execution/MLFlowCompletedExecutionResponseRef.java‎

Lines changed: 0 additions & 47 deletions
This file was deleted.

‎appconns/dss-mlflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/mlflow/execution/MLFlowExecutionAction.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import com.webank.wedatasphere.dss.standard.app.development.listener.common.LongTermRefExecutionAction;
66
import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionState;
77
import com.webank.wedatasphere.dss.standard.app.development.listener.core.ExecutionRequestRefContext;
8+
import com.webank.wedatasphere.dss.standard.app.development.listener.ref.RefExecutionRequestRef;
89

910
public class MLFlowExecutionAction extends AbstractRefExecutionAction implements LongTermRefExecutionAction {
1011

‎appconns/dss-mlflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/mlflow/execution/MLFlowExecutionOperation.java‎

Lines changed: 30 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -16,75 +16,56 @@
1616

1717
package com.webank.wedatasphere.dss.appconn.mlflow.execution;
1818

19-
import com.webank.wedatasphere.dss.appconn.mlflow.MLFlowAppConn;
2019
import com.webank.wedatasphere.dss.appconn.mlflow.job.JobManager;
21-
import com.webank.wedatasphere.dss.standard.app.development.listener.common.AsyncExecutionRequestRef;
22-
import com.webank.wedatasphere.dss.standard.app.development.listener.common.CompletedExecutionResponseRef;
20+
import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils;
2321
import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionAction;
2422
import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionState;
2523
import com.webank.wedatasphere.dss.standard.app.development.listener.core.Killable;
2624
import com.webank.wedatasphere.dss.standard.app.development.listener.core.LongTermRefExecutionOperation;
2725
import com.webank.wedatasphere.dss.standard.app.development.listener.core.Procedure;
28-
import com.webank.wedatasphere.dss.standard.app.development.ref.ExecutionRequestRef;
29-
import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService;
30-
import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation;
31-
import org.apache.linkis.httpclient.request.HttpAction;
32-
import org.apache.linkis.httpclient.response.HttpResult;
33-
import org.slf4j.Logger;
34-
import org.slf4j.LoggerFactory;
26+
import com.webank.wedatasphere.dss.standard.app.development.listener.ref.ExecutionResponseRef;
27+
import com.webank.wedatasphere.dss.standard.app.development.listener.ref.RefExecutionRequestRef;
28+
import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef;
29+
import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException;
3530

31+
import java.util.HashMap;
3632
import java.util.Map;
3733

38-
public class MLFlowExecutionOperation extends LongTermRefExecutionOperation implements Killable, Procedure {
34+
public class MLFlowExecutionOperation extends LongTermRefExecutionOperation<RefExecutionRequestRef.RefExecutionProjectWithContextRequestRef> implements Killable, Procedure {
3935

4036

41-
private final static Logger logger = LoggerFactory.getLogger(MLFlowExecutionOperation.class);
42-
DevelopmentService developmentService;
43-
// private SSORequestOperation<HttpAction, HttpResult> ssoRequestOperation;
44-
45-
public MLFlowExecutionOperation(DevelopmentService service) {
46-
this.developmentService = service;
47-
// this.ssoRequestOperation = this.developmentService.getSSORequestService().createSSORequestOperation(getAppName());
48-
}
49-
50-
private String getAppName() {
51-
return MLFlowAppConn.MLFlow_APPCONN_NAME;
52-
}
53-
5437
@Override
55-
protected RefExecutionAction submit(ExecutionRequestRef requestRef) {
56-
AsyncExecutionRequestRef asyncRequestRef = (AsyncExecutionRequestRef) requestRef;
57-
Map<String,Object> jobContent = requestRef.getJobContent();
58-
return JobManager.submit(jobContent, asyncRequestRef.getExecutionRequestRefContext());
38+
protected RefExecutionAction submit(RefExecutionRequestRef.RefExecutionProjectWithContextRequestRef requestRef) throws ExternalOperationFailedException {
39+
Map<String, Object> jobContent = requestRef.getRefJobContent();
40+
HashMap contextInfo = DSSCommonUtils.COMMON_GSON.fromJson(DSSCommonUtils.COMMON_GSON.fromJson(
41+
requestRef.getParameter("dssContextId").toString(), HashMap.class).get("value").toString(), HashMap.class);
42+
return JobManager.submit(jobContent, requestRef.getExecutionRequestRefContext(), contextInfo);
5943
}
6044

6145
@Override
62-
public RefExecutionState state(RefExecutionAction action) {
46+
public RefExecutionState state(RefExecutionAction action) throws ExternalOperationFailedException {
47+
logger.info(action.toString());
6348
MLFlowExecutionAction mlFlowExecutionAction = (MLFlowExecutionAction) action;
6449
return JobManager.state(mlFlowExecutionAction);
6550
}
6651

6752
@Override
68-
public CompletedExecutionResponseRef result(RefExecutionAction action) {
69-
MLFlowExecutionAction mlFlowExecutionAction = (MLFlowExecutionAction) action;
70-
return JobManager.result(mlFlowExecutionAction, mlFlowExecutionAction.getExecutionRequestRefContext());
71-
}
72-
73-
private String getUser(AsyncExecutionRequestRef requestRef) {
74-
return requestRef.getExecutionRequestRefContext().getRuntimeMap().get("wds.dss.workflow.submit.user").toString();
75-
}
76-
77-
private String getId(AsyncExecutionRequestRef requestRef) {
78-
return null;
79-
}
80-
81-
@Override
82-
public void setDevelopmentService(DevelopmentService service) {
83-
this.developmentService = service;
84-
}
85-
86-
private String getBaseUrl(){
87-
return developmentService.getAppInstance().getBaseUrl();
53+
public ExecutionResponseRef result(RefExecutionAction action) throws ExternalOperationFailedException {
54+
HashMap resultMap = new HashMap();
55+
int status = -1;
56+
String errorMsg = "";
57+
if (action == null) {
58+
errorMsg = "";
59+
return ExecutionResponseRef.newBuilder().setResponseRef(
60+
new MLFlowExecutionResponseRef("", status, errorMsg, resultMap)).build();
61+
}
62+
logger.info(action.toString());
63+
MLFlowExecutionAction mlssAction = (MLFlowExecutionAction) action;
64+
if (mlssAction.getState() == RefExecutionState.Success) {
65+
status = 200;
66+
}
67+
ResponseRef response = new MLFlowExecutionResponseRef("", status, errorMsg, resultMap);
68+
return ExecutionResponseRef.newBuilder().setResponseRef(response).build();
8869
}
8970

9071
@Override

appconns/dss-mlss-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/mlss/ref/NodeUpdateCSRequestRefImpl.java renamed to appconns/dss-mlflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/mlflow/execution/MLFlowExecutionResponseRef.java

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,16 @@
1414
*
1515
*/
1616

17-
package com.webank.wedatasphere.dss.appconn.mlss.ref;
17+
package com.webank.wedatasphere.dss.appconn.mlflow.execution;
1818

19-
import com.webank.wedatasphere.dss.standard.app.development.ref.UpdateCSRequestRef;
20-
import com.webank.wedatasphere.dss.standard.app.development.ref.impl.NodeRequestRefImpl;
19+
import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRefImpl;
2120

21+
import java.util.Map;
2222

23-
public class NodeUpdateCSRequestRefImpl extends NodeRequestRefImpl implements UpdateCSRequestRef {
2423

24+
public class MLFlowExecutionResponseRef extends ResponseRefImpl {
25+
26+
public MLFlowExecutionResponseRef(String responseBody, int status, String errorMsg, Map<String, Object> responseMap) {
27+
super(responseBody, status, errorMsg, responseMap);
28+
}
2529
}

‎appconns/dss-mlflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/mlflow/execution/MLFlowExecutionService.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ public RefExecutionOperation createRefExecutionOperation() {
3333
MLFlowConfig.APP_SIGN = String.valueOf(config.get("MLSS-APPSignature"));
3434
MLFlowConfig.AUTH_TYPE = String.valueOf(config.get("MLSS-Auth-Type"));
3535
MLFlowConfig.TIMESTAMP = String.valueOf(config.get("MLSS-APPSignature"));
36-
return new MLFlowExecutionOperation(this);
36+
MLFlowConfig.MODEL_MONITORING_JAR = String.valueOf(config.get("MLSS-ModelMonitoring-JAR"));
37+
return new MLFlowExecutionOperation();
3738
}
3839

3940
}

0 commit comments

Comments
 (0)