From 436f90d7a08cb495762e67f41b42ecfecb051511 Mon Sep 17 00:00:00 2001 From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com> Date: Tue, 21 Feb 2023 19:29:48 +0800 Subject: [PATCH 001/689] [Bugfix] engine version support hyphen (#4246) * EngineTypeLabel support "-" --- .../label/entity/engine/EngineTypeLabel.java | 15 ++++++ .../entity/engine/EngineTypeLabelTest.java | 47 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java index 09492b146ac..912f6c9f948 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java @@ -74,4 +74,19 @@ public void setVersion(String version) { public Boolean isEmpty() { return StringUtils.isBlank(getEngineType()) || StringUtils.isBlank(getVersion()); } + + @Override + protected void setStringValue(String stringValue) { + String version; + String engineType = stringValue.split("-")[0]; + + if (engineType.equals("*")) { + version = stringValue.replaceFirst("[" + engineType + "]-", ""); + } else { + version = stringValue.replaceFirst(engineType + "-", ""); + } + + setEngineType(engineType); + setVersion(version); + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java new file mode 100644 index 00000000000..cc4171b4377 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.label.entity.engine; + +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactory; +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** EngineTypeLabel Tester */ +public class EngineTypeLabelTest { + + @Test + public void testSetStringValue() { + String engineType = "hive"; + String version = "1.1.0-cdh5.12.0"; + + String engineType1 = "*"; + String version1 = "*"; + + LabelBuilderFactory labelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory(); + EngineTypeLabel engineTypeLabel = labelBuilderFactory.createLabel(EngineTypeLabel.class); + engineTypeLabel.setStringValue(engineType + "-" + version); + Assertions.assertEquals(engineTypeLabel.getEngineType(), engineType); + Assertions.assertEquals(engineTypeLabel.getVersion(), version); + + engineTypeLabel.setStringValue(engineType1 + "-" + version1); + Assertions.assertEquals(engineTypeLabel.getEngineType(), engineType1); + Assertions.assertEquals(engineTypeLabel.getVersion(), version1); + } +} From ee29de94374b6fb59285a0bf6b0a3710a3c6e430 Mon Sep 17 00:00:00 2001 From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com> Date: Thu, 23 Feb 2023 21:17:57 +0800 Subject: [PATCH 002/689] [feat] support different hive version (#4255) * support different hive version * fix code style * log exception --- linkis-engineconn-plugins/flink/pom.xml | 1 - linkis-engineconn-plugins/hive/pom.xml | 4 -- .../serde/CustomerDelimitedJSONSerDe.java | 54 ++++++++++++++----- .../service/hive/pom.xml | 1 - pom.xml | 1 + 5 files changed, 42 insertions(+), 19 deletions(-) diff --git a/linkis-engineconn-plugins/flink/pom.xml b/linkis-engineconn-plugins/flink/pom.xml index a6c0894a5c9..7fe87fb2d2b 100644 --- a/linkis-engineconn-plugins/flink/pom.xml +++ b/linkis-engineconn-plugins/flink/pom.xml @@ -27,7 +27,6 @@ linkis-engineconn-plugin-flink 1.12.2 - 2.3.3 1.3.1 diff --git a/linkis-engineconn-plugins/hive/pom.xml b/linkis-engineconn-plugins/hive/pom.xml index 8fe446167c6..28b60fff0a8 100644 --- a/linkis-engineconn-plugins/hive/pom.xml +++ b/linkis-engineconn-plugins/hive/pom.xml @@ -26,10 +26,6 @@ linkis-engineplugin-hive - - 2.3.3 - - diff --git a/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java b/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java index 671b0c1d19e..9c425ff0575 100644 --- a/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java +++ b/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java @@ -17,6 +17,8 @@ package org.apache.linkis.engineplugin.hive.serde; +import org.apache.linkis.common.utils.ClassUtils; + import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.hive.serde2.ByteStream; import org.apache.hadoop.hive.serde2.SerDeException; @@ -33,6 +35,7 @@ import java.io.IOException; import java.io.OutputStream; +import java.lang.reflect.InvocationTargetException; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -309,18 +312,6 @@ private static void writePrimitiveUTF8( binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); break; } - case INTERVAL_YEAR_MONTH: - { - wc = ((HiveIntervalYearMonthObjectInspector) oi).getPrimitiveWritableObject(o); - binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); - break; - } - case INTERVAL_DAY_TIME: - { - wc = ((HiveIntervalDayTimeObjectInspector) oi).getPrimitiveWritableObject(o); - binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); - break; - } case DECIMAL: { HiveDecimalObjectInspector decimalOI = (HiveDecimalObjectInspector) oi; @@ -329,7 +320,44 @@ private static void writePrimitiveUTF8( } default: { - throw new RuntimeException("Unknown primitive type: " + category); + boolean containsIntervalYearMonth = false; + boolean containsIntervalDayTime = false; + for (PrimitiveObjectInspector.PrimitiveCategory primitiveCategory : + PrimitiveObjectInspector.PrimitiveCategory.values()) { + containsIntervalYearMonth = "INTERVAL_YEAR_MONTH".equals(primitiveCategory.name()); + containsIntervalDayTime = "INTERVAL_DAY_TIME".equals(primitiveCategory.name()); + try { + if (containsIntervalYearMonth) { + wc = + (WritableComparable) + ClassUtils.getClassInstance( + "org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalYearMonthObjectInspector") + .getClass() + .getMethod("getPrimitiveWritableObject", Object.class) + .invoke(oi, o); + binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); + break; + } + if (containsIntervalDayTime) { + wc = + (WritableComparable) + ClassUtils.getClassInstance( + "org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalDayTimeObjectInspector") + .getClass() + .getMethod("getPrimitiveWritableObject", Object.class) + .invoke(oi, o); + binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); + break; + } + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + LOG.error("Fail to invoke method:[getPrimitiveWritableObject]!", e); + } + } + if (containsIntervalYearMonth || containsIntervalDayTime) { + break; + } else { + throw new RuntimeException("Unknown primitive type: " + category); + } } } if (binaryData == null) { diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml index 12791394463..b0c9cdef363 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml @@ -26,7 +26,6 @@ linkis-metadata-query-service-hive - 2.3.3 2.7.2 4.2.4 diff --git a/pom.xml b/pom.xml index 6e42ea39da7..030b00f870f 100644 --- a/pom.xml +++ b/pom.xml @@ -106,6 +106,7 @@ 1.3.2-SNAPSHOT 2.9.2 2.4.3 + 2.3.3 2.7.2 hadoop-hdfs 2.7.2 From 503f72d927c700c248239d22188641eb7ffb78fb Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Sat, 25 Feb 2023 14:50:26 +0100 Subject: [PATCH 003/689] add Apache license for wagon-http-lightweight (#4267) --- linkis-dist/release-docs/NOTICE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkis-dist/release-docs/NOTICE b/linkis-dist/release-docs/NOTICE index 69c618a7226..1e4cfcbd750 100644 --- a/linkis-dist/release-docs/NOTICE +++ b/linkis-dist/release-docs/NOTICE @@ -1864,7 +1864,7 @@ testng (6.14.2) wagon-http-lightweight (3.0.0) -* License: Pending +* License: Apache-2.0 * Project: https://maven.apache.org/wagon/ * Source: https://mvnrepository.com/artifact/org.apache.maven.wagon/wagon-http-lightweight/3.0.0 @@ -3496,4 +3496,4 @@ Apache SeaTunnel (incubating) Copyright 2021-2022 The Apache Software Foundation This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). \ No newline at end of file +The Apache Software Foundation (http://www.apache.org/). From 4217a3f7e32438bb1d7b94b328ef620c640baac4 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Mon, 7 Nov 2022 09:41:48 +0800 Subject: [PATCH 004/689] ServiceInstance add registryTime --- .../org/apache/linkis/common/ServiceInstance.scala | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/ServiceInstance.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/ServiceInstance.scala index 8fcb4af7378..9cee5fe329d 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/ServiceInstance.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/ServiceInstance.scala @@ -20,10 +20,13 @@ package org.apache.linkis.common class ServiceInstance { private var applicationName: String = _ private var instance: String = _ + private var registryTimestamp: Long = _ def setApplicationName(applicationName: String): Unit = this.applicationName = applicationName def getApplicationName: String = applicationName def setInstance(instance: String): Unit = this.instance = instance def getInstance: String = instance + def setRegistryTimestamp(registryTimestamp: Long): Unit = this.registryTimestamp = registryTimestamp + def getRegistryTimestamp: Long = registryTimestamp override def equals(other: Any): Boolean = other match { case that: ServiceInstance => @@ -42,7 +45,7 @@ class ServiceInstance { .foldLeft(0)((a, b) => 31 * a + b) } - override def toString: String = s"ServiceInstance($applicationName, $instance)" + override def toString: String = s"ServiceInstance($applicationName, $instance, $registryTimestamp)" } object ServiceInstance { @@ -54,6 +57,14 @@ object ServiceInstance { serviceInstance } + def apply(applicationName: String, instance: String, registryTimestamp: Long): ServiceInstance = { + val serviceInstance = new ServiceInstance + serviceInstance.setApplicationName(applicationName) + serviceInstance.setInstance(instance) + serviceInstance.setRegistryTimestamp(registryTimestamp) + serviceInstance + } + def unapply(serviceInstance: ServiceInstance): Option[(String, String)] = if (serviceInstance != null) { Some(serviceInstance.applicationName, serviceInstance.instance) From 08fc461ed8ed4ac5fd98879f502b7539c1604751 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Wed, 9 Nov 2022 18:10:27 +0800 Subject: [PATCH 005/689] gateway edit and entrance api edit --- .../server/conf/ServerConfiguration.scala | 2 + .../protocol/utils/ZuulEntranceUtils.scala | 2 +- .../restful/EntranceLabelRestfulApi.java | 19 ++++++- .../linkis/entrance/EntranceServer.scala | 22 +++++++++ .../entrance/utils/JobHistoryHelper.scala | 17 +++++++ .../linkis-gateway-server-support/pom.xml | 7 +++ .../parser/EntranceRequestGatewayParser.scala | 49 +++++++++++++++++-- 7 files changed, 110 insertions(+), 8 deletions(-) diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala index 582568e6262..6784c5100f9 100644 --- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala +++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala @@ -207,4 +207,6 @@ object ServerConfiguration extends Logging { val LINKIS_SERVER_SESSION_PROXY_TICKETID_KEY = CommonVars("wds.linkis.session.proxy.user.ticket.key", "linkis_user_session_proxy_ticket_id_v1") + val LINKIS_SERVER_HEADER_KEY = CommonVars("wds.linkis.session.proxy.user.ticket.key", "job_req_id") + } diff --git a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/ZuulEntranceUtils.scala b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/ZuulEntranceUtils.scala index 95c7a818739..ad30484c46f 100644 --- a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/ZuulEntranceUtils.scala +++ b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/utils/ZuulEntranceUtils.scala @@ -23,7 +23,7 @@ object ZuulEntranceUtils { private val INSTANCE_SPLIT_TOKEN = "_" - private val EXEC_ID = "exec_id" + val EXEC_ID = "exec_id" private val SPLIT_LEN = 3 diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java index 2ab457747cf..03ae97b7816 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java @@ -18,6 +18,8 @@ package org.apache.linkis.entrance.restful; import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.entrance.EntranceServer; +import org.apache.linkis.entrance.context.DefaultEntranceContext; import org.apache.linkis.instance.label.client.InstanceLabelClient; import org.apache.linkis.manager.label.constant.LabelKeyConstant; import org.apache.linkis.manager.label.constant.LabelValueConstant; @@ -26,6 +28,7 @@ import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; @@ -45,7 +48,13 @@ @RequestMapping(path = "/entrance/operation/label") public class EntranceLabelRestfulApi { - private static final Logger logger = LoggerFactory.getLogger(EntranceLabelRestfulApi.class); + private static final Logger logger = LoggerFactory.getLogger(EntranceLabelRestfulApi.class); + private EntranceServer entranceServer; + + @Autowired + public void setEntranceServer(EntranceServer entranceServer) { + this.entranceServer = entranceServer; + } @ApiOperation(value = "update", notes = "update route label", response = Message.class) @ApiOperationSupport(ignoreParameters = {"jsonNode"}) @@ -79,6 +88,12 @@ public Message updateRouteLabel(HttpServletRequest req) { insLabelRefreshRequest.setServiceInstance(Sender.getThisServiceInstance()); InstanceLabelClient.getInstance().refreshLabelsToInstance(insLabelRefreshRequest); logger.info("Finished to modify the routelabel of entry to offline"); - return Message.ok(); + + logger.info("Prepare to update the instances field for all not execution task to empty string"); + // todo ((DefaultEntranceContext) entranceServer.getEntranceContext()).setOfflineFlag(true); + entranceServer.updateAllNotExecutionTaskInstances(); + logger.info("Finished to update the instances field for all not execution task to empty string"); + + return Message.ok(); } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index f298e54251d..c369adcbc32 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -227,6 +227,28 @@ abstract class EntranceServer extends Logging { .toArray } + def getAllNotStartRunningTask(): Array[EntranceJob] = { + val consumers = getEntranceContext + .getOrCreateScheduler() + .getSchedulerContext + .getOrCreateConsumerManager + .listConsumers() + .toSet + + consumers + .flatMap { consumer => + consumer.getConsumeQueue.getWaitingEvents + } + .filter(job => job != null && job.isInstanceOf[EntranceJob]) + .map(_.asInstanceOf[EntranceJob]) + .toArray + } + + def updateAllNotExecutionTaskInstances(): Unit = { + val taskIds = getAllNotStartRunningTask().map(_.getJobRequest.getId) + JobHistoryHelper.updateBatchInstances(taskIds) + } + } object EntranceServer { diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala index 0fc7e6e486e..080e6913e05 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala @@ -123,6 +123,23 @@ object JobHistoryHelper extends Logging { sender.ask(jobReqBatchUpdate) } + /** + * Batch update instances + * + * @param taskIdList + */ + def updateBatchInstances(taskIdList: Array[java.lang.Long]): Unit = { + val jobReqList = new util.ArrayList[JobRequest]() + taskIdList.foreach(taskID => { + val jobRequest = new JobRequest + jobRequest.setId(taskID) + jobRequest.setInstances("") + jobReqList.add(jobRequest) + }) + val jobReqBatchUpdate = JobReqBatchUpdate(jobReqList) + sender.ask(jobReqBatchUpdate) + } + private def getTaskByTaskID(taskID: Long): JobRequest = { val jobRequest = new JobRequest jobRequest.setId(taskID) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/pom.xml b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/pom.xml index ef4635ae00d..38efa93b336 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/pom.xml +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/pom.xml @@ -89,6 +89,13 @@ ${project.version} + + + org.apache.linkis + linkis-jobhistory + ${project.version} + + com.fasterxml.jackson.core jackson-databind diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala index 2ee0f4b0233..3eacb8c2b6c 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala @@ -17,17 +17,28 @@ package org.apache.linkis.gateway.ujes.parser +import org.apache.commons.lang3.StringUtils import org.apache.linkis.common.ServiceInstance import org.apache.linkis.gateway.config.GatewayConfiguration import org.apache.linkis.gateway.http.GatewayContext import org.apache.linkis.gateway.parser.AbstractGatewayParser import org.apache.linkis.gateway.ujes.parser.EntranceExecutionGatewayParser._ +import org.apache.linkis.jobhistory.entity.JobHistory +import org.apache.linkis.jobhistory.service.JobHistoryQueryService import org.apache.linkis.protocol.utils.ZuulEntranceUtils - +import org.apache.linkis.rpc.interceptor.ServiceInstanceUtils +import org.apache.linkis.server.conf.ServerConfiguration import org.springframework.stereotype.Component +import javax.annotation.Resource + @Component class EntranceRequestGatewayParser extends AbstractGatewayParser { + + + @Resource + private var jobHistoryQueryService: JobHistoryQueryService = _ + override def shouldContainRequestBody(gatewayContext: GatewayContext): Boolean = false override def parse(gatewayContext: GatewayContext): Unit = @@ -36,9 +47,9 @@ class EntranceRequestGatewayParser extends AbstractGatewayParser { if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return val serviceInstance = if (execId.startsWith(EntranceRequestGatewayParser.API_REQUEST)) { if ( - gatewayContext.getRequest.getQueryParams.containsKey( - EntranceRequestGatewayParser.INSTANCE - ) + gatewayContext.getRequest.getQueryParams.containsKey( + EntranceRequestGatewayParser.INSTANCE + ) ) { val instances = gatewayContext.getRequest.getQueryParams.get(EntranceRequestGatewayParser.INSTANCE) @@ -50,13 +61,41 @@ class EntranceRequestGatewayParser extends AbstractGatewayParser { } else { ServiceInstance(GatewayConfiguration.ENTRANCE_SPRING_NAME.getValue, null) } - } else { + } else if (execId.startsWith(ZuulEntranceUtils.EXEC_ID)) { + // parse by execId ZuulEntranceUtils.parseServiceInstanceByExecID(execId)(0) + } else { + // parse by taskId + val jobHistory = parseJobHistoryByTaskID(execId.toLong, gatewayContext) + // add header + val jobReqId = if (jobHistory == null) "" else jobHistory.getJobReqId + gatewayContext.getRequest.addHeader(ServerConfiguration.LINKIS_SERVER_HEADER_KEY.getValue, Array(jobReqId)) + // select instance + val instance = if (jobHistory == null) null else jobHistory.getInstances + ServiceInstance(GatewayConfiguration.ENTRANCE_SPRING_NAME.getValue, instance) } gatewayContext.getGatewayRoute.setServiceInstance(serviceInstance) case _ => } + def parseJobHistoryByTaskID(taskId: Long, gatewayContext: GatewayContext): JobHistory = { + val histories = jobHistoryQueryService.search(taskId, null, null, null, null, null, null, null) + if (histories.isEmpty) { + sendErrorResponse(s"taskId $taskId is not exists.", gatewayContext) + } + val instances = histories.get(0).getInstances + val activeInstances = ServiceInstanceUtils.getRPCServerLoader.getServiceInstances(GatewayConfiguration.ENTRANCE_SPRING_NAME.getValue) + + if (activeInstances.exists(StringUtils.isNotBlank(instances) && _.getInstance.equals(instances)) && + activeInstances.filter(_.getInstance.equals(instances))(0).getRegistryTimestamp <= histories.get(0).getCreatedTime.getTime + ) { + histories.get(0) + } else { + null + } + + } + } object EntranceRequestGatewayParser { From 17a59318d8595bca1c100a3c334ffb9ad3014873 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Tue, 15 Nov 2022 15:58:00 +0800 Subject: [PATCH 006/689] add method of queryFailoverJobs --- .../impl/JobHistoryQueryServiceImpl.scala | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala index 3512d3fbfdd..c918ee085c8 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala @@ -243,6 +243,37 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { jobResp } + @Receiver + override def queryFailoverJobs(requestFailoverJob: RequestFailoverJob): JobRespProtocol = { + val reqMap = requestFailoverJob.reqMap + val statusList = requestFailoverJob.statusList + val startTimestamp = requestFailoverJob.startTimestamp + val limit = requestFailoverJob.limit + logger.info(s"query failover jobs, start timestamp:${startTimestamp}， limit:${limit}") + val jobResp = new JobRespProtocol + Utils.tryCatch { + val jobList = jobHistoryMapper.selectFailoverJobHistory(reqMap, statusList, startTimestamp, limit) + val jobReqList = jobList.asScala.map(jobHistory2JobRequest).toList + val map = new util.HashMap[String, Object]() + map.put(JobRequestConstants.JOB_HISTORY_LIST, jobReqList) + jobResp.setStatus(0) + jobResp.setData(map) + } { case e: Exception => + logger.error(s"Failed to query failover job, instances ${reqMap.keySet()}", e) + jobResp.setStatus(1) + jobResp.setMsg(ExceptionUtils.getRootCauseMessage(e)) + } + jobResp + } + + /* private def queryTaskList2RequestPersistTaskList(queryTask: java.util.List[QueryTask]): java.util.List[RequestPersistTask] = { + import scala.collection.JavaConversions._ + val tasks = new util.ArrayList[RequestPersistTask] + import org.apache.linkis.jobhistory.conversions.TaskConversions.queryTask2RequestPersistTask + queryTask.foreach(f => tasks.add(f)) + tasks + } */ + override def getJobHistoryByIdAndName(jobId: java.lang.Long, userName: String): JobHistory = { val jobReq = new JobHistory jobReq.setId(jobId) From 7706a6027dc6571bc15bbbd1010090128441b68f Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Tue, 15 Nov 2022 15:42:55 +0800 Subject: [PATCH 007/689] entrance ha and failover --- .../scheduler/queue/AbstractGroup.scala | 5 + .../queue/fifoqueue/FIFOUserConsumer.scala | 15 +- .../common/protocol/job/JobReqProcotol.scala | 2 + .../linkis-entrance/pom.xml | 6 + .../conf/EntranceSpringConfiguration.java | 4 +- .../entrance/constant/ServiceNameConsts.java | 2 + .../restful/EntranceLabelRestfulApi.java | 15 +- .../server/DefaultEntranceServer.java | 6 + .../server/EntranceFailoverJobServer.java | 146 +++++++++++++++++ .../linkis/entrance/EntranceServer.scala | 152 +++++++++++++++++- .../entrance/conf/EntranceConfiguration.scala | 21 +++ .../scheduler/EntranceFIFOUserConsumer.scala | 50 ++++++ .../scheduler/EntranceGroupFactory.scala | 88 ++++++---- .../EntranceParallelConsumerManager.scala | 31 ++++ .../scheduler/EntranceSchedulerContext.scala | 5 + .../entrance/utils/JobHistoryHelper.scala | 109 ++++++++++++- .../manager/label/constant/LabelConstant.java | 2 + .../jobhistory/dao/JobHistoryMapper.java | 35 ++++ .../mapper/common/JobHistoryMapper.xml | 23 +++ .../service/JobHistoryQueryService.java | 2 + 20 files changed, 664 insertions(+), 55 deletions(-) create mode 100644 linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java create mode 100644 linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala create mode 100644 linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/AbstractGroup.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/AbstractGroup.scala index 6e9ecbd26fa..cc9577941f3 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/AbstractGroup.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/AbstractGroup.scala @@ -23,11 +23,16 @@ abstract class AbstractGroup extends Group { private var _status: GroupStatus = _ private var maxRunningJobs: Int = _ + private var maxAllowRunningJobs: Int = 0 private var maxAskExecutorTimes: Long = 0L def setMaxRunningJobs(maxRunningJobs: Int): Unit = this.maxRunningJobs = maxRunningJobs def getMaxRunningJobs: Int = maxRunningJobs + def setMaxAllowRunningJobs(maxAllowRunningJobs: Int): Unit = this.maxAllowRunningJobs = maxAllowRunningJobs + def getMaxAllowRunningJobs: Int = + if(maxAllowRunningJobs <= 0) maxRunningJobs else Math.min(maxAllowRunningJobs, maxRunningJobs) + def setMaxAskExecutorTimes(maxAskExecutorTimes: Long): Unit = this.maxAskExecutorTimes = maxAskExecutorTimes diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala index 2a40c2517be..692325b75cc 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala @@ -108,8 +108,9 @@ class FIFOUserConsumer( } var event: Option[SchedulerEvent] = getWaitForRetryEvent if (event.isEmpty) { - val completedNums = runningJobs.filter(job => job == null || job.isCompleted) - if (completedNums.length < 1) { + val maxAllowRunningJobs = fifoGroup.getMaxAllowRunningJobs + val currentRunningJobs = runningJobs.filter(e => e != null && !e.isCompleted) + if (maxAllowRunningJobs <= currentRunningJobs) { Utils.tryQuietly(Thread.sleep(1000)) // TODO 还可以优化，通过实现JobListener进行优化 return } @@ -188,6 +189,16 @@ class FIFOUserConsumer( runningJobs(index) = job } + protected def scanAllRetryJobsAndRemove(): Unit = { + for (index <- runningJobs.indices) { + val job = runningJobs(index) + if (job != null && job.isJobCanRetry) { + runningJobs(index) = null + logger.info(s"Job $job can retry, remove from runningJobs") + } + } + } + override def shutdown(): Unit = { future.cancel(true) super.shutdown() diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala index 2e447397872..4d6346c9184 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala @@ -51,3 +51,5 @@ class RequestOneJob extends JobReq { } case class RequestAllJob(instance: String) extends JobReq + +case class RequestFailoverJob(reqMap: util.Map[String, java.lang.Long], statusList: util.List[String], startTimestamp: Long, limit: Int = 10) extends JobReq diff --git a/linkis-computation-governance/linkis-entrance/pom.xml b/linkis-computation-governance/linkis-entrance/pom.xml index b9ebec930e8..21008708ce5 100644 --- a/linkis-computation-governance/linkis-entrance/pom.xml +++ b/linkis-computation-governance/linkis-entrance/pom.xml @@ -102,6 +102,12 @@ ${project.version} + + org.apache.linkis + linkis-ps-common-lock + ${project.version} + + diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java index 0bf27a68b3a..cf520c3823c 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java @@ -42,6 +42,7 @@ import org.apache.linkis.entrance.persistence.QueryPersistenceManager; import org.apache.linkis.entrance.persistence.ResultSetEngine; import org.apache.linkis.entrance.scheduler.EntranceGroupFactory; +import org.apache.linkis.entrance.scheduler.EntranceParallelConsumerManager; import org.apache.linkis.entrance.scheduler.EntranceSchedulerContext; import org.apache.linkis.orchestrator.ecm.EngineConnManagerBuilder; import org.apache.linkis.orchestrator.ecm.EngineConnManagerBuilder$; @@ -51,7 +52,6 @@ import org.apache.linkis.scheduler.executer.ExecutorManager; import org.apache.linkis.scheduler.queue.ConsumerManager; import org.apache.linkis.scheduler.queue.GroupFactory; -import org.apache.linkis.scheduler.queue.parallelqueue.ParallelConsumerManager; import org.apache.linkis.scheduler.queue.parallelqueue.ParallelScheduler; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -190,7 +190,7 @@ public GroupFactory groupFactory() { @Bean @ConditionalOnMissingBean public ConsumerManager consumerManager() { - return new ParallelConsumerManager( + return new EntranceParallelConsumerManager( ENTRANCE_SCHEDULER_MAX_PARALLELISM_USERS().getValue(), "EntranceJobScheduler"); } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/constant/ServiceNameConsts.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/constant/ServiceNameConsts.java index cb37279c113..bee17b8ed4e 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/constant/ServiceNameConsts.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/constant/ServiceNameConsts.java @@ -26,4 +26,6 @@ private ServiceNameConsts() {} public static final String ENTRANCE_SERVER = "entranceServer"; public static final String ENTRANCE_INTERCEPTOR = "entranceInterceptors"; + + public static final String ENTRANCE_FAILOVER_SERVER = "entranceFailoverServer"; } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java index 03ae97b7816..e51f66266dc 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java @@ -19,12 +19,14 @@ import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.entrance.EntranceServer; -import org.apache.linkis.entrance.context.DefaultEntranceContext; +import org.apache.linkis.entrance.scheduler.EntranceSchedulerContext; import org.apache.linkis.instance.label.client.InstanceLabelClient; +import org.apache.linkis.manager.label.constant.LabelConstant; import org.apache.linkis.manager.label.constant.LabelKeyConstant; import org.apache.linkis.manager.label.constant.LabelValueConstant; import org.apache.linkis.protocol.label.InsLabelRefreshRequest; import org.apache.linkis.rpc.Sender; +import org.apache.linkis.scheduler.SchedulerContext; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; @@ -89,10 +91,13 @@ public Message updateRouteLabel(HttpServletRequest req) { InstanceLabelClient.getInstance().refreshLabelsToInstance(insLabelRefreshRequest); logger.info("Finished to modify the routelabel of entry to offline"); - logger.info("Prepare to update the instances field for all not execution task to empty string"); - // todo ((DefaultEntranceContext) entranceServer.getEntranceContext()).setOfflineFlag(true); - entranceServer.updateAllNotExecutionTaskInstances(); - logger.info("Finished to update the instances field for all not execution task to empty string"); + logger.info("Prepare to update all not execution task instances to empty string"); + SchedulerContext schedulerContext = entranceServer.getEntranceContext().getOrCreateScheduler().getSchedulerContext(); + if (schedulerContext instanceof EntranceSchedulerContext) { + ((EntranceSchedulerContext) schedulerContext).setOfflineFlag(true); + } + entranceServer.updateAllNotExecutionTaskInstances(true); + logger.info("Finished to update all not execution task instances to empty string"); return Message.ok(); } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java index a050056fe1e..999d5cbcbf9 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java @@ -19,6 +19,7 @@ import org.apache.linkis.entrance.EntranceContext; import org.apache.linkis.entrance.EntranceServer; +import org.apache.linkis.entrance.conf.EntranceConfiguration; import org.apache.linkis.entrance.constant.ServiceNameConsts; import org.apache.linkis.entrance.execute.EntranceJob; import org.apache.linkis.entrance.log.LogReader; @@ -79,6 +80,11 @@ private void shutdownEntrance(ContextClosedEvent event) { if (shutdownFlag) { logger.warn("event has been handled"); } else { + if (EntranceConfiguration.ENTRANCE_SHUTDOWN_FAILOVER_ENABLED()) { + logger.warn("Entrance exit to update all not execution task instances and clean ConsumeQueue"); + updateAllNotExecutionTaskInstances(false); + } + logger.warn("Entrance exit to stop all job"); EntranceJob[] allUndoneJobs = getAllUndoneTask(null); if (null != allUndoneJobs) { diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java new file mode 100644 index 00000000000..7e7e0de69cf --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.server; + +import org.apache.commons.compress.utils.Lists; +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.entrance.EntranceServer; +import org.apache.linkis.entrance.conf.EntranceConfiguration; +import org.apache.linkis.entrance.constant.ServiceNameConsts; +import org.apache.linkis.entrance.scheduler.EntranceSchedulerContext; +import org.apache.linkis.entrance.utils.JobHistoryHelper; +import org.apache.linkis.governance.common.entity.job.JobRequest; +import org.apache.linkis.instance.label.client.InstanceLabelClient; +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; +import org.apache.linkis.manager.label.constant.LabelConstant; +import org.apache.linkis.manager.label.constant.LabelKeyConstant; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.manager.label.entity.route.RouteLabel; +import org.apache.linkis.publicservice.common.lock.entity.CommonLock; +import org.apache.linkis.publicservice.common.lock.service.CommonLockService; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.scheduler.queue.SchedulerEventState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Component(ServiceNameConsts.ENTRANCE_FAILOVER_SERVER) +public class EntranceFailoverJobServer { + + private static final Logger logger = LoggerFactory.getLogger(DefaultEntranceServer.class); + + @Autowired + private EntranceServer entranceServer; + + @Autowired + private CommonLockService commonLockService; + + + private static String ENTRANCE_FAILOVER_LOCK = "ENTRANCE_FAILOVER_LOCK"; + + @PostConstruct + public void init() { + failoverTask(); + } + + public void failoverTask() { + if (EntranceConfiguration.ENTRANCE_FAILOVER_ENABLED()) { + Utils.defaultScheduler().scheduleAtFixedRate( + new Runnable() { + @Override + public void run() { + EntranceSchedulerContext schedulerContext = (EntranceSchedulerContext) entranceServer.getEntranceContext().getOrCreateScheduler().getSchedulerContext(); + + // entrance do not failover job when it is offline + if (schedulerContext.getOfflineFlag()) return; + + CommonLock commonLock = new CommonLock(); + commonLock.setLockObject(ENTRANCE_FAILOVER_LOCK); + Boolean locked = false; + try { + locked = commonLockService.lock(commonLock, 10 * 1000L); + if (!locked) return; + logger.info("success locked {}", ENTRANCE_FAILOVER_LOCK); + + // serverInstance to map + Map serverInstanceMap = getActiveServerInstances().stream() + .collect(Collectors.toMap(ServiceInstance::getInstance, ServiceInstance::getRegistryTimestamp, (k1, k2) -> k2)); + if (serverInstanceMap.isEmpty()) return; + + // get failover start time + long startTimestamp = 0L; + if (EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME() > 0) { + startTimestamp = System.currentTimeMillis() - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME(); + } + + // get uncompleted status + List statusList = Lists.newArrayList(); + SchedulerEventState.values().filterNot(SchedulerEventState::isCompleted).foreach(state -> statusList.add(state.toString())); + + List jobRequests = JobHistoryHelper.queryWaitForFailoverTask(serverInstanceMap, statusList, startTimestamp, EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); + if (jobRequests.isEmpty()) return; + logger.info("success query failover jobs , job ids: {}", jobRequests.stream().map(JobRequest::getId)); + + // failover to local server + jobRequests.forEach(jobRequest -> entranceServer.failoverExecute(jobRequest)); + logger.info("success execute failover jobs, job ids: {}", jobRequests.stream().map(JobRequest::getId)); + + } catch (Exception e) { + logger.error("failover failed", e); + } finally { + if (locked) commonLockService.unlock(commonLock); + } + } + }, + EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INIT_TIME(), + EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INTERVAL(), + TimeUnit.MILLISECONDS + ); + } + } + + private List getActiveServerInstances() { + // get all entrance server from eureka + ServiceInstance[] serviceInstances = Sender.getInstances(Sender.getThisServiceInstance().getApplicationName()); + if (serviceInstances == null || serviceInstances.length <= 0) return Lists.newArrayList(); + + // get all offline label server + RouteLabel routeLabel = LabelBuilderFactoryContext.getLabelBuilderFactory() + .createLabel(LabelKeyConstant.ROUTE_KEY, LabelConstant.OFFLINE); + List> labels = Lists.newArrayList(); + labels.add(routeLabel); + List labelInstances = InstanceLabelClient.getInstance().getInstanceFromLabel(labels); + + // get active entrance server + List allInstances = Lists.newArrayList(); + allInstances.addAll(Arrays.asList(serviceInstances)); + allInstances.removeAll(labelInstances); + + return allInstances; + } + +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index c369adcbc32..e91cfb3df61 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -35,9 +35,11 @@ import org.apache.linkis.server.conf.ServerConfiguration import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils +import org.apache.linkis.common.log.LogUtils import java.text.MessageFormat -import java.util +import java.{lang, util} +import java.util.Date abstract class EntranceServer extends Logging { @@ -227,7 +229,7 @@ abstract class EntranceServer extends Logging { .toArray } - def getAllNotStartRunningTask(): Array[EntranceJob] = { + def getAllConsumeQueueTask(): Array[EntranceJob] = { val consumers = getEntranceContext .getOrCreateScheduler() .getSchedulerContext @@ -244,9 +246,149 @@ abstract class EntranceServer extends Logging { .toArray } - def updateAllNotExecutionTaskInstances(): Unit = { - val taskIds = getAllNotStartRunningTask().map(_.getJobRequest.getId) - JobHistoryHelper.updateBatchInstances(taskIds) + def clearAllConsumeQueue(): Unit = { + getEntranceContext + .getOrCreateScheduler() + .getSchedulerContext + .getOrCreateConsumerManager + .listConsumers() + .foreach(_.getConsumeQueue.clearAll()) + } + + def updateAllNotExecutionTaskInstances(retryWhenUpdateFail: Boolean): Unit = { + val taskIds = getAllConsumeQueueTask().map(_.getJobRequest.getId).toList + JobHistoryHelper.updateAllConsumeQueueTask(taskIds, retryWhenUpdateFail) + logger.info("Finished to update all not execution task instances") + clearAllConsumeQueue() + logger.info("Finished to clean all ConsumeQueue") + } + + /** + * execute failover job (提交故障转移任务，返回新的execId) + * + * @param jobRequest + */ + def failoverExecute(jobRequest: JobRequest): String = { + + if (null == jobRequest || null == jobRequest.getId || jobRequest.getId <= 0) { + throw new EntranceErrorException( + PERSIST_JOBREQUEST_ERROR.getErrorCode, + PERSIST_JOBREQUEST_ERROR.getErrorDesc + ) + } + + // todo dmp kill ec + + val logAppender = new java.lang.StringBuilder() + // init properties + initJobRequestProperties(jobRequest, logAppender) + // update jobRequest + getEntranceContext + .getOrCreatePersistenceManager() + .createPersistenceEngine() + .updateIfNeeded(jobRequest) + + val job = getEntranceContext.getOrCreateEntranceParser().parseToJob(jobRequest) + Utils.tryThrow { + job.init() + job.setLogListener(getEntranceContext.getOrCreateLogManager()) + job.setProgressListener(getEntranceContext.getOrCreatePersistenceManager()) + job.setJobListener(getEntranceContext.getOrCreatePersistenceManager()) + job match { + case entranceJob: EntranceJob => { + entranceJob.setEntranceListenerBus(getEntranceContext.getOrCreateEventListenerBus) + } + case _ => + } + Utils.tryCatch { + if (logAppender.length() > 0) + job.getLogListener.foreach(_.onLogUpdate(job, logAppender.toString.trim)) + } { t => + logger.error("Failed to write init JobRequest log, reason: ", t) + } + + /** + * job.afterStateChanged() method is only called in job.run(), and job.run() is called only + * after job is scheduled so it suggest that we lack a hook for job init, currently we call + * this to trigger JobListener.onJobinit() + */ + Utils.tryAndWarn(job.getJobListener.foreach(_.onJobInited(job))) + getEntranceContext.getOrCreateScheduler().submit(job) + val msg = s"Job with jobId : ${jobRequest.getId} and execID : ${job.getId()} submitted, success to failover" + logger.info(msg) + + job match { + case entranceJob: EntranceJob => + entranceJob.getJobRequest.setReqId(job.getId()) + if (jobTimeoutManager.timeoutCheck && JobTimeoutManager.hasTimeoutLabel(entranceJob)) + jobTimeoutManager.add(job.getId(), entranceJob) + entranceJob.getLogListener.foreach(_.onLogUpdate(entranceJob, msg)) + case _ => + } + + job.getId() + } { t => + job.onFailure("Submitting the query failed!(提交查询失败！)", t) + val _jobRequest = + getEntranceContext.getOrCreateEntranceParser().parseToJobRequest(job) + getEntranceContext + .getOrCreatePersistenceManager() + .createPersistenceEngine() + .updateIfNeeded(_jobRequest) + t match { + case e: LinkisException => e + case e: LinkisRuntimeException => e + case t: Throwable => + new SubmitFailedException( + SUBMITTING_QUERY_FAILED.getErrorCode, + SUBMITTING_QUERY_FAILED.getErrorDesc + ExceptionUtils.getRootCauseMessage(t), + t + ) + } + } + + } + + private def initJobRequestProperties(jobRequest: JobRequest, logAppender: lang.StringBuilder): Unit = { + + val initInstance = Sender.getThisInstance + val initDate = new Date(System.currentTimeMillis) + val initStatus = SchedulerEventState.Inited.toString + val initProgress = "0.0" + val initReqId = "" + + logAppender.append( + LogUtils.generateInfo(s"Job ${jobRequest.getId} start to failover, Initialize the properties \n") + ) + logAppender.append( + LogUtils.generateInfo(s"the instances ${jobRequest.getInstances} -> ${initInstance} \n") + ) + logAppender.append( + LogUtils.generateInfo(s"the created_time ${jobRequest.getCreatedTime} -> ${initDate} \n") + ) + logAppender.append( + LogUtils.generateInfo(s"the status ${jobRequest.getStatus} -> $initStatus \n") + ) + logAppender.append( + LogUtils.generateInfo(s"the progress ${jobRequest.getProgress} -> $initProgress \n") + ) + logAppender.append( + LogUtils.generateInfo(s"the job_req_id ${jobRequest.getReqId} -> $initReqId \n") + ) + + jobRequest.setInstances(initInstance) + jobRequest.setCreatedTime(initDate) + jobRequest.setStatus(initStatus) + jobRequest.setProgress(initProgress) + jobRequest.setReqId(initReqId) + jobRequest.setErrorCode(0) + jobRequest.setErrorDesc("") + jobRequest.setMetrics(new util.HashMap[String, Object]()) + jobRequest.getMetrics.put(TaskConstant.ENTRANCEJOB_SUBMIT_TIME, initInstance) + + logAppender.append( + LogUtils.generateInfo(s"Job ${jobRequest.getId} success to initialize the properties \n") + ) } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index 4b6230299b4..62c42cfdd0e 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -225,4 +225,25 @@ object EntranceConfiguration { val CREATOR_IP_SWITCH = CommonVars("wds.linkis.entrance.user.creator.ip.interceptor.switch", false) + val ENTRANCE_FAILOVER_ENABLED = CommonVars("linkis.entrance.failover.enable", true).getValue + + val ENTRANCE_FAILOVER_SCAN_INIT_TIME = + CommonVars("linkis.entrance.failover.scan.init.time", 3 * 1000).getValue + + val ENTRANCE_FAILOVER_SCAN_INTERVAL = + CommonVars("linkis.entrance.failover.scan.interval", 3 * 1000).getValue + + val ENTRANCE_FAILOVER_DATA_NUM_LIMIT = CommonVars("linkis.entrance.failover.data.num.limit", 10).getValue + + val ENTRANCE_FAILOVER_DATA_INTERVAL_TIME = CommonVars("linkis.entrance.failover.data.interval.time", new TimeType("7d").toLong).getValue + + // if true, the waitForRetry job in runningJobs can be failover + val ENTRANCE_FAILOVER_RETRY_JOB_ENABLED = CommonVars("linkis.entrance.failover.retry.job.enable", true) + + val ENTRANCE_UPDATE_BATCH_SIZE = CommonVars("linkis.entrance.update.batch.size", 100) + + val ENTRANCE_SHUTDOWN_FAILOVER_ENABLED = CommonVars("linkis.entrance.shutdown.failover.enable", true).getValue + + val ENTRANCE_GROUP_SCAN_ENABLED = CommonVars("linkis.entrance.group.scan.enable", true) + } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala new file mode 100644 index 00000000000..34d3e3042c5 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.scheduler + +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.scheduler.SchedulerContext +import org.apache.linkis.scheduler.queue.Group +import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer + +import java.util.concurrent.ExecutorService + +class EntranceFIFOUserConsumer( + schedulerContext: SchedulerContext, + executeService: ExecutorService, + private var group: Group +) extends FIFOUserConsumer(schedulerContext, executeService, group) { + + override def loop(): Unit = { + schedulerContext match { + case entranceSchedulerContext: EntranceSchedulerContext => + if (entranceSchedulerContext.getOfflineFlag && EntranceConfiguration.ENTRANCE_FAILOVER_RETRY_JOB_ENABLED.getValue) { + scanAllRetryJobsAndRemove() + Utils.tryQuietly(Thread.sleep(5000)) + return + } + case _ => + } + + // general logic + super.loop() + + } + +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala index 7f16dd24630..a0a644e1d0f 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala @@ -31,6 +31,7 @@ import org.apache.linkis.governance.common.protocol.conf.{ import org.apache.linkis.instance.label.client.InstanceLabelClient import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext import org.apache.linkis.manager.label.constant.{LabelKeyConstant, LabelValueConstant} +import org.apache.linkis.governance.common.protocol.conf.{RequestQueryEngineConfigWithGlobalConfig, ResponseQueryConfig} import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.engine.{ ConcurrentEngineConnLabel, @@ -38,22 +39,25 @@ import org.apache.linkis.manager.label.entity.engine.{ UserCreatorLabel } import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.apache.linkis.manager.label.entity.engine.{ConcurrentEngineConnLabel, EngineTypeLabel, UserCreatorLabel} import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.protocol.constants.TaskConstant import org.apache.linkis.protocol.utils.TaskUtils import org.apache.linkis.rpc.Sender import org.apache.linkis.scheduler.queue.{Group, GroupFactory, SchedulerEvent} import org.apache.linkis.scheduler.queue.parallelqueue.ParallelGroup - import org.apache.commons.lang3.StringUtils import java.util import java.util.concurrent.TimeUnit import java.util.regex.Pattern - import scala.collection.JavaConverters._ - import com.google.common.cache.{Cache, CacheBuilder} +import org.apache.linkis.common.ServiceInstance +import org.apache.linkis.instance.label.client.InstanceLabelClient +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext +import org.apache.linkis.manager.label.constant.{LabelConstant, LabelKeyConstant} +import org.apache.linkis.manager.label.entity.route.RouteLabel class EntranceGroupFactory extends GroupFactory with Logging { @@ -73,6 +77,39 @@ class EntranceGroupFactory extends GroupFactory with Logging { private val GROUP_INIT_CAPACITY = CommonVars("wds.linkis.entrance.init.capacity", 100) + private val GROUP_SCAN_INIT_TIME = CommonVars("linkis.entrance.group.scan.init.time", 3 * 1000) + + private val GROUP_SCAN_INTERVAL = CommonVars("linkis.entrance.group.scan.interval", 60 * 1000) + + if (EntranceConfiguration.ENTRANCE_GROUP_SCAN_ENABLED.getValue) { + Utils.defaultScheduler.scheduleAtFixedRate( + new Runnable { + override def run(): Unit = { + // get all entrance server from eureka + val serviceInstances = Sender.getInstances(Sender.getThisServiceInstance.getApplicationName) + if (null == serviceInstances || serviceInstances.isEmpty) return + + // get all offline label server + val routeLabel = LabelBuilderFactoryContext.getLabelBuilderFactory + .createLabel[RouteLabel](LabelKeyConstant.ROUTE_KEY, LabelConstant.OFFLINE) + val labels = new util.ArrayList[Label[_]] + labels.add(routeLabel) + val labelInstances = InstanceLabelClient.getInstance.getInstanceFromLabel(labels) + + // get active entrance server + val allInstances = new util.ArrayList[ServiceInstance]() + allInstances.addAll(serviceInstances.toList.asJava) + allInstances.removeAll(labelInstances) + // refresh all group maxAllowRunningJobs + refreshAllGroupMaxAllowRunningJobs(allInstances.size()) + } + }, + GROUP_SCAN_INIT_TIME.getValue, + GROUP_SCAN_INTERVAL.getValue, + TimeUnit.MILLISECONDS + ) + } + private val specifiedUsernameRegexPattern: Pattern = if (StringUtils.isNotBlank(SPECIFIED_USERNAME_REGEX.getValue)) { Pattern.compile(SPECIFIED_USERNAME_REGEX.getValue) @@ -156,41 +193,22 @@ class EntranceGroupFactory extends GroupFactory with Logging { group } + def refreshAllGroupMaxAllowRunningJobs(activeCount: Int): Unit = { + if (activeCount <= 0) return + groupNameToGroups.asMap().asScala.foreach(item => { + item._2 match { + case group: ParallelGroup => + group.setMaxAllowRunningJobs(Math.round(group.getMaxRunningJobs / activeCount)) + case _ => + } + }) + } + private def getUserMaxRunningJobs(keyAndValue: util.Map[String, String]): Int = { - var userDefinedRunningJobs = EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue(keyAndValue) - var entranceNum = Sender.getInstances(Sender.getThisServiceInstance.getApplicationName).length - val labelList = new util.ArrayList[Label[_]]() - val offlineRouteLabel = LabelBuilderFactoryContext.getLabelBuilderFactory - .createLabel[RouteLabel](LabelKeyConstant.ROUTE_KEY, LabelValueConstant.OFFLINE_VALUE) - labelList.add(offlineRouteLabel) - var offlineIns: Array[ServiceInstance] = null - Utils.tryAndWarn { - offlineIns = InstanceLabelClient.getInstance - .getInstanceFromLabel(labelList) - .asScala - .filter(l => - null != l && l.getApplicationName - .equalsIgnoreCase(Sender.getThisServiceInstance.getApplicationName) - ) - .toArray - } - if (null != offlineIns) { - logger.info(s"There are ${offlineIns.length} offlining instance.") - entranceNum = entranceNum - offlineIns.length - } - /* - Sender.getInstances may get 0 instances due to cache in Sender. So this instance is the one instance. - */ - if (0 >= entranceNum) { - logger.error( - s"Got ${entranceNum} ${Sender.getThisServiceInstance.getApplicationName} instances." - ) - entranceNum = 1 - } Math.max( EntranceConfiguration.ENTRANCE_INSTANCE_MIN.getValue, - userDefinedRunningJobs / entranceNum - ); + EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue(keyAndValue) + ) } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala new file mode 100644 index 00000000000..91a7c4aaa67 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.scheduler + +import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer +import org.apache.linkis.scheduler.queue.parallelqueue.ParallelConsumerManager + +class EntranceParallelConsumerManager(maxParallelismUsers: Int, schedulerName: String) + extends ParallelConsumerManager(maxParallelismUsers, schedulerName){ + + override protected def createConsumer(groupName: String): FIFOUserConsumer = { + val group = getSchedulerContext.getOrCreateGroupFactory.getGroup(groupName) + new EntranceFIFOUserConsumer(getSchedulerContext, getOrCreateExecutorService, group) + } + +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceSchedulerContext.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceSchedulerContext.scala index d5de2cc2da6..1638b0fb1ce 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceSchedulerContext.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceSchedulerContext.scala @@ -28,6 +28,11 @@ class EntranceSchedulerContext extends SchedulerContext { private var consumerManager: ConsumerManager = _ private var executorManager: ExecutorManager = _ + private var offlineFlag: Boolean = false + + def setOfflineFlag(offlineFlag: Boolean): Unit = this.offlineFlag = offlineFlag + def getOfflineFlag: Boolean = this.offlineFlag + def this( groupFactory: GroupFactory, consumerManager: ConsumerManager, diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala index 080e6913e05..a5dbeaab396 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala @@ -30,15 +30,13 @@ import org.apache.linkis.protocol.constants.TaskConstant import org.apache.linkis.protocol.query.cache.{CacheTaskResult, RequestReadCache} import org.apache.linkis.rpc.Sender import org.apache.linkis.scheduler.queue.SchedulerEventState - import org.apache.commons.lang3.StringUtils import javax.servlet.http.HttpServletRequest - import java.util import java.util.Date - import scala.collection.JavaConverters._ +import sun.net.util.IPAddressUtil import com.google.common.net.InetAddresses @@ -124,11 +122,50 @@ object JobHistoryHelper extends Logging { } /** - * Batch update instances + * Get all consume queue task and batch update instances(获取所有消费队列中的任务进行批量更新) + * + * @param taskIdList + * @param retryWhenUpdateFail + */ + def updateAllConsumeQueueTask(taskIdList: List[java.lang.Long], retryWhenUpdateFail: Boolean = false): Unit = { + + if (taskIdList.isEmpty) return + + val updateTaskIds = new util.ArrayList[java.lang.Long]() + + if (EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue > 0 && + taskIdList.length > EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue) { + for (i <- 0 until EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue) { + updateTaskIds.add(taskIdList(i)) + } + } else { + updateTaskIds.addAll(taskIdList.asJava) + } + + try { + val successTaskIds = updateBatchInstances(updateTaskIds.asScala.toList) + if (retryWhenUpdateFail) { + taskIdList.asJava.removeAll(successTaskIds.asJava) + } else { + taskIdList.asJava.removeAll(updateTaskIds) + } + } catch { + case e: Exception => + logger.warn("update batch instances failed, wait for retry", e) + Thread.sleep(1000) + } + + updateAllConsumeQueueTask(taskIdList, retryWhenUpdateFail) + + } + + /** + * Batch update instances(批量更新instances字段) * * @param taskIdList + * @return */ - def updateBatchInstances(taskIdList: Array[java.lang.Long]): Unit = { + private def updateBatchInstances(taskIdList: List[java.lang.Long]): List[java.lang.Long] = { val jobReqList = new util.ArrayList[JobRequest]() taskIdList.foreach(taskID => { val jobRequest = new JobRequest @@ -137,7 +174,67 @@ object JobHistoryHelper extends Logging { jobReqList.add(jobRequest) }) val jobReqBatchUpdate = JobReqBatchUpdate(jobReqList) - sender.ask(jobReqBatchUpdate) + Utils.tryCatch { + val response = sender.ask(jobReqBatchUpdate) + response match { + case resp: util.ArrayList[JobRespProtocol] => + resp.asScala.filter(r => r.getStatus == SUCCESS_FLAG && r.getData.containsKey(JobRequestConstants.JOB_ID)) + .map(_.getData.get(JobRequestConstants.JOB_ID).asInstanceOf[java.lang.Long]).toList + case _ => + throw JobHistoryFailedException( + "update batch instances from jobhistory not a correct List type" + ) + } + } { + case errorException: ErrorException => throw errorException + case e: Exception => + val e1 = JobHistoryFailedException(s"update batch instances ${taskIdList.mkString(",")} error") + e1.initCause(e) + throw e + } + } + + /** + * query wait for failover task(获取待故障转移的任务) + * + * @param reqMap + * @param statusList + * @param startTimestamp + * @param limit + * @return + */ + def queryWaitForFailoverTask(reqMap: util.Map[String, java.lang.Long], statusList: util.List[String], startTimestamp: Long, limit: Int): util.List[JobRequest] = { + val requestFailoverJob = RequestFailoverJob(reqMap, statusList, startTimestamp, limit) + val tasks = Utils.tryCatch { + val response = sender.ask(requestFailoverJob) + response match { + case responsePersist: JobRespProtocol => + val status = responsePersist.getStatus + if (status != SUCCESS_FLAG) { + logger.error(s"query from jobHistory status failed, status is $status") + throw JobHistoryFailedException("query from jobHistory status failed") + } + val data = responsePersist.getData + data.get(JobRequestConstants.JOB_HISTORY_LIST) match { + case tasks: util.List[JobRequest] => + tasks + case _ => + throw JobHistoryFailedException( + s"query from jobhistory not a correct List type, instances ${reqMap.keySet()}" + ) + } + case _ => + logger.error("get query response incorrectly") + throw JobHistoryFailedException("get query response incorrectly") + } + } { + case errorException: ErrorException => throw errorException + case e: Exception => + val e1 = JobHistoryFailedException(s"query failover task error, instances ${reqMap.keySet()} ") + e1.initCause(e) + throw e + } + tasks } private def getTaskByTaskID(taskID: Long): JobRequest = { diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelConstant.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelConstant.java index 4db4bfca404..b43501ed9e4 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelConstant.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelConstant.java @@ -22,4 +22,6 @@ public class LabelConstant { public static final int LABEL_BUILDER_ERROR_CODE = 40001; public static final int LABEL_UTIL_CONVERT_ERROR_CODE = 40002; + + public static final String OFFLINE = "offline"; } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java index 1403a29ed0f..6568fb838bf 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java @@ -23,6 +23,7 @@ import java.util.Date; import java.util.List; +import java.util.Map; public interface JobHistoryMapper { @@ -105,4 +106,38 @@ Integer countUndoneTaskWithCreatorOnly( String selectJobHistoryStatusForUpdate(Long jobId); void updateOberverById(@Param("taskid") Long taskid, @Param("observeInfo") String observeInfo); + + /** + * query wait for failover job + * + * Sql example: + * SELECT a.* FROM linkis_ps_job_history_group_history a + * where (a.instances = '' + * or a.instances is null + * or a.instances not in ('192.168.1.123:9104','192.168.1.124:9104') + * or EXISTS ( + * select 1 from + * ( + * select '192.168.1.123:9104' as instances, 1697775054098 as registryTime + * union all + * select '192.168.1.124:9104' as instances, 1666239054098 as registryTime + * ) b + * where a.instances = b.instances and UNIX_TIMESTAMP(a.created_time) * 1000 < b.registryTime + * ) + * ) + * and + * status in ('Inited','Running','Scheduled','WaitForRetry') + * and UNIX_TIMESTAMP(a.created_time) * 1000 >= 1666239054098 + * limit 10 + * + * @param instancesMap + * @param statusList + * @param startTimestamp + * @param limit + * @return + */ + List selectFailoverJobHistory(@Param("instancesMap") Map instancesMap, + @Param("statusList") List statusList, + @Param("startTimestamp") Long startTimestamp, + @Param("limit") Integer limit); } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml index 8ac85a7c46e..b2fa7f95bad 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml @@ -221,4 +221,27 @@ update linkis_ps_job_history_group_history set observe_info = #{observeInfo} where id = #{taskid} + + diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java index b2387389075..ba92d37eccc 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java @@ -36,6 +36,8 @@ public interface JobHistoryQueryService { JobRespProtocol query(JobReqQuery jobReqQuery); + JobRespProtocol queryFailoverJobs(RequestFailoverJob requestFailoverJob); + JobHistory getJobHistoryByIdAndName(Long jobID, String userName); List search(Long jobId, String username, String creator, String status, Date sDate, Date eDate, String engineType, Long startJobId, String instance); From 00777c630c33ee89f1cbedcbea099ea93f010cc9 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Thu, 17 Nov 2022 16:21:02 +0800 Subject: [PATCH 008/689] Interface development of status,log,progress,kill --- .../entrance/restful/EntranceRestfulApi.java | 809 +++++++++++------- .../parser/EntranceRequestGatewayParser.scala | 6 +- 2 files changed, 485 insertions(+), 330 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index 324187fc288..4a946e6d0cf 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -35,6 +35,7 @@ import org.apache.linkis.scheduler.queue.Job; import org.apache.linkis.scheduler.queue.SchedulerEventState; import org.apache.linkis.server.Message; +import org.apache.linkis.server.conf.ServerConfiguration; import org.apache.linkis.server.security.SecurityFilter; import org.apache.linkis.server.utils.ModuleUserUtils; @@ -197,193 +198,277 @@ private void pushLog(String log, Job job) { entranceServer.getEntranceContext().getOrCreateLogManager().onLogUpdate(job, log); } - @ApiOperation(value = "status", notes = "get task stats", response = Message.class) - @ApiImplicitParams({ - @ApiImplicitParam(name = "taskID", required = false, dataType = "String", value = " task id"), - @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "execute id ") - }) - @Override - @RequestMapping(path = "/{id}/status", method = RequestMethod.GET) - public Message status( - HttpServletRequest req, - @PathVariable("id") String id, - @RequestParam(value = "taskID", required = false) String taskID) { - Message message = null; - String realId = ZuulEntranceUtils.parseExecID(id)[3]; - ModuleUserUtils.getOperationUser(req, "status realId: " + realId); - Option job = Option.apply(null); - try { - job = entranceServer.getJob(realId); - } catch (Exception e) { - logger.warn("获取任务 {} 状态时出现错误", realId, e.getMessage()); - long realTaskID = Long.parseLong(taskID); - String status = JobHistoryHelper.getStatusByTaskID(realTaskID); - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/status"); - message.data("status", status).data("execID", id); - return message; - } - if (job.isDefined()) { - if (job.get() instanceof EntranceJob) { - ((EntranceJob) job.get()).updateNewestAccessByClientTimestamp(); - } - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/status"); - message.data("status", job.get().getState().toString()).data("execID", id); - } else { - message = - Message.error( - "ID The corresponding job is empty and cannot obtain the corresponding task status.(ID 对应的job为空，不能获取相应的任务状态)"); - } - return message; - } + @ApiOperation(value = "status", notes = "get task stats", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "taskID", required = false, dataType = "String", value = " task id"), + @ApiImplicitParam(name = "id",required = true, dataType = "String", value = "execute id ") + }) + @Override + @RequestMapping(path = "/{id}/status", method = RequestMethod.GET) + public Message status( + HttpServletRequest req, + @PathVariable("id") String id, + @RequestParam(value = "taskID", required = false) String taskID) { + ModuleUserUtils.getOperationUser(req, "job status"); + Message message = null; + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; + } else { + // taskID + String jobReqId = req.getHeader(ServerConfiguration.LINKIS_SERVER_ENTRANCE_HEADER_KEY().toString()); + if (StringUtils.isEmpty(jobReqId)){ + logger.warn("The job wait failover, return status is Inited"); + String status = SchedulerEventState.Inited().toString(); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/status"); + message.data("status", status).data("execID", "").data("taskID", id); + return message; + } else { + realId = jobReqId; + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } + } - @ApiOperation(value = "progress", notes = "get task progress info", response = Message.class) - @ApiImplicitParams({ - @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "exectue id") - }) - @Override - @RequestMapping(path = "/{id}/progress", method = RequestMethod.GET) - public Message progress(HttpServletRequest req, @PathVariable("id") String id) { - Message message = null; - String realId = ZuulEntranceUtils.parseExecID(id)[3]; - ModuleUserUtils.getOperationUser(req, "progress realId: " + realId); - Option job = null; - try { - job = entranceServer.getJob(realId); - } catch (Exception e) { - logger.error(e.getMessage()); + Option job = Option.apply(null); + try { + job = entranceServer.getJob(realId); + } catch (Exception e) { + logger.warn("get {} status error", realId, e); + if (StringUtils.isEmpty(taskID)) { + message = + Message.error( + "Get job by ID error and cannot obtain the corresponding task status.(获取job时发生异常，不能获取相应的任务状态)"); + return message; + } + long realTaskID = Long.parseLong(taskID); + String status = JobHistoryHelper.getStatusByTaskID(realTaskID); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/status"); + message.data("status", status).data("execID", execID); + return message; + } + if (job.isDefined()) { + if (job.get() instanceof EntranceJob) { + ((EntranceJob) job.get()).updateNewestAccessByClientTimestamp(); + } + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/status"); + message.data("status", job.get().getState().toString()).data("execID", execID); + } else { + message = + Message.error( + "ID The corresponding job is empty and cannot obtain the corresponding task status.(ID 对应的job为空，不能获取相应的任务状态)"); + } + return message; } - if (job != null && job.isDefined()) { - JobProgressInfo[] jobProgressInfos = ((EntranceJob) job.get()).getProgressInfo(); - if (jobProgressInfos == null) { - message = - Message.error( - "Can not get the corresponding progress information, it may be that the corresponding progress information has not been generated(不能获取相应的进度信息,可能是相应的进度信息还未生成)"); - message.setMethod("/api/entrance/" + id + "/progress"); - } else { - List> list = new ArrayList<>(); - for (JobProgressInfo jobProgressInfo : jobProgressInfos) { - if ("true".equals(EntranceConfiguration.PROGRESS_PUSH().getValue()) - || jobProgressInfo.totalTasks() > 0) { - setJobProgressInfos(list, jobProgressInfo); - } + + @ApiOperation(value = "progress", notes = "get task progress info", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "id",required = true, dataType = "String", value = "exectue id") + }) + @Override + @RequestMapping(path = "/{id}/progress", method = RequestMethod.GET) + public Message progress(HttpServletRequest req, @PathVariable("id") String id) { + ModuleUserUtils.getOperationUser(req, "job progress"); + Message message = null; + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; + } else { + // taskID + String jobReqId = req.getHeader(ServerConfiguration.LINKIS_SERVER_ENTRANCE_HEADER_KEY().toString()); + if (StringUtils.isEmpty(jobReqId)){ + logger.warn("The job wait failover, return progress is 0"); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/progress"); + message.data("progress", 0) + .data("execID", "") + .data("taskID", id) + .data("progressInfo", new ArrayList<>()); + return message; + } else { + realId = jobReqId; + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } } - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/progress"); - message - .data("progress", Math.abs(job.get().getProgress())) - .data("execID", id) - .data("progressInfo", list); - } - } else { - message = - Message.error( - "The job corresponding to the ID is empty, and the corresponding task progress cannot be obtained.(ID 对应的job为空，不能获取相应的任务进度)"); - } - return message; - } + Option job = null; + try { + job = entranceServer.getJob(realId); + } catch (Exception e) { + logger.error(e.getMessage()); + } + if (job != null && job.isDefined()) { + JobProgressInfo[] jobProgressInfos = ((EntranceJob) job.get()).getProgressInfo(); + if (jobProgressInfos == null) { + message = + Message.error( + "Can not get the corresponding progress information, it may be that the corresponding progress information has not been generated(不能获取相应的进度信息,可能是相应的进度信息还未生成)"); + message.setMethod("/api/entrance/" + id + "/progress"); + } else { + List> list = new ArrayList<>(); + for (JobProgressInfo jobProgressInfo : jobProgressInfos) { + if ("true".equals(EntranceConfiguration.PROGRESS_PUSH().getValue()) + || jobProgressInfo.totalTasks() > 0) { + setJobProgressInfos(list, jobProgressInfo); + } + } + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/progress"); - @ApiOperation( - value = "progressWithResource", - notes = "get progress and resource info", - response = Message.class) - @ApiImplicitParams({ - @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "execute id") - }) - @Override - @RequestMapping(path = "/{id}/progressWithResource", method = RequestMethod.GET) - public Message progressWithResource(HttpServletRequest req, @PathVariable("id") String id) { - Message message = null; - String realId = ZuulEntranceUtils.parseExecID(id)[3]; - ModuleUserUtils.getOperationUser(req, "progressWithResource realId: " + realId); - Option job = null; - try { - job = entranceServer.getJob(realId); - } catch (Exception e) { - logger.error(e.getMessage()); + message.data("progress", Math.abs(job.get().getProgress())) + .data("execID", execID) + .data("progressInfo", list); + } + } else { + message = + Message.error( + "The job corresponding to the ID is empty, and the corresponding task progress cannot be obtained.(ID 对应的job为空，不能获取相应的任务进度)"); + } + return message; } - if (job != null && job.isDefined()) { - JobProgressInfo[] jobProgressInfos = ((EntranceJob) job.get()).getProgressInfo(); - if (jobProgressInfos == null) { - message = - Message.error( - "Can not get the corresponding progress information, it may be that the corresponding progress information has not been generated(不能获取相应的进度信息,可能是相应的进度信息还未生成)"); - message.setMethod("/api/entrance/" + id + "/progressWithResource"); - } else { - List> list = new ArrayList<>(); - for (JobProgressInfo jobProgressInfo : jobProgressInfos) { - if ("true".equals(EntranceConfiguration.PROGRESS_PUSH().getValue()) - || jobProgressInfo.totalTasks() > 0) { - setJobProgressInfos(list, jobProgressInfo); - } + + @ApiOperation(value = "progressWithResource", notes = "get progress and resource info", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "execute id") + }) + @Override + @RequestMapping(path = "/{id}/progressWithResource", method = RequestMethod.GET) + public Message progressWithResource(HttpServletRequest req, @PathVariable("id") String id) { + ModuleUserUtils.getOperationUser(req, "job progressWithResource"); + Message message = null; + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; + } else { + // taskID + String jobReqId = req.getHeader(ServerConfiguration.LINKIS_SERVER_ENTRANCE_HEADER_KEY().toString()); + if (StringUtils.isEmpty(jobReqId)){ + logger.warn("The job wait failover, return progress is 0 and resource is null"); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/progressWithResource"); + message.data(TaskConstant.ENTRANCEJOB_YARNRESOURCE, null) + .data("progress", 0) + .data("execID", "") + .data("taskID", id) + .data("progressInfo", new ArrayList<>()); + return message; + } else { + realId = jobReqId; + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } + } + Option job = null; + try { + job = entranceServer.getJob(realId); + } catch (Exception e) { + logger.error(e.getMessage()); } - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/progressWithResource"); - - JobRequest jobRequest = ((EntranceJob) job.get()).getJobRequest(); - Map metrics = jobRequest.getMetrics(); - Map metricsVo = new HashMap<>(); - if (metrics.containsKey(TaskConstant.ENTRANCEJOB_YARNRESOURCE)) { - HashMap resourceMap = - (HashMap) - metrics.get(TaskConstant.ENTRANCEJOB_YARNRESOURCE); - ArrayList resoureList = new ArrayList<>(12); - if (null != resourceMap && !resourceMap.isEmpty()) { - resourceMap.forEach( - (applicationId, resource) -> { - resoureList.add(new YarnResourceWithStatusVo(applicationId, resource)); - }); - metricsVo.put(TaskConstant.ENTRANCEJOB_YARNRESOURCE, resoureList); - Optional cores = - resourceMap.values().stream() - .map(resource -> resource.queueCores()) - .reduce((x, y) -> x + y); - Optional memory = - resourceMap.values().stream() - .map(resource -> resource.queueMemory()) - .reduce((x, y) -> x + y); - float corePercent = 0.0f; - float memoryPercent = 0.0f; - if (cores.isPresent() && memory.isPresent()) { - corePercent = - cores.get().floatValue() - / EntranceConfiguration.YARN_QUEUE_CORES_MAX().getHotValue(); - memoryPercent = - memory.get().floatValue() - / (EntranceConfiguration.YARN_QUEUE_MEMORY_MAX().getHotValue().longValue() - * 1024 - * 1024 - * 1024); + if (job != null && job.isDefined()) { + JobProgressInfo[] jobProgressInfos = ((EntranceJob) job.get()).getProgressInfo(); + if (jobProgressInfos == null) { + message = + Message.error( + "Can not get the corresponding progress information, it may be that the corresponding progress information has not been generated(不能获取相应的进度信息,可能是相应的进度信息还未生成)"); + message.setMethod("/api/entrance/" + id + "/progressWithResource"); + } else { + List> list = new ArrayList<>(); + for (JobProgressInfo jobProgressInfo : jobProgressInfos) { + if ("true".equals(EntranceConfiguration.PROGRESS_PUSH().getValue()) + || jobProgressInfo.totalTasks() > 0) { + setJobProgressInfos(list, jobProgressInfo); + } + } + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/progressWithResource"); + + JobRequest jobRequest = ((EntranceJob) job.get()).getJobRequest(); + Map metrics = jobRequest.getMetrics(); + Map metricsVo = new HashMap<>(); + if (metrics.containsKey(TaskConstant.ENTRANCEJOB_YARNRESOURCE)) { + HashMap resourceMap = + (HashMap) + metrics.get(TaskConstant.ENTRANCEJOB_YARNRESOURCE); + ArrayList resoureList = new ArrayList<>(12); + if (null != resourceMap && !resourceMap.isEmpty()) { + resourceMap.forEach( + (applicationId, resource) -> { + resoureList.add( + new YarnResourceWithStatusVo(applicationId, resource)); + }); + metricsVo.put(TaskConstant.ENTRANCEJOB_YARNRESOURCE, resoureList); + Optional cores = + resourceMap.values().stream() + .map(resource -> resource.queueCores()) + .reduce((x, y) -> x + y); + Optional memory = + resourceMap.values().stream() + .map(resource -> resource.queueMemory()) + .reduce((x, y) -> x + y); + float corePercent = 0.0f; + float memoryPercent = 0.0f; + if (cores.isPresent() && memory.isPresent()) { + corePercent = + cores.get().floatValue() + / EntranceConfiguration.YARN_QUEUE_CORES_MAX() + .getValue(); + memoryPercent = + memory.get().floatValue() + / (EntranceConfiguration.YARN_QUEUE_MEMORY_MAX() + .getValue() + .longValue() + * 1024 + * 1024 + * 1024); + } + String coreRGB = RGBUtils.getRGB(corePercent); + String memoryRGB = RGBUtils.getRGB(memoryPercent); + metricsVo.put(TaskConstant.ENTRANCEJOB_CORE_PERCENT, corePercent); + metricsVo.put(TaskConstant.ENTRANCEJOB_MEMORY_PERCENT, memoryPercent); + metricsVo.put(TaskConstant.ENTRANCEJOB_CORE_RGB, coreRGB); + metricsVo.put(TaskConstant.ENTRANCEJOB_MEMORY_RGB, memoryRGB); + + message.data(TaskConstant.ENTRANCEJOB_YARN_METRICS, metricsVo); + } else { + message.data(TaskConstant.ENTRANCEJOB_YARNRESOURCE, null); + } + } else { + message.data(TaskConstant.ENTRANCEJOB_YARNRESOURCE, null); + } + + message.data("progress", Math.abs(job.get().getProgress())) + .data("execID", execID) + .data("progressInfo", list); } - String coreRGB = RGBUtils.getRGB(corePercent); - String memoryRGB = RGBUtils.getRGB(memoryPercent); - metricsVo.put(TaskConstant.ENTRANCEJOB_CORE_PERCENT, corePercent); - metricsVo.put(TaskConstant.ENTRANCEJOB_MEMORY_PERCENT, memoryPercent); - metricsVo.put(TaskConstant.ENTRANCEJOB_CORE_RGB, coreRGB); - metricsVo.put(TaskConstant.ENTRANCEJOB_MEMORY_RGB, memoryRGB); - - message.data(TaskConstant.ENTRANCEJOB_YARN_METRICS, metricsVo); - } else { - message.data(TaskConstant.ENTRANCEJOB_YARNRESOURCE, null); - } } else { - message.data(TaskConstant.ENTRANCEJOB_YARNRESOURCE, null); + message = + Message.error( + "The job corresponding to the ID is empty, and the corresponding task progress cannot be obtained.(ID 对应的job为空，不能获取相应的任务进度)"); } - - message - .data("progress", Math.abs(job.get().getProgress())) - .data("execID", id) - .data("progressInfo", list); - } - } else { - message = - Message.error( - "The job corresponding to the ID is empty, and the corresponding task progress cannot be obtained.(ID 对应的job为空，不能获取相应的任务进度)"); + return message; } - return message; - } private void setJobProgressInfos( List> list, JobProgressInfo jobProgressInfo) { @@ -396,108 +481,146 @@ private void setJobProgressInfos( list.add(map); } - @ApiOperation(value = "log", notes = "get task log", response = Message.class) - @ApiImplicitParams({ - @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "execute id") - }) - @Override - @RequestMapping(path = "/{id}/log", method = RequestMethod.GET) - public Message log(HttpServletRequest req, @PathVariable("id") String id) { - String realId = ZuulEntranceUtils.parseExecID(id)[3]; - ModuleUserUtils.getOperationUser(req, "log realId: " + realId); - Option job = Option.apply(null); - Message message = null; - try { - job = entranceServer.getJob(realId); - } catch (final Throwable t) { - message = - Message.error( - "The job you just executed has ended. This interface no longer provides a query. It is recommended that you download the log file for viewing.(您刚刚执行的job已经结束，本接口不再提供查询，建议您下载日志文件进行查看)"); - message.setMethod("/api/entrance/" + id + "/log"); - return message; - } - if (job.isDefined()) { - logger.debug("begin to get log for {}(开始获取 {} 的日志)", job.get().getId(), job.get().getId()); - LogReader logReader = - entranceServer.getEntranceContext().getOrCreateLogManager().getLogReader(realId); - int fromLine = 0; - int size = 100; - boolean distinctLevel = true; - if (req != null) { + @ApiOperation(value = "log", notes = "get task log", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "execute id") + }) + @Override + @RequestMapping(path = "/{id}/log", method = RequestMethod.GET) + public Message log(HttpServletRequest req, @PathVariable("id") String id) { + ModuleUserUtils.getOperationUser(req, "get job log"); + Message message = null; + int fromLine = 0; + int size = 100; + boolean distinctLevel = true; String fromLineStr = req.getParameter("fromLine"); String sizeStr = req.getParameter("size"); if (StringUtils.isNotBlank(fromLineStr)) { - fromLine = Math.max(Integer.parseInt(fromLineStr), 0); + fromLine = Math.max(Integer.parseInt(fromLineStr), 0); } if (StringUtils.isNotBlank(sizeStr)) { - size = Integer.parseInt(sizeStr) >= 0 ? Integer.parseInt(sizeStr) : 10000; + size = Integer.parseInt(sizeStr) >= 0 ? Integer.parseInt(sizeStr) : 10000; } String distinctLevelStr = req.getParameter("distinctLevel"); if ("false".equals(distinctLevelStr)) { - distinctLevel = false; + distinctLevel = false; } - } - Object retLog = null; - int retFromLine = 0; - try { - if (distinctLevel) { - String[] logs = new String[4]; - retFromLine = logReader.readArray(logs, fromLine, size); - retLog = new ArrayList(Arrays.asList(logs)); + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; } else { - StringBuilder sb = new StringBuilder(); - retFromLine = logReader.read(sb, fromLine, size); - retLog = sb.toString(); + // taskID + String jobReqId = req.getHeader(ServerConfiguration.LINKIS_SERVER_ENTRANCE_HEADER_KEY().toString()); + if (StringUtils.isEmpty(jobReqId)){ + logger.warn("The job wait failover, return customer log"); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/log"); + String log = LogUtils.generateInfo("The job will failover soon, please try again later"); + Object retLog; + if (distinctLevel) { + String[] array = new String[4]; + array[2] = log; + array[3] = log; + retLog = new ArrayList(Arrays.asList(array)); + } else { + retLog = log; + } + message.data("log", retLog).data("execID", "").data("taskID", id).data("fromLine", 0); + return message; + } else { + realId = jobReqId; + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } } - } catch (IllegalStateException e) { - logger.debug( - "Failed to get log information for :{}(为 {} 获取日志失败)", - job.get().getId(), - job.get().getId(), - e); - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/log"); - message.data("log", "").data("execID", id).data("fromLine", retFromLine + fromLine); - } catch (final IllegalArgumentException e) { - logger.debug( - "Failed to get log information for :{}(为 {} 获取日志失败)", - job.get().getId(), - job.get().getId(), - e); - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/log"); - message.data("log", "").data("execID", id).data("fromLine", retFromLine + fromLine); - return message; - } catch (final Exception e1) { - logger.debug( - "Failed to get log information for :{}(为 {} 获取日志失败)", - job.get().getId(), - job.get().getId(), - e1); - message = Message.error("Failed to get log information(获取日志信息失败)"); - message.setMethod("/api/entrance/" + id + "/log"); - message.data("log", "").data("execID", id).data("fromLine", retFromLine + fromLine); - return message; - } finally { - if (null != logReader && job.get().isCompleted()) { - IOUtils.closeQuietly(logReader); + + Option job = Option.apply(null); + try { + job = entranceServer.getJob(realId); + } catch (final Throwable t) { + message = + Message.error( + "The job you just executed has ended. This interface no longer provides a query. It is recommended that you download the log file for viewing.(您刚刚执行的job已经结束，本接口不再提供查询，建议您下载日志文件进行查看)"); + message.setMethod("/api/entrance/" + id + "/log"); + return message; } - } - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/log"); - message.data("log", retLog).data("execID", id).data("fromLine", retFromLine + fromLine); - logger.debug("success to get log for {} (获取 {} 日志成功)", job.get().getId(), job.get().getId()); - } else { - message = - Message.error( - "Can't find execID(不能找到execID): " - + id - + "Corresponding job, can not get the corresponding log(对应的job，不能获得对应的日志)"); - message.setMethod("/api/entrance/" + id + "/log"); + if (job.isDefined()) { + logger.debug( + "begin to get log for {}(开始获取 {} 的日志)", job.get().getId(), job.get().getId()); + LogReader logReader = + entranceServer + .getEntranceContext() + .getOrCreateLogManager() + .getLogReader(realId); + + Object retLog = null; + int retFromLine = 0; + try { + if (distinctLevel) { + String[] logs = new String[4]; + retFromLine = logReader.readArray(logs, fromLine, size); + retLog = new ArrayList(Arrays.asList(logs)); + } else { + StringBuilder sb = new StringBuilder(); + retFromLine = logReader.read(sb, fromLine, size); + retLog = sb.toString(); + } + } catch (IllegalStateException e) { + logger.debug( + "Failed to get log information for :{}(为 {} 获取日志失败)", + job.get().getId(), + job.get().getId(), + e); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/log"); + message.data("log", "").data("execID", execID).data("fromLine", retFromLine + fromLine); + } catch (final IllegalArgumentException e) { + logger.debug( + "Failed to get log information for :{}(为 {} 获取日志失败)", + job.get().getId(), + job.get().getId(), + e); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/log"); + message.data("log", "").data("execID", execID).data("fromLine", retFromLine + fromLine); + return message; + } catch (final Exception e1) { + logger.debug( + "Failed to get log information for :{}(为 {} 获取日志失败)", + job.get().getId(), + job.get().getId(), + e1); + message = Message.error("Failed to get log information(获取日志信息失败)"); + message.setMethod("/api/entrance/" + id + "/log"); + message.data("log", "").data("execID", execID).data("fromLine", retFromLine + fromLine); + return message; + } finally { + if (null != logReader && job.get().isCompleted()) { + IOUtils.closeQuietly(logReader); + } + } + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/log"); + message.data("log", retLog).data("execID", execID).data("fromLine", retFromLine + fromLine); + logger.debug( + "success to get log for {} (获取 {} 日志成功)", job.get().getId(), job.get().getId()); + } else { + message = + Message.error( + "Can't find execID(不能找到execID): " + + id + + "Corresponding job, can not get the corresponding log(对应的job，不能获得对应的日志)"); + message.setMethod("/api/entrance/" + id + "/log"); + } + return message; } - return message; - } @ApiOperation(value = "killJobs", notes = "kill jobs", response = Message.class) @ApiImplicitParams({ @@ -595,71 +718,103 @@ public Message killJobs( return Message.ok("success").data("messages", messages); } - @ApiOperation(value = "kill", notes = "kill", response = Message.class) - @ApiImplicitParams({ - @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "excute id"), - @ApiImplicitParam(name = "taskID", required = false, dataType = "String", value = "task id") - }) - @Override - @RequestMapping(path = "/{id}/kill", method = RequestMethod.GET) - public Message kill( - HttpServletRequest req, - @PathVariable("id") String id, - @RequestParam(value = "taskID", required = false) Long taskID) { - String realId = ZuulEntranceUtils.parseExecID(id)[3]; - ModuleUserUtils.getOperationUser(req, "kill realId:" + realId); - Option job = Option.apply(null); - try { - job = entranceServer.getJob(realId); - } catch (Exception e) { - logger.warn("can not find a job in entranceServer, will force to kill it", e); - // 如果在内存中找不到该任务，那么该任务可能已经完成了，或者就是重启导致的 - JobHistoryHelper.forceKill(taskID); - Message message = Message.ok("Forced Kill task (强制杀死任务)"); - message.setMethod("/api/entrance/" + id + "/kill"); - message.setStatus(0); - return message; - } - Message message = null; - if (job.isEmpty()) { - logger.warn("can not find a job in entranceServer, will force to kill it"); - // 如果在内存中找不到该任务，那么该任务可能已经完成了，或者就是重启导致的 - JobHistoryHelper.forceKill(taskID); - message = Message.ok("Forced Kill task (强制杀死任务)"); - message.setMethod("/api/entrance/" + id + "/kill"); - message.setStatus(0); - return message; - } else { - try { - logger.info("begin to kill job {} ", job.get().getId()); - job.get().kill(); - message = Message.ok("Successfully killed the job(成功kill了job)"); - message.setMethod("/api/entrance/" + id + "/kill"); - message.setStatus(0); - message.data("execID", id); - // ensure the job's state is cancelled in database - if (job.get() instanceof EntranceJob) { - EntranceJob entranceJob = (EntranceJob) job.get(); - JobRequest jobReq = entranceJob.getJobRequest(); - entranceJob.updateJobRequestStatus(SchedulerEventState.Cancelled().toString()); - this.entranceServer - .getEntranceContext() - .getOrCreatePersistenceManager() - .createPersistenceEngine() - .updateIfNeeded(jobReq); + @ApiOperation(value = "kill", notes = "kill", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "exec id"), + @ApiImplicitParam(name = "taskID", required = false, dataType = "String", value = "task id") + }) + @Override + @RequestMapping(path = "/{id}/kill", method = RequestMethod.GET) + public Message kill( + HttpServletRequest req, + @PathVariable("id") String id, + @RequestParam(value = "taskID", required = false) Long taskID) { + ModuleUserUtils.getOperationUser(req, "kill job"); + Message message = null; + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; + } else { + // taskID + String jobReqId = req.getHeader(ServerConfiguration.LINKIS_SERVER_ENTRANCE_HEADER_KEY().toString()); + if (StringUtils.isEmpty(jobReqId)){ + logger.warn("The job wait failover, but now force kill"); + // TODO If failover occurs during force kill, the job status may change from Cancelled to Running + long taskId = Long.parseLong(id); + JobHistoryHelper.forceKill(taskId); + message = Message.ok("Forced Kill task (强制杀死任务)"); + message.setMethod("/api/entrance/" + id + "/kill"); + message.data("execID", "").data("taskID", id); + return message; + } else { + realId = jobReqId; + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } } - logger.info("end to kill job {} ", job.get().getId()); - } catch (Throwable t) { - logger.error("kill job {} failed ", job.get().getId(), t); - message = - Message.error( - "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)"); - message.setMethod("/api/entrance/" + id + "/kill"); - message.setStatus(1); - } + + Option job = Option.apply(null); + try { + job = entranceServer.getJob(realId); + } catch (Exception e) { + logger.warn("can not find a job in entranceServer, will force to kill it", e); + // 如果在内存中找不到该任务，那么该任务可能已经完成了，或者就是重启导致的 + if (taskID == null || taskID <= 0) { + message = + Message.error( + "Get job by ID error, kill failed.(获取job时发生异常，kill失败)"); + return message; + } + JobHistoryHelper.forceKill(taskID); + message = Message.ok("Forced Kill task (强制杀死任务)"); + message.setMethod("/api/entrance/" + id + "/kill"); + message.setStatus(0); + return message; + } + + if (job.isEmpty()) { + logger.warn("can not find a job in entranceServer, will force to kill it"); + // 如果在内存中找不到该任务，那么该任务可能已经完成了，或者就是重启导致的 + JobHistoryHelper.forceKill(taskID); + message = Message.ok("Forced Kill task (强制杀死任务)"); + message.setMethod("/api/entrance/" + id + "/kill"); + message.setStatus(0); + return message; + } else { + try { + logger.info("begin to kill job {} ", job.get().getId()); + job.get().kill(); + message = Message.ok("Successfully killed the job(成功kill了job)"); + message.setMethod("/api/entrance/" + id + "/kill"); + message.data("execID", execID); + // ensure the job's state is cancelled in database + if (job.get() instanceof EntranceJob) { + EntranceJob entranceJob = (EntranceJob) job.get(); + JobRequest jobReq = entranceJob.getJobRequest(); + entranceJob.updateJobRequestStatus(SchedulerEventState.Cancelled().toString()); + this.entranceServer + .getEntranceContext() + .getOrCreatePersistenceManager() + .createPersistenceEngine() + .updateIfNeeded(jobReq); + } + logger.info("end to kill job {} ", job.get().getId()); + } catch (Throwable t) { + logger.error("kill job {} failed ", job.get().getId(), t); + message = + Message.error( + "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)"); + message.setMethod("/api/entrance/" + id + "/kill"); + } + } + return message; } - return message; - } @ApiOperation(value = "pause ", notes = "puase a task job", response = Message.class) @ApiImplicitParams({ diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala index 3eacb8c2b6c..9fb3958ac0c 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala @@ -65,8 +65,8 @@ class EntranceRequestGatewayParser extends AbstractGatewayParser { // parse by execId ZuulEntranceUtils.parseServiceInstanceByExecID(execId)(0) } else { - // parse by taskId - val jobHistory = parseJobHistoryByTaskID(execId.toLong, gatewayContext) + // check by taskId + val jobHistory = checkJobValidityByTaskID(execId.toLong, gatewayContext) // add header val jobReqId = if (jobHistory == null) "" else jobHistory.getJobReqId gatewayContext.getRequest.addHeader(ServerConfiguration.LINKIS_SERVER_HEADER_KEY.getValue, Array(jobReqId)) @@ -78,7 +78,7 @@ class EntranceRequestGatewayParser extends AbstractGatewayParser { case _ => } - def parseJobHistoryByTaskID(taskId: Long, gatewayContext: GatewayContext): JobHistory = { + def checkJobValidityByTaskID(taskId: Long, gatewayContext: GatewayContext): JobHistory = { val histories = jobHistoryQueryService.search(taskId, null, null, null, null, null, null, null) if (histories.isEmpty) { sendErrorResponse(s"taskId $taskId is not exists.", gatewayContext) From 8c5774c02e8eb66971878b009e5d7af939eabd0a Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Tue, 22 Nov 2022 11:31:29 +0800 Subject: [PATCH 009/689] failover and kill ec --- .../protocol/constants/TaskConstant.java | 1 + .../server/EntranceFailoverJobServer.java | 35 +++++--- .../linkis/entrance/EntranceServer.scala | 89 ++++++++++++++++++- .../entrance/utils/JobHistoryHelper.scala | 27 ++++-- 4 files changed, 130 insertions(+), 22 deletions(-) diff --git a/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java b/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java index 8f5a6800899..ea4a30a0b7b 100644 --- a/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java +++ b/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java @@ -66,6 +66,7 @@ public interface TaskConstant { String TICKET_ID = "ticketId"; String ENGINE_CONN_TASK_ID = "engineConnTaskId"; String ENGINE_CONN_SUBMIT_TIME = "engineConnSubmitTime"; + String FAILOVER_FLAG = "failoverFlag"; String PARAMS_DATA_SOURCE = "dataSources"; diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java index 7e7e0de69cf..175da3be415 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java @@ -91,23 +91,34 @@ public void run() { .collect(Collectors.toMap(ServiceInstance::getInstance, ServiceInstance::getRegistryTimestamp, (k1, k2) -> k2)); if (serverInstanceMap.isEmpty()) return; - // get failover start time - long startTimestamp = 0L; - if (EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME() > 0) { - startTimestamp = System.currentTimeMillis() - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME(); - } + // get failover job expired time (获取任务故障转移过期时间，配置为0表示不过期, 过期则不处理) + long expiredTimestamp = 0L; + if (EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME() > 0) { + expiredTimestamp = + System.currentTimeMillis() + - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME(); + } // get uncompleted status List statusList = Lists.newArrayList(); SchedulerEventState.values().filterNot(SchedulerEventState::isCompleted).foreach(state -> statusList.add(state.toString())); - List jobRequests = JobHistoryHelper.queryWaitForFailoverTask(serverInstanceMap, statusList, startTimestamp, EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); - if (jobRequests.isEmpty()) return; - logger.info("success query failover jobs , job ids: {}", jobRequests.stream().map(JobRequest::getId)); - - // failover to local server - jobRequests.forEach(jobRequest -> entranceServer.failoverExecute(jobRequest)); - logger.info("success execute failover jobs, job ids: {}", jobRequests.stream().map(JobRequest::getId)); + List jobRequests = + JobHistoryHelper.queryWaitForFailoverTask( + serverInstanceMap, + statusList, + expiredTimestamp, + EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); + if (jobRequests.isEmpty()) return; + logger.info( + "success query failover jobs , job ids: {}", + jobRequests.stream().map(JobRequest::getId)); + + // failover to local server + jobRequests.forEach(jobRequest -> entranceServer.failoverExecute(jobRequest)); + logger.info( + "success execute failover jobs, job ids: {}", + jobRequests.stream().map(JobRequest::getId)); } catch (Exception e) { logger.error("failover failed", e); diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index e91cfb3df61..ec8692c84eb 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -17,6 +17,7 @@ package org.apache.linkis.entrance +import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.exception.{ErrorException, LinkisException, LinkisRuntimeException} import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{Logging, Utils} @@ -27,9 +28,14 @@ import org.apache.linkis.entrance.execute.EntranceJob import org.apache.linkis.entrance.log.LogReader import org.apache.linkis.entrance.timeout.JobTimeoutManager import org.apache.linkis.entrance.utils.JobHistoryHelper +import org.apache.linkis.governance.common.conf.GovernanceCommonConf import org.apache.linkis.governance.common.entity.job.JobRequest +import org.apache.linkis.governance.common.protocol.task.RequestTaskKill +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest +import org.apache.linkis.manager.label.entity.entrance.ExecuteOnceLabel import org.apache.linkis.protocol.constants.TaskConstant import org.apache.linkis.rpc.Sender +import org.apache.linkis.rpc.conf.RPCConfiguration import org.apache.linkis.scheduler.queue.{Job, SchedulerEventState} import org.apache.linkis.server.conf.ServerConfiguration @@ -41,6 +47,8 @@ import java.text.MessageFormat import java.{lang, util} import java.util.Date +import scala.collection.JavaConverters._ + abstract class EntranceServer extends Logging { private var entranceWebSocketService: Option[EntranceWebSocketService] = None @@ -263,6 +271,68 @@ abstract class EntranceServer extends Logging { logger.info("Finished to clean all ConsumeQueue") } + def killEC(jobRequest: JobRequest): Unit = { + Utils.tryCatch { + if ( + !SchedulerEventState.isRunning(SchedulerEventState.withName(jobRequest.getStatus)) + || !SchedulerEventState.isScheduled(SchedulerEventState.withName(jobRequest.getStatus)) + || jobRequest.getMetrics == null + || !jobRequest.getMetrics.containsKey(TaskConstant.ENTRANCEJOB_ENGINECONN_MAP) + ) { + logger.info( + s"job ${jobRequest.getId} is not running,scheduled or not have EC info, ignore it" + ) + } + + val engineMap = jobRequest.getMetrics + .get(TaskConstant.ENTRANCEJOB_ENGINECONN_MAP) + .asInstanceOf[util.Map[String, Object]] + + val engineInstance = + engineMap.asScala + .map(_._2.asInstanceOf[util.Map[String, Object]]) + .filter(_.containsKey(TaskConstant.ENGINE_INSTANCE)) + .maxBy(_.getOrDefault(TaskConstant.ENGINE_CONN_SUBMIT_TIME, 0).toString) + + if (engineInstance != null || engineInstance.containsKey(TaskConstant.FAILOVER_FLAG)) { + logger.info( + s"job ${jobRequest.getId} do not submit to EC or already failover, not need kill ec" + ) + return + } + engineInstance.put(TaskConstant.FAILOVER_FLAG, "") + + val ecInstance = ServiceInstance( + GovernanceCommonConf.ENGINE_CONN_SPRING_NAME.getValue, + engineInstance.get(TaskConstant.ENGINE_INSTANCE).toString + ) + if (jobRequest.getLabels.asScala.exists(_.isInstanceOf[ExecuteOnceLabel])) { + // kill ec by linkismanager + val engineStopRequest = new EngineStopRequest + engineStopRequest.setServiceInstance(ecInstance) + // send to linkismanager + Sender + .getSender(RPCConfiguration.LINKIS_MANAGER_APPLICATION_NAME.getValue) + .send(engineStopRequest) + logger.info( + s"job ${jobRequest.getId} send EngineStopRequest to linkismanager, kill instance $ecInstance" + ) + } else if (engineInstance.containsKey(TaskConstant.ENGINE_CONN_TASK_ID)) { + // kill ec task + val engineTaskId = engineInstance.get(TaskConstant.ENGINE_CONN_TASK_ID).toString + // send to ec + Sender + .getSender(ecInstance) + .send(RequestTaskKill(engineTaskId)) + logger.info( + s"job ${jobRequest.getId} send RequestTaskKill to kill engineConn $ecInstance, execID $engineTaskId" + ) + } + } { case e: Exception => + logger.error(s"job ${jobRequest.getId} kill ec error", e) + } + } + /** * execute failover job (提交故障转移任务，返回新的execId) * @@ -277,7 +347,8 @@ abstract class EntranceServer extends Logging { ) } - // todo dmp kill ec + // try to kill ec + killEC(jobRequest); val logAppender = new java.lang.StringBuilder() // init properties @@ -376,6 +447,18 @@ abstract class EntranceServer extends Logging { LogUtils.generateInfo(s"the job_req_id ${jobRequest.getReqId} -> $initReqId \n") ) + val metricMap = new util.HashMap[String, Object]() + if ( + jobRequest.getMetrics != null && jobRequest.getMetrics.containsKey( + TaskConstant.ENTRANCEJOB_ENGINECONN_MAP + ) + ) { + val oldEngineconnMap = jobRequest.getMetrics + .get(TaskConstant.ENTRANCEJOB_ENGINECONN_MAP) + .asInstanceOf[util.Map[String, Object]] + metricMap.put(TaskConstant.ENTRANCEJOB_ENGINECONN_MAP, oldEngineconnMap) + } + jobRequest.setInstances(initInstance) jobRequest.setCreatedTime(initDate) jobRequest.setStatus(initStatus) @@ -383,8 +466,8 @@ abstract class EntranceServer extends Logging { jobRequest.setReqId(initReqId) jobRequest.setErrorCode(0) jobRequest.setErrorDesc("") - jobRequest.setMetrics(new util.HashMap[String, Object]()) - jobRequest.getMetrics.put(TaskConstant.ENTRANCEJOB_SUBMIT_TIME, initInstance) + jobRequest.setMetrics(metricMap) + jobRequest.getMetrics.put(TaskConstant.ENTRANCEJOB_SUBMIT_TIME, initDate) logAppender.append( LogUtils.generateInfo(s"Job ${jobRequest.getId} success to initialize the properties \n") diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala index a5dbeaab396..6416fe1d470 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala @@ -133,8 +133,10 @@ object JobHistoryHelper extends Logging { val updateTaskIds = new util.ArrayList[java.lang.Long]() - if (EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue > 0 && - taskIdList.length > EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue) { + if ( + EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue > 0 && + taskIdList.length > EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue + ) { for (i <- 0 until EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue) { updateTaskIds.add(taskIdList(i)) } @@ -178,8 +180,12 @@ object JobHistoryHelper extends Logging { val response = sender.ask(jobReqBatchUpdate) response match { case resp: util.ArrayList[JobRespProtocol] => - resp.asScala.filter(r => r.getStatus == SUCCESS_FLAG && r.getData.containsKey(JobRequestConstants.JOB_ID)) - .map(_.getData.get(JobRequestConstants.JOB_ID).asInstanceOf[java.lang.Long]).toList + resp.asScala + .filter(r => + r.getStatus == SUCCESS_FLAG && r.getData.containsKey(JobRequestConstants.JOB_ID) + ) + .map(_.getData.get(JobRequestConstants.JOB_ID).asInstanceOf[java.lang.Long]) + .toList case _ => throw JobHistoryFailedException( "update batch instances from jobhistory not a correct List type" @@ -188,7 +194,8 @@ object JobHistoryHelper extends Logging { } { case errorException: ErrorException => throw errorException case e: Exception => - val e1 = JobHistoryFailedException(s"update batch instances ${taskIdList.mkString(",")} error") + val e1 = + JobHistoryFailedException(s"update batch instances ${taskIdList.mkString(",")} error") e1.initCause(e) throw e } @@ -203,7 +210,12 @@ object JobHistoryHelper extends Logging { * @param limit * @return */ - def queryWaitForFailoverTask(reqMap: util.Map[String, java.lang.Long], statusList: util.List[String], startTimestamp: Long, limit: Int): util.List[JobRequest] = { + def queryWaitForFailoverTask( + reqMap: util.Map[String, java.lang.Long], + statusList: util.List[String], + startTimestamp: Long, + limit: Int + ): util.List[JobRequest] = { val requestFailoverJob = RequestFailoverJob(reqMap, statusList, startTimestamp, limit) val tasks = Utils.tryCatch { val response = sender.ask(requestFailoverJob) @@ -230,7 +242,8 @@ object JobHistoryHelper extends Logging { } { case errorException: ErrorException => throw errorException case e: Exception => - val e1 = JobHistoryFailedException(s"query failover task error, instances ${reqMap.keySet()} ") + val e1 = + JobHistoryFailedException(s"query failover task error, instances ${reqMap.keySet()} ") e1.initCause(e) throw e } From 497f8b62e7a3e2e5e490b3bf18d0046e465d0357 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Tue, 22 Nov 2022 19:40:37 +0800 Subject: [PATCH 010/689] failover --- .../scheduler/queue/SchedulerEventState.scala | 4 ++++ .../queue/fifoqueue/FIFOUserConsumer.scala | 2 +- .../server/EntranceFailoverJobServer.java | 18 ++++++++++++------ .../linkis/entrance/EntranceServer.scala | 2 +- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/SchedulerEventState.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/SchedulerEventState.scala index 4edc1d5d17c..a64103628c9 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/SchedulerEventState.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/SchedulerEventState.scala @@ -38,4 +38,8 @@ object SchedulerEventState extends Enumeration { SchedulerEventState.withName(jobState) ) + def uncompleteStatusArray(): Array[SchedulerEventState] = { + SchedulerEventState.values.filterNot(isCompleted).toArray + } + } diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala index 692325b75cc..4483a02a767 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala @@ -109,7 +109,7 @@ class FIFOUserConsumer( var event: Option[SchedulerEvent] = getWaitForRetryEvent if (event.isEmpty) { val maxAllowRunningJobs = fifoGroup.getMaxAllowRunningJobs - val currentRunningJobs = runningJobs.filter(e => e != null && !e.isCompleted) + val currentRunningJobs = runningJobs.count(e => e != null && !e.isCompleted) if (maxAllowRunningJobs <= currentRunningJobs) { Utils.tryQuietly(Thread.sleep(1000)) // TODO 还可以优化，通过实现JobListener进行优化 return diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java index 175da3be415..1eb29a48fb3 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java @@ -86,10 +86,15 @@ public void run() { if (!locked) return; logger.info("success locked {}", ENTRANCE_FAILOVER_LOCK); - // serverInstance to map - Map serverInstanceMap = getActiveServerInstances().stream() - .collect(Collectors.toMap(ServiceInstance::getInstance, ServiceInstance::getRegistryTimestamp, (k1, k2) -> k2)); - if (serverInstanceMap.isEmpty()) return; + // serverInstance to map + Map serverInstanceMap = + getActiveServerInstances().stream() + .collect( + Collectors.toMap( + ServiceInstance::getInstance, + ServiceInstance::getRegistryTimestamp, + (k1, k2) -> k2)); + if (serverInstanceMap.isEmpty()) return; // get failover job expired time (获取任务故障转移过期时间，配置为0表示不过期, 过期则不处理) long expiredTimestamp = 0L; @@ -100,8 +105,9 @@ public void run() { } // get uncompleted status - List statusList = Lists.newArrayList(); - SchedulerEventState.values().filterNot(SchedulerEventState::isCompleted).foreach(state -> statusList.add(state.toString())); + List statusList = + Arrays.stream(SchedulerEventState.uncompleteStatusArray()) + .map(Object::toString).collect(Collectors.toList()); List jobRequests = JobHistoryHelper.queryWaitForFailoverTask( diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index ec8692c84eb..b09ef4911ad 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -292,7 +292,7 @@ abstract class EntranceServer extends Logging { engineMap.asScala .map(_._2.asInstanceOf[util.Map[String, Object]]) .filter(_.containsKey(TaskConstant.ENGINE_INSTANCE)) - .maxBy(_.getOrDefault(TaskConstant.ENGINE_CONN_SUBMIT_TIME, 0).toString) + .maxBy(_.getOrDefault(TaskConstant.ENGINE_CONN_SUBMIT_TIME, "0").toString) if (engineInstance != null || engineInstance.containsKey(TaskConstant.FAILOVER_FLAG)) { logger.info( From da5b2b7277c2755a62ca78d414a8e9160dfe03d0 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Wed, 23 Nov 2022 20:08:27 +0800 Subject: [PATCH 011/689] add log --- .../scheduler/EntranceGroupFactory.scala | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala index a0a644e1d0f..2a7432ee6e6 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala @@ -195,13 +195,19 @@ class EntranceGroupFactory extends GroupFactory with Logging { def refreshAllGroupMaxAllowRunningJobs(activeCount: Int): Unit = { if (activeCount <= 0) return - groupNameToGroups.asMap().asScala.foreach(item => { - item._2 match { - case group: ParallelGroup => - group.setMaxAllowRunningJobs(Math.round(group.getMaxRunningJobs / activeCount)) - case _ => - } - }) + groupNameToGroups + .asMap() + .asScala + .foreach(item => { + item._2 match { + case group: ParallelGroup => + val maxAllowRunningJobs = Math.round(group.getMaxRunningJobs / activeCount) + group.setMaxAllowRunningJobs(maxAllowRunningJobs) + logger + .info(s"group ${group.getGroupName} update maxAllowRunningJobs $maxAllowRunningJobs") + case _ => + } + }) } private def getUserMaxRunningJobs(keyAndValue: util.Map[String, String]): Int = { From b624a04f09907d9bc574d7f3bf55d19e9efa729b Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Thu, 24 Nov 2022 22:59:19 +0800 Subject: [PATCH 012/689] entrance taskID --- .../linkis/common/entity/JobInstance.scala | 26 + .../server/conf/ServerConfiguration.scala | 3 +- .../entrance/restful/EntranceRestfulApi.java | 1068 +++++++++-------- .../entrance/utils/JobHistoryHelper.scala | 11 +- .../parser/EntranceRequestGatewayParser.scala | 57 +- 5 files changed, 658 insertions(+), 507 deletions(-) create mode 100644 linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/entity/JobInstance.scala diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/entity/JobInstance.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/entity/JobInstance.scala new file mode 100644 index 00000000000..aa9db730ee2 --- /dev/null +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/entity/JobInstance.scala @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.common.entity + +case class JobInstance( + status: String, + instances: String, + jobReqId: String, + createTimestamp: Long, + instanceRegistryTimestamp: Long +) diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala index 6784c5100f9..8d9f9d65adf 100644 --- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala +++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala @@ -207,6 +207,7 @@ object ServerConfiguration extends Logging { val LINKIS_SERVER_SESSION_PROXY_TICKETID_KEY = CommonVars("wds.linkis.session.proxy.user.ticket.key", "linkis_user_session_proxy_ticket_id_v1") - val LINKIS_SERVER_HEADER_KEY = CommonVars("wds.linkis.session.proxy.user.ticket.key", "job_req_id") + val LINKIS_SERVER_ENTRANCE_HEADER_KEY = + CommonVars("wds.linkis.server.entrance.header.key", "jobInstanceKey") } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index 4a946e6d0cf..8b10b9eb52b 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -17,6 +17,7 @@ package org.apache.linkis.entrance.restful; +import org.apache.linkis.common.entity.JobInstance; import org.apache.linkis.common.log.LogUtils; import org.apache.linkis.entrance.EntranceServer; import org.apache.linkis.entrance.conf.EntranceConfiguration; @@ -34,6 +35,7 @@ import org.apache.linkis.scheduler.listener.LogListener; import org.apache.linkis.scheduler.queue.Job; import org.apache.linkis.scheduler.queue.SchedulerEventState; +import org.apache.linkis.server.BDPJettyServerHelper; import org.apache.linkis.server.Message; import org.apache.linkis.server.conf.ServerConfiguration; import org.apache.linkis.server.security.SecurityFilter; @@ -61,6 +63,7 @@ import scala.Option; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport; import io.swagger.annotations.Api; @@ -198,277 +201,353 @@ private void pushLog(String log, Job job) { entranceServer.getEntranceContext().getOrCreateLogManager().onLogUpdate(job, log); } - @ApiOperation(value = "status", notes = "get task stats", response = Message.class) - @ApiImplicitParams({ - @ApiImplicitParam(name = "taskID", required = false, dataType = "String", value = " task id"), - @ApiImplicitParam(name = "id",required = true, dataType = "String", value = "execute id ") - }) - @Override - @RequestMapping(path = "/{id}/status", method = RequestMethod.GET) - public Message status( - HttpServletRequest req, - @PathVariable("id") String id, - @RequestParam(value = "taskID", required = false) String taskID) { - ModuleUserUtils.getOperationUser(req, "job status"); - Message message = null; - String realId; - String execID; - if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { - // execID - realId = ZuulEntranceUtils.parseExecID(id)[3]; - execID = id; - } else { - // taskID - String jobReqId = req.getHeader(ServerConfiguration.LINKIS_SERVER_ENTRANCE_HEADER_KEY().toString()); - if (StringUtils.isEmpty(jobReqId)){ - logger.warn("The job wait failover, return status is Inited"); - String status = SchedulerEventState.Inited().toString(); - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/status"); - message.data("status", status).data("execID", "").data("taskID", id); - return message; - } else { - realId = jobReqId; - execID = - ZuulEntranceUtils.generateExecID( - realId, - Sender.getThisServiceInstance().getApplicationName(), - new String[] {Sender.getThisInstance()}); - } - } + private JobInstance parseHeaderToJobInstance(HttpServletRequest req) + throws JsonProcessingException { + String jobStr = + req.getHeader(ServerConfiguration.LINKIS_SERVER_ENTRANCE_HEADER_KEY().getValue()); + return BDPJettyServerHelper.gson().fromJson(jobStr, JobInstance.class); + } - Option job = Option.apply(null); - try { - job = entranceServer.getJob(realId); - } catch (Exception e) { - logger.warn("get {} status error", realId, e); - if (StringUtils.isEmpty(taskID)) { - message = - Message.error( - "Get job by ID error and cannot obtain the corresponding task status.(获取job时发生异常，不能获取相应的任务状态)"); - return message; - } - long realTaskID = Long.parseLong(taskID); - String status = JobHistoryHelper.getStatusByTaskID(realTaskID); - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/status"); - message.data("status", status).data("execID", execID); - return message; - } - if (job.isDefined()) { - if (job.get() instanceof EntranceJob) { - ((EntranceJob) job.get()).updateNewestAccessByClientTimestamp(); - } - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/status"); - message.data("status", job.get().getState().toString()).data("execID", execID); - } else { - message = - Message.error( - "ID The corresponding job is empty and cannot obtain the corresponding task status.(ID 对应的job为空，不能获取相应的任务状态)"); - } + @ApiOperation(value = "status", notes = "get task stats", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "taskID", required = false, dataType = "String", value = " task id"), + @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "execute id ") + }) + @Override + @RequestMapping(path = "/{id}/status", method = RequestMethod.GET) + public Message status( + HttpServletRequest req, + @PathVariable("id") String id, + @RequestParam(value = "taskID", required = false) String taskID) { + ModuleUserUtils.getOperationUser(req, "job status"); + Message message = null; + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; + } else { + // taskID + JobInstance jobInstance; + try { + jobInstance = parseHeaderToJobInstance(req); + } catch (JsonProcessingException e) { + logger.error("parse JobInstance json error, id: {}", id); + message = Message.error("parse JobInstance json error"); + message.setMethod("/api/entrance/" + id + "/status"); return message; + } + + // return ok when job complete + if (SchedulerEventState.isCompletedByStr(jobInstance.status())) { + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/status"); + message.data("status", jobInstance.status()).data("execID", "").data("taskID", id); + return message; + } else if (jobInstance.instanceRegistryTimestamp() > jobInstance.createTimestamp()) { + logger.warn("The job {} wait failover, return status is Inited", id); + String status = SchedulerEventState.Inited().toString(); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/status"); + message.data("status", status).data("execID", "").data("taskID", id); + return message; + } else { + realId = jobInstance.jobReqId(); + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } } - @ApiOperation(value = "progress", notes = "get task progress info", response = Message.class) - @ApiImplicitParams({ - @ApiImplicitParam(name = "id",required = true, dataType = "String", value = "exectue id") - }) - @Override - @RequestMapping(path = "/{id}/progress", method = RequestMethod.GET) - public Message progress(HttpServletRequest req, @PathVariable("id") String id) { - ModuleUserUtils.getOperationUser(req, "job progress"); - Message message = null; - String realId; - String execID; - if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { - // execID - realId = ZuulEntranceUtils.parseExecID(id)[3]; - execID = id; - } else { - // taskID - String jobReqId = req.getHeader(ServerConfiguration.LINKIS_SERVER_ENTRANCE_HEADER_KEY().toString()); - if (StringUtils.isEmpty(jobReqId)){ - logger.warn("The job wait failover, return progress is 0"); - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/progress"); - message.data("progress", 0) - .data("execID", "") - .data("taskID", id) - .data("progressInfo", new ArrayList<>()); - return message; - } else { - realId = jobReqId; - execID = - ZuulEntranceUtils.generateExecID( - realId, - Sender.getThisServiceInstance().getApplicationName(), - new String[] {Sender.getThisInstance()}); - } - } + Option job = Option.apply(null); + try { + job = entranceServer.getJob(realId); + } catch (Exception e) { + logger.warn("get {} status error", realId, e); + if (StringUtils.isEmpty(taskID)) { + message = + Message.error( + "Get job by ID error and cannot obtain the corresponding task status.(获取job时发生异常，不能获取相应的任务状态)"); + return message; + } + long realTaskID = Long.parseLong(taskID); + String status = JobHistoryHelper.getStatusByTaskID(realTaskID); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/status"); + message.data("status", status).data("execID", execID); + return message; + } + if (job.isDefined()) { + if (job.get() instanceof EntranceJob) { + ((EntranceJob) job.get()).updateNewestAccessByClientTimestamp(); + } + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/status"); + message.data("status", job.get().getState().toString()).data("execID", execID); + } else { + message = + Message.error( + "ID The corresponding job is empty and cannot obtain the corresponding task status.(ID 对应的job为空，不能获取相应的任务状态)"); + } + return message; + } - Option job = null; - try { - job = entranceServer.getJob(realId); - } catch (Exception e) { - logger.error(e.getMessage()); - } - if (job != null && job.isDefined()) { - JobProgressInfo[] jobProgressInfos = ((EntranceJob) job.get()).getProgressInfo(); - if (jobProgressInfos == null) { - message = - Message.error( - "Can not get the corresponding progress information, it may be that the corresponding progress information has not been generated(不能获取相应的进度信息,可能是相应的进度信息还未生成)"); - message.setMethod("/api/entrance/" + id + "/progress"); - } else { - List> list = new ArrayList<>(); - for (JobProgressInfo jobProgressInfo : jobProgressInfos) { - if ("true".equals(EntranceConfiguration.PROGRESS_PUSH().getValue()) - || jobProgressInfo.totalTasks() > 0) { - setJobProgressInfos(list, jobProgressInfo); - } - } - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/progress"); - - message.data("progress", Math.abs(job.get().getProgress())) - .data("execID", execID) - .data("progressInfo", list); - } - } else { - message = - Message.error( - "The job corresponding to the ID is empty, and the corresponding task progress cannot be obtained.(ID 对应的job为空，不能获取相应的任务进度)"); - } + @ApiOperation(value = "progress", notes = "get task progress info", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "exectue id") + }) + @Override + @RequestMapping(path = "/{id}/progress", method = RequestMethod.GET) + public Message progress(HttpServletRequest req, @PathVariable("id") String id) { + ModuleUserUtils.getOperationUser(req, "job progress"); + Message message = null; + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; + } else { + // taskID + JobInstance jobInstance; + try { + jobInstance = parseHeaderToJobInstance(req); + } catch (JsonProcessingException e) { + logger.error("parse JobInstance json error, id: {}", id); + message = Message.error("parse JobInstance json error"); + message.setMethod("/api/entrance/" + id + "/progress"); + return message; + } + + // return ok when job complete + if (SchedulerEventState.isCompletedByStr(jobInstance.status())) { + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/progress"); + message + .data("progress", "1.0") + .data("execID", "") + .data("taskID", id) + .data("progressInfo", new ArrayList<>()); + return message; + } else if (jobInstance.instanceRegistryTimestamp() > jobInstance.createTimestamp()) { + logger.warn("The job {} wait failover, return progress is 0", id); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/progress"); + message + .data("progress", 0) + .data("execID", "") + .data("taskID", id) + .data("progressInfo", new ArrayList<>()); return message; + } else { + realId = jobInstance.jobReqId(); + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } } - @ApiOperation(value = "progressWithResource", notes = "get progress and resource info", response = Message.class) - @ApiImplicitParams({ - @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "execute id") - }) - @Override - @RequestMapping(path = "/{id}/progressWithResource", method = RequestMethod.GET) - public Message progressWithResource(HttpServletRequest req, @PathVariable("id") String id) { - ModuleUserUtils.getOperationUser(req, "job progressWithResource"); - Message message = null; - String realId; - String execID; - if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { - // execID - realId = ZuulEntranceUtils.parseExecID(id)[3]; - execID = id; - } else { - // taskID - String jobReqId = req.getHeader(ServerConfiguration.LINKIS_SERVER_ENTRANCE_HEADER_KEY().toString()); - if (StringUtils.isEmpty(jobReqId)){ - logger.warn("The job wait failover, return progress is 0 and resource is null"); - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/progressWithResource"); - message.data(TaskConstant.ENTRANCEJOB_YARNRESOURCE, null) - .data("progress", 0) - .data("execID", "") - .data("taskID", id) - .data("progressInfo", new ArrayList<>()); - return message; - } else { - realId = jobReqId; - execID = - ZuulEntranceUtils.generateExecID( - realId, - Sender.getThisServiceInstance().getApplicationName(), - new String[] {Sender.getThisInstance()}); - } + Option job = null; + try { + job = entranceServer.getJob(realId); + } catch (Exception e) { + logger.error(e.getMessage()); + } + if (job != null && job.isDefined()) { + JobProgressInfo[] jobProgressInfos = ((EntranceJob) job.get()).getProgressInfo(); + if (jobProgressInfos == null) { + message = + Message.error( + "Can not get the corresponding progress information, it may be that the corresponding progress information has not been generated(不能获取相应的进度信息,可能是相应的进度信息还未生成)"); + message.setMethod("/api/entrance/" + id + "/progress"); + } else { + List> list = new ArrayList<>(); + for (JobProgressInfo jobProgressInfo : jobProgressInfos) { + if ("true".equals(EntranceConfiguration.PROGRESS_PUSH().getValue()) + || jobProgressInfo.totalTasks() > 0) { + setJobProgressInfos(list, jobProgressInfo); + } } - Option job = null; - try { - job = entranceServer.getJob(realId); - } catch (Exception e) { - logger.error(e.getMessage()); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/progress"); + + message + .data("progress", Math.abs(job.get().getProgress())) + .data("execID", execID) + .data("progressInfo", list); + } + } else { + message = + Message.error( + "The job corresponding to the ID is empty, and the corresponding task progress cannot be obtained.(ID 对应的job为空，不能获取相应的任务进度)"); + } + return message; + } + + @ApiOperation( + value = "progressWithResource", + notes = "get progress and resource info", + response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "execute id") + }) + @Override + @RequestMapping(path = "/{id}/progressWithResource", method = RequestMethod.GET) + public Message progressWithResource(HttpServletRequest req, @PathVariable("id") String id) { + ModuleUserUtils.getOperationUser(req, "job progressWithResource"); + Message message = null; + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; + } else { + // taskID + JobInstance jobInstance; + try { + jobInstance = parseHeaderToJobInstance(req); + } catch (JsonProcessingException e) { + logger.error("parse JobInstance json error, id: {}", id); + message = Message.error("parse JobInstance json error"); + message.setMethod("/api/entrance/" + id + "/progressWithResource"); + return message; + } + + // return ok when job complete + if (SchedulerEventState.isCompletedByStr(jobInstance.status())) { + long realTaskID = Long.parseLong(id); + JobRequest jobRequest = JobHistoryHelper.getTaskByTaskID(realTaskID); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/progressWithResource"); + Map metricsVo = new HashMap<>(); + buildYarnResource(jobRequest, metricsVo, message); + message + .data("progress", "1.0") + .data("execID", "") + .data("taskID", id) + .data("progressInfo", new ArrayList<>()); + return message; + } else if (jobInstance.instanceRegistryTimestamp() > jobInstance.createTimestamp()) { + logger.warn("The job {} wait failover, return progress is 0 and resource is null", id); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/progressWithResource"); + message + .data(TaskConstant.ENTRANCEJOB_YARNRESOURCE, null) + .data("progress", 0) + .data("execID", "") + .data("taskID", id) + .data("progressInfo", new ArrayList<>()); + return message; + } else { + realId = jobInstance.jobReqId(); + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } + } + Option job = null; + try { + job = entranceServer.getJob(realId); + } catch (Exception e) { + logger.error(e.getMessage()); + } + if (job != null && job.isDefined()) { + JobProgressInfo[] jobProgressInfos = ((EntranceJob) job.get()).getProgressInfo(); + if (jobProgressInfos == null) { + message = + Message.error( + "Can not get the corresponding progress information, it may be that the corresponding progress information has not been generated(不能获取相应的进度信息,可能是相应的进度信息还未生成)"); + message.setMethod("/api/entrance/" + id + "/progressWithResource"); + } else { + List> list = new ArrayList<>(); + for (JobProgressInfo jobProgressInfo : jobProgressInfos) { + if ("true".equals(EntranceConfiguration.PROGRESS_PUSH().getValue()) + || jobProgressInfo.totalTasks() > 0) { + setJobProgressInfos(list, jobProgressInfo); + } } - if (job != null && job.isDefined()) { - JobProgressInfo[] jobProgressInfos = ((EntranceJob) job.get()).getProgressInfo(); - if (jobProgressInfos == null) { - message = - Message.error( - "Can not get the corresponding progress information, it may be that the corresponding progress information has not been generated(不能获取相应的进度信息,可能是相应的进度信息还未生成)"); - message.setMethod("/api/entrance/" + id + "/progressWithResource"); - } else { - List> list = new ArrayList<>(); - for (JobProgressInfo jobProgressInfo : jobProgressInfos) { - if ("true".equals(EntranceConfiguration.PROGRESS_PUSH().getValue()) - || jobProgressInfo.totalTasks() > 0) { - setJobProgressInfos(list, jobProgressInfo); - } - } - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/progressWithResource"); - - JobRequest jobRequest = ((EntranceJob) job.get()).getJobRequest(); - Map metrics = jobRequest.getMetrics(); - Map metricsVo = new HashMap<>(); - if (metrics.containsKey(TaskConstant.ENTRANCEJOB_YARNRESOURCE)) { - HashMap resourceMap = - (HashMap) - metrics.get(TaskConstant.ENTRANCEJOB_YARNRESOURCE); - ArrayList resoureList = new ArrayList<>(12); - if (null != resourceMap && !resourceMap.isEmpty()) { - resourceMap.forEach( - (applicationId, resource) -> { - resoureList.add( - new YarnResourceWithStatusVo(applicationId, resource)); - }); - metricsVo.put(TaskConstant.ENTRANCEJOB_YARNRESOURCE, resoureList); - Optional cores = - resourceMap.values().stream() - .map(resource -> resource.queueCores()) - .reduce((x, y) -> x + y); - Optional memory = - resourceMap.values().stream() - .map(resource -> resource.queueMemory()) - .reduce((x, y) -> x + y); - float corePercent = 0.0f; - float memoryPercent = 0.0f; - if (cores.isPresent() && memory.isPresent()) { - corePercent = - cores.get().floatValue() - / EntranceConfiguration.YARN_QUEUE_CORES_MAX() - .getValue(); - memoryPercent = - memory.get().floatValue() - / (EntranceConfiguration.YARN_QUEUE_MEMORY_MAX() - .getValue() - .longValue() - * 1024 - * 1024 - * 1024); - } - String coreRGB = RGBUtils.getRGB(corePercent); - String memoryRGB = RGBUtils.getRGB(memoryPercent); - metricsVo.put(TaskConstant.ENTRANCEJOB_CORE_PERCENT, corePercent); - metricsVo.put(TaskConstant.ENTRANCEJOB_MEMORY_PERCENT, memoryPercent); - metricsVo.put(TaskConstant.ENTRANCEJOB_CORE_RGB, coreRGB); - metricsVo.put(TaskConstant.ENTRANCEJOB_MEMORY_RGB, memoryRGB); - - message.data(TaskConstant.ENTRANCEJOB_YARN_METRICS, metricsVo); - } else { - message.data(TaskConstant.ENTRANCEJOB_YARNRESOURCE, null); - } - } else { - message.data(TaskConstant.ENTRANCEJOB_YARNRESOURCE, null); - } - - message.data("progress", Math.abs(job.get().getProgress())) - .data("execID", execID) - .data("progressInfo", list); - } + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/progressWithResource"); + + JobRequest jobRequest = ((EntranceJob) job.get()).getJobRequest(); + Map metricsVo = new HashMap<>(); + buildYarnResource(jobRequest, metricsVo, message); + + message + .data("progress", Math.abs(job.get().getProgress())) + .data("execID", execID) + .data("progressInfo", list); + } + } else { + message = + Message.error( + "The job corresponding to the ID is empty, and the corresponding task progress cannot be obtained.(ID 对应的job为空，不能获取相应的任务进度)"); + } + return message; + } + + private void buildYarnResource( + JobRequest jobRequest, Map metricsVo, Message message) { + try { + Map metrics = jobRequest.getMetrics(); + if (metrics.containsKey(TaskConstant.ENTRANCEJOB_YARNRESOURCE)) { + + HashMap resourceMap = + (HashMap) + metrics.get(TaskConstant.ENTRANCEJOB_YARNRESOURCE); + ArrayList resoureList = new ArrayList<>(12); + if (null != resourceMap && !resourceMap.isEmpty()) { + resourceMap.forEach( + (applicationId, resource) -> { + resoureList.add(new YarnResourceWithStatusVo(applicationId, resource)); + }); + metricsVo.put(TaskConstant.ENTRANCEJOB_YARNRESOURCE, resoureList); + Optional cores = + resourceMap.values().stream() + .map(resource -> resource.queueCores()) + .reduce((x, y) -> x + y); + Optional memory = + resourceMap.values().stream() + .map(resource -> resource.queueMemory()) + .reduce((x, y) -> x + y); + float corePercent = 0.0f; + float memoryPercent = 0.0f; + if (cores.isPresent() && memory.isPresent()) { + corePercent = + cores.get().floatValue() / EntranceConfiguration.YARN_QUEUE_CORES_MAX().getValue(); + memoryPercent = + memory.get().floatValue() + / (EntranceConfiguration.YARN_QUEUE_MEMORY_MAX().getValue().longValue() + * 1024 + * 1024 + * 1024); + } + String coreRGB = RGBUtils.getRGB(corePercent); + String memoryRGB = RGBUtils.getRGB(memoryPercent); + metricsVo.put(TaskConstant.ENTRANCEJOB_CORE_PERCENT, corePercent); + metricsVo.put(TaskConstant.ENTRANCEJOB_MEMORY_PERCENT, memoryPercent); + metricsVo.put(TaskConstant.ENTRANCEJOB_CORE_RGB, coreRGB); + metricsVo.put(TaskConstant.ENTRANCEJOB_MEMORY_RGB, memoryRGB); + + message.data(TaskConstant.ENTRANCEJOB_YARN_METRICS, metricsVo); } else { - message = - Message.error( - "The job corresponding to the ID is empty, and the corresponding task progress cannot be obtained.(ID 对应的job为空，不能获取相应的任务进度)"); + message.data(TaskConstant.ENTRANCEJOB_YARNRESOURCE, null); } - return message; + } else { + message.data(TaskConstant.ENTRANCEJOB_YARNRESOURCE, null); + } + } catch (Exception e) { + logger.error("build yarnResource error", e); } + } private void setJobProgressInfos( List> list, JobProgressInfo jobProgressInfo) { @@ -481,146 +560,157 @@ private void setJobProgressInfos( list.add(map); } - @ApiOperation(value = "log", notes = "get task log", response = Message.class) - @ApiImplicitParams({ - @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "execute id") - }) - @Override - @RequestMapping(path = "/{id}/log", method = RequestMethod.GET) - public Message log(HttpServletRequest req, @PathVariable("id") String id) { - ModuleUserUtils.getOperationUser(req, "get job log"); - Message message = null; - int fromLine = 0; - int size = 100; - boolean distinctLevel = true; - String fromLineStr = req.getParameter("fromLine"); - String sizeStr = req.getParameter("size"); - if (StringUtils.isNotBlank(fromLineStr)) { - fromLine = Math.max(Integer.parseInt(fromLineStr), 0); - } - if (StringUtils.isNotBlank(sizeStr)) { - size = Integer.parseInt(sizeStr) >= 0 ? Integer.parseInt(sizeStr) : 10000; - } - String distinctLevelStr = req.getParameter("distinctLevel"); - if ("false".equals(distinctLevelStr)) { - distinctLevel = false; - } + @ApiOperation(value = "log", notes = "get task log", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "execute id") + }) + @Override + @RequestMapping(path = "/{id}/log", method = RequestMethod.GET) + public Message log(HttpServletRequest req, @PathVariable("id") String id) { + ModuleUserUtils.getOperationUser(req, "get job log"); + Message message = null; + int fromLine = 0; + int size = 100; + boolean distinctLevel = true; + String fromLineStr = req.getParameter("fromLine"); + String sizeStr = req.getParameter("size"); + if (StringUtils.isNotBlank(fromLineStr)) { + fromLine = Math.max(Integer.parseInt(fromLineStr), 0); + } + if (StringUtils.isNotBlank(sizeStr)) { + size = Integer.parseInt(sizeStr) >= 0 ? Integer.parseInt(sizeStr) : 10000; + } + String distinctLevelStr = req.getParameter("distinctLevel"); + if ("false".equals(distinctLevelStr)) { + distinctLevel = false; + } - String realId; - String execID; - if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { - // execID - realId = ZuulEntranceUtils.parseExecID(id)[3]; - execID = id; + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; + } else { + // taskID + JobInstance jobInstance; + try { + jobInstance = parseHeaderToJobInstance(req); + } catch (JsonProcessingException e) { + logger.error("parse JobInstance json error, id: {}", id); + message = Message.error("parse JobInstance json error"); + message.setMethod("/api/entrance/" + id + "/log"); + return message; + } + + // return ok when job complete + if (SchedulerEventState.isCompletedByStr(jobInstance.status())) { + message = + Message.error( + "The job you just executed has ended. This interface no longer provides a query. It is recommended that you download the log file for viewing.(您刚刚执行的job已经结束，本接口不再提供查询，建议您下载日志文件进行查看)"); + message.setMethod("/api/entrance/" + id + "/log"); + return message; + } else if (jobInstance.instanceRegistryTimestamp() > jobInstance.createTimestamp()) { + logger.warn("The job {} wait failover, return customer log", id); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/log"); + String log = LogUtils.generateInfo("The job will failover soon, please try again later"); + Object retLog; + if (distinctLevel) { + String[] array = new String[4]; + array[2] = log; + array[3] = log; + retLog = new ArrayList(Arrays.asList(array)); } else { - // taskID - String jobReqId = req.getHeader(ServerConfiguration.LINKIS_SERVER_ENTRANCE_HEADER_KEY().toString()); - if (StringUtils.isEmpty(jobReqId)){ - logger.warn("The job wait failover, return customer log"); - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/log"); - String log = LogUtils.generateInfo("The job will failover soon, please try again later"); - Object retLog; - if (distinctLevel) { - String[] array = new String[4]; - array[2] = log; - array[3] = log; - retLog = new ArrayList(Arrays.asList(array)); - } else { - retLog = log; - } - message.data("log", retLog).data("execID", "").data("taskID", id).data("fromLine", 0); - return message; - } else { - realId = jobReqId; - execID = - ZuulEntranceUtils.generateExecID( - realId, - Sender.getThisServiceInstance().getApplicationName(), - new String[] {Sender.getThisInstance()}); - } + retLog = log; } + message.data("log", retLog).data("execID", "").data("taskID", id).data("fromLine", 0); + return message; + } else { + realId = jobInstance.jobReqId(); + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } + } - Option job = Option.apply(null); - try { - job = entranceServer.getJob(realId); - } catch (final Throwable t) { - message = - Message.error( - "The job you just executed has ended. This interface no longer provides a query. It is recommended that you download the log file for viewing.(您刚刚执行的job已经结束，本接口不再提供查询，建议您下载日志文件进行查看)"); - message.setMethod("/api/entrance/" + id + "/log"); - return message; - } - if (job.isDefined()) { - logger.debug( - "begin to get log for {}(开始获取 {} 的日志)", job.get().getId(), job.get().getId()); - LogReader logReader = - entranceServer - .getEntranceContext() - .getOrCreateLogManager() - .getLogReader(realId); - - Object retLog = null; - int retFromLine = 0; - try { - if (distinctLevel) { - String[] logs = new String[4]; - retFromLine = logReader.readArray(logs, fromLine, size); - retLog = new ArrayList(Arrays.asList(logs)); - } else { - StringBuilder sb = new StringBuilder(); - retFromLine = logReader.read(sb, fromLine, size); - retLog = sb.toString(); - } - } catch (IllegalStateException e) { - logger.debug( - "Failed to get log information for :{}(为 {} 获取日志失败)", - job.get().getId(), - job.get().getId(), - e); - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/log"); - message.data("log", "").data("execID", execID).data("fromLine", retFromLine + fromLine); - } catch (final IllegalArgumentException e) { - logger.debug( - "Failed to get log information for :{}(为 {} 获取日志失败)", - job.get().getId(), - job.get().getId(), - e); - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/log"); - message.data("log", "").data("execID", execID).data("fromLine", retFromLine + fromLine); - return message; - } catch (final Exception e1) { - logger.debug( - "Failed to get log information for :{}(为 {} 获取日志失败)", - job.get().getId(), - job.get().getId(), - e1); - message = Message.error("Failed to get log information(获取日志信息失败)"); - message.setMethod("/api/entrance/" + id + "/log"); - message.data("log", "").data("execID", execID).data("fromLine", retFromLine + fromLine); - return message; - } finally { - if (null != logReader && job.get().isCompleted()) { - IOUtils.closeQuietly(logReader); - } - } - message = Message.ok(); - message.setMethod("/api/entrance/" + id + "/log"); - message.data("log", retLog).data("execID", execID).data("fromLine", retFromLine + fromLine); - logger.debug( - "success to get log for {} (获取 {} 日志成功)", job.get().getId(), job.get().getId()); + Option job = Option.apply(null); + try { + job = entranceServer.getJob(realId); + } catch (final Throwable t) { + message = + Message.error( + "The job you just executed has ended. This interface no longer provides a query. It is recommended that you download the log file for viewing.(您刚刚执行的job已经结束，本接口不再提供查询，建议您下载日志文件进行查看)"); + message.setMethod("/api/entrance/" + id + "/log"); + return message; + } + if (job.isDefined()) { + logger.debug("begin to get log for {}(开始获取 {} 的日志)", job.get().getId(), job.get().getId()); + LogReader logReader = + entranceServer.getEntranceContext().getOrCreateLogManager().getLogReader(realId); + + Object retLog = null; + int retFromLine = 0; + try { + if (distinctLevel) { + String[] logs = new String[4]; + retFromLine = logReader.readArray(logs, fromLine, size); + retLog = new ArrayList(Arrays.asList(logs)); } else { - message = - Message.error( - "Can't find execID(不能找到execID): " - + id - + "Corresponding job, can not get the corresponding log(对应的job，不能获得对应的日志)"); - message.setMethod("/api/entrance/" + id + "/log"); + StringBuilder sb = new StringBuilder(); + retFromLine = logReader.read(sb, fromLine, size); + retLog = sb.toString(); } + } catch (IllegalStateException e) { + logger.debug( + "Failed to get log information for :{}(为 {} 获取日志失败)", + job.get().getId(), + job.get().getId(), + e); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/log"); + message.data("log", "").data("execID", execID).data("fromLine", retFromLine + fromLine); + } catch (final IllegalArgumentException e) { + logger.debug( + "Failed to get log information for :{}(为 {} 获取日志失败)", + job.get().getId(), + job.get().getId(), + e); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/log"); + message.data("log", "").data("execID", execID).data("fromLine", retFromLine + fromLine); + return message; + } catch (final Exception e1) { + logger.debug( + "Failed to get log information for :{}(为 {} 获取日志失败)", + job.get().getId(), + job.get().getId(), + e1); + message = Message.error("Failed to get log information(获取日志信息失败)"); + message.setMethod("/api/entrance/" + id + "/log"); + message.data("log", "").data("execID", execID).data("fromLine", retFromLine + fromLine); return message; + } finally { + if (null != logReader && job.get().isCompleted()) { + IOUtils.closeQuietly(logReader); + } + } + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/log"); + message.data("log", retLog).data("execID", execID).data("fromLine", retFromLine + fromLine); + logger.debug("success to get log for {} (获取 {} 日志成功)", job.get().getId(), job.get().getId()); + } else { + message = + Message.error( + "Can't find execID(不能找到execID): " + + id + + "Corresponding job, can not get the corresponding log(对应的job，不能获得对应的日志)"); + message.setMethod("/api/entrance/" + id + "/log"); } + return message; + } @ApiOperation(value = "killJobs", notes = "kill jobs", response = Message.class) @ApiImplicitParams({ @@ -718,104 +808,116 @@ public Message killJobs( return Message.ok("success").data("messages", messages); } - @ApiOperation(value = "kill", notes = "kill", response = Message.class) - @ApiImplicitParams({ - @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "exec id"), - @ApiImplicitParam(name = "taskID", required = false, dataType = "String", value = "task id") - }) - @Override - @RequestMapping(path = "/{id}/kill", method = RequestMethod.GET) - public Message kill( - HttpServletRequest req, - @PathVariable("id") String id, - @RequestParam(value = "taskID", required = false) Long taskID) { - ModuleUserUtils.getOperationUser(req, "kill job"); - Message message = null; - String realId; - String execID; - if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { - // execID - realId = ZuulEntranceUtils.parseExecID(id)[3]; - execID = id; - } else { - // taskID - String jobReqId = req.getHeader(ServerConfiguration.LINKIS_SERVER_ENTRANCE_HEADER_KEY().toString()); - if (StringUtils.isEmpty(jobReqId)){ - logger.warn("The job wait failover, but now force kill"); - // TODO If failover occurs during force kill, the job status may change from Cancelled to Running - long taskId = Long.parseLong(id); - JobHistoryHelper.forceKill(taskId); - message = Message.ok("Forced Kill task (强制杀死任务)"); - message.setMethod("/api/entrance/" + id + "/kill"); - message.data("execID", "").data("taskID", id); - return message; - } else { - realId = jobReqId; - execID = - ZuulEntranceUtils.generateExecID( - realId, - Sender.getThisServiceInstance().getApplicationName(), - new String[] {Sender.getThisInstance()}); - } - } + @ApiOperation(value = "kill", notes = "kill", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "exec id"), + @ApiImplicitParam(name = "taskID", required = false, dataType = "String", value = "task id") + }) + @Override + @RequestMapping(path = "/{id}/kill", method = RequestMethod.GET) + public Message kill( + HttpServletRequest req, + @PathVariable("id") String id, + @RequestParam(value = "taskID", required = false) Long taskID) { + ModuleUserUtils.getOperationUser(req, "kill job"); + Message message = null; + String realId; + String execID; + if (id.startsWith(ZuulEntranceUtils.EXEC_ID())) { + // execID + realId = ZuulEntranceUtils.parseExecID(id)[3]; + execID = id; + } else { + // taskID + JobInstance jobInstance; + try { + jobInstance = parseHeaderToJobInstance(req); + } catch (JsonProcessingException e) { + logger.error("parse JobInstance json error, id: {}", id); + message = Message.error("parse JobInstance json error"); + message.setMethod("/api/entrance/" + id + "/kill"); + return message; + } - Option job = Option.apply(null); - try { - job = entranceServer.getJob(realId); - } catch (Exception e) { - logger.warn("can not find a job in entranceServer, will force to kill it", e); - // 如果在内存中找不到该任务，那么该任务可能已经完成了，或者就是重启导致的 - if (taskID == null || taskID <= 0) { - message = - Message.error( - "Get job by ID error, kill failed.(获取job时发生异常，kill失败)"); - return message; - } - JobHistoryHelper.forceKill(taskID); - message = Message.ok("Forced Kill task (强制杀死任务)"); - message.setMethod("/api/entrance/" + id + "/kill"); - message.setStatus(0); - return message; - } + // return ok when job complete + if (SchedulerEventState.isCompletedByStr(jobInstance.status())) { + message = Message.error("The job already completed. Do not support kill.(任务已经结束，不支持kill)"); + message.setMethod("/api/entrance/" + id + "/kill"); + return message; + } else if (jobInstance.instanceRegistryTimestamp() > jobInstance.createTimestamp()) { + logger.warn("The job {} wait failover, but now force kill", id); + // TODO If failover during force kill, the job status may change from Cancelled to Running + long taskId = Long.parseLong(id); + JobHistoryHelper.forceKill(taskId); + message = Message.ok("Forced Kill task (强制杀死任务)"); + message.setMethod("/api/entrance/" + id + "/kill"); + message.data("execID", "").data("taskID", id); + return message; + } else { + realId = jobInstance.jobReqId(); + execID = + ZuulEntranceUtils.generateExecID( + realId, + Sender.getThisServiceInstance().getApplicationName(), + new String[] {Sender.getThisInstance()}); + } + } - if (job.isEmpty()) { - logger.warn("can not find a job in entranceServer, will force to kill it"); - // 如果在内存中找不到该任务，那么该任务可能已经完成了，或者就是重启导致的 - JobHistoryHelper.forceKill(taskID); - message = Message.ok("Forced Kill task (强制杀死任务)"); - message.setMethod("/api/entrance/" + id + "/kill"); - message.setStatus(0); - return message; - } else { - try { - logger.info("begin to kill job {} ", job.get().getId()); - job.get().kill(); - message = Message.ok("Successfully killed the job(成功kill了job)"); - message.setMethod("/api/entrance/" + id + "/kill"); - message.data("execID", execID); - // ensure the job's state is cancelled in database - if (job.get() instanceof EntranceJob) { - EntranceJob entranceJob = (EntranceJob) job.get(); - JobRequest jobReq = entranceJob.getJobRequest(); - entranceJob.updateJobRequestStatus(SchedulerEventState.Cancelled().toString()); - this.entranceServer - .getEntranceContext() - .getOrCreatePersistenceManager() - .createPersistenceEngine() - .updateIfNeeded(jobReq); - } - logger.info("end to kill job {} ", job.get().getId()); - } catch (Throwable t) { - logger.error("kill job {} failed ", job.get().getId(), t); - message = - Message.error( - "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)"); - message.setMethod("/api/entrance/" + id + "/kill"); - } - } + Option job = Option.apply(null); + try { + job = entranceServer.getJob(realId); + } catch (Exception e) { + logger.warn("can not find a job in entranceServer, will force to kill it", e); + // 如果在内存中找不到该任务，那么该任务可能已经完成了，或者就是重启导致的 + if (taskID == null || taskID <= 0) { + message = Message.error("Get job by ID error, kill failed.(获取job时发生异常，kill失败)"); return message; + } + JobHistoryHelper.forceKill(taskID); + message = Message.ok("Forced Kill task (强制杀死任务)"); + message.setMethod("/api/entrance/" + id + "/kill"); + message.setStatus(0); + return message; } + if (job.isEmpty()) { + logger.warn("can not find a job in entranceServer, will force to kill it"); + // 如果在内存中找不到该任务，那么该任务可能已经完成了，或者就是重启导致的 + JobHistoryHelper.forceKill(taskID); + message = Message.ok("Forced Kill task (强制杀死任务)"); + message.setMethod("/api/entrance/" + id + "/kill"); + message.setStatus(0); + return message; + } else { + try { + logger.info("begin to kill job {} ", job.get().getId()); + job.get().kill(); + message = Message.ok("Successfully killed the job(成功kill了job)"); + message.setMethod("/api/entrance/" + id + "/kill"); + message.data("execID", execID); + // ensure the job's state is cancelled in database + if (job.get() instanceof EntranceJob) { + EntranceJob entranceJob = (EntranceJob) job.get(); + JobRequest jobReq = entranceJob.getJobRequest(); + entranceJob.updateJobRequestStatus(SchedulerEventState.Cancelled().toString()); + this.entranceServer + .getEntranceContext() + .getOrCreatePersistenceManager() + .createPersistenceEngine() + .updateIfNeeded(jobReq); + } + logger.info("end to kill job {} ", job.get().getId()); + } catch (Throwable t) { + logger.error("kill job {} failed ", job.get().getId(), t); + message = + Message.error( + "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)"); + message.setMethod("/api/entrance/" + id + "/kill"); + } + } + return message; + } + @ApiOperation(value = "pause ", notes = "puase a task job", response = Message.class) @ApiImplicitParams({ @ApiImplicitParam(name = "id", required = true, dataType = "String", value = "excete id") diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala index 6416fe1d470..811af8fce5c 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala @@ -67,6 +67,11 @@ object JobHistoryHelper extends Logging { else task.getStatus } + def getProgressByTaskID(taskID: Long): String = { + val task = getTaskByTaskID(taskID) + if (task == null) "0" else task.getProgress + } + def getRequestIpAddr(req: HttpServletRequest): String = { val addrList = List( Option(req.getHeader("x-forwarded-for")).getOrElse("").split(",")(0), @@ -228,8 +233,8 @@ object JobHistoryHelper extends Logging { } val data = responsePersist.getData data.get(JobRequestConstants.JOB_HISTORY_LIST) match { - case tasks: util.List[JobRequest] => - tasks + case tasks: List[JobRequest] => + tasks.asJava case _ => throw JobHistoryFailedException( s"query from jobhistory not a correct List type, instances ${reqMap.keySet()}" @@ -250,7 +255,7 @@ object JobHistoryHelper extends Logging { tasks } - private def getTaskByTaskID(taskID: Long): JobRequest = { + def getTaskByTaskID(taskID: Long): JobRequest = { val jobRequest = new JobRequest jobRequest.setId(taskID) jobRequest.setSource(null) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala index 9fb3958ac0c..a1be26de875 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala @@ -19,14 +19,16 @@ package org.apache.linkis.gateway.ujes.parser import org.apache.commons.lang3.StringUtils import org.apache.linkis.common.ServiceInstance +import org.apache.linkis.common.entity.JobInstance +import org.apache.linkis.common.utils.JsonUtils import org.apache.linkis.gateway.config.GatewayConfiguration import org.apache.linkis.gateway.http.GatewayContext import org.apache.linkis.gateway.parser.AbstractGatewayParser import org.apache.linkis.gateway.ujes.parser.EntranceExecutionGatewayParser._ -import org.apache.linkis.jobhistory.entity.JobHistory import org.apache.linkis.jobhistory.service.JobHistoryQueryService import org.apache.linkis.protocol.utils.ZuulEntranceUtils import org.apache.linkis.rpc.interceptor.ServiceInstanceUtils +import org.apache.linkis.server.BDPJettyServerHelper import org.apache.linkis.server.conf.ServerConfiguration import org.springframework.stereotype.Component @@ -65,35 +67,50 @@ class EntranceRequestGatewayParser extends AbstractGatewayParser { // parse by execId ZuulEntranceUtils.parseServiceInstanceByExecID(execId)(0) } else { - // check by taskId - val jobHistory = checkJobValidityByTaskID(execId.toLong, gatewayContext) - // add header - val jobReqId = if (jobHistory == null) "" else jobHistory.getJobReqId - gatewayContext.getRequest.addHeader(ServerConfiguration.LINKIS_SERVER_HEADER_KEY.getValue, Array(jobReqId)) - // select instance - val instance = if (jobHistory == null) null else jobHistory.getInstances - ServiceInstance(GatewayConfiguration.ENTRANCE_SPRING_NAME.getValue, instance) + // build JobInstance by taskId + val jobInstance = buildJobInstance(execId.toLong, gatewayContext) + if (jobInstance == null) return + val str = BDPJettyServerHelper.gson.toJson(jobInstance) + gatewayContext.getRequest.addHeader( + ServerConfiguration.LINKIS_SERVER_ENTRANCE_HEADER_KEY.getValue, + Array(str) + ) + + ServiceInstance(GatewayConfiguration.ENTRANCE_SPRING_NAME.getValue, jobInstance.instances) } gatewayContext.getGatewayRoute.setServiceInstance(serviceInstance) case _ => } - def checkJobValidityByTaskID(taskId: Long, gatewayContext: GatewayContext): JobHistory = { + def buildJobInstance(taskId: Long, gatewayContext: GatewayContext): JobInstance = { val histories = jobHistoryQueryService.search(taskId, null, null, null, null, null, null, null) if (histories.isEmpty) { sendErrorResponse(s"taskId $taskId is not exists.", gatewayContext) + return null } - val instances = histories.get(0).getInstances - val activeInstances = ServiceInstanceUtils.getRPCServerLoader.getServiceInstances(GatewayConfiguration.ENTRANCE_SPRING_NAME.getValue) - - if (activeInstances.exists(StringUtils.isNotBlank(instances) && _.getInstance.equals(instances)) && - activeInstances.filter(_.getInstance.equals(instances))(0).getRegistryTimestamp <= histories.get(0).getCreatedTime.getTime - ) { - histories.get(0) - } else { - null + val history = histories.get(0) + if (StringUtils.isEmpty(history.getInstances)) { + return JobInstance( + history.getStatus, + null, + history.getJobReqId, + history.getCreatedTime.getTime, + Long.MaxValue + ) } - + val activeInstances = ServiceInstanceUtils.getRPCServerLoader.getServiceInstances( + GatewayConfiguration.ENTRANCE_SPRING_NAME.getValue + ) + val instance = activeInstances + .find(_.getInstance.equals(history.getInstances)) + .getOrElse(ServiceInstance("", "", Long.MaxValue)) + JobInstance( + history.getStatus, + history.getInstances, + history.getJobReqId, + history.getCreatedTime.getTime, + instance.getRegistryTimestamp + ) } } From cbfbdff07b244fc0d4e140af468b277bb6aa7ecb Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Thu, 24 Nov 2022 23:02:08 +0800 Subject: [PATCH 013/689] update status limit for failover --- .../governance/common/entity/job/JobRequest.java | 10 ++++++++++ .../org/apache/linkis/entrance/EntranceServer.scala | 1 + .../service/impl/JobHistoryQueryServiceImpl.scala | 4 ++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java index d5d97aa3643..01f9df3f5da 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java @@ -51,6 +51,8 @@ public class JobRequest { private String observeInfo; + private Boolean updateLimitFlag = true; + private Map metrics = new HashMap<>(); public Long getId() { @@ -205,6 +207,14 @@ public void setObserveInfo(String observeInfo) { this.observeInfo = observeInfo; } + public Boolean getUpdateLimitFlag() { + return updateLimitFlag; + } + + public void setUpdateLimitFlag(Boolean updateLimitFlag) { + this.updateLimitFlag = updateLimitFlag; + } + @Override public String toString() { return "JobRequest{" diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index b09ef4911ad..42c7f8ea676 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -468,6 +468,7 @@ abstract class EntranceServer extends Logging { jobRequest.setErrorDesc("") jobRequest.setMetrics(metricMap) jobRequest.getMetrics.put(TaskConstant.ENTRANCEJOB_SUBMIT_TIME, initDate) + jobRequest.setUpdateLimitFlag(false) logAppender.append( LogUtils.generateInfo(s"Job ${jobRequest.getId} success to initialize the properties \n") diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala index c918ee085c8..bb90fee2dce 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala @@ -109,7 +109,7 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { logger.info(s"${jobReq.getErrorDesc}") } } - if (jobReq.getStatus != null) { + if (jobReq.getUpdateLimitFlag && jobReq.getStatus != null) { val oldStatus: String = jobHistoryMapper.selectJobHistoryStatusForUpdate(jobReq.getId) if (oldStatus != null && !shouldUpdate(oldStatus, jobReq.getStatus)) { throw new QueryException( @@ -174,7 +174,7 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { logger.info(s"${jobReq.getErrorDesc}") } } - if (jobReq.getStatus != null) { + if (jobReq.getUpdateLimitFlag && jobReq.getStatus != null) { val oldStatus: String = jobHistoryMapper.selectJobHistoryStatusForUpdate(jobReq.getId) if (oldStatus != null && !shouldUpdate(oldStatus, jobReq.getStatus)) { throw new QueryException( From d7eb30227d674a4339934951d8f1e475153351d7 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Fri, 25 Nov 2022 15:30:38 +0800 Subject: [PATCH 014/689] [bug-fix] failover logic --- .../scala/org/apache/linkis/entrance/EntranceServer.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index 42c7f8ea676..ad386be806e 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -282,6 +282,7 @@ abstract class EntranceServer extends Logging { logger.info( s"job ${jobRequest.getId} is not running,scheduled or not have EC info, ignore it" ) + return } val engineMap = jobRequest.getMetrics @@ -294,7 +295,7 @@ abstract class EntranceServer extends Logging { .filter(_.containsKey(TaskConstant.ENGINE_INSTANCE)) .maxBy(_.getOrDefault(TaskConstant.ENGINE_CONN_SUBMIT_TIME, "0").toString) - if (engineInstance != null || engineInstance.containsKey(TaskConstant.FAILOVER_FLAG)) { + if (engineInstance == null || engineInstance.containsKey(TaskConstant.FAILOVER_FLAG)) { logger.info( s"job ${jobRequest.getId} do not submit to EC or already failover, not need kill ec" ) @@ -328,8 +329,8 @@ abstract class EntranceServer extends Logging { s"job ${jobRequest.getId} send RequestTaskKill to kill engineConn $ecInstance, execID $engineTaskId" ) } - } { case e: Exception => - logger.error(s"job ${jobRequest.getId} kill ec error", e) + } { t => + logger.error(s"job ${jobRequest.getId} kill ec error", t) } } From 88017f36df3cabb72b6d0c9c72e4723b9229757a Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Fri, 25 Nov 2022 15:31:33 +0800 Subject: [PATCH 015/689] add failover log --- .../server/EntranceFailoverJobServer.java | 9 ++-- .../linkis/entrance/EntranceServer.scala | 45 +++++++++++++++---- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java index 1eb29a48fb3..cebb7c68b5e 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java @@ -116,15 +116,12 @@ public void run() { expiredTimestamp, EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); if (jobRequests.isEmpty()) return; - logger.info( - "success query failover jobs , job ids: {}", - jobRequests.stream().map(JobRequest::getId)); + Object[] ids = jobRequests.stream().map(JobRequest::getId).toArray(); + logger.info("success query failover jobs , job ids: {}", ids); // failover to local server jobRequests.forEach(jobRequest -> entranceServer.failoverExecute(jobRequest)); - logger.info( - "success execute failover jobs, job ids: {}", - jobRequests.stream().map(JobRequest::getId)); + logger.info("success execute failover jobs, job ids: {}", ids); } catch (Exception e) { logger.error("failover failed", e); diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index ad386be806e..4ba011a5c32 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -43,8 +43,8 @@ import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.linkis.common.log.LogUtils -import java.text.MessageFormat import java.{lang, util} +import java.text.{MessageFormat, SimpleDateFormat} import java.util.Date import scala.collection.JavaConverters._ @@ -360,6 +360,8 @@ abstract class EntranceServer extends Logging { .createPersistenceEngine() .updateIfNeeded(jobRequest) + logger.info(s"job ${jobRequest.getId} update JobRequest success") + val job = getEntranceContext.getOrCreateEntranceParser().parseToJob(jobRequest) Utils.tryThrow { job.init() @@ -386,7 +388,9 @@ abstract class EntranceServer extends Logging { */ Utils.tryAndWarn(job.getJobListener.foreach(_.onJobInited(job))) getEntranceContext.getOrCreateScheduler().submit(job) - val msg = s"Job with jobId : ${jobRequest.getId} and execID : ${job.getId()} submitted, success to failover" + val msg = LogUtils.generateInfo( + s"Job with jobId : ${jobRequest.getId} and execID : ${job.getId()} submitted, success to failover" + ) logger.info(msg) job match { @@ -421,22 +425,36 @@ abstract class EntranceServer extends Logging { } - private def initJobRequestProperties(jobRequest: JobRequest, logAppender: lang.StringBuilder): Unit = { - + private def initJobRequestProperties( + jobRequest: JobRequest, + logAppender: lang.StringBuilder + ): Unit = { + logger.info(s"Job ${jobRequest.getId} start to initialize the properties") + val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") val initInstance = Sender.getThisInstance val initDate = new Date(System.currentTimeMillis) val initStatus = SchedulerEventState.Inited.toString val initProgress = "0.0" val initReqId = "" + logAppender.append("\n\n") logAppender.append( - LogUtils.generateInfo(s"Job ${jobRequest.getId} start to failover, Initialize the properties \n") + LogUtils + .generateInfo( + s"*************************************FAILOVER************************************** \n" + ) + ) + logAppender.append( + LogUtils + .generateInfo(s"Job ${jobRequest.getId} start to failover, Initialize the properties \n") ) logAppender.append( LogUtils.generateInfo(s"the instances ${jobRequest.getInstances} -> ${initInstance} \n") ) logAppender.append( - LogUtils.generateInfo(s"the created_time ${jobRequest.getCreatedTime} -> ${initDate} \n") + LogUtils.generateInfo( + s"the created_time ${sdf.format(jobRequest.getCreatedTime)} -> ${sdf.format(initDate)} \n" + ) ) logAppender.append( LogUtils.generateInfo(s"the status ${jobRequest.getStatus} -> $initStatus \n") @@ -444,9 +462,6 @@ abstract class EntranceServer extends Logging { logAppender.append( LogUtils.generateInfo(s"the progress ${jobRequest.getProgress} -> $initProgress \n") ) - logAppender.append( - LogUtils.generateInfo(s"the job_req_id ${jobRequest.getReqId} -> $initReqId \n") - ) val metricMap = new util.HashMap[String, Object]() if ( @@ -460,6 +475,17 @@ abstract class EntranceServer extends Logging { metricMap.put(TaskConstant.ENTRANCEJOB_ENGINECONN_MAP, oldEngineconnMap) } + if ( + jobRequest.getMetrics != null && jobRequest.getMetrics.containsKey( + TaskConstant.ENTRANCEJOB_YARNRESOURCE + ) + ) { + val oldResourceMap = jobRequest.getMetrics + .get(TaskConstant.ENTRANCEJOB_YARNRESOURCE) + .asInstanceOf[util.Map[String, Object]] + metricMap.put(TaskConstant.ENTRANCEJOB_YARNRESOURCE, oldResourceMap) + } + jobRequest.setInstances(initInstance) jobRequest.setCreatedTime(initDate) jobRequest.setStatus(initStatus) @@ -474,6 +500,7 @@ abstract class EntranceServer extends Logging { logAppender.append( LogUtils.generateInfo(s"Job ${jobRequest.getId} success to initialize the properties \n") ) + logger.info(s"Job ${jobRequest.getId} success to initialize the properties") } } From e32118958d339749ef5b414ce4abfd1bda4882ab Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Fri, 25 Nov 2022 16:34:07 +0800 Subject: [PATCH 016/689] push log to entrance --- .../linkis/entrance/EntranceServer.scala | 101 ++++++++++-------- .../entrance/conf/EntranceConfiguration.scala | 6 ++ 2 files changed, 63 insertions(+), 44 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index 4ba011a5c32..b18441549ae 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -21,6 +21,7 @@ import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.exception.{ErrorException, LinkisException, LinkisRuntimeException} import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.entrance.conf.EntranceConfiguration import org.apache.linkis.entrance.cs.CSEntranceHelper import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ import org.apache.linkis.entrance.exception.{EntranceErrorException, SubmitFailedException} @@ -271,17 +272,25 @@ abstract class EntranceServer extends Logging { logger.info("Finished to clean all ConsumeQueue") } - def killEC(jobRequest: JobRequest): Unit = { + def killEC(jobRequest: JobRequest, logAppender: lang.StringBuilder): Unit = { Utils.tryCatch { if ( !SchedulerEventState.isRunning(SchedulerEventState.withName(jobRequest.getStatus)) || !SchedulerEventState.isScheduled(SchedulerEventState.withName(jobRequest.getStatus)) - || jobRequest.getMetrics == null + ) { + val msg = s"job ${jobRequest.getId} status is not running or scheduled, ignore it" + logger.info(msg) + logAppender.append(LogUtils.generateInfo(msg) + "\n") + return + } + + if ( + jobRequest.getMetrics == null || !jobRequest.getMetrics.containsKey(TaskConstant.ENTRANCEJOB_ENGINECONN_MAP) ) { - logger.info( - s"job ${jobRequest.getId} is not running,scheduled or not have EC info, ignore it" - ) + val msg = s"job ${jobRequest.getId} not have EC info, ignore it" + logger.info(msg) + logAppender.append(LogUtils.generateInfo(msg) + "\n") return } @@ -296,9 +305,10 @@ abstract class EntranceServer extends Logging { .maxBy(_.getOrDefault(TaskConstant.ENGINE_CONN_SUBMIT_TIME, "0").toString) if (engineInstance == null || engineInstance.containsKey(TaskConstant.FAILOVER_FLAG)) { - logger.info( + val msg = s"job ${jobRequest.getId} do not submit to EC or already failover, not need kill ec" - ) + logger.info(msg) + logAppender.append(LogUtils.generateInfo(msg) + "\n") return } engineInstance.put(TaskConstant.FAILOVER_FLAG, "") @@ -315,9 +325,10 @@ abstract class EntranceServer extends Logging { Sender .getSender(RPCConfiguration.LINKIS_MANAGER_APPLICATION_NAME.getValue) .send(engineStopRequest) - logger.info( + val msg = s"job ${jobRequest.getId} send EngineStopRequest to linkismanager, kill instance $ecInstance" - ) + logger.info(msg) + logAppender.append(LogUtils.generateInfo(msg) + "\n") } else if (engineInstance.containsKey(TaskConstant.ENGINE_CONN_TASK_ID)) { // kill ec task val engineTaskId = engineInstance.get(TaskConstant.ENGINE_CONN_TASK_ID).toString @@ -325,9 +336,10 @@ abstract class EntranceServer extends Logging { Sender .getSender(ecInstance) .send(RequestTaskKill(engineTaskId)) - logger.info( + val msg = s"job ${jobRequest.getId} send RequestTaskKill to kill engineConn $ecInstance, execID $engineTaskId" - ) + logger.info(msg) + logAppender.append(LogUtils.generateInfo(msg) + "\n") } } { t => logger.error(s"job ${jobRequest.getId} kill ec error", t) @@ -347,11 +359,15 @@ abstract class EntranceServer extends Logging { PERSIST_JOBREQUEST_ERROR.getErrorDesc ) } - - // try to kill ec - killEC(jobRequest); - val logAppender = new java.lang.StringBuilder() + logAppender.append( + LogUtils + .generateInfo( + s"\n\n *************************************FAILOVER************************************** \n" + ) + ) + // try to kill ec + killEC(jobRequest, logAppender); // init properties initJobRequestProperties(jobRequest, logAppender) // update jobRequest @@ -429,7 +445,7 @@ abstract class EntranceServer extends Logging { jobRequest: JobRequest, logAppender: lang.StringBuilder ): Unit = { - logger.info(s"Job ${jobRequest.getId} start to initialize the properties") + logger.info(s"job ${jobRequest.getId} start to initialize the properties") val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") val initInstance = Sender.getThisInstance val initDate = new Date(System.currentTimeMillis) @@ -437,16 +453,9 @@ abstract class EntranceServer extends Logging { val initProgress = "0.0" val initReqId = "" - logAppender.append("\n\n") logAppender.append( LogUtils - .generateInfo( - s"*************************************FAILOVER************************************** \n" - ) - ) - logAppender.append( - LogUtils - .generateInfo(s"Job ${jobRequest.getId} start to failover, Initialize the properties \n") + .generateInfo(s"job ${jobRequest.getId} start to failover, Initialize the properties \n") ) logAppender.append( LogUtils.generateInfo(s"the instances ${jobRequest.getInstances} -> ${initInstance} \n") @@ -464,26 +473,30 @@ abstract class EntranceServer extends Logging { ) val metricMap = new util.HashMap[String, Object]() - if ( - jobRequest.getMetrics != null && jobRequest.getMetrics.containsKey( - TaskConstant.ENTRANCEJOB_ENGINECONN_MAP - ) - ) { - val oldEngineconnMap = jobRequest.getMetrics - .get(TaskConstant.ENTRANCEJOB_ENGINECONN_MAP) - .asInstanceOf[util.Map[String, Object]] - metricMap.put(TaskConstant.ENTRANCEJOB_ENGINECONN_MAP, oldEngineconnMap) + if (EntranceConfiguration.ENTRANCE_FAILOVER_RETAIN_ENGINE_CONN_ENABLED.getValue) { + if ( + jobRequest.getMetrics != null && jobRequest.getMetrics.containsKey( + TaskConstant.ENTRANCEJOB_ENGINECONN_MAP + ) + ) { + val oldEngineconnMap = jobRequest.getMetrics + .get(TaskConstant.ENTRANCEJOB_ENGINECONN_MAP) + .asInstanceOf[util.Map[String, Object]] + metricMap.put(TaskConstant.ENTRANCEJOB_ENGINECONN_MAP, oldEngineconnMap) + } } - if ( - jobRequest.getMetrics != null && jobRequest.getMetrics.containsKey( - TaskConstant.ENTRANCEJOB_YARNRESOURCE - ) - ) { - val oldResourceMap = jobRequest.getMetrics - .get(TaskConstant.ENTRANCEJOB_YARNRESOURCE) - .asInstanceOf[util.Map[String, Object]] - metricMap.put(TaskConstant.ENTRANCEJOB_YARNRESOURCE, oldResourceMap) + if (EntranceConfiguration.ENTRANCE_FAILOVER_RETAIN_YARN_RESOURCE_ENABLED.getValue) { + if ( + jobRequest.getMetrics != null && jobRequest.getMetrics.containsKey( + TaskConstant.ENTRANCEJOB_YARNRESOURCE + ) + ) { + val oldResourceMap = jobRequest.getMetrics + .get(TaskConstant.ENTRANCEJOB_YARNRESOURCE) + .asInstanceOf[util.Map[String, Object]] + metricMap.put(TaskConstant.ENTRANCEJOB_YARNRESOURCE, oldResourceMap) + } } jobRequest.setInstances(initInstance) @@ -498,9 +511,9 @@ abstract class EntranceServer extends Logging { jobRequest.setUpdateLimitFlag(false) logAppender.append( - LogUtils.generateInfo(s"Job ${jobRequest.getId} success to initialize the properties \n") + LogUtils.generateInfo(s"job ${jobRequest.getId} success to initialize the properties \n") ) - logger.info(s"Job ${jobRequest.getId} success to initialize the properties") + logger.info(s"job ${jobRequest.getId} success to initialize the properties") } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index 62c42cfdd0e..ada20480977 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -246,4 +246,10 @@ object EntranceConfiguration { val ENTRANCE_GROUP_SCAN_ENABLED = CommonVars("linkis.entrance.group.scan.enable", true) + val ENTRANCE_FAILOVER_RETAIN_ENGINE_CONN_ENABLED = + CommonVars("linkis.entrance.failover.retain.engine.conn.enable", true) + + val ENTRANCE_FAILOVER_RETAIN_YARN_RESOURCE_ENABLED = + CommonVars("linkis.entrance.failover.retain.yarn.resource.enable", true) + } From 2dc0c7297a8d18051c24f74b5c3397238f0399ca Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Fri, 25 Nov 2022 20:56:55 +0800 Subject: [PATCH 017/689] add entrance log --- .../org/apache/linkis/entrance/EntranceServer.scala | 8 ++++++-- .../entrance/scheduler/EntranceGroupFactory.scala | 10 ++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index b18441549ae..354dafa118d 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -274,6 +274,10 @@ abstract class EntranceServer extends Logging { def killEC(jobRequest: JobRequest, logAppender: lang.StringBuilder): Unit = { Utils.tryCatch { + logAppender.append( + LogUtils + .generateInfo(s"job ${jobRequest.getId} start to kill ec \n") + ) if ( !SchedulerEventState.isRunning(SchedulerEventState.withName(jobRequest.getStatus)) || !SchedulerEventState.isScheduled(SchedulerEventState.withName(jobRequest.getStatus)) @@ -363,7 +367,7 @@ abstract class EntranceServer extends Logging { logAppender.append( LogUtils .generateInfo( - s"\n\n *************************************FAILOVER************************************** \n" + s"\n\n*************************************FAILOVER************************************** \n\n" ) ) // try to kill ec @@ -455,7 +459,7 @@ abstract class EntranceServer extends Logging { logAppender.append( LogUtils - .generateInfo(s"job ${jobRequest.getId} start to failover, Initialize the properties \n") + .generateInfo(s"job ${jobRequest.getId} start to Initialize the properties \n") ) logAppender.append( LogUtils.generateInfo(s"the instances ${jobRequest.getInstances} -> ${initInstance} \n") diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala index 2a7432ee6e6..c38fae5e4a9 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala @@ -193,18 +193,20 @@ class EntranceGroupFactory extends GroupFactory with Logging { group } - def refreshAllGroupMaxAllowRunningJobs(activeCount: Int): Unit = { - if (activeCount <= 0) return + def refreshAllGroupMaxAllowRunningJobs(validInsCount: Int): Unit = { + if (validInsCount <= 0) return groupNameToGroups .asMap() .asScala .foreach(item => { item._2 match { case group: ParallelGroup => - val maxAllowRunningJobs = Math.round(group.getMaxRunningJobs / activeCount) + val maxAllowRunningJobs = Math.round(group.getMaxRunningJobs / validInsCount) group.setMaxAllowRunningJobs(maxAllowRunningJobs) logger - .info(s"group ${group.getGroupName} update maxAllowRunningJobs $maxAllowRunningJobs") + .info( + s"group ${group.getGroupName} refresh maxAllowRunningJobs => ${group.getMaxRunningJobs}/$validInsCount=$maxAllowRunningJobs" + ) case _ => } }) From fe501c65eb9e7f29ba84ebf559bdae0226d52bd6 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Sun, 27 Nov 2022 20:51:10 +0800 Subject: [PATCH 018/689] update failover scan interval --- .../server/EntranceFailoverJobServer.java | 184 +++++++++--------- .../entrance/conf/EntranceConfiguration.scala | 2 +- 2 files changed, 97 insertions(+), 89 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java index cebb7c68b5e..a2bf9005361 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java @@ -67,94 +67,102 @@ public void init() { failoverTask(); } - public void failoverTask() { - if (EntranceConfiguration.ENTRANCE_FAILOVER_ENABLED()) { - Utils.defaultScheduler().scheduleAtFixedRate( - new Runnable() { - @Override - public void run() { - EntranceSchedulerContext schedulerContext = (EntranceSchedulerContext) entranceServer.getEntranceContext().getOrCreateScheduler().getSchedulerContext(); - - // entrance do not failover job when it is offline - if (schedulerContext.getOfflineFlag()) return; - - CommonLock commonLock = new CommonLock(); - commonLock.setLockObject(ENTRANCE_FAILOVER_LOCK); - Boolean locked = false; - try { - locked = commonLockService.lock(commonLock, 10 * 1000L); - if (!locked) return; - logger.info("success locked {}", ENTRANCE_FAILOVER_LOCK); - - // serverInstance to map - Map serverInstanceMap = - getActiveServerInstances().stream() - .collect( - Collectors.toMap( - ServiceInstance::getInstance, - ServiceInstance::getRegistryTimestamp, - (k1, k2) -> k2)); - if (serverInstanceMap.isEmpty()) return; - - // get failover job expired time (获取任务故障转移过期时间，配置为0表示不过期, 过期则不处理) - long expiredTimestamp = 0L; - if (EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME() > 0) { - expiredTimestamp = - System.currentTimeMillis() - - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME(); - } - - // get uncompleted status - List statusList = - Arrays.stream(SchedulerEventState.uncompleteStatusArray()) - .map(Object::toString).collect(Collectors.toList()); - - List jobRequests = - JobHistoryHelper.queryWaitForFailoverTask( - serverInstanceMap, - statusList, - expiredTimestamp, - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); - if (jobRequests.isEmpty()) return; - Object[] ids = jobRequests.stream().map(JobRequest::getId).toArray(); - logger.info("success query failover jobs , job ids: {}", ids); - - // failover to local server - jobRequests.forEach(jobRequest -> entranceServer.failoverExecute(jobRequest)); - logger.info("success execute failover jobs, job ids: {}", ids); - - } catch (Exception e) { - logger.error("failover failed", e); - } finally { - if (locked) commonLockService.unlock(commonLock); - } - } - }, - EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INIT_TIME(), - EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INTERVAL(), - TimeUnit.MILLISECONDS - ); - } - } - - private List getActiveServerInstances() { - // get all entrance server from eureka - ServiceInstance[] serviceInstances = Sender.getInstances(Sender.getThisServiceInstance().getApplicationName()); - if (serviceInstances == null || serviceInstances.length <= 0) return Lists.newArrayList(); - - // get all offline label server - RouteLabel routeLabel = LabelBuilderFactoryContext.getLabelBuilderFactory() - .createLabel(LabelKeyConstant.ROUTE_KEY, LabelConstant.OFFLINE); - List> labels = Lists.newArrayList(); - labels.add(routeLabel); - List labelInstances = InstanceLabelClient.getInstance().getInstanceFromLabel(labels); - - // get active entrance server - List allInstances = Lists.newArrayList(); - allInstances.addAll(Arrays.asList(serviceInstances)); - allInstances.removeAll(labelInstances); - - return allInstances; + public void failoverTask() { + if (EntranceConfiguration.ENTRANCE_FAILOVER_ENABLED()) { + Utils.defaultScheduler() + .scheduleWithFixedDelay( + () -> { + EntranceSchedulerContext schedulerContext = + (EntranceSchedulerContext) + entranceServer + .getEntranceContext() + .getOrCreateScheduler() + .getSchedulerContext(); + + // entrance do not failover job when it is offline + if (schedulerContext.getOfflineFlag()) return; + + CommonLock commonLock = new CommonLock(); + commonLock.setLockObject(ENTRANCE_FAILOVER_LOCK); + Boolean locked = false; + try { + locked = commonLockService.lock(commonLock, 10 * 1000L); + if (!locked) return; + logger.info("success locked {}", ENTRANCE_FAILOVER_LOCK); + + // serverInstance to map + Map serverInstanceMap = + getActiveServerInstances().stream() + .collect( + Collectors.toMap( + ServiceInstance::getInstance, + ServiceInstance::getRegistryTimestamp, + (k1, k2) -> k2)); + if (serverInstanceMap.isEmpty()) return; + + // get failover job expired time (获取任务故障转移过期时间，配置为0表示不过期, 过期则不处理) + long expiredTimestamp = 0L; + if (EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME() > 0) { + expiredTimestamp = + System.currentTimeMillis() + - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME(); + } + + // get uncompleted status + List statusList = + Arrays.stream(SchedulerEventState.uncompleteStatusArray()) + .map(Object::toString) + .collect(Collectors.toList()); + + List jobRequests = + JobHistoryHelper.queryWaitForFailoverTask( + serverInstanceMap, + statusList, + expiredTimestamp, + EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); + if (jobRequests.isEmpty()) return; + Object[] ids = jobRequests.stream().map(JobRequest::getId).toArray(); + logger.info("success query failover jobs , job ids: {}", ids); + + // failover to local server + jobRequests.forEach(jobRequest -> entranceServer.failoverExecute(jobRequest)); + logger.info("success execute failover jobs, job ids: {}", ids); + + } catch (Exception e) { + logger.error("failover failed", e); + } finally { + if (locked) commonLockService.unlock(commonLock); + } + }, + EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INIT_TIME(), + EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INTERVAL(), + TimeUnit.MILLISECONDS); } + } + + private List getActiveServerInstances() { + // get all entrance server from eureka + ServiceInstance[] serviceInstances = + Sender.getInstances(Sender.getThisServiceInstance().getApplicationName()); + if (serviceInstances == null || serviceInstances.length <= 0) return Lists.newArrayList(); + + // get all offline label server + RouteLabel routeLabel = + LabelBuilderFactoryContext.getLabelBuilderFactory() + .createLabel(LabelKeyConstant.ROUTE_KEY, LabelConstant.OFFLINE); + List> labels = Lists.newArrayList(); + labels.add(routeLabel); + List labelInstances = + InstanceLabelClient.getInstance().getInstanceFromLabel(labels); + if (labelInstances == null) labelInstances = Lists.newArrayList(); + + // get active entrance server + List allInstances = Lists.newArrayList(); + allInstances.addAll(Arrays.asList(serviceInstances)); + allInstances.removeAll(labelInstances); + + return allInstances; + } +} } \ No newline at end of file diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index ada20480977..907b67e89e7 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -231,7 +231,7 @@ object EntranceConfiguration { CommonVars("linkis.entrance.failover.scan.init.time", 3 * 1000).getValue val ENTRANCE_FAILOVER_SCAN_INTERVAL = - CommonVars("linkis.entrance.failover.scan.interval", 3 * 1000).getValue + CommonVars("linkis.entrance.failover.scan.interval", 30 * 1000).getValue val ENTRANCE_FAILOVER_DATA_NUM_LIMIT = CommonVars("linkis.entrance.failover.data.num.limit", 10).getValue From 170c0c9849ded01667b997df9246a0260ec9688b Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Mon, 28 Nov 2022 14:09:17 +0800 Subject: [PATCH 019/689] [Bug-fix] gateway choose instance --- .../gateway/ujes/parser/EntranceRequestGatewayParser.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala index a1be26de875..883f252d70d 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala @@ -103,10 +103,10 @@ class EntranceRequestGatewayParser extends AbstractGatewayParser { ) val instance = activeInstances .find(_.getInstance.equals(history.getInstances)) - .getOrElse(ServiceInstance("", "", Long.MaxValue)) + .getOrElse(ServiceInstance(null, null, Long.MaxValue)) JobInstance( history.getStatus, - history.getInstances, + instance.getInstance, history.getJobReqId, history.getCreatedTime.getTime, instance.getRegistryTimestamp From c91a6e8656644702b8bf851253a78d8f85d5db8c Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Mon, 28 Nov 2022 14:12:56 +0800 Subject: [PATCH 020/689] batch update instance --- .../server/DefaultEntranceServer.java | 15 ++-- .../linkis/entrance/EntranceServer.scala | 75 +++++++++++++++++-- .../entrance/utils/JobHistoryHelper.scala | 50 +++++++------ 3 files changed, 106 insertions(+), 34 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java index 999d5cbcbf9..443feb2a816 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java @@ -81,16 +81,19 @@ private void shutdownEntrance(ContextClosedEvent event) { logger.warn("event has been handled"); } else { if (EntranceConfiguration.ENTRANCE_SHUTDOWN_FAILOVER_ENABLED()) { - logger.warn("Entrance exit to update all not execution task instances and clean ConsumeQueue"); + logger.warn("Entrance exit to update and clean all ConsumeQueue task instances"); updateAllNotExecutionTaskInstances(false); } logger.warn("Entrance exit to stop all job"); - EntranceJob[] allUndoneJobs = getAllUndoneTask(null); - if (null != allUndoneJobs) { - for (EntranceJob job : allUndoneJobs) { - job.onFailure( - "Entrance exits the automatic cleanup task and can be rerun(服务退出自动清理任务，可以重跑)", null); + EntranceJob[] allUndoneTask = getAllUndoneTask(null); + if (null != allUndoneTask) { + String msg = "Entrance exits the automatic cleanup task and can be rerun(服务退出自动清理任务，可以重跑)"; + for (EntranceJob job : allUndoneTask) { + if (job.getLogListener().isDefined()) { + job.getLogListener().get().onLogUpdate(job, msg); + } + job.onFailure(msg, null); } } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index 354dafa118d..0c43bc8159f 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -44,6 +44,8 @@ import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.linkis.common.log.LogUtils +import org.springframework.beans.BeanUtils + import java.{lang, util} import java.text.{MessageFormat, SimpleDateFormat} import java.util.Date @@ -265,11 +267,14 @@ abstract class EntranceServer extends Logging { } def updateAllNotExecutionTaskInstances(retryWhenUpdateFail: Boolean): Unit = { - val taskIds = getAllConsumeQueueTask().map(_.getJobRequest.getId).toList - JobHistoryHelper.updateAllConsumeQueueTask(taskIds, retryWhenUpdateFail) - logger.info("Finished to update all not execution task instances") - clearAllConsumeQueue() - logger.info("Finished to clean all ConsumeQueue") + val consumeQueueTasks = getAllConsumeQueueTask() + if (consumeQueueTasks != null && consumeQueueTasks.length > 0) { + val taskIds = consumeQueueTasks.map(_.getJobRequest.getId.asInstanceOf[Long]).toList + clearAllConsumeQueue() + logger.info("Finished to clean all ConsumeQueue") + JobHistoryHelper.updateAllConsumeQueueTask(taskIds.asJava, retryWhenUpdateFail) + logger.info("Finished to update all not execution task instances") + } } def killEC(jobRequest: JobRequest, logAppender: lang.StringBuilder): Unit = { @@ -355,14 +360,18 @@ abstract class EntranceServer extends Logging { * * @param jobRequest */ - def failoverExecute(jobRequest: JobRequest): String = { + def failoverExecute(jobReq: JobRequest): String = { - if (null == jobRequest || null == jobRequest.getId || jobRequest.getId <= 0) { + if (null == jobReq || null == jobReq.getId || jobReq.getId <= 0) { throw new EntranceErrorException( PERSIST_JOBREQUEST_ERROR.getErrorCode, PERSIST_JOBREQUEST_ERROR.getErrorDesc ) } + + var jobRequest = new JobRequest + BeanUtils.copyProperties(jobReq, jobRequest) + val logAppender = new java.lang.StringBuilder() logAppender.append( LogUtils @@ -370,10 +379,62 @@ abstract class EntranceServer extends Logging { s"\n\n*************************************FAILOVER************************************** \n\n" ) ) + // try to kill ec killEC(jobRequest, logAppender); + + // if status is Inited, need to deal by all Interceptors, such as log_path + if (jobRequest.getStatus.equals(SchedulerEventState.Inited.toString)) { + Utils.tryThrow( + getEntranceContext + .getOrCreateEntranceInterceptors() + .foreach(int => jobRequest = int.apply(jobRequest, logAppender)) + ) { t => + val error = t match { + case error: ErrorException => error + case t1: Throwable => + val exception = new EntranceErrorException( + FAILED_ANALYSIS_TASK.getErrorCode, + MessageFormat.format( + FAILED_ANALYSIS_TASK.getErrorDesc, + ExceptionUtils.getRootCauseMessage(t) + ) + ) + exception.initCause(t1) + exception + case _ => + new EntranceErrorException( + FAILED_ANALYSIS_TASK.getErrorCode, + MessageFormat.format( + FAILED_ANALYSIS_TASK.getErrorDesc, + ExceptionUtils.getRootCauseMessage(t) + ) + ) + } + jobRequest match { + case t: JobRequest => + t.setErrorCode(error.getErrCode) + t.setErrorDesc(error.getDesc) + t.setStatus(SchedulerEventState.Failed.toString) + t.setProgress(EntranceJob.JOB_COMPLETED_PROGRESS.toString) + val infoMap = new util.HashMap[String, Object] + infoMap.put(TaskConstant.ENGINE_INSTANCE, "NULL") + infoMap.put(TaskConstant.TICKET_ID, "") + infoMap.put("message", "Task interception failed and cannot be retried") + JobHistoryHelper.updateJobRequestMetrics(jobRequest, null, infoMap) + case _ => + } + getEntranceContext + .getOrCreatePersistenceManager() + .createPersistenceEngine() + .updateIfNeeded(jobRequest) + error + } + } + // init properties initJobRequestProperties(jobRequest, logAppender) + // update jobRequest getEntranceContext .getOrCreatePersistenceManager() diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala index 811af8fce5c..7a55124f75e 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala @@ -132,37 +132,40 @@ object JobHistoryHelper extends Logging { * @param taskIdList * @param retryWhenUpdateFail */ - def updateAllConsumeQueueTask(taskIdList: List[java.lang.Long], retryWhenUpdateFail: Boolean = false): Unit = { + def updateAllConsumeQueueTask( + taskIdList: util.List[Long], + retryWhenUpdateFail: Boolean = false + ): Unit = { if (taskIdList.isEmpty) return - val updateTaskIds = new util.ArrayList[java.lang.Long]() + val updateTaskIds = new util.ArrayList[Long]() if ( EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue > 0 && - taskIdList.length > EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue + taskIdList.size() > EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue ) { for (i <- 0 until EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue) { - updateTaskIds.add(taskIdList(i)) + updateTaskIds.add(taskIdList.get(i)) } } else { - updateTaskIds.addAll(taskIdList.asJava) + updateTaskIds.addAll(taskIdList) } - + val list = new util.ArrayList[Long]() + list.addAll(taskIdList) try { - val successTaskIds = updateBatchInstances(updateTaskIds.asScala.toList) + val successTaskIds = updateBatchInstances(updateTaskIds) if (retryWhenUpdateFail) { - taskIdList.asJava.removeAll(successTaskIds.asJava) + list.removeAll(successTaskIds) } else { - taskIdList.asJava.removeAll(updateTaskIds) + list.removeAll(updateTaskIds) } } catch { case e: Exception => logger.warn("update batch instances failed, wait for retry", e) Thread.sleep(1000) } - - updateAllConsumeQueueTask(taskIdList, retryWhenUpdateFail) + updateAllConsumeQueueTask(list, retryWhenUpdateFail) } @@ -172,9 +175,9 @@ object JobHistoryHelper extends Logging { * @param taskIdList * @return */ - private def updateBatchInstances(taskIdList: List[java.lang.Long]): List[java.lang.Long] = { + private def updateBatchInstances(taskIdList: util.List[Long]): util.List[Long] = { val jobReqList = new util.ArrayList[JobRequest]() - taskIdList.foreach(taskID => { + taskIdList.asScala.foreach(taskID => { val jobRequest = new JobRequest jobRequest.setId(taskID) jobRequest.setInstances("") @@ -184,13 +187,16 @@ object JobHistoryHelper extends Logging { Utils.tryCatch { val response = sender.ask(jobReqBatchUpdate) response match { - case resp: util.ArrayList[JobRespProtocol] => - resp.asScala - .filter(r => - r.getStatus == SUCCESS_FLAG && r.getData.containsKey(JobRequestConstants.JOB_ID) - ) - .map(_.getData.get(JobRequestConstants.JOB_ID).asInstanceOf[java.lang.Long]) - .toList + case resp: util.List[JobRespProtocol] => + // todo filter success data, rpc have bug +// resp.asScala +// .filter(r => +// r.getStatus == SUCCESS_FLAG && r.getData.containsKey(JobRequestConstants.JOB_ID) +// ) +// .map(_.getData.get(JobRequestConstants.JOB_ID).asInstanceOf[java.lang.Long]) +// .toList + + taskIdList case _ => throw JobHistoryFailedException( "update batch instances from jobhistory not a correct List type" @@ -200,7 +206,9 @@ object JobHistoryHelper extends Logging { case errorException: ErrorException => throw errorException case e: Exception => val e1 = - JobHistoryFailedException(s"update batch instances ${taskIdList.mkString(",")} error") + JobHistoryFailedException( + s"update batch instances ${taskIdList.asScala.mkString(",")} error" + ) e1.initCause(e) throw e } From 8c55b180893ea2e8ba5b62d9ef162832e8c96d70 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Fri, 2 Dec 2022 11:12:34 +0800 Subject: [PATCH 021/689] failover status changed from Running to Cancelled --- .../linkis/entrance/EntranceServer.scala | 158 ++++++++++++------ .../entrance/conf/EntranceConfiguration.scala | 3 + 2 files changed, 106 insertions(+), 55 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index 0c43bc8159f..afeb23e8208 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -19,6 +19,7 @@ package org.apache.linkis.entrance import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.exception.{ErrorException, LinkisException, LinkisRuntimeException} +import org.apache.linkis.common.io.FsPath import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.entrance.conf.EntranceConfiguration @@ -26,7 +27,7 @@ import org.apache.linkis.entrance.cs.CSEntranceHelper import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ import org.apache.linkis.entrance.exception.{EntranceErrorException, SubmitFailedException} import org.apache.linkis.entrance.execute.EntranceJob -import org.apache.linkis.entrance.log.LogReader +import org.apache.linkis.entrance.log.{Cache, HDFSCacheLogWriter, LogReader} import org.apache.linkis.entrance.timeout.JobTimeoutManager import org.apache.linkis.entrance.utils.JobHistoryHelper import org.apache.linkis.governance.common.conf.GovernanceCommonConf @@ -39,6 +40,7 @@ import org.apache.linkis.rpc.Sender import org.apache.linkis.rpc.conf.RPCConfiguration import org.apache.linkis.scheduler.queue.{Job, SchedulerEventState} import org.apache.linkis.server.conf.ServerConfiguration +import org.apache.linkis.storage.utils.StorageUtils import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils @@ -355,23 +357,107 @@ abstract class EntranceServer extends Logging { } } + def dealInitedJobRequest(jobRequest: JobRequest, logAppender: lang.StringBuilder): Unit = { + Utils.tryThrow( + getEntranceContext + .getOrCreateEntranceInterceptors() + .foreach(int => int.apply(jobRequest, logAppender)) + ) { t => + val error = t match { + case error: ErrorException => error + case t1: Throwable => + val exception = new EntranceErrorException( + FAILED_ANALYSIS_TASK.getErrorCode, + MessageFormat.format( + FAILED_ANALYSIS_TASK.getErrorDesc, + ExceptionUtils.getRootCauseMessage(t) + ) + ) + exception.initCause(t1) + exception + case _ => + new EntranceErrorException( + FAILED_ANALYSIS_TASK.getErrorCode, + MessageFormat.format( + FAILED_ANALYSIS_TASK.getErrorDesc, + ExceptionUtils.getRootCauseMessage(t) + ) + ) + } + jobRequest match { + case t: JobRequest => + t.setErrorCode(error.getErrCode) + t.setErrorDesc(error.getDesc) + t.setStatus(SchedulerEventState.Failed.toString) + t.setProgress(EntranceJob.JOB_COMPLETED_PROGRESS.toString) + val infoMap = new util.HashMap[String, Object] + infoMap.put(TaskConstant.ENGINE_INSTANCE, "NULL") + infoMap.put(TaskConstant.TICKET_ID, "") + infoMap.put("message", "Task interception failed and cannot be retried") + JobHistoryHelper.updateJobRequestMetrics(jobRequest, null, infoMap) + case _ => + } + getEntranceContext + .getOrCreatePersistenceManager() + .createPersistenceEngine() + .updateIfNeeded(jobRequest) + error + } + + } + + def dealRunningJobRequest(jobRequest: JobRequest): Unit = { + Utils.tryCatch { + // init jobRequest properties + jobRequest.setStatus(SchedulerEventState.Cancelled.toString) + jobRequest.setProgress("1.0") + jobRequest.setInstances(Sender.getThisInstance) + + // update jobRequest + getEntranceContext + .getOrCreatePersistenceManager() + .createPersistenceEngine() + .updateIfNeeded(jobRequest) + + // append log + val logPath = jobRequest.getLogPath + if (StringUtils.isNotBlank(logPath)) { + val fsLogPath = new FsPath(logPath) + if (StorageUtils.HDFS == fsLogPath.getFsType) { + val logWriter = new HDFSCacheLogWriter( + logPath, + EntranceConfiguration.DEFAULT_LOG_CHARSET.getValue, + Cache(1), + jobRequest.getExecuteUser + ) + + val msg = + s"Job ${jobRequest.getId} failover, status changed from Running to Cancelled (任务故障转移，状态从Running变更为Cancelled)" + logWriter.write(msg) + logWriter.flush() + logWriter.close() + } + } + } { case e: Exception => + logger.error(s"Job ${jobRequest.getId} failover, change status error", e) + } + + } + /** * execute failover job (提交故障转移任务，返回新的execId) * * @param jobRequest */ - def failoverExecute(jobReq: JobRequest): String = { + def failoverExecute(jobRequest: JobRequest): Unit = { - if (null == jobReq || null == jobReq.getId || jobReq.getId <= 0) { + if (null == jobRequest || null == jobRequest.getId || jobRequest.getId <= 0) { throw new EntranceErrorException( PERSIST_JOBREQUEST_ERROR.getErrorCode, PERSIST_JOBREQUEST_ERROR.getErrorDesc ) } - var jobRequest = new JobRequest - BeanUtils.copyProperties(jobReq, jobRequest) - val logAppender = new java.lang.StringBuilder() logAppender.append( LogUtils @@ -383,53 +469,18 @@ abstract class EntranceServer extends Logging { // try to kill ec killEC(jobRequest, logAppender); - // if status is Inited, need to deal by all Interceptors, such as log_path + // deal Inited jobRequest, if status is Inited, need to deal by all Interceptors, such as log_path if (jobRequest.getStatus.equals(SchedulerEventState.Inited.toString)) { - Utils.tryThrow( - getEntranceContext - .getOrCreateEntranceInterceptors() - .foreach(int => jobRequest = int.apply(jobRequest, logAppender)) - ) { t => - val error = t match { - case error: ErrorException => error - case t1: Throwable => - val exception = new EntranceErrorException( - FAILED_ANALYSIS_TASK.getErrorCode, - MessageFormat.format( - FAILED_ANALYSIS_TASK.getErrorDesc, - ExceptionUtils.getRootCauseMessage(t) - ) - ) - exception.initCause(t1) - exception - case _ => - new EntranceErrorException( - FAILED_ANALYSIS_TASK.getErrorCode, - MessageFormat.format( - FAILED_ANALYSIS_TASK.getErrorDesc, - ExceptionUtils.getRootCauseMessage(t) - ) - ) - } - jobRequest match { - case t: JobRequest => - t.setErrorCode(error.getErrCode) - t.setErrorDesc(error.getDesc) - t.setStatus(SchedulerEventState.Failed.toString) - t.setProgress(EntranceJob.JOB_COMPLETED_PROGRESS.toString) - val infoMap = new util.HashMap[String, Object] - infoMap.put(TaskConstant.ENGINE_INSTANCE, "NULL") - infoMap.put(TaskConstant.TICKET_ID, "") - infoMap.put("message", "Task interception failed and cannot be retried") - JobHistoryHelper.updateJobRequestMetrics(jobRequest, null, infoMap) - case _ => - } - getEntranceContext - .getOrCreatePersistenceManager() - .createPersistenceEngine() - .updateIfNeeded(jobRequest) - error - } + dealInitedJobRequest(jobRequest, logAppender) + } + + // deal Running jobRequest, if enabled, status changed from Running to Cancelled + if ( + EntranceConfiguration.ENTRANCE_FAILOVER_RUNNING_KILL_ENABLED.getValue && + jobRequest.getStatus.equals(SchedulerEventState.Running.toString) + ) { + dealRunningJobRequest(jobRequest) + return } // init properties @@ -482,8 +533,6 @@ abstract class EntranceServer extends Logging { entranceJob.getLogListener.foreach(_.onLogUpdate(entranceJob, msg)) case _ => } - - job.getId() } { t => job.onFailure("Submitting the query failed!(提交查询失败！)", t) val _jobRequest = @@ -503,7 +552,6 @@ abstract class EntranceServer extends Logging { ) } } - } private def initJobRequestProperties( diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index 907b67e89e7..10db3715fef 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -252,4 +252,7 @@ object EntranceConfiguration { val ENTRANCE_FAILOVER_RETAIN_YARN_RESOURCE_ENABLED = CommonVars("linkis.entrance.failover.retain.yarn.resource.enable", true) + val ENTRANCE_FAILOVER_RUNNING_KILL_ENABLED = + CommonVars("linkis.entrance.failover.running.kill.enable", true) + } From 7971904007b4d879df5b8fe8d020f55a04c3cec8 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Tue, 6 Dec 2022 19:38:35 +0800 Subject: [PATCH 022/689] entrance refactor failoverExecute --- .../service/TaskExecutionServiceImpl.scala | 2 +- .../errorcode/EntranceErrorCodeSummary.java | 6 +- .../entrance/restful/EntranceRestfulApi.java | 11 +- .../linkis/entrance/EntranceServer.scala | 159 ++++++++++-------- 4 files changed, 105 insertions(+), 73 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala index 039c1060c49..50110088b0c 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala @@ -184,7 +184,7 @@ class TaskExecutionServiceImpl if (!lockService.isLockExist(requestTask.getLock)) { logger.error(s"Lock ${requestTask.getLock} not exist, cannot execute.") return ErrorExecuteResponse( - "Lock not exixt", + "Lock not exist", new EngineConnExecutorErrorException( EngineConnExecutorErrorCode.INVALID_LOCK, "Lock : " + requestTask.getLock + " not exist(您的锁无效，请重新获取后再提交)." diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java index 2f045a17602..b5f90e30703 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java @@ -71,7 +71,11 @@ public enum EntranceErrorCodeSummary implements LinkisErrorCode { SHELL_BLACKLISTED_CODE(50081, "Shell code contains blacklisted code(shell中包含黑名单代码)"), JOB_HISTORY_FAILED_ID(50081, ""), - LOGPATH_NOT_NULL(20301, "The logPath cannot be empty(日志路径不能为空)"); + LOGPATH_NOT_NULL(20301, "The logPath cannot be empty(日志路径不能为空)"), + + FAILOVER_RUNNING_TO_CANCELLED( + 30001, + "Job {0} failover, status changed from Running to Cancelled (任务故障转移，状态从Running变更为Cancelled)"); /** (errorCode)错误码 */ private final int errorCode; diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index 8b10b9eb52b..b32923cc0d1 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -778,11 +778,12 @@ public Message killJobs( if (null != logListener) { logListener.onLogUpdate( entranceJob, - "Job " - + jobReq.getId() - + " was kill by user successfully(任务" - + jobReq.getId() - + "已成功取消)"); + LogUtils.generateInfo( + "Job " + + jobReq.getId() + + " was kill by user successfully(任务" + + jobReq.getId() + + "已成功取消)")); } this.entranceServer .getEntranceContext() diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index afeb23e8208..81c701720e6 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -24,10 +24,12 @@ import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.entrance.conf.EntranceConfiguration import org.apache.linkis.entrance.cs.CSEntranceHelper +import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ import org.apache.linkis.entrance.exception.{EntranceErrorException, SubmitFailedException} import org.apache.linkis.entrance.execute.EntranceJob -import org.apache.linkis.entrance.log.{Cache, HDFSCacheLogWriter, LogReader} +import org.apache.linkis.entrance.log.{Cache, CacheLogWriter, HDFSCacheLogWriter, LogReader} +import org.apache.linkis.entrance.parser.ParserUtils import org.apache.linkis.entrance.timeout.JobTimeoutManager import org.apache.linkis.entrance.utils.JobHistoryHelper import org.apache.linkis.governance.common.conf.GovernanceCommonConf @@ -279,7 +281,49 @@ abstract class EntranceServer extends Logging { } } - def killEC(jobRequest: JobRequest, logAppender: lang.StringBuilder): Unit = { + /** + * execute failover job (提交故障转移任务，返回新的execId) + * + * @param jobRequest + */ + def failoverExecute(jobRequest: JobRequest): Unit = { + + if (null == jobRequest || null == jobRequest.getId || jobRequest.getId <= 0) { + throw new EntranceErrorException( + PERSIST_JOBREQUEST_ERROR.getErrorCode, + PERSIST_JOBREQUEST_ERROR.getErrorDesc + ) + } + + val logAppender = new java.lang.StringBuilder() + logAppender.append( + LogUtils + .generateInfo( + s"\n\n*************************************FAILOVER************************************** \n\n" + ) + ) + + // try to kill ec + killOldEC(jobRequest, logAppender); + + // deal Inited jobRequest, if status is Inited, need to deal by all Interceptors, such as set log_path + if (jobRequest.getStatus.equals(SchedulerEventState.Inited.toString)) { + dealInitedJobRequest(jobRequest, logAppender) + } + + if ( + EntranceConfiguration.ENTRANCE_FAILOVER_RUNNING_KILL_ENABLED.getValue && + jobRequest.getStatus.equals(SchedulerEventState.Running.toString) + ) { + // deal Running jobRequest, if enabled, status changed from Running to Cancelled + dealRunningJobRequest(jobRequest, logAppender) + } else { + // init and submit + initAndSubmitJobRequest(jobRequest, logAppender) + } + } + + def killOldEC(jobRequest: JobRequest, logAppender: lang.StringBuilder): Unit = { Utils.tryCatch { logAppender.append( LogUtils @@ -332,18 +376,18 @@ abstract class EntranceServer extends Logging { // kill ec by linkismanager val engineStopRequest = new EngineStopRequest engineStopRequest.setServiceInstance(ecInstance) - // send to linkismanager + // send to linkismanager kill ec Sender .getSender(RPCConfiguration.LINKIS_MANAGER_APPLICATION_NAME.getValue) .send(engineStopRequest) val msg = - s"job ${jobRequest.getId} send EngineStopRequest to linkismanager, kill instance $ecInstance" + s"job ${jobRequest.getId} send EngineStopRequest to linkismanager, kill EC instance $ecInstance" logger.info(msg) logAppender.append(LogUtils.generateInfo(msg) + "\n") } else if (engineInstance.containsKey(TaskConstant.ENGINE_CONN_TASK_ID)) { - // kill ec task + // get ec taskId val engineTaskId = engineInstance.get(TaskConstant.ENGINE_CONN_TASK_ID).toString - // send to ec + // send to ec kill task Sender .getSender(ecInstance) .send(RequestTaskKill(engineTaskId)) @@ -403,15 +447,22 @@ abstract class EntranceServer extends Logging { .updateIfNeeded(jobRequest) error } - } - def dealRunningJobRequest(jobRequest: JobRequest): Unit = { + def dealRunningJobRequest(jobRequest: JobRequest, logAppender: lang.StringBuilder): Unit = { Utils.tryCatch { + // error_msg + val msg = + MessageFormat.format( + EntranceErrorCodeSummary.FAILOVER_RUNNING_TO_CANCELLED.getErrorDesc, + jobRequest.getId + ) // init jobRequest properties jobRequest.setStatus(SchedulerEventState.Cancelled.toString) jobRequest.setProgress("1.0") jobRequest.setInstances(Sender.getThisInstance) + jobRequest.setErrorCode(EntranceErrorCodeSummary.FAILOVER_RUNNING_TO_CANCELLED.getErrorCode) + jobRequest.setErrorDesc(msg) // update jobRequest getEntranceContext @@ -419,70 +470,46 @@ abstract class EntranceServer extends Logging { .createPersistenceEngine() .updateIfNeeded(jobRequest) - // append log - val logPath = jobRequest.getLogPath - if (StringUtils.isNotBlank(logPath)) { - val fsLogPath = new FsPath(logPath) - if (StorageUtils.HDFS == fsLogPath.getFsType) { - val logWriter = new HDFSCacheLogWriter( - logPath, - EntranceConfiguration.DEFAULT_LOG_CHARSET.getValue, - Cache(1), - jobRequest.getExecuteUser - ) - - val msg = - s"Job ${jobRequest.getId} failover, status changed from Running to Cancelled (任务故障转移，状态从Running变更为Cancelled)" - logWriter.write(msg) - logWriter.flush() - logWriter.close() - } + // getOrGenerate log_path + var logPath = jobRequest.getLogPath + if (StringUtils.isBlank(logPath)) { + ParserUtils.generateLogPath(jobRequest, null) + logPath = jobRequest.getLogPath + logAppender.append( + LogUtils.generateInfo(s"job ${jobRequest.getId} generate new logPath $logPath \n") + ) } - } { case e: Exception => - logger.error(s"Job ${jobRequest.getId} failover, change status error", e) - } - - } - - /** - * execute failover job (提交故障转移任务，返回新的execId) - * - * @param jobRequest - */ - def failoverExecute(jobRequest: JobRequest): Unit = { - - if (null == jobRequest || null == jobRequest.getId || jobRequest.getId <= 0) { - throw new EntranceErrorException( - PERSIST_JOBREQUEST_ERROR.getErrorCode, - PERSIST_JOBREQUEST_ERROR.getErrorDesc - ) - } - - val logAppender = new java.lang.StringBuilder() - logAppender.append( - LogUtils - .generateInfo( - s"\n\n*************************************FAILOVER************************************** \n\n" + val fsLogPath = new FsPath(logPath) + val cache = Cache(EntranceConfiguration.DEFAULT_CACHE_MAX.getHotValue()) + val logWriter = if (StorageUtils.HDFS == fsLogPath.getFsType) { + new HDFSCacheLogWriter( + logPath, + EntranceConfiguration.DEFAULT_LOG_CHARSET.getValue, + cache, + jobRequest.getExecuteUser ) - ) - - // try to kill ec - killEC(jobRequest, logAppender); + } else { + new CacheLogWriter( + logPath, + EntranceConfiguration.DEFAULT_LOG_CHARSET.getValue, + cache, + jobRequest.getExecuteUser + ) + } + if (logAppender.length() > 0) { + logWriter.write(logAppender.toString.trim) + } - // deal Inited jobRequest, if status is Inited, need to deal by all Interceptors, such as log_path - if (jobRequest.getStatus.equals(SchedulerEventState.Inited.toString)) { - dealInitedJobRequest(jobRequest, logAppender) - } + logWriter.write(LogUtils.generateInfo(msg) + "\n") + logWriter.flush() + logWriter.close() - // deal Running jobRequest, if enabled, status changed from Running to Cancelled - if ( - EntranceConfiguration.ENTRANCE_FAILOVER_RUNNING_KILL_ENABLED.getValue && - jobRequest.getStatus.equals(SchedulerEventState.Running.toString) - ) { - dealRunningJobRequest(jobRequest) - return + } { case e: Exception => + logger.error(s"Job ${jobRequest.getId} failover, change status error", e) } + } + def initAndSubmitJobRequest(jobRequest: JobRequest, logAppender: lang.StringBuilder): Unit = { // init properties initJobRequestProperties(jobRequest, logAppender) From ae9a172015333ae9ec93b8ee87e55db067858625 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Wed, 7 Dec 2022 18:20:08 +0800 Subject: [PATCH 023/689] change updateOrderFlag --- .../governance/common/entity/job/JobRequest.java | 12 ++++++------ .../org/apache/linkis/entrance/EntranceServer.scala | 2 +- .../service/impl/JobHistoryQueryServiceImpl.scala | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java index 01f9df3f5da..75134bd84ab 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java @@ -49,9 +49,9 @@ public class JobRequest { /** result location */ private String resultLocation; - private String observeInfo; + private Boolean updateOrderFlag = true; - private Boolean updateLimitFlag = true; + private String observeInfo; private Map metrics = new HashMap<>(); @@ -207,12 +207,12 @@ public void setObserveInfo(String observeInfo) { this.observeInfo = observeInfo; } - public Boolean getUpdateLimitFlag() { - return updateLimitFlag; + public Boolean getUpdateOrderFlag() { + return updateOrderFlag; } - public void setUpdateLimitFlag(Boolean updateLimitFlag) { - this.updateLimitFlag = updateLimitFlag; + public void setUpdateOrderFlag(Boolean updateOrderFlag) { + this.updateOrderFlag = updateOrderFlag; } @Override diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index 81c701720e6..d2a504100d1 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -648,7 +648,7 @@ abstract class EntranceServer extends Logging { jobRequest.setErrorDesc("") jobRequest.setMetrics(metricMap) jobRequest.getMetrics.put(TaskConstant.ENTRANCEJOB_SUBMIT_TIME, initDate) - jobRequest.setUpdateLimitFlag(false) + jobRequest.setUpdateOrderFlag(false) logAppender.append( LogUtils.generateInfo(s"job ${jobRequest.getId} success to initialize the properties \n") diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala index bb90fee2dce..22084f88a69 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala @@ -109,7 +109,7 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { logger.info(s"${jobReq.getErrorDesc}") } } - if (jobReq.getUpdateLimitFlag && jobReq.getStatus != null) { + if (jobReq.getUpdateOrderFlag && jobReq.getStatus != null) { val oldStatus: String = jobHistoryMapper.selectJobHistoryStatusForUpdate(jobReq.getId) if (oldStatus != null && !shouldUpdate(oldStatus, jobReq.getStatus)) { throw new QueryException( @@ -174,7 +174,7 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { logger.info(s"${jobReq.getErrorDesc}") } } - if (jobReq.getUpdateLimitFlag && jobReq.getStatus != null) { + if (jobReq.getUpdateOrderFlag && jobReq.getStatus != null) { val oldStatus: String = jobHistoryMapper.selectJobHistoryStatusForUpdate(jobReq.getId) if (oldStatus != null && !shouldUpdate(oldStatus, jobReq.getStatus)) { throw new QueryException( From a4a41d6d6a2f51c541d1e40749798c3fd7019e15 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Wed, 7 Dec 2022 18:22:55 +0800 Subject: [PATCH 024/689] edit failoverJobServer --- .../server/EntranceFailoverJobServer.java | 185 ++++++++++-------- 1 file changed, 100 insertions(+), 85 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java index a2bf9005361..b5f06886262 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java @@ -17,7 +17,6 @@ package org.apache.linkis.entrance.server; -import org.apache.commons.compress.utils.Lists; import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.common.utils.Utils; import org.apache.linkis.entrance.EntranceServer; @@ -28,115 +27,133 @@ import org.apache.linkis.governance.common.entity.job.JobRequest; import org.apache.linkis.instance.label.client.InstanceLabelClient; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; -import org.apache.linkis.manager.label.constant.LabelConstant; import org.apache.linkis.manager.label.constant.LabelKeyConstant; +import org.apache.linkis.manager.label.constant.LabelValueConstant; import org.apache.linkis.manager.label.entity.Label; import org.apache.linkis.manager.label.entity.route.RouteLabel; import org.apache.linkis.publicservice.common.lock.entity.CommonLock; import org.apache.linkis.publicservice.common.lock.service.CommonLockService; import org.apache.linkis.rpc.Sender; import org.apache.linkis.scheduler.queue.SchedulerEventState; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import org.apache.commons.compress.utils.Lists; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; + import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + @Component(ServiceNameConsts.ENTRANCE_FAILOVER_SERVER) public class EntranceFailoverJobServer { - private static final Logger logger = LoggerFactory.getLogger(DefaultEntranceServer.class); + private static final Logger logger = LoggerFactory.getLogger(EntranceFailoverJobServer.class); - @Autowired - private EntranceServer entranceServer; + @Autowired private EntranceServer entranceServer; - @Autowired - private CommonLockService commonLockService; + @Autowired private CommonLockService commonLockService; + private static String ENTRANCE_FAILOVER_LOCK = "ENTRANCE_FAILOVER_LOCK"; - private static String ENTRANCE_FAILOVER_LOCK = "ENTRANCE_FAILOVER_LOCK"; + private ScheduledExecutorService scheduledExecutor; - @PostConstruct - public void init() { - failoverTask(); - } + @PostConstruct + public void init() { + this.scheduledExecutor = + Executors.newSingleThreadScheduledExecutor( + Utils.threadFactory("Linkis-Failover-Scheduler-Thread-", true)); + failoverTask(); + } public void failoverTask() { if (EntranceConfiguration.ENTRANCE_FAILOVER_ENABLED()) { - Utils.defaultScheduler() - .scheduleWithFixedDelay( - () -> { - EntranceSchedulerContext schedulerContext = - (EntranceSchedulerContext) - entranceServer - .getEntranceContext() - .getOrCreateScheduler() - .getSchedulerContext(); - - // entrance do not failover job when it is offline - if (schedulerContext.getOfflineFlag()) return; - - CommonLock commonLock = new CommonLock(); - commonLock.setLockObject(ENTRANCE_FAILOVER_LOCK); - Boolean locked = false; - try { - locked = commonLockService.lock(commonLock, 10 * 1000L); - if (!locked) return; - logger.info("success locked {}", ENTRANCE_FAILOVER_LOCK); - - // serverInstance to map - Map serverInstanceMap = - getActiveServerInstances().stream() - .collect( - Collectors.toMap( - ServiceInstance::getInstance, - ServiceInstance::getRegistryTimestamp, - (k1, k2) -> k2)); - if (serverInstanceMap.isEmpty()) return; - - // get failover job expired time (获取任务故障转移过期时间，配置为0表示不过期, 过期则不处理) - long expiredTimestamp = 0L; - if (EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME() > 0) { - expiredTimestamp = - System.currentTimeMillis() - - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME(); - } - - // get uncompleted status - List statusList = - Arrays.stream(SchedulerEventState.uncompleteStatusArray()) - .map(Object::toString) - .collect(Collectors.toList()); - - List jobRequests = - JobHistoryHelper.queryWaitForFailoverTask( - serverInstanceMap, - statusList, - expiredTimestamp, - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); - if (jobRequests.isEmpty()) return; - Object[] ids = jobRequests.stream().map(JobRequest::getId).toArray(); - logger.info("success query failover jobs , job ids: {}", ids); - - // failover to local server - jobRequests.forEach(jobRequest -> entranceServer.failoverExecute(jobRequest)); - logger.info("success execute failover jobs, job ids: {}", ids); - - } catch (Exception e) { - logger.error("failover failed", e); - } finally { - if (locked) commonLockService.unlock(commonLock); - } - }, - EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INIT_TIME(), - EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INTERVAL(), - TimeUnit.MILLISECONDS); + scheduledExecutor.scheduleWithFixedDelay( + () -> { + EntranceSchedulerContext schedulerContext = + (EntranceSchedulerContext) + entranceServer + .getEntranceContext() + .getOrCreateScheduler() + .getSchedulerContext(); + + // entrance do not failover job when it is offline + if (schedulerContext.getOfflineFlag()) return; + + CommonLock commonLock = new CommonLock(); + commonLock.setLockObject(ENTRANCE_FAILOVER_LOCK); + Boolean locked = false; + try { + locked = commonLockService.lock(commonLock, 10 * 1000L); + if (!locked) return; + logger.info("success locked {}", ENTRANCE_FAILOVER_LOCK); + + // serverInstance to map + Map serverInstanceMap = + getActiveServerInstances().stream() + .collect( + Collectors.toMap( + ServiceInstance::getInstance, + ServiceInstance::getRegistryTimestamp, + (k1, k2) -> k2)); + if (serverInstanceMap.isEmpty()) return; + + // It is very important to avoid repeated execute job + // when failover self job, if self instance is empty, the job can be repeated execute + if (!serverInstanceMap.containsKey(Sender.getThisInstance())) { + logger.warn( + "server has just started and has not get self info, it does not failover"); + return; + } + + // get failover job expired time (获取任务故障转移过期时间，配置为0表示不过期, 过期则不处理) + long expiredTimestamp = 0L; + if (EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME() > 0) { + expiredTimestamp = + System.currentTimeMillis() + - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME(); + } + + // get uncompleted status + List statusList = + Arrays.stream(SchedulerEventState.uncompleteStatusArray()) + .map(Object::toString) + .collect(Collectors.toList()); + + List jobRequests = + JobHistoryHelper.queryWaitForFailoverTask( + serverInstanceMap, + statusList, + expiredTimestamp, + EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); + if (jobRequests.isEmpty()) return; + Object[] ids = jobRequests.stream().map(JobRequest::getId).toArray(); + logger.info("success query failover jobs , job ids: {}", ids); + + // failover to local server + for (JobRequest jobRequest : jobRequests) { + entranceServer.failoverExecute(jobRequest); + } + logger.info("finished execute failover jobs, job ids: {}", ids); + + } catch (Exception e) { + logger.error("failover failed", e); + } finally { + if (locked) commonLockService.unlock(commonLock); + } + }, + EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INIT_TIME(), + EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INTERVAL(), + TimeUnit.MILLISECONDS); } } @@ -149,7 +166,7 @@ private List getActiveServerInstances() { // get all offline label server RouteLabel routeLabel = LabelBuilderFactoryContext.getLabelBuilderFactory() - .createLabel(LabelKeyConstant.ROUTE_KEY, LabelConstant.OFFLINE); + .createLabel(LabelKeyConstant.ROUTE_KEY, LabelValueConstant.OFFLINE_VALUE); List> labels = Lists.newArrayList(); labels.add(routeLabel); List labelInstances = @@ -164,5 +181,3 @@ private List getActiveServerInstances() { return allInstances; } } - -} \ No newline at end of file From f0755376d4c9db08cd9a71c27c8ab0b6b4b18fd1 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Wed, 14 Dec 2022 15:59:43 +0800 Subject: [PATCH 025/689] =?UTF-8?q?1.=E4=BF=AE=E6=94=B9=E8=8E=B7=E5=8F=96f?= =?UTF-8?q?ailover=E5=AE=9E=E4=BE=8B=E5=88=97=E8=A1=A8=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E8=83=BD=E5=8E=BB=E9=99=A4offline=20lable=E5=AE=9E=E4=BE=8B=20?= =?UTF-8?q?2.=E7=A7=BB=E9=99=A4retry=E4=BB=BB=E5=8A=A1=E6=97=B6=EF=BC=8C?= =?UTF-8?q?=E8=A6=81=E6=9B=B4=E6=96=B0=E6=95=B0=E6=8D=AE=E5=BA=93instance?= =?UTF-8?q?=E4=B8=BA=E7=A9=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../queue/fifoqueue/FIFOUserConsumer.scala | 6 ++- .../server/DefaultEntranceServer.java | 2 +- .../server/EntranceFailoverJobServer.java | 40 +++---------------- .../entrance/conf/EntranceConfiguration.scala | 3 +- .../scheduler/EntranceFIFOUserConsumer.scala | 26 +++++++++--- .../entrance/utils/JobHistoryHelper.scala | 2 +- 6 files changed, 36 insertions(+), 43 deletions(-) diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala index 4483a02a767..ea4f4ce6df3 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala @@ -27,6 +27,7 @@ import org.apache.linkis.scheduler.executer.Executor import org.apache.linkis.scheduler.future.{BDPFuture, BDPFutureTask} import org.apache.linkis.scheduler.queue._ +import java.util import java.util.concurrent.{ExecutorService, Future} import scala.beans.BeanProperty @@ -189,14 +190,17 @@ class FIFOUserConsumer( runningJobs(index) = job } - protected def scanAllRetryJobsAndRemove(): Unit = { + protected def scanAllRetryJobsAndRemove(): util.List[Job] = { + val jobs = new util.ArrayList[Job]() for (index <- runningJobs.indices) { val job = runningJobs(index) if (job != null && job.isJobCanRetry) { + jobs.add(job) runningJobs(index) = null logger.info(s"Job $job can retry, remove from runningJobs") } } + jobs } override def shutdown(): Unit = { diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java index 443feb2a816..ea920f4c112 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java @@ -80,7 +80,7 @@ private void shutdownEntrance(ContextClosedEvent event) { if (shutdownFlag) { logger.warn("event has been handled"); } else { - if (EntranceConfiguration.ENTRANCE_SHUTDOWN_FAILOVER_ENABLED()) { + if (EntranceConfiguration.ENTRANCE_SHUTDOWN_FAILOVER_CONSUME_QUEUE_ENABLED()) { logger.warn("Entrance exit to update and clean all ConsumeQueue task instances"); updateAllNotExecutionTaskInstances(false); } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java index b5f06886262..1c2f906a9d2 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java @@ -25,19 +25,11 @@ import org.apache.linkis.entrance.scheduler.EntranceSchedulerContext; import org.apache.linkis.entrance.utils.JobHistoryHelper; import org.apache.linkis.governance.common.entity.job.JobRequest; -import org.apache.linkis.instance.label.client.InstanceLabelClient; -import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; -import org.apache.linkis.manager.label.constant.LabelKeyConstant; -import org.apache.linkis.manager.label.constant.LabelValueConstant; -import org.apache.linkis.manager.label.entity.Label; -import org.apache.linkis.manager.label.entity.route.RouteLabel; import org.apache.linkis.publicservice.common.lock.entity.CommonLock; import org.apache.linkis.publicservice.common.lock.service.CommonLockService; import org.apache.linkis.rpc.Sender; import org.apache.linkis.scheduler.queue.SchedulerEventState; -import org.apache.commons.compress.utils.Lists; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -97,15 +89,19 @@ public void failoverTask() { if (!locked) return; logger.info("success locked {}", ENTRANCE_FAILOVER_LOCK); + // get all entrance server from eureka + ServiceInstance[] serviceInstances = + Sender.getInstances(Sender.getThisServiceInstance().getApplicationName()); + if (serviceInstances == null || serviceInstances.length <= 0) return; + // serverInstance to map Map serverInstanceMap = - getActiveServerInstances().stream() + Arrays.stream(serviceInstances) .collect( Collectors.toMap( ServiceInstance::getInstance, ServiceInstance::getRegistryTimestamp, (k1, k2) -> k2)); - if (serverInstanceMap.isEmpty()) return; // It is very important to avoid repeated execute job // when failover self job, if self instance is empty, the job can be repeated execute @@ -156,28 +152,4 @@ public void failoverTask() { TimeUnit.MILLISECONDS); } } - - private List getActiveServerInstances() { - // get all entrance server from eureka - ServiceInstance[] serviceInstances = - Sender.getInstances(Sender.getThisServiceInstance().getApplicationName()); - if (serviceInstances == null || serviceInstances.length <= 0) return Lists.newArrayList(); - - // get all offline label server - RouteLabel routeLabel = - LabelBuilderFactoryContext.getLabelBuilderFactory() - .createLabel(LabelKeyConstant.ROUTE_KEY, LabelValueConstant.OFFLINE_VALUE); - List> labels = Lists.newArrayList(); - labels.add(routeLabel); - List labelInstances = - InstanceLabelClient.getInstance().getInstanceFromLabel(labels); - if (labelInstances == null) labelInstances = Lists.newArrayList(); - - // get active entrance server - List allInstances = Lists.newArrayList(); - allInstances.addAll(Arrays.asList(serviceInstances)); - allInstances.removeAll(labelInstances); - - return allInstances; - } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index 10db3715fef..3b606cfe3ec 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -242,7 +242,8 @@ object EntranceConfiguration { val ENTRANCE_UPDATE_BATCH_SIZE = CommonVars("linkis.entrance.update.batch.size", 100) - val ENTRANCE_SHUTDOWN_FAILOVER_ENABLED = CommonVars("linkis.entrance.shutdown.failover.enable", true).getValue + val ENTRANCE_SHUTDOWN_FAILOVER_CONSUME_QUEUE_ENABLED = + CommonVars("linkis.entrance.shutdown.failover.consume.queue.enable", true).getValue val ENTRANCE_GROUP_SCAN_ENABLED = CommonVars("linkis.entrance.group.scan.enable", true) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala index 34d3e3042c5..2ff42d1eb8d 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala @@ -19,23 +19,39 @@ package org.apache.linkis.entrance.scheduler import org.apache.linkis.common.utils.Utils import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.entrance.execute.EntranceJob +import org.apache.linkis.entrance.utils.JobHistoryHelper import org.apache.linkis.scheduler.SchedulerContext import org.apache.linkis.scheduler.queue.Group import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer +import java.util import java.util.concurrent.ExecutorService +import scala.collection.JavaConverters.collectionAsScalaIterableConverter + class EntranceFIFOUserConsumer( - schedulerContext: SchedulerContext, - executeService: ExecutorService, - private var group: Group + schedulerContext: SchedulerContext, + executeService: ExecutorService, + private var group: Group ) extends FIFOUserConsumer(schedulerContext, executeService, group) { override def loop(): Unit = { schedulerContext match { case entranceSchedulerContext: EntranceSchedulerContext => - if (entranceSchedulerContext.getOfflineFlag && EntranceConfiguration.ENTRANCE_FAILOVER_RETRY_JOB_ENABLED.getValue) { - scanAllRetryJobsAndRemove() + if ( + entranceSchedulerContext.getOfflineFlag && EntranceConfiguration.ENTRANCE_FAILOVER_RETRY_JOB_ENABLED.getValue + ) { + val jobs = scanAllRetryJobsAndRemove() + if (!jobs.isEmpty) { + val ids = new util.ArrayList[Long]() + jobs.asScala.foreach { + case entranceJob: EntranceJob => + ids.add(entranceJob.getJobRequest.getId) + case _ => + } + JobHistoryHelper.updateBatchInstances(ids) + } Utils.tryQuietly(Thread.sleep(5000)) return } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala index 7a55124f75e..714f1d77dea 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala @@ -175,7 +175,7 @@ object JobHistoryHelper extends Logging { * @param taskIdList * @return */ - private def updateBatchInstances(taskIdList: util.List[Long]): util.List[Long] = { + def updateBatchInstances(taskIdList: util.List[Long]): util.List[Long] = { val jobReqList = new util.ArrayList[JobRequest]() taskIdList.asScala.foreach(taskID => { val jobRequest = new JobRequest From db2f0eda512024f4d9b1ce47ab3ae6f653e4a7b3 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Wed, 14 Dec 2022 16:20:24 +0800 Subject: [PATCH 026/689] =?UTF-8?q?1.=E6=97=A5=E5=BF=97=E4=BF=A1=E6=81=AF?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=AD=E6=96=87=E6=8F=8F=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/apache/linkis/entrance/restful/EntranceRestfulApi.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index b32923cc0d1..afa4aeb06c6 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -614,7 +614,7 @@ public Message log(HttpServletRequest req, @PathVariable("id") String id) { logger.warn("The job {} wait failover, return customer log", id); message = Message.ok(); message.setMethod("/api/entrance/" + id + "/log"); - String log = LogUtils.generateInfo("The job will failover soon, please try again later"); + String log = LogUtils.generateInfo("The job will failover soon, please try again later.(job很快就会failover，请稍后再试)"); Object retLog; if (distinctLevel) { String[] array = new String[4]; From dfb531afc19411532d2eecef04305156363def1d Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Fri, 16 Dec 2022 09:19:52 +0800 Subject: [PATCH 027/689] =?UTF-8?q?failover=E6=97=B6=EF=BC=8C=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E4=B8=BB=E5=8A=A8=E5=88=B7=E6=97=A5=E5=BF=97hdfs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/DefaultEntranceServer.java | 7 +++--- .../linkis/entrance/EntranceServer.scala | 24 ++++++++++++++----- .../scheduler/EntranceFIFOUserConsumer.scala | 6 ++--- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java index ea920f4c112..54b855ffbd3 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java @@ -22,9 +22,12 @@ import org.apache.linkis.entrance.conf.EntranceConfiguration; import org.apache.linkis.entrance.constant.ServiceNameConsts; import org.apache.linkis.entrance.execute.EntranceJob; +import org.apache.linkis.entrance.job.EntranceExecutionJob; import org.apache.linkis.entrance.log.LogReader; import org.apache.linkis.rpc.Sender; +import org.apache.commons.io.IOUtils; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.event.EventListener; @@ -90,10 +93,8 @@ private void shutdownEntrance(ContextClosedEvent event) { if (null != allUndoneTask) { String msg = "Entrance exits the automatic cleanup task and can be rerun(服务退出自动清理任务，可以重跑)"; for (EntranceJob job : allUndoneTask) { - if (job.getLogListener().isDefined()) { - job.getLogListener().get().onLogUpdate(job, msg); - } job.onFailure(msg, null); + IOUtils.closeQuietly(((EntranceExecutionJob) job).getLogWriter().get()); } } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index d2a504100d1..b5563262b8d 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -43,17 +43,16 @@ import org.apache.linkis.rpc.conf.RPCConfiguration import org.apache.linkis.scheduler.queue.{Job, SchedulerEventState} import org.apache.linkis.server.conf.ServerConfiguration import org.apache.linkis.storage.utils.StorageUtils - import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.linkis.common.log.LogUtils import org.springframework.beans.BeanUtils +import org.apache.linkis.entrance.job.EntranceExecutionJob import java.{lang, util} import java.text.{MessageFormat, SimpleDateFormat} import java.util.Date - import scala.collection.JavaConverters._ abstract class EntranceServer extends Logging { @@ -272,11 +271,24 @@ abstract class EntranceServer extends Logging { def updateAllNotExecutionTaskInstances(retryWhenUpdateFail: Boolean): Unit = { val consumeQueueTasks = getAllConsumeQueueTask() + + clearAllConsumeQueue() + logger.info("Finished to clean all ConsumeQueue") + if (consumeQueueTasks != null && consumeQueueTasks.length > 0) { - val taskIds = consumeQueueTasks.map(_.getJobRequest.getId.asInstanceOf[Long]).toList - clearAllConsumeQueue() - logger.info("Finished to clean all ConsumeQueue") - JobHistoryHelper.updateAllConsumeQueueTask(taskIds.asJava, retryWhenUpdateFail) + val taskIds = new util.ArrayList[Long]() + consumeQueueTasks.foreach(job => { + taskIds.add(job.getJobRequest.getId.asInstanceOf[Long]) + job match { + case entranceExecutionJob : EntranceExecutionJob => + val msg = LogUtils.generateWarn(s"job ${job.getJobRequest.getId} clean from ConsumeQueue, wait for failover") + entranceExecutionJob.getLogListener.foreach(_.onLogUpdate(entranceExecutionJob, msg)) + entranceExecutionJob.getLogWriter.foreach(_.close()) + case _ => + } + }) + + JobHistoryHelper.updateAllConsumeQueueTask(taskIds, retryWhenUpdateFail) logger.info("Finished to update all not execution task instances") } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala index 2ff42d1eb8d..2404db51dc4 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala @@ -19,7 +19,7 @@ package org.apache.linkis.entrance.scheduler import org.apache.linkis.common.utils.Utils import org.apache.linkis.entrance.conf.EntranceConfiguration -import org.apache.linkis.entrance.execute.EntranceJob +import org.apache.linkis.entrance.job.EntranceExecutionJob import org.apache.linkis.entrance.utils.JobHistoryHelper import org.apache.linkis.scheduler.SchedulerContext import org.apache.linkis.scheduler.queue.Group @@ -27,7 +27,6 @@ import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer import java.util import java.util.concurrent.ExecutorService - import scala.collection.JavaConverters.collectionAsScalaIterableConverter class EntranceFIFOUserConsumer( @@ -46,7 +45,8 @@ class EntranceFIFOUserConsumer( if (!jobs.isEmpty) { val ids = new util.ArrayList[Long]() jobs.asScala.foreach { - case entranceJob: EntranceJob => + case entranceJob: EntranceExecutionJob => + entranceJob.getLogWriter.foreach(_.close()) ids.add(entranceJob.getJobRequest.getId) case _ => } From 8ab841c8baf12b516a330cd73ae342013a9bebfa Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Fri, 16 Dec 2022 16:18:12 +0800 Subject: [PATCH 028/689] =?UTF-8?q?=E8=BF=81=E7=A7=BB=E5=88=B7=E6=96=B0max?= =?UTF-8?q?AllowRunningJobs=E6=9C=8D=E5=8A=A1=EF=BC=8C=E6=94=BE=E5=88=B0co?= =?UTF-8?q?nsumeManager=E9=87=8C=EF=BC=8C=E5=88=B7=E6=96=B0consumer?= =?UTF-8?q?=E9=87=8C=E7=9A=84group,=E8=80=8C=E4=B8=8D=E6=98=AF=E5=88=B7?= =?UTF-8?q?=E6=96=B0groupFactory=E9=87=8C=E7=9A=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../entrance/conf/EntranceConfiguration.scala | 4 ++ .../scheduler/EntranceGroupFactory.scala | 54 +-------------- .../EntranceParallelConsumerManager.scala | 65 ++++++++++++++++++- 3 files changed, 69 insertions(+), 54 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index 3b606cfe3ec..959d8c68bc9 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -247,6 +247,10 @@ object EntranceConfiguration { val ENTRANCE_GROUP_SCAN_ENABLED = CommonVars("linkis.entrance.group.scan.enable", true) + val ENTRANCE_GROUP_SCAN_INIT_TIME = CommonVars("linkis.entrance.group.scan.init.time", 3 * 1000) + + val ENTRANCE_GROUP_SCAN_INTERVAL = CommonVars("linkis.entrance.group.scan.interval", 60 * 1000) + val ENTRANCE_FAILOVER_RETAIN_ENGINE_CONN_ENABLED = CommonVars("linkis.entrance.failover.retain.engine.conn.enable", true) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala index c38fae5e4a9..4bd0caca1b2 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala @@ -17,7 +17,6 @@ package org.apache.linkis.entrance.scheduler -import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.conf.{CommonVars, Configuration} import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.entrance.conf.EntranceConfiguration @@ -46,6 +45,7 @@ import org.apache.linkis.protocol.utils.TaskUtils import org.apache.linkis.rpc.Sender import org.apache.linkis.scheduler.queue.{Group, GroupFactory, SchedulerEvent} import org.apache.linkis.scheduler.queue.parallelqueue.ParallelGroup + import org.apache.commons.lang3.StringUtils import java.util @@ -77,39 +77,6 @@ class EntranceGroupFactory extends GroupFactory with Logging { private val GROUP_INIT_CAPACITY = CommonVars("wds.linkis.entrance.init.capacity", 100) - private val GROUP_SCAN_INIT_TIME = CommonVars("linkis.entrance.group.scan.init.time", 3 * 1000) - - private val GROUP_SCAN_INTERVAL = CommonVars("linkis.entrance.group.scan.interval", 60 * 1000) - - if (EntranceConfiguration.ENTRANCE_GROUP_SCAN_ENABLED.getValue) { - Utils.defaultScheduler.scheduleAtFixedRate( - new Runnable { - override def run(): Unit = { - // get all entrance server from eureka - val serviceInstances = Sender.getInstances(Sender.getThisServiceInstance.getApplicationName) - if (null == serviceInstances || serviceInstances.isEmpty) return - - // get all offline label server - val routeLabel = LabelBuilderFactoryContext.getLabelBuilderFactory - .createLabel[RouteLabel](LabelKeyConstant.ROUTE_KEY, LabelConstant.OFFLINE) - val labels = new util.ArrayList[Label[_]] - labels.add(routeLabel) - val labelInstances = InstanceLabelClient.getInstance.getInstanceFromLabel(labels) - - // get active entrance server - val allInstances = new util.ArrayList[ServiceInstance]() - allInstances.addAll(serviceInstances.toList.asJava) - allInstances.removeAll(labelInstances) - // refresh all group maxAllowRunningJobs - refreshAllGroupMaxAllowRunningJobs(allInstances.size()) - } - }, - GROUP_SCAN_INIT_TIME.getValue, - GROUP_SCAN_INTERVAL.getValue, - TimeUnit.MILLISECONDS - ) - } - private val specifiedUsernameRegexPattern: Pattern = if (StringUtils.isNotBlank(SPECIFIED_USERNAME_REGEX.getValue)) { Pattern.compile(SPECIFIED_USERNAME_REGEX.getValue) @@ -193,25 +160,6 @@ class EntranceGroupFactory extends GroupFactory with Logging { group } - def refreshAllGroupMaxAllowRunningJobs(validInsCount: Int): Unit = { - if (validInsCount <= 0) return - groupNameToGroups - .asMap() - .asScala - .foreach(item => { - item._2 match { - case group: ParallelGroup => - val maxAllowRunningJobs = Math.round(group.getMaxRunningJobs / validInsCount) - group.setMaxAllowRunningJobs(maxAllowRunningJobs) - logger - .info( - s"group ${group.getGroupName} refresh maxAllowRunningJobs => ${group.getMaxRunningJobs}/$validInsCount=$maxAllowRunningJobs" - ) - case _ => - } - }) - } - private def getUserMaxRunningJobs(keyAndValue: util.Map[String, String]): Int = { Math.max( EntranceConfiguration.ENTRANCE_INSTANCE_MIN.getValue, diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala index 91a7c4aaa67..f114981c5c8 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala @@ -17,8 +17,22 @@ package org.apache.linkis.entrance.scheduler +import org.apache.linkis.common.ServiceInstance +import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.instance.label.client.InstanceLabelClient +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext +import org.apache.linkis.manager.label.constant.{LabelKeyConstant, LabelValueConstant} +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.apache.linkis.rpc.Sender import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer -import org.apache.linkis.scheduler.queue.parallelqueue.ParallelConsumerManager +import org.apache.linkis.scheduler.queue.parallelqueue.{ParallelConsumerManager, ParallelGroup} + +import java.util +import java.util.concurrent.TimeUnit +import scala.collection.JavaConverters._ class EntranceParallelConsumerManager(maxParallelismUsers: Int, schedulerName: String) extends ParallelConsumerManager(maxParallelismUsers, schedulerName){ @@ -28,4 +42,53 @@ class EntranceParallelConsumerManager(maxParallelismUsers: Int, schedulerName: S new EntranceFIFOUserConsumer(getSchedulerContext, getOrCreateExecutorService, group) } + if (EntranceConfiguration.ENTRANCE_GROUP_SCAN_ENABLED.getValue) { + Utils.defaultScheduler.scheduleAtFixedRate( + new Runnable { + override def run(): Unit = { + logger.info("start refresh consumer group maxAllowRunningJobs") + // get all entrance server from eureka + val serviceInstances = + Sender.getInstances(Sender.getThisServiceInstance.getApplicationName) + if (null == serviceInstances || serviceInstances.isEmpty) return + + // get all offline label server + val routeLabel = LabelBuilderFactoryContext.getLabelBuilderFactory + .createLabel[RouteLabel](LabelKeyConstant.ROUTE_KEY, LabelValueConstant.OFFLINE_VALUE) + val labels = new util.ArrayList[Label[_]] + labels.add(routeLabel) + val labelInstances = InstanceLabelClient.getInstance.getInstanceFromLabel(labels) + + // get active entrance server + val allInstances = new util.ArrayList[ServiceInstance]() + allInstances.addAll(serviceInstances.toList.asJava) + allInstances.removeAll(labelInstances) + // refresh all group maxAllowRunningJobs + refreshAllGroupMaxAllowRunningJobs(allInstances.size()) + logger.info("Finished to refresh consumer group maxAllowRunningJobs") + } + }, + EntranceConfiguration.ENTRANCE_GROUP_SCAN_INIT_TIME.getValue, + EntranceConfiguration.ENTRANCE_GROUP_SCAN_INTERVAL.getValue, + TimeUnit.MILLISECONDS + ) + } + + def refreshAllGroupMaxAllowRunningJobs(validInsCount: Int): Unit = { + if (validInsCount <= 0) return + listConsumers() + .foreach(item => { + item.getGroup match { + case group: ParallelGroup => + val maxAllowRunningJobs = Math.round(group.getMaxRunningJobs / validInsCount) + group.setMaxAllowRunningJobs(maxAllowRunningJobs) + logger + .info( + s"group ${group.getGroupName} refresh maxAllowRunningJobs => ${group.getMaxRunningJobs}/$validInsCount=$maxAllowRunningJobs" + ) + case _ => + } + }) + } + } From 1cf6321ef5721f93096cae256a7721eec819d179 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Mon, 19 Dec 2022 11:40:40 +0800 Subject: [PATCH 029/689] update name --- .../linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala | 2 +- .../org/apache/linkis/entrance/utils/JobHistoryHelper.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala index 2404db51dc4..1977fa68ac8 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala @@ -50,7 +50,7 @@ class EntranceFIFOUserConsumer( ids.add(entranceJob.getJobRequest.getId) case _ => } - JobHistoryHelper.updateBatchInstances(ids) + JobHistoryHelper.updateBatchInstancesEmpty(ids) } Utils.tryQuietly(Thread.sleep(5000)) return diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala index 714f1d77dea..df7b846a7d4 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala @@ -154,7 +154,7 @@ object JobHistoryHelper extends Logging { val list = new util.ArrayList[Long]() list.addAll(taskIdList) try { - val successTaskIds = updateBatchInstances(updateTaskIds) + val successTaskIds = updateBatchInstancesEmpty(updateTaskIds) if (retryWhenUpdateFail) { list.removeAll(successTaskIds) } else { @@ -175,7 +175,7 @@ object JobHistoryHelper extends Logging { * @param taskIdList * @return */ - def updateBatchInstances(taskIdList: util.List[Long]): util.List[Long] = { + def updateBatchInstancesEmpty(taskIdList: util.List[Long]): util.List[Long] = { val jobReqList = new util.ArrayList[JobRequest]() taskIdList.asScala.foreach(taskID => { val jobRequest = new JobRequest From 789404fa26877959ca85e739d118925683dbbe2f Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Tue, 27 Dec 2022 10:40:21 +0800 Subject: [PATCH 030/689] add header --- .../springcloud/http/SpringCloudGatewayHttpRequest.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/scala/org/apache/linkis/gateway/springcloud/http/SpringCloudGatewayHttpRequest.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/scala/org/apache/linkis/gateway/springcloud/http/SpringCloudGatewayHttpRequest.scala index d591e5ce94c..929ed6ae62e 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/scala/org/apache/linkis/gateway/springcloud/http/SpringCloudGatewayHttpRequest.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/scala/org/apache/linkis/gateway/springcloud/http/SpringCloudGatewayHttpRequest.scala @@ -87,8 +87,10 @@ class SpringCloudGatewayHttpRequest(request: AbstractServerHttpRequest) extends override def getHeaders: JMap[String, Array[String]] = headers - override def addHeader(headerName: String, headers: Array[String]): Unit = + override def addHeader(headerName: String, headers: Array[String]): Unit = { + this.headers.put(headerName, headers) addHeaders.put(headerName, headers) + } override def addCookie(cookieName: String, cookies: Array[Cookie]): Unit = { this.cookies.put(cookieName, cookies) From 2a39fa38099515df4b39c59e6b370878e404aec2 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Wed, 8 Feb 2023 16:32:58 +0800 Subject: [PATCH 031/689] bug - print failover ids size --- .../linkis/entrance/server/EntranceFailoverJobServer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java index 1c2f906a9d2..6d8e2971c16 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java @@ -132,8 +132,8 @@ public void failoverTask() { expiredTimestamp, EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); if (jobRequests.isEmpty()) return; - Object[] ids = jobRequests.stream().map(JobRequest::getId).toArray(); - logger.info("success query failover jobs , job ids: {}", ids); + List ids = jobRequests.stream().map(JobRequest::getId).collect(Collectors.toList()); + logger.info("success query failover jobs , job size: {}, ids: {}", ids.size(), ids); // failover to local server for (JobRequest jobRequest : jobRequests) { From a33ad78b0827cc7cc95f378e0f066ef6fe7bb285 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Wed, 8 Feb 2023 17:48:13 +0800 Subject: [PATCH 032/689] update failover log --- .../org/apache/linkis/entrance/EntranceServer.scala | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index b5563262b8d..9125abf4c6f 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -308,12 +308,7 @@ abstract class EntranceServer extends Logging { } val logAppender = new java.lang.StringBuilder() - logAppender.append( - LogUtils - .generateInfo( - s"\n\n*************************************FAILOVER************************************** \n\n" - ) - ) + logAppender.append("*************************************FAILOVER**************************************") // try to kill ec killOldEC(jobRequest, logAppender); @@ -339,7 +334,7 @@ abstract class EntranceServer extends Logging { Utils.tryCatch { logAppender.append( LogUtils - .generateInfo(s"job ${jobRequest.getId} start to kill ec \n") + .generateInfo(s"job ${jobRequest.getId} start to kill old ec \n") ) if ( !SchedulerEventState.isRunning(SchedulerEventState.withName(jobRequest.getStatus)) From c172f326a0b8cfdf8055e79592aa25b0334a3822 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Wed, 8 Feb 2023 17:57:26 +0800 Subject: [PATCH 033/689] update log --- .../main/scala/org/apache/linkis/entrance/EntranceServer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index 9125abf4c6f..efd5e76a451 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -462,7 +462,7 @@ abstract class EntranceServer extends Logging { val msg = MessageFormat.format( EntranceErrorCodeSummary.FAILOVER_RUNNING_TO_CANCELLED.getErrorDesc, - jobRequest.getId + jobRequest.getId.toString ) // init jobRequest properties jobRequest.setStatus(SchedulerEventState.Cancelled.toString) From 5e1712d0f437055cd536a983803a7a4c84ed49bb Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Tue, 28 Feb 2023 13:48:59 +0800 Subject: [PATCH 034/689] code format --- .../linkis/common/ServiceInstance.scala | 9 +++-- .../scheduler/queue/AbstractGroup.scala | 6 ++-- .../common/protocol/job/JobReqProcotol.scala | 7 +++- .../restful/EntranceLabelRestfulApi.java | 20 +++++------ .../entrance/restful/EntranceRestfulApi.java | 4 ++- .../server/EntranceFailoverJobServer.java | 3 +- .../linkis/entrance/EntranceServer.scala | 15 +++++--- .../entrance/conf/EntranceConfiguration.scala | 9 +++-- .../scheduler/EntranceFIFOUserConsumer.scala | 1 + .../scheduler/EntranceGroupFactory.scala | 13 ++----- .../EntranceParallelConsumerManager.scala | 4 +-- .../entrance/utils/JobHistoryHelper.scala | 12 ++++--- .../manager/label/constant/LabelConstant.java | 2 -- .../jobhistory/dao/JobHistoryMapper.java | 35 +++++++------------ .../impl/JobHistoryQueryServiceImpl.scala | 11 ++---- .../parser/EntranceRequestGatewayParser.scala | 15 ++++---- 16 files changed, 82 insertions(+), 84 deletions(-) diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/ServiceInstance.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/ServiceInstance.scala index 9cee5fe329d..f9e47184724 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/ServiceInstance.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/ServiceInstance.scala @@ -25,7 +25,10 @@ class ServiceInstance { def getApplicationName: String = applicationName def setInstance(instance: String): Unit = this.instance = instance def getInstance: String = instance - def setRegistryTimestamp(registryTimestamp: Long): Unit = this.registryTimestamp = registryTimestamp + + def setRegistryTimestamp(registryTimestamp: Long): Unit = this.registryTimestamp = + registryTimestamp + def getRegistryTimestamp: Long = registryTimestamp override def equals(other: Any): Boolean = other match { @@ -45,7 +48,9 @@ class ServiceInstance { .foldLeft(0)((a, b) => 31 * a + b) } - override def toString: String = s"ServiceInstance($applicationName, $instance, $registryTimestamp)" + override def toString: String = + s"ServiceInstance($applicationName, $instance, $registryTimestamp)" + } object ServiceInstance { diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/AbstractGroup.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/AbstractGroup.scala index cc9577941f3..b123682b56f 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/AbstractGroup.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/AbstractGroup.scala @@ -29,9 +29,11 @@ abstract class AbstractGroup extends Group { def setMaxRunningJobs(maxRunningJobs: Int): Unit = this.maxRunningJobs = maxRunningJobs def getMaxRunningJobs: Int = maxRunningJobs - def setMaxAllowRunningJobs(maxAllowRunningJobs: Int): Unit = this.maxAllowRunningJobs = maxAllowRunningJobs + def setMaxAllowRunningJobs(maxAllowRunningJobs: Int): Unit = this.maxAllowRunningJobs = + maxAllowRunningJobs + def getMaxAllowRunningJobs: Int = - if(maxAllowRunningJobs <= 0) maxRunningJobs else Math.min(maxAllowRunningJobs, maxRunningJobs) + if (maxAllowRunningJobs <= 0) maxRunningJobs else Math.min(maxAllowRunningJobs, maxRunningJobs) def setMaxAskExecutorTimes(maxAskExecutorTimes: Long): Unit = this.maxAskExecutorTimes = maxAskExecutorTimes diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala index 4d6346c9184..df197ddb2c3 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala @@ -52,4 +52,9 @@ class RequestOneJob extends JobReq { case class RequestAllJob(instance: String) extends JobReq -case class RequestFailoverJob(reqMap: util.Map[String, java.lang.Long], statusList: util.List[String], startTimestamp: Long, limit: Int = 10) extends JobReq +case class RequestFailoverJob( + reqMap: util.Map[String, java.lang.Long], + statusList: util.List[String], + startTimestamp: Long, + limit: Int = 10 +) extends JobReq diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java index e51f66266dc..841a6a3fb08 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java @@ -21,7 +21,6 @@ import org.apache.linkis.entrance.EntranceServer; import org.apache.linkis.entrance.scheduler.EntranceSchedulerContext; import org.apache.linkis.instance.label.client.InstanceLabelClient; -import org.apache.linkis.manager.label.constant.LabelConstant; import org.apache.linkis.manager.label.constant.LabelKeyConstant; import org.apache.linkis.manager.label.constant.LabelValueConstant; import org.apache.linkis.protocol.label.InsLabelRefreshRequest; @@ -50,13 +49,13 @@ @RequestMapping(path = "/entrance/operation/label") public class EntranceLabelRestfulApi { - private static final Logger logger = LoggerFactory.getLogger(EntranceLabelRestfulApi.class); - private EntranceServer entranceServer; + private static final Logger logger = LoggerFactory.getLogger(EntranceLabelRestfulApi.class); + private EntranceServer entranceServer; - @Autowired - public void setEntranceServer(EntranceServer entranceServer) { - this.entranceServer = entranceServer; - } + @Autowired + public void setEntranceServer(EntranceServer entranceServer) { + this.entranceServer = entranceServer; + } @ApiOperation(value = "update", notes = "update route label", response = Message.class) @ApiOperationSupport(ignoreParameters = {"jsonNode"}) @@ -92,13 +91,14 @@ public Message updateRouteLabel(HttpServletRequest req) { logger.info("Finished to modify the routelabel of entry to offline"); logger.info("Prepare to update all not execution task instances to empty string"); - SchedulerContext schedulerContext = entranceServer.getEntranceContext().getOrCreateScheduler().getSchedulerContext(); + SchedulerContext schedulerContext = + entranceServer.getEntranceContext().getOrCreateScheduler().getSchedulerContext(); if (schedulerContext instanceof EntranceSchedulerContext) { - ((EntranceSchedulerContext) schedulerContext).setOfflineFlag(true); + ((EntranceSchedulerContext) schedulerContext).setOfflineFlag(true); } entranceServer.updateAllNotExecutionTaskInstances(true); logger.info("Finished to update all not execution task instances to empty string"); - return Message.ok(); + return Message.ok(); } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index afa4aeb06c6..71b0df42503 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -614,7 +614,9 @@ public Message log(HttpServletRequest req, @PathVariable("id") String id) { logger.warn("The job {} wait failover, return customer log", id); message = Message.ok(); message.setMethod("/api/entrance/" + id + "/log"); - String log = LogUtils.generateInfo("The job will failover soon, please try again later.(job很快就会failover，请稍后再试)"); + String log = + LogUtils.generateInfo( + "The job will failover soon, please try again later.(job很快就会failover，请稍后再试)"); Object retLog; if (distinctLevel) { String[] array = new String[4]; diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java index 6d8e2971c16..77e85cba694 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java @@ -132,7 +132,8 @@ public void failoverTask() { expiredTimestamp, EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); if (jobRequests.isEmpty()) return; - List ids = jobRequests.stream().map(JobRequest::getId).collect(Collectors.toList()); + List ids = + jobRequests.stream().map(JobRequest::getId).collect(Collectors.toList()); logger.info("success query failover jobs , job size: {}, ids: {}", ids.size(), ids); // failover to local server diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index efd5e76a451..8ef5c268b5f 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -28,6 +28,7 @@ import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ import org.apache.linkis.entrance.exception.{EntranceErrorException, SubmitFailedException} import org.apache.linkis.entrance.execute.EntranceJob +import org.apache.linkis.entrance.job.EntranceExecutionJob import org.apache.linkis.entrance.log.{Cache, CacheLogWriter, HDFSCacheLogWriter, LogReader} import org.apache.linkis.entrance.parser.ParserUtils import org.apache.linkis.entrance.timeout.JobTimeoutManager @@ -43,16 +44,16 @@ import org.apache.linkis.rpc.conf.RPCConfiguration import org.apache.linkis.scheduler.queue.{Job, SchedulerEventState} import org.apache.linkis.server.conf.ServerConfiguration import org.apache.linkis.storage.utils.StorageUtils + import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils -import org.apache.linkis.common.log.LogUtils import org.springframework.beans.BeanUtils -import org.apache.linkis.entrance.job.EntranceExecutionJob import java.{lang, util} import java.text.{MessageFormat, SimpleDateFormat} import java.util.Date + import scala.collection.JavaConverters._ abstract class EntranceServer extends Logging { @@ -280,8 +281,10 @@ abstract class EntranceServer extends Logging { consumeQueueTasks.foreach(job => { taskIds.add(job.getJobRequest.getId.asInstanceOf[Long]) job match { - case entranceExecutionJob : EntranceExecutionJob => - val msg = LogUtils.generateWarn(s"job ${job.getJobRequest.getId} clean from ConsumeQueue, wait for failover") + case entranceExecutionJob: EntranceExecutionJob => + val msg = LogUtils.generateWarn( + s"job ${job.getJobRequest.getId} clean from ConsumeQueue, wait for failover" + ) entranceExecutionJob.getLogListener.foreach(_.onLogUpdate(entranceExecutionJob, msg)) entranceExecutionJob.getLogWriter.foreach(_.close()) case _ => @@ -308,7 +311,9 @@ abstract class EntranceServer extends Logging { } val logAppender = new java.lang.StringBuilder() - logAppender.append("*************************************FAILOVER**************************************") + logAppender.append( + "*************************************FAILOVER**************************************" + ) // try to kill ec killOldEC(jobRequest, logAppender); diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index 959d8c68bc9..13db69700f7 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -233,12 +233,15 @@ object EntranceConfiguration { val ENTRANCE_FAILOVER_SCAN_INTERVAL = CommonVars("linkis.entrance.failover.scan.interval", 30 * 1000).getValue - val ENTRANCE_FAILOVER_DATA_NUM_LIMIT = CommonVars("linkis.entrance.failover.data.num.limit", 10).getValue + val ENTRANCE_FAILOVER_DATA_NUM_LIMIT = + CommonVars("linkis.entrance.failover.data.num.limit", 10).getValue - val ENTRANCE_FAILOVER_DATA_INTERVAL_TIME = CommonVars("linkis.entrance.failover.data.interval.time", new TimeType("7d").toLong).getValue + val ENTRANCE_FAILOVER_DATA_INTERVAL_TIME = + CommonVars("linkis.entrance.failover.data.interval.time", new TimeType("7d").toLong).getValue // if true, the waitForRetry job in runningJobs can be failover - val ENTRANCE_FAILOVER_RETRY_JOB_ENABLED = CommonVars("linkis.entrance.failover.retry.job.enable", true) + val ENTRANCE_FAILOVER_RETRY_JOB_ENABLED = + CommonVars("linkis.entrance.failover.retry.job.enable", true) val ENTRANCE_UPDATE_BATCH_SIZE = CommonVars("linkis.entrance.update.batch.size", 100) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala index 1977fa68ac8..faee683fbf6 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala @@ -27,6 +27,7 @@ import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer import java.util import java.util.concurrent.ExecutorService + import scala.collection.JavaConverters.collectionAsScalaIterableConverter class EntranceFIFOUserConsumer( diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala index 4bd0caca1b2..0f31351b484 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala @@ -27,18 +27,12 @@ import org.apache.linkis.governance.common.protocol.conf.{ RequestQueryEngineConfigWithGlobalConfig, ResponseQueryConfig } -import org.apache.linkis.instance.label.client.InstanceLabelClient -import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext -import org.apache.linkis.manager.label.constant.{LabelKeyConstant, LabelValueConstant} -import org.apache.linkis.governance.common.protocol.conf.{RequestQueryEngineConfigWithGlobalConfig, ResponseQueryConfig} import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.engine.{ ConcurrentEngineConnLabel, EngineTypeLabel, UserCreatorLabel } -import org.apache.linkis.manager.label.entity.route.RouteLabel -import org.apache.linkis.manager.label.entity.engine.{ConcurrentEngineConnLabel, EngineTypeLabel, UserCreatorLabel} import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.protocol.constants.TaskConstant import org.apache.linkis.protocol.utils.TaskUtils @@ -51,13 +45,10 @@ import org.apache.commons.lang3.StringUtils import java.util import java.util.concurrent.TimeUnit import java.util.regex.Pattern + import scala.collection.JavaConverters._ + import com.google.common.cache.{Cache, CacheBuilder} -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.instance.label.client.InstanceLabelClient -import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext -import org.apache.linkis.manager.label.constant.{LabelConstant, LabelKeyConstant} -import org.apache.linkis.manager.label.entity.route.RouteLabel class EntranceGroupFactory extends GroupFactory with Logging { diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala index f114981c5c8..a067d658299 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala @@ -18,7 +18,6 @@ package org.apache.linkis.entrance.scheduler import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.common.utils.Utils import org.apache.linkis.entrance.conf.EntranceConfiguration import org.apache.linkis.instance.label.client.InstanceLabelClient @@ -32,10 +31,11 @@ import org.apache.linkis.scheduler.queue.parallelqueue.{ParallelConsumerManager, import java.util import java.util.concurrent.TimeUnit + import scala.collection.JavaConverters._ class EntranceParallelConsumerManager(maxParallelismUsers: Int, schedulerName: String) - extends ParallelConsumerManager(maxParallelismUsers, schedulerName){ + extends ParallelConsumerManager(maxParallelismUsers, schedulerName) { override protected def createConsumer(groupName: String): FIFOUserConsumer = { val group = getSchedulerContext.getOrCreateGroupFactory.getGroup(groupName) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala index df7b846a7d4..3fed0f78beb 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala @@ -30,13 +30,15 @@ import org.apache.linkis.protocol.constants.TaskConstant import org.apache.linkis.protocol.query.cache.{CacheTaskResult, RequestReadCache} import org.apache.linkis.rpc.Sender import org.apache.linkis.scheduler.queue.SchedulerEventState + import org.apache.commons.lang3.StringUtils import javax.servlet.http.HttpServletRequest + import java.util import java.util.Date + import scala.collection.JavaConverters._ -import sun.net.util.IPAddressUtil import com.google.common.net.InetAddresses @@ -316,15 +318,15 @@ object JobHistoryHelper extends Logging { val ecResourceMap = if (resourceInfo == null) new util.HashMap[String, ResourceWithStatus] else resourceInfo if (resourceMap != null) { - resourceMap.asInstanceOf[util.HashMap[String, ResourceWithStatus]].putAll(ecResourceMap) + resourceMap.asInstanceOf[util.Map[String, ResourceWithStatus]].putAll(ecResourceMap) } else { metricsMap.put(TaskConstant.ENTRANCEJOB_YARNRESOURCE, ecResourceMap) } - var engineInstanceMap: util.HashMap[String, AnyRef] = null + var engineInstanceMap: util.Map[String, AnyRef] = null if (metricsMap.containsKey(TaskConstant.ENTRANCEJOB_ENGINECONN_MAP)) { engineInstanceMap = metricsMap .get(TaskConstant.ENTRANCEJOB_ENGINECONN_MAP) - .asInstanceOf[util.HashMap[String, AnyRef]] + .asInstanceOf[util.Map[String, AnyRef]] } else { engineInstanceMap = new util.HashMap[String, AnyRef]() metricsMap.put(TaskConstant.ENTRANCEJOB_ENGINECONN_MAP, engineInstanceMap) @@ -334,7 +336,7 @@ object JobHistoryHelper extends Logging { val ticketId = infoMap.get(TaskConstant.TICKET_ID).asInstanceOf[String] val engineExtraInfoMap = engineInstanceMap .getOrDefault(ticketId, new util.HashMap[String, AnyRef]) - .asInstanceOf[util.HashMap[String, AnyRef]] + .asInstanceOf[util.Map[String, AnyRef]] engineExtraInfoMap.putAll(infoMap) engineInstanceMap.put(ticketId, engineExtraInfoMap) } else { diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelConstant.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelConstant.java index b43501ed9e4..4db4bfca404 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelConstant.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelConstant.java @@ -22,6 +22,4 @@ public class LabelConstant { public static final int LABEL_BUILDER_ERROR_CODE = 40001; public static final int LABEL_UTIL_CONVERT_ERROR_CODE = 40002; - - public static final String OFFLINE = "offline"; } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java index 6568fb838bf..88267e48005 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java @@ -110,25 +110,13 @@ Integer countUndoneTaskWithCreatorOnly( /** * query wait for failover job * - * Sql example: - * SELECT a.* FROM linkis_ps_job_history_group_history a - * where (a.instances = '' - * or a.instances is null - * or a.instances not in ('192.168.1.123:9104','192.168.1.124:9104') - * or EXISTS ( - * select 1 from - * ( - * select '192.168.1.123:9104' as instances, 1697775054098 as registryTime - * union all - * select '192.168.1.124:9104' as instances, 1666239054098 as registryTime - * ) b - * where a.instances = b.instances and UNIX_TIMESTAMP(a.created_time) * 1000 < b.registryTime - * ) - * ) - * and - * status in ('Inited','Running','Scheduled','WaitForRetry') - * and UNIX_TIMESTAMP(a.created_time) * 1000 >= 1666239054098 - * limit 10 + *

Sql example: SELECT a.* FROM linkis_ps_job_history_group_history a where (a.instances = '' + * or a.instances is null or a.instances not in ('192.168.1.123:9104','192.168.1.124:9104') or + * EXISTS ( select 1 from ( select '192.168.1.123:9104' as instances, 1697775054098 as + * registryTime union all select '192.168.1.124:9104' as instances, 1666239054098 as registryTime + * ) b where a.instances = b.instances and UNIX_TIMESTAMP(a.created_time) * 1000 < b.registryTime + * ) ) and status in ('Inited','Running','Scheduled','WaitForRetry') and + * UNIX_TIMESTAMP(a.created_time) * 1000 >= 1666239054098 limit 10 * * @param instancesMap * @param statusList @@ -136,8 +124,9 @@ Integer countUndoneTaskWithCreatorOnly( * @param limit * @return */ - List selectFailoverJobHistory(@Param("instancesMap") Map instancesMap, - @Param("statusList") List statusList, - @Param("startTimestamp") Long startTimestamp, - @Param("limit") Integer limit); + List selectFailoverJobHistory( + @Param("instancesMap") Map instancesMap, + @Param("statusList") List statusList, + @Param("startTimestamp") Long startTimestamp, + @Param("limit") Integer limit); } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala index 22084f88a69..a44cd0e2621 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala @@ -252,7 +252,8 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { logger.info(s"query failover jobs, start timestamp:${startTimestamp}， limit:${limit}") val jobResp = new JobRespProtocol Utils.tryCatch { - val jobList = jobHistoryMapper.selectFailoverJobHistory(reqMap, statusList, startTimestamp, limit) + val jobList = + jobHistoryMapper.selectFailoverJobHistory(reqMap, statusList, startTimestamp, limit) val jobReqList = jobList.asScala.map(jobHistory2JobRequest).toList val map = new util.HashMap[String, Object]() map.put(JobRequestConstants.JOB_HISTORY_LIST, jobReqList) @@ -266,14 +267,6 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { jobResp } - /* private def queryTaskList2RequestPersistTaskList(queryTask: java.util.List[QueryTask]): java.util.List[RequestPersistTask] = { - import scala.collection.JavaConversions._ - val tasks = new util.ArrayList[RequestPersistTask] - import org.apache.linkis.jobhistory.conversions.TaskConversions.queryTask2RequestPersistTask - queryTask.foreach(f => tasks.add(f)) - tasks - } */ - override def getJobHistoryByIdAndName(jobId: java.lang.Long, userName: String): JobHistory = { val jobReq = new JobHistory jobReq.setId(jobId) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala index 883f252d70d..930bfac73a5 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala @@ -17,10 +17,8 @@ package org.apache.linkis.gateway.ujes.parser -import org.apache.commons.lang3.StringUtils import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.entity.JobInstance -import org.apache.linkis.common.utils.JsonUtils import org.apache.linkis.gateway.config.GatewayConfiguration import org.apache.linkis.gateway.http.GatewayContext import org.apache.linkis.gateway.parser.AbstractGatewayParser @@ -30,6 +28,9 @@ import org.apache.linkis.protocol.utils.ZuulEntranceUtils import org.apache.linkis.rpc.interceptor.ServiceInstanceUtils import org.apache.linkis.server.BDPJettyServerHelper import org.apache.linkis.server.conf.ServerConfiguration + +import org.apache.commons.lang3.StringUtils + import org.springframework.stereotype.Component import javax.annotation.Resource @@ -37,7 +38,6 @@ import javax.annotation.Resource @Component class EntranceRequestGatewayParser extends AbstractGatewayParser { - @Resource private var jobHistoryQueryService: JobHistoryQueryService = _ @@ -49,9 +49,9 @@ class EntranceRequestGatewayParser extends AbstractGatewayParser { if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return val serviceInstance = if (execId.startsWith(EntranceRequestGatewayParser.API_REQUEST)) { if ( - gatewayContext.getRequest.getQueryParams.containsKey( - EntranceRequestGatewayParser.INSTANCE - ) + gatewayContext.getRequest.getQueryParams.containsKey( + EntranceRequestGatewayParser.INSTANCE + ) ) { val instances = gatewayContext.getRequest.getQueryParams.get(EntranceRequestGatewayParser.INSTANCE) @@ -83,7 +83,8 @@ class EntranceRequestGatewayParser extends AbstractGatewayParser { } def buildJobInstance(taskId: Long, gatewayContext: GatewayContext): JobInstance = { - val histories = jobHistoryQueryService.search(taskId, null, null, null, null, null, null, null) + val histories = + jobHistoryQueryService.search(taskId, null, null, null, null, null, null, null, null) if (histories.isEmpty) { sendErrorResponse(s"taskId $taskId is not exists.", gatewayContext) return null From 9e27aec833d3d20e43ff2f6c6c3d1d107860364d Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Tue, 28 Feb 2023 14:44:05 +0800 Subject: [PATCH 035/689] set default value --- .../server/DefaultEntranceServer.java | 13 +- .../server/EntranceFailoverJobServer.java | 178 +++++++++--------- .../entrance/conf/EntranceConfiguration.scala | 8 +- 3 files changed, 95 insertions(+), 104 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java index 54b855ffbd3..b077ab37bb3 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java @@ -17,6 +17,7 @@ package org.apache.linkis.entrance.server; +import org.apache.commons.io.IOUtils; import org.apache.linkis.entrance.EntranceContext; import org.apache.linkis.entrance.EntranceServer; import org.apache.linkis.entrance.conf.EntranceConfiguration; @@ -25,9 +26,8 @@ import org.apache.linkis.entrance.job.EntranceExecutionJob; import org.apache.linkis.entrance.log.LogReader; import org.apache.linkis.rpc.Sender; - -import org.apache.commons.io.IOUtils; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.event.EventListener; @@ -35,9 +35,6 @@ import javax.annotation.PostConstruct; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** Description: */ @Component(ServiceNameConsts.ENTRANCE_SERVER) public class DefaultEntranceServer extends EntranceServer { @@ -91,9 +88,9 @@ private void shutdownEntrance(ContextClosedEvent event) { logger.warn("Entrance exit to stop all job"); EntranceJob[] allUndoneTask = getAllUndoneTask(null); if (null != allUndoneTask) { - String msg = "Entrance exits the automatic cleanup task and can be rerun(服务退出自动清理任务，可以重跑)"; for (EntranceJob job : allUndoneTask) { - job.onFailure(msg, null); + job.onFailure( + "Entrance exits the automatic cleanup task and can be rerun(服务退出自动清理任务，可以重跑)", null); IOUtils.closeQuietly(((EntranceExecutionJob) job).getLogWriter().get()); } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java index 77e85cba694..73c91f6a368 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java @@ -29,12 +29,12 @@ import org.apache.linkis.publicservice.common.lock.service.CommonLockService; import org.apache.linkis.rpc.Sender; import org.apache.linkis.scheduler.queue.SchedulerEventState; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; - import java.util.Arrays; import java.util.List; import java.util.Map; @@ -43,9 +43,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - @Component(ServiceNameConsts.ENTRANCE_FAILOVER_SERVER) public class EntranceFailoverJobServer { @@ -61,96 +58,93 @@ public class EntranceFailoverJobServer { @PostConstruct public void init() { - this.scheduledExecutor = - Executors.newSingleThreadScheduledExecutor( - Utils.threadFactory("Linkis-Failover-Scheduler-Thread-", true)); - failoverTask(); + if (EntranceConfiguration.ENTRANCE_FAILOVER_ENABLED()) { + this.scheduledExecutor = + Executors.newSingleThreadScheduledExecutor( + Utils.threadFactory("Linkis-Failover-Scheduler-Thread-", true)); + failoverTask(); + } } public void failoverTask() { - if (EntranceConfiguration.ENTRANCE_FAILOVER_ENABLED()) { - scheduledExecutor.scheduleWithFixedDelay( - () -> { - EntranceSchedulerContext schedulerContext = - (EntranceSchedulerContext) - entranceServer - .getEntranceContext() - .getOrCreateScheduler() - .getSchedulerContext(); - - // entrance do not failover job when it is offline - if (schedulerContext.getOfflineFlag()) return; - - CommonLock commonLock = new CommonLock(); - commonLock.setLockObject(ENTRANCE_FAILOVER_LOCK); - Boolean locked = false; - try { - locked = commonLockService.lock(commonLock, 10 * 1000L); - if (!locked) return; - logger.info("success locked {}", ENTRANCE_FAILOVER_LOCK); - - // get all entrance server from eureka - ServiceInstance[] serviceInstances = - Sender.getInstances(Sender.getThisServiceInstance().getApplicationName()); - if (serviceInstances == null || serviceInstances.length <= 0) return; - - // serverInstance to map - Map serverInstanceMap = - Arrays.stream(serviceInstances) - .collect( - Collectors.toMap( - ServiceInstance::getInstance, - ServiceInstance::getRegistryTimestamp, - (k1, k2) -> k2)); - - // It is very important to avoid repeated execute job - // when failover self job, if self instance is empty, the job can be repeated execute - if (!serverInstanceMap.containsKey(Sender.getThisInstance())) { - logger.warn( - "server has just started and has not get self info, it does not failover"); - return; - } - - // get failover job expired time (获取任务故障转移过期时间，配置为0表示不过期, 过期则不处理) - long expiredTimestamp = 0L; - if (EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME() > 0) { - expiredTimestamp = - System.currentTimeMillis() - - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME(); - } - - // get uncompleted status - List statusList = - Arrays.stream(SchedulerEventState.uncompleteStatusArray()) - .map(Object::toString) - .collect(Collectors.toList()); - - List jobRequests = - JobHistoryHelper.queryWaitForFailoverTask( - serverInstanceMap, - statusList, - expiredTimestamp, - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); - if (jobRequests.isEmpty()) return; - List ids = - jobRequests.stream().map(JobRequest::getId).collect(Collectors.toList()); - logger.info("success query failover jobs , job size: {}, ids: {}", ids.size(), ids); - - // failover to local server - for (JobRequest jobRequest : jobRequests) { - entranceServer.failoverExecute(jobRequest); - } - logger.info("finished execute failover jobs, job ids: {}", ids); - - } catch (Exception e) { - logger.error("failover failed", e); - } finally { - if (locked) commonLockService.unlock(commonLock); + scheduledExecutor.scheduleWithFixedDelay( + () -> { + EntranceSchedulerContext schedulerContext = + (EntranceSchedulerContext) + entranceServer.getEntranceContext().getOrCreateScheduler().getSchedulerContext(); + + // entrance do not failover job when it is offline + if (schedulerContext.getOfflineFlag()) return; + + CommonLock commonLock = new CommonLock(); + commonLock.setLockObject(ENTRANCE_FAILOVER_LOCK); + Boolean locked = false; + try { + locked = commonLockService.lock(commonLock, 30 * 1000L); + if (!locked) return; + logger.info("success locked {}", ENTRANCE_FAILOVER_LOCK); + + // get all entrance server from eureka + ServiceInstance[] serviceInstances = + Sender.getInstances(Sender.getThisServiceInstance().getApplicationName()); + if (serviceInstances == null || serviceInstances.length <= 0) return; + + // serverInstance to map + Map serverInstanceMap = + Arrays.stream(serviceInstances) + .collect( + Collectors.toMap( + ServiceInstance::getInstance, + ServiceInstance::getRegistryTimestamp, + (k1, k2) -> k2)); + + // It is very important to avoid repeated execute job + // when failover self job, if self instance is empty, the job can be repeated execute + if (!serverInstanceMap.containsKey(Sender.getThisInstance())) { + logger.warn( + "server has just started and has not get self info, it does not failover"); + return; } - }, - EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INIT_TIME(), - EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INTERVAL(), - TimeUnit.MILLISECONDS); - } + + // get failover job expired time (获取任务故障转移过期时间，配置为0表示不过期, 过期则不处理) + long expiredTimestamp = 0L; + if (EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME() > 0) { + expiredTimestamp = + System.currentTimeMillis() + - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME(); + } + + // get uncompleted status + List statusList = + Arrays.stream(SchedulerEventState.uncompleteStatusArray()) + .map(Object::toString) + .collect(Collectors.toList()); + + List jobRequests = + JobHistoryHelper.queryWaitForFailoverTask( + serverInstanceMap, + statusList, + expiredTimestamp, + EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); + if (jobRequests.isEmpty()) return; + List ids = + jobRequests.stream().map(JobRequest::getId).collect(Collectors.toList()); + logger.info("success query failover jobs , job size: {}, ids: {}", ids.size(), ids); + + // failover to local server + for (JobRequest jobRequest : jobRequests) { + entranceServer.failoverExecute(jobRequest); + } + logger.info("finished execute failover jobs, job ids: {}", ids); + + } catch (Exception e) { + logger.error("failover failed", e); + } finally { + if (locked) commonLockService.unlock(commonLock); + } + }, + EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INIT_TIME(), + EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INTERVAL(), + TimeUnit.MILLISECONDS); } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index 13db69700f7..d8248620b77 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -241,7 +241,7 @@ object EntranceConfiguration { // if true, the waitForRetry job in runningJobs can be failover val ENTRANCE_FAILOVER_RETRY_JOB_ENABLED = - CommonVars("linkis.entrance.failover.retry.job.enable", true) + CommonVars("linkis.entrance.failover.retry.job.enable", false) val ENTRANCE_UPDATE_BATCH_SIZE = CommonVars("linkis.entrance.update.batch.size", 100) @@ -255,12 +255,12 @@ object EntranceConfiguration { val ENTRANCE_GROUP_SCAN_INTERVAL = CommonVars("linkis.entrance.group.scan.interval", 60 * 1000) val ENTRANCE_FAILOVER_RETAIN_ENGINE_CONN_ENABLED = - CommonVars("linkis.entrance.failover.retain.engine.conn.enable", true) + CommonVars("linkis.entrance.failover.retain.engine.conn.enable", false) val ENTRANCE_FAILOVER_RETAIN_YARN_RESOURCE_ENABLED = - CommonVars("linkis.entrance.failover.retain.yarn.resource.enable", true) + CommonVars("linkis.entrance.failover.retain.yarn.resource.enable", false) val ENTRANCE_FAILOVER_RUNNING_KILL_ENABLED = - CommonVars("linkis.entrance.failover.running.kill.enable", true) + CommonVars("linkis.entrance.failover.running.kill.enable", false) } From 3dda863d37aa94326b77c52a33251bf934e588d3 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Tue, 28 Feb 2023 15:27:17 +0800 Subject: [PATCH 036/689] add config to properties --- .../linkis/entrance/server/DefaultEntranceServer.java | 9 ++++++--- .../entrance/server/EntranceFailoverJobServer.java | 7 +++++-- .../org/apache/linkis/entrance/EntranceServer.scala | 2 +- linkis-dist/package/conf/linkis-cg-entrance.properties | 7 ++++++- linkis-dist/package/conf/linkis-mg-gateway.properties | 4 ++-- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java index b077ab37bb3..14bea604351 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java @@ -17,7 +17,6 @@ package org.apache.linkis.entrance.server; -import org.apache.commons.io.IOUtils; import org.apache.linkis.entrance.EntranceContext; import org.apache.linkis.entrance.EntranceServer; import org.apache.linkis.entrance.conf.EntranceConfiguration; @@ -26,8 +25,9 @@ import org.apache.linkis.entrance.job.EntranceExecutionJob; import org.apache.linkis.entrance.log.LogReader; import org.apache.linkis.rpc.Sender; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import org.apache.commons.io.IOUtils; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.event.EventListener; @@ -35,6 +35,9 @@ import javax.annotation.PostConstruct; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** Description: */ @Component(ServiceNameConsts.ENTRANCE_SERVER) public class DefaultEntranceServer extends EntranceServer { diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java index 73c91f6a368..d7f5ce5951b 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java @@ -29,12 +29,12 @@ import org.apache.linkis.publicservice.common.lock.service.CommonLockService; import org.apache.linkis.rpc.Sender; import org.apache.linkis.scheduler.queue.SchedulerEventState; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; + import java.util.Arrays; import java.util.List; import java.util.Map; @@ -43,6 +43,9 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + @Component(ServiceNameConsts.ENTRANCE_FAILOVER_SERVER) public class EntranceFailoverJobServer { diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index 8ef5c268b5f..8e9bbeeac00 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -390,7 +390,7 @@ abstract class EntranceServer extends Logging { engineStopRequest.setServiceInstance(ecInstance) // send to linkismanager kill ec Sender - .getSender(RPCConfiguration.LINKIS_MANAGER_APPLICATION_NAME.getValue) + .getSender(RPCConfiguration.LINKIS_MANAGER_SERVICE_NAME.getValue) .send(engineStopRequest) val msg = s"job ${jobRequest.getId} send EngineStopRequest to linkismanager, kill EC instance $ecInstance" diff --git a/linkis-dist/package/conf/linkis-cg-entrance.properties b/linkis-dist/package/conf/linkis-cg-entrance.properties index e89ced2159f..639256d5cf3 100644 --- a/linkis-dist/package/conf/linkis-cg-entrance.properties +++ b/linkis-dist/package/conf/linkis-cg-entrance.properties @@ -33,4 +33,9 @@ spring.server.port=9104 wds.linkis.entrance.user.creator.ip.interceptor.switch=false ## you may set service version if you want to distinguish different configuration version -spring.eureka.instance.metadata-map.linkis.conf.version=v1 \ No newline at end of file +spring.eureka.instance.metadata-map.linkis.conf.version=v1 + + +wds.linkis.server.mybatis.mapperLocations=classpath*:mapper/common/*.xml,classpath*:mapper/mysql/*.xml +wds.linkis.server.mybatis.BasePackage=org.apache.linkis.publicservice.common.lock.dao +wds.linkis.server.mybatis.typeAliasesPackage=org.apache.linkis.publicservice.common.lock.entity \ No newline at end of file diff --git a/linkis-dist/package/conf/linkis-mg-gateway.properties b/linkis-dist/package/conf/linkis-mg-gateway.properties index 84be3d897dd..27656f7f315 100644 --- a/linkis-dist/package/conf/linkis-mg-gateway.properties +++ b/linkis-dist/package/conf/linkis-mg-gateway.properties @@ -21,8 +21,8 @@ wds.linkis.gateway.conf.url.pass.auth=/dss/ wds.linkis.gateway.conf.enable.token.auth=true wds.linkis.is.gateway=true wds.linkis.server.mybatis.mapperLocations=classpath*:mapper/common/*.xml,classpath*:mapper/mysql/*.xml -wds.linkis.server.mybatis.typeAliasesPackage=org.apache.linkis.instance.label.entity -wds.linkis.server.mybatis.BasePackage=org.apache.linkis.instance.label.dao,org.apache.linkis.gateway.authentication.dao +wds.linkis.server.mybatis.typeAliasesPackage=org.apache.linkis.instance.label.entity,org.apache.linkis.jobhistory.entity +wds.linkis.server.mybatis.BasePackage=org.apache.linkis.instance.label.dao,org.apache.linkis.gateway.authentication.dao,org.apache.linkis.jobhistory.dao wds.linkis.label.entity.packages=org.apache.linkis.gateway.ujes.route.label wds.linkis.login_encrypt.enable=false ##LDAP From ff2871919e7903877e175f14979d797e20706dd6 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Tue, 28 Feb 2023 21:09:58 +0800 Subject: [PATCH 037/689] change HashMap to Map --- .../main/scala/org/apache/linkis/entrance/EntranceServer.scala | 2 +- .../linkis/entrance/interceptor/impl/CustomVariableUtils.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index 8e9bbeeac00..55be20fd4dd 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -312,7 +312,7 @@ abstract class EntranceServer extends Logging { val logAppender = new java.lang.StringBuilder() logAppender.append( - "*************************************FAILOVER**************************************" + "*************************************FAILOVER**************************************\n" ) // try to kill ec diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CustomVariableUtils.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CustomVariableUtils.scala index 7a7cb7463a9..a40c3fa35d2 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CustomVariableUtils.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CustomVariableUtils.scala @@ -63,7 +63,7 @@ object CustomVariableUtils extends Logging { } val variableMap = TaskUtils .getVariableMap(jobRequest.getParams) - .asInstanceOf[util.HashMap[String, String]] + .asInstanceOf[util.Map[String, String]] variables.putAll(variableMap) if (!variables.containsKey("user")) { variables.put("user", jobRequest.getExecuteUser) From 39d45d3b40d27427a2aa28d334f2bc88189b55e5 Mon Sep 17 00:00:00 2001 From: guoshupei <719126Liyuelynn> Date: Wed, 1 Mar 2023 11:03:56 +0800 Subject: [PATCH 038/689] update default value --- .../main/scala/org/apache/linkis/entrance/EntranceServer.scala | 2 +- .../org/apache/linkis/entrance/conf/EntranceConfiguration.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index 55be20fd4dd..5560cc716dc 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -312,7 +312,7 @@ abstract class EntranceServer extends Logging { val logAppender = new java.lang.StringBuilder() logAppender.append( - "*************************************FAILOVER**************************************\n" + "*************************************FAILOVER************************************** \n" ) // try to kill ec diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index d8248620b77..17f2dffd9c0 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -237,7 +237,7 @@ object EntranceConfiguration { CommonVars("linkis.entrance.failover.data.num.limit", 10).getValue val ENTRANCE_FAILOVER_DATA_INTERVAL_TIME = - CommonVars("linkis.entrance.failover.data.interval.time", new TimeType("7d").toLong).getValue + CommonVars("linkis.entrance.failover.data.interval.time", new TimeType("1d").toLong).getValue // if true, the waitForRetry job in runningJobs can be failover val ENTRANCE_FAILOVER_RETRY_JOB_ENABLED = From fdc54d45cb2b9f6f88a1eeef9e06ba3424a24fa5 Mon Sep 17 00:00:00 2001 From: guoshupei <15764973965@163.com> Date: Thu, 2 Mar 2023 12:08:08 +0800 Subject: [PATCH 039/689] Optimal refresh consumer group maxAllowRunningJobs logic --- .../entrance/scheduler/EntranceParallelConsumerManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala index a067d658299..a6e24388a68 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala @@ -75,7 +75,7 @@ class EntranceParallelConsumerManager(maxParallelismUsers: Int, schedulerName: S } def refreshAllGroupMaxAllowRunningJobs(validInsCount: Int): Unit = { - if (validInsCount <= 0) return + if (validInsCount <= 1) return listConsumers() .foreach(item => { item.getGroup match { From f1deacaef7707b762492a441e3c84b32f4827469 Mon Sep 17 00:00:00 2001 From: binbincheng <106590848+binbinCheng@users.noreply.github.com> Date: Thu, 2 Mar 2023 15:22:29 +0800 Subject: [PATCH 040/689] The abnormal class description information in the checkEngineConnDistHome method supplements the path information (#4283) --- .../errorcode/EngineconnCoreErrorCodeSummary.java | 8 +++++--- .../localize/AbstractEngineConnBmlResourceGenerator.scala | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java index f7fd0165b6e..58e3b5fad6c 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java @@ -27,12 +27,14 @@ public enum EngineconnCoreErrorCodeSummary implements LinkisErrorCode { CANNOT_DEFAULT_EF(20000, "Cannot find default ExecutorFactory(找不到默认的 ExecutorFactory)"), ETL_NOT_EXISTS(20000, "EngineTypeLabel does not exist(EngineTypeLabel 不存在)"), UCL_NOT_EXISTS(20000, "UserCreatorLabel does not exist(UserCreatorLabel 不存在)"), - CANNOT_HOME_PATH_EC(20001, "Cannot find the home path of engineConn(找不到 engineConn 的 home 路径)"), + CANNOT_HOME_PATH_EC( + 20001, "Cannot find the home path of engineConn at: {0}(找不到 engineConn 的 home 路径，该路径为：{0})"), CANNOT_HOME_PATH_DIST( - 20001, "Cannot find the home path of engineconn dist(找不到 engineconn dist 的 home 路径)"), + 20001, + "Could not find the home path for engineconn dist at: {0}(找不到 engineconn dist 的 home 路径，该路径为：{0})"), DIST_IS_EMPTY( 20001, - "The dist of EngineConn is empty,engineConnType is:{0}(EngineConn 的 dist 为空,engineConnType为：{})"), + "The dist of EngineConn is empty,engineConnType is:{0}(EngineConn 的 dist 为空,engineConnType为：{0})"), ENGINE_VERSION_NOT_FOUND( 20001, "Cannot find the path of engineConn with specified version: {0} and engineConnType: {1}(找不到版本为：{0} engineConnType 为:{1}的engineConn路径"), diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala index d62f8996f02..476a969788c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala @@ -34,7 +34,7 @@ abstract class AbstractEngineConnBmlResourceGenerator extends EngineConnBmlResou if (!new File(getEngineConnsHome).exists) { throw new EngineConnPluginErrorException( CANNOT_HOME_PATH_EC.getErrorCode, - CANNOT_HOME_PATH_EC.getErrorDesc + MessageFormat.format(CANNOT_HOME_PATH_EC.getErrorDesc, getEngineConnsHome) ) } @@ -70,7 +70,7 @@ abstract class AbstractEngineConnBmlResourceGenerator extends EngineConnBmlResou if (!engineConnPackageHome.exists()) { throw new EngineConnPluginErrorException( CANNOT_HOME_PATH_DIST.getErrorCode, - CANNOT_HOME_PATH_DIST.getErrorDesc + MessageFormat.format(CANNOT_HOME_PATH_DIST.getErrorDesc, engineConnPackageHome.getPath) ) } } From 2826602d2c6fa975a13c6f1939a93ab7710756a2 Mon Sep 17 00:00:00 2001 From: binbinCheng Date: Fri, 3 Mar 2023 16:56:21 +0800 Subject: [PATCH 041/689] mapper formatting --- .../common/EngineConnBmlResourceMapper.xml | 16 +- .../ExternalResourceProviderDaoImpl.xml | 4 +- .../mapper/common/ECResourceRecordMapper.xml | 33 +- .../mapper/common/LabelManagerMapper.xml | 166 ++++++---- .../mapper/common/LockManagerMapper.xml | 16 +- .../mapper/common/NodeManagerMapper.xml | 83 +++-- .../mapper/common/NodeMetricManagerMapper.xml | 49 ++- .../mapper/common/ResourceManagerMapper.xml | 105 ++++-- .../mapper/common/CgManagerLabelMapper.xml | 6 +- .../common/ConfigurationConfigKeyMapper.xml | 22 +- .../common/ConfigurationConfigValueMapper.xml | 11 +- .../ConfigurationKeyEngineRelationMapper.xml | 5 +- .../mapper/common/DatasourceAccessMapper.xml | 8 +- .../mapper/common/DatasourceEnvMapper.xml | 10 +- .../mapper/common/DatasourceTypeKeyMapper.xml | 4 +- .../mapper/common/DatasourceTypeMapper.xml | 10 +- .../EngineConnPluginBmlResourcesMapper.xml | 4 +- .../mapper/common/GatewayAuthTokenMapper.xml | 10 +- .../mapper/common/PsErrorCodeMapper.xml | 12 +- .../RmExternalResourceProviderMapper.xml | 12 +- .../mapper/common/UdfManagerMapper.xml | 6 +- .../resources/mapper/common/UdfTreeMapper.xml | 12 +- .../mapper/common/BmlProjectMapper.xml | 48 ++- .../mapper/common/DownloadMapper.xml | 6 +- .../mapper/common/ResourceMapper.xml | 32 +- .../resources/mapper/common/VersionMapper.xml | 107 ++++-- .../mapper/common/contextHistoryMapper.xml | 18 +- .../mapper/common/contextIDListenerMapper.xml | 11 +- .../main/resources/mapper/common/MdqDao.xml | 15 +- .../mapper/common/JobHistoryMapper.xml | 4 +- .../mapper/common/CommonLockMapper.xml | 10 +- .../main/resources/mapper/common/UDFDao.xml | 311 ++++++++++-------- .../resources/mapper/common/UDFTreeDao.xml | 45 ++- .../resources/mapper/common/UDFVersionDao.xml | 60 ++-- .../resources/mapper/common/VarMapper.xml | 8 +- .../resources/mapper/common/TokenMapper.xml | 4 +- 36 files changed, 805 insertions(+), 478 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/resources/mapper/common/EngineConnBmlResourceMapper.xml b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/resources/mapper/common/EngineConnBmlResourceMapper.xml index 1be548bbc6f..c922ff549f1 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/resources/mapper/common/EngineConnBmlResourceMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/resources/mapper/common/EngineConnBmlResourceMapper.xml @@ -56,14 +56,14 @@ - delete from linkis_cg_engine_conn_plugin_bml_resources where - engine_conn_type=#{engineConnBmlResource.engineConnType} and - `version`=#{engineConnBmlResource.version} - and file_name=#{engineConnBmlResource.fileName} + DELETE FROM linkis_cg_engine_conn_plugin_bml_resources + WHERE engine_conn_type = #{engineConnBmlResource.engineConnType} + AND `version` = #{engineConnBmlResource.version} + AND file_name = #{engineConnBmlResource.fileName} - update linkis_cg_engine_conn_plugin_bml_resources + UPDATE linkis_cg_engine_conn_plugin_bml_resources `last_modified` = #{engineConnBmlResource.lastModified}, @@ -77,12 +77,12 @@ WHERE engine_conn_type=#{engineConnBmlResource.engineConnType} and `version`=#{engineConnBmlResource.version} - and file_name=#{engineConnBmlResource.fileName} + AND file_name=#{engineConnBmlResource.fileName} - insert into linkis_cg_engine_conn_plugin_bml_resources() - values(#{engineConnBmlResource.engineConnType}, #{engineConnBmlResource.version}, + INSERT INTO linkis_cg_engine_conn_plugin_bml_resources() + VALUES (#{engineConnBmlResource.engineConnType}, #{engineConnBmlResource.version}, #{engineConnBmlResource.fileName},#{engineConnBmlResource.fileSize}, #{engineConnBmlResource.lastModified},#{engineConnBmlResource.bmlResourceId}, #{engineConnBmlResource.bmlResourceVersion}, diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/resources/mapper/common/ExternalResourceProviderDaoImpl.xml b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/resources/mapper/common/ExternalResourceProviderDaoImpl.xml index 166b629b5f2..e7e91cd05a2 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/resources/mapper/common/ExternalResourceProviderDaoImpl.xml +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/resources/mapper/common/ExternalResourceProviderDaoImpl.xml @@ -39,9 +39,9 @@ - INSERT INTO linkis_cg_ec_resource_info_record(label_value,create_user, ticket_id, service_instance, request_times, request_resource, - used_times, used_resource, release_times, released_resource,release_time,used_time,ecm_instance,log_dir_suffix,status) - VALUES (#{labelValue},#{createUser},#{ticketId},#{serviceInstance},#{requestTimes}, - #{requestResource},#{usedTimes},#{usedResource}, - #{releaseTimes},#{releasedResource},#{releaseTime},#{usedTime},#{ecmInstance},#{logDirSuffix},#{status}) + INSERT INTO linkis_cg_ec_resource_info_record (label_value, create_user, ticket_id, service_instance, request_times + , request_resource, used_times, used_resource, release_times, released_resource + , release_time, used_time, ecm_instance, log_dir_suffix, status) + VALUES (#{labelValue}, #{createUser}, #{ticketId}, #{serviceInstance}, #{requestTimes} + , #{requestResource}, #{usedTimes}, #{usedResource}, #{releaseTimes}, #{releasedResource} + , #{releaseTime}, #{usedTime}, #{ecmInstance}, #{logDirSuffix}, #{status}) @@ -58,16 +64,19 @@ - delete from linkis_cg_ec_resource_info_record WHERE ticket_id = #{ticketId} + DELETE FROM linkis_cg_ec_resource_info_record + WHERE ticket_id = #{ticketId} - delete from linkis_cg_ec_resource_info_record WHERE id = #{id} + DELETE FROM linkis_cg_ec_resource_info_record + WHERE id = #{id} - select * from linkis_cg_ec_resource_info_record - where id in ( + SELECT * FROM linkis_cg_ec_resource_info_record + WHERE id IN ( select max(ecr.id) as id from linkis_cg_ec_resource_info_record ecr WHERE 1=1 diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/LabelManagerMapper.xml b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/LabelManagerMapper.xml index 4e1929cb485..bc6b37b4891 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/LabelManagerMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/LabelManagerMapper.xml @@ -83,32 +83,43 @@ - REPLACE INTO linkis_cg_manager_label_value_relation (label_value_key,label_value_content,label_id,update_time,create_time) VALUES(#{labelKey}, #{labelStringValue},#{labelId},now(),now()) + REPLACE INTO linkis_cg_manager_label_value_relation (label_value_key,label_value_content,label_id,update_time,create_time) + VALUES(#{labelKey}, #{labelStringValue},#{labelId},now(),now()) - delete from linkis_cg_manager_label where id =#{id} + DELETE + FROM linkis_cg_manager_label + WHERE id =#{id} - delete from linkis_cg_manager_label where label_key =#{labelKey} and label_value=#{labelStringValue} + DELETE + FROM linkis_cg_manager_label + WHERE label_key =#{labelKey} + AND label_value=#{labelStringValue} - delete from linkis_cg_manager_label_value_relation where label_id =#{id} + DELETE + FROM linkis_cg_manager_label_value_relation + WHERE label_id =#{id} - update linkis_cg_manager_label set label_key = #{persistenceLabel.labelKey},label_value = #{persistenceLabel.stringValue}, - label_feature=#{persistenceLabel.feature},label_value_size=#{persistenceLabel.labelValueSize},update_time=#{persistenceLabel.updateTime} where id=#{id} + UPDATE linkis_cg_manager_label + SET label_key = #{persistenceLabel.labelKey}, label_value = #{persistenceLabel.stringValue}, label_feature = #{persistenceLabel.feature}, label_value_size = #{persistenceLabel.labelValueSize}, update_time = #{persistenceLabel.updateTime} + WHERE id = #{id} - insert into linkis_cg_manager_label_service_instance(label_id, service_instance, update_time,create_time) values + INSERT INTO linkis_cg_manager_label_service_instance(label_id, service_instance, update_time,create_time) VALUES (#{labelId}, #{instance},now(),now()) @@ -128,30 +139,45 @@ - insert into linkis_cg_manager_label_resource(label_id, resource_id,update_time, create_time) values + INSERT INTO linkis_cg_manager_label_resource(label_id, resource_id,update_time, create_time) VALUES (#{labelId},#{resourceId},now(), now()) - select service_instance from linkis_cg_manager_label_service_instance where label_id in ( + SELECT service_instance FROM linkis_cg_manager_label_service_instance WHERE label_id IN ( #{labelId} @@ -175,7 +201,7 @@ - insert into linkis_cg_manager_label_user(username, label_id,update_time, create_time) values + INSERT INTO linkis_cg_manager_label_user(username, label_id,update_time, create_time) VALUES (#{userName},#{labelId},now(), now()) - delete from linkis_cg_manager_label_resource A join linkis_cg_manager_linkis_resources B on A.resource_id = B.id and B.ticketId=#{ticketId} + DELETE FROM linkis_cg_manager_label_resource A + JOIN linkis_cg_manager_linkis_resources B + ON A.resource_id = B.id + AND B.ticketId = #{ticketId} - delete from linkis_cg_manager_label_service_instance where service_instance = #{instance} and label_id in + DELETE FROM linkis_cg_manager_label_service_instance WHERE service_instance = #{instance} AND label_id IN #{labelId} - delete from linkis_cg_manager_label_user where username = #{userName} and label_id in + DELETE FROM linkis_cg_manager_label_user WHERE username = #{userName} AND label_id IN #{labelId} - delete from linkis_cg_manager_label_user where label_id=#{labelId} + DELETE FROM linkis_cg_manager_label_user + WHERE label_id = #{labelId} - select * from linkis_cg_manager_label where label_key = #{labelKey} and label_value = #{stringValue} + SELECT * + FROM linkis_cg_manager_label + WHERE label_key = #{labelKey} + AND label_value = #{stringValue} @@ -301,15 +342,6 @@ INNER JOIN linkis_cg_manager_service_instance si ON si.instance = t.service_instance - - - - - SELECT r.* FROM linkis_cg_manager_label_resource lr, linkis_cg_manager_linkis_resources r, - linkis_cg_manager_label l - WHERE l.id = #{labelId} and - lr.label_id = #{labelId} and - lr.resource_id = r.id + SELECT r.* + FROM linkis_cg_manager_label_resource lr, linkis_cg_manager_linkis_resources r, linkis_cg_manager_label l + WHERE l.id = #{labelId} + AND lr.label_id = #{labelId} + AND lr.resource_id = r.id - delete from linkis_cg_manager_linkis_resources where id in (select resource_id from - linkis_cg_manager_label_resource where label_id = #{labelId}); + DELETE FROM linkis_cg_manager_linkis_resources + WHERE id IN ( + SELECT resource_id + FROM linkis_cg_manager_label_resource + WHERE label_id = #{labelId} + ); - delete from linkis_cg_manager_label_resource where label_id = #{labelId}; + DELETE FROM linkis_cg_manager_label_resource + WHERE label_id = #{labelId}; - delete from linkis_cg_manager_linkis_resources where id in (select resource_id from - linkis_cg_manager_label_resource - where label_id = #{id} ); + DELETE FROM linkis_cg_manager_linkis_resources + WHERE id IN ( + SELECT resource_id + FROM linkis_cg_manager_label_resource + WHERE label_id = #{id} + ); - delete from linkis_cg_manager_label_resource where label_id = #{id}; + DELETE FROM linkis_cg_manager_label_resource + WHERE label_id = #{id}; - delete from linkis_cg_manager_linkis_resources where id in (select resource_id from + DELETE FROM linkis_cg_manager_linkis_resources WHERE id IN (SELECT resource_id FROM linkis_cg_manager_label_resource - where label_id in + WHERE label_id IN #{labelId} ); - delete from linkis_cg_manager_label_resource where label_id in + DELETE FROM linkis_cg_manager_label_resource WHERE label_id IN #{labelId} @@ -417,10 +458,10 @@ - delete from linkis_cg_manager_linkis_resources where id in (select resource_id from + DELETE FROM linkis_cg_manager_linkis_resources WHERE id IN (select resource_id from linkis_cg_manager_label_resource - where label_id in ( - select tmp.id from( + WHERE label_id IN ( + SELECT tmp.id FROM( SELECT l.id,l.label_value_size FROM linkis_cg_manager_label l ,linkis_cg_manager_label_resource lr ,linkis_cg_manager_label_value_relation lvr WHERE l.id = lr.label_id AND l.id = lvr.label_id @@ -434,9 +475,9 @@ ) as tmp )); - delete from linkis_cg_manager_label_resource where label_id in + DELETE FROM linkis_cg_manager_label_resource WHERE label_id IN ( - select tmp.id from( + SELECT tmp.id FROM( SELECT l.id,l.label_value_size FROM linkis_cg_manager_label l ,linkis_cg_manager_label_resource lr ,linkis_cg_manager_label_value_relation lvr WHERE l.id = lr.label_id AND l.id = lvr.label_id @@ -534,8 +575,11 @@ \ No newline at end of file diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/LockManagerMapper.xml b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/LockManagerMapper.xml index 8c0339ea2da..ac074a9da75 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/LockManagerMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/LockManagerMapper.xml @@ -46,15 +46,21 @@ - delete from linkis_cg_manager_lock where id = #{id} + DELETE FROM linkis_cg_manager_lock + WHERE id = #{id} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml index 2294cdbca4f..6eb38c52b5d 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml @@ -20,73 +20,102 @@ - update linkis_cg_manager_engine_em set engine_instance = #{instance} where engine_instance = #{tickedId} + UPDATE linkis_cg_manager_engine_em + SET engine_instance = #{instance} + WHERE engine_instance = #{tickedId} - update linkis_cg_manager_label_service_instance set service_instance = #{instance} where service_instance = #{tickedId} + UPDATE linkis_cg_manager_label_service_instance + SET service_instance = #{instance} + WHERE service_instance = #{tickedId} - insert into linkis_cg_manager_service_instance(instance,name,owner,mark,update_time,create_time,updator,creator) - values(#{instance},#{name},#{owner},#{mark},#{updateTime},#{createTime},#{updator},#{creator}) + INSERT INTO linkis_cg_manager_service_instance (instance, name, owner, mark, update_time + , create_time, updator, creator) + VALUES (#{instance}, #{name}, #{owner}, #{mark}, #{updateTime} + , #{createTime}, #{updator}, #{creator}) - update linkis_cg_manager_service_instance set instance = #{persistenceNode.instance}, owner = #{persistenceNode.owner},mark = #{persistenceNode.mark},name = #{persistenceNode.name}, - update_time = #{persistenceNode.updateTime},updator = #{persistenceNode.updator},creator = #{persistenceNode.creator} where instance = #{instance} + UPDATE linkis_cg_manager_service_instance + SET instance = #{persistenceNode.instance}, owner = #{persistenceNode.owner}, mark = #{persistenceNode.mark}, name = #{persistenceNode.name}, update_time = #{persistenceNode.updateTime}, updator = #{persistenceNode.updator}, creator = #{persistenceNode.creator} + WHERE instance = #{instance} - delete from linkis_cg_manager_service_instance where instance = #{instance} + DELETE FROM linkis_cg_manager_service_instance + WHERE instance = #{instance} - update linkis_cg_manager_service_instance set owner = #{persistenceNode.owner},mark = #{persistenceNode.mark},name = #{persistenceNode.name}, - update_time = #{persistenceNode.updateTime},create_time = #{persistenceNode.createTime},updator = #{persistenceNode.updator} - ,creator = #{persistenceNode.creator} where instance = #{persistenceNode.instance} + UPDATE linkis_cg_manager_service_instance + SET owner = #{persistenceNode.owner}, mark = #{persistenceNode.mark}, name = #{persistenceNode.name}, update_time = #{persistenceNode.updateTime}, create_time = #{persistenceNode.createTime}, updator = #{persistenceNode.updator}, creator = #{persistenceNode.creator} + WHERE instance = #{persistenceNode.instance} - insert into linkis_cg_manager_engine_em (engine_instance, em_instance, update_time, create_time) - values(#{engineNodeInstance}, #{emNodeInstance}, now(), now()) + INSERT INTO linkis_cg_manager_engine_em (engine_instance, em_instance, update_time, create_time) + VALUES (#{engineNodeInstance}, #{emNodeInstance}, now(), now()) - delete from linkis_cg_manager_engine_em where engine_instance = #{engineNodeInstance} and em_instance = #{emNodeInstance} + DELETE FROM linkis_cg_manager_engine_em + WHERE engine_instance = #{engineNodeInstance} + AND em_instance = #{emNodeInstance} - select count(id) from linkis_cg_manager_service_instance_metrics met inner join linkis_cg_manager_service_instance ins - on met.instance = #{instance} and ins.instance = #{instance} and met.instance = ins.instance + SELECT count(id) + FROM linkis_cg_manager_service_instance_metrics met + INNER JOIN linkis_cg_manager_service_instance ins + ON met.instance = #{instance} + AND ins.instance = #{instance} + AND met.instance = ins.instance - update linkis_cg_manager_service_instance_metrics + UPDATE linkis_cg_manager_service_instance_metrics instance_status = #{nodeMetrics.status}, overload = #{nodeMetrics.overLoad}, heartbeat_msg = #{nodeMetrics.heartBeatMsg}, healthy_status=#{nodeMetrics.healthy}, update_time=#{nodeMetrics.updateTime}, - where instance = #{instance} + WHERE instance = #{instance} - delete from linkis_cg_manager_service_instance_metrics where instance in - (select instance from linkis_cg_manager_service_instance where instance=#{instance}) + DELETE FROM linkis_cg_manager_service_instance_metrics + WHERE instance IN ( + SELECT instance + FROM linkis_cg_manager_service_instance + WHERE instance = #{instance} + ) - delete from linkis_cg_manager_service_instance_metrics where instance = #{instance} + DELETE FROM linkis_cg_manager_service_instance_metrics + WHERE instance = #{instance} - update linkis_cg_manager_service_instance_metrics - set instance_status = #{instanceStatus}, - update_time= now() - where instance = #{instance} and instance_status=#{oldStatus} + UPDATE linkis_cg_manager_service_instance_metrics + SET instance_status = #{instanceStatus}, update_time = now() + WHERE instance = #{instance} + AND instance_status = #{oldStatus} diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/ResourceManagerMapper.xml b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/ResourceManagerMapper.xml index 559c5069e1e..23b03bacb68 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/ResourceManagerMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/ResourceManagerMapper.xml @@ -44,58 +44,113 @@ - update linkis_cg_manager_linkis_resources set max_resource = #{persistenceResource.maxResource},min_resource = #{persistenceResource.minResource}, - used_resource=#{persistenceResource.usedResource},left_resource=#{persistenceResource.leftResource},expected_resource=#{persistenceResource.expectedResource}, - locked_resource=#{persistenceResource.lockedResource}, - update_time=#{persistenceResource.updateTime} where ticketId = #{ticketId} + UPDATE linkis_cg_manager_linkis_resources + SET max_resource = #{persistenceResource.maxResource}, min_resource = #{persistenceResource.minResource}, used_resource = #{persistenceResource.usedResource}, left_resource = #{persistenceResource.leftResource}, expected_resource = #{persistenceResource.expectedResource}, locked_resource = #{persistenceResource.lockedResource}, update_time = #{persistenceResource.updateTime} + WHERE ticketId = #{ticketId} - update linkis_cg_manager_linkis_resources set max_resource = #{persistenceResource.maxResource},min_resource = #{persistenceResource.minResource}, - used_resource=#{persistenceResource.usedResource},left_resource=#{persistenceResource.leftResource},expected_resource=#{persistenceResource.expectedResource}, - locked_resource=#{persistenceResource.lockedResource}, - update_time=#{persistenceResource.updateTime} where id = #{resourceId} + UPDATE linkis_cg_manager_linkis_resources + SET max_resource = #{persistenceResource.maxResource}, min_resource = #{persistenceResource.minResource}, used_resource = #{persistenceResource.usedResource}, left_resource = #{persistenceResource.leftResource}, expected_resource = #{persistenceResource.expectedResource}, locked_resource = #{persistenceResource.lockedResource}, update_time = #{persistenceResource.updateTime} + WHERE id = #{resourceId} - delete from linkis_cg_manager_label_resource where label_id in - (select label_id from linkis_cg_manager_label_service_instance where service_instance=#{instance}) + DELETE FROM linkis_cg_manager_label_resource + WHERE label_id IN ( + SELECT label_id + FROM linkis_cg_manager_label_service_instance + WHERE service_instance = #{instance} + ) - delete from linkis_cg_manager_linkis_resources where id in - (select resource_id from linkis_cg_manager_label_resource A join linkis_cg_manager_label_service_instance B on A.label_id=B.label_id and B.service_instance = #{instance} ) + DELETE FROM linkis_cg_manager_linkis_resources + WHERE id IN ( + SELECT resource_id + FROM linkis_cg_manager_label_resource A + JOIN linkis_cg_manager_label_service_instance B + ON A.label_id = B.label_id + AND B.service_instance = #{instance} + ) - select * from linkis_cg_manager_linkis_resources where resourceType = #{resourceType} and - id in (select resource_id from linkis_cg_manager_label_resource A join linkis_cg_manager_label_service_instance B on A.label_id = B.label_id and B.service_instance=#{instance}) + SELECT * + FROM linkis_cg_manager_linkis_resources + WHERE resourceType = #{resourceType} + AND id IN ( + SELECT resource_id + FROM linkis_cg_manager_label_resource A + JOIN linkis_cg_manager_label_service_instance B + ON A.label_id = B.label_id + AND B.service_instance = #{instance} + ) \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/CgManagerLabelMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/CgManagerLabelMapper.xml index 3d7aceecc8c..7e8bfebd348 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/CgManagerLabelMapper.xml +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/CgManagerLabelMapper.xml @@ -19,8 +19,8 @@ diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/ConfigurationConfigKeyMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/ConfigurationConfigKeyMapper.xml index 4c4ae8e8bb3..f88e58de3be 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/ConfigurationConfigKeyMapper.xml +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/ConfigurationConfigKeyMapper.xml @@ -21,21 +21,11 @@ diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/ConfigurationConfigValueMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/ConfigurationConfigValueMapper.xml index 9678d4f15d4..526b4b5b61c 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/ConfigurationConfigValueMapper.xml +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/ConfigurationConfigValueMapper.xml @@ -20,13 +20,12 @@ - update linkis_ps_configuration_config_value - set config_value = #{configValue.configValue} - where config_key_id = #{configValue.configKeyId} + UPDATE linkis_ps_configuration_config_value + SET config_value = #{configValue.configValue} + WHERE config_key_id = #{configValue.configKeyId} - delete - from linkis_ps_configuration_config_value - where config_key_id = #{keyId} + DELETE FROM linkis_ps_configuration_config_value + WHERE config_key_id = #{keyId} diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/ConfigurationKeyEngineRelationMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/ConfigurationKeyEngineRelationMapper.xml index c4ef06107c6..e0b67a647ea 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/ConfigurationKeyEngineRelationMapper.xml +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/ConfigurationKeyEngineRelationMapper.xml @@ -19,8 +19,7 @@ - delete - from linkis_ps_configuration_key_engine_relation - where config_key_id = #{keyId} + DELETE FROM linkis_ps_configuration_key_engine_relation + WHERE config_key_id = #{keyId} diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/DatasourceAccessMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/DatasourceAccessMapper.xml index 0faa890b55e..fa62a5f0dc0 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/DatasourceAccessMapper.xml +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/DatasourceAccessMapper.xml @@ -38,12 +38,12 @@ diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/DatasourceEnvMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/DatasourceEnvMapper.xml index b7fcc14bb71..a3a6a27488a 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/DatasourceEnvMapper.xml +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/DatasourceEnvMapper.xml @@ -39,13 +39,13 @@ diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/DatasourceTypeKeyMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/DatasourceTypeKeyMapper.xml index fb17885ece1..f93f0b53045 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/DatasourceTypeKeyMapper.xml +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/DatasourceTypeKeyMapper.xml @@ -50,10 +50,10 @@ - select + SELECT - + diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/EngineConnPluginBmlResourcesMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/EngineConnPluginBmlResourcesMapper.xml index ef0e3590e69..4af20433e76 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/EngineConnPluginBmlResourcesMapper.xml +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/EngineConnPluginBmlResourcesMapper.xml @@ -42,7 +42,9 @@ diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/GatewayAuthTokenMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/GatewayAuthTokenMapper.xml index ea0ebcdc926..d187af707ea 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/GatewayAuthTokenMapper.xml +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/GatewayAuthTokenMapper.xml @@ -39,13 +39,13 @@ diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/PsErrorCodeMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/PsErrorCodeMapper.xml index 047a9408a6f..d59b0d8b041 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/PsErrorCodeMapper.xml +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/PsErrorCodeMapper.xml @@ -34,14 +34,14 @@ diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/RmExternalResourceProviderMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/RmExternalResourceProviderMapper.xml index 70f8f493ab9..78f62848a3f 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/RmExternalResourceProviderMapper.xml +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/RmExternalResourceProviderMapper.xml @@ -34,14 +34,14 @@ diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/UdfManagerMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/UdfManagerMapper.xml index 84127f13e26..b0d31fe0710 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/UdfManagerMapper.xml +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/UdfManagerMapper.xml @@ -30,11 +30,11 @@ diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/UdfTreeMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/UdfTreeMapper.xml index 4012018fb72..7f95dd151d0 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/UdfTreeMapper.xml +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/UdfTreeMapper.xml @@ -38,14 +38,14 @@ diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/BmlProjectMapper.xml b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/BmlProjectMapper.xml index 83f8472a0dd..47a894bf930 100644 --- a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/BmlProjectMapper.xml +++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/BmlProjectMapper.xml @@ -20,43 +20,59 @@ - insert ignore into linkis_ps_bml_project(name, `system`, source, description, creator, enabled, create_time) - values(#{bmlProject.name}, #{bmlProject.system}, #{bmlProject.source}, #{bmlProject.description}, - #{bmlProject.creator}, #{bmlProject.enabled}, #{bmlProject.createTime}) + INSERT IGNORE INTO linkis_ps_bml_project (name, `system`, source, description, creator + , enabled, create_time) + VALUES (#{bmlProject.name}, #{bmlProject.system}, #{bmlProject.source}, #{bmlProject.description}, #{bmlProject.creator} + , #{bmlProject.enabled}, #{bmlProject.createTime}) - insert ignore into linkis_ps_bml_project_user(project_id, username, priv, creator, create_time) values + INSERT IGNORE INTO linkis_ps_bml_project_user(project_id, username, priv, creator, create_time) VALUES #{projectId}, #{username}, #{priv}, #{creator}, #{createTime} - insert ignore into linkis_ps_bml_project_resource(project_id, resource_id) - values(#{projectId}, #{resourceId}) + INSERT IGNORE INTO linkis_ps_bml_project_resource (project_id, resource_id) + VALUES (#{projectId}, #{resourceId}) - insert ignore into linkis_ps_bml_project_resource(project_id, resource_id) - values(#{projectId}, #{resourceId}) + INSERT IGNORE INTO linkis_ps_bml_project_resource (project_id, resource_id) + VALUES (#{projectId}, #{resourceId}) - delete from linkis_ps_bml_project_user where project_id = #{projectId} + DELETE FROM linkis_ps_bml_project_user + WHERE project_id = #{projectId} diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/DownloadMapper.xml b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/DownloadMapper.xml index 83c6196744d..61c40092fe5 100644 --- a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/DownloadMapper.xml +++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/DownloadMapper.xml @@ -27,8 +27,10 @@ - insert into `linkis_ps_resources_download_history` ( `start_time`,`end_time`,`client_ip`,`state`,`resource_id`,`version`,`downloader`) - values(#{downloadModel.startTime}, #{downloadModel.endTime}, #{downloadModel.clientIp}, #{downloadModel.state}, #{downloadModel.resourceId}, #{downloadModel.version}, #{downloadModel.downloader}) + INSERT INTO `linkis_ps_resources_download_history` (`start_time`, `end_time`, `client_ip`, `state`, `resource_id` + , `version`, `downloader`) + VALUES (#{downloadModel.startTime}, #{downloadModel.endTime}, #{downloadModel.clientIp}, #{downloadModel.state}, #{downloadModel.resourceId} + , #{downloadModel.version}, #{downloadModel.downloader}) diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/ResourceMapper.xml b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/ResourceMapper.xml index 0234e49b8f7..410d039b2ee 100644 --- a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/ResourceMapper.xml +++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/ResourceMapper.xml @@ -45,21 +45,25 @@ @@ -67,7 +71,7 @@ UPDATE linkis_ps_bml_resources SET enable_flag = 0 - WHERE resource_id = #{resourceId} and enable_flag = 1 + WHERE resource_id = #{resourceId} AND enable_flag = 1 @@ -93,15 +97,25 @@ - update `linkis_ps_bml_resources` set owner = #{newOwner} where resource_id = #{resourceId} and owner=#{oldOwner} + UPDATE `linkis_ps_bml_resources` + SET owner = #{newOwner} + WHERE resource_id = #{resourceId} + AND owner = #{oldOwner} diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/VersionMapper.xml b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/VersionMapper.xml index 0600bd4331b..1fae82ce423 100644 --- a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/VersionMapper.xml +++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/VersionMapper.xml @@ -39,21 +39,25 @@ - select resource - from linkis_ps_bml_resources_version - WHERE resource_id = #{resourceId} limit 1 + SELECT resource + FROM linkis_ps_bml_resources_version + WHERE resource_id = #{resourceId} + LIMIT 1 - update linkis_ps_bml_resources_version - set enable_flag = 0 - where enable_flag = 1 and resource_id in + UPDATE linkis_ps_bml_resources_version + SET enable_flag = 0 + WHERE enable_flag = 1 + AND resource_id IN #{resourceId} @@ -160,35 +181,63 @@ - update `linkis_ps_bml_resources_version` set enable_flag = 0 where resource_id = #{resourceId} + UPDATE `linkis_ps_bml_resources_version` + SET enable_flag = 0 + WHERE resource_id = #{resourceId} diff --git a/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/main/resources/mapper/common/contextHistoryMapper.xml b/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/main/resources/mapper/common/contextHistoryMapper.xml index b29902b87ff..bb943e117bd 100644 --- a/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/main/resources/mapper/common/contextHistoryMapper.xml +++ b/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/main/resources/mapper/common/contextHistoryMapper.xml @@ -32,21 +32,31 @@ - DELETE FROM linkis_ps_cs_context_history WHERE context_id = #{contextID.contextId} AND source = #{source} + DELETE FROM linkis_ps_cs_context_history + WHERE context_id = #{contextID.contextId} + AND source = #{source} diff --git a/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/main/resources/mapper/common/contextIDListenerMapper.xml b/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/main/resources/mapper/common/contextIDListenerMapper.xml index 33b8781e938..9da399535e4 100644 --- a/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/main/resources/mapper/common/contextIDListenerMapper.xml +++ b/linkis-public-enhancements/linkis-context-service/linkis-cs-server/src/main/resources/mapper/common/contextIDListenerMapper.xml @@ -31,15 +31,20 @@ - DELETE FROM linkis_ps_cs_context_listener where context_id = #{listener.contextId} AND listener_source = #{listener.source} + DELETE FROM linkis_ps_cs_context_listener + WHERE context_id = #{listener.contextId} + AND listener_source = #{listener.source} - DELETE FROM linkis_ps_cs_context_listener where context_id = #{contextID.contextId} + DELETE FROM linkis_ps_cs_context_listener + WHERE context_id = #{contextID.contextId} diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/resources/mapper/common/MdqDao.xml b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/resources/mapper/common/MdqDao.xml index 07caadefd35..cfd75e1d026 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/resources/mapper/common/MdqDao.xml +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/resources/mapper/common/MdqDao.xml @@ -40,16 +40,23 @@ - update linkis_ps_datasource_table set is_available = true where id = #{tableId} + UPDATE linkis_ps_datasource_table + SET is_available = true + WHERE id = #{tableId} diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml index 8ac85a7c46e..7c2f24d9e1e 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml @@ -219,6 +219,8 @@ - update linkis_ps_job_history_group_history set observe_info = #{observeInfo} where id = #{taskid} + UPDATE linkis_ps_job_history_group_history + SET observe_info = #{observeInfo} + WHERE id = #{taskid} diff --git a/linkis-public-enhancements/linkis-ps-common-lock/src/main/resources/mapper/common/CommonLockMapper.xml b/linkis-public-enhancements/linkis-ps-common-lock/src/main/resources/mapper/common/CommonLockMapper.xml index dddb5ebfd7a..46bfdef9607 100644 --- a/linkis-public-enhancements/linkis-ps-common-lock/src/main/resources/mapper/common/CommonLockMapper.xml +++ b/linkis-public-enhancements/linkis-ps-common-lock/src/main/resources/mapper/common/CommonLockMapper.xml @@ -21,15 +21,17 @@ - insert into linkis_ps_common_lock (lock_object, time_out, update_time, create_time) - values(#{jsonObject}, #{timeOut}, now(), now()) + INSERT INTO linkis_ps_common_lock (lock_object, time_out, update_time, create_time) + VALUES (#{jsonObject}, #{timeOut}, now(), now()) - delete from linkis_ps_common_lock where lock_object = #{jsonObject} + DELETE FROM linkis_ps_common_lock + WHERE lock_object = #{jsonObject} diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml index 17133345b60..898ef64c0de 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml @@ -86,19 +86,19 @@ DELETE FROM linkis_ps_udf_baseinfo - where id = #{arg0} and create_user = #{arg1} + WHERE id = #{arg0} and create_user = #{arg1} DELETE FROM linkis_ps_udf_user_load - where udf_id = #{arg0} and user_name = #{arg1} + where udf_id = #{arg0} AND user_name = #{arg1} @@ -106,131 +106,150 @@ SELECT user_name - from linkis_ps_udf_manager - where user_name=#{userName} + FROM linkis_ps_udf_manager + WHERE user_name = #{userName} - INSERT into linkis_ps_udf_shared_info(udf_id,user_name) - VALUES (#{udfId},#{shareUserName}) + INSERT INTO linkis_ps_udf_shared_info (udf_id, user_name) + VALUES (#{udfId}, #{shareUserName}) UPDATE linkis_ps_udf_baseinfo SET is_shared=#{isShared} - where id=#{id} + WHERE id=#{id} - UPDATE linkis_ps_udf_baseinfo set is_expire = true where id=#{udfId} + UPDATE linkis_ps_udf_baseinfo + SET is_expire = true + WHERE id = #{udfId} - insert into linkis_ps_udf_shared_info(udf_id,user_name) + INSERT INTO linkis_ps_udf_shared_info(udf_id,user_name) VALUES (#{udfId},#{addSharedUser}) - DELETE from linkis_ps_udf_shared_info where udf_id =#{udfId} + DELETE FROM linkis_ps_udf_shared_info + WHERE udf_id = #{udfId} - DELETE from linkis_ps_udf_shared_info where udf_id =#{udfId} and user_name=#{removeSharedUser} + DELETE FROM linkis_ps_udf_shared_info + WHERE udf_id = #{udfId} + AND user_name = #{removeSharedUser} - update linkis_ps_udf_user_load set user_name=#{newUser} where udf_id=#{udfId} and user_name=#{oldUser} + UPDATE linkis_ps_udf_user_load + SET user_name = #{newUser} + WHERE udf_id = #{udfId} + AND user_name = #{oldUser} \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFTreeDao.xml b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFTreeDao.xml index 37aa1c260c3..4bbbce371f2 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFTreeDao.xml +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFTreeDao.xml @@ -56,28 +56,28 @@ DELETE FROM linkis_ps_udf_tree - where id = #{arg0} and user_name=#{arg1} + WHERE id = #{arg0} and user_name=#{arg1} \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFVersionDao.xml b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFVersionDao.xml index 71f1ad20307..51a46c90858 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFVersionDao.xml +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFVersionDao.xml @@ -47,36 +47,43 @@ - update linkis_ps_udf_version set is_published=#{isPublished} where udf_id=#{udfId} and - bml_resource_version=#{version} + UPDATE linkis_ps_udf_version + SET is_published = #{isPublished} + WHERE udf_id = #{udfId} + AND bml_resource_version = #{version} - delete from linkis_ps_udf_version where udf_id=#{udfId} + DELETE FROM linkis_ps_udf_version + WHERE udf_id = #{udfId} - update linkis_ps_udf_version set bml_resource_id=#{resourceId},path=replace(`path`,#{oldUser},#{newUser}) - where udf_id=#{udfId} + UPDATE linkis_ps_udf_version + SET bml_resource_id = #{resourceId}, path = replace(`path`, #{oldUser}, #{newUser}) + WHERE udf_id = #{udfId} @@ -106,9 +118,13 @@ \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-variable/src/main/resources/mapper/common/VarMapper.xml b/linkis-public-enhancements/linkis-variable/src/main/resources/mapper/common/VarMapper.xml index 04bec42968c..1b3ba43f077 100644 --- a/linkis-public-enhancements/linkis-variable/src/main/resources/mapper/common/VarMapper.xml +++ b/linkis-public-enhancements/linkis-variable/src/main/resources/mapper/common/VarMapper.xml @@ -34,7 +34,9 @@ @@ -54,7 +56,9 @@ - update `linkis_ps_variable_key_user` set value = #{value} where id = #{valueID} + UPDATE `linkis_ps_variable_key_user` + SET value = #{value} + WHERE id = #{valueID} \ No newline at end of file diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/resources/mapper/common/TokenMapper.xml b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/resources/mapper/common/TokenMapper.xml index 155c548d611..7242a5ee4a9 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/resources/mapper/common/TokenMapper.xml +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/resources/mapper/common/TokenMapper.xml @@ -37,7 +37,7 @@ linkis_mg_gateway_auth_token From 0b0ef7917eeccc2707754b88eb74421c1d3e3913 Mon Sep 17 00:00:00 2001 From: guoshupei <15764973965@163.com> Date: Sun, 5 Mar 2023 18:44:38 +0800 Subject: [PATCH 042/689] rename config key --- .../org/apache/linkis/server/conf/ServerConfiguration.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala index 8d9f9d65adf..3c6a25a343e 100644 --- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala +++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala @@ -208,6 +208,6 @@ object ServerConfiguration extends Logging { CommonVars("wds.linkis.session.proxy.user.ticket.key", "linkis_user_session_proxy_ticket_id_v1") val LINKIS_SERVER_ENTRANCE_HEADER_KEY = - CommonVars("wds.linkis.server.entrance.header.key", "jobInstanceKey") + CommonVars("linkis.server.entrance.header.key", "jobInstanceKey") } From 617dcf8dccf926237aa172680dd8afd514fa4f03 Mon Sep 17 00:00:00 2001 From: binbinCheng Date: Mon, 6 Mar 2023 11:04:23 +0800 Subject: [PATCH 043/689] Beautify the sql style, clear and clear --- .../common/EngineConnBmlResourceMapper.xml | 4 +- .../EngineConnPluginBmlResourcesMapper.xml | 2 +- .../resources/mapper/common/VersionMapper.xml | 4 +- .../common/DataSourceParamKeyMapper.xml | 4 +- .../mapper/common/DataSourceVersionMapper.xml | 4 +- .../resources/mapper/common/HiveMetaDao.xml | 257 +++++++++++------- .../mapper/common/InsLabelRelationMapper.xml | 2 +- .../mapper/common/InstanceInfoMapper.xml | 4 +- .../mapper/common/InstanceLabelMapper.xml | 2 +- .../main/resources/mapper/common/UDFDao.xml | 12 +- .../resources/mapper/common/TokenMapper.xml | 2 +- 11 files changed, 183 insertions(+), 114 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/resources/mapper/common/EngineConnBmlResourceMapper.xml b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/resources/mapper/common/EngineConnBmlResourceMapper.xml index c922ff549f1..61792b70540 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/resources/mapper/common/EngineConnBmlResourceMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/resources/mapper/common/EngineConnBmlResourceMapper.xml @@ -31,11 +31,11 @@ WHERE engine_conn_type=#{engineConnType} and `version`=#{version} - SELECT version FROM linkis_cg_engine_conn_plugin_bml_resources where engine_conn_type = #{type} GROUP BY version - SELECT engine_conn_type FROM linkis_cg_engine_conn_plugin_bml_resources GROUP BY engine_conn_type diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/EngineConnPluginBmlResourcesMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/EngineConnPluginBmlResourcesMapper.xml index 4af20433e76..3e0f8b6f88a 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/EngineConnPluginBmlResourcesMapper.xml +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/EngineConnPluginBmlResourcesMapper.xml @@ -41,7 +41,7 @@ last_update_time - SELECT concat(engine_conn_type, '-', substring(version, 2)) FROM linkis_cg_engine_conn_plugin_bml_resources GROUP BY engine_conn_type, version diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/VersionMapper.xml b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/VersionMapper.xml index 1fae82ce423..9ae9b72df82 100644 --- a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/VersionMapper.xml +++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/resources/mapper/common/VersionMapper.xml @@ -188,14 +188,14 @@ AND enable_flag = 1 - SELECT start_byte FROM linkis_ps_bml_resources_version WHERE resource_id = #{resourceId} AND version = #{version} - SELECT end_byte FROM linkis_ps_bml_resources_version WHERE resource_id = #{resourceId} diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/resources/mapper/common/DataSourceParamKeyMapper.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/resources/mapper/common/DataSourceParamKeyMapper.xml index 6f6c89314c2..45af3d9b117 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/resources/mapper/common/DataSourceParamKeyMapper.xml +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/resources/mapper/common/DataSourceParamKeyMapper.xml @@ -41,12 +41,12 @@ `id`, `key`, `description_en` as `description`, `name_en` as `name`, `require`, `scope`, `default_value`, `value_type`, `value_regex`, `ref_id`, `ref_value`, `data_source` - SELECT FROM `linkis_ps_dm_datasource_type_key` WHERE `data_source_type_id` = #{dataSourceTypeId}; - SELECT FROM `linkis_ps_dm_datasource_type_key` WHERE `data_source_type_id` = #{dataSourceTypeId}; diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/resources/mapper/common/DataSourceVersionMapper.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/resources/mapper/common/DataSourceVersionMapper.xml index b263f80cea9..19f361f250c 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/resources/mapper/common/DataSourceVersionMapper.xml +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/resources/mapper/common/DataSourceVersionMapper.xml @@ -31,7 +31,7 @@ `version_id`, `datasource_id`, `parameter`, `comment`, `create_time`, `create_user` - SELECT ifnull(max(version_id),0 ) FROM `linkis_ps_dm_datasource_version` WHERE datasource_id = #{dataSourceId}; @@ -50,7 +50,7 @@ #{createUser}) - SELECT `parameter` FROM `linkis_ps_dm_datasource_version` WHERE datasource_id = #{dataSourceId} and `version_id` = #{version, jdbcType=INTEGER}; diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/resources/mapper/common/HiveMetaDao.xml b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/resources/mapper/common/HiveMetaDao.xml index d4f13c3c689..ec0adef35c2 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/resources/mapper/common/HiveMetaDao.xml +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/resources/mapper/common/HiveMetaDao.xml @@ -20,49 +20,60 @@ - + SELECT LOCATION + FROM SDS + WHERE SD_ID IN ( + SELECT SD_ID + FROM TBLS + WHERE TBL_NAME = #{tableName,jdbcType=VARCHAR} + AND DB_ID IN ( + SELECT DB_ID + FROM `DBS` + WHERE NAME = #{dbName,jdbcType=VARCHAR} + ) ) - SELECT ROLE_NAME - FROM ROLES r INNER JOIN ROLE_MAP rm - ON r.ROLE_ID = rm.ROLE_ID - and rm.PRINCIPAL_TYPE = 'USER' - and rm.PRINCIPAL_NAME = #{userName,jdbcType=VARCHAR} + FROM ROLES r + INNER JOIN ROLE_MAP rm + ON r.ROLE_ID = rm.ROLE_ID + AND rm.PRINCIPAL_TYPE = 'USER' + AND rm.PRINCIPAL_NAME = #{userName,jdbcType=VARCHAR} - + SELECT t2.TBL_NAME AS NAME, t2.TBL_TYPE AS TYPE, t2.CREATE_TIME AS CREATE_TIME, t2.LAST_ACCESS_TIME AS LAST_ACCESS_TIME, t2.OWNER AS OWNER + FROM DB_PRIVS t1 + INNER JOIN TBLS t2 + ON t1.DB_ID = t2.DB_ID + AND t1.DB_PRIV IN ('SELECT', 'ALL') + INNER JOIN DBS t3 + ON t1.DB_ID = t3.DB_ID + AND t3.NAME = #{dbName,jdbcType=VARCHAR} + WHERE t1.PRINCIPAL_TYPE = 'USER' + AND t1.PRINCIPAL_NAME = #{userName,jdbcType=VARCHAR} OR (t1.PRINCIPAL_TYPE = 'ROLE' AND t1.PRINCIPAL_NAME IN #{id} ) - union - select t2.TBL_NAME as NAME, t2.TBL_TYPE as TYPE, t2.CREATE_TIME as CREATE_TIME, t2.LAST_ACCESS_TIME as LAST_ACCESS_TIME, t2.OWNER as OWNER - from TBL_PRIVS t1 - inner join TBLS t2 on t1.TBL_ID=t2.TBL_ID and t1.TBL_PRIV in ('SELECT','ALL') - inner join DBS t3 on t2.DB_ID=t3.DB_ID and t3.NAME = #{dbName,jdbcType=VARCHAR} - where (t1.PRINCIPAL_TYPE = 'USER' and t1.PRINCIPAL_NAME = #{userName,jdbcType=VARCHAR}) + UNION + SELECT t2.TBL_NAME AS NAME, t2.TBL_TYPE AS TYPE, t2.CREATE_TIME AS CREATE_TIME, t2.LAST_ACCESS_TIME AS LAST_ACCESS_TIME, t2.OWNER AS OWNER + FROM TBL_PRIVS t1 + INNER JOIN TBLS t2 + ON t1.TBL_ID = t2.TBL_ID + AND t1.TBL_PRIV IN ('SELECT', 'ALL') + INNER JOIN DBS t3 + ON t2.DB_ID = t3.DB_ID + AND t3.NAME = #{dbName,jdbcType=VARCHAR} + WHERE t1.PRINCIPAL_TYPE = 'USER' + AND t1.PRINCIPAL_NAME = #{userName,jdbcType=VARCHAR} OR (t1.PRINCIPAL_TYPE = 'ROLE' AND t1.PRINCIPAL_NAME IN #{id} ) - order by NAME; + ORDER BY NAME; - + SELECT t2.TBL_NAME AS NAME, t2.TBL_TYPE AS TYPE, t2.CREATE_TIME AS CREATE_TIME, t2.LAST_ACCESS_TIME AS LAST_ACCESS_TIME, t2.OWNER AS OWNER + FROM TBLS t2 + INNER JOIN DBS t3 ON t2.DB_ID = t3.DB_ID + WHERE t3.NAME = #{dbName,jdbcType=VARCHAR} + ORDER BY t2.TBL_NAME; - + SELECT PARAM_VALUE + FROM PARTITION_PARAMS + WHERE PARAM_KEY = 'totalSize' + AND PART_ID IN ( + SELECT PART_ID + FROM PARTITIONS + WHERE PART_NAME = #{partitionName,jdbcType=VARCHAR} + AND TBL_ID IN ( + SELECT TBL_ID + FROM `TBLS` + WHERE TBL_NAME = #{tableName,jdbcType=VARCHAR} + AND DB_ID IN ( + SELECT DB_ID + FROM `DBS` + WHERE NAME = #{dbName,jdbcType=VARCHAR} + ) + ) ); - + SELECT PART_NAME + FROM PARTITIONS + WHERE TBL_ID IN ( + SELECT TBL_ID + FROM `TBLS` + WHERE TBL_NAME = #{tableName,jdbcType=VARCHAR} + AND DB_ID IN ( + SELECT DB_ID + FROM `DBS` + WHERE NAME = #{dbName,jdbcType=VARCHAR} + ) ); - + SELECT COMMENT, COLUMN_NAME, TYPE_NAME + FROM COLUMNS_V2 + WHERE CD_ID IN ( + SELECT CD_ID + FROM SDS + WHERE SD_ID IN ( + SELECT SD_ID + FROM `TBLS` + WHERE TBL_NAME = #{tableName,jdbcType=VARCHAR} + AND DB_ID IN ( + SELECT DB_ID + FROM `DBS` + WHERE NAME = #{dbName,jdbcType=VARCHAR} + ) + ) + ) + ORDER BY INTEGER_IDX ASC; - + SELECT t2.TBL_NAME AS NAME, t2.TBL_TYPE AS TYPE, t2.CREATE_TIME AS CREATE_TIME, t2.LAST_ACCESS_TIME AS LAST_ACCESS_TIME, t2.OWNER AS OWNER + , t2.SD_ID AS SD_ID + FROM DB_PRIVS t1 + INNER JOIN TBLS t2 + ON t1.DB_ID = t2.DB_ID + AND t1.DB_PRIV IN ('SELECT', 'ALL') + AND t2.TBL_NAME = #{tableName,jdbcType=VARCHAR} + INNER JOIN DBS t3 + ON t1.DB_ID = t3.DB_ID + AND t3.NAME = #{dbName,jdbcType=VARCHAR} + WHERE t1.PRINCIPAL_TYPE = 'USER' + AND t1.PRINCIPAL_NAME = #{userName,jdbcType=VARCHAR} OR (t1.PRINCIPAL_TYPE = 'ROLE' AND t1.PRINCIPAL_NAME IN #{id} ) - union - select t2.TBL_NAME as NAME, t2.TBL_TYPE as TYPE, t2.CREATE_TIME as CREATE_TIME, t2.LAST_ACCESS_TIME as LAST_ACCESS_TIME, t2.OWNER as OWNER, t2.SD_ID as SD_ID - from TBL_PRIVS t1 - inner join TBLS t2 on t1.TBL_ID = t2.TBL_ID and t1.TBL_PRIV in ('SELECT','ALL') and t2.TBL_NAME = #{tableName,jdbcType=VARCHAR} - inner join DBS t3 on t2.DB_ID=t3.DB_ID and t3.NAME = #{dbName,jdbcType=VARCHAR} - where (t1.PRINCIPAL_TYPE = 'USER' and t1.PRINCIPAL_NAME = #{userName,jdbcType=VARCHAR}) + UNION + SELECT t2.TBL_NAME AS NAME, t2.TBL_TYPE AS TYPE, t2.CREATE_TIME AS CREATE_TIME, t2.LAST_ACCESS_TIME AS LAST_ACCESS_TIME, t2.OWNER AS OWNER + , t2.SD_ID AS SD_ID + FROM TBL_PRIVS t1 + INNER JOIN TBLS t2 + ON t1.TBL_ID = t2.TBL_ID + AND t1.TBL_PRIV IN ('SELECT', 'ALL') + AND t2.TBL_NAME = #{tableName,jdbcType=VARCHAR} + INNER JOIN DBS t3 + ON t2.DB_ID = t3.DB_ID + AND t3.NAME = #{dbName,jdbcType=VARCHAR} + WHERE t1.PRINCIPAL_TYPE = 'USER' + AND t1.PRINCIPAL_NAME = #{userName,jdbcType=VARCHAR} OR (t1.PRINCIPAL_TYPE = 'ROLE' AND t1.PRINCIPAL_NAME IN #{id} ) - order by NAME; + ORDER BY NAME; - + SELECT COMMENT, COLUMN_NAME, TYPE_NAME + FROM COLUMNS_V2 + WHERE CD_ID IN ( + SELECT CD_ID + FROM SDS + WHERE SD_ID = #{sdId,jdbcType=VARCHAR} + ) + ORDER BY INTEGER_IDX ASC; - + SELECT PKEY_COMMENT, PKEY_NAME, PKEY_TYPE + FROM PARTITION_KEYS + WHERE TBL_ID IN ( + SELECT TBL_ID + FROM `TBLS` + WHERE TBL_NAME = #{tableName,jdbcType=VARCHAR} + AND DB_ID IN ( + SELECT DB_ID + FROM `DBS` + WHERE NAME = #{dbName,jdbcType=VARCHAR} + ) ); diff --git a/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InsLabelRelationMapper.xml b/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InsLabelRelationMapper.xml index 2a5d8a3d457..71ef9a63677 100644 --- a/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InsLabelRelationMapper.xml +++ b/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InsLabelRelationMapper.xml @@ -153,7 +153,7 @@ - diff --git a/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InstanceInfoMapper.xml b/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InstanceInfoMapper.xml index d09d9dc033f..af0970da3d2 100644 --- a/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InstanceInfoMapper.xml +++ b/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InstanceInfoMapper.xml @@ -20,7 +20,7 @@ - + @@ -28,7 +28,7 @@ - + diff --git a/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InstanceLabelMapper.xml b/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InstanceLabelMapper.xml index eb3c5b5616a..57b627d54c5 100644 --- a/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InstanceLabelMapper.xml +++ b/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InstanceLabelMapper.xml @@ -39,7 +39,7 @@ `label_key`, `label_value`, `label_feature`, `label_value_size` - diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml index 898ef64c0de..8b98eb81679 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml @@ -261,12 +261,12 @@ WHERE user_name = #{userName} - SELECT DISTINCT create_user FROM linkis_ps_udf_baseinfo - SELECT COUNT(1) FROM linkis_ps_udf_shared_info busu JOIN linkis_ps_udf_baseinfo bu ON busu.udf_id = bu.id @@ -274,7 +274,7 @@ AND busu.user_name = #{userName} - SELECT COUNT(1) FROM linkis_ps_udf_shared_info busu JOIN linkis_ps_udf_baseinfo bu ON busu.udf_id = bu.id @@ -294,7 +294,7 @@ WHERE id=#{id} - SELECT id FROM linkis_ps_udf_baseinfo WHERE create_user =#{userName} AND udf_name=#{udfName} @@ -307,14 +307,14 @@ WHERE id = #{udfId} - SELECT busu.user_name FROM linkis_ps_udf_shared_info busu JOIN linkis_ps_udf_baseinfo bu ON busu.udf_id = bu.id WHERE bu.id = #{udfId} - SELECT id FROM linkis_user WHERE username =#{userName} diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/resources/mapper/common/TokenMapper.xml b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/resources/mapper/common/TokenMapper.xml index 7242a5ee4a9..8ae1a8ba9d8 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/resources/mapper/common/TokenMapper.xml +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/resources/mapper/common/TokenMapper.xml @@ -36,7 +36,7 @@ linkis_mg_gateway_auth_token - SELECT * FROM From 557f4fe57d63e2f48f687b108df353c6b513a49f Mon Sep 17 00:00:00 2001 From: binbinCheng Date: Mon, 6 Mar 2023 11:43:33 +0800 Subject: [PATCH 044/689] update LabelMapper.xml --- .../src/main/resources/mapper/common/LabelMapper.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/LabelMapper.xml b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/LabelMapper.xml index d4cea8e29ae..cc927852626 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/LabelMapper.xml +++ b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/LabelMapper.xml @@ -39,15 +39,15 @@ From eb55412389c70cd5b9cb8e39f195cea4bf786842 Mon Sep 17 00:00:00 2001 From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com> Date: Mon, 6 Mar 2023 11:43:54 +0800 Subject: [PATCH 045/689] [feat] upgrade hadoop\spark\hive default vertion to 3.x (#4263) * upgrade hive\spark\hadoop to default 3.x --- README.md | 4 +- README_CN.md | 4 +- .../linkis-computation-governance-common.md | 4 +- docs/configuration/linkis-manager-common.md | 2 +- docs/configuration/linkis-udf.md | 2 +- .../linkis-configuration-errorcode.md | 2 +- docs/trino-usage.md | 2 +- .../operator/ujes/UJESConstants.java | 5 +- .../LinkisClientApplicationTest.java | 8 +- .../client/InteractiveJobTest.java | 3 +- ...nConf.scala => GovernanceCommonConf.scala} | 7 +- .../conf/GovernanceCommonConfTest.scala | 4 +- .../parser/CommonEntranceParser.scala | 4 +- .../manager/label/conf/LabelCommonConfig.java | 4 +- .../manager/label/TestLabelBuilder.java | 5 +- .../common/conf/ManagerCommonConf.scala | 6 +- .../mapper/common/ECResourceRecordMapper.xml | 4 +- linkis-dist/bin/checkEnv.sh | 2 +- linkis-dist/bin/install.sh | 4 +- linkis-dist/deploy-config/linkis-env.sh | 6 +- linkis-dist/docker/ldh.Dockerfile | 8 +- .../docker/scripts/prepare-ldh-image.sh | 8 +- linkis-dist/helm/README.md | 14 +- linkis-dist/helm/README_CN.md | 14 +- .../linkis/templates/configmap-init-sql.yaml | 4 +- linkis-dist/helm/charts/linkis/values.yaml | 6 +- linkis-dist/helm/scripts/prepare-for-spark.sh | 8 +- linkis-dist/package/bin/linkis-cli-hive | 2 +- .../package/bin/linkis-cli-spark-submit | 6 +- linkis-dist/package/db/linkis_dml.sql | 14 +- .../db/module/linkis_configuration_dml.sql | 14 +- linkis-dist/pom.xml | 8 +- linkis-engineconn-plugins/spark/pom.xml | 113 ++-------------- .../server/response/EngineLabelResponse.java | 2 +- .../src/test/resources/data.sql | 14 +- .../LinkisConfigurationErrorCodeSummary.java | 2 +- .../api/ConfigurationRestfulApiTest.java | 8 +- .../apps/linkis/module/setting/setting.vue | 2 +- pom.xml | 47 +++---- tool/dependencies/known-dependencies.txt | 121 +++++++++++++++++- 40 files changed, 270 insertions(+), 227 deletions(-) rename linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/{GovernaceCommonConf.scala => GovernanceCommonConf.scala} (89%) diff --git a/README.md b/README.md index 9c864cca557..c8ab09b3cc7 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,8 @@ Since the first release of Linkis in 2019, it has accumulated more than **700** | **Engine Name** | **Suppor Component Version
(Default Dependent Version)** | **Linkis Version Requirements** | **Included in Release Package
By Default** | **Description** | |:---- |:---- |:---- |:---- |:---- | -|Spark|Apache 2.0.0~2.4.7,
CDH >= 5.4.0,
(default Apache Spark 2.4.3)|\>=1.0.3|Yes|Spark EngineConn, supports SQL , Scala, Pyspark and R code| -|Hive|Apache >= 1.0.0,
CDH >= 5.4.0,
(default Apache Hive 2.3.3)|\>=1.0.3|Yes |Hive EngineConn, supports HiveQL code| +|Spark|Apache >= 2.0.0,
CDH >= 5.4.0,
(default Apache Spark 3.2.1)|\>=1.0.3|Yes|Spark EngineConn, supports SQL , Scala, Pyspark and R code| +|Hive|Apache >= 1.0.0,
CDH >= 5.4.0,
(default Apache Hive 3.1.3)|\>=1.0.3|Yes |Hive EngineConn, supports HiveQL code| |Python|Python >= 2.6,
(default Python2*)|\>=1.0.3|Yes |Python EngineConn, supports python code| |Shell|Bash >= 2.0|\>=1.0.3|Yes|Shell EngineConn, supports Bash shell code| |JDBC|MySQL >= 5.0, Hive >=1.2.1,
(default Hive-jdbc 2.3.4)|\>=1.0.3|No|JDBC EngineConn, already supports MySQL and HiveQL, can be extended quickly Support other engines with JDBC Driver package, such as Oracle| diff --git a/README_CN.md b/README_CN.md index 34de3690944..b47497789ef 100644 --- a/README_CN.md +++ b/README_CN.md @@ -82,8 +82,8 @@ Linkis 自 2019 年开源发布以来，已累计积累了 700 多家试验企 | **引擎名** | **支持底层组件版本
(默认依赖版本)** | **Linkis 版本要求** | **是否默认包含在发布包中** | **说明** | |:---- |:---- |:---- |:---- |:---- | -|Spark|Apache 2.0.0~2.4.7,
CDH >= 5.4.0,
（默认 Apache Spark 2.4.3）|\>=1.0.3|是|Spark EngineConn， 支持 SQL, Scala, Pyspark 和 R 代码| -|Hive|Apache >= 1.0.0,
CDH >= 5.4.0,
（默认 Apache Hive 2.3.3）|\>=1.0.3|是|Hive EngineConn， 支持 HiveQL 代码| +|Spark|Apache >= 2.0.0,
CDH >= 5.4.0,
（默认 Apache Spark 3.2.1）|\>=1.0.3|是|Spark EngineConn， 支持 SQL, Scala, Pyspark 和 R 代码| +|Hive|Apache >= 1.0.0,
CDH >= 5.4.0,
（默认 Apache Hive 3.1.3）|\>=1.0.3|是|Hive EngineConn， 支持 HiveQL 代码| |Python|Python >= 2.6,
（默认 Python2*）|\>=1.0.3|是|Python EngineConn， 支持 python 代码| |Shell|Bash >= 2.0|\>=1.0.3|是|Shell EngineConn， 支持 Bash shell 代码| |JDBC|MySQL >= 5.0, Hive >=1.2.1,
(默认 Hive-jdbc 2.3.4)|\>=1.0.3|否|JDBC EngineConn， 已支持 MySQL 和 HiveQL，可快速扩展支持其他有 JDBC Driver 包的引擎, 如 Oracle| diff --git a/docs/configuration/linkis-computation-governance-common.md b/docs/configuration/linkis-computation-governance-common.md index 0fc5900ef32..e0bae1ae310 100644 --- a/docs/configuration/linkis-computation-governance-common.md +++ b/docs/configuration/linkis-computation-governance-common.md @@ -4,8 +4,8 @@ | Module Name (Service Name) | Parameter Name | Default Value | Description | | -------- | -------- | ----- |----- | |linkis-computation-governance-common|wds.linkis.rm| | wds.linkis.rm | -|linkis-computation-governance-common|wds.linkis.spark.engine.version|2.4.3 |spark.engine.version| -|linkis-computation-governance-common|wds.linkis.hive.engine.version| 1.2.1 |hive.engine.version| +|linkis-computation-governance-common|wds.linkis.spark.engine.version|3.2.1 |spark.engine.version| +|linkis-computation-governance-common|wds.linkis.hive.engine.version| 3.1.3 |hive.engine.version| |linkis-computation-governance-common|wds.linkis.python.engine.version|python2 | python.engine.version | |linkis-computation-governance-common|wds.linkis.python.code_parser.enabled| false |python.code_parser.enabled| |linkis-computation-governance-common|wds.linkis.scala.code_parser.enabled| false | scala.code_parser.enabled | diff --git a/docs/configuration/linkis-manager-common.md b/docs/configuration/linkis-manager-common.md index 1ef0475bd17..d84b06ea570 100644 --- a/docs/configuration/linkis-manager-common.md +++ b/docs/configuration/linkis-manager-common.md @@ -4,7 +4,7 @@ | Module Name (Service Name) | Parameter Name | Default Value | Description |Used| | -------- | -------- | ----- |----- | ----- | |linkis-manager-common|wds.linkis.default.engine.type |spark|engine.type| -|linkis-manager-common|wds.linkis.default.engine.version |2.4.3|engine.version| +|linkis-manager-common|wds.linkis.default.engine.version |3.2.1|engine.version| |linkis-manager-common|wds.linkis.manager.admin|hadoop|manager.admin| |linkis-manager-common|wds.linkis.rm.application.name|ResourceManager|rm.application.name| |linkis-manager-common|wds.linkis.rm.wait.event.time.out| 1000 * 60 * 12L |event.time.out| diff --git a/docs/configuration/linkis-udf.md b/docs/configuration/linkis-udf.md index 76a9460cfae..dd8aeed169e 100644 --- a/docs/configuration/linkis-udf.md +++ b/docs/configuration/linkis-udf.md @@ -3,7 +3,7 @@ | Module Name (Service Name) | Parameter Name | Default Value | Description |Used| | -------- | -------- | ----- |----- | ----- | -|linkis-udf|wds.linkis.udf.hive.exec.path |/appcom/Install/DataWorkCloudInstall/linkis-linkis-Udf-0.0.3-SNAPSHOT/lib/hive-exec-1.2.1.jar|udf.hive.exec.path| +|linkis-udf|wds.linkis.udf.hive.exec.path |/appcom/Install/DataWorkCloudInstall/linkis-linkis-Udf-0.0.3-SNAPSHOT/lib/hive-exec-3.1.3.jar|udf.hive.exec.path| |linkis-udf|wds.linkis.udf.tmp.path|/tmp/udf/|udf.tmp.path| |linkis-udf|wds.linkis.udf.share.path|/mnt/bdap/udf/|udf.share.path| |linkis-udf|wds.linkis.udf.share.proxy.user| hadoop|udf.share.proxy.user| diff --git a/docs/errorcode/linkis-configuration-errorcode.md b/docs/errorcode/linkis-configuration-errorcode.md index c261f1852e8..299ac0e60f8 100644 --- a/docs/errorcode/linkis-configuration-errorcode.md +++ b/docs/errorcode/linkis-configuration-errorcode.md @@ -15,7 +15,7 @@ |linkis-configuration |14100|CategoryName cannot be included '-'(类别名称不能包含 '-')|CANNOT_BE_INCLUDED|LinkisConfigurationErrorCodeSummary| |linkis-configuration |14100|Creator is null, cannot be added(创建者为空，无法添加)|CREATOR_IS_NULL_CANNOT_BE_ADDED|LinkisConfigurationErrorCodeSummary| |linkis-configuration |14100|Engine type is null, cannot be added(引擎类型为空，无法添加)|ENGINE_TYPE_IS_NULL|LinkisConfigurationErrorCodeSummary| -|linkis-configuration |14100|The saved engine type parameter is incorrect, please send it in a fixed format, such as spark-2.4.3(保存的引擎类型参数有误，请按照固定格式传送，例如spark-2.4.3)|INCORRECT_FIXED_SUCH|LinkisConfigurationErrorCodeSummary| +|linkis-configuration |14100|The saved engine type parameter is incorrect, please send it in a fixed format, such as spark-3.2.1(保存的引擎类型参数有误，请按照固定格式传送，例如spark-3.2.1)|INCORRECT_FIXED_SUCH|LinkisConfigurationErrorCodeSummary| |linkis-configuration |14100|Incomplete request parameters, please reconfirm(请求参数不完整，请重新确认)|INCOMPLETE_RECONFIRM|LinkisConfigurationErrorCodeSummary| |linkis-configuration |14100|Only admin can modify category(只有管理员才能修改目录)|ONLY_ADMIN_CAN_MODIFY|LinkisConfigurationErrorCodeSummary| |linkis-configuration |14100|The label parameter is empty(标签参数为空)|THE_LABEL_PARAMETER_IS_EMPTY|LinkisConfigurationErrorCodeSummary| diff --git a/docs/trino-usage.md b/docs/trino-usage.md index cfd199f8dba..10b7a835bf7 100644 --- a/docs/trino-usage.md +++ b/docs/trino-usage.md @@ -46,7 +46,7 @@ Linkis1.X是通过标签来进行的，所以需要在我们数据库中插入 ``` linkis_ps_configuration_config_key: 插入引擎的配置参数的key和默认values -linkis_cg_manager_label：插入引擎label如：hive-2.3.3 +linkis_cg_manager_label：插入引擎label如：hive-3.1.3 linkis_ps_configuration_category： 插入引擎的目录关联关系 linkis_ps_configuration_config_value： 插入引擎需要展示的配置 linkis_ps_configuration_key_engine_relation:配置项和引擎的关联关系 diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java index 845949079f5..a9bb5523869 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java @@ -24,7 +24,7 @@ public class UJESConstants { public static final String QUERY_PAGE_SIZE_NAME = "pageSize"; public static final int QUERY_PAGE_SIZE_DEFAULT_VALUE = 100; - public static final Long DRIVER_QUERY_SLEEP_MILLS = 500l; + public static final Long DRIVER_QUERY_SLEEP_MILLS = 500L; public static final Integer DRIVER_REQUEST_MAX_RETRY_TIME = 3; public static final String QUERY_STATUS_NAME = "status"; @@ -40,7 +40,4 @@ public class UJESConstants { public static final Integer IDX_FOR_LOG_TYPE_ALL = 3; // 0: Error 1: WARN 2:INFO 3: ALL public static final int DEFAULT_PAGE_SIZE = 500; - - public static final String DEFAULT_SPARK_ENGINE = "spark-2.4.3"; - public static final String DEFAULT_HIVE_ENGINE = "hive-1.2.1"; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java index 0af22266615..14b0bfee792 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java @@ -85,12 +85,12 @@ public void before() { /* Test different task type */ - // "-engineType", "spark-2.4.3", + // "-engineType", "spark-3.2.1", // "-codeType", "sql", // "-code", "show tables;show tables;show tables", // - // "-engineType", "hive-1.2.1", + // "-engineType", "hive-3.1.3", // "-codeType", "sql", // "-code", "show tables;", @@ -101,11 +101,11 @@ public void before() { "-code", "whoami", - // "-engineType", "spark-2.4.3", + // "-engineType", "spark-3.2.1", // "-codeType", "py", // "-code", "print ('hello')", - // "-engineType", "spark-2.4.3", + // "-engineType", "spark-3.2.1", // "-codeType", "scala", // "-codePath", "src/test/resources/testScala.scala", diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/computation/client/InteractiveJobTest.java b/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/computation/client/InteractiveJobTest.java index 4ee0384076c..843e3d30a12 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/computation/client/InteractiveJobTest.java +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/computation/client/InteractiveJobTest.java @@ -18,6 +18,7 @@ package org.apache.linkis.computation.client; import org.apache.linkis.computation.client.interactive.SubmittableInteractiveJob; +import org.apache.linkis.manager.label.conf.LabelCommonConfig; /** A test class for submit a sql to hive engineConn. */ public class InteractiveJobTest { @@ -29,7 +30,7 @@ public static void main(String[] args) { SubmittableInteractiveJob job = LinkisJobClient.interactive() .builder() - .setEngineType("hive-2.3.3") + .setEngineType("hive-" + LabelCommonConfig.HIVE_ENGINE_VERSION.getValue()) .setRunTypeStr("sql") .setCreator("IDE") .setCode("show tables") diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConf.scala similarity index 89% rename from linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala rename to linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConf.scala index 5fdc9cf7f8f..1181cd7d2ce 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConf.scala @@ -18,14 +18,17 @@ package org.apache.linkis.governance.common.conf import org.apache.linkis.common.conf.{CommonVars, Configuration} +import org.apache.linkis.manager.label.conf.LabelCommonConfig object GovernanceCommonConf { val CONF_FILTER_RM = "wds.linkis.rm" - val SPARK_ENGINE_VERSION = CommonVars("wds.linkis.spark.engine.version", "2.4.3") + val SPARK_ENGINE_VERSION = + CommonVars("wds.linkis.spark.engine.version", LabelCommonConfig.SPARK_ENGINE_VERSION.getValue) - val HIVE_ENGINE_VERSION = CommonVars("wds.linkis.hive.engine.version", "1.2.1") + val HIVE_ENGINE_VERSION = + CommonVars("wds.linkis.hive.engine.version", LabelCommonConfig.HIVE_ENGINE_VERSION.getValue) val PYTHON_ENGINE_VERSION = CommonVars("wds.linkis.python.engine.version", "python2") diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConfTest.scala b/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConfTest.scala index 7988a6c95d9..96b6e9a1c26 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConfTest.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/conf/GovernanceCommonConfTest.scala @@ -42,8 +42,8 @@ class GovernanceCommonConfTest { val errorcodedesclen = GovernanceCommonConf.ERROR_CODE_DESC_LEN Assertions.assertEquals("wds.linkis.rm", conffilterrm) - Assertions.assertEquals("2.4.3", sparkengineversion) - Assertions.assertEquals("1.2.1", hiveengineversion) + Assertions.assertEquals("3.2.1", sparkengineversion) + Assertions.assertEquals("3.1.3", hiveengineversion) Assertions.assertEquals("python2", pythonengineversion) Assertions.assertFalse(pythoncodeparserswitch) Assertions.assertFalse(scalacodeparserswitch) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala index 5108a7bf4c6..f6d20d6d5cd 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala @@ -28,6 +28,7 @@ import org.apache.linkis.manager.label.builder.factory.{ LabelBuilderFactory, LabelBuilderFactoryContext } +import org.apache.linkis.manager.label.conf.LabelCommonConfig import org.apache.linkis.manager.label.constant.LabelKeyConstant import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.engine.{CodeLanguageLabel, UserCreatorLabel} @@ -134,7 +135,8 @@ class CommonEntranceParser(val persistenceManager: PersistenceManager) private def checkEngineTypeLabel(labels: util.Map[String, Label[_]]): Unit = { val engineTypeLabel = labels.getOrDefault(LabelKeyConstant.ENGINE_TYPE_KEY, null) if (null == engineTypeLabel) { - val msg = s"You need to specify engineTypeLabel in labels, such as spark-2.4.3" + val msg = s"You need to specify engineTypeLabel in labels," + + s"such as spark-${LabelCommonConfig.SPARK_ENGINE_VERSION.getValue}" throw new EntranceIllegalParamException( EntranceErrorCode.LABEL_PARAMS_INVALID.getErrCode, EntranceErrorCode.LABEL_PARAMS_INVALID.getDesc + msg diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java index 04805860ceb..d0854186a57 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java @@ -34,10 +34,10 @@ public class LabelCommonConfig { CommonVars.apply("wds.linkis.label.entity.packages", ""); public static final CommonVars SPARK_ENGINE_VERSION = - CommonVars.apply("wds.linkis.spark.engine.version", "2.4.3"); + CommonVars.apply("wds.linkis.spark.engine.version", "3.2.1"); public static final CommonVars HIVE_ENGINE_VERSION = - CommonVars.apply("wds.linkis.hive.engine.version", "2.3.3"); + CommonVars.apply("wds.linkis.hive.engine.version", "3.1.3"); public static final CommonVars PYTHON_ENGINE_VERSION = CommonVars.apply("wds.linkis.python.engine.version", "python2"); diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/TestLabelBuilder.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/TestLabelBuilder.java index 8b6e49570ca..cffa7891d7f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/TestLabelBuilder.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/TestLabelBuilder.java @@ -19,6 +19,7 @@ import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactory; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; +import org.apache.linkis.manager.label.conf.LabelCommonConfig; import org.apache.linkis.manager.label.entity.Label; import org.apache.linkis.manager.label.entity.node.AliasServiceInstanceLabel; import org.apache.linkis.manager.label.exception.LabelErrorException; @@ -27,7 +28,9 @@ public class TestLabelBuilder { public static void main(String[] args) throws LabelErrorException { LabelBuilderFactory labelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory(); - Label engineType = labelBuilderFactory.createLabel("engineType", "hive-1.2.1"); + Label engineType = + labelBuilderFactory.createLabel( + "engineType", "hive-" + LabelCommonConfig.HIVE_ENGINE_VERSION.getValue()); System.out.println(engineType.getFeature()); AliasServiceInstanceLabel emInstanceLabel = diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/conf/ManagerCommonConf.scala b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/conf/ManagerCommonConf.scala index c37d6700f3a..81f7294ba05 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/conf/ManagerCommonConf.scala +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/conf/ManagerCommonConf.scala @@ -18,12 +18,16 @@ package org.apache.linkis.manager.common.conf import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.manager.label.conf.LabelCommonConfig object ManagerCommonConf { val DEFAULT_ENGINE_TYPE = CommonVars("wds.linkis.default.engine.type", "spark") - val DEFAULT_ENGINE_VERSION = CommonVars("wds.linkis.default.engine.version", "2.4.3") + val DEFAULT_ENGINE_VERSION = CommonVars( + "wds.linkis.default.engine.version", + LabelCommonConfig.SPARK_ENGINE_VERSION.defaultValue + ) val DEFAULT_ADMIN = CommonVars("wds.linkis.manager.admin", "hadoop") diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/ECResourceRecordMapper.xml b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/ECResourceRecordMapper.xml index 543d20234ab..ad2c710f0c0 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/ECResourceRecordMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/ECResourceRecordMapper.xml @@ -71,7 +71,7 @@ service_instance = #{instance} and create_user = #{username} - + and label_value like concat('%,',#{engineType},'%') and create_time BETWEEN #{startDate} AND #{endDate} @@ -93,7 +93,7 @@ - + and SUBSTRING_INDEX(SUBSTRING_INDEX(ecr.label_value,',',-1),"-",1) in #{i} diff --git a/linkis-dist/bin/checkEnv.sh b/linkis-dist/bin/checkEnv.sh index 0af9b12d671..82b434e5206 100644 --- a/linkis-dist/bin/checkEnv.sh +++ b/linkis-dist/bin/checkEnv.sh @@ -37,7 +37,7 @@ function checkPythonAndJava(){ function checkHdfs(){ hadoopVersion="`hdfs version`" - defaultHadoopVersion="2.7" + defaultHadoopVersion="3.3" checkversion "$hadoopVersion" $defaultHadoopVersion hadoop } diff --git a/linkis-dist/bin/install.sh b/linkis-dist/bin/install.sh index 038d2784668..87e01885cbf 100644 --- a/linkis-dist/bin/install.sh +++ b/linkis-dist/bin/install.sh @@ -219,13 +219,13 @@ SERVER_IP=$local_host ##Label set start if [ "$SPARK_VERSION" != "" ] then - sed -i ${txt} "s#spark-2.4.3#spark-$SPARK_VERSION#g" $LINKIS_HOME/db/linkis_dml.sql + sed -i ${txt} "s#spark-3.2.1#spark-$SPARK_VERSION#g" $LINKIS_HOME/db/linkis_dml.sql sed -i ${txt} "s#\#wds.linkis.spark.engine.version.*#wds.linkis.spark.engine.version=$SPARK_VERSION#g" $common_conf fi if [ "$HIVE_VERSION" != "" ] then - sed -i ${txt} "s#hive-2.3.3#hive-$HIVE_VERSION#g" $LINKIS_HOME/db/linkis_dml.sql + sed -i ${txt} "s#hive-3.1.3#hive-$HIVE_VERSION#g" $LINKIS_HOME/db/linkis_dml.sql sed -i ${txt} "s#\#wds.linkis.hive.engine.version.*#wds.linkis.hive.engine.version=$HIVE_VERSION#g" $common_conf fi diff --git a/linkis-dist/deploy-config/linkis-env.sh b/linkis-dist/deploy-config/linkis-env.sh index 9197f7be977..f4d497a4ac1 100644 --- a/linkis-dist/deploy-config/linkis-env.sh +++ b/linkis-dist/deploy-config/linkis-env.sh @@ -78,7 +78,7 @@ HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/appcom/config/hadoop-config"} HADOOP_KERBEROS_ENABLE=${HADOOP_KERBEROS_ENABLE:-"false"} HADOOP_KEYTAB_PATH=${HADOOP_KEYTAB_PATH:-"/appcom/keytab/"} ## Hadoop env version -HADOOP_VERSION=${HADOOP_VERSION:-"2.7.2"} +HADOOP_VERSION=${HADOOP_VERSION:-"3.3.4"} #Hive HIVE_HOME=/appcom/Install/hive @@ -91,10 +91,10 @@ SPARK_CONF_DIR=/appcom/config/spark-config ## Engine version conf #SPARK_VERSION -#SPARK_VERSION=2.4.3 +#SPARK_VERSION=3.2.1 ##HIVE_VERSION -#HIVE_VERSION=2.3.3 +#HIVE_VERSION=3.1.3 #PYTHON_VERSION=python2 diff --git a/linkis-dist/docker/ldh.Dockerfile b/linkis-dist/docker/ldh.Dockerfile index dcd01bdf999..05e7d77e2b8 100644 --- a/linkis-dist/docker/ldh.Dockerfile +++ b/linkis-dist/docker/ldh.Dockerfile @@ -27,10 +27,10 @@ ARG JDK_VERSION=1.8.0-openjdk ARG JDK_BUILD_REVISION=1.8.0.332.b09-1.el7_9 ARG MYSQL_JDBC_VERSION=8.0.28 -ARG HADOOP_VERSION=2.7.2 -ARG HIVE_VERSION=2.3.3 -ARG SPARK_VERSION=2.4.3 -ARG SPARK_HADOOP_VERSION=2.7 +ARG HADOOP_VERSION=3.3.4 +ARG HIVE_VERSION=3.1.3 +ARG SPARK_VERSION=3.2.1 +ARG SPARK_HADOOP_VERSION=3.2 ARG FLINK_VERSION=1.12.2 ARG ZOOKEEPER_VERSION=3.5.9 diff --git a/linkis-dist/docker/scripts/prepare-ldh-image.sh b/linkis-dist/docker/scripts/prepare-ldh-image.sh index 791c7c731b8..d37719c1a72 100755 --- a/linkis-dist/docker/scripts/prepare-ldh-image.sh +++ b/linkis-dist/docker/scripts/prepare-ldh-image.sh @@ -27,10 +27,10 @@ rm -rf ${LDH_TAR_DIR} && mkdir -p ${LDH_TAR_DIR} rm -rf ${PROJECT_TARGET}/entry-point-ldh.sh cp ${WORK_DIR}/entry-point-ldh.sh ${PROJECT_TARGET}/ -HADOOP_VERSION=${HADOOP_VERSION:-2.7.2} -HIVE_VERSION=${HIVE_VERSION:-2.3.3} -SPARK_VERSION=${SPARK_VERSION:-2.4.3} -SPARK_HADOOP_VERSION=${SPARK_HADOOP_VERSION:-2.7} +HADOOP_VERSION=${HADOOP_VERSION:-3.3.4} +HIVE_VERSION=${HIVE_VERSION:-3.1.3} +SPARK_VERSION=${SPARK_VERSION:-3.2.1} +SPARK_HADOOP_VERSION=${SPARK_HADOOP_VERSION:-3.2} FLINK_VERSION=${FLINK_VERSION:-1.12.2} ZOOKEEPER_VERSION=${ZOOKEEPER_VERSION:-3.5.9} MYSQL_JDBC_VERSION=${MYSQL_JDBC_VERSION:-8.0.28} diff --git a/linkis-dist/helm/README.md b/linkis-dist/helm/README.md index 274de3dc2a3..b1cce7ce754 100644 --- a/linkis-dist/helm/README.md +++ b/linkis-dist/helm/README.md @@ -201,9 +201,9 @@ $> kind delete cluster --name test-helm We introduced a new image, called LDH (Linkis's hadoop all-in-one image), which provides a pseudo-distributed hadoop cluster for testing quickly. This image contains the following hadoop components, the default mode for engines in LDH is on-yarn. -* Hadoop 2.7.2 , including HDFS and YARN -* Hive 2.3.3 -* Spark 2.4.3 +* Hadoop 3.3.4 , including HDFS and YARN +* Hive 3.1.3 +* Spark 3.2.1 * Flink 1.12.2 * ZooKeeper 3.5.9 @@ -245,10 +245,10 @@ drwxrwxrwx - root supergroup 0 2022-07-31 02:48 /user [root@ldh-96bdc757c-dnkbs /]# beeline -u jdbc:hive2://ldh.ldh.svc.cluster.local:10000/ -n hadoop Connecting to jdbc:hive2://ldh.ldh.svc.cluster.local:10000/ -Connected to: Apache Hive (version 2.3.3) -Driver: Hive JDBC (version 2.3.3) +Connected to: Apache Hive (version 3.1.3) +Driver: Hive JDBC (version 3.1.3) Transaction isolation: TRANSACTION_REPEATABLE_READ -Beeline version 2.3.3 by Apache Hive +Beeline version 3.1.3 by Apache Hive 0: jdbc:hive2://ldh.ldh.svc.cluster.local:100> create database demo; No rows affected (1.306 seconds) 0: jdbc:hive2://ldh.ldh.svc.cluster.local:100> use demo; @@ -271,7 +271,7 @@ No rows affected (5.491 seconds) 22/07/31 02:53:18 INFO hive.metastore: Trying to connect to metastore with URI thrift://ldh.ldh.svc.cluster.local:9083 22/07/31 02:53:18 INFO hive.metastore: Connected to metastore. ... -22/07/31 02:53:19 INFO spark.SparkContext: Running Spark version 2.4.3 +22/07/31 02:53:19 INFO spark.SparkContext: Running Spark version 3.2.1 22/07/31 02:53:19 INFO spark.SparkContext: Submitted application: SparkSQL::10.244.0.6 ... 22/07/31 02:53:27 INFO yarn.Client: Submitting application application_1659235712576_0001 to ResourceManager diff --git a/linkis-dist/helm/README_CN.md b/linkis-dist/helm/README_CN.md index e756dc73fc5..832530147a5 100644 --- a/linkis-dist/helm/README_CN.md +++ b/linkis-dist/helm/README_CN.md @@ -190,9 +190,9 @@ $> kind delete cluster --name test-helm ## 使用 LDH 进行测试 我们引入了一个新的镜像，叫做LDH（Linkis 的 hadoop 一体式镜像），它提供了一个伪分布式的 hadoop 集群，方便快速测试 On Hadoop 的部署模式。 这个镜像包含以下多个 hadoop 组件，LDH 中引擎的默认模式是 on-yarn 的。 -* Hadoop 2.7.2 , 包括 HDFS and YARN -* Hive 2.3.3 -* Spark 2.4.3 +* Hadoop 3.3.4 , 包括 HDFS and YARN +* Hive 3.1.3 +* Spark 3.2.1 * Flink 1.12.2 * ZooKeeper 3.5.9 @@ -236,10 +236,10 @@ drwxrwxrwx - root supergroup 0 2022-07-31 02:48 /user [root@ldh-96bdc757c-dnkbs /]# beeline -u jdbc:hive2://ldh.ldh.svc.cluster.local:10000/ -n hadoop Connecting to jdbc:hive2://ldh.ldh.svc.cluster.local:10000/ -Connected to: Apache Hive (version 2.3.3) -Driver: Hive JDBC (version 2.3.3) +Connected to: Apache Hive (version 3.1.3) +Driver: Hive JDBC (version 3.1.3) Transaction isolation: TRANSACTION_REPEATABLE_READ -Beeline version 2.3.3 by Apache Hive +Beeline version 3.1.3 by Apache Hive 0: jdbc:hive2://ldh.ldh.svc.cluster.local:100> create database demo; No rows affected (1.306 seconds) 0: jdbc:hive2://ldh.ldh.svc.cluster.local:100> use demo; @@ -262,7 +262,7 @@ No rows affected (5.491 seconds) 22/07/31 02:53:18 INFO hive.metastore: Trying to connect to metastore with URI thrift://ldh.ldh.svc.cluster.local:9083 22/07/31 02:53:18 INFO hive.metastore: Connected to metastore. ... -22/07/31 02:53:19 INFO spark.SparkContext: Running Spark version 2.4.3 +22/07/31 02:53:19 INFO spark.SparkContext: Running Spark version 3.2.1 22/07/31 02:53:19 INFO spark.SparkContext: Submitted application: SparkSQL::10.244.0.6 ... 22/07/31 02:53:27 INFO yarn.Client: Submitting application application_1659235712576_0001 to ResourceManager diff --git a/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml b/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml index 175f2cb7ad9..30db9e61a0d 100644 --- a/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml +++ b/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml @@ -1183,12 +1183,12 @@ data: (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = '*-*,*-*'); - -- spark2.4.3 default configuration + -- spark default configuration insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @SPARK_ALL); - -- hive1.2.1 default configuration + -- hive default configuration insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @HIVE_ALL); diff --git a/linkis-dist/helm/charts/linkis/values.yaml b/linkis-dist/helm/charts/linkis/values.yaml index 638f75134e3..89017dbb294 100644 --- a/linkis-dist/helm/charts/linkis/values.yaml +++ b/linkis-dist/helm/charts/linkis/values.yaml @@ -111,7 +111,7 @@ linkis: python: version: 2.7 hadoop: - version: 2.7.2 + version: 3.3.4 configMapName: hadoop-conf yarn: restfulUrl: http://ldh.ldh.svc.cluster.local:8088 @@ -123,10 +123,10 @@ linkis: keytab: /etc/hadoop-conf/yarn.keytab krb5: /etc/krb5.keytab spark: - version: 2.4.3 + version: 3.2.1 configMapName: spark-conf hive: - version: 2.3.3 + version: 3.1.3 configMapName: hive-conf meta: url: "jdbc:mysql://mysql.mysql.svc.cluster.local:3306/hive_metadata?&createDatabaseIfNotExist=true&characterEncoding=UTF-8&useSSL=false" # jdbc:mysql://localhost:3306/metastore?useUnicode=true diff --git a/linkis-dist/helm/scripts/prepare-for-spark.sh b/linkis-dist/helm/scripts/prepare-for-spark.sh index 2bbd1123a38..5b2b35a8245 100644 --- a/linkis-dist/helm/scripts/prepare-for-spark.sh +++ b/linkis-dist/helm/scripts/prepare-for-spark.sh @@ -28,10 +28,10 @@ ECM_POD_NAME=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-de kubectl cp ./ldh -n linkis ${ECM_POD_NAME}:/opt/ ; -kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "chmod +x /opt/ldh/1.3.0/spark-2.4.3-bin-hadoop2.7/bin/*" -kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "ln -s /opt/ldh/1.3.0/spark-2.4.3-bin-hadoop2.7 /opt/ldh/current/spark" -kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "ln -s /opt/ldh/1.3.0/hadoop-2.7.2 /opt/ldh/current/hadoop" -kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "ln -s /opt/ldh/1.3.0/apache-hive-2.3.3-bin /opt/ldh/current/hive" +kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "chmod +x /opt/ldh/1.3.0/spark-3.2.1-bin-hadoop3.2/bin/*" +kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "ln -s /opt/ldh/1.3.0/spark-3.2.1-bin-hadoop3.2 /opt/ldh/current/spark" +kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "ln -s /opt/ldh/1.3.0/hadoop-3.3.4 /opt/ldh/current/hadoop" +kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "ln -s /opt/ldh/1.3.0/apache-hive-3.1.3-bin /opt/ldh/current/hive" kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "echo 'export SPARK_HOME=/opt/ldh/current/spark' |sudo tee --append /etc/profile" diff --git a/linkis-dist/package/bin/linkis-cli-hive b/linkis-dist/package/bin/linkis-cli-hive index 31ef0c54f0e..7c8da89c4cc 100644 --- a/linkis-dist/package/bin/linkis-cli-hive +++ b/linkis-dist/package/bin/linkis-cli-hive @@ -161,6 +161,6 @@ else parse fi -exec ${WORK_DIR}/bin/linkis-cli-pre -engineType hive-2.3.3 -codeType hql "${PARSED_CMD[@]}" +exec ${WORK_DIR}/bin/linkis-cli-pre -engineType hive-3.1.3 -codeType hql "${PARSED_CMD[@]}" diff --git a/linkis-dist/package/bin/linkis-cli-spark-submit b/linkis-dist/package/bin/linkis-cli-spark-submit index 2ae23046686..c3f62efc518 100644 --- a/linkis-dist/package/bin/linkis-cli-spark-submit +++ b/linkis-dist/package/bin/linkis-cli-spark-submit @@ -192,9 +192,9 @@ else fi if [ "$IS_PYSPARK"x == "true"x ]; then - exec ${WORK_DIR}/bin/linkis-cli-pre -engineType spark-2.4.3 -codeType py "${PARSED_CMD[@]}" + exec ${WORK_DIR}/bin/linkis-cli-pre -engineType spark-3.2.1 -codeType py "${PARSED_CMD[@]}" elif [ "IS_SCALA"x == "true"x ]; then - exec ${WORK_DIR}/bin/linkis-cli-pre -engineType spark-2.4.3 -codeType scala "${PARSED_CMD[@]}" + exec ${WORK_DIR}/bin/linkis-cli-pre -engineType spark-3.2.1 -codeType scala "${PARSED_CMD[@]}" else - exec ${WORK_DIR}/bin/linkis-cli-pre -engineType spark-2.4.3 "${PARSED_CMD[@]}" + exec ${WORK_DIR}/bin/linkis-cli-pre -engineType spark-3.2.1 "${PARSED_CMD[@]}" fi \ No newline at end of file diff --git a/linkis-dist/package/db/linkis_dml.sql b/linkis-dist/package/db/linkis_dml.sql index dc9a1c1eccd..b193cc5112d 100644 --- a/linkis-dist/package/db/linkis_dml.sql +++ b/linkis-dist/package/db/linkis_dml.sql @@ -18,8 +18,8 @@ -- 变量： -SET @SPARK_LABEL="spark-2.4.3"; -SET @HIVE_LABEL="hive-2.3.3"; +SET @SPARK_LABEL="spark-3.2.1"; +SET @HIVE_LABEL="hive-3.1.3"; SET @PYTHON_LABEL="python-python2"; SET @PIPELINE_LABEL="pipeline-1"; SET @JDBC_LABEL="jdbc-4"; @@ -189,18 +189,18 @@ insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_featur insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType', @PRESTO_ALL, 'OPTIONAL', 2, now(), now()); insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType', @TRINO_ALL, 'OPTIONAL', 2, now(), now()); --- Custom correlation engine (e.g. spark-2.4.3) and configKey value +-- Custom correlation engine (e.g. spark) and configKey value -- Global Settings insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`) (select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type is null and label.label_value = "*-*,*-*"); --- spark-2.4.3(Here choose to associate all spark type Key values with spark2.4.3) +-- spark(Here choose to associate all spark type Key values with spark) insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`) (select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type = 'spark' and label.label_value = @SPARK_ALL); --- hive-1.2.1 +-- hive insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`) (select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type = 'hive' and label_value = @HIVE_ALL); @@ -318,12 +318,12 @@ insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_val (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = '*-*,*-*'); --- spark2.4.3 default configuration +-- spark default configuration insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @SPARK_ALL); --- hive1.2.1 default configuration +-- hive default configuration insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @HIVE_ALL); diff --git a/linkis-dist/package/db/module/linkis_configuration_dml.sql b/linkis-dist/package/db/module/linkis_configuration_dml.sql index 3e71eaeba0f..0d989eba38f 100644 --- a/linkis-dist/package/db/module/linkis_configuration_dml.sql +++ b/linkis-dist/package/db/module/linkis_configuration_dml.sql @@ -18,8 +18,8 @@ -- 变量： -SET @SPARK_LABEL="spark-2.4.3"; -SET @HIVE_LABEL="hive-1.2.1"; +SET @SPARK_LABEL="spark-3.2.1"; +SET @HIVE_LABEL="hive-3.1.3"; SET @PYTHON_LABEL="python-python2"; SET @PIPELINE_LABEL="pipeline-*"; SET @JDBC_LABEL="jdbc-4"; @@ -109,18 +109,18 @@ insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_featur insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@PIPELINE_ALL, 'OPTIONAL', 2, now(), now()); insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@JDBC_ALL, 'OPTIONAL', 2, now(), now()); --- Custom correlation engine (e.g. spark-2.4.3) and configKey value +-- Custom correlation engine (e.g. spark) and configKey value -- Global Settings insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`) (select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type is null and label.label_value = "*-*,*-*"); --- spark-2.4.3(Here choose to associate all spark type Key values with spark2.4.3) +-- spark(Here choose to associate all spark type Key values with spark) insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`) (select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type = 'spark' and label.label_value = @SPARK_ALL); --- hive-1.2.1 +-- hive insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`) (select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type = 'hive' and label_value = @HIVE_ALL); @@ -206,12 +206,12 @@ insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_val (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = '*-*,*-*'); --- spark2.4.3 default configuration +-- spark default configuration insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @SPARK_ALL); --- hive1.2.1 default configuration +-- hive default configuration insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @HIVE_ALL); diff --git a/linkis-dist/pom.xml b/linkis-dist/pom.xml index b847950a796..6ffc994062a 100644 --- a/linkis-dist/pom.xml +++ b/linkis-dist/pom.xml @@ -211,10 +211,10 @@ /opt/linkis /etc/linkis-conf /var/logs/linkis - 2.7.2 - 2.3.3 - 2.4.3 - 2.7 + 3.3.4 + 3.1.3 + 3.2.1 + 3.2 1.12.2 3.5.9 diff --git a/linkis-engineconn-plugins/spark/pom.xml b/linkis-engineconn-plugins/spark/pom.xml index 46ed7abab18..eebadfbb458 100644 --- a/linkis-engineconn-plugins/spark/pom.xml +++ b/linkis-engineconn-plugins/spark/pom.xml @@ -435,16 +435,21 @@ - org.apache.hadoop - hadoop-common - ${hadoop.version} + org.eclipse.jetty + jetty-client provided - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - provided + ${spark.hadoop.groupid} + ${spark.hadoop-common.artifactId} + ${spark.hadoop.version} + ${spark.hadoop.scope} + + + ${spark.hadoop.groupid} + ${spark.hadoop-hdfs.artifactId} + ${spark.hadoop.version} + ${spark.hadoop.scope} @@ -485,98 +490,4 @@ - - - - spark-2.4-hadoop-3.3 - - ${hadoop-hdfs-client-shade.version} - - - - org.apache.linkis - linkis-hadoop-hdfs-client-shade - ${project.version} - - - commmons-logging - commons-logging - - - log4j - log4j - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jetty-util - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-json - - - javax.ws.rs - jsr311-api - - - net.java.dev.jets3t - jets3t - - - com.jcraft - jsch - - - com.google.code.findbugs - jsr305 - - - xmlenc - xmlenc - - - net.java.dev.jets3t - jets3t - - - org.apache.avro - avro - - - com.jcraft - jsch - - - com.google.code.findbugs - jsr305 - - - javax.servlet - servlet-api - - - org.slf4j - slf4j-log4j12 - - - org.eclipse.jetty - * - - - - - - diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/response/EngineLabelResponse.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/response/EngineLabelResponse.java index 9bf7e51bde6..fdab5d6d1ba 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/response/EngineLabelResponse.java +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/response/EngineLabelResponse.java @@ -30,7 +30,7 @@ public class EngineLabelResponse implements Serializable { @ApiModelProperty(value = "label id.") private Integer labelId; - @ApiModelProperty(value = "engine name. eg: spark-2.4.3") + @ApiModelProperty(value = "engine name. eg: spark-3.2.1") private String engineName; @ApiModelProperty(value = "install. eg: yes") diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/data.sql b/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/data.sql index 680c5307629..4b3b29fa654 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/data.sql +++ b/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/data.sql @@ -49,21 +49,21 @@ INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_ INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (3, 'combined_userCreator_engineType', '*-Visualis,*-*', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (4, 'combined_userCreator_engineType', '*-nodeexecution,*-*', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (5, 'combined_userCreator_engineType', '*-*,*-*', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); -INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (6, 'combined_userCreator_engineType', '*-*,spark-2.4.3', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); -INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (7, 'combined_userCreator_engineType', '*-*,hive-2.3.3', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); +INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (6, 'combined_userCreator_engineType', '*-*,spark-3.2.1', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); +INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (7, 'combined_userCreator_engineType', '*-*,hive-3.1.3', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (8, 'combined_userCreator_engineType', '*-*,python-python2', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (9, 'combined_userCreator_engineType', '*-*,pipeline-1', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (10, 'combined_userCreator_engineType', '*-*,jdbc-4', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (11, 'combined_userCreator_engineType', '*-*,openlookeng-1.5.0', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); -INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (12, 'combined_userCreator_engineType', '*-IDE,spark-2.4.3', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); -INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (13, 'combined_userCreator_engineType', '*-IDE,hive-2.3.3', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); +INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (12, 'combined_userCreator_engineType', '*-IDE,spark-3.2.1', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); +INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (13, 'combined_userCreator_engineType', '*-IDE,hive-3.1.3', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (14, 'combined_userCreator_engineType', '*-IDE,python-python2', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (15, 'combined_userCreator_engineType', '*-IDE,pipeline-1', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (16, 'combined_userCreator_engineType', '*-IDE,jdbc-4', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (17, 'combined_userCreator_engineType', '*-IDE,openlookeng-1.5.0', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); -INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (18, 'combined_userCreator_engineType', '*-Visualis,spark-2.4.3', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); -INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (19, 'combined_userCreator_engineType', '*-nodeexecution,spark-2.4.3', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); -INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (20, 'combined_userCreator_engineType', '*-nodeexecution,hive-2.3.3', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); +INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (18, 'combined_userCreator_engineType', '*-Visualis,spark-3.2.1', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); +INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (19, 'combined_userCreator_engineType', '*-nodeexecution,spark-3.2.1', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); +INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (20, 'combined_userCreator_engineType', '*-nodeexecution,hive-3.1.3', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); INSERT INTO `linkis_cg_manager_label` (`id`, `label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES (21, 'combined_userCreator_engineType', '*-nodeexecution,python-python2', 'OPTIONAL', 2, '2022-11-24 20:46:21', '2022-11-24 20:46:21'); diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/errorcode/LinkisConfigurationErrorCodeSummary.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/errorcode/LinkisConfigurationErrorCodeSummary.java index 3f58930ea11..f02e0398f58 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/errorcode/LinkisConfigurationErrorCodeSummary.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/errorcode/LinkisConfigurationErrorCodeSummary.java @@ -39,7 +39,7 @@ public enum LinkisConfigurationErrorCodeSummary implements LinkisErrorCode { ENGINE_TYPE_IS_NULL(14100, "Engine type is null, cannot be added(引擎类型为空，无法添加)"), INCORRECT_FIXED_SUCH( 14100, - "The saved engine type parameter is incorrect, please send it in a fixed format, such as spark-2.4.3(保存的引擎类型参数有误，请按照固定格式传送，例如spark-2.4.3)"), + "The saved engine type parameter is incorrect, please send it in a fixed format, such as spark-3.2.1(保存的引擎类型参数有误，请按照固定格式传送，例如spark-3.2.1)"), INCOMPLETE_RECONFIRM(14100, "Incomplete request parameters, please reconfirm(请求参数不完整，请重新确认)"), ONLY_ADMIN_CAN_MODIFY(14100, "Only admin can modify category(只有管理员才能修改目录)"), THE_LABEL_PARAMETER_IS_EMPTY(14100, " The label parameter is empty(标签参数为空)"), diff --git a/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApiTest.java b/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApiTest.java index 77c77d926fb..09747430147 100644 --- a/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApiTest.java +++ b/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApiTest.java @@ -55,9 +55,9 @@ public class ConfigurationRestfulApiTest { public void TestAddKeyForEngine() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("engineType", "spark"); - paramsMap.add("version", "2.4.3"); + paramsMap.add("version", "3.2.1"); paramsMap.add("token", "e8724-e"); - paramsMap.add("keyJson", "{'engineType':'spark','version':'2.4.3'}"); + paramsMap.add("keyJson", "{'engineType':'spark','version':'3.2.1'}"); String url = "/configuration/addKeyForEngine"; sendUrl(url, paramsMap, "get", null); } @@ -66,7 +66,7 @@ public void TestAddKeyForEngine() throws Exception { public void TestGetFullTreesByAppName() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("engineType", "spark"); - paramsMap.add("version", "2.4.3"); + paramsMap.add("version", "3.2.1"); paramsMap.add("creator", "sam"); String url = "/configuration/getFullTreesByAppName"; @@ -127,7 +127,7 @@ public void TestSaveFullTree() throws Exception { // " }\n" + // " ],\n" + // " \"creator\": \"LINKISCLI\",\n" + - // " \"engineType\": \"hive-2.3.3\"\n" + + // " \"engineType\": \"hive-3.1.3\"\n" + // "}"; // String url = "/configuration/saveFullTree"; // diff --git a/linkis-web/src/apps/linkis/module/setting/setting.vue b/linkis-web/src/apps/linkis/module/setting/setting.vue index 9b7e4bceb04..97d11a17100 100644 --- a/linkis-web/src/apps/linkis/module/setting/setting.vue +++ b/linkis-web/src/apps/linkis/module/setting/setting.vue @@ -310,7 +310,7 @@ export default { { creator: parameter[0], // Specify a first-level directory(指定一级目录) engineType: parameter[1], // Specify the engine (secondary directory) if there is only a first-level directory, it will be automatically undefined and no parameters will be passed(指定引擎（二级目录）如果只有一级目录则自动为undefined不会传参) - version: parameter[2], // The corresponding engine currently only supports the corresponding version. For example, spark will pass version-2.4.3. If there is only a first-level directory, it will be automatically undefined and no parameters will be passed.(对应的引擎目前只支持对应的版本，如spark就传version-2.4.3，如果只有一级目录则自动为undefined不会传参) + version: parameter[2], // The corresponding engine currently only supports the corresponding version. For example, spark will pass version-3.2.1. If there is only a first-level directory, it will be automatically undefined and no parameters will be passed.(对应的引擎目前只支持对应的版本，如spark就传version-3.2.1，如果只有一级目录则自动为undefined不会传参) }, "get" ) diff --git a/pom.xml b/pom.xml index 030b00f870f..8549ec8e9a4 100644 --- a/pom.xml +++ b/pom.xml @@ -105,14 +105,19 @@ 1.3.2-SNAPSHOT 2.9.2 - 2.4.3 - 2.3.3 - 2.7.2 - hadoop-hdfs + 3.2.1 + 3.1.3 + 3.3.4 + hadoop-hdfs-client 2.7.2 + org.apache.hadoop + hadoop-common + hadoop-hdfs + ${hadoop.version} + provided 3.5.9 - 2.7.1 + 4.2.0 30.0-jre 4.1.86.Final @@ -120,7 +125,7 @@ 2.8.9 2.13.4.20221013 - 3.5.3 + 3.7.0-M11 1.19.4 2.23.1 @@ -175,8 +180,8 @@ 1.8 3.5.0 - 2.11.12 - 2.11 + 2.12.17 + 2.12 1.10.12 @@ -1352,30 +1357,28 @@ - - - hadoop-3.3 - - 3.3.1 - 4.2.0 - hadoop-hdfs-client - - hadoop-2.7 2.7.2 2.7.1 + hadoop-hdfs + - spark-3.2 + spark-2.4 - 3.7.0-M11 - 3.2.1 - 2.12.15 - 2.12 + org.apache.linkis + linkis-hadoop-hdfs-client-shade + linkis-hadoop-hdfs-client-shade + ${project.version} + compile + 3.5.3 + 2.4.3 + 2.11.12 + 2.11 diff --git a/tool/dependencies/known-dependencies.txt b/tool/dependencies/known-dependencies.txt index 5940c589513..3c4cbf9d338 100644 --- a/tool/dependencies/known-dependencies.txt +++ b/tool/dependencies/known-dependencies.txt @@ -585,4 +585,123 @@ seatunnel-core-spark-2.1.2.jar mongo-java-driver-3.12.7.jar clickhouse-jdbc-0.3.2-patch11.jar postgresql-42.3.8.jar - +accessors-smart-2.3.1.jar +aircompressor-0.10.jar +akka-actor_2.12-2.5.21.jar +akka-protobuf_2.12-2.5.21.jar +akka-slf4j_2.12-2.5.21.jar +akka-stream_2.12-2.5.21.jar +asm-9.3.jar +avatica-1.11.0.jar +calcite-core-1.16.0.jar +calcite-druid-1.16.0.jar +calcite-linq4j-1.16.0.jar +chill_2.12-0.7.6.jar +commons-configuration2-2.1.1.jar +commons-el-1.0.jar +commons-net-3.6.jar +curator-client-4.2.0.jar +curator-framework-4.2.0.jar +curator-recipes-4.2.0.jar +kerby-util-1.0.1.jar +kerby-xdr-1.0.1.jar +kotlin-stdlib-1.3.72.jar +kotlin-stdlib-common-1.3.72.jar +memory-0.9.0.jar +netty-3.10.6.Final.jar +nimbus-jose-jwt-8.19.jar +orc-core-1.5.8.jar +orc-shims-1.5.8.jar +re2j-1.1.jar +reload4j-1.2.22.jar +scala-compiler-2.12.17.jar +scala-java8-compat_2.12-0.8.0.jar +scala-library-2.12.17.jar +scala-parser-combinators_2.12-1.1.1.jar +scala-reflect-2.12.17.jar +scala-xml_2.12-2.1.0.jar +scalap-2.12.17.jar +scopt_2.12-3.5.0.jar +servlet-api-2.5.jar +sketches-core-0.9.0.jar +slf4j-reload4j-1.7.36.jar +snappy-java-1.1.8.2.jar +ssl-config-core_2.12-0.3.7.jar +token-provider-1.0.1.jar +woodstox-core-5.3.0.jar +dnsjava-2.1.7.jar +esri-geometry-api-2.0.0.jar +flink-clients_2.12-1.12.2.jar +flink-connector-hive_2.12-1.12.2.jar +flink-connector-kafka_2.12-1.12.2.jar +flink-optimizer_2.12-1.12.2.jar +flink-runtime_2.12-1.12.2.jar +flink-scala_2.12-1.12.2.jar +flink-sql-client_2.12-1.12.2.jar +flink-streaming-java_2.12-1.12.2.jar +flink-streaming-scala_2.12-1.12.2.jar +flink-table-api-java-bridge_2.12-1.12.2.jar +flink-table-api-scala-bridge_2.12-1.12.2.jar +flink-table-api-scala_2.12-1.12.2.jar +flink-table-planner-blink_2.12-1.12.2.jar +flink-table-runtime-blink_2.12-1.12.2.jar +flink-yarn_2.12-1.12.2.jar +grizzled-slf4j_2.12-1.3.2.jar +guice-4.0.jar +guice-servlet-4.0.jar +hadoop-annotations-3.3.4.jar +hadoop-auth-3.3.4.jar +hadoop-client-3.3.4.jar +hadoop-common-3.3.4.jar +hadoop-hdfs-2.4.1.jar +hadoop-hdfs-2.7.1.jar +hadoop-hdfs-3.3.4.jar +hadoop-hdfs-client-3.3.4.jar +hadoop-mapreduce-client-common-3.3.4.jar +hadoop-mapreduce-client-core-3.3.4.jar +hadoop-mapreduce-client-jobclient-3.3.4.jar +hadoop-shaded-guava-1.1.1.jar +hadoop-shaded-protobuf_3_7-1.1.1.jar +hadoop-yarn-api-3.3.4.jar +hadoop-yarn-client-3.3.4.jar +hadoop-yarn-common-3.3.4.jar +hadoop-yarn-registry-3.1.0.jar +hive-classification-3.1.3.jar +hive-common-3.1.3.jar +hive-exec-3.1.3.jar +hive-llap-client-3.1.3.jar +hive-llap-common-3.1.3.jar +hive-llap-tez-3.1.3.jar +hive-storage-api-2.7.0.jar +hive-upgrade-acid-3.1.3.jar +hive-vector-code-gen-3.1.3.jar +jackson-jaxrs-base-2.13.4.jar +jackson-jaxrs-json-provider-2.13.4.jar +jackson-module-jaxb-annotations-2.13.4.jar +jackson-module-scala_2.12-2.13.4.jar +jasper-runtime-5.5.23.jar +javax.servlet-api-4.0.1.jar +jcip-annotations-1.0-1.jar +jetty-6.1.26.jar +jetty-rewrite-9.4.48.v20220622.jar +jetty-util-6.1.26.jar +jline-3.9.0.jar +joda-time-2.10.10.jar +joda-time-2.9.9.jar +json-smart-2.3.1.jar +json4s-ast_2.12-3.7.0-M11.jar +json4s-core_2.12-3.7.0-M11.jar +json4s-jackson_2.12-3.7.0-M11.jar +json4s-scalap_2.12-3.7.0-M11.jar +kerb-admin-1.0.1.jar +kerb-client-1.0.1.jar +kerb-common-1.0.1.jar +kerb-core-1.0.1.jar +kerb-crypto-1.0.1.jar +kerb-identity-1.0.1.jar +kerb-server-1.0.1.jar +kerb-simplekdc-1.0.1.jar +kerb-util-1.0.1.jar +kerby-asn1-1.0.1.jar +kerby-config-1.0.1.jar +kerby-pkix-1.0.1.jar \ No newline at end of file From 75eddde7591a59c2b721a92afd8b5dc88b1143e3 Mon Sep 17 00:00:00 2001 From: guoshupei <15764973965@163.com> Date: Mon, 6 Mar 2023 11:46:01 +0800 Subject: [PATCH 046/689] rename metric config key --- .../scala/org/apache/linkis/entrance/EntranceServer.scala | 4 ++-- .../linkis/entrance/conf/EntranceConfiguration.scala | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index 5560cc716dc..e9c3da2cdab 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -625,7 +625,7 @@ abstract class EntranceServer extends Logging { ) val metricMap = new util.HashMap[String, Object]() - if (EntranceConfiguration.ENTRANCE_FAILOVER_RETAIN_ENGINE_CONN_ENABLED.getValue) { + if (EntranceConfiguration.ENTRANCE_FAILOVER_RETAIN_METRIC_ENGINE_CONN_ENABLED.getValue) { if ( jobRequest.getMetrics != null && jobRequest.getMetrics.containsKey( TaskConstant.ENTRANCEJOB_ENGINECONN_MAP @@ -638,7 +638,7 @@ abstract class EntranceServer extends Logging { } } - if (EntranceConfiguration.ENTRANCE_FAILOVER_RETAIN_YARN_RESOURCE_ENABLED.getValue) { + if (EntranceConfiguration.ENTRANCE_FAILOVER_RETAIN_METRIC_YARN_RESOURCE_ENABLED.getValue) { if ( jobRequest.getMetrics != null && jobRequest.getMetrics.containsKey( TaskConstant.ENTRANCEJOB_YARNRESOURCE diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index 17f2dffd9c0..617584f2782 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -254,11 +254,11 @@ object EntranceConfiguration { val ENTRANCE_GROUP_SCAN_INTERVAL = CommonVars("linkis.entrance.group.scan.interval", 60 * 1000) - val ENTRANCE_FAILOVER_RETAIN_ENGINE_CONN_ENABLED = - CommonVars("linkis.entrance.failover.retain.engine.conn.enable", false) + val ENTRANCE_FAILOVER_RETAIN_METRIC_ENGINE_CONN_ENABLED = + CommonVars("linkis.entrance.failover.retain.metric.engine.conn.enable", false) - val ENTRANCE_FAILOVER_RETAIN_YARN_RESOURCE_ENABLED = - CommonVars("linkis.entrance.failover.retain.yarn.resource.enable", false) + val ENTRANCE_FAILOVER_RETAIN_METRIC_YARN_RESOURCE_ENABLED = + CommonVars("linkis.entrance.failover.retain.metric.yarn.resource.enable", false) val ENTRANCE_FAILOVER_RUNNING_KILL_ENABLED = CommonVars("linkis.entrance.failover.running.kill.enable", false) From 9990b9c9a64b78ed101f91f3d55375149a2d2a2a Mon Sep 17 00:00:00 2001 From: binbinCheng Date: Mon, 6 Mar 2023 14:18:30 +0800 Subject: [PATCH 047/689] Beautify the sql style, clear and clear --- .../src/main/resources/mapper/common/UserIpMapper.xml | 8 ++++---- .../src/main/resources/mapper/common/UserTenantMapper.xml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/UserIpMapper.xml b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/UserIpMapper.xml index 62f36291ba0..d25ebf25526 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/UserIpMapper.xml +++ b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/UserIpMapper.xml @@ -53,8 +53,8 @@ - select - from linkis_cg_user_ip_config + SELECT + FROM linkis_cg_user_ip_config `user` = #{user} and `creator` = #{creator} diff --git a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/UserTenantMapper.xml b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/UserTenantMapper.xml index 1d031f730d3..bd653c90a29 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/UserTenantMapper.xml +++ b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/UserTenantMapper.xml @@ -52,8 +52,8 @@ - select - from linkis_cg_tenant_label_config + SELECT + FROM linkis_cg_tenant_label_config `user` = #{user} and `creator` = #{creator} From 6fee59f67d637dad555fc96715164e327c1eee08 Mon Sep 17 00:00:00 2001 From: guoshupei <15764973965@163.com> Date: Wed, 8 Mar 2023 11:02:34 +0800 Subject: [PATCH 048/689] - failover server close - use logger template --- .../server/EntranceFailoverJobServer.java | 182 ++++++++++-------- .../EntranceParallelConsumerManager.scala | 10 +- 2 files changed, 108 insertions(+), 84 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java index d7f5ce5951b..d162be08205 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/EntranceFailoverJobServer.java @@ -31,6 +31,8 @@ import org.apache.linkis.scheduler.queue.SchedulerEventState; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.event.ContextClosedEvent; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -38,9 +40,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -59,6 +59,8 @@ public class EntranceFailoverJobServer { private ScheduledExecutorService scheduledExecutor; + private Future future; + @PostConstruct public void init() { if (EntranceConfiguration.ENTRANCE_FAILOVER_ENABLED()) { @@ -69,85 +71,101 @@ public void init() { } } + @EventListener + private void shutdownFailover(ContextClosedEvent event) { + if (future != null && !future.isDone()) { + future.cancel(true); + } + if (scheduledExecutor != null) { + scheduledExecutor.shutdown(); + logger.info("Entrance Failover Server exit!"); + } + } + public void failoverTask() { - scheduledExecutor.scheduleWithFixedDelay( - () -> { - EntranceSchedulerContext schedulerContext = - (EntranceSchedulerContext) - entranceServer.getEntranceContext().getOrCreateScheduler().getSchedulerContext(); - - // entrance do not failover job when it is offline - if (schedulerContext.getOfflineFlag()) return; - - CommonLock commonLock = new CommonLock(); - commonLock.setLockObject(ENTRANCE_FAILOVER_LOCK); - Boolean locked = false; - try { - locked = commonLockService.lock(commonLock, 30 * 1000L); - if (!locked) return; - logger.info("success locked {}", ENTRANCE_FAILOVER_LOCK); - - // get all entrance server from eureka - ServiceInstance[] serviceInstances = - Sender.getInstances(Sender.getThisServiceInstance().getApplicationName()); - if (serviceInstances == null || serviceInstances.length <= 0) return; - - // serverInstance to map - Map serverInstanceMap = - Arrays.stream(serviceInstances) - .collect( - Collectors.toMap( - ServiceInstance::getInstance, - ServiceInstance::getRegistryTimestamp, - (k1, k2) -> k2)); - - // It is very important to avoid repeated execute job - // when failover self job, if self instance is empty, the job can be repeated execute - if (!serverInstanceMap.containsKey(Sender.getThisInstance())) { - logger.warn( - "server has just started and has not get self info, it does not failover"); - return; - } - - // get failover job expired time (获取任务故障转移过期时间，配置为0表示不过期, 过期则不处理) - long expiredTimestamp = 0L; - if (EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME() > 0) { - expiredTimestamp = - System.currentTimeMillis() - - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME(); - } - - // get uncompleted status - List statusList = - Arrays.stream(SchedulerEventState.uncompleteStatusArray()) - .map(Object::toString) - .collect(Collectors.toList()); - - List jobRequests = - JobHistoryHelper.queryWaitForFailoverTask( - serverInstanceMap, - statusList, - expiredTimestamp, - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); - if (jobRequests.isEmpty()) return; - List ids = - jobRequests.stream().map(JobRequest::getId).collect(Collectors.toList()); - logger.info("success query failover jobs , job size: {}, ids: {}", ids.size(), ids); - - // failover to local server - for (JobRequest jobRequest : jobRequests) { - entranceServer.failoverExecute(jobRequest); - } - logger.info("finished execute failover jobs, job ids: {}", ids); - - } catch (Exception e) { - logger.error("failover failed", e); - } finally { - if (locked) commonLockService.unlock(commonLock); - } - }, - EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INIT_TIME(), - EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INTERVAL(), - TimeUnit.MILLISECONDS); + future = + scheduledExecutor.scheduleWithFixedDelay( + () -> { + EntranceSchedulerContext schedulerContext = + (EntranceSchedulerContext) + entranceServer + .getEntranceContext() + .getOrCreateScheduler() + .getSchedulerContext(); + + // entrance do not failover job when it is offline + if (schedulerContext.getOfflineFlag()) return; + + CommonLock commonLock = new CommonLock(); + commonLock.setLockObject(ENTRANCE_FAILOVER_LOCK); + Boolean locked = false; + try { + locked = commonLockService.lock(commonLock, 30 * 1000L); + if (!locked) return; + logger.info("success locked {}", ENTRANCE_FAILOVER_LOCK); + + // get all entrance server from eureka + ServiceInstance[] serviceInstances = + Sender.getInstances(Sender.getThisServiceInstance().getApplicationName()); + if (serviceInstances == null || serviceInstances.length <= 0) return; + + // serverInstance to map + Map serverInstanceMap = + Arrays.stream(serviceInstances) + .collect( + Collectors.toMap( + ServiceInstance::getInstance, + ServiceInstance::getRegistryTimestamp, + (k1, k2) -> k2)); + + // It is very important to avoid repeated execute job + // when failover self job, if self instance is empty, the job can be repeated + // execute + if (!serverInstanceMap.containsKey(Sender.getThisInstance())) { + logger.warn( + "server has just started and has not get self info, it does not failover"); + return; + } + + // get failover job expired time (获取任务故障转移过期时间，配置为0表示不过期, 过期则不处理) + long expiredTimestamp = 0L; + if (EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME() > 0) { + expiredTimestamp = + System.currentTimeMillis() + - EntranceConfiguration.ENTRANCE_FAILOVER_DATA_INTERVAL_TIME(); + } + + // get uncompleted status + List statusList = + Arrays.stream(SchedulerEventState.uncompleteStatusArray()) + .map(Object::toString) + .collect(Collectors.toList()); + + List jobRequests = + JobHistoryHelper.queryWaitForFailoverTask( + serverInstanceMap, + statusList, + expiredTimestamp, + EntranceConfiguration.ENTRANCE_FAILOVER_DATA_NUM_LIMIT()); + if (jobRequests.isEmpty()) return; + List ids = + jobRequests.stream().map(JobRequest::getId).collect(Collectors.toList()); + logger.info("success query failover jobs , job size: {}, ids: {}", ids.size(), ids); + + // failover to local server + for (JobRequest jobRequest : jobRequests) { + entranceServer.failoverExecute(jobRequest); + } + logger.info("finished execute failover jobs, job ids: {}", ids); + + } catch (Exception e) { + logger.error("failover failed", e); + } finally { + if (locked) commonLockService.unlock(commonLock); + } + }, + EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INIT_TIME(), + EntranceConfiguration.ENTRANCE_FAILOVER_SCAN_INTERVAL(), + TimeUnit.MILLISECONDS); } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala index a6e24388a68..060fcbdd65a 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala @@ -30,10 +30,12 @@ import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer import org.apache.linkis.scheduler.queue.parallelqueue.{ParallelConsumerManager, ParallelGroup} import java.util -import java.util.concurrent.TimeUnit +import java.util.concurrent.{ScheduledFuture, TimeUnit} import scala.collection.JavaConverters._ +import com.sun.javafx.util.Logging + class EntranceParallelConsumerManager(maxParallelismUsers: Int, schedulerName: String) extends ParallelConsumerManager(maxParallelismUsers, schedulerName) { @@ -84,7 +86,11 @@ class EntranceParallelConsumerManager(maxParallelismUsers: Int, schedulerName: S group.setMaxAllowRunningJobs(maxAllowRunningJobs) logger .info( - s"group ${group.getGroupName} refresh maxAllowRunningJobs => ${group.getMaxRunningJobs}/$validInsCount=$maxAllowRunningJobs" + "group {} refresh maxAllowRunningJobs => {}/{}={}", + group.getGroupName, + group.getMaxRunningJobs, + validInsCount, + maxAllowRunningJobs ) case _ => } From 37567a86bbdac42952727497737fb6fc5f596843 Mon Sep 17 00:00:00 2001 From: guoshupei <15764973965@163.com> Date: Wed, 8 Mar 2023 11:41:03 +0800 Subject: [PATCH 049/689] Remove useless references --- .../entrance/scheduler/EntranceParallelConsumerManager.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala index 060fcbdd65a..d30f53a8f56 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala @@ -30,12 +30,10 @@ import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer import org.apache.linkis.scheduler.queue.parallelqueue.{ParallelConsumerManager, ParallelGroup} import java.util -import java.util.concurrent.{ScheduledFuture, TimeUnit} +import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ -import com.sun.javafx.util.Logging - class EntranceParallelConsumerManager(maxParallelismUsers: Int, schedulerName: String) extends ParallelConsumerManager(maxParallelismUsers, schedulerName) { From 9563457f8b44f6d307a79bc5b617916eccfc9164 Mon Sep 17 00:00:00 2001 From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com> Date: Wed, 8 Mar 2023 11:44:33 +0800 Subject: [PATCH 050/689] [feat] EngineConn no longer imports the dependencies of the underlying engine (#4278) * EngineConn no longer imports the dependencies of the underlying engine --- linkis-dist/release-docs/LICENSE | 8 +- .../elasticsearch/pom.xml | 4 - linkis-engineconn-plugins/flink/pom.xml | 11 -- linkis-engineconn-plugins/hive/pom.xml | 7 - linkis-engineconn-plugins/io_file/pom.xml | 10 -- .../io/executor/IoEngineConnExecutor.scala | 5 - linkis-engineconn-plugins/jdbc/pom.xml | 7 - linkis-engineconn-plugins/openlookeng/pom.xml | 4 - linkis-engineconn-plugins/pipeline/pom.xml | 10 -- linkis-engineconn-plugins/presto/pom.xml | 4 - linkis-engineconn-plugins/python/pom.xml | 10 -- linkis-engineconn-plugins/seatunnel/pom.xml | 3 - linkis-engineconn-plugins/shell/pom.xml | 10 -- linkis-engineconn-plugins/spark/pom.xml | 7 - .../spark/Interpreter/Interpreter.scala | 43 ----- .../Interpreter/ProcessInterpreter.scala | 125 -------------- .../spark/Interpreter/PythonInterpreter.scala | 155 +----------------- .../engineplugin/spark/common/SparkKind.scala | 23 --- .../spark/imexport/ExportData.scala | 13 +- .../spark/imexport/LoadData.scala | 33 +--- linkis-engineconn-plugins/sqoop/pom.xml | 4 - linkis-engineconn-plugins/trino/pom.xml | 4 +- linkis-hadoop-hdfs-client-shade/pom.xml | 2 +- pom.xml | 16 +- tool/dependencies/known-dependencies.txt | 8 +- 25 files changed, 41 insertions(+), 485 deletions(-) delete mode 100644 linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/Interpreter.scala delete mode 100644 linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/ProcessInterpreter.scala diff --git a/linkis-dist/release-docs/LICENSE b/linkis-dist/release-docs/LICENSE index 7fd83cb1c38..8fb5148924d 100644 --- a/linkis-dist/release-docs/LICENSE +++ b/linkis-dist/release-docs/LICENSE @@ -446,10 +446,10 @@ See licenses/ for text of these licenses. (Apache License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.13.4.1 - http://github.com/FasterXML/jackson) (Apache License, Version 2.0) jackson-module-scala (com.fasterxml.jackson.module:jackson-module-scala_2.11:2.13.4 - http://wiki.fasterxml.com/JacksonModuleScala) (Apache License, Version 2.0) javax.inject (javax.inject:javax.inject:1 - http://code.google.com/p/atinject/) - (Apache License, Version 2.0) json4s-ast (org.json4s:json4s-ast_2.11:3.5.3 - https://github.com/json4s/json4s) - (Apache License, Version 2.0) json4s-core (org.json4s:json4s-core_2.11:3.5.3 - https://github.com/json4s/json4s) - (Apache License, Version 2.0) json4s-jackson (org.json4s:json4s-jackson_2.11:3.5.3 - https://github.com/json4s/json4s) - (Apache License, Version 2.0) json4s-scalap (org.json4s:json4s-scalap_2.11:3.5.3 - https://github.com/json4s/json4s) + (Apache License, Version 2.0) json4s-ast (org.json4s:json4s-ast_2.11:3.7.0-M11 - https://github.com/json4s/json4s) + (Apache License, Version 2.0) json4s-core (org.json4s:json4s-core_2.11:3.7.0-M11 - https://github.com/json4s/json4s) + (Apache License, Version 2.0) json4s-jackson (org.json4s:json4s-jackson_2.11:3.7.0-M11 - https://github.com/json4s/json4s) + (Apache License, Version 2.0) json4s-scalap (org.json4s:json4s-scalap_2.11:3.7.0-M11 - https://github.com/json4s/json4s) (Apache License, Version 2.0) jna (net.java.dev.jna:jna:5.6.0 - https://github.com/java-native-access/jna) (Apache License, Version 2.0) jna-platform (net.java.dev.jna:jna-platform:5.6.0 - https://github.com/java-native-access/jna) (Apache License, Version 2.0) micrometer-core (io.micrometer:micrometer-core:1.3.1 - https://github.com/micrometer-metrics/micrometer) diff --git a/linkis-engineconn-plugins/elasticsearch/pom.xml b/linkis-engineconn-plugins/elasticsearch/pom.xml index 6b8cc7d1a04..cb54e5d77f7 100644 --- a/linkis-engineconn-plugins/elasticsearch/pom.xml +++ b/linkis-engineconn-plugins/elasticsearch/pom.xml @@ -26,10 +26,6 @@ linkis-engineplugin-elasticsearch - - 7.6.2 - - org.apache.linkis diff --git a/linkis-engineconn-plugins/flink/pom.xml b/linkis-engineconn-plugins/flink/pom.xml index 7fe87fb2d2b..ed9474e42c8 100644 --- a/linkis-engineconn-plugins/flink/pom.xml +++ b/linkis-engineconn-plugins/flink/pom.xml @@ -25,10 +25,6 @@ linkis-engineconn-plugin-flink - - 1.12.2 - 1.3.1 - @@ -414,13 +410,6 @@ provided - - org.json4s - json4s-jackson_${scala.binary.version} - ${json4s.version} - provided - - io.netty netty-all diff --git a/linkis-engineconn-plugins/hive/pom.xml b/linkis-engineconn-plugins/hive/pom.xml index 28b60fff0a8..74cc14314e2 100644 --- a/linkis-engineconn-plugins/hive/pom.xml +++ b/linkis-engineconn-plugins/hive/pom.xml @@ -312,13 +312,6 @@ provided - - org.json4s - json4s-jackson_${scala.binary.version} - ${json4s.version} - provided - - io.netty netty-all diff --git a/linkis-engineconn-plugins/io_file/pom.xml b/linkis-engineconn-plugins/io_file/pom.xml index 4aeddd125c1..cc8136e9667 100644 --- a/linkis-engineconn-plugins/io_file/pom.xml +++ b/linkis-engineconn-plugins/io_file/pom.xml @@ -25,9 +25,6 @@ linkis-engineplugin-io_file - - 1.0 - @@ -80,13 +77,6 @@ provided - - org.json4s - json4s-jackson_${scala.binary.version} - ${json4s.version} - provided - - io.netty netty-all diff --git a/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/executor/IoEngineConnExecutor.scala b/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/executor/IoEngineConnExecutor.scala index fbc4a77d120..ef9ba73b3b5 100644 --- a/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/executor/IoEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/executor/IoEngineConnExecutor.scala @@ -29,7 +29,6 @@ import org.apache.linkis.manager.common.entity.resource.{ LoadResource, NodeResource } -import org.apache.linkis.manager.engineplugin.common.conf.EngineConnPluginConf import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils import org.apache.linkis.manager.engineplugin.io.conf.IOEngineConnConfiguration import org.apache.linkis.manager.engineplugin.io.domain.FSInfo @@ -61,14 +60,10 @@ import java.util.concurrent.atomic.AtomicLong import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer -import org.json4s.DefaultFormats - class IoEngineConnExecutor(val id: Int, val outputLimit: Int = 10) extends ConcurrentComputationExecutor(outputLimit) with Logging { - implicit val formats = DefaultFormats - val fsIdCount = new AtomicLong() val FS_ID_LIMIT = IOEngineConnConfiguration.IO_FS_ID_LIMIT.getValue diff --git a/linkis-engineconn-plugins/jdbc/pom.xml b/linkis-engineconn-plugins/jdbc/pom.xml index b42cb1ae8db..bb4367308ec 100644 --- a/linkis-engineconn-plugins/jdbc/pom.xml +++ b/linkis-engineconn-plugins/jdbc/pom.xml @@ -172,13 +172,6 @@ provided - - org.json4s - json4s-jackson_${scala.binary.version} - ${json4s.version} - provided - - io.netty netty-all diff --git a/linkis-engineconn-plugins/openlookeng/pom.xml b/linkis-engineconn-plugins/openlookeng/pom.xml index 7f9f02f3aff..8ccc2b4dc52 100644 --- a/linkis-engineconn-plugins/openlookeng/pom.xml +++ b/linkis-engineconn-plugins/openlookeng/pom.xml @@ -27,10 +27,6 @@ linkis-engineplugin-openlookeng - - 1.5.0 - - org.apache.linkis diff --git a/linkis-engineconn-plugins/pipeline/pom.xml b/linkis-engineconn-plugins/pipeline/pom.xml index 4ff0bdf93a9..2f720f1e37d 100644 --- a/linkis-engineconn-plugins/pipeline/pom.xml +++ b/linkis-engineconn-plugins/pipeline/pom.xml @@ -25,9 +25,6 @@ linkis-engineplugin-pipeline - - 1 - @@ -80,13 +77,6 @@ provided - - org.json4s - json4s-jackson_${scala.binary.version} - ${json4s.version} - provided - - io.netty netty-all diff --git a/linkis-engineconn-plugins/presto/pom.xml b/linkis-engineconn-plugins/presto/pom.xml index 0394a0fcbf6..2b2d234d740 100644 --- a/linkis-engineconn-plugins/presto/pom.xml +++ b/linkis-engineconn-plugins/presto/pom.xml @@ -26,10 +26,6 @@ linkis-engineplugin-presto - - 0.234 - - org.apache.linkis diff --git a/linkis-engineconn-plugins/python/pom.xml b/linkis-engineconn-plugins/python/pom.xml index 6ede3100fca..6a9020dc8a8 100644 --- a/linkis-engineconn-plugins/python/pom.xml +++ b/linkis-engineconn-plugins/python/pom.xml @@ -25,9 +25,6 @@ linkis-engineplugin-python - - python2 - @@ -90,13 +87,6 @@ provided - - org.json4s - json4s-jackson_${scala.binary.version} - ${json4s.version} - provided - - io.netty netty-all diff --git a/linkis-engineconn-plugins/seatunnel/pom.xml b/linkis-engineconn-plugins/seatunnel/pom.xml index e831db2be41..ed843f9f2c2 100644 --- a/linkis-engineconn-plugins/seatunnel/pom.xml +++ b/linkis-engineconn-plugins/seatunnel/pom.xml @@ -25,9 +25,6 @@ linkis-engineplugin-seatunnel - - 2.1.2 - diff --git a/linkis-engineconn-plugins/shell/pom.xml b/linkis-engineconn-plugins/shell/pom.xml index b622d77567c..ad10c0c1a5b 100755 --- a/linkis-engineconn-plugins/shell/pom.xml +++ b/linkis-engineconn-plugins/shell/pom.xml @@ -25,9 +25,6 @@ linkis-engineplugin-shell - - 1 - @@ -98,13 +95,6 @@ provided - - org.json4s - json4s-jackson_${scala.binary.version} - ${json4s.version} - provided - - io.netty netty-all diff --git a/linkis-engineconn-plugins/spark/pom.xml b/linkis-engineconn-plugins/spark/pom.xml index eebadfbb458..c7993b28de2 100644 --- a/linkis-engineconn-plugins/spark/pom.xml +++ b/linkis-engineconn-plugins/spark/pom.xml @@ -159,13 +159,6 @@ provided - - org.json4s - json4s-jackson_${scala.binary.version} - ${json4s.version} - provided - - io.netty netty-all diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/Interpreter.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/Interpreter.scala deleted file mode 100644 index f6d6c797ec9..00000000000 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/Interpreter.scala +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.spark.Interpreter - -import org.apache.linkis.common.utils.Utils -import org.apache.linkis.engineplugin.spark.common.State -import org.apache.linkis.scheduler.executer.ExecuteResponse - -import scala.concurrent.TimeoutException -import scala.concurrent.duration.Duration - -/** - */ - -trait Interpreter { - def state: State - - def execute(code: String): ExecuteResponse - - def close(): Unit - - @throws(classOf[TimeoutException]) - @throws(classOf[InterruptedException]) - final def waitForStateChange(oldState: State, atMost: Duration): Unit = { - Utils.waitUntil({ () => state != oldState }, atMost) - } - -} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/ProcessInterpreter.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/ProcessInterpreter.scala deleted file mode 100644 index 171d48e4f19..00000000000 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/ProcessInterpreter.scala +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.spark.Interpreter - -import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.engineplugin.spark.common._ -import org.apache.linkis.scheduler.executer.{ - ErrorExecuteResponse, - ExecuteResponse, - SuccessExecuteResponse -} - -import org.apache.commons.io.IOUtils - -import java.io.{BufferedReader, InputStreamReader, PrintWriter} -import java.util.concurrent.TimeUnit - -import scala.concurrent.{Await, ExecutionContext, Future} -import scala.concurrent.duration.Duration - -import org.json4s._ - -/** - */ -abstract class ProcessInterpreter(process: Process) extends Interpreter with Logging { - - implicit val executor: ExecutionContext = ExecutionContext.global - - protected[this] var _state: State = Starting() - - protected[this] val stdin = new PrintWriter(process.getOutputStream) - - protected[this] val stdout = - new BufferedReader(new InputStreamReader(process.getInputStream()), 1) - - protected[this] val errOut = new LineBufferedStream(process.getErrorStream()) - - override def state: State = _state - - override def execute(code: String): ExecuteResponse = { - if (code == "sc.cancelAllJobs" || code == "sc.cancelAllJobs()") { - sendExecuteRequest(code) - } - _state match { - case (Dead() | ShuttingDown() | Error() | Success()) => - throw new IllegalStateException("interpreter is not running") - case Idle() => - require(state == Idle()) - code match { - case "SHUTDOWN" => - sendShutdownRequest() - close() - ErrorExecuteResponse("shutdown", new Exception("shutdown")) - case _ => - _state = Busy() - sendExecuteRequest(code) match { - case Some(rep) => - _state = Idle() - // ExecuteComplete(rep) - SuccessExecuteResponse() - case None => - _state = Error() - val errorMsg = errOut.lines.mkString(", ") - throw new Exception(errorMsg) - } - } - case _ => - throw new IllegalStateException(s"interpreter is in ${_state} state, cannot do query.") - } - } - - Future { - val exitCode = process.waitFor() - if (exitCode != 0) { - // scalastyle:off println - errOut.lines.foreach(println) - println(getClass.getSimpleName + " has stopped with exit code " + process.exitValue) - _state = Error() - } else { - println(getClass.getSimpleName + " has finished.") - _state = Success() - } - } - - protected def waitUntilReady(): Unit - - protected def sendExecuteRequest(request: String): Option[JValue] - - protected def sendShutdownRequest(): Unit = {} - - override def close(): Unit = { - val future = Future { - _state match { - case (Dead() | ShuttingDown() | Success()) => - Future.successful() - case _ => - sendShutdownRequest() - } - } - _state = Dead() - IOUtils.closeQuietly(stdin) - IOUtils.closeQuietly(stdout) - errOut.close - // scalastyle:off awaitresult - Utils.tryFinally(Await.result(future, Duration(10, TimeUnit.SECONDS))) { - process.destroy() - } - } - -} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/PythonInterpreter.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/PythonInterpreter.scala index 4223db8ba71..dbbac2623fb 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/PythonInterpreter.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/PythonInterpreter.scala @@ -17,178 +17,27 @@ package org.apache.linkis.engineplugin.spark.Interpreter -import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.common.io.FsPath import org.apache.linkis.common.utils.{ClassUtils, Logging, Utils} -import org.apache.linkis.engineplugin.spark.common.LineBufferedStream import org.apache.linkis.engineplugin.spark.config.SparkConfiguration import org.apache.linkis.storage.FSFactory import org.apache.commons.io.IOUtils -import org.apache.spark.{SparkContext, SparkException} +import org.apache.spark.SparkContext import org.apache.spark.sql.DataFrame import org.apache.spark.sql.catalyst.expressions.Attribute import java.io._ import java.nio.file.Files -import scala.collection.JavaConverters._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer -import org.json4s.{DefaultFormats, JValue} -import org.json4s.jackson.JsonMethods._ -import org.json4s.jackson.Serialization -import py4j.GatewayServer - /** */ object PythonInterpreter { - def create(): Interpreter = { - val pythonExec = CommonVars("PYSPARK_DRIVER_PYTHON", "python").getValue - - val gatewayServer = new GatewayServer(SQLSession, 0) - gatewayServer.start() - - val builder = new ProcessBuilder(Array(pythonExec, createFakeShell().toString).toList.asJava) - - val env = builder.environment() - env.put("PYTHONPATH", pythonPath) - env.put("PYTHONUNBUFFERED", "YES") - env.put("PYSPARK_GATEWAY_PORT", "" + gatewayServer.getListeningPort) - env.put("SPARK_HOME", SparkConfiguration.SPARK_HOME.getValue) - - val process = builder.start() - - new PythonInterpreter(process, gatewayServer) - } - - def pythonPath: String = { - val pythonPath = new ArrayBuffer[String] - val pythonHomePath = new File(SparkConfiguration.SPARK_HOME.getValue, "python").getPath - val pythonParentPath = new File(pythonHomePath, "lib") - pythonPath += pythonHomePath - pythonParentPath - .listFiles(new FileFilter { - override def accept(pathname: File): Boolean = pathname.getName.endsWith(".zip") - }) - .foreach(f => pythonPath += f.getPath) - ClassUtils.jarOfClass(classOf[SparkContext]).foreach(pythonPath += _) - pythonPath.mkString(File.pathSeparator) - } - - def createFakeShell(): File = createFakeShell("python/fake_shell.py") - - def createFakeShell(script: String, fileType: String = ".py"): File = { - val source: InputStream = getClass.getClassLoader.getResourceAsStream(script) - - val file = Files.createTempFile("", fileType).toFile - file.deleteOnExit() - - val sink = new FileOutputStream(file) - val buf = new Array[Byte](1024) - var n = source.read(buf) - - while (n > 0) { - sink.write(buf, 0, n) - n = source.read(buf) - } - - source.close() - sink.close() - - file - } - - private def createFakePySpark(): File = { - val source: InputStream = getClass.getClassLoader.getResourceAsStream("fake_pyspark.sh") - - val file = Files.createTempFile("", "").toFile - file.deleteOnExit() - - file.setExecutable(true) - - val sink = new FileOutputStream(file) - val buf = new Array[Byte](1024) - var n = source.read(buf) - - while (n > 0) { - sink.write(buf, 0, n) - n = source.read(buf) - } - - source.close() - sink.close() - - file - } - -} - -private class PythonInterpreter(process: Process, gatewayServer: GatewayServer) - extends ProcessInterpreter(process) - with Logging { - implicit val formats = DefaultFormats - - override def close(): Unit = { - try { - super.close() - } finally { - gatewayServer.shutdown() - } - } - - final override protected def waitUntilReady(): Unit = { - var running = false - val code = - try process.exitValue - catch { case t: IllegalThreadStateException => running = true; -1 } - if (!running) { - throw new SparkException( - s"Spark python application has already finished with exit code $code, now exit..." - ) - } - var continue = true - val initOut = new LineBufferedStream(process.getInputStream) - val iterable = initOut.iterator - while (continue && iterable.hasNext) { - iterable.next match { - // scalastyle:off println - case "READY" => println("Start python application succeed."); continue = false - case str: String => println(str) - case _ => - } - } - initOut.close - } - - override protected def sendExecuteRequest(code: String): Option[JValue] = { - val rep = sendRequest(Map("msg_type" -> "execute_request", "content" -> Map("code" -> code))) - rep.map { rep => - assert((rep \ "msg_type").extract[String] == "execute_reply") - - val content: JValue = rep \ "content" - - content - } - } - - override protected def sendShutdownRequest(): Unit = { - sendRequest(Map("msg_type" -> "shutdown_request", "content" -> ())).foreach { rep => - logger.warn(f"process failed to shut down while returning $rep") - } - } - - private def sendRequest(request: Map[String, Any]): Option[JValue] = { - // scalastyle:off println - stdin.println(Serialization.write(request)) - stdin.flush() - - Option(stdout.readLine()).map { line => parse(line) } - } - def pythonPath: String = { val pythonPath = new ArrayBuffer[String] val pythonHomePath = new File(SparkConfiguration.SPARK_HOME.getValue, "python").getPath @@ -296,7 +145,7 @@ object SQLSession extends Logging { logger.warn(s"Fetched $colCount col(s) : $index row(s).") sc.clearJobGroup() Utils.tryFinally({ - msg.flush(); + msg.flush() msg.toString }) { () => IOUtils.closeQuietly(msg) } } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/common/SparkKind.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/common/SparkKind.scala index 46e8b5defbf..26c8ea3fc9d 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/common/SparkKind.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/common/SparkKind.scala @@ -17,9 +17,6 @@ package org.apache.linkis.engineplugin.spark.common -import org.json4s.CustomSerializer -import org.json4s.JsonAST.JString - /** */ object SparkKind { @@ -91,23 +88,3 @@ case class SparkDataCalc() extends Kind { case class SparkMLSQL() extends Kind { override val toString = SparkKind.SPARKMLSQL_TYPE } - -case object SparkSessionKindSerializer - extends CustomSerializer[Kind](implicit formats => - ( - { - case JString(SparkKind.SPARKSCALA_TYPE) | JString(SparkKind.SCALA_LAN) => SparkScala() - case JString(SparkKind.PYSPARK_TYPE) | JString(SparkKind.PYTHON_LAN) | JString( - SparkKind.PYTHON_END - ) => - PySpark() - case JString(SparkKind.SPARKR_TYPE) | JString(SparkKind.R_LAN) => SparkR() - case JString(SparkKind.SPARKMIX_TYPE) | JString(SparkKind.MIX_TYPE) => SparkMix() - case JString(SparkKind.SQL_LAN) | JString(SparkKind.SPARKSQL_TYPE) => SparkSQL() - case JString(SparkKind.SPARKMLSQL_TYPE) | JString(SparkKind.ML_LAN) => SparkMLSQL() - }, - { case kind: Kind => - JString(kind.toString) - } - ) - ) diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/imexport/ExportData.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/imexport/ExportData.scala index 187277349dd..cbfb5195c97 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/imexport/ExportData.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/imexport/ExportData.scala @@ -20,22 +20,19 @@ package org.apache.linkis.engineplugin.spark.imexport import org.apache.linkis.common.utils.Logging import org.apache.linkis.engineplugin.spark.config.SparkConfiguration import org.apache.linkis.engineplugin.spark.imexport.util.BackGroundServiceUtils +import org.apache.linkis.server.BDPJettyServerHelper import org.apache.spark.sql.SparkSession -import org.json4s.{DefaultFormats, _} -import org.json4s.jackson.JsonMethods._ - /** */ object ExportData extends Logging { - implicit val formats = DefaultFormats def exportData(spark: SparkSession, dataInfo: String, destination: String): Unit = { exportDataFromFile( spark, - parse(dataInfo).extract[Map[String, Any]], - parse(destination).extract[Map[String, Any]] + BDPJettyServerHelper.gson.fromJson(dataInfo, classOf[Map[String, Any]]), + BDPJettyServerHelper.gson.fromJson(destination, classOf[Map[String, Any]]) ) } @@ -43,8 +40,8 @@ object ExportData extends Logging { val dataInfo = BackGroundServiceUtils.exchangeExecutionCode(dataInfoPath) exportDataFromFile( spark, - parse(dataInfo).extract[Map[String, Any]], - parse(destination).extract[Map[String, Any]] + BDPJettyServerHelper.gson.fromJson(dataInfo, classOf[Map[String, Any]]), + BDPJettyServerHelper.gson.fromJson(destination, classOf[Map[String, Any]]) ) } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/imexport/LoadData.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/imexport/LoadData.scala index 6d278175b7c..fd8e5cac500 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/imexport/LoadData.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/imexport/LoadData.scala @@ -22,6 +22,7 @@ import org.apache.linkis.engineplugin.spark.config.SparkConfiguration import org.apache.linkis.engineplugin.spark.imexport.util.{BackGroundServiceUtils, ImExportUtils} import org.apache.linkis.hadoop.common.conf.HadoopConf import org.apache.linkis.hadoop.common.utils.HDFSUtils +import org.apache.linkis.server.BDPJettyServerHelper import org.apache.linkis.storage.excel.XlsUtils import org.apache.commons.lang3.StringUtils @@ -35,26 +36,22 @@ import java.util.Locale import scala.collection.JavaConverters._ -import org.json4s._ -import org.json4s.jackson.JsonMethods._ - /** */ object LoadData { - implicit val formats = DefaultFormats def loadDataToTable(spark: SparkSession, source: String, destination: String): Unit = { - create_table_from_a_file(spark, parse(source), parse(destination)) + create_table_from_a_file(spark, source, destination) } def loadDataToTableByFile(spark: SparkSession, destinationPath: String, source: String): Unit = { val destination = BackGroundServiceUtils.exchangeExecutionCode(destinationPath) - create_table_from_a_file(spark, parse(source), parse(destination)) + create_table_from_a_file(spark, source, destination) } - def create_table_from_a_file(spark: SparkSession, src: JValue, dest: JValue): Unit = { - val source = src.extract[Map[String, Any]] - val destination = dest.extract[Map[String, Any]] + def create_table_from_a_file(spark: SparkSession, src: String, dest: String): Unit = { + val source = BDPJettyServerHelper.gson.fromJson(src, classOf[Map[String, Any]]) + val destination = BDPJettyServerHelper.gson.fromJson(src, classOf[Map[String, Any]]) var path = getMapValue[String](source, "path") val pathType = getMapValue[String](source, "pathType", "share") @@ -79,7 +76,9 @@ object LoadData { val partition = getMapValue[String](destination, "partition", "ds") val partitionValue = getMapValue[String](destination, "partitionValue", "1993-01-02") - val columns = (dest \ "columns").extract[List[Map[String, Any]]] + val columnsJson = getMapValue[String](destination, "columns", "") + val columns = BDPJettyServerHelper.gson.fromJson(columnsJson, classOf[List[Map[String, Any]]]) + val dateFormats = columns.map(_.get("dateFormat").get.toString).map(f => if (f isEmpty) "yyyy-MM-dd" else f) var isFirst = true @@ -204,20 +203,6 @@ object LoadData { hdfsPath } - def getNodeValue[T](json: JValue, node: String, default: T = null.asInstanceOf[T])(implicit - m: Manifest[T] - ): T = { - json \ node match { - case JNothing => default - case value: JValue => - if ("JString()".equals(value.toString)) default - else { - try value.extract[T] - catch { case t: Throwable => default } - } - } - } - def getMapValue[T](map: Map[String, Any], key: String, default: T = null.asInstanceOf[T]): T = { val value = map.get(key).map(_.asInstanceOf[T]).getOrElse(default) if (StringUtils.isEmpty(value.toString)) { diff --git a/linkis-engineconn-plugins/sqoop/pom.xml b/linkis-engineconn-plugins/sqoop/pom.xml index 5428047fe60..96ca23cd8da 100644 --- a/linkis-engineconn-plugins/sqoop/pom.xml +++ b/linkis-engineconn-plugins/sqoop/pom.xml @@ -25,10 +25,6 @@ linkis-engineplugin-sqoop - - 1.4.6 - 3.1.2 - diff --git a/linkis-engineconn-plugins/trino/pom.xml b/linkis-engineconn-plugins/trino/pom.xml index 246f547511e..d9ea2d6868f 100644 --- a/linkis-engineconn-plugins/trino/pom.xml +++ b/linkis-engineconn-plugins/trino/pom.xml @@ -25,9 +25,7 @@ linkis-engineplugin-trino - - 371 - + org.apache.linkis diff --git a/linkis-hadoop-hdfs-client-shade/pom.xml b/linkis-hadoop-hdfs-client-shade/pom.xml index e4990f857ed..560bbaa698d 100644 --- a/linkis-hadoop-hdfs-client-shade/pom.xml +++ b/linkis-hadoop-hdfs-client-shade/pom.xml @@ -207,7 +207,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.3.0 + ${maven-shade-plugin.version} true false diff --git a/pom.xml b/pom.xml index 8549ec8e9a4..0d4170613c2 100644 --- a/pom.xml +++ b/pom.xml @@ -105,6 +105,18 @@ 1.3.2-SNAPSHOT 2.9.2 + 7.6.2 + 1.12.2 + 1.0 + 4 + 371 + 1.5.0 + 1 + 0.234 + python2 + 2.1.2 + 1 + 1.4.6 3.2.1 3.1.3 3.3.4 @@ -124,7 +136,6 @@ 2.8.9 2.13.4.20221013 - 3.7.0-M11 1.19.4 @@ -141,6 +152,7 @@ 8.0.28 1.1.22 3.27.0-GA + 1.3.1 3.2.2 2.6 3.12.0 @@ -201,6 +213,8 @@ ${java.version} ${java.version} 3.3.0 + 3.3.0 + 3.2.0 4.7.1 2.24.1 0.8.7 diff --git a/tool/dependencies/known-dependencies.txt b/tool/dependencies/known-dependencies.txt index 3c4cbf9d338..55f0163aa27 100644 --- a/tool/dependencies/known-dependencies.txt +++ b/tool/dependencies/known-dependencies.txt @@ -311,10 +311,10 @@ jpam-1.1.jar json-0.191.jar json-0.193.jar json-1.8.jar -json4s-ast_2.11-3.5.3.jar -json4s-core_2.11-3.5.3.jar -json4s-jackson_2.11-3.5.3.jar -json4s-scalap_2.11-3.5.3.jar +json4s-ast_2.11-3.7.0-M11.jar +json4s-core_2.11-3.7.0-M11.jar +json4s-jackson_2.11-3.7.0-M11.jar +json4s-scalap_2.11-3.7.0-M11.jar jsp-api-2.1.jar jsqlparser-1.0.jar jsqlparser-4.2.jar From 72d84048a9849d0691c5c59ed89697f34f0fba88 Mon Sep 17 00:00:00 2001 From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com> Date: Wed, 8 Mar 2023 11:49:23 +0800 Subject: [PATCH 051/689] give more exception in CustomerDelimitedJSONSerDe.java (#4341) --- .../hive/serde/CustomerDelimitedJSONSerDe.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java b/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java index 9c425ff0575..fe948f79523 100644 --- a/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java +++ b/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java @@ -320,12 +320,20 @@ private static void writePrimitiveUTF8( } default: { + if (!"INTERVAL_YEAR_MONTH".equals(category.name()) + && !"INTERVAL_DAY_TIME".equals(category.name())) { + throw new RuntimeException("Unknown primitive type: " + category); + } boolean containsIntervalYearMonth = false; boolean containsIntervalDayTime = false; for (PrimitiveObjectInspector.PrimitiveCategory primitiveCategory : PrimitiveObjectInspector.PrimitiveCategory.values()) { - containsIntervalYearMonth = "INTERVAL_YEAR_MONTH".equals(primitiveCategory.name()); - containsIntervalDayTime = "INTERVAL_DAY_TIME".equals(primitiveCategory.name()); + containsIntervalYearMonth = + "INTERVAL_YEAR_MONTH".equals(primitiveCategory.name()) + && "INTERVAL_YEAR_MONTH".equals(category.name()); + containsIntervalDayTime = + "INTERVAL_DAY_TIME".equals(primitiveCategory.name()) + && "INTERVAL_DAY_TIME".equals(category.name()); try { if (containsIntervalYearMonth) { wc = From 71d3e089dcd455027bc36026711e26a4abeb8f40 Mon Sep 17 00:00:00 2001 From: guoshupei <15764973965@163.com> Date: Wed, 8 Mar 2023 12:36:25 +0800 Subject: [PATCH 052/689] cast string when use logger template --- .../scheduler/EntranceParallelConsumerManager.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala index d30f53a8f56..726d93c5004 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala @@ -86,9 +86,9 @@ class EntranceParallelConsumerManager(maxParallelismUsers: Int, schedulerName: S .info( "group {} refresh maxAllowRunningJobs => {}/{}={}", group.getGroupName, - group.getMaxRunningJobs, - validInsCount, - maxAllowRunningJobs + group.getMaxRunningJobs.toString, + validInsCount.toString, + maxAllowRunningJobs.toString ) case _ => } From 2c5abd7a0504db7b2ef5c34a2e43257c66d783c0 Mon Sep 17 00:00:00 2001 From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com> Date: Wed, 8 Mar 2023 14:41:49 +0800 Subject: [PATCH 053/689] [feat]Support spark3.3+ and spark2.2- compile (#4301) * Support spark3.3+ and spark2.2- compile * fix unknown dependency * improve code * remove unused dependency * remove unused dependency for hive sink --------- Co-authored-by: gf13871 --- linkis-engineconn-plugins/spark/pom.xml | 10 ++++++++++ .../spark/datacalc/sink/HiveSink.scala | 16 ++++++++-------- .../spark/datacalc/sink/JdbcSink.scala | 16 ++++++++++++---- tool/dependencies/known-dependencies.txt | 1 + 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/linkis-engineconn-plugins/spark/pom.xml b/linkis-engineconn-plugins/spark/pom.xml index c7993b28de2..a5e7523d0b6 100644 --- a/linkis-engineconn-plugins/spark/pom.xml +++ b/linkis-engineconn-plugins/spark/pom.xml @@ -180,12 +180,22 @@ linkis-rpc ${project.version} + + net.sf.py4j + py4j + 0.10.7 + provided + org.apache.spark spark-core_${scala.binary.version} ${spark.version} provided + + net.sf.py4j + py4j + org.apache.hadoop hadoop-common diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/HiveSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/HiveSink.scala index 8ba618776b9..1a81d6537ee 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/HiveSink.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/HiveSink.scala @@ -24,14 +24,11 @@ import org.apache.linkis.engineplugin.spark.errorcode.SparkErrorCodeSummary import org.apache.commons.lang3.StringUtils import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.catalog.HiveTableRelation import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.sql.functions.col import org.apache.spark.sql.sources.DataSourceRegister import org.apache.spark.sql.types.StructField -import org.slf4j.{Logger, LoggerFactory} - class HiveSink extends DataCalcSink[HiveSinkConfig] with Logging { def output(spark: SparkSession, ds: Dataset[Row]): Unit = { @@ -122,7 +119,9 @@ class HiveSink extends DataCalcSink[HiveSinkConfig] with Logging { logFields(sourceFields, targetFields) throw new HiveSinkException( SparkErrorCodeSummary.DATA_CALC_COLUMN_NUM_NOT_MATCH.getErrorCode, - s"$targetTable requires that the data to be inserted have the same number of columns as the target table: target table has ${targetFields.length} column(s) but the inserted data has ${sourceFields.length} column(s)" + s"$targetTable requires that the data to be inserted have the same number of columns " + + s"as the target table: target table has ${targetFields.length} column(s) " + + s"but the inserted data has ${sourceFields.length} column(s)" ) } @@ -184,17 +183,18 @@ class HiveSink extends DataCalcSink[HiveSinkConfig] with Logging { logicalRelation.relation match { case hadoopFsRelation: HadoopFsRelation => hadoopFsRelation.fileFormat match { - case _: org.apache.spark.sql.execution.datasources.orc.OrcFileFormat => - fileFormat = FileFormat.ORC case _: org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat => fileFormat = FileFormat.PARQUET case dataSourceRegister: DataSourceRegister => fileFormat = FileFormat.withName(dataSourceRegister.shortName.toUpperCase) case _ => + if (hadoopFsRelation.fileFormat.getClass.getSimpleName.equals("OrcFileFormat")) { + fileFormat = FileFormat.ORC + } } } - case hiveTableRelation: HiveTableRelation => - // todo + // case hiveTableRelation: HiveTableRelation => + // todo please note `HiveTableRelation` was added after spark 2.2.1 } fileFormat } catch { diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/JdbcSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/JdbcSink.scala index ab8a21c3f7b..e9d60bd2b39 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/JdbcSink.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/JdbcSink.scala @@ -17,14 +17,16 @@ package org.apache.linkis.engineplugin.spark.datacalc.sink +import org.apache.linkis.common.utils.ClassUtils.getFieldVal import org.apache.linkis.common.utils.Logging import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSink import org.apache.commons.lang3.StringUtils +import org.apache.spark.SPARK_VERSION import org.apache.spark.sql.{Dataset, Row, SparkSession} -import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JdbcUtils} +import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions -import java.sql.Connection +import java.sql.{Connection, DriverManager} import scala.collection.JavaConverters._ @@ -58,7 +60,8 @@ class JdbcSink extends DataCalcSink[JdbcSinkConfig] with Logging { .repartition(1) .foreachPartition((_: Iterator[Row]) => { val jdbcOptions = new JDBCOptions(options) - val conn: Connection = JdbcUtils.createConnectionFactory(jdbcOptions)() + val conn: Connection = + DriverManager.getConnection(config.getUrl, config.getUser, config.getPassword) try { config.getPreQueries.asScala.foreach(query => { logger.info(s"Execute pre query: $query") @@ -86,7 +89,12 @@ class JdbcSink extends DataCalcSink[JdbcSinkConfig] with Logging { logger.info("Execute query: {}", query) val statement = conn.prepareStatement(query) try { - statement.setQueryTimeout(jdbcOptions.queryTimeout) + // `queryTimeout` was added after spark2.4.0, more details please check SPARK-23856 + if (SPARK_VERSION >= "2.4") { + val queryTimeout = getFieldVal(jdbcOptions, "queryTimeout").asInstanceOf[Int] + statement.setQueryTimeout(queryTimeout) + } + val rows = statement.executeUpdate() logger.info("{} rows affected", rows) } catch { diff --git a/tool/dependencies/known-dependencies.txt b/tool/dependencies/known-dependencies.txt index 55f0163aa27..5b3b459d682 100644 --- a/tool/dependencies/known-dependencies.txt +++ b/tool/dependencies/known-dependencies.txt @@ -434,6 +434,7 @@ protostuff-collectionschema-1.6.2.jar protostuff-core-1.6.2.jar protostuff-runtime-1.6.2.jar py4j-0.10.4.jar +py4j-0.10.7.jar quartz-2.3.2.jar reactive-streams-1.0.3.jar reactor-core-3.3.17.RELEASE.jar From 8ae8a3de4b3d2741459adcd63d309ff4e54dfbc5 Mon Sep 17 00:00:00 2001 From: guoshupei <15764973965@163.com> Date: Wed, 8 Mar 2023 15:29:07 +0800 Subject: [PATCH 054/689] use logger template --- .../scheduler/EntranceParallelConsumerManager.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala index 726d93c5004..afaf6b16e78 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala @@ -85,10 +85,12 @@ class EntranceParallelConsumerManager(maxParallelismUsers: Int, schedulerName: S logger .info( "group {} refresh maxAllowRunningJobs => {}/{}={}", - group.getGroupName, - group.getMaxRunningJobs.toString, - validInsCount.toString, - maxAllowRunningJobs.toString + Array( + group.getGroupName, + group.getMaxRunningJobs, + validInsCount, + maxAllowRunningJobs + ) ) case _ => } From cb048534aa2a7180365b3c1f15b3cacc053c461e Mon Sep 17 00:00:00 2001 From: guoshupei <15764973965@163.com> Date: Wed, 8 Mar 2023 16:47:07 +0800 Subject: [PATCH 055/689] use logger template --- .../entrance/scheduler/EntranceParallelConsumerManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala index afaf6b16e78..5e74d489395 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala @@ -90,7 +90,7 @@ class EntranceParallelConsumerManager(maxParallelismUsers: Int, schedulerName: S group.getMaxRunningJobs, validInsCount, maxAllowRunningJobs - ) + ): _* ) case _ => } From 800074e400c834735fb04299f4422d35803cc3bf Mon Sep 17 00:00:00 2001 From: guoshupei <15764973965@163.com> Date: Wed, 8 Mar 2023 18:01:33 +0800 Subject: [PATCH 056/689] use logger template --- .../scheduler/EntranceParallelConsumerManager.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala index 5e74d489395..6d756ad1a84 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala @@ -87,9 +87,9 @@ class EntranceParallelConsumerManager(maxParallelismUsers: Int, schedulerName: S "group {} refresh maxAllowRunningJobs => {}/{}={}", Array( group.getGroupName, - group.getMaxRunningJobs, - validInsCount, - maxAllowRunningJobs + group.getMaxRunningJobs.toString, + validInsCount.toString, + maxAllowRunningJobs.toString ): _* ) case _ => From 1d9c9a30146c2feb5c48b1ea6d418fa39ab9987a Mon Sep 17 00:00:00 2001 From: binbincheng <106590848+binbinCheng@users.noreply.github.com> Date: Tue, 14 Mar 2023 17:59:01 +0800 Subject: [PATCH 057/689] Optimization of upload file interface in FsRestfulApi.java (#4357) --- linkis-dist/package/conf/linkis.properties | 4 +++- .../linkis/filesystem/restful/api/FsRestfulApi.java | 4 ++-- .../apache/linkis/filesystem/util/WorkspaceUtil.java | 10 +++++++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/linkis-dist/package/conf/linkis.properties b/linkis-dist/package/conf/linkis.properties index 9116486b0de..66ed15cba38 100644 --- a/linkis-dist/package/conf/linkis.properties +++ b/linkis-dist/package/conf/linkis.properties @@ -87,4 +87,6 @@ linkis.session.redis.port=6379 # redis password linkis.session.redis.password=test123 # redis sso switch -linkis.session.redis.cache.enabled=false \ No newline at end of file +linkis.session.redis.cache.enabled=false +wds.linkis.workspace.filesystem.owner.check=true +wds.linkis.workspace.filesystem.path.check=true \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 91a8323dbab..e0fd3ea121f 100644 --- a/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -319,8 +319,8 @@ public Message upload( FileSystem fileSystem = fsService.getFileSystem(userName, fsPath); for (MultipartFile p : files) { String fileName = p.getOriginalFilename(); + WorkspaceUtil.charCheckFileName(fileName); FsPath fsPathNew = new FsPath(fsPath.getPath() + "/" + fileName); - WorkspaceUtil.fileAndDirNameSpecialCharCheck(fsPathNew.getPath()); fileSystem.createNewFile(fsPathNew); try (InputStream is = p.getInputStream(); OutputStream outputStream = fileSystem.write(fsPathNew, true)) { @@ -442,7 +442,7 @@ public void download( // downloaded(判断目录,目录不能下载) FileSystem fileSystem = fsService.getFileSystem(userName, fsPath); if (!fileSystem.exists(fsPath)) { - throw WorkspaceExceptionManager.createException(8011, path); + throw WorkspaceExceptionManager.createException(80011, path); } inputStream = fileSystem.read(fsPath); byte[] buffer = new byte[1024]; diff --git a/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/util/WorkspaceUtil.java b/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/util/WorkspaceUtil.java index a21dc57cbf6..738167fc272 100644 --- a/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/util/WorkspaceUtil.java +++ b/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/util/WorkspaceUtil.java @@ -84,14 +84,18 @@ public static String suffixTuning(String path) { public static void fileAndDirNameSpecialCharCheck(String path) throws WorkSpaceException { String name = new File(path).getName(); - int i = name.lastIndexOf("."); + charCheckFileName(name); + } + + public static void charCheckFileName(String fileName) throws WorkSpaceException { + int i = fileName.lastIndexOf("."); if (i != -1) { - name = name.substring(0, i); + fileName = fileName.substring(0, i); } // Only support numbers, uppercase letters, underscores, Chinese(只支持数字,字母大小写,下划线,中文) String specialRegEx = "^[\\w\\u4e00-\\u9fa5]{1,200}$"; Pattern specialPattern = Pattern.compile(specialRegEx); - if (!specialPattern.matcher(name).find()) { + if (!specialPattern.matcher(fileName).find()) { WorkspaceExceptionManager.createException(80028); } } From 581ce1e6778d96b126d9318b8c27ed0c55903f2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=B9=BF?= Date: Thu, 16 Mar 2023 19:57:15 +0800 Subject: [PATCH 058/689] [fix bug] The s3a file cannot be written because FileSystem is closed prematurely (#4375) https://github.com/apache/linkis/issues/4374 --- .../linkis/storage/resultset/StorageResultSetWriter.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala index 9c7947272c0..50bc0d276b4 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala @@ -187,11 +187,11 @@ class StorageResultSetWriter[K <: MetaData, V <: Record]( } } Utils.tryFinally(if (outputStream != null) flush()) { - closeFs if (outputStream != null) { IOUtils.closeQuietly(outputStream) outputStream = null } + closeFs } } From 139aa82f17f0d0d2a5357a848f27812c7e19edf9 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Sun, 19 Mar 2023 13:11:08 +0800 Subject: [PATCH 059/689] control panel bug fix (#4380) * show udfType and datasourceType page * kafka datasource add default env * kafka datasource add default env * data duplicate * add kafka linkis_ps_dm_datasource_type --------- Co-authored-by: aiceflower --- linkis-dist/package/db/linkis_dml.sql | 8 ++++++++ linkis-web/src/apps/linkis/view/linkis/index.vue | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/linkis-dist/package/db/linkis_dml.sql b/linkis-dist/package/db/linkis_dml.sql index dffa9a7d767..f9c0f34b639 100644 --- a/linkis-dist/package/db/linkis_dml.sql +++ b/linkis-dist/package/db/linkis_dml.sql @@ -571,6 +571,12 @@ INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (@data_source_type_id, 'envId', '集群环境(Cluster env)', 'Cluster env', NULL, 'SELECT', NULL, 1, '集群环境(Cluster env)', 'Cluster env', NULL, NULL, NULL, @data_source, now(), now()); +select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'kafka'; +SET @data_source=CONCAT('/data-source-manager/env-list/all/type/',@data_source_type_id); +INSERT INTO `linkis_ps_dm_datasource_type_key` + (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) +VALUES (@data_source_type_id, 'envId', '集群环境(Cluster env)', 'Cluster env', NULL, 'SELECT', NULL, 1, '集群环境(Cluster env)', 'Cluster env', NULL, NULL, NULL, @data_source, now(), now()); + select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'elasticsearch'; INSERT INTO `linkis_ps_dm_datasource_type_key` @@ -715,3 +721,5 @@ select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `nam INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_type_id`, `parameter`, `create_time`, `create_user`, `modify_time`, `modify_user`) VALUES ('测试环境SIT', '测试环境SIT', @data_source_type_id, '{"uris":"thrift://localhost:9083", "hadoopConf":{"hive.metastore.execute.setugi":"true"}}', now(), NULL, now(), NULL); INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_type_id`, `parameter`, `create_time`, `create_user`, `modify_time`, `modify_user`) VALUES ('测试环境UAT', '测试环境UAT', @data_source_type_id, '{"uris":"thrift://localhost:9083", "hadoopConf":{"hive.metastore.execute.setugi":"true"}}', now(), NULL, now(), NULL); +select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'kafka'; +INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_type_id`, `parameter`, `create_time`, `create_user`, `modify_time`, `modify_user`) VALUES ('kafka测试环境SIT', '开源测试环境SIT', @data_source_type_id, '{"uris":"thrift://localhost:9092"}', now(), NULL, now(), NULL); diff --git a/linkis-web/src/apps/linkis/view/linkis/index.vue b/linkis-web/src/apps/linkis/view/linkis/index.vue index 88e5088af5a..1721688f322 100644 --- a/linkis-web/src/apps/linkis/view/linkis/index.vue +++ b/linkis-web/src/apps/linkis/view/linkis/index.vue @@ -203,7 +203,7 @@ export default { ), path: '/console/datasourceEnv', }, - // {key: '1-8-3', name: this.$t('message.linkis.sideNavList.function.children.datasourceType'), path: '/console/datasourceType' }, + {key: '1-8-3', name: this.$t('message.linkis.sideNavList.function.children.datasourceType'), path: '/console/datasourceType' }, // {key: '1-8-4', name: this.$t('message.linkis.sideNavList.function.children.datasourceAccess'), path: '/console/datasourceAccess' }, { key: '1-8-5', @@ -285,7 +285,7 @@ export default { path: '/console/urm/functionManagement', }, // {key: '1-9-3', name: this.$t('message.linkis.sideNavList.function.children.udfManager'), path: '/console/udfManager' }, - // {key: '1-9-4', name: this.$t('message.linkis.sideNavList.function.children.udfTree'), path: '/console/udfTree' }, + {key: '1-9-4', name: this.$t('message.linkis.sideNavList.function.children.udfTree'), path: '/console/udfTree' }, ], }, breadcrumbSecondName: this.$t( From ebe52f6d155643674c51ecd82eccab7094e605b6 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Mon, 20 Mar 2023 14:13:34 +0800 Subject: [PATCH 060/689] mysql connect error (#4382) * mysql connect error * restore datasource type key * adjust the code location --------- Co-authored-by: aiceflower --- .../apache/linkis/common/utils/SecurityUtils.java | 3 +++ linkis-dist/package/db/linkis_dml.sql | 13 ++++++------- .../db/upgrade/1.3.2_schema/mysql/linkis_dml.sql | 6 ------ .../metadata/query/service/mysql/SqlConnection.java | 6 ++++++ 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java index f7158b4899b..2e77a14c0e9 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java @@ -176,6 +176,9 @@ public static String parseParamsMapToMysqlParamUrl(Map forcePara } private static Map parseMysqlUrlParamsToMap(String paramsUrl) { + if (StringUtils.isBlank(paramsUrl)) { + return new LinkedHashMap<>(); + } String[] params = paramsUrl.split(AND_SYMBOL); Map map = new LinkedHashMap<>(params.length); for (String param : params) { diff --git a/linkis-dist/package/db/linkis_dml.sql b/linkis-dist/package/db/linkis_dml.sql index f9c0f34b639..facbf32603c 100644 --- a/linkis-dist/package/db/linkis_dml.sql +++ b/linkis-dist/package/db/linkis_dml.sql @@ -649,7 +649,7 @@ VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL (@data_source_type_id, 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 1, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), - (@data_source_type_id, 'databaseName', '数据库名(Database name)', 'Database name', NULL, 'TEXT', NULL, 0, '数据库名(Database name)', 'Database name', NULL, NULL, NULL, NULL, now(), now()); + (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()); -- https://learn.microsoft.com/zh-cn/sql/connect/jdbc/building-the-connection-url?redirectedfrom=MSDN&view=sql-server-ver16 select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'sqlserver'; @@ -662,8 +662,7 @@ VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL (@data_source_type_id, 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 1, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), - (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()), - (@data_source_type_id, 'databaseName', '数据库名(Database name)', 'Database name', NULL, 'TEXT', NULL, 0, '数据库名(Database name)', 'Database name', NULL, NULL, NULL, NULL, now(), now()); + (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()); -- https://www.ibm.com/docs/en/db2/11.5?topic=cdsudidsdjs-url-format-data-server-driver-jdbc-sqlj-type-4-connectivity select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'db2'; @@ -676,7 +675,7 @@ VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL (@data_source_type_id, 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 1, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), - (@data_source_type_id, 'databaseName', '数据库名(Database name)', 'Database name', NULL, 'TEXT', NULL, 0, '数据库名(Database name)', 'Database name', NULL, NULL, NULL, NULL, now(), now()); + (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()); -- https://greenplum.docs.pivotal.io/6-1/datadirect/datadirect_jdbc.html#topic_ylk_pbx_2bb select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'greenplum'; @@ -689,7 +688,7 @@ VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL (@data_source_type_id, 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 1, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), - (@data_source_type_id, 'databaseName', '数据库名(Database name)', 'Database name', NULL, 'TEXT', NULL, 0, '数据库名(Database name)', 'Database name', NULL, NULL, NULL, NULL, now(), now()); + (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()); select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'doris'; INSERT INTO `linkis_ps_dm_datasource_type_key` @@ -701,7 +700,7 @@ VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL (@data_source_type_id, 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 1, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), - (@data_source_type_id, 'databaseName', '数据库名(Database name)', 'Database name', NULL, 'TEXT', NULL, 0, '数据库名(Database name)', 'Database name', NULL, NULL, NULL, NULL, now(), now()); + (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()); -- https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-jdbc select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'clickhouse'; @@ -714,7 +713,7 @@ VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL (@data_source_type_id, 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 1, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), - (@data_source_type_id, 'databaseName', '数据库名(Database name)', 'Database name', NULL, 'TEXT', NULL, 0, '数据库名(Database name)', 'Database name', NULL, NULL, NULL, NULL, now(), now()); + (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()); select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'hive'; diff --git a/linkis-dist/package/db/upgrade/1.3.2_schema/mysql/linkis_dml.sql b/linkis-dist/package/db/upgrade/1.3.2_schema/mysql/linkis_dml.sql index ea07c3ee30f..25e3ad2710b 100644 --- a/linkis-dist/package/db/upgrade/1.3.2_schema/mysql/linkis_dml.sql +++ b/linkis-dist/package/db/upgrade/1.3.2_schema/mysql/linkis_dml.sql @@ -267,27 +267,21 @@ INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `n select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'postgresql'; INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL, 0, '地址(host1:port1,host2:port2...)', 'Address(host1:port1,host2:port2...)', NULL, NULL, NULL, NULL, now(), now()); -INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (@data_source_type_id, 'databaseName', '数据库名(Database name)', 'Database name', NULL, 'TEXT', NULL, 0, '数据库名(Database name)', 'Database name', NULL, NULL, NULL, NULL, now(), now()); select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'sqlserver'; INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL, 0, '地址(host1:port1,host2:port2...)', 'Address(host1:port1,host2:port2...)', NULL, NULL, NULL, NULL, now(), now()); -INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (@data_source_type_id, 'databaseName', '数据库名(Database name)', 'Database name', NULL, 'TEXT', NULL, 0, '数据库名(Database name)', 'Database name', NULL, NULL, NULL, NULL, now(), now()); select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'db2'; INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL, 0, '地址(host1:port1,host2:port2...)', 'Address(host1:port1,host2:port2...)', NULL, NULL, NULL, NULL, now(), now()); -INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (@data_source_type_id, 'databaseName', '数据库名(Database name)', 'Database name', NULL, 'TEXT', NULL, 0, '数据库名(Database name)', 'Database name', NULL, NULL, NULL, NULL, now(), now()); select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'greenplum'; INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL, 0, '地址(host1:port1,host2:port2...)', 'Address(host1:port1,host2:port2...)', NULL, NULL, NULL, NULL, now(), now()); -INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (@data_source_type_id, 'databaseName', '数据库名(Database name)', 'Database name', NULL, 'TEXT', NULL, 0, '数据库名(Database name)', 'Database name', NULL, NULL, NULL, NULL, now(), now()); select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'doris'; INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL, 0, '地址(host1:port1,host2:port2...)', 'Address(host1:port1,host2:port2...)', NULL, NULL, NULL, NULL, now(), now()); -INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (@data_source_type_id, 'databaseName', '数据库名(Database name)', 'Database name', NULL, 'TEXT', NULL, 0, '数据库名(Database name)', 'Database name', NULL, NULL, NULL, NULL, now(), now()); select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'clickhouse'; INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL, 0, '地址(host1:port1,host2:port2...)', 'Address(host1:port1,host2:port2...)', NULL, NULL, NULL, NULL, now(), now()); -INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (@data_source_type_id, 'databaseName', '数据库名(Database name)', 'Database name', NULL, 'TEXT', NULL, 0, '数据库名(Database name)', 'Database name', NULL, NULL, NULL, NULL, now(), now()); INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('mongodb', 'default', 'default', 'DEFAULT', NULL, 3); diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java index 0b5099cce25..b5a3a54faf5 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java @@ -21,6 +21,8 @@ import org.apache.linkis.common.utils.SecurityUtils; import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; +import org.apache.commons.lang3.StringUtils; + import java.io.Closeable; import java.io.IOException; import java.sql.*; @@ -221,6 +223,10 @@ private Connection getDBConnection(ConnectMessage connectMessage, String databas String url = String.format( SQL_CONNECT_URL.getValue(), connectMessage.host, connectMessage.port, database); + // deal with empty database + if (StringUtils.isBlank(database)) { + url = url.substring(0, url.length() - 1); + } if (!connectMessage.extraParams.isEmpty()) { url += "?" + extraParamString; } From c74b811b3da747c3bfd5358035d0ee4ab0e4a6fe Mon Sep 17 00:00:00 2001 From: Alexyang Date: Mon, 20 Mar 2023 20:33:18 +0800 Subject: [PATCH 061/689] add metaspace config (#4379) * engineconn-plugin-core - add config for MaxMetaspaceSize * Merge branch 'dev-1.3.2-part2-fix-metaspace-oom-not-exit' into dev-1.1.8-webank * remove jdk8 judge --------- Co-authored-by: peacewong --- .../engineplugin/common/conf/EnvConfiguration.scala | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala index 290c6211e1b..f3235ffa343 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala @@ -41,13 +41,12 @@ object EnvConfiguration { val ENGINE_CONN_CLASSPATH_FILES = CommonVars("wds.linkis.engineConn.files", "", "engineConn额外的配置文件") - val metaspaceSize = if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_1_8)) { - "-XX:MaxMetaspaceSize=256m -XX:MetaspaceSize=128m" - } else { - "-XX:MaxPermSize=256m -XX:PermSize=128m" - } + val MAX_METASPACE_SIZE = CommonVars("linkis.engineconn.metaspace.size.max", "256m") - val ENGINE_CONN_DEFAULT_JAVA_OPTS = CommonVars[String]( + lazy val metaspaceSize = + s"-XX:MaxMetaspaceSize=${MAX_METASPACE_SIZE.getValue} -XX:MetaspaceSize=128m" + + lazy val ENGINE_CONN_DEFAULT_JAVA_OPTS = CommonVars[String]( "wds.linkis.engineConn.javaOpts.default", s"-XX:+UseG1GC ${metaspaceSize} " + s"-Xloggc:%s -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Dwds.linkis.server.conf=linkis-engineconn.properties -Dwds.linkis.gateway.url=${Configuration.getGateWayURL()}" From 3d7e02faa14dd78c6c1c5b3db772f8752654d373 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Mon, 20 Mar 2023 20:44:45 +0800 Subject: [PATCH 062/689] update version number (#4392) * update version number * add file schema check --------- Co-authored-by: aiceflower --- .github/workflows/publish-docker.yaml | 2 +- linkis-dist/deploy-config/linkis-env.sh | 2 +- linkis-dist/helm/README.md | 4 ++-- linkis-dist/helm/README_CN.md | 4 ++-- linkis-dist/helm/charts/linkis/Chart.yaml | 2 +- .../apache/linkis/filesystem/validator/PathValidator.scala | 5 ++++- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml index c3b9b6e8256..f06621a72d3 100644 --- a/.github/workflows/publish-docker.yaml +++ b/.github/workflows/publish-docker.yaml @@ -31,7 +31,7 @@ jobs: TAG: ${{ github.sha }} SKIP_TEST: true HUB: ghcr.io/apache/linkis - LINKIS_VERSION: 1.3.1 + LINKIS_VERSION: 1.3.2 steps: - name: Checkout uses: actions/checkout@v2 diff --git a/linkis-dist/deploy-config/linkis-env.sh b/linkis-dist/deploy-config/linkis-env.sh index 9197f7be977..e9f7bff4dde 100644 --- a/linkis-dist/deploy-config/linkis-env.sh +++ b/linkis-dist/deploy-config/linkis-env.sh @@ -156,7 +156,7 @@ export SERVER_HEAP_SIZE="512M" ##The extended lib such mysql-connector-java-*.jar #LINKIS_EXTENDED_LIB=/appcom/common/linkisExtendedLib -LINKIS_VERSION=1.3.1 +LINKIS_VERSION=1.3.2 # for install LINKIS_PUBLIC_MODULE=lib/linkis-commons/public-module diff --git a/linkis-dist/helm/README.md b/linkis-dist/helm/README.md index 274de3dc2a3..ad82a780528 100644 --- a/linkis-dist/helm/README.md +++ b/linkis-dist/helm/README.md @@ -78,14 +78,14 @@ REVISION: 1 TEST SUITE: None NOTES: --- -Welcome to Apache Linkis (v1.3.1)! +Welcome to Apache Linkis (v1.3.2)! .___ .___ .______ .____/\ .___ .________ | | : __|: \ : / \: __|| ___/ | | | : || ||. ___/| : ||___ \ | |/\ | || | || \ | || / | / \| ||___| || \| ||__:___/ -|______/|___| |___||___\ /|___| : v1.3.1 +|______/|___| |___||___\ /|___| : v1.3.2 \/ Linkis builds a layer of computation middleware between upper applications and underlying engines. diff --git a/linkis-dist/helm/README_CN.md b/linkis-dist/helm/README_CN.md index e756dc73fc5..4ee3246e3f7 100644 --- a/linkis-dist/helm/README_CN.md +++ b/linkis-dist/helm/README_CN.md @@ -75,14 +75,14 @@ REVISION: 1 TEST SUITE: None NOTES: --- -Welcome to Apache Linkis (v1.3.1)! +Welcome to Apache Linkis (v1.3.2)! .___ .___ .______ .____/\ .___ .________ | | : __|: \ : / \: __|| ___/ | | | : || ||. ___/| : ||___ \ | |/\ | || | || \ | || / | / \| ||___| || \| ||__:___/ -|______/|___| |___||___\ /|___| : v1.3.1 +|______/|___| |___||___\ /|___| : v1.3.2 \/ Linkis builds a layer of computation middleware between upper applications and underlying engines. diff --git a/linkis-dist/helm/charts/linkis/Chart.yaml b/linkis-dist/helm/charts/linkis/Chart.yaml index 62b9b7706ae..2a677665359 100644 --- a/linkis-dist/helm/charts/linkis/Chart.yaml +++ b/linkis-dist/helm/charts/linkis/Chart.yaml @@ -36,4 +36,4 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.3.1" +appVersion: "1.3.2" diff --git a/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/scala/org/apache/linkis/filesystem/validator/PathValidator.scala b/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/scala/org/apache/linkis/filesystem/validator/PathValidator.scala index 6b63e3a7173..6bdc48620a9 100644 --- a/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/scala/org/apache/linkis/filesystem/validator/PathValidator.scala +++ b/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/scala/org/apache/linkis/filesystem/validator/PathValidator.scala @@ -85,13 +85,16 @@ class PathValidator extends Logging { } def checkPath(path: String, username: String): Unit = { + if (path.contains(StorageUtils.HDFS_SCHEMA)) { + return + } // 校验path的逻辑 val userLocalRootPath: String = WorkspaceUtil.suffixTuning(LOCAL_USER_ROOT_PATH.getValue) + username var userHdfsRootPath: String = WorkspaceUtil.suffixTuning(HDFS_USER_ROOT_PATH_PREFIX.getValue) + username + HDFS_USER_ROOT_PATH_SUFFIX.getValue - if (!(path.contains(StorageUtils.FILE_SCHEMA)) && !(path.contains(StorageUtils.HDFS_SCHEMA))) { + if (!(path.contains(StorageUtils.FILE_SCHEMA))) { throw new WorkSpaceException(80025, "the path should contain schema") } userHdfsRootPath = StringUtils.trimTrailingCharacter(userHdfsRootPath, File.separatorChar) From 4ab2670fdd4df643942ae1e3576f47238a104f89 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Mon, 20 Mar 2023 21:07:42 +0800 Subject: [PATCH 063/689] show udfManager and update datasource env (#4393) Co-authored-by: aiceflower --- .../linkis/module/datasourceEnv/EditForm/index.vue | 11 +++++++++-- .../src/apps/linkis/module/datasourceEnv/index.vue | 6 +++--- linkis-web/src/apps/linkis/view/linkis/index.vue | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/linkis-web/src/apps/linkis/module/datasourceEnv/EditForm/index.vue b/linkis-web/src/apps/linkis/module/datasourceEnv/EditForm/index.vue index 71f02e44058..d54077ffde3 100644 --- a/linkis-web/src/apps/linkis/module/datasourceEnv/EditForm/index.vue +++ b/linkis-web/src/apps/linkis/module/datasourceEnv/EditForm/index.vue @@ -239,13 +239,20 @@ export default { }) }, hiddenHandler (newV) { + let dataSourceTypeName = ''; + for(let i = 0;i < this.typeOptions.length; i++) { + if (this.typeOptions[i].value === newV.datasourceTypeId) { + dataSourceTypeName = this.typeOptions[i].label + } + } // radio - this.rule[4].hidden = !(newV.datasourceTypeId === 4); + this.rule[4].hidden = !(['hive', 'kafka'].includes(dataSourceTypeName)) + // keytab value this.rule[6].hidden = !newV.keytab; // upload this.rule[5].hidden = !this.rule[6].hidden; - if (!newV.hasKeyTab || newV.datasourceTypeId !== 4){ + if (!newV.hasKeyTab || !['hive', 'kafka'].includes(dataSourceTypeName)){ this.rule[5].hidden = true; this.rule[6].hidden = true; this.rule[8].hidden = true; diff --git a/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue b/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue index 5a2666f153c..c3e291338e2 100644 --- a/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue +++ b/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue @@ -70,7 +70,7 @@ - +

@@ -212,7 +212,7 @@ export default { } getAllEnv().then((res) => { let options = [...res.typeList].sort((a, b) => a.id - b.id) - .map(item => { return {value: +item.id, label: item.name, disabled: ![2, 4].includes(+item.id)}}) + .map(item => { return {value: +item.id, label: item.name, disabled: !['hive', 'kafka'].includes(item.name)}}) this.datasourceTypeOptions= options // 获取列表 getList(params).then((data) => { @@ -223,7 +223,7 @@ export default { let filter = options.filter(optionsItem=>{ return optionsItem.value === item.datasourceTypeId }) - item.name = filter[0].label + item.name = filter[0]?.label || ''; }) }) }) diff --git a/linkis-web/src/apps/linkis/view/linkis/index.vue b/linkis-web/src/apps/linkis/view/linkis/index.vue index 1721688f322..e848ed1b335 100644 --- a/linkis-web/src/apps/linkis/view/linkis/index.vue +++ b/linkis-web/src/apps/linkis/view/linkis/index.vue @@ -284,7 +284,7 @@ export default { ), path: '/console/urm/functionManagement', }, - // {key: '1-9-3', name: this.$t('message.linkis.sideNavList.function.children.udfManager'), path: '/console/udfManager' }, + {key: '1-9-3', name: this.$t('message.linkis.sideNavList.function.children.udfManager'), path: '/console/udfManager' }, {key: '1-9-4', name: this.$t('message.linkis.sideNavList.function.children.udfTree'), path: '/console/udfTree' }, ], }, From bfee7f1c0dbf6e61b98d767278aea44dad3d7677 Mon Sep 17 00:00:00 2001 From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com> Date: Tue, 21 Mar 2023 19:44:58 +0800 Subject: [PATCH 064/689] Backport hive compatibility to 1.3.2 (#4394) * backport hive compatibility #4255 #4246 #4341 to 1.3.2 * trigger rebuild --- .../label/entity/engine/EngineTypeLabel.java | 15 ++++++ .../entity/engine/EngineTypeLabelTest.java | 47 +++++++++++++++++ linkis-engineconn-plugins/flink/pom.xml | 1 - linkis-engineconn-plugins/hive/pom.xml | 4 -- .../serde/CustomerDelimitedJSONSerDe.java | 50 ++++++++++++++++++- .../service/hive/pom.xml | 1 - pom.xml | 1 + 7 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java index 09492b146ac..912f6c9f948 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java @@ -74,4 +74,19 @@ public void setVersion(String version) { public Boolean isEmpty() { return StringUtils.isBlank(getEngineType()) || StringUtils.isBlank(getVersion()); } + + @Override + protected void setStringValue(String stringValue) { + String version; + String engineType = stringValue.split("-")[0]; + + if (engineType.equals("*")) { + version = stringValue.replaceFirst("[" + engineType + "]-", ""); + } else { + version = stringValue.replaceFirst(engineType + "-", ""); + } + + setEngineType(engineType); + setVersion(version); + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java new file mode 100644 index 00000000000..cc4171b4377 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.label.entity.engine; + +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactory; +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** EngineTypeLabel Tester */ +public class EngineTypeLabelTest { + + @Test + public void testSetStringValue() { + String engineType = "hive"; + String version = "1.1.0-cdh5.12.0"; + + String engineType1 = "*"; + String version1 = "*"; + + LabelBuilderFactory labelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory(); + EngineTypeLabel engineTypeLabel = labelBuilderFactory.createLabel(EngineTypeLabel.class); + engineTypeLabel.setStringValue(engineType + "-" + version); + Assertions.assertEquals(engineTypeLabel.getEngineType(), engineType); + Assertions.assertEquals(engineTypeLabel.getVersion(), version); + + engineTypeLabel.setStringValue(engineType1 + "-" + version1); + Assertions.assertEquals(engineTypeLabel.getEngineType(), engineType1); + Assertions.assertEquals(engineTypeLabel.getVersion(), version1); + } +} diff --git a/linkis-engineconn-plugins/flink/pom.xml b/linkis-engineconn-plugins/flink/pom.xml index 3ceca971e20..2d88c95b652 100644 --- a/linkis-engineconn-plugins/flink/pom.xml +++ b/linkis-engineconn-plugins/flink/pom.xml @@ -27,7 +27,6 @@ linkis-engineconn-plugin-flink 1.12.2 - 2.3.3 1.3.1 diff --git a/linkis-engineconn-plugins/hive/pom.xml b/linkis-engineconn-plugins/hive/pom.xml index 8fe446167c6..28b60fff0a8 100644 --- a/linkis-engineconn-plugins/hive/pom.xml +++ b/linkis-engineconn-plugins/hive/pom.xml @@ -26,10 +26,6 @@ linkis-engineplugin-hive - - 2.3.3 - - diff --git a/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java b/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java index 671b0c1d19e..897a76e668d 100644 --- a/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java +++ b/linkis-engineconn-plugins/hive/src/main/java/org/apache/linkis/engineplugin/hive/serde/CustomerDelimitedJSONSerDe.java @@ -17,6 +17,8 @@ package org.apache.linkis.engineplugin.hive.serde; +import org.apache.linkis.common.utils.ClassUtils; + import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.hive.serde2.ByteStream; import org.apache.hadoop.hive.serde2.SerDeException; @@ -33,6 +35,7 @@ import java.io.IOException; import java.io.OutputStream; +import java.lang.reflect.InvocationTargetException; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -329,7 +332,52 @@ private static void writePrimitiveUTF8( } default: { - throw new RuntimeException("Unknown primitive type: " + category); + if (!"INTERVAL_YEAR_MONTH".equals(category.name()) + && !"INTERVAL_DAY_TIME".equals(category.name())) { + throw new RuntimeException("Unknown primitive type: " + category); + } + boolean containsIntervalYearMonth = false; + boolean containsIntervalDayTime = false; + for (PrimitiveObjectInspector.PrimitiveCategory primitiveCategory : + PrimitiveObjectInspector.PrimitiveCategory.values()) { + containsIntervalYearMonth = + "INTERVAL_YEAR_MONTH".equals(primitiveCategory.name()) + && "INTERVAL_YEAR_MONTH".equals(category.name()); + containsIntervalDayTime = + "INTERVAL_DAY_TIME".equals(primitiveCategory.name()) + && "INTERVAL_DAY_TIME".equals(category.name()); + try { + if (containsIntervalYearMonth) { + wc = + (WritableComparable) + ClassUtils.getClassInstance( + "org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalYearMonthObjectInspector") + .getClass() + .getMethod("getPrimitiveWritableObject", Object.class) + .invoke(oi, o); + binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); + break; + } + if (containsIntervalDayTime) { + wc = + (WritableComparable) + ClassUtils.getClassInstance( + "org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalDayTimeObjectInspector") + .getClass() + .getMethod("getPrimitiveWritableObject", Object.class) + .invoke(oi, o); + binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes()); + break; + } + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + LOG.error("Fail to invoke method:[getPrimitiveWritableObject]!", e); + } + } + if (containsIntervalYearMonth || containsIntervalDayTime) { + break; + } else { + throw new RuntimeException("Unknown primitive type: " + category); + } } } if (binaryData == null) { diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml index 12791394463..b0c9cdef363 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml @@ -26,7 +26,6 @@ linkis-metadata-query-service-hive - 2.3.3 2.7.2 4.2.4 diff --git a/pom.xml b/pom.xml index 5b9cb09c944..4f2381c87f9 100644 --- a/pom.xml +++ b/pom.xml @@ -106,6 +106,7 @@ 1.3.2-SNAPSHOT 2.9.2 2.4.3 + 2.3.3 2.7.2 hadoop-hdfs 2.7.2 From f5a9bfbcba523ef1038752325676357cdbb2fcfb Mon Sep 17 00:00:00 2001 From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com> Date: Tue, 21 Mar 2023 20:15:18 +0800 Subject: [PATCH 065/689] backport spark compatibility #4301 to 1.3.2 (#4395) --- linkis-engineconn-plugins/spark/pom.xml | 10 ++++++++++ .../spark/datacalc/sink/HiveSink.scala | 17 +++++++++-------- .../spark/datacalc/sink/JdbcSink.scala | 16 ++++++++++++---- tool/dependencies/known-dependencies.txt | 1 + 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/linkis-engineconn-plugins/spark/pom.xml b/linkis-engineconn-plugins/spark/pom.xml index a9631e2b74d..5fd083b90e1 100644 --- a/linkis-engineconn-plugins/spark/pom.xml +++ b/linkis-engineconn-plugins/spark/pom.xml @@ -187,12 +187,22 @@ linkis-rpc ${project.version} + + net.sf.py4j + py4j + 0.10.7 + provided + org.apache.spark spark-core_${scala.binary.version} ${spark.version} provided + + net.sf.py4j + py4j + org.apache.hadoop hadoop-common diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/HiveSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/HiveSink.scala index 8ba618776b9..bef4c39694e 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/HiveSink.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/HiveSink.scala @@ -24,14 +24,11 @@ import org.apache.linkis.engineplugin.spark.errorcode.SparkErrorCodeSummary import org.apache.commons.lang3.StringUtils import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.catalog.HiveTableRelation import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.sql.functions.col import org.apache.spark.sql.sources.DataSourceRegister import org.apache.spark.sql.types.StructField -import org.slf4j.{Logger, LoggerFactory} - class HiveSink extends DataCalcSink[HiveSinkConfig] with Logging { def output(spark: SparkSession, ds: Dataset[Row]): Unit = { @@ -122,7 +119,9 @@ class HiveSink extends DataCalcSink[HiveSinkConfig] with Logging { logFields(sourceFields, targetFields) throw new HiveSinkException( SparkErrorCodeSummary.DATA_CALC_COLUMN_NUM_NOT_MATCH.getErrorCode, - s"$targetTable requires that the data to be inserted have the same number of columns as the target table: target table has ${targetFields.length} column(s) but the inserted data has ${sourceFields.length} column(s)" + s"$targetTable requires that the data to be inserted have the same number of columns " + + s"as the target table: target table has ${targetFields.length} column(s) " + + s"but the inserted data has ${sourceFields.length} column(s)" ) } @@ -184,17 +183,19 @@ class HiveSink extends DataCalcSink[HiveSinkConfig] with Logging { logicalRelation.relation match { case hadoopFsRelation: HadoopFsRelation => hadoopFsRelation.fileFormat match { - case _: org.apache.spark.sql.execution.datasources.orc.OrcFileFormat => - fileFormat = FileFormat.ORC case _: org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat => fileFormat = FileFormat.PARQUET case dataSourceRegister: DataSourceRegister => fileFormat = FileFormat.withName(dataSourceRegister.shortName.toUpperCase) case _ => + if (hadoopFsRelation.fileFormat.getClass.getSimpleName.equals("OrcFileFormat")) { + fileFormat = FileFormat.ORC + } } } - case hiveTableRelation: HiveTableRelation => - // todo + // case hiveTableRelation: HiveTableRelation => + // todo please note `HiveTableRelation` was added after spark 2.2.1 + case _ => } fileFormat } catch { diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/JdbcSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/JdbcSink.scala index ab8a21c3f7b..e9d60bd2b39 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/JdbcSink.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/JdbcSink.scala @@ -17,14 +17,16 @@ package org.apache.linkis.engineplugin.spark.datacalc.sink +import org.apache.linkis.common.utils.ClassUtils.getFieldVal import org.apache.linkis.common.utils.Logging import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSink import org.apache.commons.lang3.StringUtils +import org.apache.spark.SPARK_VERSION import org.apache.spark.sql.{Dataset, Row, SparkSession} -import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JdbcUtils} +import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions -import java.sql.Connection +import java.sql.{Connection, DriverManager} import scala.collection.JavaConverters._ @@ -58,7 +60,8 @@ class JdbcSink extends DataCalcSink[JdbcSinkConfig] with Logging { .repartition(1) .foreachPartition((_: Iterator[Row]) => { val jdbcOptions = new JDBCOptions(options) - val conn: Connection = JdbcUtils.createConnectionFactory(jdbcOptions)() + val conn: Connection = + DriverManager.getConnection(config.getUrl, config.getUser, config.getPassword) try { config.getPreQueries.asScala.foreach(query => { logger.info(s"Execute pre query: $query") @@ -86,7 +89,12 @@ class JdbcSink extends DataCalcSink[JdbcSinkConfig] with Logging { logger.info("Execute query: {}", query) val statement = conn.prepareStatement(query) try { - statement.setQueryTimeout(jdbcOptions.queryTimeout) + // `queryTimeout` was added after spark2.4.0, more details please check SPARK-23856 + if (SPARK_VERSION >= "2.4") { + val queryTimeout = getFieldVal(jdbcOptions, "queryTimeout").asInstanceOf[Int] + statement.setQueryTimeout(queryTimeout) + } + val rows = statement.executeUpdate() logger.info("{} rows affected", rows) } catch { diff --git a/tool/dependencies/known-dependencies.txt b/tool/dependencies/known-dependencies.txt index 62f2e364a26..de0f8e52625 100644 --- a/tool/dependencies/known-dependencies.txt +++ b/tool/dependencies/known-dependencies.txt @@ -434,6 +434,7 @@ protostuff-collectionschema-1.6.2.jar protostuff-core-1.6.2.jar protostuff-runtime-1.6.2.jar py4j-0.10.4.jar +py4j-0.10.7.jar quartz-2.3.2.jar reactive-streams-1.0.3.jar reactor-core-3.3.17.RELEASE.jar From 11a37879dcc48a712f6246485dbbd4a5a0f2bac6 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Wed, 22 Mar 2023 10:49:44 +0800 Subject: [PATCH 066/689] update udfManager auth (#4399) Co-authored-by: aiceflower --- linkis-web/src/apps/linkis/view/linkis/index.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-web/src/apps/linkis/view/linkis/index.vue b/linkis-web/src/apps/linkis/view/linkis/index.vue index e848ed1b335..671da9e06ae 100644 --- a/linkis-web/src/apps/linkis/view/linkis/index.vue +++ b/linkis-web/src/apps/linkis/view/linkis/index.vue @@ -51,7 +51,7 @@ v-for="(item3, index3) in (item.key === '1-9' ? urmSideNavList.children : item.key === '1-8' ?datasourceNavList.children:basedataNavList.children)" :key="index3" @on-click="clickToRoute">
+ v-if="isLogAdmin ? true : item3.key === '1-8-1' || item3.key === '1-9-2' || item3.key === '1-9-1'">
From 1bf673851b257d3e0a25360eda44251270533323 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Wed, 22 Mar 2023 15:04:27 +0800 Subject: [PATCH 067/689] [feat-4348]change the default token to a random number (#4349) * Change the default token to a random number * replace token to random * update token * update token * update token * update token * update token * fix build error * update token * update token * update token * update token --------- Co-authored-by: aiceflower --- .../operator/ujes/UJESClientFactory.java | 6 ++++-- .../LinkisClientApplicationTest.java | 6 +++++- .../computation/client/LinkisJobBuilder.scala | 8 +++++--- .../ujes/client/JobObserveActionTest.scala | 7 +++++-- linkis-dist/bin/install.sh | 17 +++++++++++++++++ linkis-dist/package/conf/linkis.properties | 13 ++++++++++++- linkis-dist/package/db/linkis_dml.sql | 10 +++++----- .../cs/client/utils/ContextClientConf.scala | 6 +++++- .../cs/client/utils/ContextClientConfTest.java | 4 ---- .../authentication/dao/TokenDaoTest.java | 5 ++++- .../service/CachedTokenServiceTest.java | 4 +++- 11 files changed, 65 insertions(+), 21 deletions(-) diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java index 2767929e8d5..c234403c5c5 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java @@ -131,6 +131,8 @@ public static DWSClientConfig generateDWSClientConfigForBML( } DWSClientConfigBuilder builder = DWSClientConfigBuilder.newBuilder(); + String authKey = stdVarAccess.getVar(String.class, AppKeys.LINKIS_COMMON_TOKEN_KEY); + String authValue = stdVarAccess.getVar(String.class, AppKeys.LINKIS_COMMON_TOKEN_VALUE); DWSClientConfig config = ((DWSClientConfigBuilder) (builder @@ -143,8 +145,8 @@ public static DWSClientConfig generateDWSClientConfigForBML( .retryEnabled(false) .readTimeout(context.getReadTimeoutMills()) .setAuthenticationStrategy(authenticationStrategy) - .setAuthTokenKey("BML-AUTH") - .setAuthTokenValue("BML-AUTH"))) + .setAuthTokenKey(authKey) + .setAuthTokenValue(authValue))) .setDWSVersion(context.getDwsVersion()) .build(); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java index 0af22266615..bdacb4f09e4 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java @@ -17,6 +17,8 @@ package org.apache.linkis.cli.application; +import org.apache.linkis.common.conf.CommonVars; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -26,6 +28,8 @@ public class LinkisClientApplicationTest { private static final Logger logger = LoggerFactory.getLogger(LinkisClientApplicationTest.class); + String bmlToken = CommonVars.apply("wds.linkis.bml.auth.token.value", "BML-AUTH").getValue(); + String[] cmdStr; String[] cmdStr2; @@ -58,7 +62,7 @@ public void before() { "--authKey", "Validation-Code", "--authVal", - "BML-AUTH", + bmlToken, // "--help", // "--kill", "8249", // "--status", "379", diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/LinkisJobBuilder.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/LinkisJobBuilder.scala index 3daba19410a..9cc2863559d 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/LinkisJobBuilder.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/LinkisJobBuilder.scala @@ -17,7 +17,7 @@ package org.apache.linkis.computation.client -import org.apache.linkis.common.conf.Configuration +import org.apache.linkis.common.conf.{CommonVars, Configuration} import org.apache.linkis.common.exception.LinkisRetryException import org.apache.linkis.common.utils.{RetryHandler, Utils} import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy @@ -174,8 +174,10 @@ object LinkisJobBuilder { private var threadPool: ScheduledThreadPoolExecutor = Utils.defaultScheduler private var serverUrl: String = _ - private var authTokenValue: String = - "LINKIS_CLI_TEST" // This is the default authToken, we usually suggest set different ones for users. + private var authTokenValue: String = CommonVars[String]( + "wds.linkis.client.test.common.tokenValue", + "LINKIS_CLI_TEST" + ).getValue // This is the default authToken, we usually suggest set different ones for users. def setDefaultClientConfig(clientConfig: DWSClientConfig): Unit = this.clientConfig = clientConfig diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/ujes/client/JobObserveActionTest.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/ujes/client/JobObserveActionTest.scala index 683b9b47b10..b55f42159ba 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/ujes/client/JobObserveActionTest.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/test/java/org/apache/linkis/ujes/client/JobObserveActionTest.scala @@ -18,6 +18,7 @@ package org.apache.linkis.ujes.client import org.apache.commons.io.IOUtils +import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.httpclient.dws.authentication.{StaticAuthenticationStrategy, TokenAuthenticationStrategy} import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} import org.apache.linkis.ujes.client.request.{EmsListAction, JobExecuteAction, JobObserveAction, ResultSetAction} @@ -28,6 +29,8 @@ import java.util.concurrent.TimeUnit @Deprecated object JobObserveActionTest extends App { + val bmlToken = CommonVars("wds.linkis.bml.auth.token.value", "BML-AUTH").getValue + val clientConfig = DWSClientConfigBuilder.newBuilder() .addServerUrl("127.0.0.1:9001") // Change to test gateway address .connectionTimeout(30000) @@ -38,8 +41,8 @@ object JobObserveActionTest extends App { .retryEnabled(false) .readTimeout(30000) .setAuthenticationStrategy(new TokenAuthenticationStrategy()) - .setAuthTokenKey("BML-AUTH") - .setAuthTokenValue("BML-AUTH") + .setAuthTokenKey("Validation-Code") + .setAuthTokenValue(bmlToken) .setDWSVersion("v1") .build() val client = new UJESClientImpl(clientConfig) diff --git a/linkis-dist/bin/install.sh b/linkis-dist/bin/install.sh index 934d1d0a76d..4a2479137c9 100644 --- a/linkis-dist/bin/install.sh +++ b/linkis-dist/bin/install.sh @@ -106,6 +106,16 @@ cp ${LINKIS_DB_CONFIG_PATH} $LINKIS_HOME/conf common_conf=$LINKIS_HOME/conf/linkis.properties +RANDOM_BML_TOKEN="BML-`cat /proc/sys/kernel/random/uuid | awk -F- '{print $1$2$3$4$5}'`" +RANDOM_LINKIS_CLI_TEST_TOKEN="LINKIS_CLI-`cat /proc/sys/kernel/random/uuid | awk -F- '{print $1$2$3$4$5}'`" +RANDOM_WS_TOKEN="WS-`cat /proc/sys/kernel/random/uuid | awk -F- '{print $1$2$3$4$5}'`" +RANDOM_DSM_TOKEN="DSM-`cat /proc/sys/kernel/random/uuid | awk -F- '{print $1$2$3$4$5}'`" +sed -i ${txt} "s#BML-AUTH#$RANDOM_BML_TOKEN#g" $LINKIS_HOME/conf/linkis-cli/linkis-cli.properties +sed -i ${txt} "s#BML-AUTH#$RANDOM_BML_TOKEN#g" $common_conf +sed -i ${txt} "s#LINKIS_CLI_TEST#$RANDOM_LINKIS_CLI_TEST_TOKEN#g" $common_conf +sed -i ${txt} "s#WS-AUTH#$RANDOM_WS_TOKEN#g" $common_conf +sed -i ${txt} "s#DSM-AUTH#$RANDOM_DSM_TOKEN#g" $common_conf + echo "======= Step 3: Create necessary directory ==========" echo "[WORKSPACE_USER_ROOT_PATH] try to create directory" @@ -184,6 +194,13 @@ echo "[RESULT_SET_ROOT_PATH] try to create directory" echo "======= Step 4: Create linkis table ==========" ## sql init +# replace token +sed -i ${txt} "s#BML-AUTH#$RANDOM_BML_TOKEN#g" $LINKIS_HOME/db/linkis_dml.sql +sed -i ${txt} "s#LINKIS_CLI_TEST#$RANDOM_LINKIS_CLI_TEST_TOKEN#g" $LINKIS_HOME/db/linkis_dml.sql +sed -i ${txt} "s#WS-AUTH#$RANDOM_WS_TOKEN#g" $LINKIS_HOME/db/linkis_dml.sql +sed -i ${txt} "s#DSM-AUTH#$RANDOM_DSM_TOKEN#g" $LINKIS_HOME/db/linkis_dml.sql + + if [ "$YARN_RESTFUL_URL" != "" ] then sed -i ${txt} "s#@YARN_RESTFUL_URL#$YARN_RESTFUL_URL#g" $LINKIS_HOME/db/linkis_dml.sql diff --git a/linkis-dist/package/conf/linkis.properties b/linkis-dist/package/conf/linkis.properties index 66ed15cba38..5448aa9b4dc 100644 --- a/linkis-dist/package/conf/linkis.properties +++ b/linkis-dist/package/conf/linkis.properties @@ -89,4 +89,15 @@ linkis.session.redis.password=test123 # redis sso switch linkis.session.redis.cache.enabled=false wds.linkis.workspace.filesystem.owner.check=true -wds.linkis.workspace.filesystem.path.check=true \ No newline at end of file +wds.linkis.workspace.filesystem.path.check=true + +#linkis token +linkis.configuration.linkisclient.auth.token.value=BML-AUTH +wds.linkis.client.common.tokenValue=BML-AUTH +wds.linkis.bml.auth.token.value=BML-AUTH +wds.linkis.context.client.auth.value=BML-AUTH +wds.linkis.errorcode.auth.token=BML-AUTH +wds.linkis.client.test.common.tokenValue=LINKIS_CLI_TEST +wds.linkis.filesystem.token.value=WS-AUTH +wds.linkis.gateway.access.token=WS-AUTH +wds.linkis.server.dsm.auth.token.value=DSM-AUTH \ No newline at end of file diff --git a/linkis-dist/package/db/linkis_dml.sql b/linkis-dist/package/db/linkis_dml.sql index facbf32603c..c0738d10f7a 100644 --- a/linkis-dist/package/db/linkis_dml.sql +++ b/linkis-dist/package/db/linkis_dml.sql @@ -528,13 +528,13 @@ INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) -- ---------------------------- -- Default Tokens -- ---------------------------- -INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('QML-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); +INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES (concat('QML-', MD5(RAND())),'*','*','BDP',curdate(),curdate(),-1,'LINKIS'); INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('BML-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('WS-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); -INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('dss-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); -INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('QUALITIS-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); -INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('VALIDATOR-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); -INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('LINKISCLI-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); +INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES (concat('DSS-', MD5(RAND())),'*','*','BDP',curdate(),curdate(),-1,'LINKIS'); +INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES (concat('QUALITIS-', MD5(RAND())),'*','*','BDP',curdate(),curdate(),-1,'LINKIS'); +INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES (concat('VALIDATOR-', MD5(RAND())),'*','*','BDP',curdate(),curdate(),-1,'LINKIS'); +INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES (concat('LINKISCLI-', MD5(RAND())),'*','*','BDP',curdate(),curdate(),-1,'LINKIS'); INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('DSM-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('LINKIS_CLI_TEST','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); diff --git a/linkis-public-enhancements/linkis-context-service/linkis-cs-client/src/main/scala/org/apache/linkis/cs/client/utils/ContextClientConf.scala b/linkis-public-enhancements/linkis-context-service/linkis-cs-client/src/main/scala/org/apache/linkis/cs/client/utils/ContextClientConf.scala index e8de9dce4c4..c472a1add1e 100644 --- a/linkis-public-enhancements/linkis-context-service/linkis-cs-client/src/main/scala/org/apache/linkis/cs/client/utils/ContextClientConf.scala +++ b/linkis-public-enhancements/linkis-context-service/linkis-cs-client/src/main/scala/org/apache/linkis/cs/client/utils/ContextClientConf.scala @@ -28,7 +28,11 @@ object ContextClientConf { CommonVars[String]("wds.linkis.context.client.auth.value", "BML-AUTH") val URL_PREFIX: CommonVars[String] = - CommonVars[String]("wds.linkis.cs.url.prefix", "/api/rest_j/v1/contextservice", "cs服务的url前缀") + CommonVars[String]( + "wds.linkis.cs.url.prefix", + "/api/rest_j/v1/contextservice", + "The url prefix of the cs service." + ) val HEART_BEAT_ENABLED: CommonVars[String] = CommonVars[String]("wds.linkis.cs.heartbeat.enabled", "true") diff --git a/linkis-public-enhancements/linkis-context-service/linkis-cs-client/src/test/java/org/apache/linkis/cs/client/utils/ContextClientConfTest.java b/linkis-public-enhancements/linkis-context-service/linkis-cs-client/src/test/java/org/apache/linkis/cs/client/utils/ContextClientConfTest.java index 951cab6177f..2c1bd3be97c 100644 --- a/linkis-public-enhancements/linkis-context-service/linkis-cs-client/src/test/java/org/apache/linkis/cs/client/utils/ContextClientConfTest.java +++ b/linkis-public-enhancements/linkis-context-service/linkis-cs-client/src/test/java/org/apache/linkis/cs/client/utils/ContextClientConfTest.java @@ -27,13 +27,9 @@ public class ContextClientConfTest { @DisplayName("constTest") public void constTest() { - String contextClientAuthKey = ContextClientConf.CONTEXT_CLIENT_AUTH_KEY().getValue(); - String contextClientAuthValue = ContextClientConf.CONTEXT_CLIENT_AUTH_VALUE().getValue(); String urlPrefix = ContextClientConf.URL_PREFIX().getValue(); String hearBeatEnabled = ContextClientConf.HEART_BEAT_ENABLED().getValue(); - Assertions.assertEquals("Token-Code", contextClientAuthKey); - Assertions.assertEquals("BML-AUTH", contextClientAuthValue); Assertions.assertEquals("/api/rest_j/v1/contextservice", urlPrefix); Assertions.assertEquals("true", hearBeatEnabled); } diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/test/java/org/apache/linkis/gateway/authentication/dao/TokenDaoTest.java b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/test/java/org/apache/linkis/gateway/authentication/dao/TokenDaoTest.java index c2f9eed5392..2c952302333 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/test/java/org/apache/linkis/gateway/authentication/dao/TokenDaoTest.java +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/test/java/org/apache/linkis/gateway/authentication/dao/TokenDaoTest.java @@ -17,6 +17,7 @@ package org.apache.linkis.gateway.authentication.dao; +import org.apache.linkis.common.conf.CommonVars; import org.apache.linkis.gateway.authentication.entity.TokenEntity; import org.springframework.beans.factory.annotation.Autowired; @@ -33,7 +34,9 @@ class TokenDaoTest extends BaseDaoTest { private static final Logger logger = LoggerFactory.getLogger(BaseDaoTest.class); - private static String TokenName = "BML-AUTH"; + private static String TokenName = + CommonVars.apply("wds.linkis.bml.auth.token.value", "BML-AUTH").getValue(); + @Autowired TokenDao tokenDao; @Test diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/test/java/org/apache/linkis/gateway/authentication/service/CachedTokenServiceTest.java b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/test/java/org/apache/linkis/gateway/authentication/service/CachedTokenServiceTest.java index 0412197a03a..6f111352e9b 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/test/java/org/apache/linkis/gateway/authentication/service/CachedTokenServiceTest.java +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/test/java/org/apache/linkis/gateway/authentication/service/CachedTokenServiceTest.java @@ -17,6 +17,7 @@ package org.apache.linkis.gateway.authentication.service; +import org.apache.linkis.common.conf.CommonVars; import org.apache.linkis.gateway.authentication.Scan; import org.apache.linkis.gateway.authentication.WebApplicationServer; import org.apache.linkis.gateway.authentication.exception.TokenAuthException; @@ -37,7 +38,8 @@ public class CachedTokenServiceTest { private static final Logger logger = LoggerFactory.getLogger(CachedTokenServiceTest.class); - private static String TokenName = "BML-AUTH"; + private static String TokenName = + CommonVars.apply("wds.linkis.bml.auth.token.value", "BML-AUTH").getValue(); @Autowired CachedTokenService tokenService; From 61eb8c7551537560b0ba2a93df0fa58231716302 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Wed, 22 Mar 2023 16:06:12 +0800 Subject: [PATCH 068/689] build: upgrade the version to 1.3.2 (#4398) * build: update the version to 1.3.2 --- .github/workflows/build-backend.yml | 80 ++++++++++++------- .../check-third-party-dependencies.yml | 56 ------------- .../spark/config/SparkConfiguration.scala | 2 +- linkis-web/package.json | 5 +- pom.xml | 3 +- tool/dependencies/known-dependencies.txt | 23 +++--- .../regenerate_konwn_dependencies_txt.sh | 4 +- 7 files changed, 69 insertions(+), 104 deletions(-) delete mode 100644 .github/workflows/check-third-party-dependencies.yml diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index 319779f4219..fec9f02f634 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -20,48 +20,72 @@ name: Build Backend on: [push, pull_request] env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + MAVEN_OPTS: -Dmaven.resolver.transport=wagon -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 jobs: build-backend: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up JDK 8 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 8 - - name: Build backend by maven - run: - ./mvnw clean package - build-spark: runs-on: ubuntu-latest strategy: matrix: - installSpark: [ 'spark2.4-hadoop2.3', 'spark3.2-hadoop3.3','spark2.4-hadoop3.3' ] + spark: [2.4, 3.2] + hadoop: [3.3] + include: + - spark: 2.4 + hadoop: 2.7 steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up JDK 8 - uses: actions/setup-java@v2 + uses: actions/setup-java@v3 with: - distribution: 'temurin' - java-version: 8 - - if: ${{ matrix.installSpark == 'spark2.4-hadoop2.3' }} - name: build-spark on init + distribution: 'temurin' + java-version: 8 + cache: maven + - if: ${{matrix.spark == '2.4' && matrix.hadoop == '3.3'}} + name: build spark ${{matrix.spark}} and hadoop ${{matrix.hadoop}} run: - ./mvnw clean install -Pspark-2.4 -Phadoop-2.3 -Dmaven.test.skip=true - - if: ${{ matrix.installSpark == 'spark3.2-hadoop3.3' }} - name: build-spark on 3.2 + ./mvnw clean package -Pspark-${{matrix.spark}}-hadoop-${{matrix.hadoop}} -Phadoop-${{matrix.hadoop}} -Dmaven.test.skip=true + - if: ${{matrix.hadoop == '2.7'}} + name: build default run: - ./mvnw clean install -Pspark-3.2 -Phadoop-3.3 -Dmaven.test.skip=true - - if: ${{ matrix.installSpark == 'spark2.4-hadoop3.3' }} - name: build-spark on 2.4 + ./mvnw clean package + - if: ${{matrix.spark != '2.4'}} + name: build spark ${{matrix.spark}} and hadoop ${{matrix.hadoop}} run: - ./mvnw clean install -Pspark-2.4-hadoop-3.3 -Phadoop-3.3 -Dmaven.test.skip=true + ./mvnw clean package -Pspark-${{matrix.spark}} -Phadoop-${{matrix.hadoop}} -Dmaven.test.skip=true # - name: Upload coverage to Codecov # uses: codecov/codecov-action@v3.0.0 # with: # token: ${{ secrets.CODECOV_TOKEN }} + third-party-dependencies-check: + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v3 + - name: Set up JDK 8 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: 8 + cache: maven + - name: mvn install + run: + #pom.xml also introduce linkis related jar,so run mvn install in first time + ./mvnw install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true + - name: mvn dependency:copy-dependencies + run: + ./mvnw dependency:copy-dependencies -DexcludeGroupIds=org.apache.linkis -DincludeScope=runtime -DoutputDirectory=${{ github.workspace }}/current_dependencies + - name: generate current_dependencies.txt + run: | + ls ${{ github.workspace }}/current_dependencies | sort > ~/current_dependencies.txt + cat ~/current_dependencies.txt + - name: check third dependencies + run: | + #by using commond join ,to check whether there are new third-party dependencies,compared with file(tool/dependencies/known-dependencies.txt) + sort ${{ github.workspace }}/tool/dependencies/known-dependencies.txt > ~/known-dependencies.txt + join -t : -o 1.1 2.1 -a2 ~/known-dependencies.txt ~/current_dependencies.txt > ~/result.txt + #print new third-party dependencies name if it exists + awk -F ":" '{if($1=="")print $2" is not in file known-dependencies.txt!\n You can refer to this guide to repair it(你可以参考这个执行进行修复)：https://linkis.apache.org/zh-CN/docs/latest/development/development-specification/license"}' ~/result.txt + result=`awk -F ":" '{if($1=="")print $2}' ~/result.txt |wc -l` + #if has new third-party,the Action will fail + if [[ $result == 0 ]];then echo "All third dependencies is known!" ;else exit 1;fi \ No newline at end of file diff --git a/.github/workflows/check-third-party-dependencies.yml b/.github/workflows/check-third-party-dependencies.yml deleted file mode 100644 index bcf4a371c3c..00000000000 --- a/.github/workflows/check-third-party-dependencies.yml +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Third-party Dependencies Check - -on: [push, pull_request] - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 - -jobs: - third-party-dependencies-check-: - runs-on: ubuntu-latest - steps: - - name: Checkout source - uses: actions/checkout@v2 - - name: Set up JDK 8 - uses: actions/setup-java@v2 - with: - java-version: '8' - distribution: 'adopt' - - name: mvn install - run: - #pom.xml also introduce linkis related jar,so run mvn install in first time - ./mvnw install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true - - name: mvn dependency:copy-dependencies - run: - ./mvnw dependency:copy-dependencies -DincludeScope=runtime -DoutputDirectory=${{ github.workspace }}/current_dependencies - - name: generate current_dependencies.txt - run: | - ls ${{ github.workspace }}/current_dependencies |egrep -v "^linkis" |sort > ~/current_dependencies.txt - cat ~/current_dependencies.txt - - name: check third dependencies - run: | - #by using commond join ,to check whether there are new third-party dependencies,compared with file(tool/dependencies/known-dependencies.txt) - sort ${{ github.workspace }}/tool/dependencies/known-dependencies.txt > ~/known-dependencies.txt - join -t : -o 1.1 2.1 -a2 ~/known-dependencies.txt ~/current_dependencies.txt > ~/result.txt - #print new third-party dependencies name if it exists - awk -F ":" '{if($1=="")print $2" is not in file known-dependencies.txt!\n You can refer to this guide to repair it(你可以参考这个执行进行修复)：https://linkis.apache.org/zh-CN/docs/latest/development/development-specification/license"}' ~/result.txt - result=`awk -F ":" '{if($1=="")print $2}' ~/result.txt |wc -l` - #if has new third-party,the Action will fail - if [[ $result == 0 ]];then echo "All third dependencies is known!" ;else exit 1;fi \ No newline at end of file diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala index 966caa03a76..0a39667010b 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala @@ -87,7 +87,7 @@ object SparkConfiguration extends Logging { val LINKIS_SPARK_USEHIVECONTEXT = CommonVars[Boolean]("wds.linkis.spark.useHiveContext", true) val DEFAULT_SPARK_JAR_NAME = - CommonVars[String]("wds.linkis.ecp.spark.default.jar", "linkis-engineconn-core-1.3.1.jar") + CommonVars[String]("wds.linkis.ecp.spark.default.jar", "linkis-engineconn-core-1.3.2.jar") val ENGINE_JAR = CommonVars[String]("wds.linkis.enginemanager.core.jar", getMainJarName) diff --git a/linkis-web/package.json b/linkis-web/package.json index 0b1b8385563..d1ed26e814d 100644 --- a/linkis-web/package.json +++ b/linkis-web/package.json @@ -1,6 +1,6 @@ { "name": "linkis", - "version": "1.3.1", + "version": "1.3.2", "private": true, "scripts": { "serve": "vue-cli-service serve", @@ -18,8 +18,7 @@ }, "lint-staged": { "src/**/*.{js,vue}": [ - "vue-cli-service lint --no-fix", - "git add" + "vue-cli-service lint --no-fix" ] }, "dependencies": { diff --git a/pom.xml b/pom.xml index 4f2381c87f9..3b31644003b 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ - 1.3.2-SNAPSHOT + 1.3.2 2.9.2 2.4.3 2.3.3 @@ -1177,6 +1177,7 @@ **/target/** **/out/** **/*.log + **/logs/** CONTRIBUTING.md CONTRIBUTING_CN.md README.md diff --git a/tool/dependencies/known-dependencies.txt b/tool/dependencies/known-dependencies.txt index de0f8e52625..62f7f83f2bc 100644 --- a/tool/dependencies/known-dependencies.txt +++ b/tool/dependencies/known-dependencies.txt @@ -51,6 +51,7 @@ chill-java-0.7.6.jar chill_2.11-0.7.6.jar classgraph-4.1.7.jar classmate-1.5.1.jar +clickhouse-jdbc-0.3.2-patch11.jar commons-beanutils-1.9.4.jar commons-cli-1.2.jar commons-cli-1.3.1.jar @@ -76,6 +77,7 @@ commons-math3-3.6.1.jar commons-net-3.1.jar commons-net-3.9.0.jar commons-pool-1.6.jar +commons-pool2-2.8.1.jar commons-text-1.10.0.jar concurrent-0.191.jar config-1.3.3.jar @@ -91,6 +93,7 @@ derby-10.14.2.0.jar disruptor-3.3.0.jar dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar druid-1.1.22.jar +dss-gateway-support-1.1.1.jar eigenbase-properties-1.1.5.jar elasticsearch-rest-client-6.8.15.jar elasticsearch-rest-client-7.6.2.jar @@ -267,6 +270,7 @@ jcommander-1.30.jar jdbi3-core-3.4.0.jar jdbi3-sqlobject-3.4.0.jar jdo-api-3.0.1.jar +jedis-2.9.2.jar jersey-apache-client4-1.19.4.jar jersey-client-1.19.4.jar jersey-client-2.23.1.jar @@ -301,10 +305,10 @@ jline-2.14.6.jar jmxutils-1.19.jar jna-5.12.1.jar jna-platform-5.12.1.jar +joda-time-2.10.5.jar joda-time-2.3.jar joda-time-2.8.1.jar joda-time-2.9.3.jar -joda-time-2.10.5.jar jol-core-0.2.jar joni-2.1.2.jar jpam-1.1.jar @@ -316,7 +320,6 @@ json4s-core_2.11-3.5.3.jar json4s-jackson_2.11-3.5.3.jar json4s-scalap_2.11-3.5.3.jar jsp-api-2.1.jar -jsqlparser-1.0.jar jsqlparser-4.2.jar jsr305-1.3.9.jar jsr305-3.0.1.jar @@ -359,6 +362,7 @@ metrics-jvm-4.1.22.jar micrometer-core-1.5.14.jar micrometer-registry-prometheus-1.5.14.jar minlog-1.3.0.jar +mongo-java-driver-3.12.8.jar mybatis-3.5.6.jar mybatis-plus-3.4.1.jar mybatis-plus-annotation-3.4.1.jar @@ -424,6 +428,7 @@ poi-5.2.3.jar poi-ooxml-5.2.3.jar poi-ooxml-lite-5.2.3.jar poi-shared-strings-2.5.6.jar +postgresql-42.3.8.jar presto-client-0.234.jar presto-client-1.5.0.jar presto-resource-group-managers-0.234.jar @@ -460,6 +465,9 @@ scala-reflect-2.11.12.jar scala-xml_2.11-1.0.5.jar scalap-2.11.12.jar scopt_2.11-3.5.0.jar +seatunnel-core-flink-2.1.2.jar +seatunnel-core-flink-sql-2.1.2.jar +seatunnel-core-spark-2.1.2.jar security-0.191.jar security-0.193.jar servo-core-0.12.21.jar @@ -484,7 +492,6 @@ spring-boot-starter-actuator-2.3.12.RELEASE.jar spring-boot-starter-aop-2.3.12.RELEASE.jar spring-boot-starter-cache-2.3.12.RELEASE.jar spring-boot-starter-freemarker-2.3.12.RELEASE.jar -spring-boot-starter-jdbc-2.3.12.RELEASE.jar spring-boot-starter-jetty-2.3.12.RELEASE.jar spring-boot-starter-json-2.3.12.RELEASE.jar spring-boot-starter-log4j2-2.3.12.RELEASE.jar @@ -577,13 +584,3 @@ zookeeper-3.5.9.jar zookeeper-jute-3.5.9.jar zstd-jni-1.4.4-7.jar zstd-jni-1.4.5-6.jar -commons-pool2-2.8.1.jar -jedis-2.9.2.jar -dss-gateway-support-1.1.1.jar -seatunnel-core-flink-2.1.2.jar -seatunnel-core-flink-sql-2.1.2.jar -seatunnel-core-spark-2.1.2.jar -mongo-java-driver-3.12.8.jar -clickhouse-jdbc-0.3.2-patch11.jar -postgresql-42.3.8.jar - diff --git a/tool/dependencies/regenerate_konwn_dependencies_txt.sh b/tool/dependencies/regenerate_konwn_dependencies_txt.sh index 8a5743b07bd..e359ba7c5c2 100755 --- a/tool/dependencies/regenerate_konwn_dependencies_txt.sh +++ b/tool/dependencies/regenerate_konwn_dependencies_txt.sh @@ -23,6 +23,6 @@ if [[ -d $dependencies_path ]];then rm -r -f $dependencies_path fi cd ../../ -mvn dependency:copy-dependencies -DincludeScope=runtime -DoutputDirectory=$dependencies_path -ls $dependencies_path | egrep -v "^linkis" | sort -n > $basepath/known-dependencies.txt +mvn dependency:copy-dependencies -DexcludeGroupIds=org.apache.linkis -DincludeScope=runtime -DoutputDirectory=$dependencies_path +ls $dependencies_path | sort -n > $basepath/known-dependencies.txt rm -r -f $dependencies_path \ No newline at end of file From b77cb5742199bb9234cf7e545210f0392518d854 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Wed, 22 Mar 2023 19:50:31 +0800 Subject: [PATCH 069/689] fix some bugs (#4400) * udf path check --- .../linkis/httpclient/config/ClientConfigBuilder.scala | 9 ++++++++- .../linkis/filesystem/restful/api/FsRestfulApi.java | 5 ++++- .../apache/linkis/udf/service/impl/UDFServiceImpl.java | 9 ++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala index c3e5afba302..26170a45772 100644 --- a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala +++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala @@ -19,6 +19,7 @@ package org.apache.linkis.httpclient.config import org.apache.linkis.common.utils.{DefaultRetryHandler, RetryHandler} import org.apache.linkis.httpclient.authentication.AuthenticationStrategy +import org.apache.linkis.httpclient.exception.HttpClientRetryException import org.apache.linkis.httpclient.loadbalancer.LoadBalancerStrategy import scala.concurrent.duration.TimeUnit @@ -38,7 +39,13 @@ class ClientConfigBuilder protected () { protected var readTimeout: Long = _ protected var maxConnection: Int = _ protected var retryEnabled: Boolean = true - protected var retryHandler: RetryHandler = new DefaultRetryHandler + protected var retryHandler: RetryHandler = buildDefaultRetryHandler() + + def buildDefaultRetryHandler(): RetryHandler = { + retryHandler = new DefaultRetryHandler + retryHandler.addRetryException(classOf[HttpClientRetryException]) + retryHandler + } def addServerUrl(serverUrl: String): this.type = { this.serverUrl = serverUrl diff --git a/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 9adfbbd2dc3..1e41baa234e 100644 --- a/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -95,12 +95,15 @@ private boolean checkIsUsersDirectory(String requestPath, String userName) { LOGGER.debug("not check filesystem owner."); return true; } + if (requestPath.contains(WorkspaceUtil.suffixTuning(HDFS_USER_ROOT_PATH_PREFIX.getValue())) + || Configuration.isAdmin(userName)) { + return true; + } requestPath = requestPath.toLowerCase().trim() + "/"; String hdfsUserRootPathPrefix = WorkspaceUtil.suffixTuning(HDFS_USER_ROOT_PATH_PREFIX.getValue()); String hdfsUserRootPathSuffix = HDFS_USER_ROOT_PATH_SUFFIX.getValue(); String localUserRootPath = WorkspaceUtil.suffixTuning(LOCAL_USER_ROOT_PATH.getValue()); - String path; String workspacePath = hdfsUserRootPathPrefix + userName + hdfsUserRootPathSuffix; String enginconnPath = localUserRootPath + userName; diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java index 7ca1cc611d8..14ab927e8fd 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java @@ -125,7 +125,14 @@ public long addUDF(UDFAddVo udfVo, String userName) throws Exception { throw new UDFException("分类名不能为空！"); } // 支持hdfs path - FsPath fsPath = new FsPath(udfVo.getPath()); + String path = udfVo.getPath(); + if (StringUtils.isBlank(path) || path.contains("../")) { + throw new UDFException( + "The path: " + + path + + " of udf is error. Please rename it and rebuild it.(udf的路径错误，请修改后重建)"); + } + FsPath fsPath = new FsPath(path); // FileSystem fileSystem = (FileSystem) FSFactory.getFs(fsPath.getFsType()); FileSystem fileSystem = (FileSystem) FSFactory.getFsByProxyUser(fsPath, userName); if (udfVo.getUdfType() == UDF_JAR && StringUtils.isNotBlank(udfVo.getPath())) { From 344da427139461486a75e6010fcadec50ae18734 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Wed, 22 Mar 2023 21:07:06 +0800 Subject: [PATCH 070/689] [Feature] update dml sql (#4383) * update dml * update dml * update dml --- linkis-dist/package/db/linkis_dml.sql | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/linkis-dist/package/db/linkis_dml.sql b/linkis-dist/package/db/linkis_dml.sql index c0738d10f7a..0fe91f63643 100644 --- a/linkis-dist/package/db/linkis_dml.sql +++ b/linkis-dist/package/db/linkis_dml.sql @@ -538,22 +538,22 @@ INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hos INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('DSM-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); INSERT INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('LINKIS_CLI_TEST','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); -INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('kafka', 'kafka', 'kafka', '消息队列', '', 2); -INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('hive', 'hive数据库', 'hive', '大数据存储', '', 3); -INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('elasticsearch','elasticsearch数据源','es无结构化存储','分布式全文索引','',3); -INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('mongodb', 'mongodb', 'NoSQL文档存储', 'NoSQL', NULL, 3); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('kafka', 'kafka', 'kafka', '消息队列', '', 2, 'Kafka', 'Kafka', 'Message Queue'); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('hive', 'hive数据库', 'hive', '大数据存储', '', 3, 'Hive Database', 'Hive', 'Big Data storage'); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('elasticsearch', 'elasticsearch数据源', 'es无结构化存储', '分布式全文索引', '', 3, 'Elasticsearch Datasource', 'Es No Structured Storage', 'Distributed Full-Text Indexing'); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('mongodb', 'mongodb', 'NoSQL文档存储', 'NoSQL', null, 3, 'mongodb', 'NoSQL Document Storage', 'NOSQL'); -- jdbc -INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('mysql', 'mysql数据库', 'mysql数据库', '关系型数据库', '', 3); -INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('oracle','oracle数据库','oracle','关系型数据库','',3); -INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('dm','达梦数据库','dm','关系型数据库','',3); -INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('kingbase','人大金仓数据库','kingbase','关系型数据库','',3); -INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('postgresql','postgresql数据库','postgresql','关系型数据库','',3); -INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('sqlserver','sqlserver数据库','sqlserver','关系型数据库','',3); -INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('db2','db2数据库','db2','关系型数据库','',3); -INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('greenplum','greenplum数据库','greenplum','关系型数据库','',3); -INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('doris','doris数据库','doris','olap','',4); -INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('clickhouse','clickhouse数据库','clickhouse','olap','',4); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('mysql', 'mysql数据库', 'mysql数据库', '关系型数据库', '', 3, 'Mysql Database', 'Mysql Database', 'Relational Database'); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('oracle', 'oracle数据库', 'oracle', '关系型数据库', '', 3, 'Oracle Database', 'Oracle Relational Database', 'Relational Database'); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('dm', '达梦数据库', 'dm', '关系型数据库', '', 3, 'Dameng Database', 'Dm', 'Relational Database'); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('kingbase', '人大金仓数据库', 'kingbase', '关系型数据库', '', 3, 'Renmin Jincang Database', 'Kingbase', 'Relational Database'); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('postgresql', 'postgresql数据库', 'postgresql', '关系型数据库', '', 3, 'Postgresql Database', 'Postgresql', 'Relational Database'); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('sqlserver', 'sqlserver数据库', 'sqlserver', '关系型数据库', '', 3, 'Sqlserver Database', 'Sqlserver', 'Relational Database'); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('db2', 'db2数据库', 'db2', '关系型数据库', '', 3, 'Db2 Database', 'Db2', 'Relational Database'); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('greenplum', 'greenplum数据库', 'greenplum', '关系型数据库', '', 3, 'Greenplum Database', 'Greenplum', 'Relational Database'); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('doris', 'doris数据库', 'doris', 'olap', '', 4, 'Doris Database', 'Doris', 'Olap'); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('clickhouse', 'clickhouse数据库', 'clickhouse', 'olap', '', 4, 'Clickhouse Database', 'Clickhouse', 'Olap'); select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'mongodb'; From e24da00b38ba7fd04740f73cef929b16061e8882 Mon Sep 17 00:00:00 2001 From: binbincheng <106590848+binbinCheng@users.noreply.github.com> Date: Thu, 23 Mar 2023 01:25:17 +0800 Subject: [PATCH 071/689] Improve exception information and add path information (#4351) Co-authored-by: casionone --- .../org/apache/linkis/storage/excel/XlsUtils.java | 1 + .../org/apache/linkis/storage/fs/FileSystem.java | 6 ++++++ .../linkis/storage/fs/impl/LocalFileSystem.java | 15 +++++++++++++-- .../resultset/DefaultResultSetFactory.scala | 2 +- .../storage/resultset/ResultSetReader.scala | 4 +++- .../storage/resultset/StorageResultSet.scala | 1 + .../resultset/StorageResultSetWriter.scala | 1 + 7 files changed, 26 insertions(+), 4 deletions(-) diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java index f3b8f448655..246fb79bc3a 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java @@ -61,6 +61,7 @@ public static String excelToCsv( throws Exception { String hdfsPath = "/tmp/" + StorageUtils.getJvmUser() + "/" + System.currentTimeMillis() + ".csv"; + LOG.info("The excel to csv with hdfsPath:" + hdfsPath); ExcelXlsReader xlsReader = new ExcelXlsReader(); RowToCsvDeal rowToCsvDeal = new RowToCsvDeal(); OutputStream out = null; diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java index c19213a9cce..3067383b6c6 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java @@ -24,8 +24,13 @@ import java.io.File; import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public abstract class FileSystem implements Fs { + private static final Logger LOG = LoggerFactory.getLogger(FileSystem.class); + protected String user; private String defaultFilePerm = "rwxr-----"; // 740 private String defaultFolderPerm = "rwxr-x---"; // 750 @@ -94,6 +99,7 @@ protected FsPath getParentPath(String path) { } else { parentPath = path.substring(0, path.lastIndexOf("/")); } + LOG.info("Get Parent Path:" + parentPath); return new FsPath(parentPath); } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java index ce2ee43b7e7..2df547f10eb 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java @@ -75,12 +75,14 @@ public String listRoot() throws IOException { @Override public long getTotalSpace(FsPath dest) throws IOException { String path = dest.getPath(); + LOG.info("Get total space with path:" + path); return new File(path).getTotalSpace(); } @Override public long getFreeSpace(FsPath dest) throws IOException { String path = dest.getPath(); + LOG.info("Get free space with path:" + path); return new File(path).getFreeSpace(); } @@ -117,6 +119,7 @@ public boolean setOwner(FsPath dest, String user, String group) throws IOExcepti @Override public boolean setOwner(FsPath dest, String user) throws IOException { + LOG.info("Set owner with path:" + dest.getPath() + "and user:" + user); if (!StorageUtils.isIOProxy()) { LOG.info("io not proxy, setOwner skip"); return true; @@ -133,6 +136,7 @@ public boolean setOwner(FsPath dest, String user) throws IOException { @Override public boolean setGroup(FsPath dest, String group) throws IOException { + LOG.info("Set group with path:" + dest.getPath() + "and group:" + user); if (!StorageUtils.isIOProxy()) { LOG.info("io not proxy, setGroup skip"); return true; @@ -155,6 +159,7 @@ public boolean mkdir(FsPath dest) throws IOException { @Override public boolean mkdirs(FsPath dest) throws IOException { String path = dest.getPath(); + LOG.info("Try to mkdirs with path:" + path); File file = new File(path); // Create parent directories one by one and set their permissions to rwxrwxrwx. Stack dirsToMake = new Stack(); @@ -182,6 +187,7 @@ public boolean mkdirs(FsPath dest) throws IOException { } public boolean canMkdir(FsPath destParentDir) throws IOException { + LOG.info("Try to check if the directory can be created with path:" + destParentDir.getPath()); if (!StorageUtils.isIOProxy()) { LOG.debug("io not proxy, not check owner, just check if have write permission "); return this.canWrite(destParentDir); @@ -203,6 +209,7 @@ public boolean canMkdir(FsPath destParentDir) throws IOException { @Override public boolean copy(String origin, String dest) throws IOException { File file = new File(dest); + LOG.info("Try to copy file from:" + origin + " to dest:" + dest); if (!isOwner(file.getParent())) { throw new IOException("you have on permission to create file " + dest); } @@ -225,6 +232,7 @@ public boolean copy(String origin, String dest) throws IOException { @Override public boolean setPermission(FsPath dest, String permission) throws IOException { + LOG.info("Try to set permission dest with path:" + dest.getPath()); if (!StorageUtils.isIOProxy()) { LOG.info("io not proxy, setPermission as parent."); try { @@ -251,6 +259,7 @@ public boolean setPermission(FsPath dest, String permission) throws IOException public FsPathListWithError listPathWithError(FsPath path) throws IOException { File file = new File(path.getPath()); File[] files = file.listFiles(); + LOG.info("Try to list path:" + path.getPath() + " with error msg"); if (files != null) { List rtn = new ArrayList(); String message = ""; @@ -294,6 +303,7 @@ public void init(Map properties) throws IOException { String groupInfo; try { groupInfo = Utils.exec(new String[] {"id", user}); + LOG.info("Get groupinfo:" + groupInfo + " with shell command: id " + user); } catch (RuntimeException e) { group = user; return; @@ -322,7 +332,7 @@ public FsPath get(String dest) throws IOException { } else { fsPath = new FsPath(dest); } - + LOG.info("Try to get FsPath with path:" + fsPath.getPath()); PosixFileAttributes attr = null; try { attr = Files.readAttributes(Paths.get(fsPath.getPath()), PosixFileAttributes.class); @@ -365,7 +375,7 @@ public OutputStream write(FsPath dest, boolean overwrite) throws IOException { @Override public boolean create(String dest) throws IOException { - + LOG.info("try to create file with path:" + dest); File file = new File(dest); if (!isOwner(file.getParent())) { throw new IOException("you have on permission to create file " + dest); @@ -391,6 +401,7 @@ public boolean create(String dest) throws IOException { public List list(FsPath path) throws IOException { File file = new File(path.getPath()); File[] files = file.listFiles(); + LOG.info("Try to get file list with path:" + path.getPath()); if (files != null) { List rtn = new ArrayList(); for (File f : files) { diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala index 38973ae1ab4..d4836731dbd 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala @@ -108,7 +108,7 @@ class DefaultResultSetFactory extends ResultSetFactory with Logging { if (StringUtils.isEmpty(resultSetType)) { throw new StorageWarnException( THE_FILE_IS_EMPTY.getErrorCode, - s"The file (${fsPath.getPath}) is empty(文件(${fsPath.getPath}) 为空)" + MessageFormat.format(THE_FILE_IS_EMPTY.getErrorDesc, fsPath.getPath) ) } Utils.tryQuietly(inputStream.close()) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala index 663e379b5b6..e61cf36b3dd 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala @@ -19,6 +19,7 @@ package org.apache.linkis.storage.resultset import org.apache.linkis.common.io.{FsPath, MetaData, Record} import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetReader} +import org.apache.linkis.common.utils.Logging import org.apache.linkis.storage.FSFactory import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED import org.apache.linkis.storage.exception.StorageErrorException @@ -26,7 +27,7 @@ import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord, Ta import java.io.InputStream -object ResultSetReader { +object ResultSetReader extends Logging { def getResultSetReader[K <: MetaData, V <: Record]( resultSet: ResultSet[K, V], @@ -83,6 +84,7 @@ object ResultSetReader { ) } val fs = FSFactory.getFs(resPath) + logger.info("Try to init Fs with path:" + resPath.getPath) fs.init(null) ResultSetReader.getResultSetReader(resultSet.asInstanceOf[TableResultSet], fs.read(resPath)) } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala index fc303fbb5c2..7b3aca62d96 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala @@ -34,6 +34,7 @@ abstract class StorageResultSet[K <: MetaData, V <: Record] extends ResultSet[K, } else { parentDir.toPath + "/" + fileName + Dolphin.DOLPHIN_FILE_SUFFIX } + logger.info(s"Get result set path:${path}") new FsPath(path) } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala index 50bc0d276b4..de849078bd0 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala @@ -86,6 +86,7 @@ class StorageResultSetWriter[K <: MetaData, V <: Record]( WRITER_LOCK_CREATE.synchronized { if (!fileCreated) { if (storePath != null && outputStream == null) { + logger.info(s"Try to create a new file:${storePath}, with proxy user:${proxyUser}") fs = FSFactory.getFsByProxyUser(storePath, proxyUser) fs.init(null) FileSystemUtils.createNewFile(storePath, proxyUser, true) From 55267ff3579fc7daa62f12beeb9d4fcfcb4de6f1 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Fri, 24 Mar 2023 18:17:58 +0800 Subject: [PATCH 072/689] revert EngineTypeLabel (#4406) Co-authored-by: aiceflower --- .../label/entity/engine/EngineTypeLabel.java | 15 ---------- .../entity/engine/EngineTypeLabelTest.java | 28 +------------------ 2 files changed, 1 insertion(+), 42 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java index 912f6c9f948..09492b146ac 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java @@ -74,19 +74,4 @@ public void setVersion(String version) { public Boolean isEmpty() { return StringUtils.isBlank(getEngineType()) || StringUtils.isBlank(getVersion()); } - - @Override - protected void setStringValue(String stringValue) { - String version; - String engineType = stringValue.split("-")[0]; - - if (engineType.equals("*")) { - version = stringValue.replaceFirst("[" + engineType + "]-", ""); - } else { - version = stringValue.replaceFirst(engineType + "-", ""); - } - - setEngineType(engineType); - setVersion(version); - } } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java index cc4171b4377..dd90e1050bd 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/test/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabelTest.java @@ -17,31 +17,5 @@ package org.apache.linkis.manager.label.entity.engine; -import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactory; -import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - /** EngineTypeLabel Tester */ -public class EngineTypeLabelTest { - - @Test - public void testSetStringValue() { - String engineType = "hive"; - String version = "1.1.0-cdh5.12.0"; - - String engineType1 = "*"; - String version1 = "*"; - - LabelBuilderFactory labelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory(); - EngineTypeLabel engineTypeLabel = labelBuilderFactory.createLabel(EngineTypeLabel.class); - engineTypeLabel.setStringValue(engineType + "-" + version); - Assertions.assertEquals(engineTypeLabel.getEngineType(), engineType); - Assertions.assertEquals(engineTypeLabel.getVersion(), version); - - engineTypeLabel.setStringValue(engineType1 + "-" + version1); - Assertions.assertEquals(engineTypeLabel.getEngineType(), engineType1); - Assertions.assertEquals(engineTypeLabel.getVersion(), version1); - } -} +public class EngineTypeLabelTest {} From 24dc97820a347322f184079ef71f6964501647b0 Mon Sep 17 00:00:00 2001 From: wangzhen Date: Fri, 24 Mar 2023 19:16:51 +0800 Subject: [PATCH 073/689] Fix socket session resource leakage (#4407) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 王振 --- .../springcloud/websocket/WebsocketGatewaySession.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/scala/org/apache/linkis/gateway/springcloud/websocket/WebsocketGatewaySession.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/scala/org/apache/linkis/gateway/springcloud/websocket/WebsocketGatewaySession.scala index c9c05776dcc..29c42996f17 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/scala/org/apache/linkis/gateway/springcloud/websocket/WebsocketGatewaySession.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/scala/org/apache/linkis/gateway/springcloud/websocket/WebsocketGatewaySession.scala @@ -84,8 +84,7 @@ class GatewayWebSocketSessionConnection( } } - // todo - def getAddress: InetSocketAddress = null + def getAddress: InetSocketAddress = webSocketSession.getHandshakeInfo.getRemoteAddress; def getProxyWebSocketSession( serviceInstance: ServiceInstance @@ -192,8 +191,7 @@ class GatewayWebSocketSession private ( protected var webSocketConnection: WebSocketConnection = _ - // todo - def isAlive: Boolean = true + def isAlive: Boolean = !webSocketConnection.getInbound.receiveCloseStatus().subscribe().isDisposed override def receive(): Flux[WebSocketMessage] = webSocketConnection.getInbound .aggregateFrames(ServerConfiguration.BDP_SERVER_SOCKET_TEXT_MESSAGE_SIZE_MAX.getValue.toInt) From c5164494e7a837af92fec1ca7674908b13b11e51 Mon Sep 17 00:00:00 2001 From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com> Date: Fri, 24 Mar 2023 20:32:23 +0800 Subject: [PATCH 074/689] remove hadoop version from metadata query service (#4403) Co-authored-by: gf13871 --- .../linkis-datasource/linkis-metadata-query/service/hive/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml index b0c9cdef363..771a90cdd04 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml @@ -26,7 +26,6 @@ linkis-metadata-query-service-hive - 2.7.2 4.2.4 From 304fb3f4624f45e5c25f6e6d41a8ea2e9904b32a Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Mon, 27 Mar 2023 14:43:01 +0800 Subject: [PATCH 075/689] Feat move dss gateway support (#4408) * feat(linkis-mg): move dss-gateway-support code to linkis --- .github/workflows/build-backend.yml | 2 +- .mvn/wrapper/maven-wrapper.properties | 2 +- linkis-dist/package/conf/log4j2.xml | 8 +- linkis-dist/pom.xml | 5 + .../query/service/HdfsMetaService.java | 1 + .../linkis-eureka/pom.xml | 6 +- .../gateway/parser}/RouteLabelParser.scala | 2 +- .../linkis-gateway-server-support/pom.xml | 16 +- .../src/main/assembly/distribution.xml | 5 - .../dss/parser/DSSGatewayConfiguration.scala | 31 +++ .../gateway/dss/parser/DSSGatewayParser.scala | 258 ++++++++++++++++++ .../dss/parser/DSSRouteLabelParser.scala | 35 +++ .../EntranceExecutionGatewayParser.scala | 3 +- .../route/DefaultLabelGatewayRouter.scala | 2 +- .../route/GatewayRouterConfiguration.scala | 2 +- linkis-spring-cloud-services/pom.xml | 1 - 16 files changed, 348 insertions(+), 31 deletions(-) rename linkis-spring-cloud-services/linkis-service-gateway/{linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/route/label => linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/parser}/RouteLabelParser.scala (98%) create mode 100644 linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayConfiguration.scala create mode 100644 linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala create mode 100644 linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSRouteLabelParser.scala diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml index fec9f02f634..dcbb141870c 100644 --- a/.github/workflows/build-backend.yml +++ b/.github/workflows/build-backend.yml @@ -20,7 +20,7 @@ name: Build Backend on: [push, pull_request] env: - MAVEN_OPTS: -Dmaven.resolver.transport=wagon -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + MAVEN_OPTS: -Dmaven.resolver.transport=wagon -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 jobs: build-backend: diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index 08ea486aa5a..d8b2495a1e0 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -14,5 +14,5 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.0/apache-maven-3.9.0-bin.zip +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.1/apache-maven-3.9.1-bin.zip wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar diff --git a/linkis-dist/package/conf/log4j2.xml b/linkis-dist/package/conf/log4j2.xml index c10f78e6c73..f91717325b8 100644 --- a/linkis-dist/package/conf/log4j2.xml +++ b/linkis-dist/package/conf/log4j2.xml @@ -17,9 +17,13 @@ --> + + ${env:LINKIS_LOG_DIR:-logs} + ${sys:serviceName:-linkis} + - + diff --git a/linkis-dist/pom.xml b/linkis-dist/pom.xml index b847950a796..54012cdb86d 100644 --- a/linkis-dist/pom.xml +++ b/linkis-dist/pom.xml @@ -26,6 +26,11 @@ linkis-dist pom + + true + true + + org.apache.linkis diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsMetaService.java index 7d8e1058721..e546a32a1da 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsMetaService.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsMetaService.java @@ -78,6 +78,7 @@ public Map queryConnectionInfo( HdfsConnection connection, Map queryParams) { List filterRules = new ArrayList<>(); AtomicReference uriReference = new AtomicReference<>(); + LOG.info("query hdfs ConnectionInfo for uri: {}", queryParams.get("uri")); Optional.ofNullable(queryParams.get("uri")) .ifPresent( uri -> { diff --git a/linkis-spring-cloud-services/linkis-service-discovery/linkis-eureka/pom.xml b/linkis-spring-cloud-services/linkis-service-discovery/linkis-eureka/pom.xml index 72a57fa25cc..e432fcd2963 100644 --- a/linkis-spring-cloud-services/linkis-service-discovery/linkis-eureka/pom.xml +++ b/linkis-spring-cloud-services/linkis-service-discovery/linkis-eureka/pom.xml @@ -23,10 +23,14 @@ ${revision} ../../../pom.xml - linkis-eureka jar + + true + true + + org.springframework.cloud diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/route/label/RouteLabelParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/parser/RouteLabelParser.scala similarity index 98% rename from linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/route/label/RouteLabelParser.scala rename to linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/parser/RouteLabelParser.scala index 3bddda3de4c..9a01fbbbffb 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/route/label/RouteLabelParser.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/parser/RouteLabelParser.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.gateway.ujes.route.label +package org.apache.linkis.gateway.parser import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.gateway.http.GatewayContext diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/pom.xml b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/pom.xml index ef4635ae00d..8d8c131abb8 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/pom.xml +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/pom.xml @@ -94,19 +94,6 @@ jackson-databind - - - com.webank.wedatasphere.dss - dss-gateway-support - 1.1.1 - - - org.apache.linkis - linkis-gateway-server-support - - - - @@ -115,7 +102,6 @@ net.alchim31.maven scala-maven-plugin - org.apache.maven.plugins maven-assembly-plugin @@ -126,7 +112,7 @@ false false - /src/main/assembly/distribution.xml + src/main/assembly/distribution.xml diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/assembly/distribution.xml b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/assembly/distribution.xml index 7180fea9cec..cf2fd123222 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/assembly/distribution.xml +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/assembly/distribution.xml @@ -21,7 +21,6 @@ linkis-gateway dir - zip false linkis-gateway @@ -44,9 +43,5 @@ - - - - diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayConfiguration.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayConfiguration.scala new file mode 100644 index 00000000000..8dd859e02d2 --- /dev/null +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayConfiguration.scala @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.gateway.dss.parser + +import org.apache.linkis.common.conf.CommonVars + +object DSSGatewayConfiguration { + val DSS_SPRING_NAME = CommonVars("wds.linkis.dss.name", "dss-server") + + val DSS_URL_LABEL_PREFIX = CommonVars("wds.dss.gateway.url.prefix.name", "labels") + + val DSS_URL_ROUTE_LABEL_PREFIX = CommonVars("wds.dss.gateway.url.prefix.name", "labelsRoute") + + val DSS_URL_APPCONNS = CommonVars("wds.dss.gateway.url.appconns", "visualis") + +} diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala new file mode 100644 index 00000000000..bdf8dfbe5f5 --- /dev/null +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.gateway.dss.parser + +import org.apache.linkis.common.ServiceInstance +import org.apache.linkis.gateway.exception.TooManyServiceException +import org.apache.linkis.gateway.http.GatewayContext +import org.apache.linkis.gateway.parser.AbstractGatewayParser +import org.apache.linkis.gateway.springcloud.SpringCloudGatewayConfiguration.{ + normalPath, + API_URL_PREFIX +} +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.apache.linkis.protocol.constants.TaskConstant +import org.apache.linkis.rpc.sender.SpringCloudFeignConfigurationCache +import org.apache.linkis.server.BDPJettyServerHelper + +import org.springframework.stereotype.Component + +import java.util +import java.util.Locale + +import scala.collection.JavaConverters._ + +@Component +class DSSGatewayParser extends AbstractGatewayParser { + + val appConns = DSSGatewayConfiguration.DSS_URL_APPCONNS.getValue.split(",") + + override def shouldContainRequestBody(gatewayContext: GatewayContext): Boolean = { + var contentType = gatewayContext.getRequest.getHeaders.get("Content-Type") + if (null == contentType) { + contentType = gatewayContext.getRequest.getHeaders.get("content-type") + } + + if ( + contentType != null && contentType.nonEmpty + && contentType(0).contains("form-data") + ) { + logger.info("DSS gateway get request type is form-data") + return false + } + + gatewayContext.getRequest.getRequestURI match { + case DSSGatewayParser.DSS_URL_DEFAULT_REGEX(_, _) => true + case DSSGatewayParser.DSS_URL_REGEX(_, _, _) => true + case DSSGatewayParser.APPCONN_URL_DEFAULT_REGEX(_, appconn, _) + if appConns.contains(appconn) => + true + case _ => false + } + } + + override def parse(gatewayContext: GatewayContext): Unit = + gatewayContext.getRequest.getRequestURI match { + + case DSSGatewayParser.DSS_URL_REGEX(version, firstName, secondName) => + if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return + var tmpServerName = "dss-" + firstName + "-" + secondName + "-server" + tmpServerName = getServiceNameFromLabel(gatewayContext, tmpServerName) + val serviceName: Option[String] = + findCommonService("dss/" + firstName + "/" + secondName, tmpServerName) + if (serviceName.isDefined) { + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(serviceName.get, null)) + } else { + logger.info( + "Now set default serviceInstance name " + DSSGatewayConfiguration.DSS_SPRING_NAME.getValue + "," + gatewayContext.getRequest.getRequestURI + ) + gatewayContext.getGatewayRoute.setServiceInstance( + ServiceInstance(DSSGatewayConfiguration.DSS_SPRING_NAME.getValue, null) + ) + } + case DSSGatewayParser.DSS_URL_DEFAULT_REGEX(version, firstName) => + if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return + var tmpServerName = "dss-" + firstName + "-server" + tmpServerName = getServiceNameFromLabel(gatewayContext, tmpServerName) + val serviceName: Option[String] = findCommonService("dss/" + firstName, tmpServerName) + if (serviceName.isDefined) { + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(serviceName.get, null)) + } else { + logger.info( + "Now set default serviceInstance name " + DSSGatewayConfiguration.DSS_SPRING_NAME.getValue + "," + gatewayContext.getRequest.getRequestURI + ) + gatewayContext.getGatewayRoute.setServiceInstance( + ServiceInstance(DSSGatewayConfiguration.DSS_SPRING_NAME.getValue, null) + ) + } + case DSSGatewayParser.APPCONN_URL_DEFAULT_REGEX(version, serverName, _) + if appConns.contains(serverName) => + if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return + var tmpServerName = serverName + tmpServerName = getServiceNameFromLabel(gatewayContext, tmpServerName) + val serviceName: Option[String] = findCommonService(tmpServerName, tmpServerName) + if (serviceName.isDefined) { + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(serviceName.get, null)) + } else { + logger.info( + "Now set default serviceInstance name " + DSSGatewayConfiguration.DSS_SPRING_NAME.getValue + "," + gatewayContext.getRequest.getRequestURI + ) + gatewayContext.getGatewayRoute.setServiceInstance( + ServiceInstance(DSSGatewayConfiguration.DSS_SPRING_NAME.getValue, null) + ) + } + case _ => + } + + private def getServiceNameFromLabel( + gatewayContext: GatewayContext, + tmpServiceName: String + ): String = { + var requestUrlLabels = gatewayContext.getRequest.getQueryParams + .getOrDefault(DSSGatewayConfiguration.DSS_URL_LABEL_PREFIX.getValue, null) + if (requestUrlLabels == null) { + requestUrlLabels = gatewayContext.getRequest.getQueryParams + .getOrDefault(DSSGatewayConfiguration.DSS_URL_ROUTE_LABEL_PREFIX.getValue, null) + } + logger.info( + "Get ServiceName From Label and method is " + gatewayContext.getRequest.getMethod.toString + ",and urlLabels is " + requestUrlLabels + ) + val requestMethod = gatewayContext.getRequest.getMethod.toLowerCase(Locale.getDefault()) + if ( + requestUrlLabels == null && (requestMethod + .equals("post") || requestMethod.equals("put") || requestMethod.equals("delete")) + ) { + val requestBody = Option(gatewayContext.getRequest.getRequestBody) + val routeLabelList = new util.ArrayList[RouteLabel]() + + requestBody match { + // todo form-data resolve + case Some(body) => + val labelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory + val json = + BDPJettyServerHelper.gson.fromJson(body, classOf[java.util.Map[String, Object]]) + val labels: util.List[Label[_]] = json.get(TaskConstant.LABELS) match { + case map: util.Map[String, Object] => labelBuilderFactory.getLabels(map) + case map: util.Map[String, Any] => labelBuilderFactory.getLabels(map.asInstanceOf) + case _ => new util.ArrayList[Label[_]]() + } + labels.asScala + .filter(label => label.isInstanceOf[RouteLabel]) + .foreach(label => { + routeLabelList.add(label.asInstanceOf[RouteLabel]) + }) + + case _ => null + } + val labelNameList = routeLabelList.asScala.map(routeLabel => routeLabel.getStringValue).toList + if (labelNameList != null && labelNameList.size > 0) { + genServiceNameByDSSLabel(labelNameList, tmpServiceName) + } else if (null != requestUrlLabels) { + genServiceNameByDSSLabel(requestUrlLabels.toList, tmpServiceName) + } else tmpServiceName + + } else { + if (requestUrlLabels != null) { + genServiceNameByDSSLabel(requestUrlLabels.toList, tmpServiceName) + } else tmpServiceName + } + } + + private def genServiceNameByDSSLabel(labelList: List[String], tmpServiceName: String): String = { + var resultName = tmpServiceName + if (null != labelList && labelList.size > 0) { + val labelNameList = labelList(0).replace(" ", "").split(",").toList + if (labelNameList.size > 0) { + if (labelNameList.find(name => name.equalsIgnoreCase("dev")).isDefined) { + resultName = tmpServiceName + "-dev" + } else if (labelNameList.find(name => name.equalsIgnoreCase("prod")).isDefined) { + resultName = tmpServiceName + "-prod" + } else if (labelNameList.find(name => name.equalsIgnoreCase("test")).isDefined) { + resultName = tmpServiceName + "-test" + } else { + resultName = tmpServiceName + } + } + } + resultName + } + + private def findCommonService(parsedServiceId: String, tmpServerName: String) = findService( + parsedServiceId, + tmpServerName, + services => { + val errorMsg = new TooManyServiceException( + s"Cannot find a correct serviceId for parsedServiceId $parsedServiceId, service list is: " + services + ) + warn("", errorMsg) + throw errorMsg + } + ) + + protected def findService( + parsedServiceId: String, + tmpServerName: String, + tooManyDeal: List[String] => Option[String] + ): Option[String] = { + val findIt: (String => Boolean) => Option[String] = op => { + val services = + SpringCloudFeignConfigurationCache.getDiscoveryClient.getServices.asScala.filter(op).toList + if (services.length == 1) Some(services.head) + else if (services.length > 1) tooManyDeal(services) + else None + } + // 通过匹配到最多的url中的path进行路由,如/dss/framework/workspace/ 会匹配到 dss-framework-workspace-server 而不是 dss-server + // 如果产生了相等的情况，则按照短的service名字为准 比如/dss/getProject, + // 我们可能会匹配到dss-server以及 dss-framework-workspace-server,则选择短名称的dss-server + val findMostCorrect: (String => (String, Int)) => Option[String] = { op => + { + val serviceMap = + SpringCloudFeignConfigurationCache.getDiscoveryClient.getServices.asScala.map(op).toMap + var count = 0 + var retService: Option[String] = None + serviceMap.foreach { case (k, v) => + if (v > count) { + count = v + retService = Some(k) + } else if (retService.isDefined && v == count && k.length < retService.get.length) { + retService = Some(k) + } + } + retService + } + } + val lowerServiceId = parsedServiceId.toLowerCase(Locale.getDefault()) + val serverName = tmpServerName.toLowerCase(Locale.getDefault()) + findIt(_.toLowerCase(Locale.getDefault()) == serverName).orElse(findMostCorrect(service => { + (service, lowerServiceId.split("/").count(word => service.contains(word))) + })) + } + +} + +object DSSGatewayParser { + val DSS_HEADER = normalPath(API_URL_PREFIX) + "rest_[a-zA-Z][a-zA-Z_0-9]*/(v\\d+)/dss/" + val DSS_URL_REGEX = (DSS_HEADER + "([^/]+)/" + "([^/]+)/.+").r + val DSS_URL_DEFAULT_REGEX = (DSS_HEADER + "([^/]+).+").r + + val APPCONN_HEADER = normalPath(API_URL_PREFIX) + "rest_[a-zA-Z][a-zA-Z_0-9]*/(v\\d+)/([^/]+)/" + val APPCONN_URL_DEFAULT_REGEX = (APPCONN_HEADER + "([^/]+).+").r + +} diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSRouteLabelParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSRouteLabelParser.scala new file mode 100644 index 00000000000..48ec059cadd --- /dev/null +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSRouteLabelParser.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.gateway.dss.parser + +import org.apache.linkis.gateway.http.GatewayContext +import org.apache.linkis.gateway.parser.RouteLabelParser +import org.apache.linkis.manager.label.entity.route.RouteLabel + +import org.springframework.stereotype.Component + +import java.util + +@Component +class DSSRouteLabelParser extends RouteLabelParser { + + override def parse(gatewayContext: GatewayContext): util.List[RouteLabel] = { + new util.ArrayList[RouteLabel]() + } + +} diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceExecutionGatewayParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceExecutionGatewayParser.scala index d5c4df23e85..f4610519045 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceExecutionGatewayParser.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/parser/EntranceExecutionGatewayParser.scala @@ -20,9 +20,8 @@ package org.apache.linkis.gateway.ujes.parser import org.apache.linkis.common.ServiceInstance import org.apache.linkis.gateway.config.GatewayConfiguration import org.apache.linkis.gateway.http.GatewayContext -import org.apache.linkis.gateway.parser.AbstractGatewayParser +import org.apache.linkis.gateway.parser.{AbstractGatewayParser, RouteLabelParser} import org.apache.linkis.gateway.springcloud.SpringCloudGatewayConfiguration._ -import org.apache.linkis.gateway.ujes.route.label.RouteLabelParser import org.apache.linkis.instance.label.service.InsLabelService import org.apache.linkis.manager.label.entity.route.RouteLabel diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/route/DefaultLabelGatewayRouter.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/route/DefaultLabelGatewayRouter.scala index d1c2318ae0f..70b0bfd4b6d 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/route/DefaultLabelGatewayRouter.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/route/DefaultLabelGatewayRouter.scala @@ -21,7 +21,7 @@ import org.apache.linkis.common.ServiceInstance import org.apache.linkis.gateway.errorcode.LinkisGatewayCoreErrorCodeSummary._ import org.apache.linkis.gateway.exception.GatewayErrorException import org.apache.linkis.gateway.http.GatewayContext -import org.apache.linkis.gateway.ujes.route.label.RouteLabelParser +import org.apache.linkis.gateway.parser.RouteLabelParser import org.apache.linkis.manager.label.entity.route.RouteLabel import org.apache.commons.lang3.StringUtils diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/route/GatewayRouterConfiguration.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/route/GatewayRouterConfiguration.scala index 418d1925979..208cd7faece 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/route/GatewayRouterConfiguration.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/ujes/route/GatewayRouterConfiguration.scala @@ -18,8 +18,8 @@ package org.apache.linkis.gateway.ujes.route import org.apache.linkis.common.utils.Logging +import org.apache.linkis.gateway.parser.{GenericRoueLabelParser, RouteLabelParser} import org.apache.linkis.gateway.springcloud.SpringCloudGatewayConfiguration -import org.apache.linkis.gateway.ujes.route.label.{GenericRoueLabelParser, RouteLabelParser} import org.springframework.boot.autoconfigure.AutoConfigureBefore import org.springframework.context.annotation.{Bean, Configuration, Scope} diff --git a/linkis-spring-cloud-services/pom.xml b/linkis-spring-cloud-services/pom.xml index 74473095b99..f2a6159169b 100644 --- a/linkis-spring-cloud-services/pom.xml +++ b/linkis-spring-cloud-services/pom.xml @@ -28,7 +28,6 @@ pom - linkis-service-discovery/linkis-eureka linkis-service-gateway From 7005c01d7f7bca78322447f4f2f32b8398645687 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Mon, 27 Mar 2023 16:00:37 +0800 Subject: [PATCH 076/689] update jdbc connect logic (#4412) * update jdbc connect logic --- .../linkis/common/utils/SecurityUtils.java | 194 +++++++--- .../common/utils/SecurityUtilsTest.java | 338 +++++++++++++----- .../engineplugin/jdbc/ConnectionManager.java | 13 +- .../jdbc/utils/JdbcParamUtils.java | 39 -- .../jdbc/utils/JdbcParamUtilsTest.java | 124 ------- .../query/service/mysql/SqlConnection.java | 36 +- 6 files changed, 408 insertions(+), 336 deletions(-) diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java index 2e77a14c0e9..c145f4728d0 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java @@ -25,10 +25,9 @@ import java.io.UnsupportedEncodingException; import java.net.URLDecoder; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -46,8 +45,12 @@ public abstract class SecurityUtils { private static final String QUESTION_MARK = "?"; + private static final String REGEX_QUESTION_MARK = "\\?"; + + private static final int JDBC_URL_ITEM_COUNT = 2; + /** allowLoadLocalInfile,allowLoadLocalInfiled,# */ - public static final CommonVars MYSQL_SENSITIVE_PARAMS = + private static final CommonVars MYSQL_SENSITIVE_PARAMS = CommonVars$.MODULE$.apply( "linkis.mysql.sensitive.params", "allowLoadLocalInfile,autoDeserialize,allowLocalInfile,allowUrlInLocalInfile,#"); @@ -55,16 +58,113 @@ public abstract class SecurityUtils { /** * "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false" */ - public static final CommonVars MYSQL_FORCE_PARAMS = + private static final CommonVars MYSQL_FORCE_PARAMS = CommonVars$.MODULE$.apply( "linkis.mysql.force.params", "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"); - public static final CommonVars MYSQL_STRONG_SECURITY_ENABLE = + private static final CommonVars MYSQL_STRONG_SECURITY_ENABLE = CommonVars$.MODULE$.apply("linkis.mysql.strong.security.enable", "false"); + private static final CommonVars MYSQL_CONNECT_URL = + CommonVars.apply("linkis.security.mysql.url.template", "jdbc:mysql://%s:%s/%s"); + + private static final String JDBC_MATCH_REGEX = + "(?i)jdbc:(?i)(mysql)://([a-zA-Z0-9]+\\.)*[a-zA-Z0-9]+(:[0-9]+)?(/[a-zA-Z0-9_-]*[\\.\\-]?)?"; + + private static final String JDBC_MYSQL_PROTOCOL = "jdbc:mysql"; + + /** + * check mysql connection params + * + * @param host + * @param port + * @param username + * @param password + * @param database + * @param extraParams + */ + public static void checkJdbcConnParams( + String host, + Integer port, + String username, + String password, + String database, + Map extraParams) { + // 1. Check blank params + if (StringUtils.isAnyBlank(host, username, password)) { + logger.info( + "Invalid mysql connection params: host: {}, username: {}, password: {}, database: {}", + host, + username, + password, + database); + throw new LinkisSecurityException(35000, "Invalid mysql connection params."); + } + + // 2. Check url format + String url = String.format(MYSQL_CONNECT_URL.getValue(), host.trim(), port, database.trim()); + checkUrl(url); + + // 3. Check params. Mainly vulnerability parameters. Note the url encoding + checkParams(extraParams); + } + + /** @param url */ + public static void checkJdbcConnUrl(String url) { + logger.info("jdbc url: {}", url); + if (StringUtils.isBlank(url)) { + throw new LinkisSecurityException(35000, "Invalid jdbc connection url."); + } + + // temporarily only check mysql jdbc url. + if (!url.toLowerCase().startsWith(JDBC_MYSQL_PROTOCOL)) { + return; + } + + String[] urlItems = url.split(REGEX_QUESTION_MARK); + if (urlItems.length > JDBC_URL_ITEM_COUNT) { + throw new LinkisSecurityException(35000, "Invalid jdbc connection url."); + } + + // check url + checkUrl(urlItems[0]); + + // check params + if (urlItems.length == JDBC_URL_ITEM_COUNT) { + Map params = parseMysqlUrlParamsToMap(urlItems[1]); + checkParams(params); + } + } + + /** + * call after checkJdbcConnUrl + * + * @param url + * @return + */ + public static String getJdbcUrl(String url) { + // preventing NPE + if (StringUtils.isBlank(url)) { + return url; + } + // temporarily deal with only mysql jdbc url. + if (!url.toLowerCase().startsWith(JDBC_MYSQL_PROTOCOL)) { + return url; + } + String[] items = url.split(REGEX_QUESTION_MARK); + String result = items[0]; + if (items.length == JDBC_URL_ITEM_COUNT) { + Map params = parseMysqlUrlParamsToMap(items[1]); + appendMysqlForceParams(params); + String paramUrl = parseParamsMapToMysqlParamUrl(params); + result += QUESTION_MARK + paramUrl; + } + return result; + } + /** - * mysql url append force params + * append force params, Should be called after the checkJdbcConnParams method * * @param url * @return @@ -73,6 +173,9 @@ public static String appendMysqlForceParams(String url) { if (StringUtils.isBlank(url)) { return ""; } + if (!Boolean.valueOf(MYSQL_STRONG_SECURITY_ENABLE.getValue())) { + return url; + } String extraParamString = MYSQL_FORCE_PARAMS.getValue(); @@ -86,36 +189,41 @@ public static String appendMysqlForceParams(String url) { return url; } + /** + * append force params, Should be called after the checkJdbcConnParams method + * + * @param extraParams + */ public static void appendMysqlForceParams(Map extraParams) { - extraParams.putAll(parseMysqlUrlParamsToMap(MYSQL_FORCE_PARAMS.getValue())); + if (Boolean.valueOf(MYSQL_STRONG_SECURITY_ENABLE.getValue())) { + extraParams.putAll(parseMysqlUrlParamsToMap(MYSQL_FORCE_PARAMS.getValue())); + } } - public static String checkJdbcSecurity(String url) { - logger.info("checkJdbcSecurity origin url: {}", url); - if (StringUtils.isBlank(url)) { - throw new LinkisSecurityException(35000, "Invalid mysql connection cul, url is empty"); - } - // deal with url encode - try { - url = URLDecoder.decode(url, "UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new LinkisSecurityException(35000, "mysql connection cul decode error: " + e); + public static String parseParamsMapToMysqlParamUrl(Map params) { + if (params == null || params.isEmpty()) { + return ""; } - if (url.endsWith(QUESTION_MARK) || !url.contains(QUESTION_MARK)) { - logger.info("checkJdbcSecurity target url: {}", url); - return url; + return params.entrySet().stream() + .map(e -> String.join(EQUAL_SIGN, e.getKey(), String.valueOf(e.getValue()))) + .collect(Collectors.joining(AND_SYMBOL)); + } + + /** + * check url, format: jdbc:mysql://host:port/dbname + * + * @param url + */ + public static void checkUrl(String url) { + if (url != null && !url.toLowerCase().startsWith(JDBC_MYSQL_PROTOCOL)) { + return; } - String[] items = url.split("\\?"); - if (items.length != 2) { - logger.warn("Invalid url: {}", url); - throw new LinkisSecurityException(35000, "Invalid mysql connection cul: " + url); + Pattern regex = Pattern.compile(JDBC_MATCH_REGEX); + Matcher matcher = regex.matcher(url); + if (!matcher.matches()) { + logger.info("Invalid mysql connection url: {}", url); + throw new LinkisSecurityException(35000, "Invalid mysql connection url."); } - Map params = parseMysqlUrlParamsToMap(items[1]); - Map securityMap = checkJdbcSecurity(params); - String paramUrl = parseParamsMapToMysqlParamUrl(securityMap); - url = items[0] + QUESTION_MARK + paramUrl; - logger.info("checkJdbcSecurity target url: {}", url); - return url; } /** @@ -123,15 +231,9 @@ public static String checkJdbcSecurity(String url) { * * @param paramsMap */ - public static Map checkJdbcSecurity(Map paramsMap) { - if (paramsMap == null) { - return new HashMap<>(); - } - - // mysql url strong security - if (Boolean.valueOf(MYSQL_STRONG_SECURITY_ENABLE.getValue())) { - paramsMap.clear(); - return paramsMap; + private static void checkParams(Map paramsMap) { + if (paramsMap == null || paramsMap.isEmpty()) { + return; } // deal with url encode @@ -163,16 +265,6 @@ public static Map checkJdbcSecurity(Map paramsMa "Invalid mysql connection parameters: " + parseParamsMapToMysqlParamUrl(paramsMap)); } } - return paramsMap; - } - - public static String parseParamsMapToMysqlParamUrl(Map forceParams) { - if (forceParams == null) { - return ""; - } - return forceParams.entrySet().stream() - .map(e -> String.join(EQUAL_SIGN, e.getKey(), String.valueOf(e.getValue()))) - .collect(Collectors.joining(AND_SYMBOL)); } private static Map parseMysqlUrlParamsToMap(String paramsUrl) { diff --git a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/SecurityUtilsTest.java b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/SecurityUtilsTest.java index 4fdca7b82ac..ed06f9da868 100644 --- a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/SecurityUtilsTest.java +++ b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/SecurityUtilsTest.java @@ -17,166 +17,326 @@ package org.apache.linkis.common.utils; +import org.apache.linkis.common.conf.BDPConfiguration; import org.apache.linkis.common.exception.LinkisSecurityException; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; /** SecurityUtils Tester */ public class SecurityUtilsTest { + @BeforeAll + public static void init() { + BDPConfiguration.set("linkis.mysql.strong.security.enable", "true"); + } + @Test - public void testAppendMysqlForceParamsUrl() throws Exception { - // allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false + public void testCheckUrl() { + // true String url = "jdbc:mysql://127.0.0.1:10000/db_name"; - String newUrl = SecurityUtils.appendMysqlForceParams(url); - Assertions.assertEquals( - url - + "?allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false", - newUrl); - - url = "jdbc:mysql://127.0.0.1:10000/db_name?"; - newUrl = SecurityUtils.appendMysqlForceParams(url); - Assertions.assertEquals( - url - + "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false", - newUrl); - - url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1"; - newUrl = SecurityUtils.appendMysqlForceParams(url); - Assertions.assertEquals( - url - + "&" - + "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false", - newUrl); + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkUrl(url); + }); + // false + String url1 = "jdbc:mysql://127.0.0.1:10000/db_name?"; + Assertions.assertThrows( + LinkisSecurityException.class, + () -> { + SecurityUtils.checkUrl(url1); + }); + // false + String url11 = "jdbc:mysql://127.0.0.1:10000/db_name?abc"; + Assertions.assertThrows( + LinkisSecurityException.class, + () -> { + SecurityUtils.checkUrl(url11); + }); + // true + String url2 = "jdbc:mysql://127.0.0.1:10000/"; + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkUrl(url2); + }); + // true + String url3 = "jdbc:mysql://127.0.0.1:10000"; + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkUrl(url3); + }); + // true + String url4 = "JDBC:mysql://127.0.0.1:10000/db_name"; + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkUrl(url4); + }); + // true + String url5 = "JDBC:H2://127.0.0.1:10000/db_name"; + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkUrl(url5); + }); } @Test - public void testAppendMysqlForceParamsExtraParams() throws Exception { - Map extraParams = new HashMap<>(); - extraParams.put("testKey", "testValue"); - SecurityUtils.appendMysqlForceParams(extraParams); - Assertions.assertEquals("false", extraParams.get("allowLoadLocalInfile")); - Assertions.assertEquals("false", extraParams.get("autoDeserialize")); - Assertions.assertEquals("false", extraParams.get("allowLocalInfile")); - Assertions.assertEquals("false", extraParams.get("allowUrlInLocalInfile")); - Assertions.assertEquals("testValue", extraParams.get("testKey")); - Assertions.assertEquals(null, extraParams.get("otherKey")); + public void testGetUrl() { + BDPConfiguration.set("linkis.mysql.strong.security.enable", "true"); + String baseUrl = "jdbc:mysql://127.0.0.1:10000/db_name"; + String securityStr = + "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"; + String url1 = "jdbc:mysql://127.0.0.1:10000/db_name"; + Assertions.assertEquals(baseUrl, SecurityUtils.getJdbcUrl(url1)); + String url11 = "jdbc:mysql://127.0.0.1:10000/db_name?"; + Assertions.assertEquals(baseUrl, SecurityUtils.getJdbcUrl(url11)); + String url2 = "jdbc:mysql://127.0.0.1:10000/db_name?k1=v1&"; + Assertions.assertEquals(baseUrl + "?k1=v1&" + securityStr, SecurityUtils.getJdbcUrl(url2)); + String url3 = "jdbc:mysql://127.0.0.1:10000/db_name?k1=v1&k2"; + Assertions.assertEquals(baseUrl + "?k1=v1&" + securityStr, SecurityUtils.getJdbcUrl(url3)); } @Test - public void testCheckJdbcSecurityUrl() throws Exception { - String url = "jdbc:mysql://127.0.0.1:10000/db_name"; - String newUrl = SecurityUtils.checkJdbcSecurity(url); - Assertions.assertEquals(url, newUrl); + public void testCheckJdbcConnParams() { + String host = "127.0.0.1"; + Integer port = 3306; + String username = "test"; + String password = "test"; + String database = "tdb"; + Map extraParams = new HashMap<>(); + extraParams.put("k1", "v1"); - url = "jdbc:mysql://127.0.0.1:10000/db_name?"; - newUrl = SecurityUtils.checkJdbcSecurity(url); - Assertions.assertEquals(url, newUrl); + // match ip + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams); + }); + String host1 = "localhost"; + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkJdbcConnParams(host1, port, username, password, database, extraParams); + }); - url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1"; - newUrl = SecurityUtils.checkJdbcSecurity(url); - Assertions.assertEquals(url, newUrl); + // match domain + String host2 = "www.apache.com"; + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkJdbcConnParams(host2, port, username, password, database, extraParams); + }); - // key is not security - url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1&allowLocalInfile=true"; - AtomicReference atomUrl = new AtomicReference<>(url); + // error host + String host3 = "localhost:3306"; Assertions.assertThrows( LinkisSecurityException.class, () -> { - SecurityUtils.checkJdbcSecurity(atomUrl.get()); + SecurityUtils.checkJdbcConnParams(host3, port, username, password, database, extraParams); }); - // url encode - url = "jdbc:mysql://127.0.0.1:10000/db_name?allowLocalInfil%65=true"; - atomUrl.set(url); + String host4 = "localhost:3306/test"; Assertions.assertThrows( LinkisSecurityException.class, () -> { - SecurityUtils.checkJdbcSecurity(atomUrl.get()); + SecurityUtils.checkJdbcConnParams(host4, port, username, password, database, extraParams); }); - // value is not security - url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=allowLocalInfile"; - atomUrl.set(url); + String host5 = "localhost/test"; Assertions.assertThrows( LinkisSecurityException.class, () -> { - SecurityUtils.checkJdbcSecurity(atomUrl.get()); + SecurityUtils.checkJdbcConnParams(host5, port, username, password, database, extraParams); }); - // contains # - url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1&#p2=v2"; - atomUrl.set(url); + // error port Assertions.assertThrows( LinkisSecurityException.class, () -> { - SecurityUtils.checkJdbcSecurity(atomUrl.get()); + SecurityUtils.checkJdbcConnParams(host, null, username, password, database, extraParams); }); - } - @Test - public void testCheckJdbcSecurityParamsMap() throws Exception { - Map paramsMap = new HashMap<>(); - paramsMap.put("p1", "v1"); - Map newMap = SecurityUtils.checkJdbcSecurity(paramsMap); - Assertions.assertEquals("v1", newMap.get("p1")); + // error username + Assertions.assertThrows( + LinkisSecurityException.class, + () -> { + SecurityUtils.checkJdbcConnParams(host, port, " ", password, database, extraParams); + }); + Assertions.assertThrows( + LinkisSecurityException.class, + () -> { + SecurityUtils.checkJdbcConnParams(host, port, null, password, database, extraParams); + }); - // key not security - paramsMap.put("allowLocalInfil%67", "true"); - SecurityUtils.checkJdbcSecurity(paramsMap); - Assertions.assertEquals("true", newMap.get("allowLocalInfilg")); + // error password + Assertions.assertThrows( + LinkisSecurityException.class, + () -> { + SecurityUtils.checkJdbcConnParams(host, port, username, " ", database, extraParams); + }); - // key not security - paramsMap.put("allowLocalInfile", "false"); Assertions.assertThrows( LinkisSecurityException.class, () -> { - SecurityUtils.checkJdbcSecurity(paramsMap); + SecurityUtils.checkJdbcConnParams(host, port, username, null, database, extraParams); }); - // value not security - paramsMap.clear(); - paramsMap.put("p1", "allowLocalInfile"); + // check database, The database name can be empty + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkJdbcConnParams(host, port, username, password, " ", extraParams); + }); + + String database1 = "test?k1=v1"; Assertions.assertThrows( LinkisSecurityException.class, () -> { - SecurityUtils.checkJdbcSecurity(paramsMap); + SecurityUtils.checkJdbcConnParams(host, port, username, password, database1, extraParams); }); - // value not security - paramsMap.clear(); - paramsMap.put("p1", "allowLocalInfil%65"); + // error param + extraParams.put("allowLoadLocalInfile", "true"); Assertions.assertThrows( LinkisSecurityException.class, () -> { - SecurityUtils.checkJdbcSecurity(paramsMap); + SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams); }); - // contains # - paramsMap.clear(); - paramsMap.put("p1#", "v1"); + extraParams.clear(); + extraParams.put("autoDeserialize", "true"); Assertions.assertThrows( LinkisSecurityException.class, () -> { - SecurityUtils.checkJdbcSecurity(paramsMap); + SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams); }); - paramsMap.clear(); - paramsMap.put("p1", "v1#"); + extraParams.clear(); + extraParams.put("allowLocalInfile", "true"); + Assertions.assertThrows( + LinkisSecurityException.class, + () -> { + SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams); + }); + + extraParams.clear(); + extraParams.put("allowUrlInLocalInfile", "false"); + Assertions.assertThrows( + LinkisSecurityException.class, + () -> { + SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams); + }); + + extraParams.clear(); + extraParams.put("allowLocalInfil%65", "true"); + Assertions.assertThrows( + LinkisSecurityException.class, + () -> { + SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams); + }); + + extraParams.clear(); + extraParams.put("#", "true"); + Assertions.assertThrows( + LinkisSecurityException.class, + () -> { + SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams); + }); + + extraParams.clear(); + extraParams.put("test", "#"); Assertions.assertThrows( LinkisSecurityException.class, () -> { - SecurityUtils.checkJdbcSecurity(paramsMap); + SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams); + }); + } + + @Test + public void testCheckJdbcConnUrl() { + // true + String url = "jdbc:mysql://127.0.0.1:10000/db_name"; + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkJdbcConnUrl(url); + }); + // true + String url1 = "jdbc:mysql://127.0.0.1:10000/db_name?"; + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkJdbcConnUrl(url1); + }); + // true + String url11 = "jdbc:mysql://127.0.0.1/db_name?"; + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkJdbcConnUrl(url11); + }); + // true + String url2 = "JDBC:mysql://127.0.0.1:10000/db_name?"; + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkJdbcConnUrl(url2); + }); + // true + String url21 = "JDBC:h2://127.0.0.1:10000/db_name?"; + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkJdbcConnUrl(url21); }); + // true + String url3 = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1"; + Assertions.assertDoesNotThrow( + () -> { + SecurityUtils.checkJdbcConnUrl(url3); + }); + // false url error + String url33 = + "jdbc:mysql://127.0.0.1:10000:/db_name?jdbc:mysql://127.0.0.1:10000?allowLocalInfile=true"; + Assertions.assertThrows( + LinkisSecurityException.class, + () -> { + SecurityUtils.checkJdbcConnUrl(url33); + }); + // false key is not security + String url4 = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1&allowLocalInfile=true"; + Assertions.assertThrows( + LinkisSecurityException.class, + () -> { + SecurityUtils.checkJdbcConnUrl(url4); + }); + + // false value is not security + String url5 = "jdbc:mysql://127.0.0.1:10000/db_name?p1=allowLocalInfile"; + Assertions.assertThrows( + LinkisSecurityException.class, + () -> { + SecurityUtils.checkJdbcConnUrl(url5); + }); + + // false contains # + String url6 = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1&#p2=v2"; + Assertions.assertThrows( + LinkisSecurityException.class, + () -> { + SecurityUtils.checkJdbcConnUrl(url6); + }); + } + + @Test + public void testAppendMysqlForceParamsExtraParams() { + Map extraParams = new HashMap<>(); + extraParams.put("testKey", "testValue"); + SecurityUtils.appendMysqlForceParams(extraParams); + Assertions.assertEquals("false", extraParams.get("allowLoadLocalInfile")); + Assertions.assertEquals("false", extraParams.get("autoDeserialize")); + Assertions.assertEquals("false", extraParams.get("allowLocalInfile")); + Assertions.assertEquals("false", extraParams.get("allowUrlInLocalInfile")); + Assertions.assertEquals("testValue", extraParams.get("testKey")); + Assertions.assertEquals(null, extraParams.get("otherKey")); } @Test - public void testMapToString() throws Exception { + public void testMapToString() { Map paramsMap = new HashMap<>(); paramsMap.put("p1", "v1"); String str = SecurityUtils.parseParamsMapToMysqlParamUrl(paramsMap); diff --git a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java index 73fdf1adab0..b9cd4794577 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java +++ b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java @@ -17,6 +17,7 @@ package org.apache.linkis.manager.engineplugin.jdbc; +import org.apache.linkis.common.utils.SecurityUtils; import org.apache.linkis.hadoop.common.utils.KerberosUtils; import org.apache.linkis.manager.engineplugin.jdbc.constant.JDBCEngineConnConstant; import org.apache.linkis.manager.engineplugin.jdbc.exception.JDBCParamsIllegalException; @@ -296,12 +297,12 @@ public Connection getConnection(String dataSourceIdentifier, Map private String getJdbcUrl(Map properties) throws SQLException { String url = properties.get(JDBCEngineConnConstant.JDBC_URL); if (StringUtils.isBlank(url)) { - throw new SQLException(JDBCEngineConnConstant.JDBC_URL + " is not empty."); + throw new SQLException(JDBCEngineConnConstant.JDBC_URL + " cannot be empty."); } url = JdbcParamUtils.clearJdbcUrl(url); - url = JdbcParamUtils.filterJdbcUrl(url); - JdbcParamUtils.validateJdbcUrl(url); - return url.trim(); + SecurityUtils.checkJdbcConnUrl(url); + url = SecurityUtils.getJdbcUrl(url); + return url; } private String appendProxyUserToJDBCUrl( @@ -329,7 +330,9 @@ private String appendProxyUserToJDBCUrl( private JdbcAuthType getJdbcAuthType(Map properties) { String authType = properties.getOrDefault(JDBCEngineConnConstant.JDBC_AUTH_TYPE, USERNAME.getAuthType()); - if (authType == null || authType.trim().length() == 0) return of(USERNAME.getAuthType()); + if (authType == null || authType.trim().length() == 0) { + return of(USERNAME.getAuthType()); + } return of(authType.trim().toUpperCase()); } diff --git a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtils.java b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtils.java index d4ddee737e5..850513637d9 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtils.java +++ b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtils.java @@ -19,7 +19,6 @@ import org.apache.linkis.common.conf.CommonVars; import org.apache.linkis.common.conf.CommonVars$; -import org.apache.linkis.common.utils.SecurityUtils; import org.apache.linkis.manager.engineplugin.jdbc.JDBCPropertiesParser; import org.apache.linkis.manager.engineplugin.jdbc.constant.JDBCEngineConnConstant; import org.apache.linkis.manager.engineplugin.jdbc.exception.JDBCParamsIllegalException; @@ -35,12 +34,6 @@ public class JdbcParamUtils { private static final Logger LOG = LoggerFactory.getLogger(JdbcParamUtils.class); - private static final String JDBC_MATCH_REGEX = "jdbc:\\w+://\\S+:[0-9]{2,6}(/\\S*)?(\\?\\S*)?"; - private static final String JDBC_H2_PROTOCOL = "jdbc:h2"; - - private static final String JDBC_MYSQL_PROTOCOL = "jdbc:mysql"; - - private static final String SENSITIVE_PARAM = "autoDeserialize=true"; private static final String AUTO_DESERIALIZE = "autoDeserialize"; private static final String APPEND_PARAMS = @@ -49,12 +42,8 @@ public class JdbcParamUtils { public static final CommonVars MYSQL_STRONG_SECURITY_ENABLE = CommonVars$.MODULE$.apply("linkis.mysql.strong.security.enable", "false"); - private static final char AND_SYMBOL = '&'; - private static final String QUOTATION_MARKS = "\""; - private static final char QUESTION_MARK = '?'; - public static String clearJdbcUrl(String url) { if (url.startsWith(QUOTATION_MARKS) && url.endsWith(QUOTATION_MARKS)) { url = url.trim(); @@ -63,34 +52,6 @@ public static String clearJdbcUrl(String url) { return url; } - public static void validateJdbcUrl(String url) { - if (!url.matches(JDBC_MATCH_REGEX) && !url.startsWith(JDBC_H2_PROTOCOL)) { - throw new IllegalArgumentException("JDBC url format error!" + url); - } - } - - public static String filterJdbcUrl(String url) { - - LOG.info("the filter source url is: {}", url); - - if (StringUtils.isBlank(url)) { - return url; - } - // temporarily filter only mysql jdbc url. & Handles cases that start with JDBC - if (!url.toLowerCase().contains(JDBC_MYSQL_PROTOCOL)) { - return url; - } - - // security check - url = SecurityUtils.checkJdbcSecurity(url); - - // append force params - url = SecurityUtils.appendMysqlForceParams(url); - - LOG.info("The filtered jdbc url is: {}", url); - return url; - } - public static String getJdbcUsername(Map properties) throws JDBCParamsIllegalException { String username = diff --git a/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtilsTest.java b/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtilsTest.java index 701f94effe1..78984e01e06 100644 --- a/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtilsTest.java +++ b/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/utils/JdbcParamUtilsTest.java @@ -17,141 +17,17 @@ package org.apache.linkis.manager.engineplugin.jdbc.utils; -import org.apache.linkis.common.exception.LinkisSecurityException; import org.apache.linkis.manager.engineplugin.jdbc.constant.JDBCEngineConnConstant; import org.apache.linkis.manager.engineplugin.jdbc.exception.JDBCParamsIllegalException; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; public class JdbcParamUtilsTest { - @Test - @DisplayName("testFilterJdbcUrl") - public void testFilterJdbcUrl() { - String securityParam = - "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"; - String url = "jdbc:mysql://127.0.0.1:10000/db_name"; - String newUrl = JdbcParamUtils.filterJdbcUrl(url); - Assertions.assertEquals(url + "?" + securityParam, newUrl); - - // not mysql url - url = "h2:mysql"; - newUrl = JdbcParamUtils.filterJdbcUrl(url); - Assertions.assertEquals(url, newUrl); - - // start with JDBC - url = "JDBC:mysql://127.0.0.1:10000/db_name?"; - newUrl = JdbcParamUtils.filterJdbcUrl(url); - Assertions.assertEquals(url + securityParam, newUrl); - - url = "jdbc:mysql://127.0.0.1:10000/db_name?"; - newUrl = JdbcParamUtils.filterJdbcUrl(url); - Assertions.assertEquals(url + securityParam, newUrl); - - url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1"; - newUrl = JdbcParamUtils.filterJdbcUrl(url); - Assertions.assertEquals(url + "&" + securityParam, newUrl); - - // key is not security - url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1&allowLocalInfile=true"; - AtomicReference atomUrl = new AtomicReference<>(url); - Assertions.assertThrows( - LinkisSecurityException.class, - () -> { - JdbcParamUtils.filterJdbcUrl(atomUrl.get()); - }); - - // value is not security - url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=allowLocalInfile"; - atomUrl.set(url); - Assertions.assertThrows( - LinkisSecurityException.class, - () -> { - JdbcParamUtils.filterJdbcUrl(atomUrl.get()); - }); - - // contains # - url = "jdbc:mysql://127.0.0.1:10000/db_name?p1=v1&#p2=v2"; - atomUrl.set(url); - Assertions.assertThrows( - LinkisSecurityException.class, - () -> { - JdbcParamUtils.filterJdbcUrl(atomUrl.get()); - }); - } - - @Test - @DisplayName("testValidateJdbcUrl") - public void testValidateJdbcUrl() { - AtomicReference ar = new AtomicReference<>(); - // true - ar.set("jdbc:mysql://127.0.0.1:10000/abc?p1=v1&p2=v2"); - Assertions.assertDoesNotThrow( - () -> { - JdbcParamUtils.validateJdbcUrl(ar.get()); - }); - // true - ar.set("jdbc:mysql://127.0.0.1:10000/?p1=v1&p2=v2"); - Assertions.assertDoesNotThrow( - () -> { - JdbcParamUtils.validateJdbcUrl(ar.get()); - }); - // true - ar.set("jdbc:mysql://127.0.0.1:10000?p1=v1&p2=v2"); - Assertions.assertDoesNotThrow( - () -> { - JdbcParamUtils.validateJdbcUrl(ar.get()); - }); - // true - ar.set("jdbc:mysql://127.0.0.1:10000/abc?"); - Assertions.assertDoesNotThrow( - () -> { - JdbcParamUtils.validateJdbcUrl(ar.get()); - }); - // true - ar.set("jdbc:mysql://127.0.0.1:10000/abc"); - Assertions.assertDoesNotThrow( - () -> { - JdbcParamUtils.validateJdbcUrl(ar.get()); - }); - // true - ar.set("jdbc:mysql://127.0.0.1:10000/"); - Assertions.assertDoesNotThrow( - () -> { - JdbcParamUtils.validateJdbcUrl(ar.get()); - }); - // true - ar.set("jdbc:mysql://127.0.0.1:10000"); - Assertions.assertDoesNotThrow( - () -> { - JdbcParamUtils.validateJdbcUrl(ar.get()); - }); - // true - ar.set("jdbc:mysql://127.0.0.1:10000/?v1=v2"); - Assertions.assertDoesNotThrow( - () -> { - JdbcParamUtils.validateJdbcUrl(ar.get()); - }); - // false - ar.set("jdbc:mysql://127.0.0.1:10000000/?v1=v2"); - Assertions.assertThrows( - IllegalArgumentException.class, - () -> { - JdbcParamUtils.validateJdbcUrl(ar.get()); - }); - // false - ar.set("jdbc:mysql://127.0.0.1:10000ab/?v1=v2"); - Assertions.assertThrows( - IllegalArgumentException.class, - () -> { - JdbcParamUtils.validateJdbcUrl(ar.get()); - }); - } @Test @DisplayName("testGetJdbcUsername") diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java index b5a3a54faf5..bab34059e0b 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java @@ -48,9 +48,6 @@ public class SqlConnection implements Closeable { private static final CommonVars SQL_SOCKET_TIMEOUT = CommonVars.apply("wds.linkis.server.mdm.service.sql.socket.timeout", 6000); - private static final CommonVars MYSQL_STRONG_SECURITY_ENABLE = - CommonVars.apply("linkis.mysql.strong.security.enable", false); - private Connection conn; private ConnectMessage connectMessage; @@ -63,34 +60,16 @@ public SqlConnection( String database, Map extraParams) throws ClassNotFoundException, SQLException { - // Handle mysql security vulnerabilities - validateParams(extraParams); - connectMessage = new ConnectMessage(host, port, username, password, extraParams); - conn = getDBConnection(connectMessage, database); - // Try to create statement - Statement statement = conn.createStatement(); - statement.close(); - } - - /** - * Handle mysql security vulnerabilities - * - * @param extraParams - */ - private void validateParams(Map extraParams) { - if (extraParams == null) { - return; - } - // security check - SecurityUtils.checkJdbcSecurity(extraParams); - - // append force params + SecurityUtils.checkJdbcConnParams(host, port, username, password, database, extraParams); SecurityUtils.appendMysqlForceParams(extraParams); - // print extraParams - String logStr = SecurityUtils.parseParamsMapToMysqlParamUrl(extraParams); - LOG.info("mysql metadata url extraParams: {}", logStr); + connectMessage = + new ConnectMessage(host.trim(), port, username.trim(), password.trim(), extraParams); + conn = getDBConnection(connectMessage, database.trim()); + // Try to create statement + Statement statement = conn.createStatement(); + statement.close(); } public List getAllDatabases() throws SQLException { @@ -230,6 +209,7 @@ private Connection getDBConnection(ConnectMessage connectMessage, String databas if (!connectMessage.extraParams.isEmpty()) { url += "?" + extraParamString; } + LOG.info("jdbc connection url: {}", url); return DriverManager.getConnection(url, connectMessage.username, connectMessage.password); } From bab4b47c81178c691bb8f222e853a542d1f911e8 Mon Sep 17 00:00:00 2001 From: ChengJie1053 <125547374+ChengJie1053@users.noreply.github.com> Date: Mon, 27 Mar 2023 16:02:14 +0800 Subject: [PATCH 077/689] =?UTF-8?q?Delete=20useless=20files=EF=BC=88DataSo?= =?UTF-8?q?urceInfoService.java=EF=BC=89and=20Modify=20comment=20(#4416)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: chengJie --- .../core/service/DataSourceInfoService.java | 2 +- .../core/service/DataSourceOpService.java | 20 ------------------- 2 files changed, 1 insertion(+), 21 deletions(-) delete mode 100644 linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/DataSourceOpService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/DataSourceInfoService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/DataSourceInfoService.java index a6d62f94068..5dd24497a73 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/DataSourceInfoService.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/DataSourceInfoService.java @@ -183,7 +183,7 @@ public interface DataSourceInfoService { List queryDataSourceEnvPage(DataSourceEnvVo dataSourceEnvVo); /** - * exoire data source + * expire data source * * @param dataSourceId * @return diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/DataSourceOpService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/DataSourceOpService.java deleted file mode 100644 index b79945d579d..00000000000 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/DataSourceOpService.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.datasourcemanager.core.service; - -public interface DataSourceOpService {} From 1004308408e0ac571d0f0518ad26c1a53f05f75f Mon Sep 17 00:00:00 2001 From: rarexixi Date: Mon, 27 Mar 2023 19:07:31 +0800 Subject: [PATCH 078/689] load user's and shared udf (#4417) --- .../org/apache/linkis/udf/dao/UDFDao.java | 4 ++- .../apache/linkis/udf/service/UDFService.java | 2 +- .../udf/service/impl/UDFServiceImpl.java | 4 +-- .../main/resources/mapper/common/UDFDao.xml | 30 ++++++++++++------- .../linkis/udf/api/rpc/UdfReceiver.scala | 3 +- 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/UDFDao.java b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/UDFDao.java index 98028be80b6..b8ae15c38d3 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/UDFDao.java +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/UDFDao.java @@ -62,7 +62,9 @@ List getUDFInfoByTreeId( Long treeId, String userName, Collection categoryCodes); List getUDFInfoByIds( - @Param("ids") Long[] ids, @Param("categoryCodes") Collection categoryCodes); + @Param("username") String username, + @Param("ids") Long[] ids, + @Param("categoryCodes") Collection categoryCodes); List getLoadedUDFs(String userName); diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/UDFService.java b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/UDFService.java index 5b25b744e0f..c319dc14a99 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/UDFService.java +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/UDFService.java @@ -52,7 +52,7 @@ List getUDFSByTreeIdAndUser(Long treeId, String userName, String cate List getUDFInfoByTreeId(Long treeId, String userName, String category) throws UDFException; - List getUDFInfoByIds(Long[] ids, String category) throws UDFException; + List getUDFInfoByIds(String username, Long[] ids, String category) throws UDFException; Map> generateInitSql(String userName) throws UDFException; diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java index 14ab927e8fd..9d6144ba07c 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java @@ -815,11 +815,11 @@ public List getUDFInfoByTreeId(Long treeId, String userName, String c } @Override - public List getUDFInfoByIds(Long[] ids, String category) { + public List getUDFInfoByIds(String username, Long[] ids, String category) { if (ids == null || ids.length == 0) { return new ArrayList<>(0); } - return udfDao.getUDFInfoByIds(ids, categoryToCodes.get(category)); + return udfDao.getUDFInfoByIds(username, ids, categoryToCodes.get(category)); } /** diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml index 17133345b60..59e32797678 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml @@ -165,17 +165,25 @@ - + UPDATE linkis_cg_manager_service_instance - SET owner = #{persistenceNode.owner}, mark = #{persistenceNode.mark}, name = #{persistenceNode.name}, update_time = #{persistenceNode.updateTime}, create_time = #{persistenceNode.createTime}, updator = #{persistenceNode.updator}, creator = #{persistenceNode.creator} + SET + + mark = #{persistenceNode.mark}, + + + name = #{persistenceNode.name}, + + + update_time = #{persistenceNode.updateTime}, + + + updator = #{persistenceNode.updator}, + + + creator = #{persistenceNode.creator}, + + + identifier = #{persistenceNode.identifier} + WHERE instance = #{persistenceNode.instance} diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/org/apache/linkis/manager/dao/NodeManagerMapperTest.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/org/apache/linkis/manager/dao/NodeManagerMapperTest.java index 67b4bc354c3..861b557fb32 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/org/apache/linkis/manager/dao/NodeManagerMapperTest.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/org/apache/linkis/manager/dao/NodeManagerMapperTest.java @@ -93,7 +93,7 @@ void updateNodeInstanceOverload() { persistenceNode.setMark("testmark3"); persistenceNode.setUpdator("testupdator3"); persistenceNode.setCreator("testcreator3"); - nodeManagerMapper.updateNodeInstanceOverload(persistenceNode); + nodeManagerMapper.updateNodeInstanceByInstance(persistenceNode); PersistenceNode persistenceNodes = nodeManagerMapper.getNodeInstance("instance2"); assertTrue(persistenceNode.getName().equals(persistenceNodes.getName())); } diff --git a/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml b/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml new file mode 100644 index 00000000000..6d26ae863a3 --- /dev/null +++ b/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml @@ -0,0 +1,1407 @@ +--- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: ConfigMap +apiVersion: v1 +metadata: + name: {{ include "linkis.fullname" . }}-init-sql +data: + linkis_ddl.sql: | + SET FOREIGN_KEY_CHECKS=0; + {{- if eq .Values.linkis.datasource.initSchema "Reset" }} + DROP TABLE IF EXISTS `linkis_ps_configuration_config_key`; + DROP TABLE IF EXISTS `linkis_ps_configuration_key_engine_relation`; + DROP TABLE IF EXISTS `linkis_ps_configuration_config_value`; + DROP TABLE IF EXISTS `linkis_ps_configuration_category`; + DROP TABLE IF EXISTS `linkis_ps_job_history_group_history`; + DROP TABLE IF EXISTS `linkis_ps_job_history_detail`; + DROP TABLE IF EXISTS `linkis_ps_common_lock`; + DROP TABLE IF EXISTS `linkis_ps_udf_manager`; + DROP TABLE IF EXISTS `linkis_ps_udf_shared_group`; + DROP TABLE IF EXISTS `linkis_ps_udf_shared_info`; + DROP TABLE IF EXISTS `linkis_ps_udf_tree`; + DROP TABLE IF EXISTS `linkis_ps_udf_user_load`; + DROP TABLE IF EXISTS `linkis_ps_udf_baseinfo`; + DROP TABLE IF EXISTS `linkis_ps_udf_version`; + DROP TABLE IF EXISTS `linkis_ps_variable_key_user`; + DROP TABLE IF EXISTS `linkis_ps_variable_key`; + DROP TABLE IF EXISTS `linkis_ps_datasource_access`; + DROP TABLE IF EXISTS `linkis_ps_datasource_field`; + DROP TABLE IF EXISTS `linkis_ps_datasource_import`; + DROP TABLE IF EXISTS `linkis_ps_datasource_lineage`; + DROP TABLE IF EXISTS `linkis_ps_datasource_table`; + DROP TABLE IF EXISTS `linkis_ps_datasource_table_info`; + DROP TABLE IF EXISTS `linkis_ps_cs_context_map`; + DROP TABLE IF EXISTS `linkis_ps_cs_context_map_listener`; + DROP TABLE IF EXISTS `linkis_ps_cs_context_history`; + DROP TABLE IF EXISTS `linkis_ps_cs_context_id`; + DROP TABLE IF EXISTS `linkis_ps_cs_context_listener`; + DROP TABLE IF EXISTS `linkis_ps_bml_resources`; + DROP TABLE IF EXISTS `linkis_ps_bml_resources_version`; + DROP TABLE IF EXISTS `linkis_ps_bml_resources_permission`; + DROP TABLE IF EXISTS `linkis_ps_resources_download_history`; + DROP TABLE IF EXISTS `linkis_ps_bml_resources_task`; + DROP TABLE IF EXISTS `linkis_ps_bml_project`; + DROP TABLE IF EXISTS `linkis_ps_bml_project_user`; + DROP TABLE IF EXISTS `linkis_ps_bml_project_resource`; + DROP TABLE IF EXISTS `linkis_ps_instance_label`; + DROP TABLE IF EXISTS `linkis_ps_instance_label_value_relation`; + DROP TABLE IF EXISTS `linkis_ps_instance_label_relation`; + DROP TABLE IF EXISTS `linkis_ps_instance_info`; + DROP TABLE IF EXISTS `linkis_ps_error_code`; + DROP TABLE IF EXISTS `linkis_cg_manager_service_instance`; + DROP TABLE IF EXISTS `linkis_cg_manager_linkis_resources`; + DROP TABLE IF EXISTS `linkis_cg_manager_lock`; + DROP TABLE IF EXISTS `linkis_cg_rm_external_resource_provider`; + DROP TABLE IF EXISTS `linkis_cg_manager_engine_em`; + DROP TABLE IF EXISTS `linkis_cg_manager_label`; + DROP TABLE IF EXISTS `linkis_cg_manager_label_value_relation`; + DROP TABLE IF EXISTS `linkis_cg_manager_label_resource`; + DROP TABLE IF EXISTS `linkis_cg_ec_resource_info_record`; + DROP TABLE IF EXISTS `linkis_cg_manager_label_service_instance`; + DROP TABLE IF EXISTS `linkis_cg_manager_label_user`; + DROP TABLE IF EXISTS `linkis_cg_manager_metrics_history`; + DROP TABLE IF EXISTS `linkis_cg_manager_service_instance_metrics`; + DROP TABLE IF EXISTS `linkis_cg_engine_conn_plugin_bml_resources`; + DROP TABLE IF EXISTS `linkis_ps_dm_datasource`; + DROP TABLE IF EXISTS `linkis_ps_dm_datasource_env`; + DROP TABLE IF EXISTS `linkis_ps_dm_datasource_type`; + DROP TABLE IF EXISTS `linkis_ps_dm_datasource_type_key`; + DROP TABLE IF EXISTS `linkis_ps_dm_datasource_version`; + DROP TABLE IF EXISTS `linkis_mg_gateway_auth_token`; + {{- end }} + + CREATE TABLE IF NOT EXISTS `linkis_ps_configuration_config_key`( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `key` varchar(50) DEFAULT NULL COMMENT 'Set key, e.g. spark.executor.instances', + `description` varchar(200) DEFAULT NULL, + `name` varchar(50) DEFAULT NULL, + `default_value` varchar(200) DEFAULT NULL COMMENT 'Adopted when user does not set key', + `validate_type` varchar(50) DEFAULT NULL COMMENT 'Validate type, one of the following: None, NumInterval, FloatInterval, Include, Regex, OPF, Custom Rules', + `validate_range` varchar(50) DEFAULT NULL COMMENT 'Validate range', + `engine_conn_type` varchar(50) DEFAULT NULL COMMENT 'engine type,such as spark,hive etc', + `is_hidden` tinyint(1) DEFAULT NULL COMMENT 'Whether it is hidden from user. If set to 1(true), then user cannot modify, however, it could still be used in back-end', + `is_advanced` tinyint(1) DEFAULT NULL COMMENT 'Whether it is an advanced parameter. If set to 1(true), parameters would be displayed only when user choose to do so', + `level` tinyint(1) DEFAULT NULL COMMENT 'Basis for displaying sorting in the front-end. Higher the level is, higher the rank the parameter gets', + `treeName` varchar(20) DEFAULT NULL COMMENT 'Reserved field, representing the subdirectory of engineType', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + + CREATE TABLE IF NOT EXISTS `linkis_ps_configuration_key_engine_relation`( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `config_key_id` bigint(20) NOT NULL COMMENT 'config key id', + `engine_type_label_id` bigint(20) NOT NULL COMMENT 'engine label id', + PRIMARY KEY (`id`), + UNIQUE INDEX(`config_key_id`, `engine_type_label_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + + CREATE TABLE IF NOT EXISTS `linkis_ps_configuration_config_value`( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `config_key_id` bigint(20), + `config_value` varchar(200), + `config_label_id`int(20), + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE INDEX(`config_key_id`, `config_label_id`) + )ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + CREATE TABLE IF NOT EXISTS `linkis_ps_configuration_category` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `label_id` int(20) NOT NULL, + `level` int(20) NOT NULL, + `description` varchar(200), + `tag` varchar(200), + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE INDEX(`label_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + -- + -- New linkis job + -- + + CREATE TABLE IF NOT EXISTS `linkis_ps_job_history_group_history` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'Primary Key, auto increment', + `job_req_id` varchar(64) DEFAULT NULL COMMENT 'job execId', + `submit_user` varchar(50) DEFAULT NULL COMMENT 'who submitted this Job', + `execute_user` varchar(50) DEFAULT NULL COMMENT 'who actually executed this Job', + `source` text DEFAULT NULL COMMENT 'job source', + `labels` text DEFAULT NULL COMMENT 'job labels', + `params` text DEFAULT NULL COMMENT 'job params', + `progress` varchar(32) DEFAULT NULL COMMENT 'Job execution progress', + `status` varchar(50) DEFAULT NULL COMMENT 'Script execution status, must be one of the following: Inited, WaitForRetry, Scheduled, Running, Succeed, Failed, Cancelled, Timeout', + `log_path` varchar(200) DEFAULT NULL COMMENT 'File path of the job log', + `error_code` int DEFAULT NULL COMMENT 'Error code. Generated when the execution of the script fails', + `error_desc` varchar(1000) DEFAULT NULL COMMENT 'Execution description. Generated when the execution of script fails', + `created_time` datetime(3) DEFAULT CURRENT_TIMESTAMP(3) COMMENT 'Creation time', + `updated_time` datetime(3) DEFAULT CURRENT_TIMESTAMP(3) COMMENT 'Update time', + `instances` varchar(250) DEFAULT NULL COMMENT 'Entrance instances', + `metrics` text DEFAULT NULL COMMENT 'Job Metrics', + `engine_type` varchar(32) DEFAULT NULL COMMENT 'Engine type', + `execution_code` text DEFAULT NULL COMMENT 'Job origin code or code path', + `result_location` varchar(500) DEFAULT NULL COMMENT 'File path of the resultsets', + PRIMARY KEY (`id`), + KEY `created_time` (`created_time`), + KEY `submit_user` (`submit_user`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + + CREATE TABLE IF NOT EXISTS `linkis_ps_job_history_detail` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'Primary Key, auto increment', + `job_history_id` bigint(20) NOT NULL COMMENT 'ID of JobHistory', + `result_location` varchar(500) DEFAULT NULL COMMENT 'File path of the resultsets', + `execution_content` text DEFAULT NULL COMMENT 'The script code or other execution content executed by this Job', + `result_array_size` int(4) DEFAULT 0 COMMENT 'size of result array', + `job_group_info` text DEFAULT NULL COMMENT 'Job group info/path', + `created_time` datetime(3) DEFAULT CURRENT_TIMESTAMP(3) COMMENT 'Creation time', + `updated_time` datetime(3) DEFAULT CURRENT_TIMESTAMP(3) COMMENT 'Update time', + `status` varchar(32) DEFAULT NULL COMMENT 'status', + `priority` int(4) DEFAULT 0 COMMENT 'order of subjob', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + CREATE TABLE IF NOT EXISTS `linkis_ps_common_lock` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `lock_object` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `time_out` longtext COLLATE utf8_bin, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `lock_object` (`lock_object`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + SET FOREIGN_KEY_CHECKS=0; + + -- ---------------------------- + -- Table structure for linkis_ps_udf_manager + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_udf_manager` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `user_name` varchar(20) DEFAULT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + + -- ---------------------------- + -- Table structure for linkis_ps_udf_shared_group + -- An entry would be added when a user share a function to other user group + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_udf_shared_group` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `udf_id` bigint(20) NOT NULL, + `shared_group` varchar(50) NOT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + CREATE TABLE IF NOT EXISTS `linkis_ps_udf_shared_info` + ( + `id` bigint(20) PRIMARY KEY NOT NULL AUTO_INCREMENT, + `udf_id` bigint(20) NOT NULL, + `user_name` varchar(50) NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + -- ---------------------------- + -- Table structure for linkis_ps_udf_tree + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_udf_tree` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `parent` bigint(20) NOT NULL, + `name` varchar(100) DEFAULT NULL COMMENT 'Category name of the function. It would be displayed in the front-end', + `user_name` varchar(50) NOT NULL, + `description` varchar(255) DEFAULT NULL, + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `category` varchar(50) DEFAULT NULL COMMENT 'Used to distinguish between udf and function', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + + -- ---------------------------- + -- Table structure for linkis_ps_udf_user_load + -- Used to store the function a user selects in the front-end + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_udf_user_load` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `udf_id` bigint(20) NOT NULL, + `user_name` varchar(50) NOT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + CREATE TABLE IF NOT EXISTS `linkis_ps_udf_baseinfo` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `create_user` varchar(50) NOT NULL, + `udf_name` varchar(255) NOT NULL, + `udf_type` int(11) DEFAULT '0', + `tree_id` bigint(20) NOT NULL, + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `sys` varchar(255) NOT NULL DEFAULT 'ide' COMMENT 'source system', + `cluster_name` varchar(255) NOT NULL, + `is_expire` bit(1) DEFAULT NULL, + `is_shared` bit(1) DEFAULT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + -- bdp_easy_ide.linkis_ps_udf_version definition + CREATE TABLE IF NOT EXISTS `linkis_ps_udf_version` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `udf_id` bigint(20) NOT NULL, + `path` varchar(255) NOT NULL COMMENT 'Source path for uploading files', + `bml_resource_id` varchar(50) NOT NULL, + `bml_resource_version` varchar(20) NOT NULL, + `is_published` bit(1) DEFAULT NULL COMMENT 'is published', + `register_format` varchar(255) DEFAULT NULL, + `use_format` varchar(255) DEFAULT NULL, + `description` varchar(255) NOT NULL COMMENT 'version desc', + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `md5` varchar(100) DEFAULT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + SET FOREIGN_KEY_CHECKS=0; + + -- ---------------------------- + -- Table structure for linkis_ps_variable_key_user + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_variable_key_user` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `application_id` bigint(20) DEFAULT NULL COMMENT 'Reserved word', + `key_id` bigint(20) DEFAULT NULL, + `user_name` varchar(50) DEFAULT NULL, + `value` varchar(200) DEFAULT NULL COMMENT 'Value of the global variable', + PRIMARY KEY (`id`), + UNIQUE KEY `application_id_2` (`application_id`,`key_id`,`user_name`), + KEY `key_id` (`key_id`), + KEY `application_id` (`application_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + + -- ---------------------------- + -- Table structure for linkis_ps_variable_key + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_variable_key` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `key` varchar(50) DEFAULT NULL COMMENT 'Key of the global variable', + `description` varchar(200) DEFAULT NULL COMMENT 'Reserved word', + `name` varchar(50) DEFAULT NULL COMMENT 'Reserved word', + `application_id` bigint(20) DEFAULT NULL COMMENT 'Reserved word', + `default_value` varchar(200) DEFAULT NULL COMMENT 'Reserved word', + `value_type` varchar(50) DEFAULT NULL COMMENT 'Reserved word', + `value_regex` varchar(100) DEFAULT NULL COMMENT 'Reserved word', + PRIMARY KEY (`id`), + KEY `application_id` (`application_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + -- ---------------------------- + -- Table structure for linkis_ps_datasource_access + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_datasource_access` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `table_id` bigint(20) NOT NULL, + `visitor` varchar(16) COLLATE utf8_bin NOT NULL, + `fields` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `application_id` int(4) NOT NULL, + `access_time` datetime NOT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + -- ---------------------------- + -- Table structure for linkis_ps_datasource_field + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_datasource_field` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `table_id` bigint(20) NOT NULL, + `name` varchar(64) COLLATE utf8_bin NOT NULL, + `alias` varchar(64) COLLATE utf8_bin DEFAULT NULL, + `type` varchar(64) COLLATE utf8_bin NOT NULL, + `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `express` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `rule` varchar(128) COLLATE utf8_bin DEFAULT NULL, + `is_partition_field` tinyint(1) NOT NULL, + `is_primary` tinyint(1) NOT NULL, + `length` int(11) DEFAULT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + -- ---------------------------- + -- Table structure for linkis_ps_datasource_import + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_datasource_import` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `table_id` bigint(20) NOT NULL, + `import_type` int(4) NOT NULL, + `args` varchar(255) COLLATE utf8_bin NOT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + -- ---------------------------- + -- Table structure for linkis_ps_datasource_lineage + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_datasource_lineage` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `table_id` bigint(20) DEFAULT NULL, + `source_table` varchar(64) COLLATE utf8_bin DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + -- ---------------------------- + -- Table structure for linkis_ps_datasource_table + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_datasource_table` ( + `id` bigint(255) NOT NULL AUTO_INCREMENT, + `database` varchar(64) COLLATE utf8_bin NOT NULL, + `name` varchar(64) COLLATE utf8_bin NOT NULL, + `alias` varchar(64) COLLATE utf8_bin DEFAULT NULL, + `creator` varchar(16) COLLATE utf8_bin NOT NULL, + `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `create_time` datetime NOT NULL, + `product_name` varchar(64) COLLATE utf8_bin DEFAULT NULL, + `project_name` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `usage` varchar(128) COLLATE utf8_bin DEFAULT NULL, + `lifecycle` int(4) NOT NULL, + `use_way` int(4) NOT NULL, + `is_import` tinyint(1) NOT NULL, + `model_level` int(4) NOT NULL, + `is_external_use` tinyint(1) NOT NULL, + `is_partition_table` tinyint(1) NOT NULL, + `is_available` tinyint(1) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `database` (`database`,`name`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + -- ---------------------------- + -- Table structure for linkis_ps_datasource_table_info + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_datasource_table_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `table_id` bigint(20) NOT NULL, + `table_last_update_time` datetime NOT NULL, + `row_num` bigint(20) NOT NULL, + `file_num` int(11) NOT NULL, + `table_size` varchar(32) COLLATE utf8_bin NOT NULL, + `partitions_num` int(11) NOT NULL, + `update_time` datetime NOT NULL, + `field_num` int(11) NOT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + -- ---------------------------- + -- Table structure for linkis_ps_cs_context_map + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_cs_context_map` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `key` varchar(128) DEFAULT NULL, + `context_scope` varchar(32) DEFAULT NULL, + `context_type` varchar(32) DEFAULT NULL, + `props` text, + `value` mediumtext, + `context_id` int(11) DEFAULT NULL, + `keywords` varchar(255) DEFAULT NULL, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', + `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time', + PRIMARY KEY (`id`), + UNIQUE KEY `key` (`key`,`context_id`,`context_type`), + KEY `keywords` (`keywords`(191)) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + -- ---------------------------- + -- Table structure for linkis_ps_cs_context_map_listener + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_cs_context_map_listener` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `listener_source` varchar(255) DEFAULT NULL, + `key_id` int(11) DEFAULT NULL, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', + `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + -- ---------------------------- + -- Table structure for linkis_ps_cs_context_history + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_cs_context_history` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `context_id` int(11) DEFAULT NULL, + `source` text, + `context_type` varchar(32) DEFAULT NULL, + `history_json` text, + `keyword` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', + `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time', + KEY `keyword` (`keyword`(191)) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + -- ---------------------------- + -- Table structure for linkis_ps_cs_context_id + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_cs_context_id` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `user` varchar(32) DEFAULT NULL, + `application` varchar(32) DEFAULT NULL, + `source` varchar(255) DEFAULT NULL, + `expire_type` varchar(32) DEFAULT NULL, + `expire_time` datetime DEFAULT NULL, + `instance` varchar(128) DEFAULT NULL, + `backup_instance` varchar(255) DEFAULT NULL, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', + `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time', + PRIMARY KEY (`id`), + KEY `instance` (`instance`(128)), + KEY `backup_instance` (`backup_instance`(191)), + KEY `instance_2` (`instance`(128),`backup_instance`(128)) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + -- ---------------------------- + -- Table structure for linkis_ps_cs_context_listener + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_cs_context_listener` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `listener_source` varchar(255) DEFAULT NULL, + `context_id` int(11) DEFAULT NULL, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', + `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + + CREATE TABLE IF NOT EXISTS `linkis_ps_bml_resources` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'Primary key', + `resource_id` varchar(50) NOT NULL COMMENT 'resource uuid', + `is_private` TINYINT(1) DEFAULT 0 COMMENT 'Whether the resource is private, 0 means private, 1 means public', + `resource_header` TINYINT(1) DEFAULT 0 COMMENT 'Classification, 0 means unclassified, 1 means classified', + `downloaded_file_name` varchar(200) DEFAULT NULL COMMENT 'File name when downloading', + `sys` varchar(100) NOT NULL COMMENT 'Owning system', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Created time', + `owner` varchar(200) NOT NULL COMMENT 'Resource owner', + `is_expire` TINYINT(1) DEFAULT 0 COMMENT 'Whether expired, 0 means not expired, 1 means expired', + `expire_type` varchar(50) DEFAULT null COMMENT 'Expiration type, date refers to the expiration on the specified date, TIME refers to the time', + `expire_time` varchar(50) DEFAULT null COMMENT 'Expiration time, one day by default', + `max_version` int(20) DEFAULT 10 COMMENT 'The default is 10, which means to keep the latest 10 versions', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Updated time', + `updator` varchar(50) DEFAULT NULL COMMENT 'updator', + `enable_flag` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'Status, 1: normal, 0: frozen', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB AUTO_INCREMENT=9 DEFAULT CHARSET=utf8mb4; + + + CREATE TABLE IF NOT EXISTS `linkis_ps_bml_resources_version` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'Primary key', + `resource_id` varchar(50) NOT NULL COMMENT 'Resource uuid', + `file_md5` varchar(32) NOT NULL COMMENT 'Md5 summary of the file', + `version` varchar(20) NOT NULL COMMENT 'Resource version (v plus five digits)', + `size` int(10) NOT NULL COMMENT 'File size', + `start_byte` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0, + `end_byte` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0, + `resource` varchar(2000) NOT NULL COMMENT 'Resource content (file information including path and file name)', + `description` varchar(2000) DEFAULT NULL COMMENT 'description', + `start_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Started time', + `end_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Stoped time', + `client_ip` varchar(200) NOT NULL COMMENT 'Client ip', + `updator` varchar(50) DEFAULT NULL COMMENT 'updator', + `enable_flag` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'Status, 1: normal, 0: frozen', + unique key `resource_id_version`(`resource_id`, `version`), + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + + + CREATE TABLE IF NOT EXISTS `linkis_ps_bml_resources_permission` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'Primary key', + `resource_id` varchar(50) NOT NULL COMMENT 'Resource uuid', + `permission` varchar(10) NOT NULL COMMENT 'permission', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time', + `system` varchar(50) default "dss" COMMENT 'creator', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'updated time', + `updator` varchar(50) NOT NULL COMMENT 'updator', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + + + CREATE TABLE IF NOT EXISTS `linkis_ps_resources_download_history` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'primary key', + `start_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'start time', + `end_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'stop time', + `client_ip` varchar(200) NOT NULL COMMENT 'client ip', + `state` TINYINT(1) NOT NULL COMMENT 'Download status, 0 download successful, 1 download failed', + `resource_id` varchar(50) not null, + `version` varchar(20) not null, + `downloader` varchar(50) NOT NULL COMMENT 'Downloader', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + + + + -- 创建资源任务表,包括上传,更新,下载 + CREATE TABLE IF NOT EXISTS `linkis_ps_bml_resources_task` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `resource_id` varchar(50) DEFAULT NULL COMMENT 'resource uuid', + `version` varchar(20) DEFAULT NULL COMMENT 'Resource version number of the current operation', + `operation` varchar(20) NOT NULL COMMENT 'Operation type. upload = 0, update = 1', + `state` varchar(20) NOT NULL DEFAULT 'Schduled' COMMENT 'Current status of the task:Schduled, Running, Succeed, Failed,Cancelled', + `submit_user` varchar(20) NOT NULL DEFAULT '' COMMENT 'Job submission user name', + `system` varchar(20) DEFAULT 'dss' COMMENT 'Subsystem name: wtss', + `instance` varchar(128) NOT NULL COMMENT 'Material library example', + `client_ip` varchar(50) DEFAULT NULL COMMENT 'Request IP', + `extra_params` text COMMENT 'Additional key information. Such as the resource IDs and versions that are deleted in batches, and all versions under the resource are deleted', + `err_msg` varchar(2000) DEFAULT NULL COMMENT 'Task failure information.e.getMessage', + `start_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Starting time', + `end_time` datetime DEFAULT NULL COMMENT 'End Time', + `last_update_time` datetime NOT NULL COMMENT 'Last update time', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + + + + CREATE TABLE IF NOT EXISTS `linkis_ps_bml_project` ( + `id` int(10) NOT NULL AUTO_INCREMENT, + `name` varchar(128) DEFAULT NULL, + `system` varchar(64) not null default "dss", + `source` varchar(1024) default null, + `description` varchar(1024) default null, + `creator` varchar(128) not null, + `enabled` tinyint default 1, + `create_time` datetime DEFAULT now(), + unique key(`name`), + PRIMARY KEY (`id`) + )ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT; + + + + CREATE TABLE IF NOT EXISTS `linkis_ps_bml_project_user` ( + `id` int(10) NOT NULL AUTO_INCREMENT, + `project_id` int(10) NOT NULL, + `username` varchar(64) DEFAULT NULL, + `priv` int(10) not null default 7, -- rwx 421 The permission value is 7. 8 is the administrator, which can authorize other users + `creator` varchar(128) not null, + `create_time` datetime DEFAULT now(), + `expire_time` datetime default null, + unique key user_project(`username`, `project_id`), + PRIMARY KEY (`id`) + )ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT; + + + CREATE TABLE IF NOT EXISTS `linkis_ps_bml_project_resource` ( + `id` int(10) NOT NULL AUTO_INCREMENT, + `project_id` int(10) NOT NULL, + `resource_id` varchar(128) DEFAULT NULL, + PRIMARY KEY (`id`) + )ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT; + + + CREATE TABLE IF NOT EXISTS `linkis_ps_instance_label` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `label_key` varchar(32) COLLATE utf8_bin NOT NULL COMMENT 'string key', + `label_value` varchar(255) COLLATE utf8_bin NOT NULL COMMENT 'string value', + `label_feature` varchar(16) COLLATE utf8_bin NOT NULL COMMENT 'store the feature of label, but it may be redundant', + `label_value_size` int(20) NOT NULL COMMENT 'size of key -> value map', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp', + PRIMARY KEY (`id`), + UNIQUE KEY `label_key_value` (`label_key`,`label_value`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + + CREATE TABLE IF NOT EXISTS `linkis_ps_instance_label_value_relation` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `label_value_key` varchar(255) COLLATE utf8_bin NOT NULL COMMENT 'value key', + `label_value_content` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT 'value content', + `label_id` int(20) DEFAULT NULL COMMENT 'id reference linkis_ps_instance_label -> id', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'create unix timestamp', + PRIMARY KEY (`id`), + UNIQUE KEY `label_value_key_label_id` (`label_value_key`,`label_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + CREATE TABLE IF NOT EXISTS `linkis_ps_instance_label_relation` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `label_id` int(20) DEFAULT NULL COMMENT 'id reference linkis_ps_instance_label -> id', + `service_instance` varchar(128) NOT NULL COLLATE utf8_bin COMMENT 'structure like ${host|machine}:${port}', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'create unix timestamp', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + + CREATE TABLE IF NOT EXISTS `linkis_ps_instance_info` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `instance` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT 'structure like ${host|machine}:${port}', + `name` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT 'equal application name in registry', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'create unix timestamp', + PRIMARY KEY (`id`), + UNIQUE KEY `instance` (`instance`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + CREATE TABLE IF NOT EXISTS `linkis_ps_error_code` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `error_code` varchar(50) NOT NULL, + `error_desc` varchar(1024) NOT NULL, + `error_regex` varchar(1024) DEFAULT NULL, + `error_type` int(3) DEFAULT 0, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + CREATE TABLE IF NOT EXISTS `linkis_cg_manager_service_instance` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `instance` varchar(128) COLLATE utf8_bin DEFAULT NULL, + `name` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `owner` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `mark` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `identifier` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + `updator` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `creator` varchar(32) COLLATE utf8_bin DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `instance` (`instance`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + CREATE TABLE IF NOT EXISTS `linkis_cg_manager_linkis_resources` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `max_resource` varchar(1020) COLLATE utf8_bin DEFAULT NULL, + `min_resource` varchar(1020) COLLATE utf8_bin DEFAULT NULL, + `used_resource` varchar(1020) COLLATE utf8_bin DEFAULT NULL, + `left_resource` varchar(1020) COLLATE utf8_bin DEFAULT NULL, + `expected_resource` varchar(1020) COLLATE utf8_bin DEFAULT NULL, + `locked_resource` varchar(1020) COLLATE utf8_bin DEFAULT NULL, + `resourceType` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `ticketId` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + `updator` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `creator` varchar(255) COLLATE utf8_bin DEFAULT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + CREATE TABLE IF NOT EXISTS `linkis_cg_manager_lock` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `lock_object` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `time_out` longtext COLLATE utf8_bin, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `lock_object` (`lock_object`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + CREATE TABLE IF NOT EXISTS `linkis_cg_rm_external_resource_provider` ( + `id` int(10) NOT NULL AUTO_INCREMENT, + `resource_type` varchar(32) NOT NULL, + `name` varchar(32) NOT NULL, + `labels` varchar(32) DEFAULT NULL, + `config` text NOT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + CREATE TABLE IF NOT EXISTS `linkis_cg_manager_engine_em` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `engine_instance` varchar(128) COLLATE utf8_bin DEFAULT NULL, + `em_instance` varchar(128) COLLATE utf8_bin DEFAULT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + CREATE TABLE IF NOT EXISTS `linkis_cg_manager_label` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `label_key` varchar(32) COLLATE utf8_bin NOT NULL, + `label_value` varchar(255) COLLATE utf8_bin NOT NULL, + `label_feature` varchar(16) COLLATE utf8_bin NOT NULL, + `label_value_size` int(20) NOT NULL, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `label_key_value` (`label_key`,`label_value`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + CREATE TABLE IF NOT EXISTS `linkis_cg_manager_label_value_relation` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `label_value_key` varchar(255) COLLATE utf8_bin NOT NULL, + `label_value_content` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `label_id` int(20) DEFAULT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `label_value_key_label_id` (`label_value_key`,`label_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + CREATE TABLE IF NOT EXISTS `linkis_cg_manager_label_resource` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `label_id` int(20) DEFAULT NULL, + `resource_id` int(20) DEFAULT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `label_id` (`label_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + CREATE TABLE IF NOT EXISTS `linkis_cg_ec_resource_info_record` ( + `id` INT(20) NOT NULL AUTO_INCREMENT, + `label_value` VARCHAR(255) NOT NULL COMMENT 'ec labels stringValue', + `create_user` VARCHAR(128) NOT NULL COMMENT 'ec create user', + `service_instance` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT 'ec instance info', + `ecm_instance` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT 'ecm instance info ', + `ticket_id` VARCHAR(100) NOT NULL COMMENT 'ec ticket id', + `log_dir_suffix` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT 'log path', + `request_times` INT(8) COMMENT 'resource request times', + `request_resource` VARCHAR(1020) COMMENT 'request resource', + `used_times` INT(8) COMMENT 'resource used times', + `used_resource` VARCHAR(1020) COMMENT 'used resource', + `release_times` INT(8) COMMENT 'resource released times', + `released_resource` VARCHAR(1020) COMMENT 'released resource', + `release_time` datetime DEFAULT NULL COMMENT 'released time', + `used_time` datetime DEFAULT NULL COMMENT 'used time', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', + PRIMARY KEY (`id`), + KEY (`ticket_id`), + UNIQUE KEY `label_value_ticket_id` (`ticket_id`,`label_value`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + CREATE TABLE IF NOT EXISTS `linkis_cg_manager_label_service_instance` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `label_id` int(20) DEFAULT NULL, + `service_instance` varchar(128) COLLATE utf8_bin DEFAULT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY label_serviceinstance(label_id,service_instance) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + + CREATE TABLE IF NOT EXISTS `linkis_cg_manager_label_user` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `username` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `label_id` int(20) DEFAULT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + + CREATE TABLE IF NOT EXISTS `linkis_cg_manager_metrics_history` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `instance_status` int(20) DEFAULT NULL, + `overload` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `heartbeat_msg` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `healthy_status` int(20) DEFAULT NULL, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + `creator` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `ticketID` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `serviceName` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `instance` varchar(255) COLLATE utf8_bin DEFAULT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + CREATE TABLE IF NOT EXISTS `linkis_cg_manager_service_instance_metrics` ( + `instance` varchar(128) COLLATE utf8_bin NOT NULL, + `instance_status` int(11) DEFAULT NULL, + `overload` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `heartbeat_msg` text COLLATE utf8_bin DEFAULT NULL, + `healthy_status` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`instance`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + CREATE TABLE IF NOT EXISTS `linkis_cg_engine_conn_plugin_bml_resources` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'Primary key', + `engine_conn_type` varchar(100) NOT NULL COMMENT 'Engine type', + `version` varchar(100) COMMENT 'version', + `file_name` varchar(255) COMMENT 'file name', + `file_size` bigint(20) DEFAULT 0 NOT NULL COMMENT 'file size', + `last_modified` bigint(20) COMMENT 'File update time', + `bml_resource_id` varchar(100) NOT NULL COMMENT 'Owning system', + `bml_resource_version` varchar(200) NOT NULL COMMENT 'Resource owner', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time', + `last_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'updated time', + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + -- ---------------------------- + -- Table structure for linkis_ps_dm_datasource + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_dm_datasource` + ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `datasource_name` varchar(255) COLLATE utf8_bin NOT NULL, + `datasource_desc` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `datasource_type_id` int(11) NOT NULL, + `create_identify` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `create_system` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `parameter` varchar(255) COLLATE utf8_bin NULL DEFAULT NULL, + `create_time` datetime NULL DEFAULT CURRENT_TIMESTAMP, + `modify_time` datetime NULL DEFAULT CURRENT_TIMESTAMP, + `create_user` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `modify_user` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `labels` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `version_id` int(11) DEFAULT NULL COMMENT 'current version id', + `expire` tinyint(1) DEFAULT 0, + `published_version_id` int(11) DEFAULT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + -- ---------------------------- + -- Table structure for linkis_ps_dm_datasource_env + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_dm_datasource_env` + ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `env_name` varchar(32) COLLATE utf8_bin NOT NULL, + `env_desc` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `datasource_type_id` int(11) NOT NULL, + `parameter` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `create_user` varchar(255) COLLATE utf8_bin NULL DEFAULT NULL, + `modify_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `modify_user` varchar(255) COLLATE utf8_bin NULL DEFAULT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + + -- ---------------------------- + -- Table structure for linkis_ps_dm_datasource_type + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_dm_datasource_type` + ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(32) COLLATE utf8_bin NOT NULL, + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `option` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `classifier` varchar(32) COLLATE utf8_bin NOT NULL, + `icon` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `layers` int(3) NOT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + -- ---------------------------- + -- Table structure for linkis_ps_dm_datasource_type_key + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_dm_datasource_type_key` + ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `data_source_type_id` int(11) NOT NULL, + `key` varchar(32) COLLATE utf8_bin NOT NULL, + `name` varchar(32) COLLATE utf8_bin NOT NULL, + `name_en` varchar(32) COLLATE utf8_bin NOT NULL, + `default_value` varchar(50) COLLATE utf8_bin NULL DEFAULT NULL, + `value_type` varchar(50) COLLATE utf8_bin NOT NULL, + `scope` varchar(50) COLLATE utf8_bin NULL DEFAULT NULL, + `require` tinyint(1) NULL DEFAULT 0, + `description` varchar(200) COLLATE utf8_bin NULL DEFAULT NULL, + `description_en` varchar(200) COLLATE utf8_bin NULL DEFAULT NULL, + `value_regex` varchar(200) COLLATE utf8_bin NULL DEFAULT NULL, + `ref_id` bigint(20) NULL DEFAULT NULL, + `ref_value` varchar(50) COLLATE utf8_bin NULL DEFAULT NULL, + `data_source` varchar(200) COLLATE utf8_bin NULL DEFAULT NULL, + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + -- ---------------------------- + -- Table structure for linkis_ps_dm_datasource_version + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_ps_dm_datasource_version` + ( + `version_id` int(11) NOT NULL AUTO_INCREMENT, + `datasource_id` int(11) NOT NULL, + `parameter` varchar(2048) COLLATE utf8_bin NULL DEFAULT NULL, + `comment` varchar(255) COLLATE utf8_bin NULL DEFAULT NULL, + `create_time` datetime(0) NULL DEFAULT CURRENT_TIMESTAMP, + `create_user` varchar(255) COLLATE utf8_bin NULL DEFAULT NULL, + PRIMARY KEY (`version_id`, `datasource_id`) USING BTREE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + -- ---------------------------- + -- Table structure for linkis_mg_gateway_auth_token + -- ---------------------------- + CREATE TABLE IF NOT EXISTS `linkis_mg_gateway_auth_token` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `token_name` varchar(128) NOT NULL, + `legal_users` text, + `legal_hosts` text, + `business_owner` varchar(32), + `create_time` DATE DEFAULT NULL, + `update_time` DATE DEFAULT NULL, + `elapse_day` BIGINT DEFAULT NULL, + `update_by` varchar(32), + PRIMARY KEY (`id`), + UNIQUE KEY `token_name` (`token_name`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + linkis_dml.sql: | + -- 变量： + SET @SPARK_LABEL="spark-{{ .Values.linkis.deps.spark.version }}"; + SET @HIVE_LABEL="hive-{{ .Values.linkis.deps.hive.version }}"; + SET @PYTHON_LABEL="python-{{ .Values.linkis.deps.python.version }}"; + SET @PIPELINE_LABEL="pipeline-1"; + SET @JDBC_LABEL="jdbc-4"; + SET @PRESTO_LABEL="presto-0.234"; + SET @IO_FILE_LABEL="io_file-1.0"; + SET @OPENLOOKENG_LABEL="openlookeng-1.5.0"; + -- 衍生变量： + SET @SPARK_ALL=CONCAT('*-*,',@SPARK_LABEL); + SET @SPARK_IDE=CONCAT('*-IDE,',@SPARK_LABEL); + SET @SPARK_NODE=CONCAT('*-nodeexecution,',@SPARK_LABEL); + SET @SPARK_VISUALIS=CONCAT('*-Visualis,',@SPARK_LABEL); + + SET @HIVE_ALL=CONCAT('*-*,',@HIVE_LABEL); + SET @HIVE_IDE=CONCAT('*-IDE,',@HIVE_LABEL); + SET @HIVE_NODE=CONCAT('*-nodeexecution,',@HIVE_LABEL); + + SET @PYTHON_ALL=CONCAT('*-*,',@PYTHON_LABEL); + SET @PYTHON_IDE=CONCAT('*-IDE,',@PYTHON_LABEL); + SET @PYTHON_NODE=CONCAT('*-nodeexecution,',@PYTHON_LABEL); + + SET @PIPELINE_ALL=CONCAT('*-*,',@PIPELINE_LABEL); + SET @PIPELINE_IDE=CONCAT('*-IDE,',@PIPELINE_LABEL); + + SET @JDBC_ALL=CONCAT('*-*,',@JDBC_LABEL); + SET @JDBC_IDE=CONCAT('*-IDE,',@JDBC_LABEL); + + SET @PRESTO_ALL=CONCAT('*-*,',@PRESTO_LABEL); + SET @PRESTO_IDE=CONCAT('*-IDE,',@PRESTO_LABEL); + + SET @IO_FILE_ALL=CONCAT('*-*,',@IO_FILE_LABEL); + SET @IO_FILE_IDE=CONCAT('*-IDE,',@IO_FILE_LABEL); + + SET @OPENLOOKENG_ALL=CONCAT('*-*,',@OPENLOOKENG_LABEL); + SET @OPENLOOKENG_IDE=CONCAT('*-IDE,',@OPENLOOKENG_LABEL); + + -- Global Settings + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.rm.yarnqueue', 'yarn队列名', 'yarn队列名', 'default', 'None', NULL, '0', '0', '1', '队列资源'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.rm.yarnqueue.instance.max', '取值范围：1-128，单位：个', '队列实例最大个数', '30', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', '0', '0', '1', '队列资源'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.rm.yarnqueue.cores.max', '取值范围：1-500，单位：个', '队列CPU使用上限', '150', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|500)$', '0', '0', '1', '队列资源'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.rm.yarnqueue.memory.max', '取值范围：1-1000，单位：G', '队列内存使用上限', '300G', 'Regex', '^([1-9]\\d{0,2}|1000)(G|g)$', '0', '0', '1', '队列资源'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.rm.client.memory.max', '取值范围：1-100，单位：G', '全局各个引擎内存使用上限', '20G', 'Regex', '^([1-9]\\d{0,1}|100)(G|g)$', '0', '0', '1', '队列资源'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.rm.client.core.max', '取值范围：1-128，单位：个', '全局各个引擎核心个数上限', '10', 'Regex', '^(?:[1-9]\\d?|[1][0-2][0-8])$', '0', '0', '1', '队列资源'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.rm.instance', '范围：1-20，单位：个', '全局各个引擎最大并发数', '10', 'NumInterval', '[1,20]', '0', '0', '1', '队列资源'); + -- spark + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.rm.instance', '范围：1-20，单位：个', 'spark引擎最大并发数', '10', 'NumInterval', '[1,20]', '0', '0', '1', '队列资源', 'spark'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('spark.executor.instances', '取值范围：1-40，单位：个', 'spark执行器实例最大并发数', '1', 'NumInterval', '[1,40]', '0', '0', '2', 'spark资源设置', 'spark'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('spark.executor.cores', '取值范围：1-8，单位：个', 'spark执行器核心个数', '1', 'NumInterval', '[1,8]', '0', '0', '1','spark资源设置', 'spark'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('spark.executor.memory', '取值范围：1-15，单位：G', 'spark执行器内存大小', '1g', 'Regex', '^([1-9]|1[0-5])(G|g)$', '0', '0', '3', 'spark资源设置', 'spark'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('spark.driver.cores', '取值范围：只能取1，单位：个', 'spark驱动器核心个数', '1', 'NumInterval', '[1,1]', '0', '1', '1', 'spark资源设置','spark'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('spark.driver.memory', '取值范围：1-15，单位：G', 'spark驱动器内存大小','1g', 'Regex', '^([1-9]|1[0-5])(G|g)$', '0', '0', '1', 'spark资源设置', 'spark'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.engineconn.max.free.time', '取值范围：3m,15m,30m,1h,2h', '引擎空闲退出时间','1h', 'OFT', '[\"1h\",\"2h\",\"30m\",\"15m\",\"3m\"]', '0', '0', '1', 'spark引擎设置', 'spark'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('spark.tispark.pd.addresses', NULL, NULL, 'pd0:2379', 'None', NULL, '0', '0', '1', 'tidb设置', 'spark'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('spark.tispark.tidb.addr', NULL, NULL, 'tidb', 'None', NULL, '0', '0', '1', 'tidb设置', 'spark'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('spark.tispark.tidb.password', NULL, NULL, NULL, 'None', NULL, '0', '0', '1', 'tidb设置', 'spark'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('spark.tispark.tidb.port', NULL, NULL, '4000', 'None', NULL, '0', '0', '1', 'tidb设置', 'spark'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('spark.tispark.tidb.user', NULL, NULL, 'root', 'None', NULL, '0', '0', '1', 'tidb设置', 'spark'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('spark.python.version', '取值范围：python2,python3', 'python版本','python2', 'OFT', '[\"python3\",\"python2\"]', '0', '0', '1', 'spark引擎设置', 'spark'); + -- hive + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.rm.instance', '范围：1-20，单位：个', 'hive引擎最大并发数', '10', 'NumInterval', '[1,20]', '0', '0', '1', '队列资源', 'hive'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.engineconn.java.driver.memory', '取值范围：1-10，单位：G', 'hive引擎初始化内存大小','1g', 'Regex', '^([1-9]|10)(G|g)$', '0', '0', '1', 'hive引擎设置', 'hive'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('hive.client.java.opts', 'hive客户端进程参数', 'hive引擎启动时jvm参数','', 'None', NULL, '1', '1', '1', 'hive引擎设置', 'hive'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('mapred.reduce.tasks', '范围：1-20，单位：个', 'reduce数', '10', 'NumInterval', '[1,20]', '0', '1', '1', 'hive资源设置', 'hive'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('dfs.block.size', '取值范围：2-10，单位：G', 'map数据块大小', '10', 'NumInterval', '[2,10]', '0', '1', '1', 'hive资源设置', 'hive'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('hive.exec.reduce.bytes.per.reducer', '取值范围：2-10，单位：G', 'reduce处理的数据量', '10', 'NumInterval', '[2,10]', '0', '1', '1', 'hive资源设置', 'hive'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.engineconn.max.free.time', '取值范围：3m,15m,30m,1h,2h', '引擎空闲退出时间','1h', 'OFT', '[\"1h\",\"2h\",\"30m\",\"15m\",\"3m\"]', '0', '0', '1', 'hive引擎设置', 'hive'); + + -- python + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.rm.client.memory.max', '取值范围：1-100，单位：G', 'python驱动器内存使用上限', '20G', 'Regex', '^([1-9]\\d{0,1}|100)(G|g)$', '0', '0', '1', '队列资源', 'python'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.rm.client.core.max', '取值范围：1-128，单位：个', 'python驱动器核心个数上限', '10', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', '0', '0', '1', '队列资源', 'python'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.rm.instance', '范围：1-20，单位：个', 'python引擎最大并发数', '10', 'NumInterval', '[1,20]', '0', '0', '1', '队列资源', 'python'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.engineconn.java.driver.memory', '取值范围：1-2，单位：G', 'python引擎初始化内存大小', '1g', 'Regex', '^([1-2])(G|g)$', '0', '0', '1', 'python引擎设置', 'python'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('python.version', '取值范围：python2,python3', 'python版本','python2', 'OFT', '[\"python3\",\"python2\"]', '0', '0', '1', 'python引擎设置', 'python'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.engineconn.max.free.time', '取值范围：3m,15m,30m,1h,2h', '引擎空闲退出时间','1h', 'OFT', '[\"1h\",\"2h\",\"30m\",\"15m\",\"3m\"]', '0', '0', '1', 'python引擎设置', 'python'); + + -- pipeline + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('pipeline.output.mold', '取值范围：csv或excel', '结果集导出类型','csv', 'OFT', '[\"csv\",\"excel\"]', '0', '0', '1', 'pipeline引擎设置', 'pipeline'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('pipeline.field.split', '取值范围：，或\\t', 'csv分隔符',',', 'OFT', '[\",\",\"\\\\t\"]', '0', '0', '1', 'pipeline引擎设置', 'pipeline'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('pipeline.output.charset', '取值范围：utf-8或gbk', '结果集导出字符集','gbk', 'OFT', '[\"utf-8\",\"gbk\"]', '0', '0', '1', 'pipeline引擎设置', 'pipeline'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('pipeline.output.isoverwrite', '取值范围：true或false', '是否覆写','true', 'OFT', '[\"true\",\"false\"]', '0', '0', '1', 'pipeline引擎设置', 'pipeline'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.rm.instance', '范围：1-3，单位：个', 'pipeline引擎最大并发数','3', 'NumInterval', '[1,3]', '0', '0', '1', 'pipeline引擎设置', 'pipeline'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.engineconn.java.driver.memory', '取值范围：1-10，单位：G', 'pipeline引擎初始化内存大小','2g', 'Regex', '^([1-9]|10)(G|g)$', '0', '0', '1', 'pipeline资源设置', 'pipeline'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('pipeline.output.shuffle.null.type', '取值范围：NULL或者BLANK', '空值替换','NULL', 'OFT', '[\"NULL\",\"BLANK\"]', '0', '0', '1', 'pipeline引擎设置', 'pipeline'); + -- jdbc + insert into `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.jdbc.connect.url', '例如:jdbc:hive2://127.0.0.1:10000', 'jdbc连接地址', 'jdbc:hive2://127.0.0.1:10000', 'Regex', '^\\s*jdbc:\\w+://([^:]+)(:\\d+)(/[^\\?]+)?(\\?\\S*)?$', '0', '0', '1', '数据源配置', 'jdbc'); + insert into `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.jdbc.driver', '例如:org.apache.hive.jdbc.HiveDriver', 'jdbc连接驱动', '', 'None', '', '0', '0', '1', '用户配置', 'jdbc'); + insert into `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.jdbc.version', '取值范围：jdbc3,jdbc4', 'jdbc版本','jdbc4', 'OFT', '[\"jdbc3\",\"jdbc4\"]', '0', '0', '1', '用户配置', 'jdbc'); + insert into `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.jdbc.username', 'username', '数据库连接用户名', '', 'None', '', '0', '0', '1', '用户配置', 'jdbc'); + insert into `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.jdbc.password', 'password', '数据库连接密码', '', 'None', '', '0', '0', '1', '用户配置', 'jdbc'); + insert into `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.jdbc.connect.max', '范围：1-20，单位：个', 'jdbc引擎最大连接数', '10', 'NumInterval', '[1,20]', '0', '0', '1', '数据源配置', 'jdbc'); + + -- io_file + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.rm.instance', '范围：1-20，单位：个', 'io_file引擎最大并发数', '10', 'NumInterval', '[1,20]', '0', '0', '1', 'io_file引擎资源上限', 'io_file'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.rm.client.memory.max', '取值范围：1-50，单位：G', 'io_file引擎最大内存', '20G', 'Regex', '^([1-9]\\d{0,1}|100)(G|g)$', '0', '0', '1', 'io_file引擎资源上限', 'io_file'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('wds.linkis.rm.client.core.max', '取值范围：1-100，单位：个', 'io_file引擎最大核心数', '40', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|128)$', '0', '0', '1', 'io_file引擎资源上限', 'io_file'); + + -- openlookeng + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('linkis.openlookeng.url', '例如:http://127.0.0.1:8080', '连接地址', 'http://127.0.0.1:8080', 'Regex', '^\\s*http://([^:]+)(:\\d+)(/[^\\?]+)?(\\?\\S*)?$', 'openlookeng', 0, 0, 1, '数据源配置'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('linkis.openlookeng.catalog', 'catalog', 'catalog', 'system', 'None', '', 'openlookeng', 0, 0, 1, '数据源配置'); + INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('linkis.openlookeng.source', 'source', 'source', 'global', 'None', '', 'openlookeng', 0, 0, 1, '数据源配置'); + + + -- Configuration first level directory + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType','*-全局设置,*-*', 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType','*-IDE,*-*', 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType','*-Visualis,*-*', 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType','*-nodeexecution,*-*', 'OPTIONAL', 2, now(), now()); + + + -- Engine level default configuration + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType','*-*,*-*', 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@SPARK_ALL, 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@HIVE_ALL, 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@PYTHON_ALL, 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@PIPELINE_ALL, 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@JDBC_ALL, 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@OPENLOOKENG_ALL, 'OPTIONAL', 2, now(), now()); + + -- Custom correlation engine (e.g. spark-{{ .Values.linkis.deps.spark.version }}) and configKey value + -- Global Settings + insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`) + (select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config + INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type is null and label.label_value = "*-*,*-*"); + + -- spark-{{ .Values.linkis.deps.spark.version }} (Here choose to associate all spark type Key values with spark-{{ .Values.linkis.deps.spark.version }}) + insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`) + (select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config + INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type = 'spark' and label.label_value = @SPARK_ALL); + + -- hive-{{ .Values.linkis.deps.hive.version }} + insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`) + (select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config + INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type = 'hive' and label_value = @HIVE_ALL); + + -- python-{{ .Values.linkis.deps.python.version }} + insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`) + (select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config + INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type = 'python' and label_value = @PYTHON_ALL); + + -- pipeline-* + insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`) + (select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config + INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type = 'pipeline' and label_value = @PIPELINE_ALL); + + -- jdbc-4 + insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`) + (select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config + INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type = 'jdbc' and label_value = @JDBC_ALL); + + -- io_file-1.0 + INSERT INTO `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`) + (SELECT config.id AS `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config + INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type = 'io_file' and label_value = @IO_FILE_ALL); + + -- openlookeng-* + insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`) + (select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config + INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type = 'openlookeng' and label_value = @OPENLOOKENG_ALL); + + -- If you need to customize the parameters of the new engine, the following configuration does not need to write SQL initialization + -- Just write the SQL above, and then add applications and engines to the management console to automatically initialize the configuration + + + -- Configuration secondary directory (creator level default configuration) + -- IDE + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@SPARK_IDE, 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@HIVE_IDE, 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@PYTHON_IDE, 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@PIPELINE_IDE, 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@JDBC_IDE, 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@OPENLOOKENG_IDE, 'OPTIONAL', 2, now(), now()); + + -- Visualis + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@SPARK_VISUALIS, 'OPTIONAL', 2, now(), now()); + -- nodeexecution + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@SPARK_NODE, 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@HIVE_NODE, 'OPTIONAL', 2, now(), now()); + replace into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType',@PYTHON_NODE, 'OPTIONAL', 2, now(), now()); + + + -- Associate first-level and second-level directories + select @label_id := id from linkis_cg_manager_label where `label_value` = '*-全局设置,*-*'; + insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 1); + + select @label_id := id from linkis_cg_manager_label where `label_value` = '*-IDE,*-*'; + insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 1); + + select @label_id := id from linkis_cg_manager_label where `label_value` = '*-Visualis,*-*'; + insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 1); + + select @label_id := id from linkis_cg_manager_label where `label_value` = '*-nodeexecution,*-*'; + insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 1); + + select @label_id := id from linkis_cg_manager_label where `label_value` = @SPARK_IDE; + insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 2); + + select @label_id := id from linkis_cg_manager_label where `label_value` = @HIVE_IDE; + insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 2); + + select @label_id := id from linkis_cg_manager_label where `label_value` = @PYTHON_IDE; + insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 2); + + select @label_id := id from linkis_cg_manager_label where `label_value` = @PIPELINE_IDE; + insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 2); + + select @label_id := id from linkis_cg_manager_label where `label_value` = @JDBC_IDE; + insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 2); + + select @label_id := id from linkis_cg_manager_label where `label_value` = @OPENLOOKENG_IDE; + insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 2); + + select @label_id := id from linkis_cg_manager_label where `label_value` = @SPARK_VISUALIS; + insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 2); + + select @label_id := id from linkis_cg_manager_label where `label_value` = @SPARK_NODE; + insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 2); + + select @label_id := id from linkis_cg_manager_label where `label_value` = @HIVE_NODE; + insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 2); + + select @label_id := id from linkis_cg_manager_label where `label_value` = @PYTHON_NODE; + insert into linkis_ps_configuration_category (`label_id`, `level`) VALUES (@label_id, 2); + + -- Associate label and default configuration + insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) + (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation + INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = '*-*,*-*'); + + -- spark default configuration + insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) + (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation + INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @SPARK_ALL); + + -- hive default configuration + insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) + (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation + INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @HIVE_ALL); + + -- python default configuration + insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) + (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation + INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @PYTHON_ALL); + + -- pipeline default configuration + insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) + (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation + INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @PIPELINE_ALL); + + -- jdbc default configuration + insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) + (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation + INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @JDBC_ALL); + + -- openlookeng default configuration + insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) + (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation + INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @OPENLOOKENG_ALL); + + insert into `linkis_cg_rm_external_resource_provider`(`id`,`resource_type`,`name`,`labels`,`config`) values + (1,'Yarn','default',NULL,'{\r\n\"rmWebAddress\": \"{{ .Values.linkis.deps.yarn.restfulUrl }}\",\r\n\"hadoopVersion\": \"{{ .Values.linkis.deps.hadoop.version }}\",\r\n\"authorEnable\":{{ .Values.linkis.deps.yarn.authEnable }},\r\n\"user\":\"{{ .Values.linkis.deps.yarn.authUser }}\",\r\n\"pwd\":\"{{ .Values.linkis.deps.yarn.authPassword }}\",\r\n\"kerberosEnable\":{{ .Values.linkis.deps.yarn.kerberosEnable }},\r\n\"principalName\":\"{{ .Values.linkis.deps.yarn.principal }}\",\r\n\"keytabPath\":\"{{ .Values.linkis.deps.yarn.keytab }}\",\r\n\"krb5Path\":\"{{ .Values.linkis.deps.yarn.krb5 }}\"\r\n}') + ON DUPLICATE KEY UPDATE resource_type='Yarn', config='{\r\n\"rmWebAddress\": \"{{ .Values.linkis.deps.yarn.restfulUrl }}\",\r\n\"hadoopVersion\": \"{{ .Values.linkis.deps.hadoop.version }}\",\r\n\"authorEnable\":{{ .Values.linkis.deps.yarn.authEnable }},\r\n\"user\":\"{{ .Values.linkis.deps.yarn.authUser }}\",\r\n\"pwd\":\"{{ .Values.linkis.deps.yarn.authPassword }}\",\r\n\"kerberosEnable\":{{ .Values.linkis.deps.yarn.kerberosEnable }},\r\n\"principalName\":\"{{ .Values.linkis.deps.yarn.principal }}\",\r\n\"keytabPath\":\"{{ .Values.linkis.deps.yarn.keytab }}\",\r\n\"krb5Path\":\"{{ .Values.linkis.deps.yarn.krb5 }}\"\r\n}'; + + -- errorcode + -- 01 linkis server + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01001','您的任务没有路由到后台ECM，请联系管理员','The em of labels',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01002','Linkis服务负载过高，请联系管理员扩容','Unexpected end of file from server',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01003','Linkis服务负载过高，请联系管理员扩容','failed to ask linkis Manager Can be retried SocketTimeoutException',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01004','引擎在启动时被Kill，请联系管理员',' [0-9]+ Killed',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01005','请求Yarn获取队列信息重试2次仍失败，请联系管理员','Failed to request external resourceClassCastException',0); + + + -- 11 linkis resource 12 user resource 13 user task resouce + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01101','ECM资源不足，请联系管理员扩容','ECM resources are insufficient',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01102','ECM 内存资源不足，请联系管理员扩容','ECM memory resources are insufficient',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01103','ECM CPU资源不足，请联系管理员扩容','ECM CPU resources are insufficient',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01004','ECM 实例资源不足，请联系管理员扩容','ECM Insufficient number of instances',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01005','机器内存不足，请联系管理员扩容','Cannot allocate memory',0); + + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12001','队列CPU资源不足，可以调整Spark执行器个数','Queue CPU resources are insufficient',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12002','队列内存资源不足，可以调整Spark执行器个数','Insufficient queue memory',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12003','队列实例数超过限制','Insufficient number of queue instances',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12004','全局驱动器内存使用上限，可以设置更低的驱动内存','Drive memory resources are insufficient',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12005','超出全局驱动器CPU个数上限，可以清理空闲引擎','Drive core resources are insufficient',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12006','超出引擎最大并发数上限，可以清理空闲引擎','Insufficient number of instances',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12008','获取Yarn队列信息异常,可能是您设置的yarn队列不存在','获取Yarn队列信息异常',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12009','会话创建失败，%s队列不存在，请检查队列设置是否正确','queue (\\S+) does not exist in YARN',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12010','集群队列内存资源不足，可以联系组内人员释放资源','Insufficient cluster queue memory',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12011','集群队列CPU资源不足，可以联系组内人员释放资源','Insufficient cluster queue cpu',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12012','集群队列实例数超过限制','Insufficient cluster queue instance',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12013','资源不足导致启动引擎超时，您可以进行任务重试','wait for DefaultEngineConn',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12014','请求引擎超时，可能是因为队列资源不足导致，请重试','wait for engineConn initial timeout',0); + + + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('13001','Java进程内存溢出，建议优化脚本内容','OutOfMemoryError',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('13002','使用资源过大，请调优sql或者加大资源','Container killed by YARN for exceeding memory limits',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('13003','使用资源过大，请调优sql或者加大资源','read record exception',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('13004','引擎意外退出，可能是使用资源过大导致','failed because the engine quitted unexpectedly',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('13005','Spark app应用退出，可能是复杂任务导致','Spark application has already stopped',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('13006','Spark context退出，可能是复杂任务导致','Spark application sc has already stopped',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('13007','Pyspark子进程意外退出，可能是复杂任务导致','Pyspark process has stopped',0); + -- 21 cluster Authority 22 db Authority + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('21001','会话创建失败，用户%s不能提交应用到队列：%s，请联系提供队列给您的人员','User (\\S+) cannot submit applications to queue (\\S+)',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('21002','创建Python解释器失败，请联系管理员','initialize python executor failed',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('21003','创建单机Python解释器失败，请联系管理员','PythonSession process cannot be initialized',0); + + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('22001','%s无权限访问，请申请开通数据表权限，请联系您的数据管理人员','Permission denied:\\s*user=[a-zA-Z0-9_]+,\\s*access=[A-Z]+\\s*,\\s*inode="([a-zA-Z0-9/_\\.]+)"',0); + -- INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('22002','您可能没有相关权限','Permission denied',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('22003','所查库表无权限','Authorization failed:No privilege',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('22004','用户%s在机器不存在，请确认是否申请了相关权限','user (\\S+) does not exist',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('22005','用户在机器不存在，请确认是否申请了相关权限','engineConnExec.sh: Permission denied',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('22006','用户在机器不存在，请确认是否申请了相关权限','at com.sun.security.auth.UnixPrincipal',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('22007','用户在机器不存在，请确认是否申请了相关权限','LoginException: java.lang.NullPointerException: invalid null input: name',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('22008','用户在机器不存在，请确认是否申请了相关权限','User not known to the underlying authentication module',0); + + -- 30 Space exceeded 31 user operation + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('30001','库超过限制','is exceeded',0); + + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('31001','用户主动kill任务','is killed by user',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('31002','您提交的EngineTypeLabel没有对应的引擎版本','EngineConnPluginNotFoundException',0); + + -- 41 not exist 44 sql 43 python 44 shell 45 scala 46 importExport + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('41001','数据库%s不存在，请检查引用的数据库是否有误','Database ''([a-zA-Z_0-9]+)'' not found',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('41001','数据库%s不存在，请检查引用的数据库是否有误','Database does not exist: ([a-zA-Z_0-9]+)',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('41002','表%s不存在，请检查引用的表是否有误','Table or view not found: ([`\\.a-zA-Z_0-9]+)',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('41002','表%s不存在，请检查引用的表是否有误','Table not found ''([a-zA-Z_0-9]+)''',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('41002','表%s不存在，请检查引用的表是否有误','Table ([a-zA-Z_0-9]+) not found',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('41003','字段%s不存在，请检查引用的字段是否有误','cannot resolve ''`(.+)`'' given input columns',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('41003','字段%s不存在，请检查引用的字段是否有误',' Invalid table alias or column reference ''(.+)'':',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('41003','字段%s不存在，请检查引用的字段是否有误','Column ''(.+)'' cannot be resolved',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('41004','分区字段%s不存在，请检查引用的表%s是否为分区表或分区字段有误','([a-zA-Z_0-9]+) is not a valid partition column in table ([`\\.a-zA-Z_0-9]+)',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('41004','分区字段%s不存在，请检查引用的表是否为分区表或分区字段有误','Partition spec \\{(\\S+)\\} contains non-partition columns',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('41004','分区字段%s不存在，请检查引用的表是否为分区表或分区字段有误','table is not partitioned but partition spec exists:\\{(.+)\\}',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('41004','表对应的路径不存在，请联系您的数据管理人员','Path does not exist: viewfs',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('41005','文件%s不存在','Caused by:\\s*java.io.FileNotFoundException',0); + + -- 42 sql + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42001','括号不匹配，请检查代码中括号是否前后匹配','extraneous input ''\\)''',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42002','非聚合函数%s必须写在group by中，请检查代码的group by语法','expression ''(\\S+)'' is neither present in the group by',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42002','非聚合函数%s必须写在group by中，请检查代码的group by语法','grouping expressions sequence is empty,\\s?and ''(\\S+)'' is not an aggregate function',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42002','非聚合函数%s必须写在group by中，请检查代码的group by语法','Expression not in GROUP BY key ''(\\S+)''',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42003','未知函数%s，请检查代码中引用的函数是否有误','Undefined function: ''(\\S+)''',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42003','未知函数%s，请检查代码中引用的函数是否有误','Invalid function ''(\\S+)''',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42004','字段%s存在名字冲突，请检查子查询内是否有同名字段','Reference ''(\\S+)'' is ambiguous',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42004','字段%s存在名字冲突，请检查子查询内是否有同名字段','Ambiguous column Reference ''(\\S+)'' in subquery',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42005','字段%s必须指定表或者子查询别名，请检查该字段来源','Column ''(\\S+)'' Found in more than One Tables/Subqueries',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42006','表%s在数据库%s中已经存在，请删除相应表后重试','Table or view ''(\\S+)'' already exists in database ''(\\S+)''',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42006','表%s在数据库中已经存在，请删除相应表后重试','Table (\\S+) already exists',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42006','表%s在数据库中已经存在，请删除相应表后重试','Table already exists',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42006','表%s在数据库中已经存在，请删除相应表后重试','AnalysisException: (S+) already exists',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42007','插入目标表字段数量不匹配,请检查代码！','requires that the data to be inserted have the same number of columns as the target table',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42008','数据类型不匹配，请检查代码！','due to data type mismatch: differing types in',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42009','字段%s引用有误，请检查字段是否存在！','Invalid column reference (S+)',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42010','字段%s提取数据失败','Can''t extract value from (S+): need',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42011','括号或者关键字不匹配，请检查代码！','mismatched input ''(\\S+)'' expecting',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42012','group by 位置2不在select列表中，请检查代码！','GROUP BY position (S+) is not in select list',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42013','字段提取数据失败请检查字段类型','Can''t extract value from (S+): need struct type but got string',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42014','插入数据未指定目标表字段%s，请检查代码！','Cannot insert into target table because column number/types are different ''(S+)''',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42015','表别名%s错误，请检查代码！','Invalid table alias ''(\\S+)''',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42016','UDF函数未指定参数，请检查代码！','UDFArgumentException Argument expected',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42017','聚合函数%s不能写在group by 中，请检查代码！','aggregate functions are not allowed in GROUP BY',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42018','您的代码有语法错误，请您修改代码之后执行','SemanticException Error in parsing',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42019','表不存在，请检查引用的表是否有误','table not found',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42020','函数使用错误，请检查您使用的函数方式','No matching method',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42021','您的sql代码可能有语法错误，请检查sql代码','FAILED: ParseException',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42022','您的sql代码可能有语法错误，请检查sql代码','org.apache.spark.sql.catalyst.parser.ParseException',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42022','您的sql代码可能有语法错误，请检查sql代码','org.apache.hadoop.hive.ql.parse.ParseException',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42023','聚合函数不能嵌套','aggregate function in the argument of another aggregate function',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42024','聚合函数不能嵌套','aggregate function parameters overlap with the aggregation',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42025','union 的左右查询字段不一致','Union can only be performed on tables',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42025','hql报错，union 的左右查询字段不一致','both sides of union should match',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42025','union左表和右表类型不一致','on first table and type',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42026','您的建表sql不能推断出列信息','Unable to infer the schema',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42027','动态分区的严格模式需要指定列，您可用通过设置set hive.exec.dynamic.partition.mode=nostrict','requires at least one static partition',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42028','函数输入参数有误','Invalid number of arguments for function',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42029','sql语法报错，select * 与group by无法一起使用','not allowed in select list when GROUP BY ordinal',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42030','where/having子句之外不支持引用外部查询的表达式','the outer query are not supported outside of WHERE',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42031','sql语法报错，group by 后面不能跟一个表','show up in the GROUP BY list',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42032','hql报错，窗口函数中的字段重复','check for circular dependencies',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42033','sql中出现了相同的字段','Found duplicate column',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42034','sql语法不支持','not supported in current context',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42035','hql语法报错，嵌套子查询语法问题','Unsupported SubQuery Expression',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('42036','hql报错，子查询中in 用法有误','in definition of SubQuery',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43037','表字段类型修改导致的转型失败，请联系修改人员','cannot be cast to',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43038','select 的表可能有误','Invalid call to toAttribute on unresolved object',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43039','语法问题，请检查脚本','Distinct window functions are not supported',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43040','Presto查询一定要指定数据源和库信息','Schema must be specified when session schema is not set',0); + + -- 43 python + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43001','代码中存在NoneType空类型变量，请检查代码','''NoneType'' object',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43002','数组越界','IndexError:List index out of range',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43003','您的代码有语法错误，请您修改代码之后执行','SyntaxError',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43004','python代码变量%s未定义','name ''(S+)'' is not defined',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43005','python udf %s 未定义','Undefined function:s+''(S+)''',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43006','python执行不能将%s和%s两种类型进行连接','cannot concatenate ''(S+)'' and ''(S+)''',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43007','pyspark执行失败，可能是语法错误或stage失败','Py4JJavaError: An error occurred',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43008','python代码缩进对齐有误','unexpected indent',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43009','python代码缩进有误','unexpected indent',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43010','python代码反斜杠后面必须换行','unexpected character after line',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43011','导出Excel表超过最大限制1048575','Invalid row number',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43012','python save as table未指定格式，默认用parquet保存，hive查询报错','parquet.io.ParquetDecodingException',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43013','索引使用错误','IndexError',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('43014','sql语法有问题','raise ParseException',0); + + -- 46 importExport + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('46001','找不到导入文件地址：%s','java.io.FileNotFoundException: (\\S+) \\(No such file or directory\\)',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('46002','导出为excel时临时文件目录权限异常','java.io.IOException: Permission denied(.+)at org.apache.poi.xssf.streaming.SXSSFWorkbook.createAndRegisterSXSSFSheet',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('46003','导出文件时无法创建目录：%s','java.io.IOException: Mkdirs failed to create (\\S+) (.+)',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('46004','导入模块错误，系统没有%s模块，请联系运维人员安装','ImportError: No module named (S+)',0); + + -- 91 wtss + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('91001','找不到变量值，请确认您是否设置相关变量','not find variable substitution for',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('91002','不存在的代理用户，请检查你是否申请过平台层（bdp或者bdap）用户','failed to change current working directory ownership',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('91003','请检查提交用户在WTSS内是否有该代理用户的权限，代理用户中是否存在特殊字符，是否用错了代理用户，OS层面是否有该用户，系统设置里面是否设置了该用户为代理用户','没有权限执行当前任务',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('91004','平台层不存在您的执行用户，请在ITSM申请平台层（bdp或者bdap）用户','使用chown命令修改',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('91005','未配置代理用户，请在ITSM走WTSS用户变更单，为你的用户授权改代理用户','请联系系统管理员为您的用户添加该代理用户',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('91006','您的用户初始化有问题，请联系管理员','java: No such file or directory',0); + INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('91007','JobServer中不存在您的脚本文件，请将你的脚本文件放入对应的JobServer路径中', 'Could not open input file for reading%does not exist',0); + + -- ---------------------------- + -- Default Tokens + -- ---------------------------- + REPLACE INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('QML-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); + REPLACE INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('BML-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); + REPLACE INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('WS-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); + REPLACE INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('dss-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); + REPLACE INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('QUALITIS-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); + REPLACE INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('VALIDATOR-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); + REPLACE INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('LINKISCLI-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); + REPLACE INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('DSM-AUTH','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); + REPLACE INTO `linkis_mg_gateway_auth_token`(`token_name`,`legal_users`,`legal_hosts`,`business_owner`,`create_time`,`update_time`,`elapse_day`,`update_by`) VALUES ('LINKIS_CLI_TEST','*','*','BDP',curdate(),curdate(),-1,'LINKIS'); + + INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('mysql', 'mysql数据库', 'mysql数据库', '关系型数据库', '', 3); + INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('kafka', 'kafka', 'kafka', '消息队列', '', 2); + INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('presto', 'presto SQL', 'presto', '大数据存储', '', 3); + INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('hive', 'hive数据库', 'hive', '大数据存储', '', 3); + INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`) VALUES ('mongodb', 'default', 'default', 'DEFAULT', NULL, 3); + + INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (1, 'host', '主机名(Host)', 'Host', NULL, 'TEXT', NULL, 1, '主机名(Host)', 'Host', NULL, NULL, NULL, NULL, now(), now()); + INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (1, 'port', '端口号(Port)','Port', NULL, 'TEXT', NULL, 1, '端口号(Port)','Port', NULL, NULL, NULL, NULL, now(), now()); + INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (1, 'driverClassName', '驱动类名(Driver class name)', 'Driver class name', 'com.mysql.jdbc.Driver', 'TEXT', NULL, 1, '驱动类名(Driver class name)', 'Driver class name', NULL, NULL, NULL, NULL, now(), now()); + INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (1, 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()); + INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (1, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()); + INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (1, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 1, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()); + INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (1, 'databaseName', '数据库名(Database name)', 'Database name', NULL, 'TEXT', NULL, 0, '数据库名(Database name)', 'Database name', NULL, NULL, NULL, NULL, now(), now()); + INSERT INTO `linkis_ps_dm_datasource_type_key` (`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) VALUES (4, 'envId', '集群环境(Cluster env)', 'Cluster env', NULL, 'SELECT', NULL, 1, '集群环境(Cluster env)', 'Cluster env', NULL, NULL, NULL, '/data-source-manager/env-list/all/type/4', now(), now()); + + INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_type_id`, `parameter`, `create_time`, `create_user`, `modify_time`, `modify_user`) VALUES ('测试环境SIT', '测试环境SIT', 4, '{"uris":"thrift://localhost:9083", "hadoopConf":{"hive.metastore.execute.setugi":"true"}}', now(), NULL, now(), NULL); + INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_type_id`, `parameter`, `create_time`, `create_user`, `modify_time`, `modify_user`) VALUES ('测试环境UAT', '测试环境UAT', 4, '{"uris":"thrift://localhost:9083", "hadoopConf":{"hive.metastore.execute.setugi":"true"}}', now(), NULL, now(), NULL); + diff --git a/linkis-dist/package/db/linkis_ddl.sql b/linkis-dist/package/db/linkis_ddl.sql index 8fe51ceef8a..568b055ff97 100644 --- a/linkis-dist/package/db/linkis_ddl.sql +++ b/linkis-dist/package/db/linkis_ddl.sql @@ -670,6 +670,7 @@ CREATE TABLE `linkis_cg_manager_service_instance` ( `name` varchar(32) COLLATE utf8_bin DEFAULT NULL, `owner` varchar(32) COLLATE utf8_bin DEFAULT NULL, `mark` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `identifier` varchar(32) COLLATE utf8_bin DEFAULT NULL, `update_time` datetime DEFAULT CURRENT_TIMESTAMP, `create_time` datetime DEFAULT CURRENT_TIMESTAMP, `updator` varchar(32) COLLATE utf8_bin DEFAULT NULL, diff --git a/linkis-dist/package/db/module/linkis_manager.sql b/linkis-dist/package/db/module/linkis_manager.sql index 1a404af9bbd..c3128633e59 100644 --- a/linkis-dist/package/db/module/linkis_manager.sql +++ b/linkis-dist/package/db/module/linkis_manager.sql @@ -23,6 +23,7 @@ CREATE TABLE `linkis_cg_manager_service_instance` ( `name` varchar(32) COLLATE utf8_bin DEFAULT NULL, `owner` varchar(32) COLLATE utf8_bin DEFAULT NULL, `mark` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `identifier` varchar(32) COLLATE utf8_bin DEFAULT NULL, `update_time` datetime DEFAULT CURRENT_TIMESTAMP, `create_time` datetime DEFAULT CURRENT_TIMESTAMP, `updator` varchar(32) COLLATE utf8_bin DEFAULT NULL, diff --git a/linkis-dist/package/sbin/ext/linkis-cg-linkismanager b/linkis-dist/package/sbin/ext/linkis-cg-linkismanager index 7a3849534ad..9a3cd3317d5 100644 --- a/linkis-dist/package/sbin/ext/linkis-cg-linkismanager +++ b/linkis-dist/package/sbin/ext/linkis-cg-linkismanager @@ -22,7 +22,7 @@ export SERVER_SUFFIX="linkis-computation-governance/linkis-cg-linkismanager" export SERVER_HEAP_SIZE="1G" -export SERVER_CLASS=org.apache.linkis.manager.am.LinkisManagerApplication +export SERVER_CLASS=org.apache.linkis.manager.LinkisManagerApplication if test -z "$MANAGER_HEAP_SIZE" then From 734932a053048a5eb2f0224689cd146b24e0554a Mon Sep 17 00:00:00 2001 From: ChengJie1053 <18033291053@163.com> Date: Wed, 26 Apr 2023 16:19:58 +0800 Subject: [PATCH 124/689] Support tidb dataSources (#4486) * tidb support --- .../linkis/common/utils/SecurityUtils.java | 7 +- .../common/utils/SecurityUtilsTest.java | 13 --- linkis-dist/package/db/linkis_dml.sql | 13 +++ .../upgrade/1.4.0_schema/mysql/linkis_dml.sql | 31 ++++++ .../common/cache/CacheConfiguration.java | 2 +- .../query/service/TidbMetaService.java | 94 +++++++++++++++++++ 6 files changed, 142 insertions(+), 18 deletions(-) create mode 100644 linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql create mode 100644 linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/TidbMetaService.java diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java index c145f4728d0..bd5f0880241 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java @@ -92,12 +92,11 @@ public static void checkJdbcConnParams( String database, Map extraParams) { // 1. Check blank params - if (StringUtils.isAnyBlank(host, username, password)) { - logger.info( - "Invalid mysql connection params: host: {}, username: {}, password: {}, database: {}", + if (StringUtils.isAnyBlank(host, username)) { + logger.error( + "Invalid mysql connection params: host: {}, username: {}, database: {}", host, username, - password, database); throw new LinkisSecurityException(35000, "Invalid mysql connection params."); } diff --git a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/SecurityUtilsTest.java b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/SecurityUtilsTest.java index ed06f9da868..d17f3d08f36 100644 --- a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/SecurityUtilsTest.java +++ b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/SecurityUtilsTest.java @@ -168,19 +168,6 @@ public void testCheckJdbcConnParams() { SecurityUtils.checkJdbcConnParams(host, port, null, password, database, extraParams); }); - // error password - Assertions.assertThrows( - LinkisSecurityException.class, - () -> { - SecurityUtils.checkJdbcConnParams(host, port, username, " ", database, extraParams); - }); - - Assertions.assertThrows( - LinkisSecurityException.class, - () -> { - SecurityUtils.checkJdbcConnParams(host, port, username, null, database, extraParams); - }); - // check database, The database name can be empty Assertions.assertDoesNotThrow( () -> { diff --git a/linkis-dist/package/db/linkis_dml.sql b/linkis-dist/package/db/linkis_dml.sql index 629171af869..f64387866aa 100644 --- a/linkis-dist/package/db/linkis_dml.sql +++ b/linkis-dist/package/db/linkis_dml.sql @@ -554,6 +554,7 @@ INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `cl INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('greenplum', 'greenplum数据库', 'greenplum', '关系型数据库', '', 3, 'Greenplum Database', 'Greenplum', 'Relational Database'); INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('doris', 'doris数据库', 'doris', 'olap', '', 4, 'Doris Database', 'Doris', 'Olap'); INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('clickhouse', 'clickhouse数据库', 'clickhouse', 'olap', '', 4, 'Clickhouse Database', 'Clickhouse', 'Olap'); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('tidb', 'tidb数据库', 'tidb', '关系型数据库', '', 3, 'TiDB Database', 'TiDB', 'Relational Database'); select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'mongodb'; @@ -722,3 +723,15 @@ INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_t select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'kafka'; INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_type_id`, `parameter`, `create_time`, `create_user`, `modify_time`, `modify_user`) VALUES ('kafka测试环境SIT', '开源测试环境SIT', @data_source_type_id, '{"uris":"thrift://localhost:9092"}', now(), NULL, now(), NULL); + +select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'tidb'; +INSERT INTO `linkis_ps_dm_datasource_type_key` +(`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) +VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL, 0, '地址(host1:port1,host2:port2...)', 'Address(host1:port1,host2:port2...)', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'host', '主机名(Host)', 'Host', NULL, 'TEXT', NULL, 1, '主机名(Host)', 'Host', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'port', '端口号(Port)', 'Port', NULL, 'TEXT', NULL, 1, '端口号(Port)', 'Port', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'driverClassName', '驱动类名(Driver class name)', 'Driver class name', 'com.mysql.jdbc.Driver', 'TEXT', NULL, 1, '驱动类名(Driver class name)', 'Driver class name', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 0, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()); diff --git a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql new file mode 100644 index 00000000000..fc099f20a57 --- /dev/null +++ b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('tidb', 'tidb数据库', 'tidb', '关系型数据库', '', 3, 'TiDB Database', 'TiDB', 'Relational Database'); + +select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'tidb'; +INSERT INTO `linkis_ps_dm_datasource_type_key` +(`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) +VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL, 0, '地址(host1:port1,host2:port2...)', 'Address(host1:port1,host2:port2...)', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'host', '主机名(Host)', 'Host', NULL, 'TEXT', NULL, 1, '主机名(Host)', 'Host', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'port', '端口号(Port)', 'Port', NULL, 'TEXT', NULL, 1, '端口号(Port)', 'Port', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'driverClassName', '驱动类名(Driver class name)', 'Driver class name', 'com.mysql.jdbc.Driver', 'TEXT', NULL, 1, '驱动类名(Driver class name)', 'Driver class name', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 0, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()); + diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java index e3510537ae1..54211b6ae64 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java @@ -34,5 +34,5 @@ public class CacheConfiguration { public static final CommonVars MYSQL_RELATIONSHIP_LIST = CommonVars.apply( "wds.linkis.server.mdq.mysql.relationship", - "mysql,oracle,kingbase,postgresql,sqlserver,db2,greenplum,dm,doris,clickhouse"); + "mysql,oracle,kingbase,postgresql,sqlserver,db2,greenplum,dm,doris,clickhouse,tidb"); } diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/TidbMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/TidbMetaService.java new file mode 100644 index 00000000000..3692cd632e2 --- /dev/null +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/TidbMetaService.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.metadata.query.service; + +import org.apache.linkis.datasourcemanager.common.util.json.Json; +import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; +import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService; +import org.apache.linkis.metadata.query.common.service.MetadataConnection; +import org.apache.linkis.metadata.query.service.conf.SqlParamsMapper; +import org.apache.linkis.metadata.query.service.mysql.SqlConnection; + +import org.springframework.stereotype.Component; + +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +public class TidbMetaService extends AbstractDbMetaService { + @Override + public MetadataConnection getConnection( + String operator, Map params) throws Exception { + String host = + String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_HOST.getValue(), "")); + Integer port = + (Double.valueOf( + String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_PORT.getValue(), 0)))) + .intValue(); + String username = + String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_USERNAME.getValue(), "")); + String password = + String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_PASSWORD.getValue(), "")); + + String database = + String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_DATABASE.getValue(), "")); + Map extraParams = new HashMap<>(); + Object sqlParamObj = params.get(SqlParamsMapper.PARAM_SQL_EXTRA_PARAMS.getValue()); + if (null != sqlParamObj) { + if (!(sqlParamObj instanceof Map)) { + extraParams = + Json.fromJson(String.valueOf(sqlParamObj), Map.class, String.class, Object.class); + } else { + extraParams = (Map) sqlParamObj; + } + } + assert extraParams != null; + return new MetadataConnection<>( + new SqlConnection(host, port, username, password, database, extraParams)); + } + + @Override + public List queryDatabases(SqlConnection connection) { + try { + return connection.getAllDatabases(); + } catch (SQLException e) { + throw new RuntimeException("Fail to get Sql databases(获取数据库列表失败)", e); + } + } + + @Override + public List queryTables(SqlConnection connection, String database) { + try { + return connection.getAllTables(database); + } catch (SQLException e) { + throw new RuntimeException("Fail to get Sql tables(获取表列表失败)", e); + } + } + + @Override + public List queryColumns( + SqlConnection connection, String database, String table) { + try { + return connection.getColumns(database, table); + } catch (SQLException | ClassNotFoundException e) { + throw new RuntimeException("Fail to get Sql columns(获取字段列表失败)", e); + } + } +} From 3e17fd80433e0f317f0d91b3b3ad582d672d31c4 Mon Sep 17 00:00:00 2001 From: MrFengqin <45750623+MrFengqin@users.noreply.github.com> Date: Wed, 26 Apr 2023 20:03:15 +0800 Subject: [PATCH 125/689] hive adaptive ranger (#4485) * adaptive ranger * hive adaptive ranger --- .../hive/conf/HiveEngineConfiguration.scala | 2 + .../hive/creation/HiveEngineConnFactory.scala | 42 ++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/conf/HiveEngineConfiguration.scala b/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/conf/HiveEngineConfiguration.scala index eedcfada302..8865dc66de3 100644 --- a/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/conf/HiveEngineConfiguration.scala +++ b/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/conf/HiveEngineConfiguration.scala @@ -47,4 +47,6 @@ object HiveEngineConfiguration { val HIVE_ENGINE_CONCURRENT_SUPPORT = CommonVars[Boolean]("linkis.hive.engineconn.concurrent.support", false).getValue + val HIVE_RANGER_ENABLE = CommonVars[Boolean]("linkis.hive.ranger.enabled", false).getValue + } diff --git a/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/creation/HiveEngineConnFactory.scala b/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/creation/HiveEngineConnFactory.scala index 7cbcd6e2a7a..25c6182f286 100644 --- a/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/creation/HiveEngineConnFactory.scala +++ b/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/creation/HiveEngineConnFactory.scala @@ -37,17 +37,20 @@ import org.apache.linkis.engineplugin.hive.executor.{ HiveEngineConnExecutor } import org.apache.linkis.hadoop.common.utils.HDFSUtils +import org.apache.linkis.manager.engineplugin.common.conf.EnvConfiguration import org.apache.linkis.manager.label.entity.engine.{EngineType, RunType} import org.apache.linkis.manager.label.entity.engine.EngineType.EngineType import org.apache.linkis.manager.label.entity.engine.RunType.RunType import org.apache.commons.lang3.StringUtils +import org.apache.hadoop.fs.Path import org.apache.hadoop.hive.conf.HiveConf import org.apache.hadoop.hive.ql.Driver import org.apache.hadoop.hive.ql.session.SessionState import org.apache.hadoop.security.UserGroupInformation import java.io.{ByteArrayOutputStream, PrintStream} +import java.nio.file.Paths import java.security.PrivilegedExceptionAction import java.util @@ -57,6 +60,7 @@ class HiveEngineConnFactory extends ComputationSingleExecutorEngineConnFactory w private val HIVE_QUEUE_NAME: String = "mapreduce.job.queuename" private val BDP_QUEUE_NAME: String = "wds.linkis.rm.yarnqueue" + private val HIVE_TEZ_QUEUE_NAME: String = "tez.queue.name" override protected def newExecutor( id: Int, @@ -133,6 +137,37 @@ class HiveEngineConnFactory extends ComputationSingleExecutorEngineConnFactory w private def getHiveConf(options: util.Map[String, String]) = { val hiveConf: HiveConf = HiveUtils.getHiveConf + + if (HiveEngineConfiguration.HIVE_RANGER_ENABLE) { + hiveConf.addResource( + new Path( + Paths + .get(EnvConfiguration.HIVE_CONF_DIR.getValue, "ranger-hive-security.xml") + .toAbsolutePath + .toFile + .getAbsolutePath + ) + ) + hiveConf.addResource( + new Path( + Paths + .get(EnvConfiguration.HIVE_CONF_DIR.getValue, "ranger-hive-audit.xml") + .toAbsolutePath + .toFile + .getAbsolutePath + ) + ) + hiveConf.set("hive.security.authorization.enabled", "true") + hiveConf.set( + "hive.security.authorization.manager", + "org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory" + ) + hiveConf.set( + "hive.conf.restricted.list", + "hive.security.authorization.manager,hive.security.metastore.authorization.manager," + + "hive.security.metastore.authenticator.manager,hive.users.in.admin.role,hive.server2.xsrf.filter.enabled,hive.security.authorization.enabled" + ) + } hiveConf.setVar( HiveConf.ConfVars.HIVEJAR, HiveUtils @@ -152,7 +187,12 @@ class HiveEngineConnFactory extends ComputationSingleExecutorEngineConnFactory w } .foreach { case (k, v) => logger.info(s"key is $k, value is $v") - if (BDP_QUEUE_NAME.equals(k)) hiveConf.set(HIVE_QUEUE_NAME, v) else hiveConf.set(k, v) + if (BDP_QUEUE_NAME.equals(k)) { + hiveConf.set(HIVE_QUEUE_NAME, v) + if ("tez".equals(HiveEngineConfiguration.HIVE_ENGINE_TYPE)) { + hiveConf.set(HIVE_TEZ_QUEUE_NAME, v) + } + } else hiveConf.set(k, v) } hiveConf.setVar( HiveConf.ConfVars.HIVE_HADOOP_CLASSPATH, From db37eed39c3770c580e9c1a5d649c0c94064e5d2 Mon Sep 17 00:00:00 2001 From: ChengJie1053 <18033291053@163.com> Date: Thu, 27 Apr 2023 21:36:53 +0800 Subject: [PATCH 126/689] Translate engineconn-plugins-shell service classes from Scala to Java (#4473) * Remove useless code --- .../shell/ShellEngineConnPlugin.java | 81 ++++ .../ShellProcessEngineConnLaunchBuilder.java} | 6 +- .../common/ShellEngineConnPluginConst.java} | 9 +- .../shell/conf/ShellEngineConnConf.java | 28 ++ .../exception/NoCorrectUserException.java} | 12 +- .../exception/ShellCodeErrorException.java} | 16 +- .../shell/executor/ReaderThread.java | 97 +++++ .../shell/executor/ShellECTaskInfo.java | 54 +++ .../ShellEngineConnConcurrentExecutor.java | 358 ++++++++++++++++++ .../executor/ShellEngineConnExecutor.java | 330 ++++++++++++++++ .../shell/executor/YarnAppIdExtractor.java | 53 +++ .../shell/ShellEngineConnPlugin.scala | 76 ---- .../exception/NoCorrectUserException.scala | 27 -- .../shell/executor/ReaderThread.scala | 96 ----- .../ShellEngineConnConcurrentExecutor.scala | 348 ----------------- .../executor/ShellEngineConnExecutor.scala | 319 ---------------- .../shell/executor/YarnAppIdExtractor.scala | 81 ---- .../factory/ShellEngineConnFactory.scala | 0 .../shell/TestShellEngineConnPlugin.java} | 20 +- .../TestShellEngineConnPluginConst.java} | 17 +- .../TestNoCorrectUserException.java} | 20 +- .../executor/TestShellEngineConnExecutor.java | 61 +++ .../TestShellEngineConnExecutor.scala | 62 --- 23 files changed, 1113 insertions(+), 1058 deletions(-) create mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/ShellEngineConnPlugin.java rename linkis-engineconn-plugins/shell/src/main/{scala/org/apache/linkis/manager/engineplugin/shell/builder/ShellProcessEngineConnLaunchBuilder.scala => java/org/apache/linkis/manager/engineplugin/shell/builder/ShellProcessEngineConnLaunchBuilder.java} (81%) rename linkis-engineconn-plugins/shell/src/main/{scala/org/apache/linkis/manager/engineplugin/shell/common/ShellEnginePluginConst.scala => java/org/apache/linkis/manager/engineplugin/shell/common/ShellEngineConnPluginConst.java} (73%) create mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/conf/ShellEngineConnConf.java rename linkis-engineconn-plugins/shell/src/main/{scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellECTaskInfo.scala => java/org/apache/linkis/manager/engineplugin/shell/exception/NoCorrectUserException.java} (65%) rename linkis-engineconn-plugins/shell/src/main/{scala/org/apache/linkis/manager/engineplugin/shell/conf/ShellEngineConnConf.scala => java/org/apache/linkis/manager/engineplugin/shell/exception/ShellCodeErrorException.java} (66%) create mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ReaderThread.java create mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellECTaskInfo.java create mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.java create mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.java create mode 100644 linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/YarnAppIdExtractor.java delete mode 100644 linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/ShellEngineConnPlugin.scala delete mode 100644 linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/exception/NoCorrectUserException.scala delete mode 100644 linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ReaderThread.scala delete mode 100644 linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.scala delete mode 100644 linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.scala delete mode 100644 linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/YarnAppIdExtractor.scala mode change 100644 => 100755 linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/factory/ShellEngineConnFactory.scala rename linkis-engineconn-plugins/shell/src/test/{scala/org/apache/linkis/manager/engineplugin/shell/TestShellEngineConnPlugin.scala => java/org/apache/linkis/manager/engineplugin/shell/TestShellEngineConnPlugin.java} (75%) rename linkis-engineconn-plugins/shell/src/test/{scala/org/apache/linkis/manager/engineplugin/shell/common/TestShellEngineConnPluginConst.scala => java/org/apache/linkis/manager/engineplugin/shell/common/TestShellEngineConnPluginConst.java} (70%) rename linkis-engineconn-plugins/shell/src/test/{scala/org/apache/linkis/manager/engineplugin/shell/exception/TestNoCorrectUserException.scala => java/org/apache/linkis/manager/engineplugin/shell/exception/TestNoCorrectUserException.java} (67%) create mode 100644 linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/executor/TestShellEngineConnExecutor.java delete mode 100644 linkis-engineconn-plugins/shell/src/test/scala/org/apache/linkis/manager/engineplugin/shell/executor/TestShellEngineConnExecutor.scala diff --git a/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/ShellEngineConnPlugin.java b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/ShellEngineConnPlugin.java new file mode 100644 index 00000000000..704f1cd6eb0 --- /dev/null +++ b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/ShellEngineConnPlugin.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.shell; + +import org.apache.linkis.manager.engineplugin.common.EngineConnPlugin; +import org.apache.linkis.manager.engineplugin.common.creation.EngineConnFactory; +import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuilder; +import org.apache.linkis.manager.engineplugin.common.resource.EngineResourceFactory; +import org.apache.linkis.manager.engineplugin.common.resource.GenericEngineResourceFactory; +import org.apache.linkis.manager.engineplugin.shell.builder.ShellProcessEngineConnLaunchBuilder; +import org.apache.linkis.manager.engineplugin.shell.factory.ShellEngineConnFactory; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.manager.label.entity.engine.EngineType; +import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class ShellEngineConnPlugin implements EngineConnPlugin { + private Object resourceLocker = new Object(); + private Object engineFactoryLocker = new Object(); + private volatile EngineResourceFactory engineResourceFactory; + private volatile EngineConnFactory engineFactory; + private List> defaultLabels = new ArrayList<>(); + + public void init(Map params) { + Label engineTypeLabel = + EngineTypeLabelCreator.createEngineTypeLabel(EngineType.SHELL().toString()); + this.defaultLabels.add(engineTypeLabel); + } + + @Override + public EngineResourceFactory getEngineResourceFactory() { + if (engineResourceFactory == null) { + synchronized (resourceLocker) { + if (engineResourceFactory == null) { + engineResourceFactory = new GenericEngineResourceFactory(); + } + } + } + return engineResourceFactory; + } + + @Override + public EngineConnLaunchBuilder getEngineConnLaunchBuilder() { + return new ShellProcessEngineConnLaunchBuilder(); + } + + @Override + public EngineConnFactory getEngineConnFactory() { + if (engineFactory == null) { + synchronized (engineFactoryLocker) { + if (engineFactory == null) { + engineFactory = new ShellEngineConnFactory(); + } + } + } + return engineFactory; + } + + @Override + public List> getDefaultLabels() { + return this.defaultLabels; + } +} diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/builder/ShellProcessEngineConnLaunchBuilder.scala b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/builder/ShellProcessEngineConnLaunchBuilder.java similarity index 81% rename from linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/builder/ShellProcessEngineConnLaunchBuilder.scala rename to linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/builder/ShellProcessEngineConnLaunchBuilder.java index 23668f32ae0..c030db4e208 100644 --- a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/builder/ShellProcessEngineConnLaunchBuilder.scala +++ b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/builder/ShellProcessEngineConnLaunchBuilder.java @@ -15,8 +15,8 @@ * limitations under the License. */ -package org.apache.linkis.manager.engineplugin.shell.builder +package org.apache.linkis.manager.engineplugin.shell.builder; -import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder +import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder; -class ShellProcessEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder {} +public class ShellProcessEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder {} diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/common/ShellEnginePluginConst.scala b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/common/ShellEngineConnPluginConst.java similarity index 73% rename from linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/common/ShellEnginePluginConst.scala rename to linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/common/ShellEngineConnPluginConst.java index 34771e68b07..30fd9cf68cf 100644 --- a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/common/ShellEnginePluginConst.scala +++ b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/common/ShellEngineConnPluginConst.java @@ -15,9 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.manager.engineplugin.shell.common +package org.apache.linkis.manager.engineplugin.shell.common; -object ShellEngineConnPluginConst { - final val RUNTIME_ARGS_KEY: String = "extraArguments" - final val SHELL_RUNTIME_WORKING_DIRECTORY: String = "wds.linkis.shell.runtime.working.directory" +public class ShellEngineConnPluginConst { + public static final String RUNTIME_ARGS_KEY = "extraArguments"; + public static final String SHELL_RUNTIME_WORKING_DIRECTORY = + "wds.linkis.shell.runtime.working.directory"; } diff --git a/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/conf/ShellEngineConnConf.java b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/conf/ShellEngineConnConf.java new file mode 100644 index 00000000000..3849bc1ee0e --- /dev/null +++ b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/conf/ShellEngineConnConf.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.shell.conf; + +import org.apache.linkis.common.conf.CommonVars; + +public class ShellEngineConnConf { + public static final int SHELL_ENGINECONN_CONCURRENT_LIMIT = + CommonVars.apply("linkis.engineconn.shell.concurrent.limit", 30).getValue(); + + public static final int LOG_SERVICE_MAX_THREAD_SIZE = + CommonVars.apply("linkis.engineconn.shell.log.max.thread.size", 50).getValue(); +} diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellECTaskInfo.scala b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/exception/NoCorrectUserException.java similarity index 65% rename from linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellECTaskInfo.scala rename to linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/exception/NoCorrectUserException.java index d8d7edaa396..6c5d106f6a5 100644 --- a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellECTaskInfo.scala +++ b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/exception/NoCorrectUserException.java @@ -15,6 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.manager.engineplugin.shell.executor +package org.apache.linkis.manager.engineplugin.shell.exception; -case class ShellECTaskInfo(taskId: String, process: Process, yarnAppIdExtractor: YarnAppIdExtractor) +import org.apache.linkis.common.exception.ErrorException; + +import static org.apache.linkis.manager.engineplugin.shell.errorcode.LinkisCommonsErrorCodeSummary.*; + +public class NoCorrectUserException extends ErrorException { + public NoCorrectUserException() { + super(NO_ILLEGAL_USER_HOLDS.getErrorCode(), NO_ILLEGAL_USER_HOLDS.getErrorDesc()); + } +} diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/conf/ShellEngineConnConf.scala b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/exception/ShellCodeErrorException.java similarity index 66% rename from linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/conf/ShellEngineConnConf.scala rename to linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/exception/ShellCodeErrorException.java index ba74dbbad6f..49b41ad03c5 100644 --- a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/conf/ShellEngineConnConf.scala +++ b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/exception/ShellCodeErrorException.java @@ -15,16 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.manager.engineplugin.shell.conf +package org.apache.linkis.manager.engineplugin.shell.exception; -import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.exception.ErrorException; -object ShellEngineConnConf { - - val SHELL_ENGINECONN_CONCURRENT_LIMIT: Int = - CommonVars[Int]("linkis.engineconn.shell.concurrent.limit", 30).getValue - - val LOG_SERVICE_MAX_THREAD_SIZE: Int = - CommonVars("linkis.engineconn.shell.log.max.thread.size", 50).getValue +import static org.apache.linkis.manager.engineplugin.shell.errorcode.LinkisCommonsErrorCodeSummary.*; +public class ShellCodeErrorException extends ErrorException { + public ShellCodeErrorException() { + super(SHELL_CODE_IS_WRONG.getErrorCode(), SHELL_CODE_IS_WRONG.getErrorDesc()); + } } diff --git a/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ReaderThread.java b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ReaderThread.java new file mode 100644 index 00000000000..8b3e7cad593 --- /dev/null +++ b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ReaderThread.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.shell.executor; + +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ReaderThread extends Thread { + private static final Logger logger = LoggerFactory.getLogger(ReaderThread.class); + + private EngineExecutionContext engineExecutionContext; + private BufferedReader inputReader; + private YarnAppIdExtractor extractor; + private boolean isStdout; + private final int logListCount = + CommonVars.apply("wds.linkis.engineconn.log.list.count", 50).getValue(); + private CountDownLatch counter; + + private boolean isReaderAlive = true; + + public ReaderThread( + EngineExecutionContext engineExecutionContext, + BufferedReader inputReader, + YarnAppIdExtractor extractor, + boolean isStdout, + CountDownLatch counter) { + this.engineExecutionContext = engineExecutionContext; + this.inputReader = inputReader; + this.extractor = extractor; + this.isStdout = isStdout; + this.counter = counter; + } + + public void onDestroy() { + isReaderAlive = false; + } + + @Override + public void run() { + String line = null; + List logArray = new ArrayList<>(); + while (true) { + try { + line = inputReader.readLine(); + if (!(line != null && isReaderAlive)) break; + } catch (IOException e) { + logger.warn("inputReader reading the input stream"); + break; + } + logger.info("read logger line :{}", line); + logArray.add(line); + extractor.appendLineToExtractor(line); + if (isStdout) { + engineExecutionContext.appendTextResultSet(line); + } + if (logArray.size() > logListCount) { + String linelist = StringUtils.join(logArray, "\n"); + engineExecutionContext.appendStdout(linelist); + logArray.clear(); + } + } + if (logArray.size() > 0) { + String linelist = StringUtils.join(logArray, "\n"); + engineExecutionContext.appendStdout(linelist); + logArray.clear(); + } + IOUtils.closeQuietly(inputReader); + counter.countDown(); + } +} diff --git a/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellECTaskInfo.java b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellECTaskInfo.java new file mode 100644 index 00000000000..dc843327dcd --- /dev/null +++ b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellECTaskInfo.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.shell.executor; + +public class ShellECTaskInfo { + private String taskId; + private Process process; + private YarnAppIdExtractor yarnAppIdExtractor; + + public ShellECTaskInfo(String taskId, Process process, YarnAppIdExtractor yarnAppIdExtractor) { + this.taskId = taskId; + this.process = process; + this.yarnAppIdExtractor = yarnAppIdExtractor; + } + + public String getTaskId() { + return taskId; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + public Process getProcess() { + return process; + } + + public void setProcess(Process process) { + this.process = process; + } + + public YarnAppIdExtractor getYarnAppIdExtractor() { + return yarnAppIdExtractor; + } + + public void setYarnAppIdExtractor(YarnAppIdExtractor yarnAppIdExtractor) { + this.yarnAppIdExtractor = yarnAppIdExtractor; + } +} diff --git a/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.java b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.java new file mode 100644 index 00000000000..3f445ddd5d7 --- /dev/null +++ b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.java @@ -0,0 +1,358 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.shell.executor; + +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.engineconn.computation.executor.execute.ConcurrentComputationExecutor; +import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext; +import org.apache.linkis.engineconn.core.EngineConnObject; +import org.apache.linkis.governance.common.utils.GovernanceUtils; +import org.apache.linkis.manager.common.entity.resource.CommonNodeResource; +import org.apache.linkis.manager.common.entity.resource.NodeResource; +import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils; +import org.apache.linkis.manager.engineplugin.shell.common.ShellEngineConnPluginConst; +import org.apache.linkis.manager.engineplugin.shell.conf.ShellEngineConnConf; +import org.apache.linkis.manager.engineplugin.shell.exception.ShellCodeErrorException; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.protocol.engine.JobProgressInfo; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.scheduler.executer.ErrorExecuteResponse; +import org.apache.linkis.scheduler.executer.ExecuteResponse; +import org.apache.linkis.scheduler.executer.SuccessExecuteResponse; + +import org.apache.commons.lang3.StringUtils; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +import scala.concurrent.ExecutionContextExecutorService; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ShellEngineConnConcurrentExecutor extends ConcurrentComputationExecutor { + private static final Logger logger = + LoggerFactory.getLogger(ShellEngineConnConcurrentExecutor.class); + + private EngineExecutionContext engineExecutionContext; + + private List> executorLabels = new ArrayList<>(); + + private Map shellECTaskInfoCache = new ConcurrentHashMap<>(); + + private int id; + private int maxRunningNumber; + + public ShellEngineConnConcurrentExecutor(int id, int maxRunningNumber) { + super(maxRunningNumber); + this.id = id; + this.maxRunningNumber = maxRunningNumber; + } + + private final ExecutionContextExecutorService logAsyncService = + Utils.newCachedExecutionContext( + ShellEngineConnConf.LOG_SERVICE_MAX_THREAD_SIZE, "ShelLogService-Thread-", true); + + @Override + public void init() { + logger.info("Ready to change engine state!"); + super.init(); + } + + @Override + public ExecuteResponse executeCompletely( + EngineExecutionContext engineExecutionContext, String code, String completedLine) { + String newcode = completedLine + code; + logger.debug("newcode is " + newcode); + return executeLine(engineExecutionContext, newcode); + } + + @Override + public ExecuteResponse executeLine(EngineExecutionContext engineExecutionContext, String code) { + if (engineExecutionContext != null) { + this.engineExecutionContext = engineExecutionContext; + logger.info("Shell executor reset new engineExecutionContext!"); + } + + if (engineExecutionContext.getJobId().isEmpty()) { + return new ErrorExecuteResponse("taskID is null", null); + } + + String taskId = engineExecutionContext.getJobId().get(); + BufferedReader bufferedReader = null; + BufferedReader errorsReader = null; + + AtomicBoolean completed = new AtomicBoolean(false); + ReaderThread errReaderThread = null; + ReaderThread inputReaderThread = null; + + try { + engineExecutionContext.appendStdout(getId() + " >> " + code.trim()); + + String[] argsArr; + if (engineExecutionContext.getTotalParagraph() == 1 + && engineExecutionContext.getProperties() != null + && engineExecutionContext + .getProperties() + .containsKey(ShellEngineConnPluginConst.RUNTIME_ARGS_KEY)) { + ArrayList argsList = + (ArrayList) + engineExecutionContext + .getProperties() + .get(ShellEngineConnPluginConst.RUNTIME_ARGS_KEY); + argsArr = argsList.toArray(new String[argsList.size()]); + logger.info( + "Will execute shell task with user-specified arguments: '{}'", + StringUtils.join(argsArr, "' '")); + } else { + argsArr = null; + } + + String workingDirectory; + if (engineExecutionContext.getTotalParagraph() == 1 + && engineExecutionContext.getProperties() != null + && engineExecutionContext + .getProperties() + .containsKey(ShellEngineConnPluginConst.SHELL_RUNTIME_WORKING_DIRECTORY)) { + String wdStr = + (String) + engineExecutionContext + .getProperties() + .get(ShellEngineConnPluginConst.SHELL_RUNTIME_WORKING_DIRECTORY); + if (isExecutePathExist(wdStr)) { + logger.info( + "Will execute shell task under user-specified working-directory: '{}'", wdStr); + workingDirectory = wdStr; + } else { + logger.warn( + "User-specified working-directory: '{}' does not exist or user does not have access permission. Will execute shell task under default working-directory. Please contact the administrator!", + wdStr); + workingDirectory = null; + } + } else { + workingDirectory = null; + } + + String[] generatedCode = + argsArr == null || argsArr.length == 0 + ? generateRunCode(code) + : generateRunCodeWithArgs(code, argsArr); + + ProcessBuilder processBuilder = new ProcessBuilder(generatedCode); + + if (StringUtils.isNotBlank(workingDirectory)) { + processBuilder.directory(new File(workingDirectory)); + } + + processBuilder.redirectErrorStream(false); + YarnAppIdExtractor extractor = new YarnAppIdExtractor(); + Process process = processBuilder.start(); + bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + errorsReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); + + // add task id and task Info cache + shellECTaskInfoCache.put(taskId, new ShellECTaskInfo(taskId, process, extractor)); + + CountDownLatch counter = new CountDownLatch(2); + inputReaderThread = + new ReaderThread(engineExecutionContext, bufferedReader, extractor, true, counter); + errReaderThread = + new ReaderThread(engineExecutionContext, errorsReader, extractor, false, counter); + + logAsyncService.execute(inputReaderThread); + logAsyncService.execute(errReaderThread); + + int exitCode = process.waitFor(); + counter.await(); + + completed.set(true); + + if (exitCode != 0) { + return new ErrorExecuteResponse("run shell failed", new ShellCodeErrorException()); + } else { + return new SuccessExecuteResponse(); + } + } catch (Exception e) { + logger.error("Execute shell code failed, reason:", e); + return new ErrorExecuteResponse("run shell failed", e); + } finally { + if (errorsReader != null) { + errReaderThread.onDestroy(); + } + if (inputReaderThread != null) { + inputReaderThread.onDestroy(); + } + shellECTaskInfoCache.remove(taskId); + } + } + + private boolean isExecutePathExist(String executePath) { + File etlHomeDir = new File(executePath); + return (etlHomeDir.exists() && etlHomeDir.isDirectory()); + } + + private String[] generateRunCode(String code) { + return new String[] {"sh", "-c", code}; + } + + private String[] generateRunCodeWithArgs(String code, String[] args) { + return new String[] { + "sh", + "-c", + "echo \"dummy " + StringUtils.join(args, " ") + "\" | xargs sh -c \'" + code + "\'" + }; + } + + @Override + public String getId() { + return Sender.getThisServiceInstance().getInstance() + "_" + id; + } + + @Override + public JobProgressInfo[] getProgressInfo(String taskID) { + List jobProgressInfos = new ArrayList<>(); + if (this.engineExecutionContext == null) { + return jobProgressInfos.toArray(new JobProgressInfo[0]); + } + + String jobId = + engineExecutionContext.getJobId().isDefined() + ? engineExecutionContext.getJobId().get() + : ""; + if (progress(taskID) == 0.0f) { + jobProgressInfos.add(new JobProgressInfo(jobId, 1, 1, 0, 0)); + } else { + jobProgressInfos.add(new JobProgressInfo(jobId, 1, 0, 0, 1)); + } + return jobProgressInfos.toArray(new JobProgressInfo[0]); + } + + @Override + public float progress(String taskID) { + if (this.engineExecutionContext != null) { + return ((float) this.engineExecutionContext.getCurrentParagraph()) + / this.engineExecutionContext.getTotalParagraph(); + } else { + return 0.0f; + } + } + + @Override + public boolean supportCallBackLogs() { + // todo + return true; + } + + @Override + public NodeResource requestExpectedResource(NodeResource expectedResource) { + return null; + } + + @Override + public NodeResource getCurrentNodeResource() { + CommonNodeResource resource = new CommonNodeResource(); + resource.setUsedResource( + NodeResourceUtils.applyAsLoadInstanceResource( + EngineConnObject.getEngineCreationContext().getOptions())); + return resource; + } + + @Override + public List> getExecutorLabels() { + return executorLabels; + } + + @Override + public void setExecutorLabels(List> labels) { + if (labels != null) { + executorLabels.clear(); + executorLabels.addAll(labels); + } + } + + @Override + public void killTask(String taskID) { + ShellECTaskInfo shellECTaskInfo = shellECTaskInfoCache.remove(taskID); + if (shellECTaskInfo == null) { + return; + } + + /* + Kill sub-processes + */ + int pid = getPid(shellECTaskInfo.getProcess()); + GovernanceUtils.killProcess(String.valueOf(pid), "kill task " + taskID + " process", false); + + /* + Kill yarn-applications + */ + List yarnAppIds = shellECTaskInfo.getYarnAppIdExtractor().getExtractedYarnAppIds(); + GovernanceUtils.killYarnJobApp(yarnAppIds); + logger.info( + "Finished kill yarn app ids in the engine of ({}). The YARN app ids are {}.", + getId(), + yarnAppIds); + super.killTask(taskID); + } + + private int getPid(Process process) { + try { + Class clazz = Class.forName("java.lang.UNIXProcess"); + Field field = clazz.getDeclaredField("pid"); + field.setAccessible(true); + return field.getInt(process); + } catch (Exception e) { + logger.warn("Failed to acquire pid for shell process"); + return -1; + } + } + + @Override + public void close() { + try { + killAll(); + logAsyncService.shutdown(); + } catch (Exception e) { + logger.error("Shell ec failed to close "); + } + super.close(); + } + + @Override + public void killAll() { + Iterator iterator = shellECTaskInfoCache.values().iterator(); + while (iterator.hasNext()) { + ShellECTaskInfo shellECTaskInfo = iterator.next(); + killTask(shellECTaskInfo.getTaskId()); + } + } + + @Override + public int getConcurrentLimit() { + return maxRunningNumber; + } +} diff --git a/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.java b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.java new file mode 100644 index 00000000000..96441776599 --- /dev/null +++ b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.shell.executor; + +import org.apache.linkis.engineconn.computation.executor.execute.ComputationExecutor; +import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext; +import org.apache.linkis.engineconn.core.EngineConnObject; +import org.apache.linkis.governance.common.utils.GovernanceUtils; +import org.apache.linkis.manager.common.entity.resource.CommonNodeResource; +import org.apache.linkis.manager.common.entity.resource.NodeResource; +import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils; +import org.apache.linkis.manager.engineplugin.shell.common.ShellEngineConnPluginConst; +import org.apache.linkis.manager.engineplugin.shell.exception.ShellCodeErrorException; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.protocol.engine.JobProgressInfo; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.scheduler.executer.ErrorExecuteResponse; +import org.apache.linkis.scheduler.executer.ExecuteResponse; +import org.apache.linkis.scheduler.executer.SuccessExecuteResponse; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ShellEngineConnExecutor extends ComputationExecutor { + private static final Logger logger = LoggerFactory.getLogger(ShellEngineConnExecutor.class); + + private int id; + private EngineExecutionContext engineExecutionContext; + private List> executorLabels = new ArrayList<>(); + private Process process; + private YarnAppIdExtractor extractor; + + public ShellEngineConnExecutor(int id) { + super(id); + this.id = id; + } + + @Override + public void init() { + logger.info("Ready to change engine state!"); + super.init(); + } + + @Override + public ExecuteResponse executeCompletely( + EngineExecutionContext engineExecutionContext, String code, String completedLine) { + final String newcode = completedLine + code; + logger.debug("newcode is " + newcode); + return executeLine(engineExecutionContext, newcode); + } + + @Override + public ExecuteResponse executeLine(EngineExecutionContext engineExecutionContext, String code) { + if (engineExecutionContext != null) { + this.engineExecutionContext = engineExecutionContext; + logger.info("Shell executor reset new engineExecutionContext!"); + } + + BufferedReader bufferedReader = null; + BufferedReader errorsReader = null; + + AtomicBoolean completed = new AtomicBoolean(false); + ReaderThread errReaderThread = null; + ReaderThread inputReaderThread = null; + + try { + engineExecutionContext.appendStdout(getId() + " >> " + code.trim()); + + String[] argsArr = null; + if (engineExecutionContext.getTotalParagraph() == 1 + && engineExecutionContext.getProperties() != null + && engineExecutionContext + .getProperties() + .containsKey(ShellEngineConnPluginConst.RUNTIME_ARGS_KEY)) { + + ArrayList argsList = + (ArrayList) + engineExecutionContext + .getProperties() + .get(ShellEngineConnPluginConst.RUNTIME_ARGS_KEY); + + try { + argsArr = argsList.toArray(new String[argsList.size()]); + logger.info( + "Will execute shell task with user-specified arguments: '{}'", + Arrays.toString(argsArr)); + } catch (Exception t) { + logger.warn( + "Cannot read user-input shell arguments. Will execute shell task without them.", t); + } + } + + String workingDirectory = null; + if (engineExecutionContext.getTotalParagraph() == 1 + && engineExecutionContext.getProperties() != null + && engineExecutionContext + .getProperties() + .containsKey(ShellEngineConnPluginConst.SHELL_RUNTIME_WORKING_DIRECTORY)) { + + String wdStr = + (String) + engineExecutionContext + .getProperties() + .get(ShellEngineConnPluginConst.SHELL_RUNTIME_WORKING_DIRECTORY); + + try { + if (isExecutePathExist(wdStr)) { + logger.info( + "Will execute shell task under user-specified working-directory: '" + wdStr + "'"); + workingDirectory = wdStr; + } else { + logger.warn( + "User-specified working-directory: '" + + wdStr + + "' does not exist or user does not have access permission. " + + "Will execute shell task under default working-directory. Please contact the administrator!"); + } + } catch (Exception t) { + logger.warn( + "Cannot read user-input working-directory. Will execute shell task under default working-directory.", + t); + } + } + + String[] generatedCode = + argsArr == null || argsArr.length == 0 + ? generateRunCode(code) + : generateRunCodeWithArgs(code, argsArr); + + ProcessBuilder processBuilder = new ProcessBuilder(generatedCode); + if (StringUtils.isNotBlank(workingDirectory)) { + processBuilder.directory(new File(workingDirectory)); + } + + processBuilder.redirectErrorStream(false); + extractor = new YarnAppIdExtractor(); + process = processBuilder.start(); + + bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + errorsReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); + CountDownLatch counter = new CountDownLatch(2); + inputReaderThread = + new ReaderThread(engineExecutionContext, bufferedReader, extractor, true, counter); + errReaderThread = + new ReaderThread(engineExecutionContext, errorsReader, extractor, false, counter); + + inputReaderThread.start(); + errReaderThread.start(); + + int exitCode = process.waitFor(); + counter.await(); + + completed.set(true); + + if (exitCode != 0) { + return new ErrorExecuteResponse("run shell failed", new ShellCodeErrorException()); + } else { + return new SuccessExecuteResponse(); + } + + } catch (Exception e) { + logger.error("Execute shell code failed, reason:", e); + return new ErrorExecuteResponse("run shell failed", e); + + } finally { + if (errorsReader != null) { + inputReaderThread.onDestroy(); + } + if (inputReaderThread != null) { + errReaderThread.onDestroy(); + } + IOUtils.closeQuietly(bufferedReader); + IOUtils.closeQuietly(errorsReader); + } + } + + private boolean isExecutePathExist(String executePath) { + File etlHomeDir = new File(executePath); + return (etlHomeDir.exists() && etlHomeDir.isDirectory()); + } + + private String[] generateRunCode(String code) { + return new String[] {"sh", "-c", code}; + } + + private String[] generateRunCodeWithArgs(String code, String[] args) { + return new String[] { + "sh", + "-c", + "echo \"dummy " + StringUtils.join(args, " ") + "\" | xargs sh -c \'" + code + "\'" + }; + } + + @Override + public String getId() { + return Sender.getThisServiceInstance().getInstance() + "_" + id; + } + + @Override + public JobProgressInfo[] getProgressInfo(String taskID) { + List jobProgressInfo = new ArrayList<>(); + if (this.engineExecutionContext == null) { + return jobProgressInfo.toArray(new JobProgressInfo[0]); + } + + String jobId = + engineExecutionContext.getJobId().isDefined() + ? engineExecutionContext.getJobId().get() + : ""; + if (progress(taskID) == 0.0f) { + jobProgressInfo.add(new JobProgressInfo(jobId, 1, 1, 0, 0)); + } else { + jobProgressInfo.add(new JobProgressInfo(jobId, 1, 0, 0, 1)); + } + return jobProgressInfo.toArray(new JobProgressInfo[0]); + } + + @Override + public float progress(String taskID) { + if (this.engineExecutionContext != null) { + return this.engineExecutionContext.getCurrentParagraph() + / (float) this.engineExecutionContext.getTotalParagraph(); + } else { + return 0.0f; + } + } + + @Override + public boolean supportCallBackLogs() { + // todo + return true; + } + + @Override + public NodeResource requestExpectedResource(NodeResource expectedResource) { + return null; + } + + @Override + public NodeResource getCurrentNodeResource() { + CommonNodeResource resource = new CommonNodeResource(); + resource.setUsedResource( + NodeResourceUtils.applyAsLoadInstanceResource( + EngineConnObject.getEngineCreationContext().getOptions())); + return resource; + } + + @Override + public List> getExecutorLabels() { + return executorLabels; + } + + @Override + public void setExecutorLabels(List> labels) { + if (labels != null) { + executorLabels.clear(); + executorLabels.addAll(labels); + } + } + + @Override + public void killTask(String taskID) { + + /* + Kill sub-processes + */ + int pid = getPid(process); + GovernanceUtils.killProcess(String.valueOf(pid), "kill task " + taskID + " process", false); + + /* + Kill yarn-applications + */ + List yarnAppIds = extractor.getExtractedYarnAppIds(); + GovernanceUtils.killYarnJobApp(yarnAppIds); + logger.info("Finished kill yarn app ids in the engine of ({})", getId()); + super.killTask(taskID); + } + + private int getPid(Process process) { + try { + Class clazz = Class.forName("java.lang.UNIXProcess"); + Field field = clazz.getDeclaredField("pid"); + field.setAccessible(true); + return field.getInt(process); + } catch (Exception e) { + logger.warn("Failed to acquire pid for shell process"); + return -1; + } + } + + @Override + public void close() { + try { + process.destroy(); + } catch (Exception e) { + logger.error("kill process " + process.toString() + " failed ", e); + } catch (Throwable t) { + logger.error("kill process " + process.toString() + " failed ", t); + } + super.close(); + } +} diff --git a/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/YarnAppIdExtractor.java b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/YarnAppIdExtractor.java new file mode 100644 index 00000000000..c138e79e668 --- /dev/null +++ b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/YarnAppIdExtractor.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.shell.executor; + +import org.apache.linkis.engineconn.common.conf.EngineConnConf; + +import org.apache.commons.lang3.StringUtils; + +import java.util.*; +import java.util.Collections; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class YarnAppIdExtractor { + + private final Set appIdList = Collections.synchronizedSet(new HashSet<>()); + + private final String regex = + EngineConnConf.SPARK_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); + private final Pattern pattern = Pattern.compile(regex); + + public void appendLineToExtractor(String content) { + if (StringUtils.isBlank(content)) { + return; + } + Matcher yarnAppIDMatcher = pattern.matcher(content); + if (yarnAppIDMatcher.find()) { + String yarnAppID = yarnAppIDMatcher.group(2); + appIdList.add(yarnAppID); + } + } + + public List getExtractedYarnAppIds() { + synchronized (appIdList) { + return new ArrayList<>(appIdList); + } + } +} diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/ShellEngineConnPlugin.scala b/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/ShellEngineConnPlugin.scala deleted file mode 100644 index 115b4763e5d..00000000000 --- a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/ShellEngineConnPlugin.scala +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.engineplugin.shell - -import org.apache.linkis.manager.engineplugin.common.EngineConnPlugin -import org.apache.linkis.manager.engineplugin.common.creation.EngineConnFactory -import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuilder -import org.apache.linkis.manager.engineplugin.common.resource.{ - EngineResourceFactory, - GenericEngineResourceFactory -} -import org.apache.linkis.manager.engineplugin.shell.builder.ShellProcessEngineConnLaunchBuilder -import org.apache.linkis.manager.engineplugin.shell.factory.ShellEngineConnFactory -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.engine.EngineType -import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator - -import java.util - -class ShellEngineConnPlugin extends EngineConnPlugin { - - private val resourceLocker = new Object() - - private val engineLaunchBuilderLocker = new Object() - - private val engineFactoryLocker = new Object() - - private var engineResourceFactory: EngineResourceFactory = _ - - private var engineLaunchBuilder: EngineConnLaunchBuilder = _ - - private var engineFactory: EngineConnFactory = _ - - private val defaultLabels: util.List[Label[_]] = new util.ArrayList[Label[_]]() - - override def init(params: util.Map[String, AnyRef]): Unit = { - val engineTypeLabel = EngineTypeLabelCreator.createEngineTypeLabel(EngineType.SHELL.toString) - this.defaultLabels.add(engineTypeLabel) - } - - override def getEngineResourceFactory: EngineResourceFactory = { - if (null == engineResourceFactory) resourceLocker synchronized { - engineResourceFactory = new GenericEngineResourceFactory - } - engineResourceFactory - } - - override def getEngineConnLaunchBuilder: EngineConnLaunchBuilder = { - new ShellProcessEngineConnLaunchBuilder - } - - override def getEngineConnFactory: EngineConnFactory = { - if (null == engineFactory) engineFactoryLocker synchronized { - engineFactory = new ShellEngineConnFactory - } - engineFactory - } - - override def getDefaultLabels: util.List[Label[_]] = this.defaultLabels - -} diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/exception/NoCorrectUserException.scala b/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/exception/NoCorrectUserException.scala deleted file mode 100644 index e91c5923cae..00000000000 --- a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/exception/NoCorrectUserException.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.engineplugin.shell.exception - -import org.apache.linkis.common.exception.ErrorException -import org.apache.linkis.manager.engineplugin.shell.errorcode.LinkisCommonsErrorCodeSummary._ - -case class NoCorrectUserException() - extends ErrorException(NO_ILLEGAL_USER_HOLDS.getErrorCode, NO_ILLEGAL_USER_HOLDS.getErrorDesc) - -case class ShellCodeErrorException() - extends ErrorException(SHELL_CODE_IS_WRONG.getErrorCode, SHELL_CODE_IS_WRONG.getErrorDesc) diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ReaderThread.scala b/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ReaderThread.scala deleted file mode 100644 index 8277cb11641..00000000000 --- a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ReaderThread.scala +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.engineplugin.shell.executor - -import org.apache.linkis.common.conf.CommonVars -import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext - -import org.apache.commons.io.IOUtils -import org.apache.commons.lang3.StringUtils - -import java.io.BufferedReader -import java.util -import java.util.concurrent.CountDownLatch - -class ReaderThread extends Thread with Logging { - - private var engineExecutionContext: EngineExecutionContext = _ - private var inputReader: BufferedReader = _ - private var extractor: YarnAppIdExtractor = _ - private var isStdout: Boolean = false - private val logListCount = CommonVars[Int]("wds.linkis.engineconn.log.list.count", 50) - private var counter: CountDownLatch = _ - - private var isReaderAlive = true - - def this( - engineExecutionContext: EngineExecutionContext, - inputReader: BufferedReader, - extractor: YarnAppIdExtractor, - isStdout: Boolean, - counter: CountDownLatch - ) { - this() - this.inputReader = inputReader - this.engineExecutionContext = engineExecutionContext - this.extractor = extractor - this.isStdout = isStdout - this.counter = counter - } - - def onDestroy(): Unit = { - isReaderAlive = false - } - - def startReaderThread(): Unit = { - Utils.tryCatch { - this.start() - } { t => - throw t - } - } - - override def run(): Unit = { - Utils.tryCatch { - var line: String = null - val logArray: util.List[String] = new util.ArrayList[String] - while ({ line = inputReader.readLine(); line != null && isReaderAlive }) { - logger.info("read logger line :{}", line) - logArray.add(line) - extractor.appendLineToExtractor(line) - if (isStdout) engineExecutionContext.appendTextResultSet(line) - if (logArray.size > logListCount.getValue) { - val linelist = StringUtils.join(logArray, "\n") - engineExecutionContext.appendStdout(linelist) - logArray.clear() - } - } - if (logArray.size > 0) { - val linelist = StringUtils.join(logArray, "\n") - engineExecutionContext.appendStdout(linelist) - logArray.clear() - } - } { t => - logger.warn("inputReader reading the input stream", t) - } - IOUtils.closeQuietly(inputReader) - counter.countDown() - } - -} diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.scala b/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.scala deleted file mode 100644 index b66353ca09a..00000000000 --- a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.scala +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.engineplugin.shell.executor - -import org.apache.linkis.common.utils.{Logging, OverloadUtils, Utils} -import org.apache.linkis.engineconn.computation.executor.execute.{ - ConcurrentComputationExecutor, - EngineExecutionContext -} -import org.apache.linkis.engineconn.core.EngineConnObject -import org.apache.linkis.governance.common.utils.GovernanceUtils -import org.apache.linkis.manager.common.entity.resource.{ - CommonNodeResource, - LoadResource, - NodeResource -} -import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils -import org.apache.linkis.manager.engineplugin.shell.common.ShellEngineConnPluginConst -import org.apache.linkis.manager.engineplugin.shell.conf.ShellEngineConnConf -import org.apache.linkis.manager.engineplugin.shell.exception.ShellCodeErrorException -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.protocol.engine.JobProgressInfo -import org.apache.linkis.rpc.Sender -import org.apache.linkis.scheduler.executer.{ - ErrorExecuteResponse, - ExecuteResponse, - SuccessExecuteResponse -} - -import org.apache.commons.lang3.StringUtils - -import java.io.{BufferedReader, File, InputStreamReader} -import java.util -import java.util.concurrent.{ConcurrentHashMap, CountDownLatch} -import java.util.concurrent.atomic.AtomicBoolean - -import scala.collection.mutable.ArrayBuffer -import scala.concurrent.ExecutionContextExecutorService - -class ShellEngineConnConcurrentExecutor(id: Int, maxRunningNumber: Int) - extends ConcurrentComputationExecutor - with Logging { - - private var engineExecutionContext: EngineExecutionContext = _ - - private val executorLabels: util.List[Label[_]] = new util.ArrayList[Label[_]]() - - private val shellECTaskInfoCache: util.Map[String, ShellECTaskInfo] = - new ConcurrentHashMap[String, ShellECTaskInfo]() - - private implicit val logAsyncService: ExecutionContextExecutorService = - Utils.newCachedExecutionContext( - ShellEngineConnConf.LOG_SERVICE_MAX_THREAD_SIZE, - "ShelLogService-Thread-" - ) - - override def init(): Unit = { - logger.info(s"Ready to change engine state!") - super.init - } - - override def executeCompletely( - engineExecutionContext: EngineExecutionContext, - code: String, - completedLine: String - ): ExecuteResponse = { - val newcode = completedLine + code - logger.debug("newcode is " + newcode) - executeLine(engineExecutionContext, newcode) - } - - override def executeLine( - engineExecutionContext: EngineExecutionContext, - code: String - ): ExecuteResponse = { - - if (null != engineExecutionContext) { - this.engineExecutionContext = engineExecutionContext - logger.info("Shell executor reset new engineExecutionContext!") - } - - if (engineExecutionContext.getJobId.isEmpty) { - return ErrorExecuteResponse("taskID is null", null) - } - - val taskId = engineExecutionContext.getJobId.get - var bufferedReader: BufferedReader = null - var errorsReader: BufferedReader = null - - val completed = new AtomicBoolean(false) - var errReaderThread: ReaderThread = null - var inputReaderThread: ReaderThread = null - - try { - engineExecutionContext.appendStdout(s"$getId >> ${code.trim}") - - val argsArr = - if ( - engineExecutionContext.getTotalParagraph == 1 && - engineExecutionContext.getProperties != null && - engineExecutionContext.getProperties.containsKey( - ShellEngineConnPluginConst.RUNTIME_ARGS_KEY - ) - ) { - Utils.tryCatch { - val argsList = engineExecutionContext.getProperties - .get(ShellEngineConnPluginConst.RUNTIME_ARGS_KEY) - .asInstanceOf[util.ArrayList[String]] - logger.info( - "Will execute shell task with user-specified arguments: \'" + argsList - .toArray(new Array[String](argsList.size())) - .mkString("\' \'") + "\'" - ) - argsList.toArray(new Array[String](argsList.size())) - } { t => - logger.warn( - "Cannot read user-input shell arguments. Will execute shell task without them.", - t - ) - null - } - } else { - null - } - - val workingDirectory = - if ( - engineExecutionContext.getTotalParagraph == 1 && - engineExecutionContext.getProperties != null && - engineExecutionContext.getProperties.containsKey( - ShellEngineConnPluginConst.SHELL_RUNTIME_WORKING_DIRECTORY - ) - ) { - Utils.tryCatch { - val wdStr = engineExecutionContext.getProperties - .get(ShellEngineConnPluginConst.SHELL_RUNTIME_WORKING_DIRECTORY) - .asInstanceOf[String] - if (isExecutePathExist(wdStr)) { - logger.info( - "Will execute shell task under user-specified working-directory: \'" + wdStr - ) - wdStr - } else { - logger.warn( - "User-specified working-directory: \'" + wdStr + "\' does not exist or user does not have access permission. " + - "Will execute shell task under default working-directory. Please contact the administrator!" - ) - null - } - } { t => - logger.warn( - "Cannot read user-input working-directory. Will execute shell task under default working-directory.", - t - ) - null - } - } else { - null - } - - val generatedCode = if (argsArr == null || argsArr.isEmpty) { - generateRunCode(code) - } else { - generateRunCodeWithArgs(code, argsArr) - } - - val processBuilder: ProcessBuilder = new ProcessBuilder(generatedCode: _*) - if (StringUtils.isNotBlank(workingDirectory)) { - processBuilder.directory(new File(workingDirectory)) - } - - processBuilder.redirectErrorStream(false) - val extractor = new YarnAppIdExtractor - val process = processBuilder.start() - bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream)) - errorsReader = new BufferedReader(new InputStreamReader(process.getErrorStream)) - // add task id and task Info cache - shellECTaskInfoCache.put(taskId, ShellECTaskInfo(taskId, process, extractor)) - - val counter: CountDownLatch = new CountDownLatch(2) - inputReaderThread = - new ReaderThread(engineExecutionContext, bufferedReader, extractor, true, counter) - errReaderThread = - new ReaderThread(engineExecutionContext, errorsReader, extractor, false, counter) - - logAsyncService.execute(inputReaderThread) - logAsyncService.execute(errReaderThread) - - val exitCode = process.waitFor() - counter.await() - - completed.set(true) - - if (exitCode != 0) { - ErrorExecuteResponse("run shell failed", ShellCodeErrorException()) - } else SuccessExecuteResponse() - - } catch { - case e: Exception => - logger.error("Execute shell code failed, reason:", e) - ErrorExecuteResponse("run shell failed", e) - } finally { - if (null != errorsReader) { - inputReaderThread.onDestroy() - } - if (null != inputReaderThread) { - errReaderThread.onDestroy() - } - shellECTaskInfoCache.remove(taskId) - } - } - - private def isExecutePathExist(executePath: String): Boolean = { - val etlHomeDir: File = new File(executePath) - (etlHomeDir.exists() && etlHomeDir.isDirectory) - } - - private def generateRunCode(code: String): Array[String] = { - Array("sh", "-c", code) - } - - private def generateRunCodeWithArgs(code: String, args: Array[String]): Array[String] = { - Array("sh", "-c", "echo \"dummy " + args.mkString(" ") + "\" | xargs sh -c \'" + code + "\'") - } - - override def getId(): String = Sender.getThisServiceInstance.getInstance + "_" + id - - override def getProgressInfo(taskID: String): Array[JobProgressInfo] = { - val jobProgressInfo = new ArrayBuffer[JobProgressInfo]() - if (null == this.engineExecutionContext) { - return jobProgressInfo.toArray - } - if (0.0f == progress(taskID)) { - jobProgressInfo += JobProgressInfo(engineExecutionContext.getJobId.getOrElse(""), 1, 1, 0, 0) - } else { - jobProgressInfo += JobProgressInfo(engineExecutionContext.getJobId.getOrElse(""), 1, 0, 0, 1) - } - jobProgressInfo.toArray - } - - override def progress(taskID: String): Float = { - if (null != this.engineExecutionContext) { - this.engineExecutionContext.getCurrentParagraph / this.engineExecutionContext.getTotalParagraph - .asInstanceOf[Float] - } else { - 0.0f - } - } - - override def supportCallBackLogs(): Boolean = { - // todo - true - } - - override def requestExpectedResource(expectedResource: NodeResource): NodeResource = { - null - } - - override def getCurrentNodeResource(): NodeResource = { - val resource = new CommonNodeResource - resource.setUsedResource( - NodeResourceUtils - .applyAsLoadInstanceResource(EngineConnObject.getEngineCreationContext.getOptions) - ) - resource - } - - override def getExecutorLabels(): util.List[Label[_]] = executorLabels - - override def setExecutorLabels(labels: util.List[Label[_]]): Unit = { - if (null != labels) { - executorLabels.clear() - executorLabels.addAll(labels) - } - } - - override def killTask(taskID: String): Unit = { - val shellECTaskInfo = shellECTaskInfoCache.remove(taskID) - if (null == shellECTaskInfo) { - return - } - /* - Kill sub-processes - */ - val pid = getPid(shellECTaskInfo.process) - GovernanceUtils.killProcess(String.valueOf(pid), s"kill task $taskID process", false) - /* - Kill yarn-applications - */ - val yarnAppIds = shellECTaskInfo.yarnAppIdExtractor.getExtractedYarnAppIds() - GovernanceUtils.killYarnJobApp(yarnAppIds) - logger.info( - s"Finished kill yarn app ids in the engine of (${getId()}). The yarn app ids are ${yarnAppIds}" - ) - super.killTask(taskID) - - } - - private def getPid(process: Process): Int = { - Utils.tryCatch { - val clazz = Class.forName("java.lang.UNIXProcess"); - val field = clazz.getDeclaredField("pid"); - field.setAccessible(true); - field.get(process).asInstanceOf[Int] - } { t => - logger.warn("Failed to acquire pid for shell process") - -1 - } - } - - override def close(): Unit = { - Utils.tryCatch { - killAll() - logAsyncService.shutdown() - } { t: Throwable => - logger.error(s"Shell ec failed to close ", t) - } - super.close() - } - - override def killAll(): Unit = { - val iterator = shellECTaskInfoCache.values().iterator() - while (iterator.hasNext) { - val shellECTaskInfo = iterator.next() - Utils.tryAndWarn(killTask(shellECTaskInfo.taskId)) - } - } - - override def getConcurrentLimit: Int = { - maxRunningNumber - } - -} diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.scala b/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.scala deleted file mode 100644 index c9f32062592..00000000000 --- a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.scala +++ /dev/null @@ -1,319 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.engineplugin.shell.executor - -import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.engineconn.computation.executor.execute.{ - ComputationExecutor, - EngineExecutionContext -} -import org.apache.linkis.engineconn.core.EngineConnObject -import org.apache.linkis.governance.common.utils.GovernanceUtils -import org.apache.linkis.manager.common.entity.resource.{ - CommonNodeResource, - LoadInstanceResource, - NodeResource -} -import org.apache.linkis.manager.engineplugin.common.conf.EngineConnPluginConf -import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils -import org.apache.linkis.manager.engineplugin.shell.common.ShellEngineConnPluginConst -import org.apache.linkis.manager.engineplugin.shell.exception.ShellCodeErrorException -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.protocol.engine.JobProgressInfo -import org.apache.linkis.rpc.Sender -import org.apache.linkis.scheduler.executer.{ - ErrorExecuteResponse, - ExecuteResponse, - SuccessExecuteResponse -} - -import org.apache.commons.io.IOUtils -import org.apache.commons.lang3.StringUtils - -import java.io.{BufferedReader, File, FileReader, InputStreamReader, IOException} -import java.util -import java.util.concurrent.CountDownLatch -import java.util.concurrent.atomic.AtomicBoolean - -import scala.collection.JavaConverters._ -import scala.collection.mutable.ArrayBuffer - -class ShellEngineConnExecutor(id: Int) extends ComputationExecutor with Logging { - - private var engineExecutionContext: EngineExecutionContext = _ - - private val executorLabels: util.List[Label[_]] = new util.ArrayList[Label[_]]() - - private var process: Process = _ - - private var extractor: YarnAppIdExtractor = _ - - override def init(): Unit = { - logger.info(s"Ready to change engine state!") - super.init - } - - override def executeCompletely( - engineExecutionContext: EngineExecutionContext, - code: String, - completedLine: String - ): ExecuteResponse = { - val newcode = completedLine + code - logger.debug("newcode is " + newcode) - executeLine(engineExecutionContext, newcode) - } - - override def executeLine( - engineExecutionContext: EngineExecutionContext, - code: String - ): ExecuteResponse = { - - if (null != engineExecutionContext) { - this.engineExecutionContext = engineExecutionContext - logger.info("Shell executor reset new engineExecutionContext!") - } - - var bufferedReader: BufferedReader = null - var errorsReader: BufferedReader = null - - val completed = new AtomicBoolean(false) - var errReaderThread: ReaderThread = null - var inputReaderThread: ReaderThread = null - - try { - engineExecutionContext.appendStdout(s"$getId >> ${code.trim}") - - val argsArr = - if ( - engineExecutionContext.getTotalParagraph == 1 && - engineExecutionContext.getProperties != null && - engineExecutionContext.getProperties.containsKey( - ShellEngineConnPluginConst.RUNTIME_ARGS_KEY - ) - ) { - Utils.tryCatch { - val argsList = engineExecutionContext.getProperties - .get(ShellEngineConnPluginConst.RUNTIME_ARGS_KEY) - .asInstanceOf[util.ArrayList[String]] - logger.info( - "Will execute shell task with user-specified arguments: \'" + argsList - .toArray(new Array[String](argsList.size())) - .mkString("\' \'") + "\'" - ) - argsList.toArray(new Array[String](argsList.size())) - } { t => - logger.warn( - "Cannot read user-input shell arguments. Will execute shell task without them.", - t - ) - null - } - } else { - null - } - - val workingDirectory = - if ( - engineExecutionContext.getTotalParagraph == 1 && - engineExecutionContext.getProperties != null && - engineExecutionContext.getProperties.containsKey( - ShellEngineConnPluginConst.SHELL_RUNTIME_WORKING_DIRECTORY - ) - ) { - Utils.tryCatch { - val wdStr = engineExecutionContext.getProperties - .get(ShellEngineConnPluginConst.SHELL_RUNTIME_WORKING_DIRECTORY) - .asInstanceOf[String] - if (isExecutePathExist(wdStr)) { - logger.info( - "Will execute shell task under user-specified working-directory: \'" + wdStr - ) - wdStr - } else { - logger.warn( - "User-specified working-directory: \'" + wdStr + "\' does not exist or user does not have access permission. " + - "Will execute shell task under default working-directory. Please contact the administrator!" - ) - null - } - } { t => - logger.warn( - "Cannot read user-input working-directory. Will execute shell task under default working-directory.", - t - ) - null - } - } else { - null - } - - val generatedCode = if (argsArr == null || argsArr.isEmpty) { - generateRunCode(code) - } else { - generateRunCodeWithArgs(code, argsArr) - } - - val processBuilder: ProcessBuilder = new ProcessBuilder(generatedCode: _*) - if (StringUtils.isNotBlank(workingDirectory)) { - processBuilder.directory(new File(workingDirectory)) - } - - processBuilder.redirectErrorStream(false) - extractor = new YarnAppIdExtractor - process = processBuilder.start() - - bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream)) - errorsReader = new BufferedReader(new InputStreamReader(process.getErrorStream)) - val counter: CountDownLatch = new CountDownLatch(2) - inputReaderThread = - new ReaderThread(engineExecutionContext, bufferedReader, extractor, true, counter) - errReaderThread = - new ReaderThread(engineExecutionContext, errorsReader, extractor, false, counter) - - inputReaderThread.start() - errReaderThread.start() - - val exitCode = process.waitFor() - counter.await() - - completed.set(true) - - if (exitCode != 0) { - ErrorExecuteResponse("run shell failed", ShellCodeErrorException()) - } else SuccessExecuteResponse() - } catch { - case e: Exception => - logger.error("Execute shell code failed, reason:", e) - ErrorExecuteResponse("run shell failed", e) - } finally { - if (null != errorsReader) { - inputReaderThread.onDestroy() - } - if (null != inputReaderThread) { - errReaderThread.onDestroy() - } - IOUtils.closeQuietly(bufferedReader) - IOUtils.closeQuietly(errorsReader) - } - } - - private def isExecutePathExist(executePath: String): Boolean = { - val etlHomeDir: File = new File(executePath) - (etlHomeDir.exists() && etlHomeDir.isDirectory()) - } - - private def generateRunCode(code: String): Array[String] = { - Array("sh", "-c", code) - } - - private def generateRunCodeWithArgs(code: String, args: Array[String]): Array[String] = { - Array("sh", "-c", "echo \"dummy " + args.mkString(" ") + "\" | xargs sh -c \'" + code + "\'") - } - - override def getId(): String = Sender.getThisServiceInstance.getInstance + "_" + id - - override def getProgressInfo(taskID: String): Array[JobProgressInfo] = { - val jobProgressInfo = new ArrayBuffer[JobProgressInfo]() - if (null == this.engineExecutionContext) { - return jobProgressInfo.toArray - } - if (0.0f == progress(taskID)) { - jobProgressInfo += JobProgressInfo(engineExecutionContext.getJobId.getOrElse(""), 1, 1, 0, 0) - } else { - jobProgressInfo += JobProgressInfo(engineExecutionContext.getJobId.getOrElse(""), 1, 0, 0, 1) - } - jobProgressInfo.toArray - } - - override def progress(taskID: String): Float = { - if (null != this.engineExecutionContext) { - this.engineExecutionContext.getCurrentParagraph / this.engineExecutionContext.getTotalParagraph - .asInstanceOf[Float] - } else { - 0.0f - } - } - - override def supportCallBackLogs(): Boolean = { - // todo - true - } - - override def requestExpectedResource(expectedResource: NodeResource): NodeResource = { - null - } - - override def getCurrentNodeResource(): NodeResource = { - val resource = new CommonNodeResource - resource.setUsedResource( - NodeResourceUtils - .applyAsLoadInstanceResource(EngineConnObject.getEngineCreationContext.getOptions) - ) - resource - } - - override def getExecutorLabels(): util.List[Label[_]] = executorLabels - - override def setExecutorLabels(labels: util.List[Label[_]]): Unit = { - if (null != labels) { - executorLabels.clear() - executorLabels.addAll(labels) - } - } - - override def killTask(taskID: String): Unit = { - /* - Kill sub-processes - */ - val pid = getPid(process) - GovernanceUtils.killProcess(String.valueOf(pid), s"kill task $taskID process", false) - /* - Kill yarn-applications - */ - val yarnAppIds = extractor.getExtractedYarnAppIds() - GovernanceUtils.killYarnJobApp(yarnAppIds) - logger.info(s"Finished kill yarn app ids in the engine of (${getId()}).}") - super.killTask(taskID) - - } - - private def getPid(process: Process): Int = { - Utils.tryCatch { - val clazz = Class.forName("java.lang.UNIXProcess"); - val field = clazz.getDeclaredField("pid"); - field.setAccessible(true); - field.get(process).asInstanceOf[Int] - } { t => - logger.warn("Failed to acquire pid for shell process") - -1 - } - } - - override def close(): Unit = { - try { - process.destroy() - } catch { - case e: Exception => - logger.error(s"kill process ${process.toString} failed ", e) - case t: Throwable => - logger.error(s"kill process ${process.toString} failed ", t) - } - super.close() - } - -} diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/YarnAppIdExtractor.scala b/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/YarnAppIdExtractor.scala deleted file mode 100644 index 2f5f796632f..00000000000 --- a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/YarnAppIdExtractor.scala +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.engineplugin.shell.executor - -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.engineconn.common.conf.EngineConnConf - -import org.apache.commons.lang3.StringUtils - -import java.io._ -import java.util -import java.util.Collections -import java.util.regex.Pattern - -class YarnAppIdExtractor extends Logging { - - private val appIdList: util.Set[String] = Collections.synchronizedSet(new util.HashSet[String]()) - - private val regex = EngineConnConf.SPARK_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX.getValue - private val pattern = Pattern.compile(regex) - - def appendLineToExtractor(content: String): Unit = { - if (StringUtils.isBlank(content)) return - val yarnAppIDMatcher = pattern.matcher(content) - if (yarnAppIDMatcher.find) { - val yarnAppID = yarnAppIDMatcher.group(2) - appIdList.add(yarnAppID) - } - } - - private def doExtractYarnAppId(content: String): Array[String] = { - - if (StringUtils.isBlank(content)) return new Array[String](0) - // spark: Starting|Submitted|Activating.{1,100}(application_\d{13}_\d+) - // sqoop, importtsv: Submitted application application_1609166102854_970911 - - val stringReader = new StringReader(content) - - val reader = new BufferedReader(stringReader) - - val ret = new util.ArrayList[String] - - var line = reader.readLine() - while ({ - line != null - }) { // match application_xxx_xxx - val mApp = pattern.matcher(line) - if (mApp.find) { - val candidate1 = mApp.group(2) - if (!ret.contains(candidate1)) { - ret.add(candidate1) - } - } - line = reader.readLine - } - stringReader.close() - ret.toArray(new Array[String](ret.size())) - } - - def getExtractedYarnAppIds(): util.List[String] = { - appIdList.synchronized { - new util.ArrayList[String](appIdList) - } - } - -} diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/factory/ShellEngineConnFactory.scala b/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/factory/ShellEngineConnFactory.scala old mode 100644 new mode 100755 diff --git a/linkis-engineconn-plugins/shell/src/test/scala/org/apache/linkis/manager/engineplugin/shell/TestShellEngineConnPlugin.scala b/linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/TestShellEngineConnPlugin.java similarity index 75% rename from linkis-engineconn-plugins/shell/src/test/scala/org/apache/linkis/manager/engineplugin/shell/TestShellEngineConnPlugin.scala rename to linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/TestShellEngineConnPlugin.java index 228e5312f17..34ce7a67123 100644 --- a/linkis-engineconn-plugins/shell/src/test/scala/org/apache/linkis/manager/engineplugin/shell/TestShellEngineConnPlugin.scala +++ b/linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/TestShellEngineConnPlugin.java @@ -15,19 +15,19 @@ * limitations under the License. */ -package org.apache.linkis.manager.engineplugin.shell +package org.apache.linkis.manager.engineplugin.shell; -import org.junit.jupiter.api.{Assertions, Test} +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; -class TestShellEngineConnPlugin { +public class TestShellEngineConnPlugin { @Test - def testShellEngineConnExecutor: Unit = { - val shellEngineConnPlugin = new ShellEngineConnPlugin - Assertions.assertNotNull(shellEngineConnPlugin.getEngineConnFactory) - Assertions.assertNotNull(shellEngineConnPlugin.getEngineConnLaunchBuilder) - Assertions.assertNotNull(shellEngineConnPlugin.getEngineResourceFactory) - Assertions.assertNotNull(shellEngineConnPlugin.getDefaultLabels) + public void testShellEngineConnExecutor() { + ShellEngineConnPlugin shellEngineConnPlugin = new ShellEngineConnPlugin(); + Assertions.assertNotNull(shellEngineConnPlugin.getEngineConnFactory()); + Assertions.assertNotNull(shellEngineConnPlugin.getEngineConnLaunchBuilder()); + Assertions.assertNotNull(shellEngineConnPlugin.getEngineResourceFactory()); + Assertions.assertNotNull(shellEngineConnPlugin.getDefaultLabels()); } - } diff --git a/linkis-engineconn-plugins/shell/src/test/scala/org/apache/linkis/manager/engineplugin/shell/common/TestShellEngineConnPluginConst.scala b/linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/common/TestShellEngineConnPluginConst.java similarity index 70% rename from linkis-engineconn-plugins/shell/src/test/scala/org/apache/linkis/manager/engineplugin/shell/common/TestShellEngineConnPluginConst.scala rename to linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/common/TestShellEngineConnPluginConst.java index 59f00e8ffd6..36444cf448c 100644 --- a/linkis-engineconn-plugins/shell/src/test/scala/org/apache/linkis/manager/engineplugin/shell/common/TestShellEngineConnPluginConst.scala +++ b/linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/common/TestShellEngineConnPluginConst.java @@ -15,19 +15,18 @@ * limitations under the License. */ -package org.apache.linkis.manager.engineplugin.shell.common +package org.apache.linkis.manager.engineplugin.shell.common; -import org.junit.jupiter.api.{Assertions, Test} +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; -class TestShellEngineConnPluginConst { +public class TestShellEngineConnPluginConst { @Test - def testShellEngineConnPluginConst: Unit = { - Assertions.assertEquals("extraArguments", ShellEngineConnPluginConst.RUNTIME_ARGS_KEY) + public void testShellEngineConnPluginConst() { + Assertions.assertEquals("extraArguments", ShellEngineConnPluginConst.RUNTIME_ARGS_KEY); Assertions.assertEquals( - "wds.linkis.shell.runtime.working.directory", - ShellEngineConnPluginConst.SHELL_RUNTIME_WORKING_DIRECTORY - ) + "wds.linkis.shell.runtime.working.directory", + ShellEngineConnPluginConst.SHELL_RUNTIME_WORKING_DIRECTORY); } - } diff --git a/linkis-engineconn-plugins/shell/src/test/scala/org/apache/linkis/manager/engineplugin/shell/exception/TestNoCorrectUserException.scala b/linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/exception/TestNoCorrectUserException.java similarity index 67% rename from linkis-engineconn-plugins/shell/src/test/scala/org/apache/linkis/manager/engineplugin/shell/exception/TestNoCorrectUserException.scala rename to linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/exception/TestNoCorrectUserException.java index 50ba61bb0e6..c3cc8e682c9 100644 --- a/linkis-engineconn-plugins/shell/src/test/scala/org/apache/linkis/manager/engineplugin/shell/exception/TestNoCorrectUserException.scala +++ b/linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/exception/TestNoCorrectUserException.java @@ -15,24 +15,20 @@ * limitations under the License. */ -package org.apache.linkis.manager.engineplugin.shell.exception +package org.apache.linkis.manager.engineplugin.shell.exception; -import org.junit.jupiter.api.{Assertions, Test} +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; -class TestNoCorrectUserException { +public class TestNoCorrectUserException { @Test - def testNoCorrectUserException: Unit = { - - val exception = NoCorrectUserException - Assertions.assertNotNull(exception) + public void testNoCorrectUserException() { + Assertions.assertNotNull(new NoCorrectUserException()); } @Test - def testShellCodeErrorException: Unit = { - - val exception = ShellCodeErrorException - Assertions.assertNotNull(exception) + public void testShellCodeErrorException() { + Assertions.assertNotNull(new ShellCodeErrorException()); } - } diff --git a/linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/executor/TestShellEngineConnExecutor.java b/linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/executor/TestShellEngineConnExecutor.java new file mode 100644 index 00000000000..d327282b282 --- /dev/null +++ b/linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/executor/TestShellEngineConnExecutor.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.shell.executor; + +import org.apache.linkis.DataWorkCloudApplication; +import org.apache.linkis.common.conf.DWCArgumentsParser; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext; +import org.apache.linkis.scheduler.executer.ExecuteResponse; + +import scala.collection.mutable.HashMap; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestShellEngineConnExecutor { + + @Test + public void testShellEngineConnExecutor() throws ReflectiveOperationException { + boolean isWindows = System.getProperty("os.name").startsWith("Windows"); + + System.setProperty("wds.linkis.server.version", "v1"); + System.setProperty("HADOOP_CONF_DIR", "./"); + System.setProperty( + "wds.linkis.engineconn.plugin.default.class", + "org.apache.linkis.manager.engineplugin.shell.ShellEngineConnPlugin"); + + HashMap map = new HashMap<>(); + map.put("spring.mvc.servlet.path", "/api/rest_j/v1"); + map.put("server.port", "26380"); + map.put("spring.application.name", "shellEngineExecutor"); + map.put("eureka.client.register-with-eureka", "false"); + map.put("eureka.client.fetch-registry", "false"); + DataWorkCloudApplication.main(DWCArgumentsParser.formatSpringOptions(map.toMap(null))); + ShellEngineConnExecutor shellEngineConnExecutor = new ShellEngineConnExecutor(1); + shellEngineConnExecutor.init(); + Assertions.assertTrue(shellEngineConnExecutor.isEngineInitialized()); + if (!isWindows) { + EngineExecutionContext engineExecutionContext = + new EngineExecutionContext(shellEngineConnExecutor, Utils.getJvmUser()); + ExecuteResponse response = shellEngineConnExecutor.executeLine(engineExecutionContext, "id"); + Assertions.assertNotNull(response); + shellEngineConnExecutor.close(); + } + } +} diff --git a/linkis-engineconn-plugins/shell/src/test/scala/org/apache/linkis/manager/engineplugin/shell/executor/TestShellEngineConnExecutor.scala b/linkis-engineconn-plugins/shell/src/test/scala/org/apache/linkis/manager/engineplugin/shell/executor/TestShellEngineConnExecutor.scala deleted file mode 100644 index 833db129f4d..00000000000 --- a/linkis-engineconn-plugins/shell/src/test/scala/org/apache/linkis/manager/engineplugin/shell/executor/TestShellEngineConnExecutor.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.engineplugin.shell.executor - -import org.apache.linkis.DataWorkCloudApplication -import org.apache.linkis.common.conf.DWCArgumentsParser -import org.apache.linkis.common.utils.Utils -import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext - -import scala.collection.mutable - -import org.junit.jupiter.api.{Assertions, Test} - -class TestShellEngineConnExecutor { - - @Test - def testShellEngineConnExecutor: Unit = { - val isWindows = System.getProperty("os.name").startsWith("Windows") - - System.setProperty("wds.linkis.server.version", "v1") - System.setProperty("HADOOP_CONF_DIR", "./") - System.setProperty( - "wds.linkis.engineconn.plugin.default.class", - "org.apache.linkis.manager.engineplugin.shell.ShellEngineConnPlugin" - ) - val map = new mutable.HashMap[String, String]() - map.put("spring.mvc.servlet.path", "/api/rest_j/v1") - map.put("server.port", "26380") - map.put("spring.application.name", "shellEngineExecutor") - map.put("eureka.client.register-with-eureka", "false") - map.put("eureka.client.fetch-registry", "false") - DataWorkCloudApplication.main(DWCArgumentsParser.formatSpringOptions(map.toMap)) - val shellEngineConnExecutor = new ShellEngineConnExecutor(1) - shellEngineConnExecutor.init() - Assertions.assertTrue(shellEngineConnExecutor.isEngineInitialized) - if (!isWindows) { - - val engineExecutionContext = - new EngineExecutionContext(shellEngineConnExecutor, Utils.getJvmUser) - val response = shellEngineConnExecutor.executeLine(engineExecutionContext, "id") - Assertions.assertNotNull(response) - shellEngineConnExecutor.close() - } - - } - -} From 179abedf87490488f86a53a5d99208b956727fa7 Mon Sep 17 00:00:00 2001 From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com> Date: Fri, 28 Apr 2023 11:57:50 +0800 Subject: [PATCH 127/689] improve yarn resource requester (#4492) Co-authored-by: gf13871 --- .../external/yarn/YarnResourceRequester.scala | 94 ++++++++++--------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.scala index 4ba592d5ba8..0109f7d321f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.scala @@ -80,14 +80,13 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { def maxEffectiveHandle(queueValue: Option[Any]): Option[YarnResource] = { val metrics = getResponseByUrl("metrics", rmWebAddress) - val ctx = JsonPath.parse(metrics) - val totalMB = ctx.read("$.clusterMetrics.totalMB").asInstanceOf[Long] - val totalVirtualCores = - ctx.read("$.clusterMetrics.totalVirtualCores").asInstanceOf[Long] + .get("clusterMetrics") + .asInstanceOf[util.Map[String, Object]] + val totalMB = metrics.get("totalMB").asInstanceOf[Integer] + val totalVirtualCores = metrics.get("totalVirtualCores").asInstanceOf[Integer] val totalResouceInfoResponse = (totalMB, totalVirtualCores) - queueValue.map(r => { - val absoluteCapacity = JsonPath.read(r, "$.absoluteCapacity") + val absoluteCapacity = r.asInstanceOf[util.Map[String, Object]].get("absoluteCapacity") val effectiveResource = { if (absoluteCapacity.isInstanceOf[BigDecimal]) { @@ -113,8 +112,8 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { var realQueueName = "root." + queueName def getQueue(queues: Any): Option[Any] = { - if (queues.isInstanceOf[List[Any]]) { - queues.asInstanceOf[List[Any]].foreach { q => + if (queues.isInstanceOf[util.List[Any]]) { + queues.asInstanceOf[util.List[Any]].asScala.foreach { q => val yarnQueueName = JsonPath.read(q, "$.queueName").asInstanceOf[String] if (yarnQueueName == realQueueName) return Some(q) else if (realQueueName.startsWith(yarnQueueName + ".")) { @@ -122,16 +121,18 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { } } None - } else if (queues.isInstanceOf[Map[Any, Any]]) { + } else if (queues.isInstanceOf[util.Map[Any, Any]]) { if ( queues - .asInstanceOf[Map[Any, Any]] + .asInstanceOf[util.Map[Any, Any]] + .asScala .find(_._1 == "queueName") .exists(_._2.toString == realQueueName) ) { Some(queues) } else { - val childQueues = queues.asInstanceOf[Map[Any, Any]].find(_._1 == "childQueues") + val childQueues = + queues.asInstanceOf[util.Map[Any, Any]].asScala.find(_._1 == "childQueues") if (childQueues.isEmpty) None else getQueue(childQueues.map(_._2).get) } @@ -144,39 +145,39 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { val ctx = JsonPath.parse(resp) val childQueuesValue = ctx.read("$.childQueues") val queues = - ctx.read("$.childQueues.queue").asInstanceOf[List[Any]] + ctx.read("$.childQueues.queue").asInstanceOf[util.List[Any]] - if (queues != null && queues.nonEmpty) { + if (queues != null && queues.size() > 0) { logger.info(s"queues:$queues") queues } else childQueuesValue } def getQueueOfCapacity(queues: Any): Option[Any] = { - if (queues.isInstanceOf[List[Any]]) { - queues.asInstanceOf[List[Any]].foreach { q => + if (queues.isInstanceOf[util.List[Any]]) { + queues.asInstanceOf[util.List[Any]].asScala.foreach { q => val ctx = JsonPath.parse(q) val yarnQueueName = ctx.read("$.queueName").asInstanceOf[String] - val queuesValue = ctx.read("$.queues").asInstanceOf[String] if (yarnQueueName == realQueueName) return Some(q) - else if (queuesValue.nonEmpty) { + else if (ctx.read("$.queues").asInstanceOf[String].nonEmpty) { val matchQueue = getQueueOfCapacity(getChildQueuesOfCapacity(q)) - if (matchQueue.nonEmpty) return matchQueue + if (matchQueue.nonEmpty) return Some(matchQueue) } } None - } else if (queues.isInstanceOf[Map[Any, Any]]) { + } else if (queues.isInstanceOf[util.Map[Any, Any]]) { val queuesValue = JsonPath.read(queues, "$.queues").asInstanceOf[String] if ( queues - .asInstanceOf[Map[Any, Any]] + .asInstanceOf[util.Map[Any, Any]] + .asScala .find(_._1 == "queueName") .exists(_._2.toString == realQueueName) ) { return Some(queues) } else if (queuesValue.nonEmpty) { - val matchQueue = getQueueOfCapacity(getChildQueuesOfCapacity(queues.toString)) + val matchQueue = getQueueOfCapacity(getChildQueuesOfCapacity(queues)) if (matchQueue.nonEmpty) return matchQueue } None @@ -186,18 +187,16 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { } def getChildQueuesOfCapacity(resp: Any) = { - JsonPath.read(resp, "$.queues.queue").asInstanceOf[String] + JsonPath.read(resp, "$.queues.queue").asInstanceOf[Any] } def getResources() = { val resp = getResponseByUrl("scheduler", rmWebAddress) val ctx = JsonPath.parse(resp) val schedulerInfoValue = - ctx.read("$.scheduler.schedulerInfo").asInstanceOf[String] + ctx.read("$.scheduler.schedulerInfo").asInstanceOf[util.Map[String, Object]] val schedulerType = ctx.read("$.scheduler.schedulerInfo.type").asInstanceOf[String] - val rootQueueValue = - ctx.read("$.scheduler.schedulerInfo.rootQueue").asInstanceOf[String] if ("capacityScheduler".equals(schedulerType)) { realQueueName = queueName @@ -211,13 +210,15 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { ) } - val resourceCtx = JsonPath.parse(queue) - val usedMemory = resourceCtx.read("$.resourcesUsed.memory").asInstanceOf[Long] - val usedvCores = resourceCtx.read("$.resourcesUsed.vCores").asInstanceOf[Int] + val resourceCtx = JsonPath.parse(queue.get) + val usedMemory = resourceCtx.read("$.resourcesUsed.memory").asInstanceOf[Integer] + val usedvCores = resourceCtx.read("$.resourcesUsed.vCores").asInstanceOf[Integer] val resourcesUsed = new YarnResource(usedMemory * 1024L * 1024L, usedvCores, 0, queueName) (maxEffectiveHandle(queue).get, resourcesUsed) } else if ("fairScheduler".equals(schedulerType)) { + val rootQueueValue = + ctx.read("$.scheduler.schedulerInfo.rootQueue").asInstanceOf[util.Map[String, Object]] val childQueues = getChildQueues(rootQueueValue) val queue = getQueue(childQueues) if (queue.isEmpty) { @@ -228,13 +229,13 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { ) } val resourceCtx = JsonPath.parse(queue) - val maxResourceMemory = resourceCtx.read("$.maxResources.memory").asInstanceOf[Long] - val maxResourcevCores = resourceCtx.read("$.maxResources.vCores").asInstanceOf[Int] + val maxResourceMemory = resourceCtx.read("$.maxResources.memory").asInstanceOf[Integer] + val maxResourcevCores = resourceCtx.read("$.maxResources.vCores").asInstanceOf[Integer] val maxResources = new YarnResource(maxResourceMemory * 1024L * 1024L, maxResourcevCores, 0, queueName) - val usedResourceMemory = resourceCtx.read("$.usedResources.memory").asInstanceOf[Long] - val usedResourcevCores = resourceCtx.read("$.usedResources.vCores").asInstanceOf[Int] + val usedResourceMemory = resourceCtx.read("$.usedResources.memory").asInstanceOf[Integer] + val usedResourcevCores = resourceCtx.read("$.usedResources.vCores").asInstanceOf[Integer] val usedResourcesUsed = new YarnResource(usedResourceMemory * 1024L * 1024L, usedResourcevCores, 0, queueName) @@ -278,14 +279,12 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { val realQueueName = "root." + queueName def getAppInfos(): Array[ExternalAppInfo] = { - val resp = getResponseByUrl("apps", rmWebAddress) - val ctx = JsonPath.parse(resp) - val apps = ctx.read("$.apps") - val app = ctx.read("$.apps.app") + val apps = getResponseByUrl("apps", rmWebAddress).get("apps") + val app = apps.asInstanceOf[util.Map[String, Object]].get("app") - if (app.isInstanceOf[List[Any]]) { + if (app.isInstanceOf[util.List[Any]]) { val appInfoBuffer = new ArrayBuffer[YarnAppInfo]() - apps.asInstanceOf[List[Any]].foreach { app => + app.asInstanceOf[util.List[Any]].asScala.foreach { app => val appCtx = JsonPath.parse(app) val queueValue = appCtx.read("$.queue").asInstanceOf[String] val stateValue = appCtx.read("$.state").asInstanceOf[String] @@ -294,8 +293,8 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { val applicationTypeValue = appCtx.read("$.applicationType").asInstanceOf[String] val yarnQueueName = queueValue - val allocatedMB = appCtx.read("$.allocatedMB").asInstanceOf[Long] - val allocatedVCores = appCtx.read("$.allocatedVCores").asInstanceOf[Int] + val allocatedMB = appCtx.read("$.allocatedMB").asInstanceOf[Integer] + val allocatedVCores = appCtx.read("$.allocatedVCores").asInstanceOf[Integer] val yarnResource = new YarnResource(allocatedMB * 1024L * 1024L, allocatedVCores, 0, queueName) @@ -353,7 +352,12 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { val response = YarnResourceRequester.httpClient.execute(httpGet) httpResponse = response } - JsonUtils.jackson.readValue(EntityUtils.toString(httpResponse.getEntity()), classOf[String]) + val response = YarnResourceRequester.httpClient.execute(httpGet) + httpResponse = response + JsonUtils.jackson.readValue( + EntityUtils.toString(httpResponse.getEntity()), + classOf[util.Map[String, Object]] + ) } def getAndUpdateActiveRmWebAddress(haAddress: String): String = { @@ -371,11 +375,13 @@ class YarnResourceRequester extends ExternalResourceRequester with Logging { .split(RMConfiguration.DEFAULT_YARN_RM_WEB_ADDRESS_DELIMITER.getValue) .foreach(address => { Utils.tryCatch { - val response = getResponseByUrl("info", address).asInstanceOf[Any] - val haState = JsonPath.read(response, "$.clusterInfo.haState") + val haState = getResponseByUrl("info", address) + .get("clusterInfo") + .asInstanceOf[util.Map[String, Object]] + .get("haState") if (haState.isInstanceOf[String]) { - if (HASTATE_ACTIVE.equalsIgnoreCase(haState)) { + if (HASTATE_ACTIVE.equalsIgnoreCase(haState.toString)) { activeAddress = address } else { logger.warn(s"Resourcemanager : ${address} haState : ${haState}") From dca07627d8ba398fad085ee42a2593e9b50f14dc Mon Sep 17 00:00:00 2001 From: peacewong Date: Fri, 28 Apr 2023 10:58:04 +0800 Subject: [PATCH 128/689] Remove the default compilation of impala --- linkis-engineconn-plugins/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-engineconn-plugins/pom.xml b/linkis-engineconn-plugins/pom.xml index 9fdb4b85b86..f85bb401440 100644 --- a/linkis-engineconn-plugins/pom.xml +++ b/linkis-engineconn-plugins/pom.xml @@ -41,7 +41,7 @@ trino elasticsearch seatunnel - impala + From 41798ead450319694a39b4e858c419522b54c015 Mon Sep 17 00:00:00 2001 From: jacktao007 Date: Sat, 29 Apr 2023 14:39:52 +0800 Subject: [PATCH 129/689] Fix front-end compilation exceptions (#4503) --- linkis-web/package.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/linkis-web/package.json b/linkis-web/package.json index d1ed26e814d..006750ec17e 100644 --- a/linkis-web/package.json +++ b/linkis-web/package.json @@ -74,8 +74,5 @@ "vue-cli-plugin-mockjs": "0.1.3", "vue-template-compiler": "2.6.12", "webpack-virtual-modules": "0.3.2" - }, - "resolutions": { - "postcss": "7.0.36" } } From 806d37ae77ddd09c25922c574bfb9bb236758e72 Mon Sep 17 00:00:00 2001 From: jacktao007 Date: Sat, 29 Apr 2023 14:40:55 +0800 Subject: [PATCH 130/689] fixed the front-end startup exception (#4505) --- linkis-web/.env | 1 + 1 file changed, 1 insertion(+) diff --git a/linkis-web/.env b/linkis-web/.env index 27591f1bfd7..d9f101acc35 100644 --- a/linkis-web/.env +++ b/linkis-web/.env @@ -1,4 +1,5 @@ VUE_APP_HOST= +BACKEND_URL=http://127.0.0.1:9001 VUE_APP_MN_CONFIG_PREFIX= VUE_APP_MN_CONFIG_SOCKET=/ws/api/entrance/connect VUE_APP_VERSION=1.3.2 From ba581c60c1dc06f7c2c9d6315fc2bc3c725a1c2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BA=BA=E7=94=9F=E6=9C=89=E5=A6=82=E4=B8=A4=E4=B8=AA?= =?UTF-8?q?=E6=A9=98=E5=AD=90?= <15764973965@163.com> Date: Sat, 29 Apr 2023 14:44:41 +0800 Subject: [PATCH 131/689] [Bugfix] update refresh time of providerCache is 30m (#4498) * fix refresh time is 30m close #4497 * Correct the word * code format --- .../rm/external/service/impl/ExternalResourceServiceImpl.java | 3 +-- .../scala/org/apache/linkis/manager/rm/utils/RMUtils.scala | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/service/impl/ExternalResourceServiceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/service/impl/ExternalResourceServiceImpl.java index 73d785643fe..081f7ee5a2a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/service/impl/ExternalResourceServiceImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/service/impl/ExternalResourceServiceImpl.java @@ -74,8 +74,7 @@ public class ExternalResourceServiceImpl implements ExternalResourceService, Ini CacheBuilder.newBuilder() .maximumSize(20) .expireAfterAccess(1, TimeUnit.HOURS) - .refreshAfterWrite( - RMUtils.EXTERNAL_RESOURCE_REFRESH_TIME().getValue().toLong(), TimeUnit.MINUTES) + .refreshAfterWrite(RMUtils.EXTERNAL_RESOURCE_REFRESH_TIME(), TimeUnit.MINUTES) .build( new CacheLoader>() { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala index bb300ca5039..cc1ea02c0a9 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala @@ -47,7 +47,7 @@ object RMUtils extends Logging { CommonVars("wds.linkis.manager.rm.debug.log.path", "file:///tmp/linkis/rmLog") val EXTERNAL_RESOURCE_REFRESH_TIME = - CommonVars("wds.linkis.manager.rm.external.resource.regresh.time", new TimeType("30m")) + CommonVars[Long]("wds.linkis.manager.rm.external.resource.refresh.time", 30L).getValue val COMBINED_USERCREATOR_ENGINETYPE = "combined_userCreator_engineType" From 6f4dad9acd8e132a46264eaaf77cd9f4fe5c230b Mon Sep 17 00:00:00 2001 From: ahaoyao <129247228+ahaoyao@users.noreply.github.com> Date: Sat, 29 Apr 2023 14:45:26 +0800 Subject: [PATCH 132/689] Upgrade node to version 16.0.0 (#4495) Co-authored-by: ahaoyao --- .github/workflows/publish-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml index 3a9f6c2a0d4..9a56415b417 100644 --- a/.github/workflows/publish-docker.yaml +++ b/.github/workflows/publish-docker.yaml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [14.17.3] + node-version: [16.0.0] timeout-minutes: 90 env: TAG: ${{ github.sha }} From b3c105d81a494cb280c0892373012b671ad9a43f Mon Sep 17 00:00:00 2001 From: ChengJie1053 <18033291053@163.com> Date: Sat, 29 Apr 2023 22:32:03 +0800 Subject: [PATCH 133/689] Support starrocks dataSources (#4496) * Formatting code * Modify sql --- linkis-dist/package/db/linkis_dml.sql | 13 +++ .../upgrade/1.4.0_schema/mysql/linkis_dml.sql | 13 +++ .../common/cache/CacheConfiguration.java | 2 +- .../query/service/StarrocksMetaService.java | 94 +++++++++++++++++++ 4 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/StarrocksMetaService.java diff --git a/linkis-dist/package/db/linkis_dml.sql b/linkis-dist/package/db/linkis_dml.sql index f64387866aa..2ec20f3541c 100644 --- a/linkis-dist/package/db/linkis_dml.sql +++ b/linkis-dist/package/db/linkis_dml.sql @@ -555,6 +555,7 @@ INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `cl INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('doris', 'doris数据库', 'doris', 'olap', '', 4, 'Doris Database', 'Doris', 'Olap'); INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('clickhouse', 'clickhouse数据库', 'clickhouse', 'olap', '', 4, 'Clickhouse Database', 'Clickhouse', 'Olap'); INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('tidb', 'tidb数据库', 'tidb', '关系型数据库', '', 3, 'TiDB Database', 'TiDB', 'Relational Database'); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('starrocks', 'starrocks数据库', 'starrocks', 'olap', '', 4, 'StarRocks Database', 'StarRocks', 'Olap'); select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'mongodb'; @@ -735,3 +736,15 @@ VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 0, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()); + +select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'starrocks'; +INSERT INTO `linkis_ps_dm_datasource_type_key` +(`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) +VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL, 0, '地址(host1:port1,host2:port2...)', 'Address(host1:port1,host2:port2...)', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'host', '主机名(Host)', 'Host', NULL, 'TEXT', NULL, 1, '主机名(Host)', 'Host', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'port', '端口号(Port)', 'Port', NULL, 'TEXT', NULL, 1, '端口号(Port)', 'Port', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'driverClassName', '驱动类名(Driver class name)', 'Driver class name', 'com.mysql.jdbc.Driver', 'TEXT', NULL, 1, '驱动类名(Driver class name)', 'Driver class name', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 0, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()); diff --git a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql index fc099f20a57..66d9beee1be 100644 --- a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql +++ b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql @@ -29,3 +29,16 @@ VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 0, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()); +INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('starrocks', 'starrocks数据库', 'starrocks', 'olap', '', 4, 'StarRocks Database', 'StarRocks', 'Olap'); + +select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'starrocks'; +INSERT INTO `linkis_ps_dm_datasource_type_key` +(`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`) +VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL, 0, '地址(host1:port1,host2:port2...)', 'Address(host1:port1,host2:port2...)', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'host', '主机名(Host)', 'Host', NULL, 'TEXT', NULL, 1, '主机名(Host)', 'Host', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'port', '端口号(Port)', 'Port', NULL, 'TEXT', NULL, 1, '端口号(Port)', 'Port', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'driverClassName', '驱动类名(Driver class name)', 'Driver class name', 'com.mysql.jdbc.Driver', 'TEXT', NULL, 1, '驱动类名(Driver class name)', 'Driver class name', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 0, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), + (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()); diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java index 54211b6ae64..54baec4b7e4 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java @@ -34,5 +34,5 @@ public class CacheConfiguration { public static final CommonVars MYSQL_RELATIONSHIP_LIST = CommonVars.apply( "wds.linkis.server.mdq.mysql.relationship", - "mysql,oracle,kingbase,postgresql,sqlserver,db2,greenplum,dm,doris,clickhouse,tidb"); + "mysql,oracle,kingbase,postgresql,sqlserver,db2,greenplum,dm,doris,clickhouse,tidb,starrocks"); } diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/StarrocksMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/StarrocksMetaService.java new file mode 100644 index 00000000000..6d4a146d74f --- /dev/null +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/StarrocksMetaService.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.metadata.query.service; + +import org.apache.linkis.datasourcemanager.common.util.json.Json; +import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; +import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService; +import org.apache.linkis.metadata.query.common.service.MetadataConnection; +import org.apache.linkis.metadata.query.service.conf.SqlParamsMapper; +import org.apache.linkis.metadata.query.service.mysql.SqlConnection; + +import org.springframework.stereotype.Component; + +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +public class StarrocksMetaService extends AbstractDbMetaService { + @Override + public MetadataConnection getConnection( + String operator, Map params) throws Exception { + String host = + String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_HOST.getValue(), "")); + Integer port = + (Double.valueOf( + String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_PORT.getValue(), 0)))) + .intValue(); + String username = + String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_USERNAME.getValue(), "")); + String password = + String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_PASSWORD.getValue(), "")); + + String database = + String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_DATABASE.getValue(), "")); + Map extraParams = new HashMap<>(); + Object sqlParamObj = params.get(SqlParamsMapper.PARAM_SQL_EXTRA_PARAMS.getValue()); + if (null != sqlParamObj) { + if (!(sqlParamObj instanceof Map)) { + extraParams = + Json.fromJson(String.valueOf(sqlParamObj), Map.class, String.class, Object.class); + } else { + extraParams = (Map) sqlParamObj; + } + } + assert extraParams != null; + return new MetadataConnection<>( + new SqlConnection(host, port, username, password, database, extraParams)); + } + + @Override + public List queryDatabases(SqlConnection connection) { + try { + return connection.getAllDatabases(); + } catch (SQLException e) { + throw new RuntimeException("Fail to get Sql databases(获取数据库列表失败)", e); + } + } + + @Override + public List queryTables(SqlConnection connection, String database) { + try { + return connection.getAllTables(database); + } catch (SQLException e) { + throw new RuntimeException("Fail to get Sql tables(获取表列表失败)", e); + } + } + + @Override + public List queryColumns( + SqlConnection connection, String database, String table) { + try { + return connection.getColumns(database, table); + } catch (SQLException | ClassNotFoundException e) { + throw new RuntimeException("Fail to get Sql columns(获取字段列表失败)", e); + } + } +} From bfde76c6fba767ca1d6eafd3223878365ace0f58 Mon Sep 17 00:00:00 2001 From: jacktao007 Date: Sun, 30 Apr 2023 22:36:50 +0800 Subject: [PATCH 134/689] Release refresh button (#4507) --- .../src/apps/linkis/module/resourceManagement/index.vue | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/linkis-web/src/apps/linkis/module/resourceManagement/index.vue b/linkis-web/src/apps/linkis/module/resourceManagement/index.vue index 966120522f3..12f438b5ac3 100644 --- a/linkis-web/src/apps/linkis/module/resourceManagement/index.vue +++ b/linkis-web/src/apps/linkis/module/resourceManagement/index.vue @@ -22,8 +22,12 @@ size="large" fix/>
- +
+
+ {{$t('message.linkis.resourceManagement.resourceUsage')}} + +
+
Date: Mon, 1 May 2023 00:02:52 +0800 Subject: [PATCH 135/689] Engineplugin shell Code optimization (#4499) --- .../shell/conf/ShellEngineConnConf.java | 2 + .../ShellEngineConnConcurrentExecutor.java | 295 +++--------------- .../executor/ShellEngineConnExecutor.java | 113 ++++--- .../conf/linkis-engineconn.properties | 3 +- .../executor/TestShellEngineConnExecutor.java | 1 + 5 files changed, 110 insertions(+), 304 deletions(-) diff --git a/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/conf/ShellEngineConnConf.java b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/conf/ShellEngineConnConf.java index 3849bc1ee0e..9c8bccd8a33 100644 --- a/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/conf/ShellEngineConnConf.java +++ b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/conf/ShellEngineConnConf.java @@ -25,4 +25,6 @@ public class ShellEngineConnConf { public static final int LOG_SERVICE_MAX_THREAD_SIZE = CommonVars.apply("linkis.engineconn.shell.log.max.thread.size", 50).getValue(); + public static final int SHELL_ENGINECONN_OUTPUT_PRINT_LIMIT = + CommonVars.apply("wds.linkis.engineconn.shell.output.print.limit", 1000).getValue(); } diff --git a/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.java b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.java index 3f445ddd5d7..b426a22ebd9 100644 --- a/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.java +++ b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.java @@ -17,286 +17,91 @@ package org.apache.linkis.manager.engineplugin.shell.executor; -import org.apache.linkis.common.utils.Utils; import org.apache.linkis.engineconn.computation.executor.execute.ConcurrentComputationExecutor; import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext; -import org.apache.linkis.engineconn.core.EngineConnObject; import org.apache.linkis.governance.common.utils.GovernanceUtils; -import org.apache.linkis.manager.common.entity.resource.CommonNodeResource; import org.apache.linkis.manager.common.entity.resource.NodeResource; -import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils; -import org.apache.linkis.manager.engineplugin.shell.common.ShellEngineConnPluginConst; import org.apache.linkis.manager.engineplugin.shell.conf.ShellEngineConnConf; -import org.apache.linkis.manager.engineplugin.shell.exception.ShellCodeErrorException; import org.apache.linkis.manager.label.entity.Label; import org.apache.linkis.protocol.engine.JobProgressInfo; -import org.apache.linkis.rpc.Sender; -import org.apache.linkis.scheduler.executer.ErrorExecuteResponse; import org.apache.linkis.scheduler.executer.ExecuteResponse; -import org.apache.linkis.scheduler.executer.SuccessExecuteResponse; -import org.apache.commons.lang3.StringUtils; - -import java.io.BufferedReader; -import java.io.File; -import java.io.InputStreamReader; -import java.lang.reflect.Field; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicBoolean; - -import scala.concurrent.ExecutionContextExecutorService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ShellEngineConnConcurrentExecutor extends ConcurrentComputationExecutor { + private static final Logger logger = LoggerFactory.getLogger(ShellEngineConnConcurrentExecutor.class); - private EngineExecutionContext engineExecutionContext; + private ShellEngineConnExecutor shellEngineConnExecutor; - private List> executorLabels = new ArrayList<>(); - - private Map shellECTaskInfoCache = new ConcurrentHashMap<>(); - - private int id; private int maxRunningNumber; public ShellEngineConnConcurrentExecutor(int id, int maxRunningNumber) { - super(maxRunningNumber); - this.id = id; + super(ShellEngineConnConf.SHELL_ENGINECONN_OUTPUT_PRINT_LIMIT); + this.shellEngineConnExecutor = new ShellEngineConnExecutor(id); this.maxRunningNumber = maxRunningNumber; } - private final ExecutionContextExecutorService logAsyncService = - Utils.newCachedExecutionContext( - ShellEngineConnConf.LOG_SERVICE_MAX_THREAD_SIZE, "ShelLogService-Thread-", true); - @Override - public void init() { - logger.info("Ready to change engine state!"); - super.init(); + public ExecuteResponse executeLine(EngineExecutionContext engineExecutorContext, String code) { + return shellEngineConnExecutor.executeLine(engineExecutorContext, code); } @Override public ExecuteResponse executeCompletely( - EngineExecutionContext engineExecutionContext, String code, String completedLine) { - String newcode = completedLine + code; - logger.debug("newcode is " + newcode); - return executeLine(engineExecutionContext, newcode); - } - - @Override - public ExecuteResponse executeLine(EngineExecutionContext engineExecutionContext, String code) { - if (engineExecutionContext != null) { - this.engineExecutionContext = engineExecutionContext; - logger.info("Shell executor reset new engineExecutionContext!"); - } - - if (engineExecutionContext.getJobId().isEmpty()) { - return new ErrorExecuteResponse("taskID is null", null); - } - - String taskId = engineExecutionContext.getJobId().get(); - BufferedReader bufferedReader = null; - BufferedReader errorsReader = null; - - AtomicBoolean completed = new AtomicBoolean(false); - ReaderThread errReaderThread = null; - ReaderThread inputReaderThread = null; - - try { - engineExecutionContext.appendStdout(getId() + " >> " + code.trim()); - - String[] argsArr; - if (engineExecutionContext.getTotalParagraph() == 1 - && engineExecutionContext.getProperties() != null - && engineExecutionContext - .getProperties() - .containsKey(ShellEngineConnPluginConst.RUNTIME_ARGS_KEY)) { - ArrayList argsList = - (ArrayList) - engineExecutionContext - .getProperties() - .get(ShellEngineConnPluginConst.RUNTIME_ARGS_KEY); - argsArr = argsList.toArray(new String[argsList.size()]); - logger.info( - "Will execute shell task with user-specified arguments: '{}'", - StringUtils.join(argsArr, "' '")); - } else { - argsArr = null; - } - - String workingDirectory; - if (engineExecutionContext.getTotalParagraph() == 1 - && engineExecutionContext.getProperties() != null - && engineExecutionContext - .getProperties() - .containsKey(ShellEngineConnPluginConst.SHELL_RUNTIME_WORKING_DIRECTORY)) { - String wdStr = - (String) - engineExecutionContext - .getProperties() - .get(ShellEngineConnPluginConst.SHELL_RUNTIME_WORKING_DIRECTORY); - if (isExecutePathExist(wdStr)) { - logger.info( - "Will execute shell task under user-specified working-directory: '{}'", wdStr); - workingDirectory = wdStr; - } else { - logger.warn( - "User-specified working-directory: '{}' does not exist or user does not have access permission. Will execute shell task under default working-directory. Please contact the administrator!", - wdStr); - workingDirectory = null; - } - } else { - workingDirectory = null; - } - - String[] generatedCode = - argsArr == null || argsArr.length == 0 - ? generateRunCode(code) - : generateRunCodeWithArgs(code, argsArr); - - ProcessBuilder processBuilder = new ProcessBuilder(generatedCode); - - if (StringUtils.isNotBlank(workingDirectory)) { - processBuilder.directory(new File(workingDirectory)); - } - - processBuilder.redirectErrorStream(false); - YarnAppIdExtractor extractor = new YarnAppIdExtractor(); - Process process = processBuilder.start(); - bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream())); - errorsReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); - - // add task id and task Info cache - shellECTaskInfoCache.put(taskId, new ShellECTaskInfo(taskId, process, extractor)); - - CountDownLatch counter = new CountDownLatch(2); - inputReaderThread = - new ReaderThread(engineExecutionContext, bufferedReader, extractor, true, counter); - errReaderThread = - new ReaderThread(engineExecutionContext, errorsReader, extractor, false, counter); - - logAsyncService.execute(inputReaderThread); - logAsyncService.execute(errReaderThread); - - int exitCode = process.waitFor(); - counter.await(); - - completed.set(true); - - if (exitCode != 0) { - return new ErrorExecuteResponse("run shell failed", new ShellCodeErrorException()); - } else { - return new SuccessExecuteResponse(); - } - } catch (Exception e) { - logger.error("Execute shell code failed, reason:", e); - return new ErrorExecuteResponse("run shell failed", e); - } finally { - if (errorsReader != null) { - errReaderThread.onDestroy(); - } - if (inputReaderThread != null) { - inputReaderThread.onDestroy(); - } - shellECTaskInfoCache.remove(taskId); - } - } - - private boolean isExecutePathExist(String executePath) { - File etlHomeDir = new File(executePath); - return (etlHomeDir.exists() && etlHomeDir.isDirectory()); - } - - private String[] generateRunCode(String code) { - return new String[] {"sh", "-c", code}; - } - - private String[] generateRunCodeWithArgs(String code, String[] args) { - return new String[] { - "sh", - "-c", - "echo \"dummy " + StringUtils.join(args, " ") + "\" | xargs sh -c \'" + code + "\'" - }; + EngineExecutionContext engineExecutorContext, String code, String completedLine) { + return shellEngineConnExecutor.executeCompletely(engineExecutorContext, code, completedLine); } @Override - public String getId() { - return Sender.getThisServiceInstance().getInstance() + "_" + id; + public float progress(String taskID) { + return shellEngineConnExecutor.progress(taskID); } @Override public JobProgressInfo[] getProgressInfo(String taskID) { - List jobProgressInfos = new ArrayList<>(); - if (this.engineExecutionContext == null) { - return jobProgressInfos.toArray(new JobProgressInfo[0]); - } - - String jobId = - engineExecutionContext.getJobId().isDefined() - ? engineExecutionContext.getJobId().get() - : ""; - if (progress(taskID) == 0.0f) { - jobProgressInfos.add(new JobProgressInfo(jobId, 1, 1, 0, 0)); - } else { - jobProgressInfos.add(new JobProgressInfo(jobId, 1, 0, 0, 1)); - } - return jobProgressInfos.toArray(new JobProgressInfo[0]); - } - - @Override - public float progress(String taskID) { - if (this.engineExecutionContext != null) { - return ((float) this.engineExecutionContext.getCurrentParagraph()) - / this.engineExecutionContext.getTotalParagraph(); - } else { - return 0.0f; - } + return shellEngineConnExecutor.getProgressInfo(taskID); } @Override public boolean supportCallBackLogs() { - // todo - return true; + return shellEngineConnExecutor.supportCallBackLogs(); } @Override - public NodeResource requestExpectedResource(NodeResource expectedResource) { - return null; - } - - @Override - public NodeResource getCurrentNodeResource() { - CommonNodeResource resource = new CommonNodeResource(); - resource.setUsedResource( - NodeResourceUtils.applyAsLoadInstanceResource( - EngineConnObject.getEngineCreationContext().getOptions())); - return resource; + public String getId() { + return shellEngineConnExecutor.getId(); } @Override - public List> getExecutorLabels() { - return executorLabels; + public void close() { + try { + killAll(); + shellEngineConnExecutor.logAsyncService.shutdown(); + } catch (Exception e) { + logger.error("Shell ec failed to close "); + } + super.close(); } @Override - public void setExecutorLabels(List> labels) { - if (labels != null) { - executorLabels.clear(); - executorLabels.addAll(labels); + public void killAll() { + Iterator iterator = + shellEngineConnExecutor.shellECTaskInfoCache.values().iterator(); + while (iterator.hasNext()) { + ShellECTaskInfo shellECTaskInfo = iterator.next(); + killTask(shellECTaskInfo.getTaskId()); } } @Override public void killTask(String taskID) { - ShellECTaskInfo shellECTaskInfo = shellECTaskInfoCache.remove(taskID); + ShellECTaskInfo shellECTaskInfo = shellEngineConnExecutor.shellECTaskInfoCache.remove(taskID); if (shellECTaskInfo == null) { return; } @@ -304,7 +109,7 @@ public void killTask(String taskID) { /* Kill sub-processes */ - int pid = getPid(shellECTaskInfo.getProcess()); + int pid = shellEngineConnExecutor.getPid(shellECTaskInfo.getProcess()); GovernanceUtils.killProcess(String.valueOf(pid), "kill task " + taskID + " process", false); /* @@ -319,40 +124,28 @@ public void killTask(String taskID) { super.killTask(taskID); } - private int getPid(Process process) { - try { - Class clazz = Class.forName("java.lang.UNIXProcess"); - Field field = clazz.getDeclaredField("pid"); - field.setAccessible(true); - return field.getInt(process); - } catch (Exception e) { - logger.warn("Failed to acquire pid for shell process"); - return -1; - } + @Override + public int getConcurrentLimit() { + return maxRunningNumber; } @Override - public void close() { - try { - killAll(); - logAsyncService.shutdown(); - } catch (Exception e) { - logger.error("Shell ec failed to close "); - } - super.close(); + public List> getExecutorLabels() { + return shellEngineConnExecutor.getExecutorLabels(); } @Override - public void killAll() { - Iterator iterator = shellECTaskInfoCache.values().iterator(); - while (iterator.hasNext()) { - ShellECTaskInfo shellECTaskInfo = iterator.next(); - killTask(shellECTaskInfo.getTaskId()); - } + public void setExecutorLabels(List> labels) { + shellEngineConnExecutor.setExecutorLabels(labels); } @Override - public int getConcurrentLimit() { - return maxRunningNumber; + public NodeResource requestExpectedResource(NodeResource expectedResource) { + return shellEngineConnExecutor.requestExpectedResource(expectedResource); + } + + @Override + public NodeResource getCurrentNodeResource() { + return shellEngineConnExecutor.getCurrentNodeResource(); } } diff --git a/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.java b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.java index 96441776599..28e451d73cd 100644 --- a/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.java +++ b/linkis-engineconn-plugins/shell/src/main/java/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.java @@ -17,6 +17,7 @@ package org.apache.linkis.manager.engineplugin.shell.executor; +import org.apache.linkis.common.utils.Utils; import org.apache.linkis.engineconn.computation.executor.execute.ComputationExecutor; import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext; import org.apache.linkis.engineconn.core.EngineConnObject; @@ -25,6 +26,7 @@ import org.apache.linkis.manager.common.entity.resource.NodeResource; import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils; import org.apache.linkis.manager.engineplugin.shell.common.ShellEngineConnPluginConst; +import org.apache.linkis.manager.engineplugin.shell.conf.ShellEngineConnConf; import org.apache.linkis.manager.engineplugin.shell.exception.ShellCodeErrorException; import org.apache.linkis.manager.label.entity.Label; import org.apache.linkis.protocol.engine.JobProgressInfo; @@ -33,7 +35,6 @@ import org.apache.linkis.scheduler.executer.ExecuteResponse; import org.apache.linkis.scheduler.executer.SuccessExecuteResponse; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import java.io.BufferedReader; @@ -41,28 +42,41 @@ import java.io.InputStreamReader; import java.lang.reflect.Field; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import scala.concurrent.ExecutionContextExecutorService; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ShellEngineConnExecutor extends ComputationExecutor { private static final Logger logger = LoggerFactory.getLogger(ShellEngineConnExecutor.class); - private int id; private EngineExecutionContext engineExecutionContext; + private List> executorLabels = new ArrayList<>(); + + Map shellECTaskInfoCache = new ConcurrentHashMap<>(); + + private int id; + private Process process; + private YarnAppIdExtractor extractor; public ShellEngineConnExecutor(int id) { - super(id); + super(ShellEngineConnConf.SHELL_ENGINECONN_OUTPUT_PRINT_LIMIT); this.id = id; } + final ExecutionContextExecutorService logAsyncService = + Utils.newCachedExecutionContext( + ShellEngineConnConf.LOG_SERVICE_MAX_THREAD_SIZE, "ShelLogService-Thread-", true); + @Override public void init() { logger.info("Ready to change engine state!"); @@ -72,7 +86,7 @@ public void init() { @Override public ExecuteResponse executeCompletely( EngineExecutionContext engineExecutionContext, String code, String completedLine) { - final String newcode = completedLine + code; + String newcode = completedLine + code; logger.debug("newcode is " + newcode); return executeLine(engineExecutionContext, newcode); } @@ -84,6 +98,11 @@ public ExecuteResponse executeLine(EngineExecutionContext engineExecutionContext logger.info("Shell executor reset new engineExecutionContext!"); } + if (engineExecutionContext.getJobId().isEmpty()) { + return new ErrorExecuteResponse("taskID is null", null); + } + + String taskId = engineExecutionContext.getJobId().get(); BufferedReader bufferedReader = null; BufferedReader errorsReader = null; @@ -94,60 +113,48 @@ public ExecuteResponse executeLine(EngineExecutionContext engineExecutionContext try { engineExecutionContext.appendStdout(getId() + " >> " + code.trim()); - String[] argsArr = null; + String[] argsArr; if (engineExecutionContext.getTotalParagraph() == 1 && engineExecutionContext.getProperties() != null && engineExecutionContext .getProperties() .containsKey(ShellEngineConnPluginConst.RUNTIME_ARGS_KEY)) { - ArrayList argsList = (ArrayList) engineExecutionContext .getProperties() .get(ShellEngineConnPluginConst.RUNTIME_ARGS_KEY); - - try { - argsArr = argsList.toArray(new String[argsList.size()]); - logger.info( - "Will execute shell task with user-specified arguments: '{}'", - Arrays.toString(argsArr)); - } catch (Exception t) { - logger.warn( - "Cannot read user-input shell arguments. Will execute shell task without them.", t); - } + argsArr = argsList.toArray(new String[argsList.size()]); + logger.info( + "Will execute shell task with user-specified arguments: '{}'", + StringUtils.join(argsArr, "' '")); + } else { + argsArr = null; } - String workingDirectory = null; + String workingDirectory; if (engineExecutionContext.getTotalParagraph() == 1 && engineExecutionContext.getProperties() != null && engineExecutionContext .getProperties() .containsKey(ShellEngineConnPluginConst.SHELL_RUNTIME_WORKING_DIRECTORY)) { - String wdStr = (String) engineExecutionContext .getProperties() .get(ShellEngineConnPluginConst.SHELL_RUNTIME_WORKING_DIRECTORY); - - try { - if (isExecutePathExist(wdStr)) { - logger.info( - "Will execute shell task under user-specified working-directory: '" + wdStr + "'"); - workingDirectory = wdStr; - } else { - logger.warn( - "User-specified working-directory: '" - + wdStr - + "' does not exist or user does not have access permission. " - + "Will execute shell task under default working-directory. Please contact the administrator!"); - } - } catch (Exception t) { + if (isExecutePathExist(wdStr)) { + logger.info( + "Will execute shell task under user-specified working-directory: '{}'", wdStr); + workingDirectory = wdStr; + } else { logger.warn( - "Cannot read user-input working-directory. Will execute shell task under default working-directory.", - t); + "User-specified working-directory: '{}' does not exist or user does not have access permission. Will execute shell task under default working-directory. Please contact the administrator!", + wdStr); + workingDirectory = null; } + } else { + workingDirectory = null; } String[] generatedCode = @@ -156,6 +163,7 @@ public ExecuteResponse executeLine(EngineExecutionContext engineExecutionContext : generateRunCodeWithArgs(code, argsArr); ProcessBuilder processBuilder = new ProcessBuilder(generatedCode); + if (StringUtils.isNotBlank(workingDirectory)) { processBuilder.directory(new File(workingDirectory)); } @@ -163,17 +171,20 @@ public ExecuteResponse executeLine(EngineExecutionContext engineExecutionContext processBuilder.redirectErrorStream(false); extractor = new YarnAppIdExtractor(); process = processBuilder.start(); - bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream())); errorsReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); + + // add task id and task Info cache + shellECTaskInfoCache.put(taskId, new ShellECTaskInfo(taskId, process, extractor)); + CountDownLatch counter = new CountDownLatch(2); inputReaderThread = new ReaderThread(engineExecutionContext, bufferedReader, extractor, true, counter); errReaderThread = new ReaderThread(engineExecutionContext, errorsReader, extractor, false, counter); - inputReaderThread.start(); - errReaderThread.start(); + logAsyncService.execute(inputReaderThread); + logAsyncService.execute(errReaderThread); int exitCode = process.waitFor(); counter.await(); @@ -185,20 +196,17 @@ public ExecuteResponse executeLine(EngineExecutionContext engineExecutionContext } else { return new SuccessExecuteResponse(); } - } catch (Exception e) { logger.error("Execute shell code failed, reason:", e); return new ErrorExecuteResponse("run shell failed", e); - } finally { if (errorsReader != null) { - inputReaderThread.onDestroy(); + errReaderThread.onDestroy(); } if (inputReaderThread != null) { - errReaderThread.onDestroy(); + inputReaderThread.onDestroy(); } - IOUtils.closeQuietly(bufferedReader); - IOUtils.closeQuietly(errorsReader); + shellECTaskInfoCache.remove(taskId); } } @@ -226,9 +234,9 @@ public String getId() { @Override public JobProgressInfo[] getProgressInfo(String taskID) { - List jobProgressInfo = new ArrayList<>(); + List jobProgressInfos = new ArrayList<>(); if (this.engineExecutionContext == null) { - return jobProgressInfo.toArray(new JobProgressInfo[0]); + return jobProgressInfos.toArray(new JobProgressInfo[0]); } String jobId = @@ -236,18 +244,18 @@ public JobProgressInfo[] getProgressInfo(String taskID) { ? engineExecutionContext.getJobId().get() : ""; if (progress(taskID) == 0.0f) { - jobProgressInfo.add(new JobProgressInfo(jobId, 1, 1, 0, 0)); + jobProgressInfos.add(new JobProgressInfo(jobId, 1, 1, 0, 0)); } else { - jobProgressInfo.add(new JobProgressInfo(jobId, 1, 0, 0, 1)); + jobProgressInfos.add(new JobProgressInfo(jobId, 1, 0, 0, 1)); } - return jobProgressInfo.toArray(new JobProgressInfo[0]); + return jobProgressInfos.toArray(new JobProgressInfo[0]); } @Override public float progress(String taskID) { if (this.engineExecutionContext != null) { - return this.engineExecutionContext.getCurrentParagraph() - / (float) this.engineExecutionContext.getTotalParagraph(); + return ((float) this.engineExecutionContext.getCurrentParagraph()) + / this.engineExecutionContext.getTotalParagraph(); } else { return 0.0f; } @@ -299,12 +307,13 @@ public void killTask(String taskID) { Kill yarn-applications */ List yarnAppIds = extractor.getExtractedYarnAppIds(); + GovernanceUtils.killYarnJobApp(yarnAppIds); logger.info("Finished kill yarn app ids in the engine of ({})", getId()); super.killTask(taskID); } - private int getPid(Process process) { + int getPid(Process process) { try { Class clazz = Class.forName("java.lang.UNIXProcess"); Field field = clazz.getDeclaredField("pid"); diff --git a/linkis-engineconn-plugins/shell/src/main/resources/conf/linkis-engineconn.properties b/linkis-engineconn-plugins/shell/src/main/resources/conf/linkis-engineconn.properties index dad0da1c43f..87e5025b4de 100644 --- a/linkis-engineconn-plugins/shell/src/main/resources/conf/linkis-engineconn.properties +++ b/linkis-engineconn-plugins/shell/src/main/resources/conf/linkis-engineconn.properties @@ -19,4 +19,5 @@ wds.linkis.engineconn.plugin.default.class=org.apache.linkis.manager.engineplugi wds.linkis.engineconn.max.free.time=30m wds.linkis.engineconn.log.list.count=50 wds.linkis.engineconn.support.parallelism=true -linkis.engineconn.shell.concurrent.limit=15 \ No newline at end of file +linkis.engineconn.shell.concurrent.limit=15 +wds.linkis.engineconn.shell.output.print.limit=1000 \ No newline at end of file diff --git a/linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/executor/TestShellEngineConnExecutor.java b/linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/executor/TestShellEngineConnExecutor.java index d327282b282..3baa00b6fe1 100644 --- a/linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/executor/TestShellEngineConnExecutor.java +++ b/linkis-engineconn-plugins/shell/src/test/java/org/apache/linkis/manager/engineplugin/shell/executor/TestShellEngineConnExecutor.java @@ -53,6 +53,7 @@ public void testShellEngineConnExecutor() throws ReflectiveOperationException { if (!isWindows) { EngineExecutionContext engineExecutionContext = new EngineExecutionContext(shellEngineConnExecutor, Utils.getJvmUser()); + engineExecutionContext.setJobId("1"); ExecuteResponse response = shellEngineConnExecutor.executeLine(engineExecutionContext, "id"); Assertions.assertNotNull(response); shellEngineConnExecutor.close(); From 4b787a438ade0f4e9ba9369249acf42793b32728 Mon Sep 17 00:00:00 2001 From: CharlieYan <16526279+CharlieYan24@users.noreply.github.com> Date: Mon, 1 May 2023 00:06:15 +0800 Subject: [PATCH 136/689] fix: do not clean ec resource and service info when handler ecm close event(#4184) (#4510) --- .../rm/service/impl/DefaultResourceManager.scala | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala index 8c6266528c9..86ee2e67f79 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala @@ -190,8 +190,8 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ } /** - * The registration method is mainly used to notify all RM nodes (including the node), and the - * instance is offline. 该注册方法，主要是用于通知所有的RM节点（包括本节点），下线该实例 + * The registration method is mainly used to notify all RM nodes , and the instance is offline. + * 该注册方法，主要是用于通知所有的RM节点（包括本节点），下线该实例 */ override def unregister(serviceInstance: ServiceInstance): Unit = { @@ -199,7 +199,6 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ LabelBuilderFactoryContext.getLabelBuilderFactory.createLabel(classOf[EMInstanceLabel]) eMInstanceLabel.setServiceName(serviceInstance.getApplicationName) eMInstanceLabel.setInstance(serviceInstance.getInstance) - val ecNodes = nodeManagerPersistence.getEngineNodeByEM(serviceInstance).asScala val lock = tryLockOneLabel(eMInstanceLabel, -1, Utils.getJvmUser) try { labelResourceService.removeResourceByLabel(eMInstanceLabel) @@ -217,13 +216,6 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ resourceLockService.unLock(lock) logger.info(s"Finished to clear ecm resource:${serviceInstance}") } - ecNodes.foreach { engineNode => - Utils.tryAndWarn { - engineNode.setLabels(nodeLabelService.getNodeLabels(engineNode.getServiceInstance)) - engineNode.setNodeStatus(NodeStatus.Failed) - engineStopService.engineConnInfoClear(engineNode) - } - } logger.info(s"Finished to clear ec for ecm ${serviceInstance}") } From be9d230f033503d33099c757dc68f9be6e169448 Mon Sep 17 00:00:00 2001 From: CharlieYan <16526279+CharlieYan24@users.noreply.github.com> Date: Mon, 1 May 2023 09:36:10 +0800 Subject: [PATCH 137/689] feat: Reduce scala module: linkis-mange#4192 --- .../rpc/message/method/MessageExecutor.java | 1 + .../governance/common/utils/ECPathUtils.java | 55 ++ .../governance/common/utils/ECPathUtils.scala | 58 -- .../operator/EngineConnLogOperator.java | 203 ++++ .../operator/EngineConnYarnLogOperator.java | 127 +++ .../operator/EngineConnLogOperator.scala | 191 ---- .../operator/EngineConnYarnLogOperator.scala | 124 --- .../AbstractEngineConnLaunchService.scala | 5 +- .../impl/DefaultECMRegisterService.scala | 6 +- .../service/impl/DefaultOperateService.scala | 19 +- .../impl/ProcessEngineConnLaunchService.scala | 2 +- .../OperableOnceEngineConnOperator.java | 71 ++ .../OperableOnceEngineConnOperator.scala | 72 -- .../service/TaskExecutionServiceImpl.scala | 10 +- .../EngineConnApplicationInfoOperator.java | 37 + .../AccessibleEngineConnExecution.scala | 2 +- .../EngineConnApplicationInfoOperator.scala | 50 - .../service/DefaultManagerService.scala | 2 +- .../service/DefaultOperateService.scala | 24 +- .../service/EngineConnTimedLockService.scala | 12 +- .../executor/service/LockService.scala | 6 +- .../hook/CallbackEngineConnHook.scala | 4 +- .../callback/service/EngineConnCallback.scala | 2 +- .../entrance/restful/EntranceRestfulApi.java | 2 +- .../loader/EngineConnPluginLoaderConf.java | 50 + .../CacheablesEngineConnPluginLoader.java | 3 +- .../DefaultEngineConnPluginLoader.java | 33 +- .../BmlEngineConnPluginResourceLoader.java | 4 +- .../conf/EngineConnPluginConfiguration.java | 42 + .../EngineConnLaunchInterceptor.java} | 14 +- .../EngineConnPluginsLoaderFactory.java | 31 + ...bstractEngineConnBmlResourceGenerator.java | 136 +++ ...DefaultEngineConnBmlResourceGenerator.java | 109 +++ .../EngineConnBmlResourceGenerator.java | 30 + .../localize/EngineConnLocalizeResource.java | 30 + .../EngineConnLocalizeResourceImpl.java | 63 ++ .../DefaultEngineConnLaunchService.java | 103 ++ ...faultEngineConnResourceFactoryService.java | 75 ++ .../DefaultEngineConnResourceService.java | 351 +++++++ .../service/EngineConnLaunchService.java} | 15 +- .../EngineConnResourceFactoryService.java} | 19 +- .../service/EngineConnResourceRequest.java | 53 ++ .../service/EngineConnResourceService.java | 43 + .../service/GetEngineConnResourceRequest.java | 25 + .../RefreshAllEngineConnResourceRequest.java | 28 + .../RefreshEngineConnResourceRequest.java | 25 + .../manager/am/conf/AMConfiguration.java | 125 +++ ...ApplicationManagerSpringConfiguration.java | 32 + .../am/conf/ConfigurationMapCache.java | 69 ++ ...DefaultEngineConnConfigurationService.java | 72 ++ .../conf/EngineConnConfigurationService.java | 28 + .../manager/am/conf/ManagerMonitorConf.java | 45 + .../am/converter/DefaultMetricsConverter.java | 140 +++ .../manager/am/label/AMLabelChecker.java | 56 ++ .../manager/am/label/AMLabelFilter.java | 59 ++ .../am/label/DefaultManagerLabelService.java | 63 ++ .../am/label/EngineReuseLabelChooser.java} | 11 +- .../MultiUserEngineReuseLabelChooser.java | 104 ++ .../am/locker/DefaultEngineNodeLocker.java | 74 ++ .../manager/am/locker/EngineNodeLocker.java} | 13 +- .../am/manager/DefaultEMNodeManager.java | 223 +++++ .../am/manager/DefaultEngineNodeManager.java | 348 +++++++ .../manager/am/manager/EMNodeManager.java | 73 ++ .../manager/am/manager/EngineNodeManager.java | 69 ++ .../am/pointer/AbstractNodePointer.java | 98 ++ .../am/pointer/DefaultEMNodPointer.java | 110 +++ .../am/pointer/DefaultEngineNodPointer.java | 82 ++ .../am/pointer/DefaultNodePointerBuilder.java | 39 + .../am/recycle/AssignNodeRuleExecutor.java | 42 + .../am/recycle/RecyclingRuleExecutor.java} | 13 +- .../manager/am/restful/EMRestfulApi.java | 25 +- .../manager/am/restful/EngineRestfulApi.java | 4 +- .../am/selector/DefaultECAvailableRule.java | 43 + .../am/selector/DefaultNodeSelector.java | 75 ++ .../manager/am/selector/ECAvailableRule.java | 33 + .../manager/am/selector/NodeSelector.java} | 15 +- .../rule/AvailableNodeSelectRule.java | 68 ++ .../rule/ConcurrencyNodeSelectRule.java} | 20 +- .../selector/rule/HotspotExclusionRule.java} | 56 +- .../am/selector/rule/NewECMStandbyRule.java | 82 ++ .../am/selector/rule/NodeSelectRule.java} | 9 +- .../selector/rule/ResourceNodeSelectRule.java | 120 +++ .../am/selector/rule/ScoreNodeSelectRule.java | 70 ++ .../selector/rule/TaskInfoNodeSelectRule.java | 73 ++ .../manager/am/service/EMEngineService.java} | 27 +- .../manager/am/service/EngineService.java} | 11 +- .../manager/am/service/HeartbeatService.java} | 9 +- .../ConfCacheRemoveBroadcastListener.java | 75 ++ .../service/em/DefaultECMOperateService.java} | 25 +- .../am/service/em/DefaultEMEngineService.java | 234 +++++ .../am/service/em/DefaultEMInfoService.java | 140 +++ .../service/em/DefaultEMRegisterService.java | 124 +++ .../em/DefaultEMUnregisterService.java | 98 ++ .../am/service/em/ECMOperateService.java | 27 + .../manager/am/service/em/EMInfoService.java} | 22 +- .../am/service/em/EMRegisterService.java | 26 + .../am/service/em/EMUnregisterService.java | 28 + .../engine/AbstractEngineService.java} | 35 +- .../engine/DefaultEngineAskEngineService.java | 196 ++++ .../DefaultEngineConnCanKillService.java | 196 ++++ .../engine/DefaultEngineCreateService.java | 442 +++++++++ .../engine/DefaultEngineInfoService.java | 142 +++ .../engine/DefaultEngineOperateService.java | 39 + .../engine/DefaultEngineRecycleService.java | 92 ++ .../engine/DefaultEngineReuseService.java | 255 +++++ .../engine/DefaultEngineStopService.java | 314 ++++++ .../engine/DefaultEngineSwitchService.java} | 18 +- .../engine/EngineAskEngineService.java} | 11 +- .../engine/EngineConnCanKillService.java | 26 + .../EngineConnStatusCallbackService.java | 25 + .../service/engine/EngineCreateService.java} | 18 +- .../am/service/engine/EngineInfoService.java | 52 + .../service/engine/EngineOperateService.java | 28 + .../service/engine/EngineRecycleService.java} | 15 +- .../service/engine/EngineReuseService.java} | 17 +- .../am/service/engine/EngineStopService.java | 61 ++ .../service/engine/EngineSwitchService.java} | 14 +- .../service/heartbeat/AMHeartbeatService.java | 85 ++ .../service/monitor/NodeHeartbeatMonitor.java | 364 +++++++ .../apache/linkis/manager/am/util/Utils.java | 324 +++++++ .../linkis/manager/am/utils/AMUtils.java | 338 +++++++ .../linkis/manager/am/vo/AMEngineNodeVo.java | 6 +- .../apache/linkis/manager/am/vo/EMNodeVo.java | 6 +- .../manager/label/LabelManagerUtils.java} | 26 +- .../manager/label/conf/LabelManagerConf.java | 26 + .../label/score/DefaultNodeLabelScorer.java | 201 ++++ .../score/LabelScoreServiceInstance.java} | 38 +- .../manager/label/score/NodeLabelScorer.java} | 35 +- .../label/service/NodeLabelAddService.java} | 12 +- .../label/service/NodeLabelRemoveService.java | 25 + .../label/service/NodeLabelService.java | 83 ++ .../label/service/ResourceLabelService.java} | 32 +- .../label/service/UserLabelService.java} | 20 +- .../impl/DefaultNodeLabelAddService.java | 69 ++ .../impl/DefaultNodeLabelRemoveService.java | 82 ++ .../service/impl/DefaultNodeLabelService.java | 467 +++++++++ .../impl/DefaultResourceLabelService.java | 166 ++++ .../service/impl/DefaultUserLabelService.java | 180 ++++ .../manager/rm/conf/ResourceStatus.java | 4 +- .../manager/rm/domain/RMLabelContainer.java | 11 +- .../rm/entity/LabelResourceMapping.java | 68 ++ .../rm/entity/ResourceOperationType.java | 25 + .../impl/ExternalResourceServiceImpl.java | 11 +- .../manager/rm/external/yarn/YarnAppInfo.java | 78 ++ .../external/yarn/YarnResourceRequester.java | 424 +++++++++ .../manager/rm/message/RMMessageService.java | 75 ++ .../manager/rm/restful/RMMonitorRest.java | 892 ++++++++++++++++++ .../rm/service/LabelResourceService.java} | 33 +- .../rm/service/RequestResourceService.java | 422 +++++++++ .../rm/service/ResourceLockService.java | 95 ++ .../manager/rm/service/ResourceManager.java} | 41 +- .../manager/rm/service/impl/ChangeType.java} | 22 +- .../impl/DefaultReqResourceService.java} | 21 +- .../service/impl/DefaultResourceManager.java | 874 +++++++++++++++++ .../impl/DriverAndYarnReqResourceService.java | 158 ++++ .../impl/LabelResourceServiceImpl.java | 102 ++ .../rm/service/impl/ResourceLogService.java | 224 +++++ .../rm/service/impl/UserResourceService.java | 131 +++ .../linkis/manager/rm/utils/RMUtils.java | 283 ++++++ .../manager/rm/utils/UserConfiguration.java | 210 +++++ .../service/common/label/LabelChecker.java} | 14 +- .../service/common/label/LabelFilter.java} | 11 +- .../common/label/ManagerLabelService.java | 32 + .../common/metrics/MetricsConverter.java | 55 ++ .../service/common/pointer/EMNodPointer.java} | 20 +- .../common/pointer/EngineNodePointer.java} | 24 +- .../service/common/pointer/NodePointer.java} | 32 +- .../common/pointer/NodePointerBuilder.java | 28 + .../loader/EngineConnPluginLoaderConf.scala | 50 - .../conf/EngineConnPluginConfiguration.scala | 42 - ...stractEngineConnBmlResourceGenerator.scala | 122 --- ...efaultEngineConnBmlResourceGenerator.scala | 109 --- .../DefaultEngineConnLaunchService.scala | 90 -- ...aultEngineConnResourceFactoryService.scala | 64 -- .../DefaultEngineConnResourceService.scala | 294 ------ .../service/EngineConnResourceService.scala | 78 -- .../manager/am/conf/AMConfiguration.scala | 118 --- .../am/conf/ConfigurationMapCache.scala | 64 -- .../conf/EngineConnConfigurationService.scala | 70 -- .../manager/am/conf/ManagerMonitorConf.scala | 43 - .../converter/DefaultMetricsConverter.scala | 102 -- .../manager/am/label/AMLabelChecker.scala | 50 - .../manager/am/label/AMLabelFilter.scala | 56 -- .../am/label/DefaultManagerLabelService.scala | 72 -- .../MultiUserEngineReuseLabelChooser.scala | 83 -- .../am/locker/DefaultEngineNodeLocker.scala | 61 -- .../am/manager/DefaultEMNodeManager.scala | 193 ---- .../am/manager/DefaultEngineNodeManager.scala | 317 ------- .../manager/am/manager/EMNodeManager.scala | 71 -- .../am/manager/EngineNodeManager.scala | 82 -- .../am/pointer/AbstractNodePointer.scala | 83 -- .../am/pointer/DefaultEMNodPointer.scala | 83 -- .../am/pointer/DefaultEngineNodPointer.scala | 68 -- .../am/selector/DefaultNodeSelector.scala | 75 -- .../manager/am/selector/ECAvailableRule.scala | 55 -- .../rule/AvailableNodeSelectRule.scala | 60 -- .../am/selector/rule/NewECMStandbyRule.scala | 83 -- .../rule/OverLoadNodeSelectRule.scala | 66 -- .../rule/ResourceNodeSelectRule.scala | 101 -- .../selector/rule/ScoreNodeSelectRule.scala | 55 -- .../rule/TaskInfoNodeSelectRule.scala | 63 -- .../ConfCacheRemoveBroadcastListener.scala | 70 -- .../service/em/DefaultEMEngineService.scala | 209 ---- .../am/service/em/DefaultEMInfoService.scala | 124 --- .../service/em/DefaultEMRegisterService.scala | 121 --- .../em/DefaultEMUnregisterService.scala | 89 -- .../am/service/em/ECMOperateService.scala | 27 - .../am/service/em/EMRegisterService.scala | 33 - .../DefaultEngineAskEngineService.scala | 170 ---- .../DefaultEngineConnCanKillService.scala | 173 ---- .../engine/DefaultEngineCreateService.scala | 404 -------- .../engine/DefaultEngineInfoService.scala | 145 --- .../engine/DefaultEngineRecycleService.scala | 82 -- .../engine/DefaultEngineReuseService.scala | 219 ----- .../engine/DefaultEngineStopService.scala | 292 ------ .../engine/EngineConnCanKillService.scala | 26 - .../am/service/engine/EngineInfoService.scala | 50 - .../am/service/engine/EngineKillService.scala | 31 - .../service/engine/EngineOperateService.scala | 33 - .../am/service/engine/EngineStopService.scala | 69 -- .../heartbeat/AMHeartbeatService.scala | 83 -- .../monitor/NodeHeartbeatMonitor.scala | 493 ---------- .../linkis/manager/am/utils/AMUtils.scala | 305 ------ .../manager/label/conf/LabelManagerConf.scala | 27 - .../label/score/DefaultNodeLabelScorer.scala | 182 ---- .../service/NodeLabelRemoveService.scala | 26 - .../label/service/NodeLabelService.scala | 93 -- .../impl/DefaultNodeLabelAddService.scala | 56 -- .../impl/DefaultNodeLabelRemoveService.scala | 75 -- .../impl/DefaultNodeLabelService.scala | 411 -------- .../impl/DefaultResourceLabelService.scala | 162 ---- .../impl/DefaultUserLabelService.scala | 122 --- .../manager/rm/entity/LabelResourceMap.scala | 52 - .../rm/external/yarn/YarnAppInfo.scala | 29 - .../external/yarn/YarnResourceRequester.scala | 442 --------- .../manager/rm/message/RMMessageService.scala | 68 -- .../manager/rm/restful/RMMonitorRest.scala | 842 ----------------- .../rm/service/RequestResourceService.scala | 399 -------- .../rm/service/ResourceLockService.scala | 80 -- .../service/impl/DefaultResourceManager.scala | 881 ----------------- .../DriverAndYarnReqResourceService.scala | 121 --- .../impl/LabelResourceServiceImpl.scala | 100 -- .../rm/service/impl/ResourceLogService.scala | 234 ----- .../rm/service/impl/UserResourceService.scala | 114 --- .../linkis/manager/rm/utils/RMUtils.scala | 256 ----- .../manager/rm/utils/UserConfiguration.scala | 174 ---- .../common/metrics/MetricsConverter.scala | 57 -- .../common/pointer/EngineNodePointer.scala | 34 - .../common/conf/ManagerCommonConf.java | 35 + .../manager/common/conf/RMConfiguration.java | 114 +++ .../manager/common/entity/node/AMEMNode.java | 6 +- .../common/entity/node/AMEngineNode.java | 8 +- .../manager/common/entity/node/LabelNode.java | 4 +- .../entity/recycle/AssignEMNodeRule.java | 41 + .../common/entity/recycle/AssignNodeRule.java | 41 + .../entity/recycle/AssignUserRule.java} | 16 +- .../common/entity/recycle/RecyclingRule.java} | 10 +- .../common/entity/resource/CPUResource.java | 132 +++ .../resource/DriverAndYarnResource.java | 222 +++++ .../entity/resource/InstanceResource.java | 54 ++ .../entity/resource/LoadInstanceResource.java | 159 ++++ .../common/entity/resource/LoadResource.java | 129 +++ .../entity/resource/MemoryResource.java | 120 +++ .../common/entity/resource/Resource.java | 134 +++ .../entity/resource/SpecialResource.java | 355 +++++++ .../resource/UserAvailableResource.java | 36 + .../common/entity/resource/YarnResource.java | 176 ++++ .../common/monitor/ManagerMonitor.java | 20 + .../manager/common/operator/Operator.java | 57 ++ .../common/operator/OperatorFactory.java} | 16 +- .../common/operator/OperatorFactoryImpl.java | 72 ++ .../manager/common/protocol/EngineLock.java | 22 + .../common/protocol/OperateRequest.java | 39 + .../common/protocol/OperateResponse.java} | 10 +- .../common/protocol/RequestEngineLock.java} | 39 +- .../common/protocol/RequestEngineUnlock.java | 36 + .../common/protocol/RequestManagerUnlock.java | 59 ++ .../common/protocol/ResponseEngineLock.java | 56 ++ .../protocol/ResponseEngineUnlock.java} | 23 +- .../common/protocol/ServiceHealthReport.java | 27 + .../common/protocol/ServiceState.java} | 2 +- .../common/protocol/em/ECMOperateRequest.java | 51 + .../protocol/em/ECMOperateResponse.java} | 42 +- .../em/EMResourceRegisterRequest.java | 79 ++ .../common/protocol/em/RegisterEMRequest.java | 79 ++ .../protocol/em/RegisterEMResponse.java | 50 + .../engine/EngineAskAsyncResponse.java | 48 + .../protocol/engine/EngineAsyncResponse.java} | 11 +- .../engine/EngineConnStatusCallback.java | 61 ++ .../engine/EngineConnStatusCallbackToAM.java | 70 ++ .../protocol/engine/EngineCreateError.java | 61 ++ .../protocol/engine/EngineCreateSuccess.java} | 38 +- .../engine/EngineOperateRequest.java} | 29 +- .../engine/EngineOperateResponse.java} | 43 +- .../protocol/label/LabelReportRequest.java | 43 + .../protocol/label/LabelUpdateRequest.java | 36 + .../protocol/resource/ResourceProtocol.java | 22 + .../resource/ResourceUsedProtocol.java | 51 + .../protocol/resource/ResourceWithStatus.java | 103 ++ .../resource/ResponseTaskRunningInfo.java | 66 ++ .../resource/ResponseTaskYarnResource.java | 41 + .../manager/common/utils/ManagerUtils.java | 4 +- .../manager/common/utils/ResourceUtils.java | 312 ++++++ .../linkis/manager/rm/AvailableResource.java | 34 + .../linkis/manager/rm/NotEnoughResource.java | 36 + .../manager/rm/RequestExpectedResource.java | 41 + .../rm/RequestExpectedResourceAndWait.java | 48 + .../linkis/manager/rm/RequestResource.java | 41 + .../manager/rm/RequestResourceAndWait.java | 47 + .../manager/rm/RequestResourceInfo.java | 32 + .../linkis/manager/rm/ResourceInfo.java | 34 + .../linkis/manager/rm/ResourceInited.java} | 29 +- .../linkis/manager/rm/ResourceReleased.java | 34 + .../linkis/manager/rm/ResourceUpdated.java | 41 + .../linkis/manager/rm/ResultResource.java} | 4 +- .../common/conf/ManagerCommonConf.scala | 34 - .../manager/common/conf/RMConfiguration.scala | 91 -- .../common/entity/recycle/RecyclingRule.scala | 32 - .../common/entity/resource/Resource.scala | 773 --------------- .../manager/common/operator/Operator.scala | 44 - .../common/operator/OperatorFactory.scala | 64 -- .../common/protocol/OperateRequest.scala | 45 - .../protocol/em/ECMOperateRequest.scala | 37 - .../em/EMResourceRegisterRequest.scala | 67 -- .../protocol/em/RegisterEMRequest.scala | 67 -- .../protocol/em/RegisterEMResponse.scala | 24 - .../protocol/engine/EngineAsyncResponse.scala | 35 - .../engine/EngineConnStatusCallback.scala | 55 -- .../engine/EngineOperateRequest.scala | 31 - .../protocol/label/LabelUpdateRequest.scala | 27 - .../protocol/resource/ResourceProtocol.scala | 48 - .../resource/ResourceWithStatus.scala | 33 - .../manager/common/utils/ResourceUtils.scala | 278 ------ .../manager/rm/ResourceManagerParam.scala | 51 - .../linkis/manager/rm/ResultResource.scala | 24 - .../persistence/NodeManagerPersistence.java | 20 +- .../NodeMetricManagerPersistence.java | 10 +- .../ResourceManagerPersistence.java | 6 +- .../impl/DefaultNodeManagerPersistence.java | 32 +- .../DefaultNodeMetricManagerPersistence.java | 12 +- .../DefaultResourceManagerPersistence.java | 5 +- .../operator/TriggerSavepointOperator.java | 64 ++ .../operator/TriggerSavepointOperator.scala | 49 - .../executor/SparkEngineConnExecutor.scala | 8 +- ...SubmitProcessEngineConnLaunchBuilder.scala | 2 +- .../ComputationTaskExecutionReceiver.scala | 10 +- .../ecm/ComputationEngineConnManager.scala | 23 +- .../ecm/cache/EngineAsyncResponseCache.scala | 16 +- .../impl/ComputationEngineConnExecutor.scala | 2 +- .../DefaultEngineAsyncResponseService.scala | 12 +- pom.xml | 98 +- 351 files changed, 17969 insertions(+), 14494 deletions(-) create mode 100644 linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/ECPathUtils.java delete mode 100644 linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala create mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.java create mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.java delete mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala delete mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/java/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.java delete mode 100644 linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.java delete mode 100644 linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/EngineConnPluginLoaderConf.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/engineplugin/server/interceptor/EngineConnLaunchInterceptor.scala => java/org/apache/linkis/engineplugin/server/interceptor/EngineConnLaunchInterceptor.java} (76%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/loader/EngineConnPluginsLoaderFactory.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/DefaultEngineConnBmlResourceGenerator.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnBmlResourceGenerator.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnLocalizeResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnLocalizeResourceImpl.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnLaunchService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceFactoryService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/engineplugin/server/service/EngineConnLaunchService.scala => java/org/apache/linkis/engineplugin/server/service/EngineConnLaunchService.java} (73%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/engineplugin/server/service/EngineConnResourceFactoryService.scala => java/org/apache/linkis/engineplugin/server/service/EngineConnResourceFactoryService.java} (67%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceRequest.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/GetEngineConnResourceRequest.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshAllEngineConnResourceRequest.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshEngineConnResourceRequest.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ApplicationManagerSpringConfiguration.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ConfigurationMapCache.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/DefaultEngineConnConfigurationService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/EngineConnConfigurationService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ManagerMonitorConf.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelChecker.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelFilter.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/DefaultManagerLabelService.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/label/EngineReuseLabelRester.scala => java/org/apache/linkis/manager/am/label/EngineReuseLabelChooser.java} (78%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/service/common/label/LabelFilter.scala => java/org/apache/linkis/manager/am/locker/EngineNodeLocker.java} (73%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EMNodeManager.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EngineNodeManager.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/AbstractNodePointer.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultNodePointerBuilder.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/recycle/AssignNodeRuleExecutor.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.scala => java/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.java} (77%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/DefaultECAvailableRule.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/DefaultNodeSelector.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/ECAvailableRule.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/selector/NodeSelector.scala => java/org/apache/linkis/manager/am/selector/NodeSelector.java} (74%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/AvailableNodeSelectRule.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/selector/rule/ConcurrencyNodeSelectRule.scala => java/org/apache/linkis/manager/am/selector/rule/ConcurrencyNodeSelectRule.java} (61%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/selector/rule/HotspotExclusionRule.scala => java/org/apache/linkis/manager/am/selector/rule/HotspotExclusionRule.java} (53%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/NewECMStandbyRule.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/selector/rule/NodeSelectRule.scala => java/org/apache/linkis/manager/am/selector/rule/NodeSelectRule.java} (81%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/ResourceNodeSelectRule.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/ScoreNodeSelectRule.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/selector/rule/TaskInfoNodeSelectRule.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/service/EMEngineService.scala => java/org/apache/linkis/manager/am/service/EMEngineService.java} (52%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/service/EngineService.scala => java/org/apache/linkis/manager/am/service/EngineService.java} (78%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/service/HeartbeatService.scala => java/org/apache/linkis/manager/am/service/HeartbeatService.java} (85%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/cache/ConfCacheRemoveBroadcastListener.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/service/em/DefaultECMOperateService.scala => java/org/apache/linkis/manager/am/service/em/DefaultECMOperateService.java} (53%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/DefaultEMRegisterService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/DefaultEMUnregisterService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/ECMOperateService.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/service/em/EMInfoService.scala => java/org/apache/linkis/manager/am/service/em/EMInfoService.java} (66%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/EMRegisterService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/em/EMUnregisterService.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/service/engine/DefaultEngineOperateService.scala => java/org/apache/linkis/manager/am/service/engine/AbstractEngineService.java} (55%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineConnCanKillService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineInfoService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineOperateService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineRecycleService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineReuseService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/service/engine/DefaultEngineSwitchService.scala => java/org/apache/linkis/manager/am/service/engine/DefaultEngineSwitchService.java} (68%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/service/engine/EngineAskEngineService.scala => java/org/apache/linkis/manager/am/service/engine/EngineAskEngineService.java} (80%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/EngineConnCanKillService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/EngineConnStatusCallbackService.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/service/engine/EngineCreateService.scala => java/org/apache/linkis/manager/am/service/engine/EngineCreateService.java} (67%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/EngineInfoService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/EngineOperateService.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/service/engine/EngineRecycleService.scala => java/org/apache/linkis/manager/am/service/engine/EngineRecycleService.java} (72%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/service/engine/EngineReuseService.scala => java/org/apache/linkis/manager/am/service/engine/EngineReuseService.java} (69%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/engine/EngineStopService.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/service/engine/EngineSwitchService.scala => java/org/apache/linkis/manager/am/service/engine/EngineSwitchService.java} (72%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/monitor/NodeHeartbeatMonitor.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/Utils.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/utils/AMUtils.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/label/LabelManagerUtils.scala => java/org/apache/linkis/manager/label/LabelManagerUtils.java} (61%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/conf/LabelManagerConf.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/label/score/LabelScoreServiceInstance.scala => java/org/apache/linkis/manager/label/score/LabelScoreServiceInstance.java} (55%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/label/score/NodeLabelScorer.scala => java/org/apache/linkis/manager/label/score/NodeLabelScorer.java} (60%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/label/service/NodeLabelAddService.scala => java/org/apache/linkis/manager/label/service/NodeLabelAddService.java} (75%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/NodeLabelRemoveService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/NodeLabelService.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/label/service/ResourceLabelService.scala => java/org/apache/linkis/manager/label/service/ResourceLabelService.java} (62%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/service/common/label/ManagerLabelService.scala => java/org/apache/linkis/manager/label/service/UserLabelService.java} (62%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelAddService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelRemoveService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/impl/DefaultResourceLabelService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/label/service/impl/DefaultUserLabelService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/entity/LabelResourceMapping.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/entity/ResourceOperationType.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnAppInfo.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/message/RMMessageService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/restful/RMMonitorRest.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/rm/service/LabelResourceService.scala => java/org/apache/linkis/manager/rm/service/LabelResourceService.java} (54%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/RequestResourceService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/ResourceLockService.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/rm/service/ResourceManager.scala => java/org/apache/linkis/manager/rm/service/ResourceManager.java} (68%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/label/service/UserLabelService.scala => java/org/apache/linkis/manager/rm/service/impl/ChangeType.java} (56%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/rm/service/impl/DefaultReqResourceService.scala => java/org/apache/linkis/manager/rm/service/impl/DefaultReqResourceService.java} (60%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/impl/LabelResourceServiceImpl.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/impl/ResourceLogService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/service/impl/UserResourceService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/RMUtils.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/UserConfiguration.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/service/em/EMUnregisterService.scala => java/org/apache/linkis/manager/service/common/label/LabelChecker.java} (70%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/service/common/pointer/NodePointerBuilder.scala => java/org/apache/linkis/manager/service/common/label/LabelFilter.java} (74%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/label/ManagerLabelService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/metrics/MetricsConverter.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/service/common/pointer/EMNodPointer.scala => java/org/apache/linkis/manager/service/common/pointer/EMNodPointer.java} (62%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/recycle/AssignNodeRuleExecutor.scala => java/org/apache/linkis/manager/service/common/pointer/EngineNodePointer.java} (55%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/service/common/pointer/NodePointer.scala => java/org/apache/linkis/manager/service/common/pointer/NodePointer.java} (62%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/service/common/pointer/NodePointerBuilder.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/loader/EngineConnPluginLoaderConf.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/DefaultEngineConnBmlResourceGenerator.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnLaunchService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceFactoryService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/EngineConnResourceService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/AMConfiguration.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/ConfigurationMapCache.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/EngineConnConfigurationService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/ManagerMonitorConf.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/AMLabelChecker.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/AMLabelFilter.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/DefaultManagerLabelService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/EMNodeManager.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/EngineNodeManager.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/AbstractNodePointer.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/DefaultNodeSelector.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/ECAvailableRule.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/AvailableNodeSelectRule.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/NewECMStandbyRule.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/OverLoadNodeSelectRule.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/ResourceNodeSelectRule.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/ScoreNodeSelectRule.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/TaskInfoNodeSelectRule.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/cache/ConfCacheRemoveBroadcastListener.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMRegisterService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMUnregisterService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/ECMOperateService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/EMRegisterService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineConnCanKillService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineInfoService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineRecycleService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineReuseService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineConnCanKillService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineInfoService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineKillService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineOperateService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineStopService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/monitor/NodeHeartbeatMonitor.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/utils/AMUtils.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/conf/LabelManagerConf.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/score/DefaultNodeLabelScorer.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/NodeLabelRemoveService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/NodeLabelService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelAddService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelRemoveService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultResourceLabelService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultUserLabelService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/entity/LabelResourceMap.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnAppInfo.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/message/RMMessageService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/RequestResourceService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/ResourceLockService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/LabelResourceServiceImpl.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/ResourceLogService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/UserResourceService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/UserConfiguration.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/metrics/MetricsConverter.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/pointer/EngineNodePointer.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/conf/ManagerCommonConf.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/conf/RMConfiguration.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/recycle/AssignEMNodeRule.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/recycle/AssignNodeRule.java rename linkis-computation-governance/linkis-manager/{linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/locker/EngineNodeLocker.scala => linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/recycle/AssignUserRule.java} (75%) rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/{scala/org/apache/linkis/manager/common/protocol/ServiceHealthReport.scala => java/org/apache/linkis/manager/common/entity/recycle/RecyclingRule.java} (81%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/CPUResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/DriverAndYarnResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/InstanceResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/LoadInstanceResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/LoadResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/MemoryResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/Resource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/SpecialResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/UserAvailableResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/YarnResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/monitor/ManagerMonitor.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/operator/Operator.java rename linkis-computation-governance/linkis-manager/{linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/label/LabelChecker.scala => linkis-manager-common/src/main/java/org/apache/linkis/manager/common/operator/OperatorFactory.java} (69%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/operator/OperatorFactoryImpl.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/EngineLock.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/OperateRequest.java rename linkis-computation-governance/linkis-manager/{linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/entity/ResourceOperationType.scala => linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/OperateResponse.java} (81%) rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/{scala/org/apache/linkis/manager/common/protocol/EngineLock.scala => java/org/apache/linkis/manager/common/protocol/RequestEngineLock.java} (56%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/RequestEngineUnlock.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/RequestManagerUnlock.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/ResponseEngineLock.java rename linkis-computation-governance/linkis-manager/{linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/loader/EngineConnPluginsLoader.scala => linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/ResponseEngineUnlock.java} (67%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/ServiceHealthReport.java rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/{scala/org/apache/linkis/manager/common/protocol/ServiceState.scala => java/org/apache/linkis/manager/common/protocol/ServiceState.java} (94%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/em/ECMOperateRequest.java rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/{scala/org/apache/linkis/manager/common/protocol/em/ECMOperateResponse.scala => java/org/apache/linkis/manager/common/protocol/em/ECMOperateResponse.java} (52%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/em/EMResourceRegisterRequest.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/em/RegisterEMRequest.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/em/RegisterEMResponse.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineAskAsyncResponse.java rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/{scala/org/apache/linkis/manager/common/protocol/OperateResponse.scala => java/org/apache/linkis/manager/common/protocol/engine/EngineAsyncResponse.java} (80%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineConnStatusCallback.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineConnStatusCallbackToAM.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineCreateError.java rename linkis-computation-governance/linkis-manager/{linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/EngineConnBmlResourceGenerator.scala => linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineCreateSuccess.java} (56%) rename linkis-computation-governance/linkis-manager/{linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/AbstractEngineService.scala => linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineOperateRequest.java} (57%) rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/{scala/org/apache/linkis/manager/common/protocol/engine/EngineOperateResponse.scala => java/org/apache/linkis/manager/common/protocol/engine/EngineOperateResponse.java} (52%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/label/LabelReportRequest.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/label/LabelUpdateRequest.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/resource/ResourceProtocol.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/resource/ResourceUsedProtocol.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/resource/ResourceWithStatus.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/resource/ResponseTaskRunningInfo.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/resource/ResponseTaskYarnResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/utils/ResourceUtils.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/AvailableResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/NotEnoughResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/RequestExpectedResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/RequestExpectedResourceAndWait.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/RequestResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/RequestResourceAndWait.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/RequestResourceInfo.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/ResourceInfo.java rename linkis-computation-governance/linkis-manager/{linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultNodePointerBuilder.scala => linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/ResourceInited.java} (58%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/ResourceReleased.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/rm/ResourceUpdated.java rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/{scala/org/apache/linkis/manager/common/monitor/ManagerMonitor.scala => java/org/apache/linkis/manager/rm/ResultResource.java} (89%) delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/conf/ManagerCommonConf.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/conf/RMConfiguration.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/entity/recycle/RecyclingRule.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/entity/resource/Resource.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/operator/Operator.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/operator/OperatorFactory.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/OperateRequest.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/ECMOperateRequest.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/EMResourceRegisterRequest.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/RegisterEMRequest.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/em/RegisterEMResponse.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineAsyncResponse.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineConnStatusCallback.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/engine/EngineOperateRequest.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/label/LabelUpdateRequest.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/resource/ResourceProtocol.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/resource/ResourceWithStatus.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/utils/ResourceUtils.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/rm/ResourceManagerParam.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/rm/ResultResource.scala create mode 100644 linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/operator/TriggerSavepointOperator.java delete mode 100644 linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/operator/TriggerSavepointOperator.scala diff --git a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/message/method/MessageExecutor.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/message/method/MessageExecutor.java index 0644f539d68..fd7e4cf6ab3 100644 --- a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/message/method/MessageExecutor.java +++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/message/method/MessageExecutor.java @@ -31,6 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import static org.apache.linkis.rpc.errorcode.LinkisRpcErrorCodeSummary.METHOD_CALL_FAILED; diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/ECPathUtils.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/ECPathUtils.java new file mode 100644 index 00000000000..46da41c5977 --- /dev/null +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/ECPathUtils.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.governance.common.utils; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; + +import java.io.File; +import java.nio.file.Paths; + +public class ECPathUtils { + + public static String getECWOrkDirPathSuffix(String user, String ticketId, String engineType) { + String engineTypeRes = ""; + if (StringUtils.isNotBlank(engineType)) { + engineTypeRes = engineType; + } + File file = Paths.get(user, DateFormatUtils.format(System.currentTimeMillis(), "yyyyMMdd"), engineTypeRes).toFile(); + return file.getPath() + File.separator + ticketId; + } + + + public static String getECLogDirSuffix( + EngineTypeLabel engineTypeLabel, + UserCreatorLabel userCreatorLabel, + String ticketId + ){ + if (null == engineTypeLabel || null == userCreatorLabel) { + return ""; + } + String ecwOrkDirPathSuffix = ECPathUtils.getECWOrkDirPathSuffix( + userCreatorLabel.getUser(), + ticketId, + engineTypeLabel.getEngineType() + ); + return ecwOrkDirPathSuffix + File.separator + "logs"; + } +} diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala deleted file mode 100644 index 747ef4e920c..00000000000 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.governance.common.utils - -import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} - -import org.apache.commons.lang3.StringUtils -import org.apache.commons.lang3.time.DateFormatUtils - -import java.io.File -import java.nio.file.Paths - -object ECPathUtils { - - def getECWOrkDirPathSuffix(user: String, ticketId: String, engineType: String): String = { - Paths - .get( - user, - DateFormatUtils.format(System.currentTimeMillis(), "yyyyMMdd"), - if (StringUtils.isBlank(engineType)) "" else engineType - ) - .toFile - .getPath + File.separator + ticketId - - } - - def getECLogDirSuffix( - engineTypeLabel: EngineTypeLabel, - userCreatorLabel: UserCreatorLabel, - ticketId: String - ): String = { - if (null == engineTypeLabel || null == userCreatorLabel) { - return "" - } - val suffix = ECPathUtils.getECWOrkDirPathSuffix( - userCreatorLabel.getUser, - ticketId, - engineTypeLabel.getEngineType - ) - suffix + File.separator + "logs" - } - -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.java new file mode 100644 index 00000000000..14446192a8b --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.java @@ -0,0 +1,203 @@ +package org.apache.linkis.ecm.server.operator; + +import org.apache.commons.lang3.tuple.Triple; +import org.apache.linkis.DataWorkCloudApplication; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.common.utils.Logging; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary; +import org.apache.linkis.ecm.server.conf.ECMConfiguration; +import org.apache.linkis.ecm.server.exception.ECMErrorException; +import org.apache.linkis.ecm.server.service.LocalDirsHandleService; +import org.apache.linkis.manager.common.operator.Operator; +import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.text.MessageFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.input.ReversedLinesFileReader; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary.*; + +public class EngineConnLogOperator implements Operator { + private static final Logger logger = LoggerFactory.getLogger(EngineConnLogOperator.class); + + public static final String OPERATOR_NAME = "engineConnLog"; + public static final CommonVars LOG_FILE_NAME = CommonVars.apply("linkis.engineconn.log.filename", "stdout"); + public static final CommonVars MAX_LOG_FETCH_SIZE = CommonVars.apply("linkis.engineconn.log.fetch.lines.max", 5000); + public static final CommonVars MAX_LOG_TAIL_START_SIZE = CommonVars.apply("linkis.engineconn.log.tail.start.size"); + public static final CommonVars MULTILINE_PATTERN = CommonVars.apply("linkis.engineconn.log.multiline.pattern", "^\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\.\\d{3}"); + public static final CommonVars MULTILINE_MAX = CommonVars.apply("linkis.engineconn.log.multiline.max", 500); + + private LocalDirsHandleService localDirsHandleService; + + @Override + public String[] getNames() { + return new String[]{OPERATOR_NAME}; + } + + @Override + public Map apply(Map parameters) { + File logPath = getLogPath(parameters); + int lastRows = getAs(parameters, "lastRows", 0); + int pageSize = getAs(parameters, "pageSize", 100); + int fromLine = getAs(parameters, "fromLine", 1); + boolean enableTail = getAs(parameters, "enableTail", false); + if (lastRows > EngineConnLogOperator.MAX_LOG_FETCH_SIZE.getValue()) { + throw new WarnException( + CANNOT_FETCH_MORE_THAN.getErrorCode(), + MessageFormat.format( + CANNOT_FETCH_MORE_THAN.getErrorDesc(), + EngineConnLogOperator.MAX_LOG_FETCH_SIZE.getValue().toString() + ) + ); + } else if (lastRows > 0) { + String logs = Utils.exec(new String[]{"tail", "-n", lastRows + "", logPath.getPath()}, 5000); + Map stringObjectHashMap = new HashMap<>(); + stringObjectHashMap.put("logs", logs.split("\n")); + stringObjectHashMap.put("rows", logs.length()); + return stringObjectHashMap; + } + + + String ignoreKeywords = getAs(parameters, "ignoreKeywords", ""); + String[] ignoreKeywordList = StringUtils.isNotEmpty(ignoreKeywords) ? ignoreKeywords.split(",") : new String[0]; + + String onlyKeywords = getAs(parameters, "onlyKeywords", ""); + String[] onlyKeywordList = StringUtils.isNotEmpty(onlyKeywords) ? onlyKeywords.split(",") : new String[0]; + + RandomAccessFile randomReader = null; + ReversedLinesFileReader reversedReader = null; + try { + if (enableTail) { + logger.info("enable log operator from tail to read"); + reversedReader = new ReversedLinesFileReader(logPath, Charset.defaultCharset()); + } else { + randomReader = new RandomAccessFile(logPath, "r"); + } + + ArrayList logs = new ArrayList<>(pageSize); + int readLine = 0, skippedLine = 0, lineNum = 0; + boolean rowIgnore = false; + int ignoreLine = 0; + Pattern linePattern = null != EngineConnLogOperator.MULTILINE_PATTERN.getValue() ? Pattern.compile(EngineConnLogOperator.MULTILINE_PATTERN.getValue()) : null; + + int maxMultiline = MULTILINE_MAX.getValue(); + String line = randomAndReversedReadLine(randomReader, reversedReader); + + while (readLine < pageSize && line != null) { + lineNum += 1; + if (skippedLine < fromLine - 1) { + skippedLine += 1; + } else { + if (rowIgnore) { + Matcher matcher = linePattern.matcher(line); + if (matcher.matches()) { + ignoreLine = 0; + rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList); + } else { + ignoreLine += 1; + if (ignoreLine >= maxMultiline) { + rowIgnore = false; + } + } + if (!matcher.matches()) { + rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList); + } + } else { + rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList); + } + if (!rowIgnore) { + logs.add(line); + readLine += 1; + } + } + line = randomAndReversedReadLine(randomReader, reversedReader); + } + + IOUtils.closeQuietly(randomReader); + IOUtils.closeQuietly(reversedReader); + if (enableTail) { + Collections.reverse(logs); + } + + Map resultMap = new HashMap<>(); + resultMap.put("logPath", logPath.getPath()); + resultMap.put("logs", logs); + resultMap.put("endLine", lineNum); + resultMap.put("rows", readLine); + return resultMap; + } catch (IOException e) { + //ing + throw new RuntimeException(e); + } finally { + IOUtils.closeQuietly(randomReader); + IOUtils.closeQuietly(reversedReader); + } + } + + private String randomAndReversedReadLine(RandomAccessFile randomReader, ReversedLinesFileReader reversedReader) throws IOException { + if (randomReader != null) { + String line = randomReader.readLine(); + if (line != null) { + return new String(line.getBytes(StandardCharsets.ISO_8859_1), Charset.defaultCharset()); + } else { + return null; + } + } else { + return reversedReader.readLine(); + } + } + + protected File getLogPath(Map parameters) { + String logType = getAs(parameters, "logType", EngineConnLogOperator.LOG_FILE_NAME.getValue()); + String logDIrSuffix = getAs(parameters, "logDirSuffix", ""); + + String engineConnLogDir = ECMConfiguration.ENGINECONN_ROOT_DIR() + File.separator + logDIrSuffix; + String ticketId = getAs(parameters, "ticketId", ""); + String engineConnInstance = ""; + + File logPath = new File(engineConnLogDir, logType); + if (!logPath.exists() || !logPath.isFile()) { + throw new WarnException( + LOGFILE_IS_NOT_EXISTS.getErrorCode(), + MessageFormat.format(LOGFILE_IS_NOT_EXISTS.getErrorDesc(), logPath.toString()) + ); + } + logger.info( + String.format("Try to fetch EngineConn(id: %s, instance: %s) logs from %s.", ticketId, engineConnInstance, logPath.getPath()) + ); + return logPath; + } + + protected Triple getEngineConnInfo(Map parameters){ + String logDIrSuffix = getAs(parameters, "logDirSuffix", ""); + + localDirsHandleService = DataWorkCloudApplication.getApplicationContext().getBean(LocalDirsHandleService.class); + String engineConnLogDir = ECMConfiguration.ENGINECONN_ROOT_DIR() + File.separator + logDIrSuffix; + String ticketId = getAs(parameters, "ticketId", ""); + String engineConnInstance = ""; + + return Triple.of(engineConnLogDir, engineConnInstance, ticketId); + } + + private boolean includeLine(String line, String[] onlyKeywordList, String[] ignoreKeywordList) { + boolean accept = ignoreKeywordList.length == 0 || !Arrays.stream(ignoreKeywordList).anyMatch(line::contains); + if (accept) { + accept = onlyKeywordList.length == 0 || Arrays.stream(onlyKeywordList).anyMatch(line::contains); + } + return accept; + } +} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.java new file mode 100644 index 00000000000..134033ff45f --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.java @@ -0,0 +1,127 @@ +package org.apache.linkis.ecm.server.operator; + +import org.apache.commons.lang3.tuple.Triple; +import org.apache.linkis.common.exception.LinkisCommonErrorException; +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.ecm.core.conf.ECMErrorCode; +import org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary; +import org.apache.linkis.ecm.server.exception.ECMErrorException; +import org.apache.linkis.ecm.server.exception.ECMWarnException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.text.MessageFormat; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary.LOG_IS_NOT_EXISTS; + +public class EngineConnYarnLogOperator extends EngineConnLogOperator { + private static final Logger logger = LoggerFactory.getLogger(EngineConnYarnLogOperator.class); + + @Override + public String[] getNames() { + return new String[]{EngineConnYarnLogOperator.OPERATOR_NAME}; + } + + @Override + public Map apply(Map parameters) { + Map result = new HashMap<>(); + try { + result = super.apply(parameters); + } finally { + Object logPath = result.get("logPath"); + if (logPath instanceof String) { + File logFile = new File((String) logPath); + if (logFile.exists() && logFile.getName().startsWith(".")) { + // If is a temporary file, drop it + logger.info(String.format("Delete the temporary yarn log file: [%s]", logPath)); + if (!logFile.delete()) { + logger.warn(String.format("Fail to delete the temporary yarn log file: [%s]", logPath)); + } + } + } + } + return result; + } + + + @Override + public File getLogPath(Map parameters) { + String ticketId, engineConnInstance, engineConnLogDir; + Triple engineConnInfo = getEngineConnInfo(parameters); + ticketId = engineConnInfo.getRight(); + engineConnInstance = engineConnInfo.getMiddle(); + engineConnLogDir = engineConnInfo.getLeft(); + + File rootLogDir = new File(engineConnLogDir); + if (!rootLogDir.exists() || !rootLogDir.isDirectory()) { + throw new ECMWarnException( + LOG_IS_NOT_EXISTS.getErrorCode(), + MessageFormat.format(LOG_IS_NOT_EXISTS.getErrorDesc(), rootLogDir) + ); + } + + String creator = getAsThrow(parameters, "creator"); + String applicationId = getAsThrow(parameters, "yarnApplicationId"); + File logPath = new File(engineConnLogDir, "yarn_" + applicationId); + if (!logPath.exists()) { + String tempLogFile = + String.format(".yarn_%s_%d_%d", applicationId, System.currentTimeMillis(), Thread.currentThread().getId()); + try { + String command = String.format("yarn logs -applicationId %s >> %s/%s", applicationId, rootLogDir, tempLogFile); + logger.info(String.format("Fetch yarn logs to temporary file: [%s]", command)); + + ProcessBuilder processBuilder = new ProcessBuilder(sudoCommands(creator, command)); + processBuilder.environment().putAll(System.getenv()); + processBuilder.redirectErrorStream(false); + Process process = processBuilder.start(); + boolean waitFor = process.waitFor(5, TimeUnit.SECONDS); + logger.trace(String.format("waitFor: %b, result: %d", waitFor, process.exitValue())); + if (waitFor && process.waitFor() == 0) { + command = String.format("mv %s/%s %s/yarn_%s", rootLogDir, tempLogFile, rootLogDir, applicationId); + logger.info(String.format("Move and save yarn logs: [%s]", command)); + Utils.exec(sudoCommands(creator, command)); + } else { + logPath = new File(engineConnLogDir, tempLogFile); + if (!logPath.exists()) { + throw new WarnException( + -1, + String.format("Fetch yarn logs timeout, log aggregation has not completed or is not enabled") + ); + } + } + } catch (Exception e) { + throw new WarnException( + -1, + String.format("Fail to fetch yarn logs application: %s, message: %s", applicationId, e.getMessage()) + ); + } + } + if (!logPath.exists() || !logPath.isFile()) { + throw new WarnException( + EngineconnServerErrorCodeSummary.LOGFILE_IS_NOT_EXISTS.getErrorCode(), + MessageFormat.format(EngineconnServerErrorCodeSummary.LOGFILE_IS_NOT_EXISTS.getErrorDesc(), logPath) + ); + } + logger.info( + String.format("Try to fetch EngineConn(id: %s, instance: %s) yarn logs from %s in application id: %s", ticketId, engineConnInstance, logPath.getPath(), applicationId)); + + + return logPath; + } + + + private String[] sudoCommands(String creator, String command) { + return new String[] { + "/bin/bash", + "-c", + "sudo su " + creator + " -c \"source ~/.bashrc 2>/dev/null; " + command + "\"" + }; + } + +} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala deleted file mode 100644 index 40de16849fd..00000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.operator - -import org.apache.linkis.common.conf.CommonVars -import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary._ -import org.apache.linkis.ecm.server.conf.ECMConfiguration -import org.apache.linkis.ecm.server.exception.ECMErrorException -import org.apache.linkis.ecm.server.service.LocalDirsHandleService -import org.apache.linkis.manager.common.operator.Operator - -import org.apache.commons.io.IOUtils -import org.apache.commons.io.input.ReversedLinesFileReader -import org.apache.commons.lang3.StringUtils - -import java.io.{File, RandomAccessFile} -import java.nio.charset.{Charset, StandardCharsets} -import java.text.MessageFormat -import java.util -import java.util.Collections - -import scala.util.matching.Regex - -class EngineConnLogOperator extends Operator with Logging { - - private var localDirsHandleService: LocalDirsHandleService = _ - - override def getNames: Array[String] = Array(EngineConnLogOperator.OPERATOR_NAME) - - override def apply(implicit parameters: Map[String, Any]): Map[String, Any] = { - val logPath = getLogPath - val lastRows = getAs("lastRows", 0) - val pageSize = getAs("pageSize", 100) - val fromLine = getAs("fromLine", 1) - val enableTail = getAs("enableTail", false) - if (lastRows > EngineConnLogOperator.MAX_LOG_FETCH_SIZE.getValue) { - throw new ECMErrorException( - CANNOT_FETCH_MORE_THAN.getErrorCode, - MessageFormat.format( - CANNOT_FETCH_MORE_THAN.getErrorDesc, - EngineConnLogOperator.MAX_LOG_FETCH_SIZE.getValue.toString - ) - ) - } else if (lastRows > 0) { - val logs = Utils.exec(Array("tail", "-n", lastRows + "", logPath.getPath), 5000).split("\n") - return Map("logs" -> logs, "rows" -> logs.length) - } - - val ignoreKeywords = getAs("ignoreKeywords", "") - val ignoreKeywordList = - if (StringUtils.isNotEmpty(ignoreKeywords)) ignoreKeywords.split(",") - else Array.empty[String] - val onlyKeywords = getAs("onlyKeywords", "") - val onlyKeywordList = - if (StringUtils.isNotEmpty(onlyKeywords)) onlyKeywords.split(",") else Array.empty[String] - var randomReader: RandomAccessFile = null - var reversedReader: ReversedLinesFileReader = null - if (enableTail) { - logger.info("enable log operator from tail to read") - reversedReader = new ReversedLinesFileReader(logPath, Charset.defaultCharset()) - } else { - randomReader = new RandomAccessFile(logPath, "r") - } - def randomAndReversedReadLine(): String = { - if (null != randomReader) { - val line = randomReader.readLine() - if (line != null) { - new String(line.getBytes(StandardCharsets.ISO_8859_1), Charset.defaultCharset()) - } else null - } else { - reversedReader.readLine() - } - } - val logs = new util.ArrayList[String](pageSize) - var readLine, skippedLine, lineNum = 0 - var rowIgnore = false - var ignoreLine = 0 - val linePattern = Option(EngineConnLogOperator.MULTILINE_PATTERN.getValue) match { - case Some(pattern) => pattern.r - case _ => null - } - val maxMultiline = EngineConnLogOperator.MULTILINE_MAX.getValue - Utils.tryFinally { - var line = randomAndReversedReadLine() - while (readLine < pageSize && line != null) { - lineNum += 1 - if (skippedLine < fromLine - 1) { - skippedLine += 1 - } else { - if (rowIgnore) { - linePattern match { - case reg: Regex => - if (reg.findFirstIn(line).isDefined) { - ignoreLine = 0 - rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList) - } else { - ignoreLine += 1 - if (ignoreLine >= maxMultiline) { - rowIgnore = false - } - } - case _ => rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList) - } - } else { - rowIgnore = !includeLine(line, onlyKeywordList, ignoreKeywordList) - } - if (!rowIgnore) { - logs.add(line) - readLine += 1 - } - } - line = randomAndReversedReadLine() - } - } { - IOUtils.closeQuietly(randomReader) - IOUtils.closeQuietly(reversedReader) - } - if (enableTail) Collections.reverse(logs) - Map("logPath" -> logPath.getPath, "logs" -> logs, "endLine" -> lineNum, "rows" -> readLine) - } - - protected def getLogPath(implicit parameters: Map[String, Any]): File = { - val (ticketId, engineConnInstance, engineConnLogDir) = getEngineConnInfo(parameters) - val logPath = - new File(engineConnLogDir, getAs("logType", EngineConnLogOperator.LOG_FILE_NAME.getValue)); - if (!logPath.exists() || !logPath.isFile) { - throw new ECMErrorException( - LOGFILE_IS_NOT_EXISTS.getErrorCode, - MessageFormat.format(LOGFILE_IS_NOT_EXISTS.getErrorDesc, logPath) - ) - } - logger.info( - s"Try to fetch EngineConn(id: $ticketId, instance: $engineConnInstance) logs from ${logPath.getPath}." - ) - logPath - } - - protected def getEngineConnInfo(implicit - parameters: Map[String, Any] - ): (String, String, String) = { - val logDIrSuffix = getAs("logDirSuffix", "") - val ecLogPath = ECMConfiguration.ENGINECONN_ROOT_DIR + File.separator + logDIrSuffix - val ticketId = getAs("ticketId", "") - (ticketId, "", ecLogPath) - } - - private def includeLine( - line: String, - onlyKeywordList: Array[String], - ignoreKeywordList: Array[String] - ): Boolean = { - var accept: Boolean = ignoreKeywordList.isEmpty || !ignoreKeywordList.exists(line.contains) - if (accept) { - accept = onlyKeywordList.isEmpty || onlyKeywordList.exists(line.contains) - } - accept - } - -} - -object EngineConnLogOperator { - val OPERATOR_NAME = "engineConnLog" - val LOG_FILE_NAME = CommonVars("linkis.engineconn.log.filename", "stdout") - val MAX_LOG_FETCH_SIZE = CommonVars("linkis.engineconn.log.fetch.lines.max", 5000) - - val MAX_LOG_TAIL_START_SIZE = CommonVars("linkis.engineconn.log.tail.start.size", 20000) - - // yyyy-MM-dd HH:mm:ss.SSS - val MULTILINE_PATTERN = CommonVars( - "linkis.engineconn.log.multiline.pattern", - "^\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\.\\d{3}" - ) - - val MULTILINE_MAX = CommonVars("linkis.engineconn.log.multiline.max", 500) -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala deleted file mode 100644 index e00d16b5199..00000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.operator - -import org.apache.linkis.common.exception.LinkisCommonErrorException -import org.apache.linkis.common.utils.Utils -import org.apache.linkis.ecm.core.conf.ECMErrorCode -import org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary._ -import org.apache.linkis.ecm.server.exception.ECMErrorException - -import java.io.File -import java.text.MessageFormat -import java.util.concurrent.TimeUnit - -import scala.collection.JavaConverters._ - -class EngineConnYarnLogOperator extends EngineConnLogOperator { - - override def getNames: Array[String] = Array(EngineConnYarnLogOperator.OPERATOR_NAME) - - override def apply(implicit parameters: Map[String, Any]): Map[String, Any] = { - var result: Map[String, Any] = Map() - Utils.tryFinally { - result = super.apply(parameters) - result - } { - result.get("logPath") match { - case Some(path: String) => - val logFile = new File(path) - if (logFile.exists() && logFile.getName.startsWith(".")) { - // If is a temporary file, drop it - logger.info(s"Delete the temporary yarn log file: [$path]") - if (!logFile.delete()) { - logger.warn(s"Fail to delete the temporary yarn log file: [$path]") - } - } - } - } - } - - override def getLogPath(implicit parameters: Map[String, Any]): File = { - val (ticketId, engineConnInstance, engineConnLogDir) = getEngineConnInfo(parameters) - val rootLogDir = new File(engineConnLogDir) - if (!rootLogDir.exists() || !rootLogDir.isDirectory) { - throw new ECMErrorException( - LOG_IS_NOT_EXISTS.getErrorCode, - MessageFormat.format(LOG_IS_NOT_EXISTS.getErrorDesc, rootLogDir) - ) - } - val creator = getAsThrow[String]("creator") - val applicationId = getAsThrow[String]("yarnApplicationId") - var logPath = new File(engineConnLogDir, "yarn_" + applicationId) - if (!logPath.exists()) { - val tempLogFile = - s".yarn_${applicationId}_${System.currentTimeMillis()}_${Thread.currentThread().getId}" - Utils.tryCatch { - var command = s"yarn logs -applicationId $applicationId >> $rootLogDir/$tempLogFile" - logger.info(s"Fetch yarn logs to temporary file: [$command]") - val processBuilder = new ProcessBuilder(sudoCommands(creator, command): _*) - processBuilder.environment.putAll(sys.env.asJava) - processBuilder.redirectErrorStream(false) - val process = processBuilder.start() - val waitFor = process.waitFor(5, TimeUnit.SECONDS) - logger.trace(s"waitFor: ${waitFor}, result: ${process.exitValue()}") - if (waitFor && process.waitFor() == 0) { - command = s"mv $rootLogDir/$tempLogFile $rootLogDir/yarn_$applicationId" - logger.info(s"Move and save yarn logs: [$command]") - Utils.exec(sudoCommands(creator, command)) - } else { - logPath = new File(engineConnLogDir, tempLogFile) - if (!logPath.exists()) { - throw new LinkisCommonErrorException( - -1, - s"Fetch yarn logs timeout, log aggregation has not completed or is not enabled" - ) - } - } - } { case e: Exception => - throw new LinkisCommonErrorException( - -1, - s"Fail to fetch yarn logs application: $applicationId, message: ${e.getMessage}" - ) - } - } - if (!logPath.exists() || !logPath.isFile) { - throw new ECMErrorException( - LOGFILE_IS_NOT_EXISTS.getErrorCode, - MessageFormat.format(LOGFILE_IS_NOT_EXISTS.getErrorDesc, logPath) - ) - } - logger.info( - s"Try to fetch EngineConn(id: $ticketId, instance: $engineConnInstance) yarn logs from ${logPath.getPath} in application id: $applicationId" - ) - logPath - } - - private def sudoCommands(creator: String, command: String): Array[String] = { - Array( - "/bin/bash", - "-c", - "sudo su " + creator + " -c \"source ~/.bashrc 2>/dev/null; " + command + "\"" - ) - } - -} - -object EngineConnYarnLogOperator { - val OPERATOR_NAME = "engineConnYarnLog" -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala index c3246791751..8ad3031b89f 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala @@ -149,10 +149,11 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w Sender .getSender(MANAGER_SERVICE_NAME) .send( - EngineConnStatusCallbackToAM( + new EngineConnStatusCallbackToAM( conn.getServiceInstance, NodeStatus.ShuttingDown, - " wait init failed , reason " + ExceptionUtils.getRootCauseMessage(t) + " wait init failed , reason " + ExceptionUtils.getRootCauseMessage(t), + false ) ) LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala index 4fcddf4c35f..a6cfa6d11de 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala @@ -98,9 +98,9 @@ class DefaultECMRegisterService extends ECMRegisterService with ECMEventListener logger.info("start register ecm") val response = Sender.getSender(MANAGER_SERVICE_NAME).ask(request) response match { - case RegisterEMResponse(isSuccess, msg) => - if (!isSuccess) { - logger.error(s"Failed to register info to linkis manager, reason: $msg") + case registerEMResponse: RegisterEMResponse => + if (!registerEMResponse.getIsSuccess) { + logger.error(s"Failed to register info to linkis manager, reason: ${registerEMResponse.getMsg}") System.exit(1) } case _ => diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala index 3a541df8b91..ba3dea6d4b4 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala @@ -17,36 +17,35 @@ package org.apache.linkis.ecm.server.service.impl +import java.util + import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.ecm.server.service.OperateService import org.apache.linkis.manager.common.operator.OperatorFactory import org.apache.linkis.manager.common.protocol.em.{ECMOperateRequest, ECMOperateResponse} import org.apache.linkis.rpc.message.annotation.Receiver - import org.apache.commons.lang3.exception.ExceptionUtils - import org.springframework.stereotype.Service -import scala.collection.JavaConverters.mapAsScalaMapConverter @Service class DefaultOperateService extends OperateService with Logging { @Receiver override def executeOperation(ecmOperateRequest: ECMOperateRequest): ECMOperateResponse = { - val parameters = ecmOperateRequest.parameters.asScala.toMap - val operator = Utils.tryCatch(OperatorFactory().getOperatorRequest(parameters)) { t => - logger.error(s"Get operator failed, parameters is ${ecmOperateRequest.parameters}.", t) - return ECMOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) + val parameters = ecmOperateRequest.getParameters() + val operator = Utils.tryCatch(OperatorFactory.apply().getOperatorRequest(parameters)) { t => + logger.error(s"Get operator failed, parameters is ${ecmOperateRequest.getParameters}.", t) + return new ECMOperateResponse( new util.HashMap[String,Object](), true, ExceptionUtils.getRootCauseMessage(t)) } logger.info( - s"Try to execute operator ${operator.getClass.getSimpleName} with parameters ${ecmOperateRequest.parameters}." + s"Try to execute operator ${operator.getClass.getSimpleName} with parameters ${ecmOperateRequest.getParameters}." ) val result = Utils.tryCatch(operator(parameters)) { t => logger.error(s"Execute ${operator.getClass.getSimpleName} failed.", t) - return ECMOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) + return new ECMOperateResponse( new util.HashMap[String,Object](), true, ExceptionUtils.getRootCauseMessage(t)) } - ECMOperateResponse(result) + new ECMOperateResponse(result) } } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala index e6ab513f156..11bd5345635 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala @@ -90,7 +90,7 @@ abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchSe Sender .getSender(MANAGER_SERVICE_NAME) .send( - EngineConnStatusCallbackToAM( + new EngineConnStatusCallbackToAM( conn.getServiceInstance, NodeStatus.ShuttingDown, "Failed to start EngineConn, reason: " + ExceptionUtils.getRootCauseMessage( diff --git a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/java/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.java b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/java/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.java new file mode 100644 index 00000000000..6011ca62594 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/java/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.java @@ -0,0 +1,71 @@ +package org.apache.linkis.engineconn.once.executor.operator; + +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.engineconn.common.exception.EngineConnException; +import org.apache.linkis.engineconn.once.executor.OnceExecutor; +import org.apache.linkis.engineconn.once.executor.OperableOnceExecutor; +import org.apache.linkis.engineconn.once.executor.creation.OnceExecutorManager; +import org.apache.linkis.manager.common.operator.Operator; +import org.apache.linkis.manager.common.operator.OperatorFactory; +import org.apache.linkis.protocol.engine.JobProgressInfo; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class OperableOnceEngineConnOperator implements Operator { + + public static final String PROGRESS_OPERATOR_NAME = "engineConnProgress"; + + public static final String METRICS_OPERATOR_NAME = "engineConnMetrics"; + + public static final String DIAGNOSIS_OPERATOR_NAME = "engineConnDiagnosis"; + + @Override + public String[] getNames() { + return new String[]{PROGRESS_OPERATOR_NAME, METRICS_OPERATOR_NAME, DIAGNOSIS_OPERATOR_NAME}; + } + + @Override + public Map apply(Map parameters) { + String operatorName = OperatorFactory.apply().getOperatorName(parameters); + OnceExecutor reportExecutor = OnceExecutorManager.getInstance().getReportExecutor(); + if (reportExecutor instanceof OperableOnceExecutor) { + OperableOnceExecutor operableOnceExecutor = (OperableOnceExecutor) reportExecutor; + switch (operatorName) { + case PROGRESS_OPERATOR_NAME: + List> progressInfoMap = new ArrayList<>(); + JobProgressInfo[] progressInfoList = operableOnceExecutor.getProgressInfo(); + if (progressInfoList != null && progressInfoList.length != 0) { + for (JobProgressInfo progressInfo : progressInfoList) { + Map infoMap = new HashMap<>(); + infoMap.put("id", progressInfo.id()); + infoMap.put("totalTasks", progressInfo.totalTasks()); + infoMap.put("runningTasks", progressInfo.runningTasks()); + infoMap.put("failedTasks", progressInfo.failedTasks()); + infoMap.put("succeedTasks", progressInfo.succeedTasks()); + progressInfoMap.add(infoMap); + } + } + Map resultMap = new HashMap<>(); + resultMap.put("progress", operableOnceExecutor.getProgress()); + resultMap.put("progressInfo", progressInfoMap); + return resultMap; + case METRICS_OPERATOR_NAME: + return new HashMap() {{ + put("metrics", operableOnceExecutor.getMetrics()); + }}; + case DIAGNOSIS_OPERATOR_NAME: + return new HashMap() {{ + put("diagnosis", operableOnceExecutor.getDiagnosis()); + }}; + default: + throw new WarnException(20308, "This engineConn don't support " + operatorName + " operator."); + } + } else { + throw new WarnException(20308, "This engineConn don't support " + operatorName + " operator."); + } + } +} + diff --git a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala deleted file mode 100644 index ae1828d1c0c..00000000000 --- a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/operator/OperableOnceEngineConnOperator.scala +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineconn.once.executor.operator - -import org.apache.linkis.engineconn.common.exception.EngineConnException -import org.apache.linkis.engineconn.once.executor.OperableOnceExecutor -import org.apache.linkis.engineconn.once.executor.creation.OnceExecutorManager -import org.apache.linkis.manager.common.operator.{Operator, OperatorFactory} - -class OperableOnceEngineConnOperator extends Operator { - - import OperableOnceEngineConnOperator._ - - override def getNames: Array[String] = - Array(PROGRESS_OPERATOR_NAME, METRICS_OPERATOR_NAME, DIAGNOSIS_OPERATOR_NAME) - - override def apply(implicit parameters: Map[String, Any]): Map[String, Any] = { - val operatorName = OperatorFactory().getOperatorName(parameters) - OnceExecutorManager.getInstance.getReportExecutor match { - case operableOnceExecutor: OperableOnceExecutor => - operatorName match { - case PROGRESS_OPERATOR_NAME => - val progressInfo = operableOnceExecutor.getProgressInfo - val progressInfoMap = if (progressInfo != null && progressInfo.nonEmpty) { - progressInfo.map(progressInfo => - Map( - "id" -> progressInfo.id, - "totalTasks" -> progressInfo.totalTasks, - "runningTasks" -> progressInfo.runningTasks, - "failedTasks" -> progressInfo.failedTasks, - "succeedTasks" -> progressInfo.succeedTasks - ) - ) - } else Array.empty[Map[String, Any]] - Map("progress" -> operableOnceExecutor.getProgress, "progressInfo" -> progressInfoMap) - case METRICS_OPERATOR_NAME => - Map("metrics" -> operableOnceExecutor.getMetrics) - case DIAGNOSIS_OPERATOR_NAME => - Map("diagnosis" -> operableOnceExecutor.getDiagnosis) - case _ => - throw EngineConnException( - 20308, - s"This engineConn don't support $operatorName operator." - ) - } - case _ => - throw EngineConnException(20308, s"This engineConn don't support $operatorName operator.") - } - } - -} - -object OperableOnceEngineConnOperator { - val PROGRESS_OPERATOR_NAME = "engineConnProgress" - val METRICS_OPERATOR_NAME = "engineConnMetrics" - val DIAGNOSIS_OPERATOR_NAME = "engineConnDiagnosis" -} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala index 651fc0f3dca..16ea3fa0a41 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala @@ -429,7 +429,7 @@ class TaskExecutionServiceImpl taskYarnResource(task.getTaskId) match { case responseTaskYarnResource: ResponseTaskYarnResource => if ( - responseTaskYarnResource.resourceMap != null && !responseTaskYarnResource.resourceMap.isEmpty + responseTaskYarnResource.getResourceMap != null && !responseTaskYarnResource.getResourceMap.isEmpty ) { responseTaskYarnResource } else { @@ -459,15 +459,15 @@ class TaskExecutionServiceImpl // todo add other info var respRunningInfo: ResponseTaskRunningInfo = null if (null != resourceResponse) { - respRunningInfo = ResponseTaskRunningInfo( + respRunningInfo = new ResponseTaskRunningInfo( progressResponse.execId, progressResponse.progress, progressResponse.progressInfo, - resourceResponse.resourceMap, + resourceResponse.getResourceMap, extraInfoMap ) } else { - respRunningInfo = ResponseTaskRunningInfo( + respRunningInfo = new ResponseTaskRunningInfo( progressResponse.execId, progressResponse.progress, progressResponse.progressInfo, @@ -488,7 +488,7 @@ class TaskExecutionServiceImpl executor match { case executor: ResourceFetchExecutor => val resourceWithApplicationId = executor.FetchResource - ResponseTaskYarnResource(taskID, resourceWithApplicationId) + new ResponseTaskYarnResource(taskID, resourceWithApplicationId) case _ => null } } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.java new file mode 100644 index 00000000000..0ef62c2dbfb --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/java/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.java @@ -0,0 +1,37 @@ +package org.apache.linkis.engineconn.acessible.executor.operator.impl; + +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.engineconn.common.exception.EngineConnException; +import org.apache.linkis.engineconn.core.executor.ExecutorManager; +import org.apache.linkis.engineconn.executor.entity.Executor; +import org.apache.linkis.engineconn.executor.entity.YarnExecutor; +import org.apache.linkis.manager.common.operator.Operator; + +import java.util.HashMap; +import java.util.Map; + +public class EngineConnApplicationInfoOperator implements Operator { + + public static final String OPERATOR_NAME = "engineConnYarnApplication"; + + @Override + public String[] getNames() { + return new String[]{OPERATOR_NAME}; + } + + @Override + public Map apply(Map parameters) { + Executor reportExecutor = ExecutorManager.getInstance().getReportExecutor(); + if ( reportExecutor instanceof YarnExecutor ) { + YarnExecutor yarnExecutor = (YarnExecutor) reportExecutor; + Map result = new HashMap<>(); + result.put("applicationId", yarnExecutor.getApplicationId()); + result.put("applicationUrl", yarnExecutor.getApplicationURL()); + result.put("queue", yarnExecutor.getQueue()); + result.put("yarnMode", yarnExecutor.getYarnMode()); + return result; + } else { + throw new WarnException(20301, "EngineConn is not a yarn application, cannot fetch applicaiton info."); + } + } +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala index 1b5713e5689..0bd7ececefd 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala @@ -204,7 +204,7 @@ class AccessibleEngineConnExecution extends EngineConnExecution with Logging { case resourceExecutor: ResourceExecutor => ManagerService.getManagerService .reportUsedResource( - ResourceUsedProtocol( + new ResourceUsedProtocol( Sender.getThisServiceInstance, resourceExecutor.getCurrentNodeResource(), engineCreationContext.getTicketId diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala deleted file mode 100644 index ad762892ef4..00000000000 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/operator/impl/EngineConnApplicationInfoOperator.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineconn.acessible.executor.operator.impl - -import org.apache.linkis.engineconn.common.exception.EngineConnException -import org.apache.linkis.engineconn.core.executor.ExecutorManager -import org.apache.linkis.engineconn.executor.entity.YarnExecutor -import org.apache.linkis.manager.common.operator.Operator - -class EngineConnApplicationInfoOperator extends Operator { - - override def getNames: Array[String] = Array(EngineConnApplicationInfoOperator.OPERATOR_NAME) - - override def apply(implicit parameters: Map[String, Any]): Map[String, Any] = { - ExecutorManager.getInstance.getReportExecutor match { - case yarnExecutor: YarnExecutor => - Map( - "applicationId" -> yarnExecutor.getApplicationId, - "applicationUrl" -> yarnExecutor.getApplicationURL, - "queue" -> yarnExecutor.getQueue, - "yarnMode" -> yarnExecutor.getYarnMode - ) - case _ => - throw EngineConnException( - 20301, - "EngineConn is not a yarn application, cannot fetch applicaiton info." - ) - } - } - -} - -object EngineConnApplicationInfoOperator { - val OPERATOR_NAME = "engineConnYarnApplication" -} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala index 1ab5a16265f..9ee7c1059e2 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala @@ -57,7 +57,7 @@ class DefaultManagerService extends ManagerService with Logging { logger.info("engineType labels is empty, Not reported") return } - val labelReportRequest = LabelReportRequest(reportLabel.asJava, Sender.getThisServiceInstance) + val labelReportRequest = new LabelReportRequest(reportLabel.asJava, Sender.getThisServiceInstance) getManagerSender.send(labelReportRequest) } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala index df34626c204..738dc55b5d6 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala @@ -17,19 +17,15 @@ package org.apache.linkis.engineconn.acessible.executor.service +import java.util + import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.manager.common.operator.OperatorFactory -import org.apache.linkis.manager.common.protocol.engine.{ - EngineOperateRequest, - EngineOperateResponse -} +import org.apache.linkis.manager.common.protocol.engine.{EngineOperateRequest, EngineOperateResponse} import org.apache.linkis.rpc.message.annotation.Receiver - import org.apache.commons.lang3.exception.ExceptionUtils - import org.springframework.stereotype.Service -import scala.collection.JavaConverters.mapAsScalaMapConverter @Service class DefaultOperateService extends OperateService with Logging { @@ -38,19 +34,19 @@ class DefaultOperateService extends OperateService with Logging { override def executeOperation( engineOperateRequest: EngineOperateRequest ): EngineOperateResponse = { - val parameters = engineOperateRequest.parameters.asScala.toMap - val operator = Utils.tryCatch(OperatorFactory().getOperatorRequest(parameters)) { t => - logger.error(s"Get operator failed, parameters is ${engineOperateRequest.parameters}.", t) - return EngineOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) + val parameters = engineOperateRequest.getParameters() + val operator = Utils.tryCatch(OperatorFactory.apply().getOperatorRequest(parameters)) { t => + logger.error(s"Get operator failed, parameters is ${engineOperateRequest.getParameters}.", t) + return new EngineOperateResponse(new util.HashMap, true, ExceptionUtils.getRootCauseMessage(t)) } logger.info( - s"Try to execute operator ${operator.getClass.getSimpleName} with parameters ${engineOperateRequest.parameters}." + s"Try to execute operator ${operator.getClass.getSimpleName} with parameters ${engineOperateRequest.getParameters}." ) val result = Utils.tryCatch(operator(parameters)) { t => logger.error(s"Execute ${operator.getClass.getSimpleName} failed.", t) - return EngineOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t)) + return new EngineOperateResponse(new util.HashMap, true, ExceptionUtils.getRootCauseMessage(t)) } - EngineOperateResponse(result) + new EngineOperateResponse(result) } } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala index 452c6305b0f..957a03b6f79 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala @@ -81,12 +81,12 @@ class EngineConnTimedLockService extends LockService with Logging { @throws[EngineConnExecutorErrorException] override def tryLock(requestEngineLock: RequestEngineLock): Option[String] = synchronized { if (null != engineConnLock && engineConnLock.isAcquired()) return None - this.lockType = requestEngineLock.lockType + this.lockType = requestEngineLock.getLockType lockType match { case EngineLockType.Always => timedLock(-1) case EngineLockType.Timed => - timedLock(requestEngineLock.timeout) + timedLock(requestEngineLock.getTimeout) case o: Any => logger.error("Invalid lockType : " + BDPJettyServerHelper.gson.toJson(o)) return Some(null) @@ -174,11 +174,11 @@ class EngineConnTimedLockService extends LockService with Logging { @Receiver override def requestUnLock(requestEngineUnlock: RequestEngineUnlock): ResponseEngineUnlock = { - if (StringUtils.isBlank(requestEngineUnlock.lock)) { + if (StringUtils.isBlank(requestEngineUnlock.getLock)) { logger.error("Invalid requestEngineUnlock: ") - ResponseEngineUnlock(false) + new ResponseEngineUnlock(false) } else { - ResponseEngineUnlock(unlock(requestEngineUnlock.lock)) + new ResponseEngineUnlock(unlock(requestEngineUnlock.getLock)) } } @@ -223,7 +223,7 @@ class EngineConnConcurrentLockService extends LockService { @Receiver override def requestUnLock(requestEngineUnlock: RequestEngineUnlock): ResponseEngineUnlock = - ResponseEngineUnlock(true) + new ResponseEngineUnlock(true) override def onAddLock(addLockEvent: ExecutorLockEvent): Unit = {} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/LockService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/LockService.scala index 0860076e40f..ded4cee95e3 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/LockService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/LockService.scala @@ -49,14 +49,14 @@ trait LockService extends ExecutorLockListener with Logging { // Engine can be locked if (!StringUtils.isBlank(lockStr)) { // lock success - response = ResponseEngineLock(true, lockStr, s"Lock for ${requestEngineLock.timeout} ms") + response = new ResponseEngineLock(true, lockStr, s"Lock for ${requestEngineLock.getTimeout} ms") } else { // lock failed - response = ResponseEngineLock(false, lockStr, "lock str is blank") + response = new ResponseEngineLock(false, lockStr, "lock str is blank") } case None => // Engine is busy - response = ResponseEngineLock(false, null, "Engine is busy.") + response = new ResponseEngineLock(false, null, "Engine is busy.") } logger.info( "RequestLock : " + BDPJettyServerHelper.gson.toJson( diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala index fa7d9ff0102..4261e8efd42 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala @@ -82,7 +82,7 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { val prefixMsg = Sender.getThisServiceInstance + s": log dir: ${EngineConnConf.getLogDir}," Utils.tryAndError( engineConnAfterStartCallback.callback( - EngineConnStatusCallback( + new EngineConnStatusCallback( Sender.getThisServiceInstance, engineCreationContext.getTicketId, NodeStatus.Failed, @@ -106,7 +106,7 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { val engineConnAfterStartCallback = new EngineConnAfterStartCallback Utils.tryAndError( engineConnAfterStartCallback.callback( - EngineConnStatusCallback( + new EngineConnStatusCallback( Sender.getThisServiceInstance, engineCreationContext.getTicketId, getNodeStatusOfStartSuccess(engineCreationContext, engineConn), diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala index 1b629cf30cc..39ed9afff40 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala @@ -35,7 +35,7 @@ abstract class AbstractEngineConnStartUpCallback() extends EngineConnCallback wi def callback(protocol: RequestProtocol): Unit = { protocol match { case protocol: EngineConnStatusCallback => - if (protocol.status.equals(NodeStatus.Failed)) { + if (protocol.getStatus().equals(NodeStatus.Failed)) { logger.error(s"protocol will send to lm: ${protocol}") } else { logger.info(s"protocol will send to lm: ${protocol}") diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index a5d3ace35fe..87ac92f6b2d 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -338,7 +338,7 @@ public Message progressWithResource(HttpServletRequest req, @PathVariable("id") metricsVo.put(TaskConstant.JOB_YARNRESOURCE, resoureList); Optional cores = resourceMap.values().stream() - .map(resource -> resource.queueCores()) + .map(resource -> resource.getQueueCores()) .reduce((x, y) -> x + y); Optional memory = resourceMap.values().stream() diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/EngineConnPluginLoaderConf.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/EngineConnPluginLoaderConf.java new file mode 100644 index 00000000000..eab566ff0eb --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/EngineConnPluginLoaderConf.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.loader; + +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.Configuration; + +public class EngineConnPluginLoaderConf { + + public static final String ENGINE_PLUGIN_RESOURCE_ID_NAME_PREFIX = + "wds.linkis.engineConn.plugin.loader.resource-id."; + + public static final CommonVars CLASS_LOADER_CLASS_NAME = + CommonVars.apply("wds.linkis.engineconn.plugin.loader.classname", ""); + + public static final CommonVars ENGINE_PLUGIN_LOADER_DEFAULT_USER = + CommonVars.apply("wds.linkis.engineconn.plugin.loader.defaultUser", "hadoop"); + + public static final CommonVars ENGINE_PLUGIN_STORE_PATH = + CommonVars.apply( + "wds.linkis.engineconn.plugin.loader.store.path", + CommonVars.apply( + "ENGINE_CONN_HOME", + Configuration.getLinkisHome() + "/lib/linkis-engineconn-plugins") + .getValue()); + + public static final CommonVars ENGINE_PLUGIN_PROPERTIES_NAME = + CommonVars.apply("wds.linkis.engineconn.plugin.loader.properties.name", "plugins.properties"); + + public static final CommonVars ENGINE_PLUGIN_LOADER_CACHE_REFRESH_INTERVAL = + CommonVars.apply("wds.linkis.engineconn.plugin.loader.cache.refresh-interval", "300"); + + public static final CommonVars DOWNLOAD_TEMP_DIR_PREFIX = + CommonVars.apply("wds.linkis.engineconn.plugin.loader.download.tmpdir.prefix", ".BML_TMP_"); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/CacheablesEngineConnPluginLoader.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/CacheablesEngineConnPluginLoader.java index ef726644605..059f984fa47 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/CacheablesEngineConnPluginLoader.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/CacheablesEngineConnPluginLoader.java @@ -59,8 +59,7 @@ public CacheablesEngineConnPluginLoader() { @Override public long interval() { return Long.parseLong( - EngineConnPluginLoaderConf.ENGINE_PLUGIN_LOADER_CACHE_REFRESH_INTERVAL() - .getValue()); + EngineConnPluginLoaderConf.ENGINE_PLUGIN_LOADER_CACHE_REFRESH_INTERVAL.getValue()); } @Override diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/DefaultEngineConnPluginLoader.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/DefaultEngineConnPluginLoader.java index 0e54ed8c4e5..26191563060 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/DefaultEngineConnPluginLoader.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/DefaultEngineConnPluginLoader.java @@ -23,7 +23,6 @@ import org.apache.linkis.engineplugin.loader.loaders.resource.LocalEngineConnPluginResourceLoader; import org.apache.linkis.engineplugin.loader.loaders.resource.PluginResource; import org.apache.linkis.engineplugin.loader.utils.EngineConnPluginUtils; -import org.apache.linkis.engineplugin.loader.utils.ExceptionHelper; import org.apache.linkis.manager.engineplugin.common.EngineConnPlugin; import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginLoadException; import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginNotFoundException; @@ -58,17 +57,17 @@ public class DefaultEngineConnPluginLoader extends CacheablesEngineConnPluginLoa private static final String PLUGIN_DIR = "plugin"; - public DefaultEngineConnPluginLoader() throws ErrorException { + public DefaultEngineConnPluginLoader() { // Check store path (is necessary) - String storePath = EngineConnPluginLoaderConf.ENGINE_PLUGIN_STORE_PATH().getValue(); + String storePath = EngineConnPluginLoaderConf.ENGINE_PLUGIN_STORE_PATH.getValue(); LOG.info("DefaultEngineConnPluginLoader, storePath:" + storePath); if (StringUtils.isBlank(storePath)) { - ExceptionHelper.dealErrorException( - 70061, - "You should defined [" - + EngineConnPluginLoaderConf.ENGINE_PLUGIN_STORE_PATH().key() - + "] in properties file", - null); + throw new RuntimeException( + new ErrorException( + 70061, + "You should defined [" + + EngineConnPluginLoaderConf.ENGINE_PLUGIN_STORE_PATH.key() + + "] in properties file")); } // The path can be uri try { @@ -80,17 +79,15 @@ public DefaultEngineConnPluginLoader() throws ErrorException { } catch (URISyntaxException e) { // Ignore } catch (IllegalArgumentException e) { - ExceptionHelper.dealErrorException( - 70061, - "The value:[" - + storePath - + "] of [" - + EngineConnPluginLoaderConf.ENGINE_PLUGIN_STORE_PATH().key() - + "] is incorrect", - e); + throw new RuntimeException( + new ErrorException( + 70061, + "You should defined [" + + EngineConnPluginLoaderConf.ENGINE_PLUGIN_STORE_PATH.key() + + "] in properties file")); } this.rootStorePath = storePath; - this.pluginPropsName = EngineConnPluginLoaderConf.ENGINE_PLUGIN_PROPERTIES_NAME().getValue(); + this.pluginPropsName = EngineConnPluginLoaderConf.ENGINE_PLUGIN_PROPERTIES_NAME.getValue(); // Prepare inner loaders // resourceLoaders.add(new BmlEngineConnPluginResourceLoader()); resourceLoaders.add(new LocalEngineConnPluginResourceLoader()); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/resource/BmlEngineConnPluginResourceLoader.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/resource/BmlEngineConnPluginResourceLoader.java index 8b8a071480f..23607a7063c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/resource/BmlEngineConnPluginResourceLoader.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/resource/BmlEngineConnPluginResourceLoader.java @@ -58,13 +58,13 @@ public class BmlEngineConnPluginResourceLoader implements EngineConnPluginsResou private String downloadTmpDir; public BmlEngineConnPluginResourceLoader() { - this(EngineConnPluginLoaderConf.ENGINE_PLUGIN_LOADER_DEFAULT_USER().getValue(), null); + this(EngineConnPluginLoaderConf.ENGINE_PLUGIN_LOADER_DEFAULT_USER.getValue(), null); } public BmlEngineConnPluginResourceLoader(String clientUser, Map clientProps) { this.clientUser = clientUser; this.bmlClient = BmlClientFactory.createBmlClient(clientUser, clientProps); - this.downloadTmpDir = EngineConnPluginLoaderConf.DOWNLOAD_TEMP_DIR_PREFIX().getValue(); + this.downloadTmpDir = EngineConnPluginLoaderConf.DOWNLOAD_TEMP_DIR_PREFIX.getValue(); } @Override diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.java new file mode 100644 index 00000000000..395471e78b0 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.conf; + +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.Configuration; + +public class EngineConnPluginConfiguration { + + public static final CommonVars ENGINE_CONN_HOME = + CommonVars.apply( + "wds.linkis.engineconn.home", + CommonVars.apply( + "ENGINE_CONN_HOME", + Configuration.getLinkisHome() + "/lib/linkis-engineconn-plugins") + .getValue()); + + public static final CommonVars ENGINE_CONN_DIST_LOAD_ENABLE = + CommonVars.apply("wds.linkis.engineconn.dist.load.enable", true); + + public static final CommonVars ENABLED_BML_UPLOAD_FAILED_EXIT = + CommonVars.apply("wds.linkis.engineconn.bml.upload.failed.enable", true); + + // for third party eg appconn/datax, if all update, can set to false then to remove + public static final CommonVars EC_BML_VERSION_MAY_WITH_PREFIX_V = + CommonVars.apply("linkis.engineconn.bml.version.may.with.prefix", true); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/interceptor/EngineConnLaunchInterceptor.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/interceptor/EngineConnLaunchInterceptor.java similarity index 76% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/interceptor/EngineConnLaunchInterceptor.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/interceptor/EngineConnLaunchInterceptor.java index e88783185fd..0b5e7ad129f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/interceptor/EngineConnLaunchInterceptor.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/interceptor/EngineConnLaunchInterceptor.java @@ -15,20 +15,16 @@ * limitations under the License. */ -package org.apache.linkis.engineplugin.server.interceptor +package org.apache.linkis.engineplugin.server.interceptor; -import org.apache.linkis.manager.engineplugin.common.launch.entity.{ - EngineConnBuildRequest, - RicherEngineConnBuildRequest -} - -trait EngineConnLaunchInterceptor { +import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest; +import org.apache.linkis.manager.engineplugin.common.launch.entity.RicherEngineConnBuildRequest; +interface EngineConnLaunchInterceptor { /** * 补充资源文件信息，如：UDF、用户jar、Python文件等 补充启动参数信息等 * * @param engineConnBuildRequest */ - def intercept(engineConnBuildRequest: EngineConnBuildRequest): RicherEngineConnBuildRequest - + RicherEngineConnBuildRequest intercept(EngineConnBuildRequest engineConnBuildRequest); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/loader/EngineConnPluginsLoaderFactory.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/loader/EngineConnPluginsLoaderFactory.java new file mode 100644 index 00000000000..280c35d6602 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/loader/EngineConnPluginsLoaderFactory.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.loader; + +import org.apache.linkis.engineplugin.loader.loaders.DefaultEngineConnPluginLoader; + +public class EngineConnPluginsLoaderFactory { + + private static final org.apache.linkis.engineplugin.loader.loaders.EngineConnPluginsLoader + engineConnPluginsLoader = new DefaultEngineConnPluginLoader(); + + public static org.apache.linkis.engineplugin.loader.loaders.EngineConnPluginsLoader + getEngineConnPluginsLoader() { + return engineConnPluginsLoader; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.java new file mode 100644 index 00000000000..1a3493fe613 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.localize; + +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.engineplugin.server.conf.EngineConnPluginConfiguration; +import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginErrorException; +import org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary.*; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; + +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.nio.file.Paths; +import java.text.MessageFormat; +import java.util.Arrays; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary.*; + +public abstract class AbstractEngineConnBmlResourceGenerator + implements EngineConnBmlResourceGenerator { + + private static final Logger logger = + LoggerFactory.getLogger(AbstractEngineConnBmlResourceGenerator.class); + + public AbstractEngineConnBmlResourceGenerator() { + if (!new File(getEngineConnsHome()).exists()) { + throw new WarnException( + CANNOT_HOME_PATH_EC.getErrorCode(), + MessageFormat.format(CANNOT_HOME_PATH_EC.getErrorDesc(), getEngineConnsHome())); + } + } + + public String getEngineConnsHome() { + return EngineConnPluginConfiguration.ENGINE_CONN_HOME.getValue(); + } + + protected String getEngineConnDistHome(EngineTypeLabel engineConnTypeLabel) { + return getEngineConnDistHome( + engineConnTypeLabel.getEngineType(), engineConnTypeLabel.getVersion()); + } + + protected String getEngineConnDistHome(String engineConnType, String version) { + String engineConnDistHome = + Paths.get(getEngineConnsHome(), engineConnType, "dist").toFile().getPath(); + checkEngineConnDistHome(engineConnDistHome); + if (StringUtils.isBlank(version) + || EngineConnBmlResourceGenerator.NO_VERSION_MARK.equals(version)) { + return engineConnDistHome; + } + String engineConnPackageHome = Paths.get(engineConnDistHome, version).toFile().getPath(); + logger.info("getEngineConnDistHome, engineConnPackageHome path:" + engineConnPackageHome); + File engineConnPackageHomeFile = new File(engineConnPackageHome); + if (!engineConnPackageHomeFile.exists()) { + if (!version.startsWith("v") + && (boolean) EngineConnPluginConfiguration.EC_BML_VERSION_MAY_WITH_PREFIX_V.getValue()) { + String versionOld = "v" + version; + String engineConnPackageHomeOld = + Paths.get(engineConnDistHome, versionOld).toFile().getPath(); + logger.info( + "try to getEngineConnDistHome with prefix v, engineConnPackageHome path:" + + engineConnPackageHomeOld); + File engineConnPackageHomeFileOld = new File(engineConnPackageHomeOld); + if (!engineConnPackageHomeFileOld.exists()) { + throw new WarnException( + ENGINE_VERSION_NOT_FOUND.getErrorCode(), + MessageFormat.format( + ENGINE_VERSION_NOT_FOUND.getErrorDesc(), version, engineConnType)); + } else { + return engineConnPackageHomeOld; + } + } else { + throw new WarnException( + ENGINE_VERSION_NOT_FOUND.getErrorCode(), + MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc(), version, engineConnType)); + } + } else { + return engineConnPackageHome; + } + } + + private void checkEngineConnDistHome(String engineConnPackageHomePath) { + File engineConnPackageHomeFile = new File(engineConnPackageHomePath); + checkEngineConnDistHome(engineConnPackageHomeFile); + } + + private void checkEngineConnDistHome(File engineConnPackageHome) { + if (!engineConnPackageHome.exists()) { + throw new WarnException( + CANNOT_HOME_PATH_DIST.getErrorCode(), + MessageFormat.format( + CANNOT_HOME_PATH_DIST.getErrorDesc(), engineConnPackageHome.getPath())); + } + } + + protected String[] getEngineConnDistHomeList(String engineConnType) { + String engineConnDistHome = + Paths.get(getEngineConnsHome(), engineConnType, "dist").toFile().getPath(); + File engineConnDistHomeFile = new File(engineConnDistHome); + checkEngineConnDistHome(engineConnDistHomeFile); + File[] children = engineConnDistHomeFile.listFiles(); + if (children.length == 0) { + throw new WarnException( + DIST_IS_EMPTY.getErrorCode(), + MessageFormat.format(DIST_IS_EMPTY.getErrorDesc(), engineConnType)); + } else { + return Arrays.stream(children).map(File::getPath).toArray(String[]::new); + } + } + + @Override + public String[] getEngineConnTypeListFromDisk() { + return Arrays.stream(new File(getEngineConnsHome()).listFiles()) + .filter(file -> !file.isHidden()) + .map(file -> file.getName()) + .toArray(String[]::new); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/DefaultEngineConnBmlResourceGenerator.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/DefaultEngineConnBmlResourceGenerator.java new file mode 100644 index 00000000000..38460f73812 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/DefaultEngineConnBmlResourceGenerator.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.localize; + +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.common.utils.ZipUtils; + +import java.io.File; +import java.text.MessageFormat; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary.CONTAINS_SPECIAL_CHARCATERS; +import static org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary.NO_PERMISSION_FILE; + +public class DefaultEngineConnBmlResourceGenerator extends AbstractEngineConnBmlResourceGenerator { + + private static final Logger logger = + LoggerFactory.getLogger(DefaultEngineConnBmlResourceGenerator.class); + + public DefaultEngineConnBmlResourceGenerator() {} + + @Override + public Map generate(String engineConnType) { + String[] engineConnDistHomes = getEngineConnDistHomeList(engineConnType); + Map resultMap = new HashMap<>(); + for (String path : engineConnDistHomes) { + + File versionFile = new File(path); + logger.info("generate, versionFile:" + path); + String key = versionFile.getName(); + if (key.contains("-")) { + throw new WarnException( + CONTAINS_SPECIAL_CHARCATERS.getErrorCode(), + MessageFormat.format(CONTAINS_SPECIAL_CHARCATERS.getErrorDesc(), engineConnType)); + } + + try { + EngineConnLocalizeResource[] engineConnLocalizeResources = + generateDir(versionFile.getPath()); + resultMap.put(key, engineConnLocalizeResources); + } catch (Throwable t) { + logger.error("Generate dir : " + path + " error, msg : " + t.getMessage(), t); + throw t; + } + } + + return resultMap; + } + + @Override + public EngineConnLocalizeResource[] generate(String engineConnType, String version) { + String path = getEngineConnDistHome(engineConnType, version); + return generateDir(path); + } + + private EngineConnLocalizeResource[] generateDir(String path) { + File distFile = new File(path); + logger.info("generateDir, distFile:" + path); + File[] validFiles = + distFile.listFiles( + f -> + !f.getName().endsWith(".zip") + || !new File(path, f.getName().replace(".zip", "")).exists()); + + return Arrays.stream(validFiles) + .map( + file -> { + if (file.isFile()) { + return new EngineConnLocalizeResourceImpl( + file.getPath(), file.getName(), file.lastModified(), file.length()); + } else { + File newFile = new File(path, file.getName() + ".zip"); + if (newFile.exists() && !newFile.delete()) { + throw new WarnException( + NO_PERMISSION_FILE.getErrorCode(), + MessageFormat.format(NO_PERMISSION_FILE.getErrorDesc(), newFile)); + } + + ZipUtils.fileToZip(file.getPath(), path, file.getName() + ".zip"); + // If it is a folder, the last update time here is the last update time of the + // folder, not the last update time of + // ZIP.(如果是文件夹，这里的最后更新时间，采用文件夹的最后更新时间，而不是ZIP的最后更新时间.) + return new EngineConnLocalizeResourceImpl( + newFile.getPath(), newFile.getName(), file.lastModified(), newFile.length()); + } + }) + .toArray(EngineConnLocalizeResource[]::new); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnBmlResourceGenerator.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnBmlResourceGenerator.java new file mode 100644 index 00000000000..35a46fff607 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnBmlResourceGenerator.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.localize; + +import java.util.Map; + +public interface EngineConnBmlResourceGenerator { + String NO_VERSION_MARK = "_default_"; + + String[] getEngineConnTypeListFromDisk(); + + Map generate(String engineConnType); + + EngineConnLocalizeResource[] generate(String engineConnType, String version); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnLocalizeResource.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnLocalizeResource.java new file mode 100644 index 00000000000..8e6408c72b8 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnLocalizeResource.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.localize; + +import java.io.InputStream; + +public interface EngineConnLocalizeResource { + String fileName(); + + long lastModified(); + + long fileSize(); + + InputStream getFileInputStream(); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnLocalizeResourceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnLocalizeResourceImpl.java new file mode 100644 index 00000000000..181e45172fa --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/localize/EngineConnLocalizeResourceImpl.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.localize; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; + +public class EngineConnLocalizeResourceImpl implements EngineConnLocalizeResource { + private final String filePath; + private final String fileName; + private final long lastModified; + private final long fileSize; + + public EngineConnLocalizeResourceImpl( + String filePath, String fileName, long lastModified, long fileSize) { + this.filePath = filePath; + this.fileName = fileName; + this.lastModified = lastModified; + this.fileSize = fileSize; + } + + @Override + public InputStream getFileInputStream() { + try { + return new FileInputStream(filePath); + } catch (FileNotFoundException e) { + // e.printStackTrace(); + } + return null; + } + + public String filePath() { + return filePath; + } + + public String fileName() { + return fileName; + } + + public long lastModified() { + return lastModified; + } + + public long fileSize() { + return fileSize; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnLaunchService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnLaunchService.java new file mode 100644 index 00000000000..3074e7ba577 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnLaunchService.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.service; + +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.engineplugin.server.loader.EngineConnPluginsLoaderFactory; +import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuilder; +import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest; +import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest; +import org.apache.linkis.manager.engineplugin.common.launch.process.EngineConnResourceGenerator; +import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder; +import org.apache.linkis.manager.engineplugin.common.loader.entity.EngineConnPluginInstance; +import org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.rpc.message.annotation.Receiver; + +import org.apache.commons.lang3.exception.ExceptionUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Component +public class DefaultEngineConnLaunchService implements EngineConnLaunchService { + + private static final Logger logger = + LoggerFactory.getLogger(DefaultEngineConnLaunchService.class); + + @Autowired private EngineConnResourceGenerator engineConnResourceGenerator; + + private EngineConnLaunchBuilder getEngineLaunchBuilder( + EngineTypeLabel engineTypeLabel, EngineConnBuildRequest engineBuildRequest) { + final EngineConnPluginInstance engineConnPluginInstance; + try { + engineConnPluginInstance = + EngineConnPluginsLoaderFactory.getEngineConnPluginsLoader() + .getEngineConnPlugin(engineTypeLabel); + } catch (Exception e) { + throw new RuntimeException(e); + } + final EngineConnLaunchBuilder builder = + engineConnPluginInstance.plugin().getEngineConnLaunchBuilder(); + if (builder instanceof JavaProcessEngineConnLaunchBuilder) { + ((JavaProcessEngineConnLaunchBuilder) builder) + .setEngineConnResourceGenerator(engineConnResourceGenerator); + } + builder.setBuildRequest(engineBuildRequest); + return builder; + } + + @Override + @Receiver + public EngineConnLaunchRequest createEngineConnLaunchRequest( + EngineConnBuildRequest engineBuildRequest) { + final Optional engineTypeOption = + engineBuildRequest.labels().stream() + .filter(label -> label instanceof EngineTypeLabel) + .map(label -> (EngineTypeLabel) label) + .findFirst(); + + if (!engineTypeOption.isPresent()) { + throw new WarnException( + EngineconnCoreErrorCodeSummary.ETL_REQUESTED.getErrorCode(), + EngineconnCoreErrorCodeSummary.ETL_REQUESTED.getErrorDesc()); + } + + final EngineTypeLabel engineTypeLabel = engineTypeOption.get(); + return Utils.tryCatch( + () -> getEngineLaunchBuilder(engineTypeLabel, engineBuildRequest).buildEngineConn(), + (Throwable t) -> { + logger.error( + String.format( + "Failed to createEngineConnLaunchRequest(%s)", engineBuildRequest.ticketId()), + t); + throw new WarnException( + EngineconnCoreErrorCodeSummary.FAILED_CREATE_ELR.getErrorCode(), + String.format( + "%s, %s", + EngineconnCoreErrorCodeSummary.FAILED_CREATE_ELR.getErrorDesc(), + ExceptionUtils.getRootCauseMessage(t))); + }); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceFactoryService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceFactoryService.java new file mode 100644 index 00000000000..a19155ff753 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceFactoryService.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.service; + +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.engineplugin.server.loader.EngineConnPluginsLoaderFactory; +import org.apache.linkis.manager.common.entity.resource.NodeResource; +import org.apache.linkis.manager.engineplugin.common.loader.entity.EngineConnPluginInstance; +import org.apache.linkis.manager.engineplugin.common.resource.EngineResourceFactory; +import org.apache.linkis.manager.engineplugin.common.resource.EngineResourceRequest; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.rpc.message.annotation.Receiver; + +import org.springframework.stereotype.Component; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary.ETL_REQUESTED; + +@Component +public class DefaultEngineConnResourceFactoryService implements EngineConnResourceFactoryService { + + private static final Logger logger = + LoggerFactory.getLogger(DefaultEngineConnResourceFactoryService.class); + + @Override + public EngineResourceFactory getResourceFactoryBy(EngineTypeLabel engineType) { + final EngineConnPluginInstance engineConnPluginInstance; + try { + engineConnPluginInstance = + EngineConnPluginsLoaderFactory.getEngineConnPluginsLoader() + .getEngineConnPlugin(engineType); + } catch (Exception e) { + // e.printStackTrace(); + throw new RuntimeException(e); + } + return engineConnPluginInstance.plugin().getEngineResourceFactory(); + } + + @Override + @Receiver + public NodeResource createEngineResource(final EngineResourceRequest engineResourceRequest) { + logger.info(String.format("To invoke createEngineResource %s", engineResourceRequest)); + final Optional engineTypeOption = + engineResourceRequest.labels().stream() + .filter(label -> label instanceof EngineTypeLabel) + .map(label -> (EngineTypeLabel) label) + .findFirst(); + + if (!engineTypeOption.isPresent()) { + throw new WarnException(ETL_REQUESTED.getErrorCode(), ETL_REQUESTED.getErrorDesc()); + } + + final EngineTypeLabel engineTypeLabel = engineTypeOption.get(); + return getResourceFactoryBy(engineTypeLabel).createEngineResource(engineResourceRequest); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.java new file mode 100644 index 00000000000..b62b9b96d93 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.service; + +import org.apache.linkis.bml.client.BmlClient; +import org.apache.linkis.bml.client.BmlClientFactory; +import org.apache.linkis.bml.protocol.BmlUpdateResponse; +import org.apache.linkis.bml.protocol.BmlUploadResponse; +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.engineplugin.server.conf.EngineConnPluginConfiguration; +import org.apache.linkis.engineplugin.server.dao.EngineConnBmlResourceDao; +import org.apache.linkis.engineplugin.server.entity.EngineConnBmlResource; +import org.apache.linkis.engineplugin.server.localize.EngineConnBmlResourceGenerator; +import org.apache.linkis.engineplugin.server.localize.EngineConnLocalizeResource; +import org.apache.linkis.manager.common.protocol.bml.BmlResource; +import org.apache.linkis.manager.common.protocol.bml.BmlResource.BmlResourceVisibility; +import org.apache.linkis.manager.engineplugin.common.launch.process.EngineConnResource; +import org.apache.linkis.manager.engineplugin.common.launch.process.LaunchConstants; +import org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary; +import org.apache.linkis.rpc.message.annotation.Receiver; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +import java.text.MessageFormat; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.linkis.manager.engineplugin.errorcode.EngineconnCoreErrorCodeSummary.EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION; + +@Component +public class DefaultEngineConnResourceService extends EngineConnResourceService { + private static final Logger logger = + LoggerFactory.getLogger(DefaultEngineConnResourceService.class); + + @Autowired private EngineConnBmlResourceGenerator engineConnBmlResourceGenerator; + + @Autowired private EngineConnBmlResourceDao engineConnBmlResourceDao; + + private final BmlClient bmlClient = BmlClientFactory.createBmlClient(); + private boolean isRefreshing = false; + + @PostConstruct + @Override + public void init() { + if ((boolean) EngineConnPluginConfiguration.ENGINE_CONN_DIST_LOAD_ENABLE.getValue()) { + logger.info("Start to refresh all engineconn plugins when inited."); + refreshAll(false, false); + } + } + + private BmlResource uploadToBml(final EngineConnLocalizeResource localizeResource) { + final BmlUploadResponse response = + bmlClient.uploadResource( + Utils.getJvmUser(), localizeResource.fileName(), localizeResource.getFileInputStream()); + final BmlResource bmlResource = new BmlResource(); + bmlResource.setResourceId(response.resourceId()); + bmlResource.setVersion(response.version()); + return bmlResource; + } + + private BmlResource uploadToBml( + final EngineConnLocalizeResource localizeResource, final String resourceId) { + final BmlUpdateResponse response = + bmlClient.updateResource( + Utils.getJvmUser(), + resourceId, + localizeResource.fileName(), + localizeResource.getFileInputStream()); + final BmlResource bmlResource = new BmlResource(); + bmlResource.setResourceId(response.resourceId()); + bmlResource.setVersion(response.version()); + return bmlResource; + } + + @Override + public void refreshAll(boolean iswait, boolean force) { + if (!isRefreshing) { + synchronized (this) { + if (!isRefreshing) { + + final Runnable refreshTask = + new Runnable() { + @Override + public void run() { + isRefreshing = true; + logger.info("Try to initialize the dist resources of all EngineConns. "); + String[] engineConnTypeList = + engineConnBmlResourceGenerator.getEngineConnTypeListFromDisk(); + for (String engineConnType : engineConnTypeList) { + try { + logger.info( + "Try to initialize all versions of {}EngineConn.", engineConnType); + Map version2Localize = + engineConnBmlResourceGenerator.generate(engineConnType); + for (Map.Entry entry : + version2Localize.entrySet()) { + logger.info( + "Try to initialize {}EngineConn-{}.", engineConnType, entry.getKey()); + refresh(entry.getValue(), engineConnType, entry.getKey(), force); + } + + } catch (Exception t) { + if (!iswait + && EngineConnPluginConfiguration.ENABLED_BML_UPLOAD_FAILED_EXIT + .getValue()) { + logger.error("Failed to upload engine conn to bml, now exit!", t); + System.exit(1); + } + logger.error("Failed to upload engine conn to bml", t); + } + } + isRefreshing = false; + } + }; + Future future = Utils.defaultScheduler().submit(refreshTask); + + if (iswait) { + try { + future.get(); + } catch (InterruptedException | ExecutionException e) { + logger.info("DefaultEngineConnResourceService refreshTask execution failed", e); + } + } else { + logger.info("DefaultEngineConnResourceService IsRefreshing EngineConns..."); + } + } + } + } + } + + @Receiver + public boolean refreshAll(final RefreshAllEngineConnResourceRequest engineConnRefreshAllRequest) { + logger.info("Start to refresh all engineconn plugins."); + refreshAll(true, false); + return true; + } + + @Receiver + @Override + public boolean refresh( + final RefreshEngineConnResourceRequest engineConnRefreshRequest, final boolean force) { + final String engineConnType = engineConnRefreshRequest.getEngineConnType(); + final String version = engineConnRefreshRequest.getVersion(); + if ("*".equals(version) || StringUtils.isEmpty(version)) { + logger.info("Try to refresh all versions of {}EngineConn.", engineConnType); + Map version2Localize = + engineConnBmlResourceGenerator.generate(engineConnType); + for (Map.Entry entry : version2Localize.entrySet()) { + logger.info("Try to initialize {}EngineConn-{}.", engineConnType, entry.getKey()); + refresh(entry.getValue(), engineConnType, entry.getKey(), force); + } + + } else { + logger.info("Try to refresh {}EngineConn-{}.", engineConnType, version); + EngineConnLocalizeResource[] localize = + engineConnBmlResourceGenerator.generate(engineConnType, version); + refresh(localize, engineConnType, version, force); + } + return true; + } + + private void refresh( + final EngineConnLocalizeResource[] localize, + final String engineConnType, + final String version, + final boolean force) { + final List engineConnBmlResources = + engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, version); + + if (Stream.of(localize) + .filter( + localizeResource -> + StringUtils.equals( + LaunchConstants.ENGINE_CONN_CONF_DIR_NAME() + ".zip", + localizeResource.fileName()) + || StringUtils.equals( + LaunchConstants.ENGINE_CONN_LIB_DIR_NAME() + ".zip", + localizeResource.fileName())) + .count() + < 2) { + + throw new WarnException( + EngineconnCoreErrorCodeSummary.LIB_CONF_DIR_NECESSARY.getErrorCode(), + MessageFormat.format( + EngineconnCoreErrorCodeSummary.LIB_CONF_DIR_NECESSARY.getErrorDesc(), + engineConnType)); + } + + for (EngineConnLocalizeResource localizeResource : localize) { + + Optional resource = + engineConnBmlResources.stream() + .filter(r -> r.getFileName().equals(localizeResource.fileName())) + .findFirst(); + if (!resource.isPresent()) { + logger.info( + "Ready to upload a new bmlResource for {}EngineConn-{}. path: {}", + engineConnType, + version, + localizeResource.fileName()); + final BmlResource bmlResource = uploadToBml(localizeResource); + final EngineConnBmlResource engineConnBmlResource = new EngineConnBmlResource(); + engineConnBmlResource.setBmlResourceId(bmlResource.getResourceId()); + engineConnBmlResource.setBmlResourceVersion(bmlResource.getVersion()); + engineConnBmlResource.setCreateTime(new Date()); + engineConnBmlResource.setLastUpdateTime(new Date()); + engineConnBmlResource.setEngineConnType(engineConnType); + engineConnBmlResource.setFileName(localizeResource.fileName()); + engineConnBmlResource.setFileSize(localizeResource.fileSize()); + engineConnBmlResource.setLastModified(localizeResource.lastModified()); + engineConnBmlResource.setVersion(version); + engineConnBmlResourceDao.save(engineConnBmlResource); + } else { + boolean isChanged = + resource.get().getFileSize() != localizeResource.fileSize() + || resource.get().getLastModified() != localizeResource.lastModified(); + + if (isChanged || (!isChanged && force)) { + if (!isChanged && force) { + logger.info( + "The file has no change in {}EngineConn-{}, path: {}, but force to refresh", + engineConnType, + version, + localizeResource.fileName()); + } + logger.info( + "Ready to upload a refreshed bmlResource for {}EngineConn-{}. path: {}", + engineConnType, + version, + localizeResource.fileName()); + final EngineConnBmlResource engineConnBmlResource = resource.get(); + final BmlResource bmlResource = + uploadToBml(localizeResource, engineConnBmlResource.getBmlResourceId()); + engineConnBmlResource.setBmlResourceVersion(bmlResource.getVersion()); + engineConnBmlResource.setLastUpdateTime(new Date()); + engineConnBmlResource.setFileSize(localizeResource.fileSize()); + engineConnBmlResource.setLastModified(localizeResource.lastModified()); + engineConnBmlResourceDao.update(engineConnBmlResource); + } else { + logger.info( + "The file has no change in {}EngineConn-{}, path: {}", + engineConnType, + version, + localizeResource.fileName()); + } + } + } + } + + @Receiver + @Override + public EngineConnResource getEngineConnBMLResources( + final GetEngineConnResourceRequest engineConnBMLResourceRequest) { + final String engineConnType = engineConnBMLResourceRequest.getEngineConnType(); + final String version = engineConnBMLResourceRequest.getVersion(); + + List engineConnBmlResources = + engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, version); + if (engineConnBmlResources.size() == 0 + && (boolean) EngineConnPluginConfiguration.EC_BML_VERSION_MAY_WITH_PREFIX_V.getValue()) { + logger.info("Try to get engine conn bml resource with prefex v"); + engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, "v" + version); + } + + Optional confBmlResourceMap = + engineConnBmlResources.stream() + .filter( + r -> r.getFileName().equals(LaunchConstants.ENGINE_CONN_CONF_DIR_NAME() + ".zip")) + .map(this::parseToBmlResource) + .findFirst(); + Optional libBmlResourceMap = + engineConnBmlResources.stream() + .filter( + r -> r.getFileName().equals(LaunchConstants.ENGINE_CONN_LIB_DIR_NAME() + ".zip")) + .map(this::parseToBmlResource) + .findFirst(); + + if (!confBmlResourceMap.isPresent() || !libBmlResourceMap.isPresent()) { + throw new WarnException( + EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION.getErrorCode(), + EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION.getErrorDesc()); + } + final BmlResource confBmlResource = confBmlResourceMap.get(); + final BmlResource libBmlResource = libBmlResourceMap.get(); + BmlResource[] otherBmlResources = + engineConnBmlResources.stream() + .filter( + r -> + !r.getFileName().equals(LaunchConstants.ENGINE_CONN_CONF_DIR_NAME() + ".zip") + || r.getFileName() + .equals(LaunchConstants.ENGINE_CONN_LIB_DIR_NAME() + ".zip")) + .map(this::parseToBmlResource) + .toArray(BmlResource[]::new); + + return new EngineConnResource() { + @Override + public BmlResource getConfBmlResource() { + return confBmlResource; + } + + @Override + public BmlResource getLibBmlResource() { + return libBmlResource; + } + + @Override + public BmlResource[] getOtherBmlResources() { + return otherBmlResources; + } + }; + } + + private BmlResource parseToBmlResource(final EngineConnBmlResource engineConnBmlResource) { + final BmlResource bmlResource = new BmlResource(); + bmlResource.setFileName(engineConnBmlResource.getFileName()); + bmlResource.setOwner(Utils.getJvmUser()); + bmlResource.setResourceId(engineConnBmlResource.getBmlResourceId()); + bmlResource.setVersion(engineConnBmlResource.getBmlResourceVersion()); + bmlResource.setVisibility(BmlResourceVisibility.Public); + return bmlResource; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/EngineConnLaunchService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnLaunchService.java similarity index 73% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/EngineConnLaunchService.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnLaunchService.java index 92f8c27edde..370fdbee137 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/EngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnLaunchService.java @@ -15,17 +15,12 @@ * limitations under the License. */ -package org.apache.linkis.engineplugin.server.service +package org.apache.linkis.engineplugin.server.service; -import org.apache.linkis.manager.engineplugin.common.launch.entity.{ - EngineConnBuildRequest, - EngineConnLaunchRequest -} - -trait EngineConnLaunchService { +import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest; +import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest; - def createEngineConnLaunchRequest( - engineBuildRequest: EngineConnBuildRequest - ): EngineConnLaunchRequest +public interface EngineConnLaunchService { + EngineConnLaunchRequest createEngineConnLaunchRequest(EngineConnBuildRequest engineBuildRequest); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/EngineConnResourceFactoryService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceFactoryService.java similarity index 67% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/EngineConnResourceFactoryService.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceFactoryService.java index 35be84e8493..6fa6db52711 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/EngineConnResourceFactoryService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceFactoryService.java @@ -15,19 +15,16 @@ * limitations under the License. */ -package org.apache.linkis.engineplugin.server.service +package org.apache.linkis.engineplugin.server.service; -import org.apache.linkis.manager.common.entity.resource.NodeResource -import org.apache.linkis.manager.engineplugin.common.resource.{ - EngineResourceFactory, - EngineResourceRequest -} -import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel - -trait EngineConnResourceFactoryService { +import org.apache.linkis.manager.common.entity.resource.NodeResource; +import org.apache.linkis.manager.engineplugin.common.resource.EngineResourceFactory; +import org.apache.linkis.manager.engineplugin.common.resource.EngineResourceRequest; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; - def getResourceFactoryBy(engineType: EngineTypeLabel): EngineResourceFactory +public interface EngineConnResourceFactoryService { - def createEngineResource(engineResourceRequest: EngineResourceRequest): NodeResource + EngineResourceFactory getResourceFactoryBy(EngineTypeLabel engineType); + NodeResource createEngineResource(EngineResourceRequest engineResourceRequest); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceRequest.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceRequest.java new file mode 100644 index 00000000000..62f7aea6612 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceRequest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.service; + +import org.apache.linkis.protocol.message.RequestMethod; +import org.apache.linkis.protocol.message.RequestProtocol; + +public abstract class EngineConnResourceRequest implements RequestProtocol, RequestMethod { + + private String engineConnType; + private String version; + + private boolean force; + + public String getEngineConnType() { + return engineConnType; + } + + public void setEngineConnType(String engineConnType) { + this.engineConnType = engineConnType; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public boolean getForce() { + return force; + } + + public void setForce(boolean force) { + this.force = force; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceService.java new file mode 100644 index 00000000000..5f2fb6cf691 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/EngineConnResourceService.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.service; + +import org.apache.linkis.manager.engineplugin.common.launch.process.EngineConnResource; +import org.apache.linkis.manager.engineplugin.common.launch.process.EngineConnResourceGenerator; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; + +public abstract class EngineConnResourceService implements EngineConnResourceGenerator { + + public abstract void init(); + + public abstract void refreshAll(boolean wait, boolean force); + + public abstract boolean refresh( + RefreshEngineConnResourceRequest engineConnRefreshRequest, boolean force); + + public abstract EngineConnResource getEngineConnBMLResources( + GetEngineConnResourceRequest engineConnBMLResourceRequest); + + @Override + public EngineConnResource getEngineConnBMLResources(EngineTypeLabel engineTypeLabel) { + GetEngineConnResourceRequest engineConnBMLResourceRequest = new GetEngineConnResourceRequest(); + engineConnBMLResourceRequest.setEngineConnType(engineTypeLabel.getEngineType()); + engineConnBMLResourceRequest.setVersion(engineTypeLabel.getVersion()); + return getEngineConnBMLResources(engineConnBMLResourceRequest); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/GetEngineConnResourceRequest.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/GetEngineConnResourceRequest.java new file mode 100644 index 00000000000..046a0dd7ec3 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/GetEngineConnResourceRequest.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.service; + +public class GetEngineConnResourceRequest extends EngineConnResourceRequest { + @Override + public String method() { + return "/enginePlugin/engineConn/getResource"; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshAllEngineConnResourceRequest.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshAllEngineConnResourceRequest.java new file mode 100644 index 00000000000..6bd41d2599e --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshAllEngineConnResourceRequest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.service; + +import org.apache.linkis.protocol.message.RequestMethod; +import org.apache.linkis.protocol.message.RequestProtocol; + +public class RefreshAllEngineConnResourceRequest implements RequestProtocol, RequestMethod { + @Override + public String method() { + return "/enginePlugin/engineConn/refreshAll"; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshEngineConnResourceRequest.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshEngineConnResourceRequest.java new file mode 100644 index 00000000000..f518a72059a --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/RefreshEngineConnResourceRequest.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.server.service; + +public class RefreshEngineConnResourceRequest extends EngineConnResourceRequest { + @Override + public String method() { + return "/enginePlugin/engineConn/refresh"; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java new file mode 100644 index 00000000000..d916387d298 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.conf; + +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.TimeType; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.manager.common.entity.enumeration.MaintainType; + +import java.util.Arrays; +import java.util.Optional; + +public class AMConfiguration { + + public static final CommonVars ECM_ADMIN_OPERATIONS = + CommonVars.apply("wds.linkis.governance.admin.operations", ""); + + public static final CommonVars ENGINE_START_MAX_TIME = + CommonVars.apply("wds.linkis.manager.am.engine.start.max.time", new TimeType("11m")); + + public static final CommonVars ENGINE_CONN_START_REST_MAX_WAIT_TIME = + CommonVars.apply("wds.linkis.manager.am.engine.rest.start.max.time", new TimeType("40s")); + + public static final CommonVars ENGINE_REUSE_MAX_TIME = + CommonVars.apply("wds.linkis.manager.am.engine.reuse.max.time", new TimeType("5m")); + + public static final CommonVars ENGINE_REUSE_COUNT_LIMIT = + CommonVars.apply("wds.linkis.manager.am.engine.reuse.count.limit", 2); + + public static final CommonVars NODE_STATUS_HEARTBEAT_TIME = + CommonVars.apply("wds.linkis.manager.am.node.heartbeat", new TimeType("3m")); + + public static final CommonVars NODE_HEARTBEAT_MAX_UPDATE_TIME = + CommonVars.apply("wds.linkis.manager.am.node.heartbeat", new TimeType("5m")); + + public static final CommonVars DEFAULT_NODE_OWNER = + CommonVars.apply("wds.linkis.manager.am.default.node.owner", "hadoop"); + + public static final CommonVars STOP_ENGINE_WAIT = + CommonVars.apply("wds.linkis.manager.am.stop.engine.wait", new TimeType("5m")); + + public static final CommonVars STOP_EM_WAIT = + CommonVars.apply("wds.linkis.manager.am.stop.em.wait", new TimeType("5m")); + + public static final CommonVars EM_LABEL_INIT_WAIT = + CommonVars.apply("wds.linkis.manager.am.em.label.init.wait", new TimeType("5m")); + + public static final CommonVars EM_NEW_WAIT_MILLS = + CommonVars.apply("wds.linkis.manager.am.em.new.wait.mills", 1000 * 60L); + + public static final CommonVars ENGINECONN_DEBUG_ENABLED = + CommonVars.apply("wds.linkis.engineconn.debug.mode.enable", false); + + public static final CommonVars MULTI_USER_ENGINE_TYPES = + CommonVars.apply( + "wds.linkis.multi.user.engine.types", "jdbc,es,presto,io_file,appconn,openlookeng,trino"); + + public static final CommonVars ALLOW_BATCH_KILL_ENGINE_TYPES = + CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "spark,hive,python"); + + public static final CommonVars MULTI_USER_ENGINE_USER = + CommonVars.apply("wds.linkis.multi.user.engine.user", getDefaultMultiEngineUser()); + + public static final CommonVars ENGINE_LOCKER_MAX_TIME = + CommonVars.apply("wds.linkis.manager.am.engine.locker.max.time", 1000 * 60 * 5); + + public static final CommonVars AM_CAN_RETRY_LOGS = + CommonVars.apply( + "wds.linkis.manager.am.can.retry.logs", "already in use;Cannot allocate memory"); + + public static final int ASK_ENGINE_ASYNC_MAX_THREAD_SIZE = + CommonVars.apply("wds.linkis.ecm.launch.max.thread.size", 200).getValue(); + + public static final int ASYNC_STOP_ENGINE_MAX_THREAD_SIZE = + CommonVars.apply("wds.linkis.async.stop.engine.size", 20).getValue(); + + public static final CommonVars EC_MAINTAIN_TIME_STR = + CommonVars.apply("wds.linkis.ec.maintain.time.key", MaintainType.Default.toString()); + + public static final int EC_MAINTAIN_WORK_START_TIME = + CommonVars.apply("wds.linkis.ec.maintain.time.work.start.time", 8).getValue(); + + public static final int EC_MAINTAIN_WORK_END_TIME = + CommonVars.apply("wds.linkis.ec.maintain.time.work.end.time", 19).getValue(); + + public static final Boolean NODE_SELECT_HOTSPOT_EXCLUSION_RULE = + CommonVars.apply("linkis.node.select.hotspot.exclusion.rule.enable", true).getValue(); + + public static String getDefaultMultiEngineUser() { + String jvmUser = Utils.getJvmUser(); + return String.format( + "{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", io_file:\"root\"}", + jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser); + } + + public static boolean isMultiUserEngine(String engineType) { + String[] multiUserEngine = AMConfiguration.MULTI_USER_ENGINE_TYPES.getValue().split(","); + Optional findResult = + Arrays.stream(multiUserEngine).filter(e -> e.equalsIgnoreCase(engineType)).findFirst(); + return findResult.isPresent(); + } + + public static boolean isAllowKilledEngineType(String engineType) { + String[] allowBatchKillEngine = + AMConfiguration.ALLOW_BATCH_KILL_ENGINE_TYPES.getValue().split(","); + Optional findResult = + Arrays.stream(allowBatchKillEngine).filter(e -> e.equalsIgnoreCase(engineType)).findFirst(); + return findResult.isPresent(); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ApplicationManagerSpringConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ApplicationManagerSpringConfiguration.java new file mode 100644 index 00000000000..96d28a185b1 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ApplicationManagerSpringConfiguration.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.conf; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class ApplicationManagerSpringConfiguration { + + @ConditionalOnMissingBean + @Bean + public EngineConnConfigurationService getDefaultEngineConnConfigurationService() { + return new DefaultEngineConnConfigurationService(); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ConfigurationMapCache.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ConfigurationMapCache.java new file mode 100644 index 00000000000..039704351ee --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ConfigurationMapCache.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.conf; + +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.governance.common.protocol.conf.*; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; +import org.apache.linkis.protocol.CacheableProtocol; +import org.apache.linkis.rpc.RPCMapCache; + +import java.util.Map; + +import scala.Tuple2; + +public class ConfigurationMapCache { + + public static RPCMapCache globalMapCache = + new RPCMapCache( + Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME().getValue()) { + @Override + public CacheableProtocol createRequest(UserCreatorLabel userCreatorLabel) { + return new RequestQueryGlobalConfig(userCreatorLabel.getUser()); + } + + @Override + public Map createMap(Object any) { + if (any instanceof ResponseQueryConfig) { + return ((ResponseQueryConfig) any).getKeyAndValue(); + } + return null; + } + }; + + public static RPCMapCache, String, String> + engineMapCache = + new RPCMapCache, String, String>( + Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME().getValue()) { + @Override + public CacheableProtocol createRequest( + Tuple2 labelTuple) { + return new RequestQueryEngineConfigWithGlobalConfig( + labelTuple._1(), labelTuple._2(), null); + } + + @Override + public Map createMap(Object any) { + if (any instanceof ResponseQueryConfig) { + return ((ResponseQueryConfig) any).getKeyAndValue(); + } + return null; + } + }; +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/DefaultEngineConnConfigurationService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/DefaultEngineConnConfigurationService.java new file mode 100644 index 00000000000..6a07d313953 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/DefaultEngineConnConfigurationService.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.conf; + +import org.apache.linkis.manager.am.util.Utils; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import scala.Tuple2; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class DefaultEngineConnConfigurationService implements EngineConnConfigurationService { + + private static final Logger logger = + LoggerFactory.getLogger(DefaultEngineConnConfigurationService.class); + + @Override + public Map getConsoleConfiguration(List> labelList) { + Map properties = new HashMap<>(); + + Optional userCreatorLabelOption = + labelList.stream() + .filter(l -> l instanceof UserCreatorLabel) + .map(l -> (UserCreatorLabel) l) + .findFirst(); + + Optional engineTypeLabelOption = + labelList.stream() + .filter(l -> l instanceof EngineTypeLabel) + .map(l -> (EngineTypeLabel) l) + .findFirst(); + userCreatorLabelOption.ifPresent( + userCreatorLabel -> { + engineTypeLabelOption.ifPresent( + engineTypeLabel -> { + Map engineConfig = + Utils.tryAndWarn( + () -> + ConfigurationMapCache.engineMapCache.getCacheMap( + new Tuple2(userCreatorLabel, engineTypeLabel)), + logger); + if (engineConfig != null) { + properties.putAll(engineConfig); + } + }); + }); + return properties; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/EngineConnConfigurationService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/EngineConnConfigurationService.java new file mode 100644 index 00000000000..2d2d01d0573 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/EngineConnConfigurationService.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.conf; + +import org.apache.linkis.manager.label.entity.Label; + +import java.util.List; +import java.util.Map; + +public interface EngineConnConfigurationService { + + Map getConsoleConfiguration(List> labelList); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ManagerMonitorConf.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ManagerMonitorConf.java new file mode 100644 index 00000000000..c3a35f7921d --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ManagerMonitorConf.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.conf; + +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.TimeType; + +public class ManagerMonitorConf { + + public static final CommonVars NODE_MAX_CREATE_TIME = + CommonVars.apply("wds.linkis.manager.am.node.create.time", new TimeType("12m")); + + public static final CommonVars NODE_HEARTBEAT_MAX_UPDATE_TIME = + CommonVars.apply("wds.linkis.manager.am.node.heartbeat", new TimeType("12m")); + + public static final CommonVars ENGINE_KILL_TIMEOUT = + CommonVars.apply("wds.linkis.manager.am.engine.kill.timeout", new TimeType("2m")); + + public static final CommonVars EM_KILL_TIMEOUT = + CommonVars.apply("wds.linkis.manager.am.em.kill.timeout", new TimeType("2m")); + + public static final CommonVars MANAGER_MONITOR_ASYNC_POLL_SIZE = + CommonVars.apply("wds.linkis.manager.monitor.async.poll.size", 5); + + public static final CommonVars MONITOR_SWITCH_ON = + CommonVars.apply("wds.linkis.manager.am.monitor.switch.on", true); + + public static final CommonVars ECM_HEARTBEAT_MAX_UPDATE_TIME = + CommonVars.apply("wds.linkis.manager.am.ecm.heartbeat", new TimeType("5m")); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java new file mode 100644 index 00000000000..4736433ada0 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.converter; + +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; +import org.apache.linkis.manager.common.entity.metrics.NodeHealthyInfo; +import org.apache.linkis.manager.common.entity.metrics.NodeMetrics; +import org.apache.linkis.manager.common.entity.metrics.NodeOverLoadInfo; +import org.apache.linkis.manager.common.entity.metrics.NodeTaskInfo; +import org.apache.linkis.manager.common.entity.node.AMNode; +import org.apache.linkis.manager.service.common.metrics.MetricsConverter; +import org.apache.linkis.server.BDPJettyServerHelper; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.stereotype.Component; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Component +public class DefaultMetricsConverter implements MetricsConverter { + + private static final Logger logger = LoggerFactory.getLogger(DefaultMetricsConverter.class); + + @Override + public NodeTaskInfo parseTaskInfo(NodeMetrics nodeMetrics) { + String msg = nodeMetrics.getHeartBeatMsg(); + if (StringUtils.isNotBlank(msg)) { + try { + JsonNode jsonNode = BDPJettyServerHelper.jacksonJson().readTree(msg); + if (jsonNode != null && jsonNode.has("taskInfo")) { + NodeTaskInfo taskInfo = + BDPJettyServerHelper.jacksonJson() + .readValue(jsonNode.get("taskInfo").asText(), NodeTaskInfo.class); + return taskInfo; + } + } catch (IOException e) { + logger.error("parse task info failed", e); + } + } + return null; + } + + @Override + public NodeHealthyInfo parseHealthyInfo(NodeMetrics nodeMetrics) { + String healthyInfo = nodeMetrics.getHealthy(); + if (StringUtils.isNotBlank(healthyInfo)) { + try { + return BDPJettyServerHelper.jacksonJson().readValue(healthyInfo, NodeHealthyInfo.class); + } catch (IOException e) { + logger.error("parse healthy info failed", e); + } + } + return null; + } + + @Override + public NodeOverLoadInfo parseOverLoadInfo(NodeMetrics nodeMetrics) { + String overLoad = nodeMetrics.getOverLoad(); + if (StringUtils.isNotBlank(overLoad)) { + try { + return BDPJettyServerHelper.jacksonJson().readValue(overLoad, NodeOverLoadInfo.class); + } catch (IOException e) { + logger.error("parse over load info failed", e); + } + } + return null; + } + + @Override + public NodeStatus parseStatus(NodeMetrics nodeMetrics) { + return NodeStatus.values()[nodeMetrics.getStatus()]; + } + + @Override + public String convertTaskInfo(NodeTaskInfo nodeTaskInfo) { + try { + return BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeTaskInfo); + } catch (JsonProcessingException e) { + logger.error("convert task info failed", e); + } + return null; + } + + @Override + public String convertHealthyInfo(NodeHealthyInfo nodeHealthyInfo) { + try { + return BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeHealthyInfo); + } catch (JsonProcessingException e) { + logger.error("convert healthy info failed", e); + } + return null; + } + + @Override + public String convertOverLoadInfo(NodeOverLoadInfo nodeOverLoadInfo) { + try { + return BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeOverLoadInfo); + } catch (JsonProcessingException e) { + logger.error("convert over load info failed", e); + } + return null; + } + + @Override + public int convertStatus(NodeStatus nodeStatus) { + return nodeStatus.ordinal(); + } + + @Override + public AMNode fillMetricsToNode(AMNode amNode, NodeMetrics metrics) { + if (metrics == null) return amNode; + amNode.setNodeStatus(parseStatus(metrics)); + amNode.setNodeTaskInfo(parseTaskInfo(metrics)); + amNode.setNodeHealthyInfo(parseHealthyInfo(metrics)); + amNode.setNodeOverLoadInfo(parseOverLoadInfo(metrics)); + amNode.setUpdateTime(metrics.getUpdateTime()); + return amNode; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelChecker.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelChecker.java new file mode 100644 index 00000000000..a0928db981c --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelChecker.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.label; + +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.manager.label.entity.em.EMInstanceLabel; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; +import org.apache.linkis.manager.service.common.label.LabelChecker; + +import org.springframework.stereotype.Component; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +@Component +class AMLabelChecker implements LabelChecker { + + @Override + public boolean checkEngineLabel(List> labelList) { + return checkCorrespondingLabel(labelList, EngineTypeLabel.class, UserCreatorLabel.class); + } + + @Override + public boolean checkEMLabel(List> labelList) { + return checkCorrespondingLabel(labelList, EMInstanceLabel.class); + } + + @Override + public boolean checkCorrespondingLabel(List> labelList, Class... clazz) { + // TODO: 是否需要做子类的判断 + List> classes = Arrays.asList(clazz); + return labelList.stream() + .filter(Objects::nonNull) + .map(Label::getClass) + .collect(Collectors.toList()) + .containsAll(classes); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelFilter.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelFilter.java new file mode 100644 index 00000000000..5fa8e7db12a --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelFilter.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.label; + +import org.apache.linkis.governance.common.conf.GovernanceCommonConf; +import org.apache.linkis.manager.label.entity.*; +import org.apache.linkis.manager.label.entity.node.AliasServiceInstanceLabel; +import org.apache.linkis.manager.service.common.label.LabelFilter; + +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.stream.Collectors; + +@Component +public class AMLabelFilter implements LabelFilter { + + @Override + public List> choseEngineLabel(List> labelList) { + return labelList.stream() + .filter( + label -> + label instanceof EngineNodeLabel + || (label instanceof AliasServiceInstanceLabel + && ((AliasServiceInstanceLabel) label) + .getAlias() + .equals(GovernanceCommonConf.ENGINE_CONN_SPRING_NAME().getValue()))) + .collect(Collectors.toList()); + } + + @Override + public List> choseEMLabel(List> labelList) { + return labelList.stream() + .filter( + label -> + label instanceof EMNodeLabel + || (label instanceof AliasServiceInstanceLabel + && ((AliasServiceInstanceLabel) label) + .getAlias() + .equals( + GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME().getValue()))) + .collect(Collectors.toList()); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/DefaultManagerLabelService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/DefaultManagerLabelService.java new file mode 100644 index 00000000000..50c7e1ec6e9 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/DefaultManagerLabelService.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.label; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.manager.label.entity.em.EMInstanceLabel; +import org.apache.linkis.manager.label.entity.engine.EngineInstanceLabel; +import org.apache.linkis.manager.label.service.NodeLabelService; +import org.apache.linkis.manager.service.common.label.ManagerLabelService; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Service +public class DefaultManagerLabelService implements ManagerLabelService { + + private static final Logger logger = LoggerFactory.getLogger(DefaultManagerLabelService.class); + + @Autowired private NodeLabelService nodeLabelService; + + @Override + public boolean isEngine(ServiceInstance serviceInstance) { + List> labelList = nodeLabelService.getNodeLabels(serviceInstance); + return isEngine(labelList); + } + + @Override + public boolean isEM(ServiceInstance serviceInstance) { + List> labelList = nodeLabelService.getNodeLabels(serviceInstance); + boolean isEngine = labelList.stream().anyMatch(label -> label instanceof EngineInstanceLabel); + if (!isEngine) { + return labelList.stream().anyMatch(label -> label instanceof EMInstanceLabel); + } else { + return false; + } + } + + @Override + public boolean isEngine(List> labels) { + return labels.stream().anyMatch(label -> label instanceof EngineInstanceLabel); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/EngineReuseLabelRester.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/EngineReuseLabelChooser.java similarity index 78% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/EngineReuseLabelRester.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/EngineReuseLabelChooser.java index e890fab04fa..edbaf638fae 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/EngineReuseLabelRester.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/EngineReuseLabelChooser.java @@ -15,14 +15,13 @@ * limitations under the License. */ -package org.apache.linkis.manager.am.label +package org.apache.linkis.manager.am.label; -import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.Label; -import java.util +import java.util.List; -trait EngineReuseLabelChooser { - - def chooseLabels(labelList: util.List[Label[_]]): util.List[Label[_]] +public interface EngineReuseLabelChooser { + List> chooseLabels(List> labelList); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.java new file mode 100644 index 00000000000..522fc207f02 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.label; + +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.manager.am.conf.AMConfiguration; +import org.apache.linkis.manager.am.exception.AMErrorCode; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; +import org.apache.linkis.server.BDPJettyServerHelper; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static scala.collection.JavaConverters.*; + +@Component +public class MultiUserEngineReuseLabelChooser implements EngineReuseLabelChooser { + private static final Logger logger = + LoggerFactory.getLogger(MultiUserEngineReuseLabelChooser.class); + + private final String[] multiUserEngine = + AMConfiguration.MULTI_USER_ENGINE_TYPES.getValue().split(","); + private final Map userMap = getMultiUserEngineUserMap(); + + private Map getMultiUserEngineUserMap() { + String userJson = AMConfiguration.MULTI_USER_ENGINE_USER.getValue(); + if (StringUtils.isNotBlank(userJson)) { + Map userMap = BDPJettyServerHelper.gson().fromJson(userJson, Map.class); + return userMap; + } else { + throw new WarnException( + AMErrorCode.AM_CONF_ERROR.getErrorCode(), + String.format( + "Multi-user engine parameter configuration error, please check key %s", + AMConfiguration.MULTI_USER_ENGINE_USER.key())); + } + } + + /** + * Filter out UserCreator Label that supports multi-user engine + * + * @param labelList + * @return + */ + @Override + public List> chooseLabels(List> labelList) { + List> labels = new ArrayList<>(labelList); + Optional engineTypeLabelOption = + labels.stream() + .filter(label -> label instanceof EngineTypeLabel) + .map(label -> (EngineTypeLabel) label) + .findFirst(); + if (engineTypeLabelOption.isPresent()) { + EngineTypeLabel engineTypeLabel = engineTypeLabelOption.get(); + Optional maybeString = + Stream.of(multiUserEngine) + .filter(engineTypeLabel.getEngineType()::equalsIgnoreCase) + .findFirst(); + Optional userCreatorLabelOption = + labels.stream() + .filter(label -> label instanceof UserCreatorLabel) + .map(label -> (UserCreatorLabel) label) + .findFirst(); + if (maybeString.isPresent() && userCreatorLabelOption.isPresent()) { + String userAdmin = userMap.get(engineTypeLabel.getEngineType()); + UserCreatorLabel userCreatorLabel = userCreatorLabelOption.get(); + logger.info( + String.format( + "For multi user engine to reset userCreatorLabel user %s to Admin %s", + userCreatorLabel.getUser(), userAdmin)); + userCreatorLabel.setUser(userAdmin); + return labels; + } + } + return labelList; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.java new file mode 100644 index 00000000000..b36a51bd6bf --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.locker; + +import org.apache.linkis.manager.common.entity.node.AMEngineNode; +import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.common.protocol.RequestEngineLock; +import org.apache.linkis.manager.common.protocol.RequestEngineUnlock; +import org.apache.linkis.manager.common.protocol.RequestManagerUnlock; +import org.apache.linkis.manager.common.protocol.engine.EngineLockType; +import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder; +import org.apache.linkis.rpc.message.annotation.Receiver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Component +public class DefaultEngineNodeLocker implements EngineNodeLocker { + private static final Logger logger = LoggerFactory.getLogger(DefaultEngineNodeLocker.class); + + @Autowired private NodePointerBuilder nodeBuilder; + + @Override + public Optional lockEngine(EngineNode engineNode, long timeout) { + // TODO 判断engine需要的锁类型进行不同的实例化 + return nodeBuilder + .buildEngineNodePointer(engineNode) + .lockEngine(new RequestEngineLock(timeout, EngineLockType.Timed)); + } + + @Override + public void releaseLock(EngineNode engineNode, String lock) { + nodeBuilder.buildEngineNodePointer(engineNode).releaseLock(new RequestEngineUnlock(lock)); + } + + @Receiver + public void releaseLock(RequestManagerUnlock requestManagerUnlock) { + try { + logger.info( + String.format( + "client%s Start to unlock engine %s", + requestManagerUnlock.getClientInstance(), requestManagerUnlock.getEngineInstance())); + AMEngineNode engineNode = new AMEngineNode(); + engineNode.setServiceInstance(requestManagerUnlock.getEngineInstance()); + releaseLock(engineNode, requestManagerUnlock.getLock()); + logger.info( + String.format( + "client%s Finished to unlock engine %s", + requestManagerUnlock.getClientInstance(), requestManagerUnlock.getEngineInstance())); + } catch (Exception e) { + logger.error("release lock failed", e); + } + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/label/LabelFilter.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/EngineNodeLocker.java similarity index 73% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/label/LabelFilter.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/EngineNodeLocker.java index f65cc491aac..171714b3330 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/label/LabelFilter.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/EngineNodeLocker.java @@ -15,16 +15,15 @@ * limitations under the License. */ -package org.apache.linkis.manager.service.common.label +package org.apache.linkis.manager.am.locker; -import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.common.entity.node.EngineNode; -import java.util +import java.util.Optional; -trait LabelFilter { +public interface EngineNodeLocker { - def choseEngineLabel(labelList: util.List[Label[_]]): util.List[Label[_]] - - def choseEMLabel(labelList: util.List[Label[_]]): util.List[Label[_]] + Optional lockEngine(EngineNode engineNode, long timeout); + void releaseLock(EngineNode engineNode, String lock); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.java new file mode 100644 index 00000000000..c5b8e23709c --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.manager; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.manager.common.entity.metrics.NodeMetrics; +import org.apache.linkis.manager.common.entity.node.*; +import org.apache.linkis.manager.common.entity.persistence.PersistenceNodeEntity; +import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; +import org.apache.linkis.manager.common.protocol.em.ECMOperateResponse; +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; +import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest; +import org.apache.linkis.manager.exception.PersistenceErrorException; +import org.apache.linkis.manager.persistence.NodeManagerPersistence; +import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence; +import org.apache.linkis.manager.rm.ResourceInfo; +import org.apache.linkis.manager.rm.service.ResourceManager; +import org.apache.linkis.manager.service.common.metrics.MetricsConverter; +import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Component +public class DefaultEMNodeManager implements EMNodeManager { + private static final Logger logger = LoggerFactory.getLogger(DefaultEMNodeManager.class); + + @Autowired private NodeManagerPersistence nodeManagerPersistence; + + @Autowired private NodeMetricManagerPersistence nodeMetricManagerPersistence; + + @Autowired private MetricsConverter metricsConverter; + + @Autowired private NodePointerBuilder nodePointerBuilder; + + @Autowired private ResourceManager resourceManager; + + @Override + public void emRegister(EMNode emNode) { + try { + nodeManagerPersistence.addNodeInstance(emNode); + } catch (PersistenceErrorException e) { + logger.warn("DefaultEMNodeManager emRegister failed", e); + throw new RuntimeException(e); + } + // init metric + nodeMetricManagerPersistence.addOrupdateNodeMetrics( + metricsConverter.getInitMetric(emNode.getServiceInstance())); + } + + @Override + public void addEMNodeInstance(EMNode emNode) { + try { + nodeManagerPersistence.addNodeInstance(emNode); + } catch (PersistenceErrorException e) { + logger.warn("DefaultEMNodeManager addEMNodeInstance failed", e); + throw new RuntimeException(e); + } + } + + @Override + public void initEMNodeMetrics(EMNode emNode) { + nodeMetricManagerPersistence.addOrupdateNodeMetrics( + metricsConverter.getInitMetric(emNode.getServiceInstance())); + } + + @Override + public List listEngines(EMNode emNode) { + List result = new ArrayList<>(); + List nodes = nodeManagerPersistence.getEngineNodeByEM(emNode.getServiceInstance()); + Map metricses = new HashMap<>(); + for (NodeMetrics m : nodeMetricManagerPersistence.getNodeMetrics(nodes)) { + metricses.put(m.getServiceInstance().toString(), m); + } + for (EngineNode node : nodes) { + NodeMetrics metrics = metricses.get(node.getServiceInstance().toString()); + if (metrics != null) { + metricsConverter.fillMetricsToNode(node, metrics); + } + result.add(node); + } + return result; + } + + @Override + public List listUserEngines(EMNode emNode, String user) { + List result = new ArrayList<>(); + for (EngineNode node : listEngines(emNode)) { + if (node.getOwner().equals(user)) { + result.add(node); + } + } + return result; + } + + @Override + public List listUserNodes(String user) { + return nodeManagerPersistence.getNodes(user); + } + + /** + * Get detailed em information from the persistence TODO add label to node ? + * + * @param scoreServiceInstances + * @return + */ + @Override + public EMNode[] getEMNodes(ScoreServiceInstance[] scoreServiceInstances) { + if (null == scoreServiceInstances || scoreServiceInstances.length == 0) { + return null; + } + EMNode[] emNodes = + Arrays.stream(scoreServiceInstances) + .map( + scoreServiceInstance -> { + AMEMNode emNode = new AMEMNode(); + emNode.setScore(scoreServiceInstance.getScore()); + emNode.setServiceInstance(scoreServiceInstance.getServiceInstance()); + return emNode; + }) + .toArray(EMNode[]::new); + + // 1. add nodeMetrics 2 add RM info + ResourceInfo resourceInfo = + resourceManager.getResourceInfo( + Arrays.stream(scoreServiceInstances) + .map(ScoreServiceInstance::getServiceInstance) + .toArray(ServiceInstance[]::new)); + List nodeMetrics = + nodeMetricManagerPersistence.getNodeMetrics(Arrays.asList(emNodes)); + + for (EMNode emNode : emNodes) { + Optional optionMetrics = + nodeMetrics.stream() + .filter(metrics -> metrics.getServiceInstance().equals(emNode.getServiceInstance())) + .findFirst(); + Optional optionRMNode = + resourceInfo.getResourceInfo().stream() + .filter(rmNode -> rmNode.getServiceInstance().equals(emNode.getServiceInstance())) + .findFirst(); + + optionMetrics.ifPresent(metrics -> metricsConverter.fillMetricsToNode(emNode, metrics)); + optionRMNode.ifPresent(rmNode -> emNode.setNodeResource(rmNode.getNodeResource())); + } + return emNodes; + } + + @Override + public EMNode getEM(ServiceInstance serviceInstance) { + Node node = nodeManagerPersistence.getNode(serviceInstance); + if (null == node) { + logger.info("This em of " + serviceInstance + " not exists in db"); + return null; + } + AMEMNode emNode = new AMEMNode(); + emNode.setOwner(node.getOwner()); + emNode.setServiceInstance(node.getServiceInstance()); + if (node instanceof PersistenceNodeEntity) { + emNode.setStartTime(((PersistenceNodeEntity) node).getStartTime()); + } + emNode.setMark(emNode.getMark()); + metricsConverter.fillMetricsToNode(emNode, nodeMetricManagerPersistence.getNodeMetrics(emNode)); + return emNode; + } + + @Override + public void stopEM(EMNode emNode) { + nodePointerBuilder.buildEMNodePointer(emNode).stopNode(); + } + + @Override + public void deleteEM(EMNode emNode) { + nodeManagerPersistence.removeNodeInstance(emNode); + logger.info("Finished to clear emNode instance(" + emNode.getServiceInstance() + ") info "); + nodeMetricManagerPersistence.deleteNodeMetrics(emNode); + logger.info("Finished to clear emNode(" + emNode.getServiceInstance() + ") metrics info"); + } + + @Override + public void pauseEM(ServiceInstance serviceInstance) {} + + /** + * 1. request engineManager to launch engine + * + * @param engineBuildRequest + * @param emNode + * @return + */ + @Override + public EngineNode createEngine(EngineConnLaunchRequest engineConnLaunchRequest, EMNode emNode) { + return nodePointerBuilder.buildEMNodePointer(emNode).createEngine(engineConnLaunchRequest); + } + + @Override + public void stopEngine(EngineStopRequest engineStopRequest, EMNode emNode) { + nodePointerBuilder.buildEMNodePointer(emNode).stopEngine(engineStopRequest); + } + + @Override + public ECMOperateResponse executeOperation(EMNode ecmNode, ECMOperateRequest request) { + return nodePointerBuilder.buildEMNodePointer(ecmNode).executeOperation(request); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java new file mode 100644 index 00000000000..3caccf630ed --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.manager; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.common.exception.LinkisRetryException; +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.common.utils.DefaultRetryHandler; +import org.apache.linkis.common.utils.RetryHandler; +import org.apache.linkis.manager.am.conf.AMConfiguration; +import org.apache.linkis.manager.am.exception.AMErrorCode; +import org.apache.linkis.manager.am.locker.EngineNodeLocker; +import org.apache.linkis.manager.common.constant.AMConstant; +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; +import org.apache.linkis.manager.common.entity.metrics.NodeMetrics; +import org.apache.linkis.manager.common.entity.node.*; +import org.apache.linkis.manager.common.entity.persistence.PersistenceLabel; +import org.apache.linkis.manager.common.protocol.engine.EngineOperateRequest; +import org.apache.linkis.manager.common.protocol.engine.EngineOperateResponse; +import org.apache.linkis.manager.common.protocol.node.NodeHeartbeatMsg; +import org.apache.linkis.manager.exception.PersistenceErrorException; +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactory; +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; +import org.apache.linkis.manager.label.entity.engine.EngineInstanceLabel; +import org.apache.linkis.manager.persistence.LabelManagerPersistence; +import org.apache.linkis.manager.persistence.NodeManagerPersistence; +import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence; +import org.apache.linkis.manager.rm.ResourceInfo; +import org.apache.linkis.manager.rm.service.ResourceManager; +import org.apache.linkis.manager.service.common.metrics.MetricsConverter; +import org.apache.linkis.manager.service.common.pointer.EngineNodePointer; +import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.lang.reflect.UndeclaredThrowableException; +import java.util.*; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Service +public class DefaultEngineNodeManager implements EngineNodeManager { + private static final Logger logger = LoggerFactory.getLogger(DefaultEngineNodeManager.class); + + @Autowired private EngineNodeLocker engineLocker; + + @Autowired private NodeManagerPersistence nodeManagerPersistence; + + @Autowired private NodeMetricManagerPersistence nodeMetricManagerPersistence; + + @Autowired private MetricsConverter metricsConverter; + + @Autowired private NodePointerBuilder nodePointerBuilder; + + @Autowired private ResourceManager resourceManager; + + @Autowired private LabelManagerPersistence labelManagerPersistence; + + private final LabelBuilderFactory labelBuilderFactory = + LabelBuilderFactoryContext.getLabelBuilderFactory(); + + @Override + public List listEngines(String user) { + List userNodes = nodeManagerPersistence.getNodes(user); + + List nodes = + userNodes.stream() + .map(Node::getServiceInstance) + .map(nodeManagerPersistence::getEngineNode) + .collect(Collectors.toList()); + + List nodeMetrics = nodeMetricManagerPersistence.getNodeMetrics(nodes); + Map metricses = + nodeMetrics.stream() + .collect(Collectors.toMap(m -> m.getServiceInstance().toString(), m -> m)); + + nodes.forEach( + node -> { + Optional nodeMetricsOptional = + Optional.ofNullable(metricses.get(node.getServiceInstance().toString())); + nodeMetricsOptional.ifPresent(m -> metricsConverter.fillMetricsToNode(node, m)); + }); + return nodes; + } + + @Override + public EngineNode getEngineNodeInfo(EngineNode engineNode) { + /** 修改为实时请求对应的EngineNode */ + EngineNodePointer engine = nodePointerBuilder.buildEngineNodePointer(engineNode); + NodeHeartbeatMsg heartMsg = engine.getNodeHeartbeatMsg(); + engineNode.setNodeHealthyInfo(heartMsg.getHealthyInfo()); + engineNode.setNodeOverLoadInfo(heartMsg.getOverLoadInfo()); + engineNode.setNodeStatus(heartMsg.getStatus()); + return engineNode; + } + + @Override + public EngineNode getEngineNodeInfoByDB(EngineNode engineNode) { + // 1. 从持久化器中获取EngineNode信息，需要获取Task信息和Status信息，方便后面使用 + engineNode = nodeManagerPersistence.getEngineNode(engineNode.getServiceInstance()); + metricsConverter.fillMetricsToNode( + engineNode, nodeMetricManagerPersistence.getNodeMetrics(engineNode)); + return engineNode; + } + + @Override + public void updateEngineStatus( + ServiceInstance serviceInstance, NodeStatus fromState, NodeStatus toState) {} + + @Override + public void updateEngine(EngineNode engineNode) { + nodeManagerPersistence.updateNodeInstance(engineNode); + } + + @Override + public EngineNode switchEngine(EngineNode engineNode) { + return null; + } + + @Override + public EngineNode reuseEngine(EngineNode engineNode) { + EngineNode node = getEngineNodeInfo(engineNode); + if (!NodeStatus.isAvailable(node.getNodeStatus())) { + return null; + } + if (!NodeStatus.isLocked(node.getNodeStatus())) { + Optional lockStr = + engineLocker.lockEngine(node, (long) AMConfiguration.ENGINE_LOCKER_MAX_TIME.getValue()); + if (!lockStr.isPresent()) { + throw new WarnException( + AMConstant.ENGINE_ERROR_CODE, + String.format( + "Failed to request lock from engine by reuse %s", node.getServiceInstance())); + } + node.setLock(lockStr.get()); + return node; + } else { + return null; + } + } + + /** + * TODO use Engine需要考虑流式引擎的场景，后续需要通过Label加额外的处理 + * + * @param engineNode + * @param timeout + * @return + */ + @Override + public EngineNode useEngine(EngineNode engineNode, long timeout) { + RetryHandler retryHandler = new DefaultRetryHandler(); + retryHandler.addRetryException(feign.RetryableException.class); + retryHandler.addRetryException(UndeclaredThrowableException.class); + EngineNode node = retryHandler.retry(() -> getEngineNodeInfo(engineNode), "getEngineNodeInfo"); + if (!NodeStatus.isAvailable(node.getNodeStatus())) { + return null; + } + if (!NodeStatus.isLocked(node.getNodeStatus())) { + Optional lockStr = engineLocker.lockEngine(node, timeout); + if (!lockStr.isPresent()) { + throw new WarnException( + AMConstant.ENGINE_ERROR_CODE, + String.format("Failed to request lock from engine %s", node.getServiceInstance())); + } + node.setLock(lockStr.get()); + return node; + } else { + return null; + } + } + + @Override + public EngineNode useEngine(EngineNode engineNode) { + return useEngine(engineNode, (long) AMConfiguration.ENGINE_LOCKER_MAX_TIME.getValue()); + } + + /** + * Get detailed engine information from the persistence //TODO 是否增加owner到node + * + * @param scoreServiceInstances + * @return + */ + @Override + public EngineNode[] getEngineNodes(ScoreServiceInstance[] scoreServiceInstances) { + if (scoreServiceInstances == null || scoreServiceInstances.length == 0) { + return null; + } + EngineNode[] engineNodes = + Arrays.stream(scoreServiceInstances) + .map( + scoreServiceInstance -> { + AMEngineNode engineNode = new AMEngineNode(); + engineNode.setScore(scoreServiceInstance.getScore()); + engineNode.setServiceInstance(scoreServiceInstance.getServiceInstance()); + return engineNode; + }) + .toArray(EngineNode[]::new); + // 1. add nodeMetrics 2 add RM info + ServiceInstance[] serviceInstances = + Arrays.stream(scoreServiceInstances) + .map(ScoreServiceInstance::getServiceInstance) + .toArray(ServiceInstance[]::new); + ResourceInfo resourceInfo = resourceManager.getResourceInfo(serviceInstances); + + List nodeMetrics = + nodeMetricManagerPersistence.getNodeMetrics( + Arrays.stream(engineNodes).collect(Collectors.toList())); + Arrays.stream(engineNodes) + .forEach( + engineNode -> { + Optional optionMetrics = + nodeMetrics.stream() + .filter( + nodeMetric -> + nodeMetric + .getServiceInstance() + .equals(engineNode.getServiceInstance())) + .findFirst(); + + Optional optionRMNode = + resourceInfo.getResourceInfo().stream() + .filter( + resourceNode -> + resourceNode + .getServiceInstance() + .equals(engineNode.getServiceInstance())) + .findFirst(); + + optionMetrics.ifPresent( + metrics -> metricsConverter.fillMetricsToNode(engineNode, metrics)); + optionRMNode.ifPresent( + rmNode -> engineNode.setNodeResource(rmNode.getNodeResource())); + }); + return engineNodes; + } + + /** + * add info to persistence + * + * @param engineNode + */ + @Override + public void addEngineNode(EngineNode engineNode) { + try { + nodeManagerPersistence.addEngineNode(engineNode); + } catch (PersistenceErrorException e) { + // e.printStackTrace(); + } + // init metric + nodeMetricManagerPersistence.addOrupdateNodeMetrics( + metricsConverter.getInitMetric(engineNode.getServiceInstance())); + } + + /** + * delete info to persistence + * + * @param engineNode + */ + @Override + public void deleteEngineNode(EngineNode engineNode) { + nodeManagerPersistence.deleteEngineNode(engineNode); + } + + @Override + public EngineNode getEngineNode(ServiceInstance serviceInstance) { + return nodeManagerPersistence.getEngineNode(serviceInstance); + } + + /** + * 1.serviceInstance中取出instance（实际是ticketId） 2.update serviceInstance 表，包括 + * instance替换，替换mark，owner，updator，creator的空值，更新updateTime 3.update engine_em关联表 4.update label + * ticket_id ==> instance + * + * @param serviceInstance + * @param engineNode + */ + @Override + public void updateEngineNode(ServiceInstance serviceInstance, EngineNode engineNode) { + try { + nodeManagerPersistence.updateEngineNode(serviceInstance, engineNode); + } catch (PersistenceErrorException e) { + // e.printStackTrace(); + } catch (LinkisRetryException e) { + // e.printStackTrace(); + } + + EngineInstanceLabel engineLabel = labelBuilderFactory.createLabel(EngineInstanceLabel.class); + engineLabel.setInstance(engineNode.getServiceInstance().getInstance()); + engineLabel.setServiceName(engineNode.getServiceInstance().getApplicationName()); + + EngineInstanceLabel oldEngineLabel = labelBuilderFactory.createLabel(EngineInstanceLabel.class); + oldEngineLabel.setInstance(serviceInstance.getInstance()); + oldEngineLabel.setServiceName(engineNode.getServiceInstance().getApplicationName()); + PersistenceLabel oldPersistenceLabel = + labelBuilderFactory.convertLabel(oldEngineLabel, PersistenceLabel.class); + PersistenceLabel label = + labelManagerPersistence.getLabelByKeyValue( + oldPersistenceLabel.getLabelKey(), oldPersistenceLabel.getStringValue()); + + PersistenceLabel persistenceLabel = + labelBuilderFactory.convertLabel(engineLabel, PersistenceLabel.class); + persistenceLabel.setLabelValueSize(persistenceLabel.getValue().size()); + labelManagerPersistence.updateLabel(label.getId(), persistenceLabel); + } + + public EngineOperateResponse executeOperation( + EngineNode engineNode, EngineOperateRequest request) { + EngineNodePointer engine = nodePointerBuilder.buildEngineNodePointer(engineNode); + return engine.executeOperation(request); + } + + public EngineNode getEngineNodeInfo(ServiceInstance serviceInstance) { + EngineNode engineNode = getEngineNode(serviceInstance); + if (Objects.isNull(engineNode)) { + throw new WarnException( + AMErrorCode.NOT_EXISTS_ENGINE_CONN.getErrorCode(), + AMErrorCode.NOT_EXISTS_ENGINE_CONN.getErrorDesc()); + } + + if (engineNode.getNodeStatus() == null) { + NodeMetrics nodeMetric = nodeMetricManagerPersistence.getNodeMetrics(engineNode); + if (Objects.nonNull(nodeMetric) && Objects.nonNull(nodeMetric.getStatus())) { + engineNode.setNodeStatus(NodeStatus.values()[nodeMetric.getStatus()]); + } else { + engineNode.setNodeStatus(NodeStatus.Starting); + } + } + return engineNode; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EMNodeManager.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EMNodeManager.java new file mode 100644 index 00000000000..8cb8362ecc4 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EMNodeManager.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.manager; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.manager.common.entity.node.EMNode; +import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.common.entity.node.ScoreServiceInstance; +import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; +import org.apache.linkis.manager.common.protocol.em.ECMOperateResponse; +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; +import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest; + +import java.util.List; + +public interface EMNodeManager { + + void emRegister(EMNode emNode); + + List listEngines(EMNode emNode); + + List listUserEngines(EMNode emNode, String user); + + List listUserNodes(String user); + + /** + * Get detailed em information from the persistence + * + * @param scoreServiceInstances + * @return + */ + EMNode[] getEMNodes(ScoreServiceInstance[] scoreServiceInstances); + + EMNode getEM(ServiceInstance serviceInstance); + + void stopEM(EMNode emNode); + + void deleteEM(EMNode emNode); + + void pauseEM(ServiceInstance serviceInstance); + + /** + * 1. request engineManager to launch engine 2. persist engine info + * + * @param engineConnLaunchRequest engine launch request + * @param emNode ecm node + * @return engine node + */ + EngineNode createEngine(EngineConnLaunchRequest engineConnLaunchRequest, EMNode emNode); + + void stopEngine(EngineStopRequest engineStopRequest, EMNode emNode); + + void addEMNodeInstance(EMNode emNode); + + void initEMNodeMetrics(EMNode emNode); + + ECMOperateResponse executeOperation(EMNode ecmNode, ECMOperateRequest request); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EngineNodeManager.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EngineNodeManager.java new file mode 100644 index 00000000000..083000c3e42 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EngineNodeManager.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.manager; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; +import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.common.entity.node.ScoreServiceInstance; +import org.apache.linkis.manager.common.protocol.engine.EngineOperateRequest; +import org.apache.linkis.manager.common.protocol.engine.EngineOperateResponse; + +import java.util.List; + +public interface EngineNodeManager { + + List listEngines(String user); + + EngineNode getEngineNode(ServiceInstance serviceInstance); + + EngineNode getEngineNodeInfo(EngineNode engineNode); + + EngineNode getEngineNodeInfo(ServiceInstance serviceInstance); + + EngineNode getEngineNodeInfoByDB(EngineNode engineNode); + + /** + * Get detailed engine information from the persistence + * + * @param scoreServiceInstances + * @return + */ + EngineNode[] getEngineNodes(ScoreServiceInstance[] scoreServiceInstances); + + void updateEngineStatus( + ServiceInstance serviceInstance, NodeStatus fromState, NodeStatus toState); + + void addEngineNode(EngineNode engineNode); + + void updateEngineNode(ServiceInstance serviceInstance, EngineNode engineNode); + + void updateEngine(EngineNode engineNode); + + void deleteEngineNode(EngineNode engineNode); + + EngineNode switchEngine(EngineNode engineNode); + + EngineNode reuseEngine(EngineNode engineNode); + + EngineNode useEngine(EngineNode engineNode); + + EngineNode useEngine(EngineNode engineNode, long timeout); + + EngineOperateResponse executeOperation(EngineNode engineNode, EngineOperateRequest request); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/AbstractNodePointer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/AbstractNodePointer.java new file mode 100644 index 00000000000..80390d0883c --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/AbstractNodePointer.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.pointer; + +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; +import org.apache.linkis.manager.common.entity.node.Node; +import org.apache.linkis.manager.common.protocol.node.*; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.manager.service.common.pointer.NodePointer; +import org.apache.linkis.rpc.Sender; + +public abstract class AbstractNodePointer implements NodePointer { + + protected Sender getSender() { + return Sender.getSender(getNode().getServiceInstance()); + } + + /** + * 向对应的Node发送请求获取节点状态 + * + * @return + */ + @Override + public NodeStatus getNodeStatus() { + Sender sender = getSender(); + try { + ResponseNodeStatus responseStatus = (ResponseNodeStatus) sender.ask(new RequestNodeStatus()); + return responseStatus.getNodeStatus(); + } catch (WarnException e) { + throw e; + } + } + + /** + * 向对应的Node发送请求获取节点心跳信息 + * + * @return + */ + @Override + public NodeHeartbeatMsg getNodeHeartbeatMsg() { + Sender sender = getSender(); + try { + NodeHeartbeatMsg heartbeatMsg = (NodeHeartbeatMsg) sender.ask(new NodeHeartbeatRequest()); + return heartbeatMsg; + } catch (WarnException e) { + throw e; + } + } + + /** + * 向对应的Node发送Kill 请求 + * + * @return + */ + @Override + public void stopNode() { + Sender sender = getSender(); + sender.send(new StopNodeRequest()); + } + + /** + * 向对应的Node Label 更新请求 + * + * @return + */ + @Override + public void updateLabels(Label[] labels) {} + + @Override + public void updateNodeHealthyRequest(NodeHealthyRequest nodeHealthyRequest) { + getSender().send(nodeHealthyRequest); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof Node) { + Node nodeB = (Node) obj; + return getNode().getServiceInstance().equals(nodeB.getServiceInstance()); + } + return false; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.java new file mode 100644 index 00000000000..dacaeed4e98 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.pointer; + +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.manager.common.constant.AMConstant; +import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.common.entity.node.Node; +import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; +import org.apache.linkis.manager.common.protocol.em.ECMOperateResponse; +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; +import org.apache.linkis.manager.common.protocol.engine.EngineStopResponse; +import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest; +import org.apache.linkis.manager.service.common.pointer.EMNodPointer; +import org.apache.linkis.server.BDPJettyServerHelper; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DefaultEMNodPointer extends AbstractNodePointer implements EMNodPointer { + private static final Logger logger = LoggerFactory.getLogger(DefaultEMNodPointer.class); + + private Node node; + + public DefaultEMNodPointer(Node node) { + this.node = node; + } + + /** + * 与该远程指针关联的node信息 + * + * @return + */ + @Override + public Node getNode() { + return node; + } + + @Override + public EngineNode createEngine(EngineConnLaunchRequest engineConnLaunchRequest) { + logger.info("Start to createEngine ask em " + getNode().getServiceInstance()); + Object result = getSender().ask(engineConnLaunchRequest); + if (result instanceof EngineNode) { + EngineNode engineNode = (EngineNode) result; + logger.info( + "Succeed to createEngine ask em " + + getNode().getServiceInstance() + + ", engineNode " + + engineNode); + return engineNode; + } else { + throw new WarnException( + AMConstant.ENGINE_ERROR_CODE, + "Failed to createEngine ask em " + getNode().getServiceInstance()); + } + } + + @Override + public void stopEngine(EngineStopRequest engineStopRequest) { + Object result = getSender().ask(engineStopRequest); + if (result instanceof EngineStopResponse) { + EngineStopResponse engineStopResponse = (EngineStopResponse) result; + if (!engineStopResponse.getStopStatus()) { + logger.info( + "Kill engine : " + + engineStopRequest.getServiceInstance().toString() + + " failed, because " + + engineStopResponse.getMsg() + + " . Will ask engine to suicide."); + } else { + logger.info( + "Succeed to kill engine " + engineStopRequest.getServiceInstance().toString() + "."); + } + } else { + logger.warn( + "Ask em : " + + getNode().getServiceInstance().toString() + + " to kill engine : " + + engineStopRequest.getServiceInstance().toString() + + " failed, response is : " + + BDPJettyServerHelper.gson().toJson(result) + + "."); + } + } + + @Override + public ECMOperateResponse executeOperation(ECMOperateRequest ecmOperateRequest) { + Object result = getSender().ask(ecmOperateRequest); + if (result instanceof ECMOperateResponse) { + return (ECMOperateResponse) result; + } else { + throw new WarnException(AMConstant.ENGINE_ERROR_CODE, "Failed to execute ECM operation."); + } + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.java new file mode 100644 index 00000000000..56c77b315a2 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.pointer; + +import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.manager.common.entity.node.Node; +import org.apache.linkis.manager.common.protocol.RequestEngineLock; +import org.apache.linkis.manager.common.protocol.RequestEngineUnlock; +import org.apache.linkis.manager.common.protocol.ResponseEngineLock; +import org.apache.linkis.manager.common.protocol.engine.EngineOperateRequest; +import org.apache.linkis.manager.common.protocol.engine.EngineOperateResponse; +import org.apache.linkis.manager.service.common.pointer.EngineNodePointer; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DefaultEngineNodPointer extends AbstractNodePointer implements EngineNodePointer { + private static final Logger logger = LoggerFactory.getLogger(DefaultEngineNodPointer.class); + + private Node node; + + public DefaultEngineNodPointer(Node node) { + this.node = node; + } + + @Override + public Node getNode() { + return node; + } + + @Override + public Optional lockEngine(RequestEngineLock requestEngineLock) { + Object result = getSender().ask(requestEngineLock); + if (result instanceof ResponseEngineLock) { + ResponseEngineLock responseEngineLock = (ResponseEngineLock) result; + if (responseEngineLock.getLockStatus()) { + return Optional.of(responseEngineLock.getLock()); + } else { + logger.info( + "Failed to get locker," + + node.getServiceInstance() + + ": " + + responseEngineLock.getMsg()); + return Optional.empty(); + } + } else { + return Optional.empty(); + } + } + + @Override + public void releaseLock(RequestEngineUnlock requestEngineUnlock) { + getSender().send(requestEngineUnlock); + } + + @Override + public EngineOperateResponse executeOperation(EngineOperateRequest engineOperateRequest) { + Object result = getSender().ask(engineOperateRequest); + if (result instanceof EngineOperateResponse) { + return (EngineOperateResponse) result; + } else { + throw new WarnException(-1, "Illegal response of operation."); + } + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultNodePointerBuilder.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultNodePointerBuilder.java new file mode 100644 index 00000000000..03e0bc8087e --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultNodePointerBuilder.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.pointer; + +import org.apache.linkis.manager.common.entity.node.EMNode; +import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.service.common.pointer.EMNodPointer; +import org.apache.linkis.manager.service.common.pointer.EngineNodePointer; +import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder; + +import org.springframework.stereotype.Component; + +@Component +public class DefaultNodePointerBuilder implements NodePointerBuilder { + @Override + public EMNodPointer buildEMNodePointer(EMNode node) { + return new DefaultEMNodPointer(node); + } + + @Override + public EngineNodePointer buildEngineNodePointer(EngineNode node) { + return new DefaultEngineNodPointer(node); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/recycle/AssignNodeRuleExecutor.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/recycle/AssignNodeRuleExecutor.java new file mode 100644 index 00000000000..f9a3acce4f2 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/recycle/AssignNodeRuleExecutor.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.recycle; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.manager.common.entity.recycle.AssignNodeRule; +import org.apache.linkis.manager.common.entity.recycle.RecyclingRule; + +import org.springframework.stereotype.Component; + +@Component +public class AssignNodeRuleExecutor implements RecyclingRuleExecutor { + @Override + public boolean ifAccept(RecyclingRule recyclingRule) { + return recyclingRule instanceof AssignNodeRule; + } + + @Override + public ServiceInstance[] executeRule(RecyclingRule recyclingRule) { + if (recyclingRule instanceof AssignNodeRule) { + AssignNodeRule assignNodeRule = (AssignNodeRule) recyclingRule; + return new ServiceInstance[] {assignNodeRule.serviceInstance()}; + } else { + return null; + } + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.java similarity index 77% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.java index 8a4db6fb03d..ab53aa82812 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.java @@ -15,15 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.manager.am.recycle +package org.apache.linkis.manager.am.recycle; -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.manager.common.entity.recycle.RecyclingRule +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.manager.common.entity.recycle.RecyclingRule; -trait RecyclingRuleExecutor { +public interface RecyclingRuleExecutor { - def ifAccept(recyclingRule: RecyclingRule): Boolean - - def executeRule(recyclingRule: RecyclingRule): Array[ServiceInstance] + boolean ifAccept(RecyclingRule recyclingRule); + ServiceInstance[] executeRule(RecyclingRule recyclingRule); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java index 05cda79c57d..6a1d65d8b50 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java @@ -35,9 +35,8 @@ import org.apache.linkis.manager.common.entity.node.EMNode; import org.apache.linkis.manager.common.entity.node.EngineNode; import org.apache.linkis.manager.common.entity.persistence.ECResourceInfoRecord; -import org.apache.linkis.manager.common.protocol.OperateRequest$; +import org.apache.linkis.manager.common.protocol.OperateRequest; import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; -import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest$; import org.apache.linkis.manager.common.protocol.em.ECMOperateResponse; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactory; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; @@ -99,7 +98,7 @@ public class EMRestfulApi { private Logger logger = LoggerFactory.getLogger(EMRestfulApi.class); - private String[] adminOperations = AMConfiguration.ECM_ADMIN_OPERATIONS().getValue().split(","); + private String[] adminOperations = AMConfiguration.ECM_ADMIN_OPERATIONS.getValue().split(","); private void checkAdmin(String userName) throws AMErrorException { if (Configuration.isNotAdmin(userName)) { @@ -149,7 +148,7 @@ public Message listAllEMs( stream = stream.filter( em -> { - List