This is an automated email from the ASF dual-hosted git repository. aiceflower pushed a commit to branch release-0.9.4 in repository https://gitbox.apache.org/repos/asf/linkis.git
commit 918e2a3707053be36d0b96884dbf6e9c49318caf Author: [email protected] <[email protected]> AuthorDate: Sun Feb 23 16:51:24 2020 +0800 #299 --- datasourcemanager/server/pom.xml | 131 +++++++++++++++++++++ .../validate/DataSourceParameterValidator.java | 84 +++++++++++++ .../core/validate/ParameterValidateException.java | 35 ++++++ .../core/validate/ParameterValidateStrategy.java | 39 ++++++ .../core/validate/ParameterValidator.java | 47 ++++++++ .../strategy/RegExpParameterValidateStrategy.java | 62 ++++++++++ .../strategy/TypeParameterValidateStrategy.java | 69 +++++++++++ .../datasourcemanager/core/vo/DataSourceEnvVo.java | 51 ++++++++ .../datasourcemanager/core/vo/DataSourceVo.java | 82 +++++++++++++ .../datasourcemanager/core/vo/PageViewVo.java | 39 ++++++ .../server/src/main/resources/application.yml | 24 ++++ .../server/src/main/resources/linkis.properties | 35 ++++++ .../server/src/main/resources/log4j.properties | 33 ++++++ .../server/src/main/resources/log4j2.xml | 35 ++++++ .../core/receivers/DsmReceiver.scala | 91 ++++++++++++++ 15 files changed, 857 insertions(+) diff --git a/datasourcemanager/server/pom.xml b/datasourcemanager/server/pom.xml new file mode 100644 index 0000000000..ea7dff386e --- /dev/null +++ b/datasourcemanager/server/pom.xml @@ -0,0 +1,131 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + ~ Copyright 2019 WeBank + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>linkis</artifactId> + <groupId>com.webank.wedatasphere.linkis</groupId> + <version>0.9.3</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>linkis-datasourcemanager-server</artifactId> + + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + + <dependencies> + <dependency> + <groupId>com.webank.wedatasphere.linkis</groupId> + <artifactId>linkis-module</artifactId> + <version>${linkis.version}</version> + <exclusions> + <exclusion> + <artifactId>asm</artifactId> + <groupId>org.ow2.asm</groupId> + </exclusion> + </exclusions> + </dependency> + <!-- data source manager common dependency--> + <dependency> + <groupId>com.webank.wedatasphere.linkis</groupId> + <artifactId>linkis-datasourcemanager-common</artifactId> + <version>${linkis.version}</version> + <exclusions> + <exclusion> + <artifactId>asm</artifactId> + <groupId>org.ow2.asm</groupId> + </exclusion> + </exclusions> + </dependency> + <!--jersey bean validation--> + <dependency> + <groupId>org.glassfish.jersey.ext</groupId> + <artifactId>jersey-bean-validation</artifactId> + <version>${jersey.version}</version> + <exclusions> + <exclusion> + <artifactId>javax.ws.rs-api</artifactId> + <groupId>javax.ws.rs</groupId> + </exclusion> + <exclusion> + <artifactId>hk2-locator</artifactId> + <groupId>org.glassfish.hk2</groupId> + </exclusion> + <exclusion> + <artifactId>hk2-api</artifactId> + <groupId>org.glassfish.hk2</groupId> + </exclusion> + </exclusions> + </dependency> + <!--bml client--> + <dependency> + <groupId>com.webank.wedatasphere.linkis</groupId> + <artifactId>linkis-bmlclient</artifactId> + <version>${linkis.version}</version> + </dependency> + + <dependency> + <groupId>com.webank.wedatasphere.linkis</groupId> + <artifactId>linkis-mybatis</artifactId> + <version>${linkis.version}</version> + </dependency> + <!--rpc--> + <dependency> + <groupId>com.webank.wedatasphere.linkis</groupId> + <artifactId>linkis-cloudRPC</artifactId> + <version>${linkis.version}</version> + </dependency> + <!--Metadata common--> + <dependency> + <groupId>com.webank.wedatasphere.linkis</groupId> + <artifactId>linkis-metadatamanager-common</artifactId> + <version>${linkis.version}</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-deploy-plugin</artifactId> + </plugin> + + <plugin> + <groupId>net.alchim31.maven</groupId> + <artifactId>scala-maven-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + </plugin> + </plugins> + <resources> + <resource> + <directory>src/main/resources</directory> + </resource> + <resource> + <directory>src/main/java</directory> + <includes> + <include>**/*.xml</include> + </includes> + </resource> + </resources> + <finalName>${project.artifactId}-${project.version}</finalName> + </build> +</project> diff --git a/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/DataSourceParameterValidator.java b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/DataSourceParameterValidator.java new file mode 100644 index 0000000000..29e9e72ef7 --- /dev/null +++ b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/DataSourceParameterValidator.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.datasourcemanager.core.validate; + +import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition; +import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.strategy.RegExpParameterValidateStrategy; +import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.strategy.TypeParameterValidateStrategy; +import org.apache.commons.lang.StringUtils; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * @author georgeqiao + * 2020/02/11 + */ + + +@Component +public class DataSourceParameterValidator implements ParameterValidator { + @PostConstruct + public void initToRegister(){ + registerStrategy(new TypeParameterValidateStrategy()); + registerStrategy(new RegExpParameterValidateStrategy()); + } + /** + * strategies list + */ + private List<ParameterValidateStrategy> strategies = new ArrayList<>(); + + @Override + public void registerStrategy(ParameterValidateStrategy strategy) { + strategies.add(strategy); + } + + @Override + public void validate(List<DataSourceParamKeyDefinition> paramKeyDefinitions, + Map<String, Object> parameters) throws ParameterValidateException{ + for(DataSourceParamKeyDefinition paramKeyDefinition : paramKeyDefinitions){ + String keyName = paramKeyDefinition.getKey(); + Object keyValue = parameters.get(keyName); + DataSourceParamKeyDefinition.ValueType valueType = paramKeyDefinition.getValueType(); + if(null == keyValue){ + String defaultValue = paramKeyDefinition.getDefaultValue(); + if(StringUtils.isNotBlank(defaultValue) && + valueType == DataSourceParamKeyDefinition.ValueType.SELECT){ + defaultValue = defaultValue.split(",")[0].trim(); + } + keyValue = defaultValue; + } + if(null == keyValue || StringUtils.isBlank(String.valueOf(keyValue))){ + if(paramKeyDefinition.isRequire()) { + throw new ParameterValidateException("Param Validate Failed[参数校验出错], [the value of key: '" + + keyName + " cannot be blank']"); + } + continue; + } + for(ParameterValidateStrategy validateStrategy : strategies){ + if(validateStrategy.accept(valueType)) { + validateStrategy.validate(paramKeyDefinition, keyValue); + } + } + } + } + + @Override + public List<ParameterValidateStrategy> getStrategies() { + return strategies; + } +} diff --git a/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidateException.java b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidateException.java new file mode 100644 index 0000000000..448cd734f0 --- /dev/null +++ b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidateException.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.datasourcemanager.core.validate; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; + +import static com.webank.wedatasphere.linkis.datasourcemanager.common.ServiceErrorCode.PARAM_VALIDATE_ERROR; + + +/** + * @author georgeqiao + * 2020/02/11 + */ + + +public class ParameterValidateException extends ErrorException { + public ParameterValidateException(String desc) { + super(PARAM_VALIDATE_ERROR.getValue(), desc); + } + + public ParameterValidateException(String desc, String ip, int port, String serviceKind) { + super(PARAM_VALIDATE_ERROR.getValue(), desc, ip, port, serviceKind); + } +} diff --git a/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidateStrategy.java b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidateStrategy.java new file mode 100644 index 0000000000..1d25b30190 --- /dev/null +++ b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidateStrategy.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.datasourcemanager.core.validate; + +import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition; + +/** + * Parameter validate strategy + * @author georgeqiao + * 2020/02/11 + */ +public interface ParameterValidateStrategy { + /** + * If accept value type + * @param valueType validate type + * @return result + */ + boolean accept(DataSourceParamKeyDefinition.ValueType valueType); + + /** + * Validate actual value by key definition + * @param keyDefinition key definition + * @param actualValue actual value + * @return new value + */ + Object validate(DataSourceParamKeyDefinition keyDefinition, + Object actualValue) throws ParameterValidateException; +} diff --git a/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidator.java b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidator.java new file mode 100644 index 0000000000..89ff0f76c4 --- /dev/null +++ b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidator.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.datasourcemanager.core.validate; + +import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition; + +import java.util.List; +import java.util.Map; + +/** + * @author georgeqiao + * 2020/02/11 + */ + + +public interface ParameterValidator { + /** + * Register validate strategy + * @param strategy strategy + */ + void registerStrategy(ParameterValidateStrategy strategy); + + /** + * Validate parameter dictionary + * @param paramKeyDefinitions definitions + * @param parameters parameters + */ + void validate(List<DataSourceParamKeyDefinition> paramKeyDefinitions, + Map<String, Object> parameters) throws ParameterValidateException; + + /** + * Get all strategies + * @return + */ + List<ParameterValidateStrategy> getStrategies(); +} diff --git a/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/strategy/RegExpParameterValidateStrategy.java b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/strategy/RegExpParameterValidateStrategy.java new file mode 100644 index 0000000000..64c639a0e3 --- /dev/null +++ b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/strategy/RegExpParameterValidateStrategy.java @@ -0,0 +1,62 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.datasourcemanager.core.validate.strategy; + +import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition; +import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.ParameterValidateException; +import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.ParameterValidateStrategy; +import org.apache.commons.lang.StringUtils; + +import java.util.List; + +/** + * RegExpression validate strategy + * @author georgeqiao + * 2020/02/11 + */ +public class RegExpParameterValidateStrategy implements ParameterValidateStrategy { + + + @Override + public boolean accept(DataSourceParamKeyDefinition.ValueType valueType) { + return valueType == DataSourceParamKeyDefinition.ValueType.EMAIL || valueType == DataSourceParamKeyDefinition.ValueType.TEXT || valueType == DataSourceParamKeyDefinition.ValueType.LIST; + } + + @Override + public Object validate(DataSourceParamKeyDefinition keyDefinition, + Object actualValue) throws ParameterValidateException { + String valueRegex = keyDefinition.getValueRegex(); + if(StringUtils.isNotBlank(valueRegex)){ + if(actualValue instanceof List){ + List valueList = ((List)actualValue); + for(Object value : valueList){ + match(keyDefinition.getKey(), keyDefinition.getName(), String.valueOf(value), valueRegex); + } + }else{ + match(keyDefinition.getKey(), keyDefinition.getName(), String.valueOf(actualValue), valueRegex); + } + + } + return actualValue; + } + + private void match(String key, String name, String value, String valueRegex) throws ParameterValidateException { + boolean match = String.valueOf(value).matches(valueRegex); + if(!match){ + throw new ParameterValidateException("Param Validate Failed[参数校验出错], [the value: '" + + String.valueOf(value) + "' to key: '" + + key + "(" + name +")' doesn't match]"); + } + } +} diff --git a/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/strategy/TypeParameterValidateStrategy.java b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/strategy/TypeParameterValidateStrategy.java new file mode 100644 index 0000000000..bbc0af75ab --- /dev/null +++ b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/strategy/TypeParameterValidateStrategy.java @@ -0,0 +1,69 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.datasourcemanager.core.validate.strategy; + +import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition; +import com.webank.wedatasphere.linkis.datasourcemanager.core.formdata.FormStreamContent; +import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.ParameterValidateException; +import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.ParameterValidateStrategy; +import com.webank.wedatasphere.linkis.metadatamanager.common.Json; + +import java.util.List; +import java.util.Map; + +import static com.webank.wedatasphere.linkis.datasourcemanager.core.formdata.CustomMultiPartFormDataTransformer.*; + +/** + * Type validate strategy + * @author georgeqiao + * 2020/02/13 + */ +public class TypeParameterValidateStrategy implements ParameterValidateStrategy { + @Override + public boolean accept(DataSourceParamKeyDefinition.ValueType valueType) { + //Accept all value + return true; + } + + @Override + public Object validate(DataSourceParamKeyDefinition keyDefinition, + Object actualValue) throws ParameterValidateException { + DataSourceParamKeyDefinition.ValueType valueType = keyDefinition.getValueType(); + Class<?> javaType = valueType.getJavaType(); + if(valueType == DataSourceParamKeyDefinition.ValueType.FILE ){ + if(!actualValue.getClass().equals(FormStreamContent.class)){ + throw new ParameterValidateException("Param Validate Failed[参数校验出错], [the value of '" + + keyDefinition.getKey() + "' must be 'File']"); + } + return actualValue; + } + if(!javaType.isAssignableFrom(actualValue.getClass())){ + try { + if(javaType.equals(List.class)){ + return Json.fromJson(String.valueOf(actualValue), List.class, String.class); + }else if(javaType.equals(Map.class)){ + return Json.fromJson(String.valueOf(actualValue), Map.class, String.class, String.class); + }else if(PrimitiveUtils.isPrimitive(javaType)){ + return PrimitiveUtils.primitiveTypeConverse(actualValue, javaType); + } + }catch(Exception e){ + throw new ParameterValidateException("Param Validate Failed[参数校验出错], [type of value: '" + + actualValue + "' is not '" + javaType.getSimpleName() + "']"); + } + throw new ParameterValidateException("Param Validate Failed[参数校验出错], [type of value: '" + + actualValue + "' is not '" + javaType.getSimpleName() + "']"); + } + return actualValue; + } +} diff --git a/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/DataSourceEnvVo.java b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/DataSourceEnvVo.java new file mode 100644 index 0000000000..10a6b80ea5 --- /dev/null +++ b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/DataSourceEnvVo.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.datasourcemanager.core.vo; + + +/** + * @author georgeqiao + * 2020/02/11 + */ +public class DataSourceEnvVo extends PageViewVo{ + + private String envName; + + private Long dataSourceTypeId; + + public DataSourceEnvVo(){ + + } + + public DataSourceEnvVo(String envName, Long dataSourceTypeId){ + this.envName = envName; + this.dataSourceTypeId = dataSourceTypeId; + } + + public String getEnvName() { + return envName; + } + + public void setEnvName(String envName) { + this.envName = envName; + } + + public Long getDataSourceTypeId() { + return dataSourceTypeId; + } + + public void setDataSourceTypeId(Long dataSourceTypeId) { + this.dataSourceTypeId = dataSourceTypeId; + } +} diff --git a/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/DataSourceVo.java b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/DataSourceVo.java new file mode 100644 index 0000000000..7a8d2e4b2e --- /dev/null +++ b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/DataSourceVo.java @@ -0,0 +1,82 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.datasourcemanager.core.vo; + + +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * @author georgeqiao + * 2020/02/11 + */ +public class DataSourceVo extends PageViewVo{ + + private String dataSourceName; + + private Long dataSourceTypeId; + + private List<String> createIdentifyList = new ArrayList<>(); + + private String createSystem; + + public DataSourceVo(){ + + } + + public DataSourceVo(String dataSourceName, Long dataSourceTypeId, + String createIdentifies, String createSystem){ + this.dataSourceName = dataSourceName; + this.dataSourceTypeId = dataSourceTypeId; + if(StringUtils.isNotBlank(createIdentifies)){ + createIdentifyList.addAll(Arrays.asList(createIdentifies.split(","))); + } + this.createSystem = createSystem; + } + public String getDataSourceName() { + return dataSourceName; + } + + public void setDataSourceName(String dataSourceName) { + this.dataSourceName = dataSourceName; + } + + public Long getDataSourceTypeId() { + return dataSourceTypeId; + } + + public void setDataSourceTypeId(Long dataSourceTypeId) { + this.dataSourceTypeId = dataSourceTypeId; + } + + + public String getCreateSystem() { + return createSystem; + } + + public void setCreateSystem(String createSystem) { + this.createSystem = createSystem; + } + + public List<String> getCreateIdentifyList() { + return createIdentifyList; + } + + public void setCreateIdentifyList(List<String> createIdentifyList) { + this.createIdentifyList = createIdentifyList; + } +} diff --git a/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/PageViewVo.java b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/PageViewVo.java new file mode 100644 index 0000000000..9db3b1f79a --- /dev/null +++ b/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/PageViewVo.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.datasourcemanager.core.vo; + +/** + * @author georgeqiao + * 2020/02/11 + */ +public class PageViewVo { + private int currentPage; + private int pageSize; + + public int getCurrentPage() { + return currentPage; + } + + public void setCurrentPage(int currentPage) { + this.currentPage = currentPage; + } + + public int getPageSize() { + return pageSize; + } + + public void setPageSize(int pageSize) { + this.pageSize = pageSize; + } +} diff --git a/datasourcemanager/server/src/main/resources/application.yml b/datasourcemanager/server/src/main/resources/application.yml new file mode 100644 index 0000000000..eae5f49758 --- /dev/null +++ b/datasourcemanager/server/src/main/resources/application.yml @@ -0,0 +1,24 @@ +server: + port: 8196 +spring: + application: + name: dsm-server + +eureka: + client: + serviceUrl: + defaultZone: http://${ip}:${port}/eureka/ + instance: + metadata-map: + test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml + + + diff --git a/datasourcemanager/server/src/main/resources/linkis.properties b/datasourcemanager/server/src/main/resources/linkis.properties new file mode 100644 index 0000000000..70b45bd1f3 --- /dev/null +++ b/datasourcemanager/server/src/main/resources/linkis.properties @@ -0,0 +1,35 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +wds.linkis.server.mybatis.mapperLocations=classpath:com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.linkis.datasourcemanager.common.domain +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.linkis.datasourcemanager.core.dao + +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.linkis.datasourcemanager.core.restful + +#sit +wds.linkis.server.version=v1 + +#test +wds.linkis.test.mode=true +wds.linkis.test.user= +wds.linkis.server.mybatis.datasource.url= +wds.linkis.server.mybatis.datasource.username= +wds.linkis.server.mybatis.datasource.password= + +#dsm +wds.linkis.server.dsm.admin.users= + +#bml +wds.linkis.gateway.ip= +wds.linkis.gateway.port= \ No newline at end of file diff --git a/datasourcemanager/server/src/main/resources/log4j.properties b/datasourcemanager/server/src/main/resources/log4j.properties new file mode 100644 index 0000000000..d5ee44b86b --- /dev/null +++ b/datasourcemanager/server/src/main/resources/log4j.properties @@ -0,0 +1,33 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/datasourcemanager/server/src/main/resources/log4j2.xml b/datasourcemanager/server/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..1c68190669 --- /dev/null +++ b/datasourcemanager/server/src/main/resources/log4j2.xml @@ -0,0 +1,35 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Copyright 2019 WeBank + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<configuration status="error" monitorInterval="30"> + <appenders> + <Console name="Console" target="SYSTEM_OUT"> + <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/> + <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/> + </Console> + <RollingFile name="RollingFile" fileName="logs/linkis.log" + filePattern="logs/$${date:yyyy-MM}/linkis-log-%d{yyyy-MM-dd}-%i.log"> + <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/> + <SizeBasedTriggeringPolicy size="100MB"/> + <DefaultRolloverStrategy max="20"/> + </RollingFile> + </appenders> + <loggers> + <root level="INFO"> + <appender-ref ref="RollingFile"/> + <appender-ref ref="Console"/> + </root> + </loggers> +</configuration> + diff --git a/datasourcemanager/server/src/main/scala/com/webank/wedatasphere/linkis/datasourcemanager/core/receivers/DsmReceiver.scala b/datasourcemanager/server/src/main/scala/com/webank/wedatasphere/linkis/datasourcemanager/core/receivers/DsmReceiver.scala new file mode 100644 index 0000000000..58f106ec2c --- /dev/null +++ b/datasourcemanager/server/src/main/scala/com/webank/wedatasphere/linkis/datasourcemanager/core/receivers/DsmReceiver.scala @@ -0,0 +1,91 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.datasourcemanager.core.receivers + +import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} +import com.webank.wedatasphere.linkis.datasourcemanager.core.service.{DataSourceInfoService, DataSourceRelateService} +import com.webank.wedatasphere.linkis.datasourcemanager.common.protocol.DsInfoResponse +import com.webank.wedatasphere.linkis.rpc.{Receiver, Sender} +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +import scala.concurrent.duration.Duration +import java.util + +import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSource +import com.webank.wedatasphere.linkis.datasourcemanager.common.protocol.{DsInfoQueryRequest, DsInfoResponse} +import com.webank.wedatasphere.linkis.datasourcemanager.core.restful.RestfulApiHelper +import com.webank.wedatasphere.linkis.datasourcemanager.core.service.{DataSourceInfoService, DataSourceRelateService} +import com.webank.wedatasphere.linkis.datasourcemanager.core.restful.RestfulApiHelper +/** + * @author georgeqiao + * 2020/02/10 + */ +@Component +class DsmReceiver extends Receiver with Logging{ + + @Autowired + private var dataSourceInfoService: DataSourceInfoService = _ + + @Autowired + private var dataSourceRelateService: DataSourceRelateService = _ + + override def receive(message: Any, sender: Sender): Unit = ??? + + override def receiveAndReply(message: Any, sender: Sender): Any = message match { + case DsInfoQueryRequest(id, system) => + if ( id.toLong > 0 && Some(system).isDefined ) { + Utils.tryCatch { + val dataSource: DataSource = dataSourceInfoService.getDataSourceInfo(id.toLong, system) + if ( null != dataSource ) { + RestfulApiHelper.decryptPasswordKey(dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId), + dataSource.getConnectParams) + DsInfoResponse(status = true, dataSource.getDataSourceType.getName, + dataSource.getConnectParams, dataSource.getCreateUser) + }else DsInfoResponse(status = true, "", new util.HashMap[String, Object](), "") + }{ + case e: Exception => logger.error(s"Fail to get data source information, id:$id system:$system", e) + DsInfoResponse(status = false, "", new util.HashMap[String, Object](), "") + case t: Throwable => logger.error(s"Fail to get data source information, id:$id system:$system", t) + DsInfoResponse(status = false, "", new util.HashMap[String, Object](), "") + } + } else { + DsInfoResponse(status = true, "", new util.HashMap[String, Object](), "") + } + case _ => new Object() + } + + override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = message match { + case DsInfoQueryRequest(id, system) => + if ( id.toLong > 0 && Some(system).isDefined ) { + Utils.tryCatch { + val dataSource: DataSource = dataSourceInfoService.getDataSourceInfo(id.toLong, system) + if ( null != dataSource ) { + RestfulApiHelper.decryptPasswordKey(dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId), + dataSource.getConnectParams) + DsInfoResponse(status = true, dataSource.getDataSourceType.getName, + dataSource.getConnectParams, dataSource.getCreateUser) + }else DsInfoResponse(status = true, "", new util.HashMap[String, Object](), "") + }{ + case e: Exception => logger.error(s"Fail to get data source information, id:$id system:$system", e) + DsInfoResponse(status = false, "", new util.HashMap[String, Object](), "") + case t: Throwable => logger.error(s"Fail to get data source information, id:$id system:$system", t) + DsInfoResponse(status = false, "", new util.HashMap[String, Object](), "") + } + } else { + DsInfoResponse(status = true, "", new util.HashMap[String, Object](), "") + } + case _ => new Object() + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
