caishunfeng commented on a change in pull request #8637:
URL: https://github.com/apache/dolphinscheduler/pull/8637#discussion_r834008630
##########
File path:
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java
##########
@@ -74,21 +73,22 @@
*/
boolean check(Map<String, Object> result, boolean bool, Status
userNoOperationPerm);
- /**
- * create tenant dir if not exists
- *
- * @param tenantCode tenant code
- * @throws IOException if hdfs operation exception
- */
- void createTenantDirIfNotExists(String tenantCode) throws IOException;
+// /**
Review comment:
remove unuse code
##########
File path: dolphinscheduler-api/pom.xml
##########
@@ -145,6 +151,12 @@
<dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-models</artifactId>
+ <exclusions>
Review comment:
Why to exclude swagger module?
##########
File path:
dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
##########
@@ -300,13 +310,15 @@ private void downloadResource(String execLocalPath,
Map<String, String> projectR
if (!resFile.exists()) {
try {
// query the tenant code of the resource according to the
name of the resource
- String resHdfsPath =
HadoopUtils.getHdfsResourceFileName(tenantCode, fullName);
+ //XXX
Review comment:
what's the `xxx` mean?
##########
File path:
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java
##########
@@ -162,14 +149,14 @@ private void init() {
logger.warn("hdfs.root.user is not set value!");
fs = FileSystem.get(configuration);
}
- } else if (resUploadType == ResUploadType.S3) {
- System.setProperty(Constants.AWS_S3_V4, Constants.STRING_TRUE);
- configuration.set(Constants.FS_DEFAULTFS,
PropertyUtils.getString(Constants.FS_DEFAULTFS));
- configuration.set(Constants.FS_S3A_ENDPOINT,
PropertyUtils.getString(Constants.FS_S3A_ENDPOINT));
- configuration.set(Constants.FS_S3A_ACCESS_KEY,
PropertyUtils.getString(Constants.FS_S3A_ACCESS_KEY));
- configuration.set(Constants.FS_S3A_SECRET_KEY,
PropertyUtils.getString(Constants.FS_S3A_SECRET_KEY));
- fs = FileSystem.get(configuration);
- }
+// } else if (resUploadType == ResUploadType.S3) {
Review comment:
remove the unuse code.
##########
File path:
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java
##########
@@ -37,10 +29,14 @@
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
+import static
org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
+import static
org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static
org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
public class TenantControllerTest extends AbstractControllerTest {
private static final Logger logger =
LoggerFactory.getLogger(TenantControllerTest.class);
- @Test
+ // @Test
Review comment:
why skip this test?
##########
File path:
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/S3Utils.java
##########
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.common.utils;
+
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.*;
+import com.amazonaws.services.s3.transfer.MultipleFileDownload;
+import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
+import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.storage.StorageOperate;
+import org.apache.dolphinscheduler.spi.enums.ResourceType;
+import org.jets3t.service.ServiceException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.dolphinscheduler.common.Constants.*;
+
+public class S3Utils implements Closeable, StorageOperate {
+
+ private static final Logger logger =
LoggerFactory.getLogger(S3Utils.class);
+
+ public static final String ACCESS_KEY_ID =
PropertyUtils.getString(Constants.AWS_ACCESS_KEY_ID);
+
+ public static final String SECRET_KEY_ID =
PropertyUtils.getString(Constants.AWS_SECRET_ACCESS_KEY);
+
+ public static final String REGION =
PropertyUtils.getString(Constants.AWS_REGION);
+
+
+ private AmazonS3 s3Client = null;
+
+ private S3Utils() {
+ if (PropertyUtils.getString(RESOURCE_STORAGE_TYPE).equals(STORAGE_S3))
{
+
+ if (!StringUtils.isEmpty(PropertyUtils.getString(AWS_END_POINT))) {
+ s3Client = AmazonS3ClientBuilder
+ .standard()
+ .withPathStyleAccessEnabled(true)
+ .withEndpointConfiguration(new
AwsClientBuilder.EndpointConfiguration(PropertyUtils.getString(AWS_END_POINT),
Regions.fromName(REGION).getName()))
+ .withCredentials(new AWSStaticCredentialsProvider(new
BasicAWSCredentials(ACCESS_KEY_ID, SECRET_KEY_ID)))
+ .build();
+ } else {
+ s3Client = AmazonS3ClientBuilder
+ .standard()
+ .withCredentials(new AWSStaticCredentialsProvider(new
BasicAWSCredentials(ACCESS_KEY_ID, SECRET_KEY_ID)))
+ .withRegion(Regions.fromName(REGION))
+ .build();
+ }
+ logger.info("s3client is null :{}", null == s3Client);
Review comment:
```suggestion
if (null == s3Client) {
logger.error("s3client is null");
}
```
##########
File path:
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/StoreConfiguration.java
##########
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.common.config;
+
+import org.apache.dolphinscheduler.common.storage.StorageOperate;
+import org.apache.dolphinscheduler.common.utils.HadoopUtils;
+import org.apache.dolphinscheduler.common.utils.PropertyUtils;
+import org.apache.dolphinscheduler.common.utils.S3Utils;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.stereotype.Component;
+
+import static org.apache.dolphinscheduler.common.Constants.*;
+
+
+/**
+ * @version since 2.0.4
Review comment:
please remove it and add some description for this Class
##########
File path:
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
##########
@@ -432,65 +426,63 @@ public Result queryUserList(User loginUser, String
searchVal, Integer pageNo, In
user.setUpdateTime(now);
//if user switches the tenant, the user's resources need to be copied
to the new tenant
- if (user.getTenantId() != tenantId) {
- Tenant oldTenant = tenantMapper.queryById(user.getTenantId());
- //query tenant
- Tenant newTenant = tenantMapper.queryById(tenantId);
- if (newTenant != null) {
- // if hdfs startup
- if (PropertyUtils.getResUploadStartupState() && oldTenant !=
null) {
- String newTenantCode = newTenant.getTenantCode();
- String oldResourcePath =
HadoopUtils.getHdfsResDir(oldTenant.getTenantCode());
- String oldUdfsPath =
HadoopUtils.getHdfsUdfDir(oldTenant.getTenantCode());
-
- // if old tenant dir exists
- if (HadoopUtils.getInstance().exists(oldResourcePath)) {
- String newResourcePath =
HadoopUtils.getHdfsResDir(newTenantCode);
- String newUdfsPath =
HadoopUtils.getHdfsUdfDir(newTenantCode);
-
- //file resources list
- List<Resource> fileResourcesList =
resourceMapper.queryResourceList(
- null, userId, ResourceType.FILE.ordinal());
- if (CollectionUtils.isNotEmpty(fileResourcesList)) {
- ResourceTreeVisitor resourceTreeVisitor = new
ResourceTreeVisitor(fileResourcesList);
- ResourceComponent resourceComponent =
resourceTreeVisitor.visit();
- copyResourceFiles(resourceComponent,
oldResourcePath, newResourcePath);
- }
-
- //udf resources
- List<Resource> udfResourceList =
resourceMapper.queryResourceList(
- null, userId, ResourceType.UDF.ordinal());
- if (CollectionUtils.isNotEmpty(udfResourceList)) {
- ResourceTreeVisitor resourceTreeVisitor = new
ResourceTreeVisitor(udfResourceList);
- ResourceComponent resourceComponent =
resourceTreeVisitor.visit();
- copyResourceFiles(resourceComponent, oldUdfsPath,
newUdfsPath);
- }
-
- //Delete the user from the old tenant directory
- String oldUserPath =
HadoopUtils.getHdfsUserDir(oldTenant.getTenantCode(), userId);
- HadoopUtils.getInstance().delete(oldUserPath, true);
- } else {
- // if old tenant dir not exists , create
- createTenantDirIfNotExists(oldTenant.getTenantCode());
- }
-
- if
(HadoopUtils.getInstance().exists(HadoopUtils.getHdfsTenantDir(newTenant.getTenantCode())))
{
- //create user in the new tenant directory
- String newUserPath =
HadoopUtils.getHdfsUserDir(newTenant.getTenantCode(), user.getId());
- HadoopUtils.getInstance().mkdir(newUserPath);
- } else {
- // if new tenant dir not exists , create
- createTenantDirIfNotExists(newTenant.getTenantCode());
- }
-
- }
- }
- user.setTenantId(tenantId);
- }
-
+// if (user.getTenantId() != tenantId) {
+// Tenant oldTenant = tenantMapper.queryById(user.getTenantId());
+// //query tenant
+// Tenant newTenant = tenantMapper.queryById(tenantId);
+// // if hdfs startup
+// if (null != newTenant &&
PropertyUtils.getResUploadStartupState() && oldTenant != null) {
+// String newTenantCode = newTenant.getTenantCode();
+// String oldResourcePath =
storageOperate.getResDir(oldTenant.getTenantCode());
+// String oldUdfsPath =
storageOperate.getUdfDir(oldTenant.getTenantCode());
+//
+// try {// if old tenant dir exists
+// if (storageOperate.exists(oldTenant.getTenantCode(),
oldResourcePath)) {
+// String newResourcePath =
storageOperate.getResDir(newTenantCode);
+// String newUdfsPath =
storageOperate.getUdfDir(newTenantCode);
+//
+// //file resources list
+// List<Resource> fileResourcesList =
resourceMapper.queryResourceList(
+// null, userId, ResourceType.FILE.ordinal());
+// if (CollectionUtils.isNotEmpty(fileResourcesList)) {
+// ResourceTreeVisitor resourceTreeVisitor = new
ResourceTreeVisitor(fileResourcesList);
+// ResourceComponent resourceComponent =
resourceTreeVisitor.visit();
+// copyResourceFiles(oldTenant.getTenantCode(),
newTenantCode, resourceComponent, oldResourcePath, newResourcePath);
Review comment:
Is there any problem with commenting out this code?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
