This is an automated email from the ASF dual-hosted git repository.
journey pushed a commit to branch 1.3.2-release
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/1.3.2-release by this push:
new 5f5c084 [Fix-3469][api]Should filter the resource by the different
program type (#3498)
5f5c084 is described below
commit 5f5c08402fdcecca8c35f4dc3021cc089949ef13
Author: lgcareer <18610854...@163.com>
AuthorDate: Fri Aug 14 16:47:01 2020 +0800
[Fix-3469][api]Should filter the resource by the different program type
(#3498)
* [Fix-3463][api]Fixed that run the sql task will be failure after rename
the udf resource
* [Fix-3469][api]Should list python file and jar file
* [Fix-3469][api]Should filter the resource by the different program type
* [Fix-3469][api]fix the code smell
---
.../api/controller/ResourcesController.java | 13 +++++++----
.../api/dto/resources/filter/ResourceFilter.java | 25 ++++++++++++++++------
.../api/service/ResourcesService.java | 19 +++++++++++++---
.../api/controller/ResourcesControllerTest.java | 25 ++++++++++++++++++++++
.../dto/resources/filter/ResourceFilterTest.java | 8 +++++--
.../dolphinscheduler/dao/upgrade/UpgradeDao.java | 8 +++++++
6 files changed, 82 insertions(+), 16 deletions(-)
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
index 2f2cb9c..697d364 100644
---
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
@@ -27,6 +27,7 @@ import
org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.service.UdfFuncService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
@@ -275,7 +276,7 @@ public class ResourcesController extends BaseController {
* @param type resource type
* @return resource list
*/
- @ApiOperation(value = "queryResourceJarList", notes =
"QUERY_RESOURCE_LIST_NOTES")
+ @ApiOperation(value = "queryResourceByProgramType", notes =
"QUERY_RESOURCE_LIST_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required
= true, dataType = "ResourceType")
})
@@ -283,10 +284,14 @@ public class ResourcesController extends BaseController {
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR)
public Result queryResourceJarList(@ApiIgnore @RequestAttribute(value =
Constants.SESSION_USER) User loginUser,
- @RequestParam(value = "type")
ResourceType type
+ @RequestParam(value = "type")
ResourceType type,
+ @RequestParam(value =
"programType",required = false) ProgramType programType
) {
- logger.info("query resource list, login user:{}, resource type:{}",
loginUser.getUserName(), type.toString());
- Map<String, Object> result =
resourceService.queryResourceJarList(loginUser, type);
+ String programTypeName = programType == null ? "" : programType.name();
+ String userName = loginUser.getUserName();
+ userName = userName.replaceAll("[\n|\r|\t]", "_");
+ logger.info("query resource list, login user:{}, resource type:{},
program type:{}", userName,programTypeName);
+ Map<String, Object> result =
resourceService.queryResourceByProgramType(loginUser, type,programType);
return returnDataList(result);
}
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
index c918a16..9c3a84c 100644
---
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
@@ -31,24 +31,29 @@ public class ResourceFilter implements IFilter {
/**
* resource suffix
*/
- private String suffix;
+ private Set<String> suffixs = new HashSet<>();
/**
* resource list
*/
private List<Resource> resourceList;
/**
- * parent list
+ * constructor
+ * @param suffix resource suffix
+ * @param resourceList resource list
*/
- //Set<Resource> parentList = new HashSet<>();
+ public ResourceFilter(String suffix, List<Resource> resourceList) {
+ this.suffixs.add(suffix);
+ this.resourceList = resourceList;
+ }
/**
* constructor
- * @param suffix resource suffix
+ * @param suffixs resource suffixs
* @param resourceList resource list
*/
- public ResourceFilter(String suffix, List<Resource> resourceList) {
- this.suffix = suffix;
+ public ResourceFilter(Set<String> suffixs, List<Resource> resourceList) {
+ this.suffixs = suffixs;
this.resourceList = resourceList;
}
@@ -59,7 +64,13 @@ public class ResourceFilter implements IFilter {
public Set<Resource> fileFilter(){
Set<Resource> resources = resourceList.stream().filter(t -> {
String alias = t.getAlias();
- return alias.endsWith(suffix);
+ boolean result = false;
+ for (String suffix : suffixs) {
+ if (alias.endsWith(suffix)) {
+ result = true;
+ }
+ }
+ return result;
}).collect(Collectors.toSet());
return resources;
}
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
index e1a9c85..f66e035 100644
---
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
@@ -30,6 +30,7 @@ import
org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.dao.entity.*;
@@ -654,21 +655,33 @@ public class ResourcesService extends BaseService {
}
/**
- * query resource list
+ * query resource list by program type
*
* @param loginUser login user
* @param type resource type
* @return resource list
*/
- public Map<String, Object> queryResourceJarList(User loginUser,
ResourceType type) {
+ public Map<String, Object> queryResourceByProgramType(User loginUser,
ResourceType type, ProgramType programType) {
Map<String, Object> result = new HashMap<>(5);
+ String suffix = ".jar";
int userId = loginUser.getId();
if(isAdmin(loginUser)){
userId = 0;
}
+ if (programType != null) {
+ switch (programType) {
+ case JAVA:
+ break;
+ case SCALA:
+ break;
+ case PYTHON:
+ suffix = ".py";
+ break;
+ }
+ }
List<Resource> allResourceList =
resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0);
- List<Resource> resources = new ResourceFilter(".jar",new
ArrayList<>(allResourceList)).filter();
+ List<Resource> resources = new ResourceFilter(suffix,new
ArrayList<>(allResourceList)).filter();
Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources);
result.put(Constants.DATA_LIST,
resourceTreeVisitor.visit().getChildren());
putMsg(result,Status.SUCCESS);
diff --git
a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
index a56e3f8..a2ebd27 100644
---
a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
+++
b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
@@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.api.controller;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
+import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
@@ -452,4 +453,28 @@ public class ResourcesControllerTest extends
AbstractControllerTest{
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
+
+ @Test
+ public void testqueryResourceJarList() throws Exception {
+
+ MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
+ paramsMap.add("type", ResourceType.FILE.name());
+ //paramsMap.add("programType", ProgramType.PYTHON.name());
+ paramsMap.add("programType", "JAVA");
+
+
+ MvcResult mvcResult = mockMvc.perform(get("/resources/list/jar")
+ .header(SESSION_ID, sessionId)
+ .params(paramsMap))
+ .andExpect(status().isOk())
+
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
+ .andReturn();
+
+ Result result =
JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(),
Result.class);
+ result.getCode().equals(Status.SUCCESS.getCode());
+ JSONObject object = (JSONObject)
JSON.parse(mvcResult.getResponse().getContentAsString());
+
+
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
+ logger.info(mvcResult.getResponse().getContentAsString());
+ }
}
diff --git
a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
index 8a4a16c..8e3be6a 100644
---
a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
+++
b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
@@ -23,7 +23,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
/**
* resource filter test
@@ -48,9 +50,11 @@ public class ResourceFilterTest {
allList.add(resource5);
allList.add(resource6);
allList.add(resource7);
+ Set<String> suffixSet = new HashSet<>();
+ suffixSet.add(".jar");
+ suffixSet.add(".txt");
-
- ResourceFilter resourceFilter = new ResourceFilter(".jar",allList);
+ ResourceFilter resourceFilter = new ResourceFilter(suffixSet,allList);
List<Resource> resourceList = resourceFilter.filter();
Assert.assertNotNull(resourceList);
resourceList.stream().forEach(t-> logger.info(t.toString()));
diff --git
a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
index 3d35b68..d111a8c 100644
---
a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
+++
b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
@@ -343,6 +343,14 @@ public abstract class UpgradeDao extends AbstractBaseDao {
if (param != null) {
List<ResourceInfo> resourceList =
JSONUtils.toList(param.getString("resourceList"), ResourceInfo.class);
+ ResourceInfo mainJar =
JSONUtils.parseObject(param.getString("mainJar"), ResourceInfo.class);
+ if (mainJar != null && mainJar.getId() == 0) {
+ String fullName = mainJar.getRes().startsWith("/")
? mainJar.getRes() : String.format("/%s",mainJar.getRes());
+ if (resourcesMap.containsKey(fullName)) {
+ mainJar.setId(resourcesMap.get(fullName));
+
param.put("mainJar",JSONUtils.parseObject(JSONObject.toJSONString(mainJar)));
+ }
+ }
if (CollectionUtils.isNotEmpty(resourceList)) {
List<ResourceInfo> newResourceList =
resourceList.stream().map(resInfo -> {
