This is an automated email from the ASF dual-hosted git repository. aiceflower pushed a commit to branch release-0.9.4 in repository https://gitbox.apache.org/repos/asf/linkis.git
commit f6bc39ec3269415a88185f69e42fa35fe16ed442 Author: jftang <[email protected]> AuthorDate: Wed Jun 3 16:22:41 2020 +0800 1.fix ds field repeats bug 2.hive table modificationTime in Interface("/getTableStatisticInfo") --- .../domain/mdq/vo/MdqTablePartitionStatisticInfoVO.java | 10 ++++++++++ .../metadata/domain/mdq/vo/MdqTableStatisticInfoVO.java | 1 - .../linkis/metadata/service/impl/MdqServiceImpl.java | 17 ++++++++++++++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTablePartitionStatisticInfoVO.java b/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTablePartitionStatisticInfoVO.java index 16e05a98a5..99651acc43 100644 --- a/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTablePartitionStatisticInfoVO.java +++ b/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTablePartitionStatisticInfoVO.java @@ -16,6 +16,7 @@ package com.webank.wedatasphere.linkis.metadata.domain.mdq.vo; import java.util.ArrayList; +import java.util.Date; import java.util.List; @@ -23,6 +24,15 @@ public class MdqTablePartitionStatisticInfoVO { private Integer fileNum; private String partitionSize; private String name; + private Date modificationTime; + + public Date getModificationTime() { + return modificationTime; + } + + public void setModificationTime(Date modificationTime) { + this.modificationTime = modificationTime; + } public String getName() { return name; diff --git a/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTableStatisticInfoVO.java b/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTableStatisticInfoVO.java index 09565cf6d2..13192b98f3 100644 --- a/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTableStatisticInfoVO.java +++ b/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTableStatisticInfoVO.java @@ -15,7 +15,6 @@ */ package com.webank.wedatasphere.linkis.metadata.domain.mdq.vo; -import java.util.ArrayList; import java.util.Date; import java.util.List; diff --git a/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/service/impl/MdqServiceImpl.java b/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/service/impl/MdqServiceImpl.java index 0634dfdfa7..2986b57874 100644 --- a/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/service/impl/MdqServiceImpl.java +++ b/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/service/impl/MdqServiceImpl.java @@ -17,7 +17,6 @@ package com.webank.wedatasphere.linkis.metadata.service.impl; import com.google.common.collect.Maps; import com.google.gson.Gson; - import com.webank.wedatasphere.linkis.common.utils.ByteTimeUtils; import com.webank.wedatasphere.linkis.hadoop.common.utils.HDFSUtils; import com.webank.wedatasphere.linkis.metadata.dao.MdqDao; @@ -84,6 +83,13 @@ public class MdqServiceImpl implements MdqService { mdqDao.insertTable(table); List<MdqTableFieldsInfoBO> tableFieldsInfo = mdqTableBO.getTableFieldsInfo(); List<MdqField> mdqFieldList = DomainCoversionUtils.mdqTableFieldsInfoBOListToMdqFieldList(tableFieldsInfo, table.getId()); + if(table.getPartitionTable() && table.getImport()){ + //创建表是导入,并且是分区表的话,自动去掉最后一个ds列 + List<MdqField> collect = mdqFieldList.stream().filter(f -> "ds".equals(f.getName())).collect(Collectors.toList()); + if(collect.size() > 1){ + mdqFieldList.remove(collect.get(1)); + } + } mdqDao.insertFields(mdqFieldList); if (mdqTableBO.getImportInfo() != null) { MdqTableImportInfoBO importInfo = mdqTableBO.getImportInfo(); @@ -237,6 +243,7 @@ public class MdqServiceImpl implements MdqService { mdqTablePartitionStatisticInfoVO.setName(new Path(path).getName()); mdqTablePartitionStatisticInfoVO.setFileNum(getTableFileNum(path)); mdqTablePartitionStatisticInfoVO.setPartitionSize(getTableSize(path)); + mdqTablePartitionStatisticInfoVO.setModificationTime(getTableModificationTime(path)); FileStatus tableFile = getRootHdfs().getFileStatus(new Path(path)); FileStatus[] fileStatuses = getRootHdfs().listStatus(tableFile.getPath()); List<FileStatus> collect = Arrays.stream(fileStatuses).filter(f -> f.isDirectory()).collect(Collectors.toList()); @@ -246,6 +253,14 @@ public class MdqServiceImpl implements MdqService { return mdqTablePartitionStatisticInfoVO; } + private Date getTableModificationTime(String tableLocation) throws IOException { + if (StringUtils.isNotBlank(tableLocation)) { + FileStatus tableFile = getRootHdfs().getFileStatus(new Path(tableLocation)); + return new Date(tableFile.getModificationTime()); + } + return null; + } + private int getPartitionsNum(String tableLocation) throws IOException { int partitionsNum = 0; if (StringUtils.isNotBlank(tableLocation)) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
