Repository: carbondata Updated Branches: refs/heads/branch-1.3 0283c938b -> 96e26d7a5
[CARBONDATA-2244]fix creating pre-aggregate table bug when there are invisibility INSERT_IN_PROGRESS/INSERT_OVERWRITE_IN_PROGRESS segments on main table When there are some invisibility INSERT_IN_PROGRESS/INSERT_OVERWRITE_IN_PROGRESS segments on main table, it can not create preaggregate table on it. This closes #2050 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/96e26d7a Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/96e26d7a Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/96e26d7a Branch: refs/heads/branch-1.3 Commit: 96e26d7a5da347c3638827f5986aebc7b1eaa237 Parents: 0283c93 Author: Zhang Zhichao <441586...@qq.com> Authored: Fri Mar 9 23:36:08 2018 +0800 Committer: Jacky Li <jacky.li...@qq.com> Committed: Fri Mar 16 14:47:22 2018 +0800 ---------------------------------------------------------------------- .../CreatePreAggregateTableCommand.scala | 23 ++++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/96e26d7a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala index 59c43aa..4270d47 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala @@ -179,17 +179,22 @@ case class CreatePreAggregateTableCommand( // This will be used to check if the parent table has any segments or not. If not then no // need to fire load for pre-aggregate table. Therefore reading the load details for PARENT // table. - DataLoadingUtil.deleteLoadsAndUpdateMetadata(isForceDeletion = false, + DataLoadingUtil.deleteLoadsAndUpdateMetadata( + isForceDeletion = false, parentTable, - CarbonFilters.getCurrentPartitions(sparkSession, - TableIdentifier(parentTable.getTableName, - Some(parentTable.getDatabaseName))).map(_.asJava).orNull) - val loadAvailable = SegmentStatusManager.readLoadMetadata(parentTable.getMetaDataFilepath) - if (loadAvailable.exists(load => load.getSegmentStatus == SegmentStatus.INSERT_IN_PROGRESS || - load.getSegmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS)) { + CarbonFilters.getCurrentPartitions( + sparkSession, + TableIdentifier(parentTable.getTableName, + Some(parentTable.getDatabaseName)) + ).map(_.asJava).orNull + ) + + if (SegmentStatusManager.isLoadInProgressInTable(parentTable)) { throw new UnsupportedOperationException( - "Cannot create pre-aggregate table when insert is in progress on main table") - } else if (loadAvailable.nonEmpty) { + "Cannot create pre-aggregate table when insert is in progress on main table") + } + val loadAvailable = SegmentStatusManager.readLoadMetadata(parentTable.getMetaDataFilepath) + if (loadAvailable.nonEmpty) { // Passing segmentToLoad as * because we want to load all the segments into the // pre-aggregate table even if the user has set some segments on the parent table. loadCommand.dataFrame = Some(PreAggregateUtil
