This is an automated email from the ASF dual-hosted git repository. nic pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 20353a339054b71ae50386bf08397c780d2244e7 Author: Zhong, Yanghong <nju_y...@apache.org> AuthorDate: Tue Mar 10 14:45:03 2020 +0800 KYLIN-4415 HTable Creation with Retry --- .../org/apache/kylin/common/KylinConfigBase.java | 4 ++ .../kylin/storage/hbase/steps/CreateHTableJob.java | 2 +- .../kylin/storage/hbase/steps/CubeHTableUtil.java | 78 +++++++++++++++++++--- 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 16c07c1..7136a51 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -1259,6 +1259,10 @@ public abstract class KylinConfigBase implements Serializable { } } + public int getHBaseHTableAvailableRetry() { + return Integer.parseInt(getOptional("kylin.storage.hbase.htable-available-retry", "3")); + } + public int getHBaseRegionCountMin() { return Integer.parseInt(getOptional("kylin.storage.hbase.min-region-count", "1")); } diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java index 4b2218b..7c970c2 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java @@ -117,7 +117,7 @@ public class CreateHTableJob extends AbstractHadoopJob { splitKeys = getRegionSplitsFromCuboidStatistics(cuboidSizeMap, kylinConfig, cubeSegment, partitionFilePath.getParent()); - CubeHTableUtil.createHTable(cubeSegment, splitKeys); + CubeHTableUtil.createHTable(cubeSegment, splitKeys, true); // export configuration in advance to avoid connecting to hbase from spark if (cubeDesc.getEngineType()== IEngineAware.ID_SPARK){ diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java index d06c993..369c7bc 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java @@ -19,16 +19,20 @@ package org.apache.kylin.storage.hbase.steps; import java.io.IOException; - import java.util.Locale; + import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotEnabledException; +import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.exceptions.TimeoutIOException; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.regionserver.BloomType; @@ -54,13 +58,14 @@ public class CubeHTableUtil { private static final Logger logger = LoggerFactory.getLogger(CubeHTableUtil.class); - public static void createHTable(CubeSegment cubeSegment, byte[][] splitKeys) throws IOException { - String tableName = cubeSegment.getStorageLocationIdentifier(); + public static void createHTable(CubeSegment cubeSegment, byte[][] splitKeys, boolean continueOnExists) + throws IOException { + TableName tableName = TableName.valueOf(cubeSegment.getStorageLocationIdentifier()); CubeInstance cubeInstance = cubeSegment.getCubeInstance(); CubeDesc cubeDesc = cubeInstance.getDescriptor(); KylinConfig kylinConfig = cubeDesc.getConfig(); - HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(cubeSegment.getStorageLocationIdentifier())); + HTableDescriptor tableDesc = new HTableDescriptor(tableName); tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); tableDesc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix()); tableDesc.setValue(IRealizationConstants.HTableCreationTime, String.valueOf(System.currentTimeMillis())); @@ -95,16 +100,69 @@ public class CubeHTableUtil { tableDesc.addFamily(cf); } - if (admin.tableExists(TableName.valueOf(tableName))) { - // admin.disableTable(tableName); - // admin.deleteTable(tableName); - throw new RuntimeException("HBase table " + tableName + " exists!"); + if (admin.tableExists(tableName)) { + if (!continueOnExists) { + throw new RuntimeException("HBase table " + tableName.toString() + " exists!"); + } else { + logger.warn("HBase table " + tableName + " exists when create HTable, continue the process!"); + if (admin.isTableEnabled(tableName)) { + try { + admin.disableTable(tableName); + logger.warn("Disabled existing enabled HBase table " + tableName.toString()); + } catch (TableNotEnabledException e) { + logger.warn("HBase table " + tableName + " already disabled.", e); + } + } else { + logger.warn("HBase table exists but in disabled state."); + } + try { + admin.deleteTable(tableName); + logger.info("Deleted existing HBase table " + tableName.toString()); + } catch (TableNotFoundException e) { + logger.warn("HBase table " + tableName + " already deleted.", e); + } + } } DeployCoprocessorCLI.deployCoprocessor(tableDesc); - admin.createTable(tableDesc, splitKeys); - Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons"); + try { + admin.createTable(tableDesc, splitKeys); + } catch (TableExistsException e) { + if (admin.isTableEnabled(tableName)) { + logger.warn( + "Duplicate create table request send to HMaster, ignore it and continue the process, table " + + tableName.toString()); + } else { + logger.warn( + "Duplicate create table request send to HMaster, ignore it and continue enable the table " + + tableName.toString()); + admin.enableTable(tableName); + } + } catch (TimeoutIOException e) { + if (admin.isTableEnabled(tableName)) { + logger.warn("False alerting?? Detect TimeoutIOException when creating HBase table " + + tableName.toString(), e); + } else { + throw e; + } + } + + int availableRetry = kylinConfig.getHBaseHTableAvailableRetry(); + while (availableRetry > 0) { + if (!admin.isTableAvailable(tableName)) { + logger.warn("Table created but not available, wait for a while..."); + try { + availableRetry--; + Thread.sleep(60000); + } catch (InterruptedException e) { + } + } else { + break; + } + } + + Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons"); logger.info("create hbase table " + tableName + " done."); } finally { IOUtils.closeQuietly(admin);