This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HASE-24950 in repository https://gitbox.apache.org/repos/asf/hbase.git
commit ca1d75c8669ea026574492cb0d03fd7bed5eb326 Author: Duo Zhang <zhang...@apache.org> AuthorDate: Tue Sep 15 21:11:55 2020 +0800 HBASE-24929 Introduce a special CellComparator for master local region (#2378) Signed-off-by: Guanghao Zhang <zg...@apache.org> --- .../org/apache/hadoop/hbase/PrivateCellUtil.java | 13 +++ .../hadoop/hbase/master/region/MasterRegion.java | 5 +- .../master/region/MasterRegionCellComparator.java | 92 ++++++++++++++++++++++ .../hbase/master/region/MasterRegionFactory.java | 2 +- .../hbase/master/region/MasterRegionParams.java | 11 --- .../apache/hadoop/hbase/regionserver/HRegion.java | 25 ++++-- 6 files changed, 125 insertions(+), 23 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java index 2aadc42..fcfae26 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java @@ -724,6 +724,19 @@ public final class PrivateCellUtil { length); } + public static boolean rowsStartWith(Cell left, byte[] startsWith) { + if (left.getRowLength() < startsWith.length) { + return false; + } + if (left instanceof ByteBufferExtendedCell) { + return ByteBufferUtils.equals(((ByteBufferExtendedCell) left).getRowByteBuffer(), + ((ByteBufferExtendedCell) left).getRowPosition(), startsWith.length, startsWith, 0, + startsWith.length); + } + return Bytes.equals(left.getRowArray(), left.getRowOffset(), startsWith.length, startsWith, 0, + startsWith.length); + } + public static boolean matchingFamily(final Cell left, final byte[] buf, final int offset, final int length) { if (left instanceof ByteBufferExtendedCell) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java index 81da59d..cbfc2a3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.TableName; @@ -290,9 +291,7 @@ public final class MasterRegion { if (params.useHsync() != null) { conf.setBoolean(HRegion.WAL_HSYNC_CONF_KEY, params.useHsync()); } - if (params.useMetaCellComparator() != null) { - conf.setBoolean(HRegion.USE_META_CELL_COMPARATOR, params.useMetaCellComparator()); - } + conf.setBoolean(HRegion.USE_MASTER_REGION_CELL_COMPARATOR, true); conf.setInt(AbstractFSWAL.RING_BUFFER_SLOT_COUNT, IntMath.ceilingPowerOfTwo(params.ringBufferSlotCount())); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionCellComparator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionCellComparator.java new file mode 100644 index 0000000..3960c41 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionCellComparator.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.region; + +import java.util.Comparator; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellComparatorImpl; +import org.apache.hadoop.hbase.MetaCellComparator; +import org.apache.hadoop.hbase.PrivateCellUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * Cell comparator implementation for master local region. + * <p/> + * In general, for catalog family, we need to use {@link MetaCellComparator} while for other + * families, we need to use {@link CellComparatorImpl}. + * <p/> + * The trick here is to check the row key format, if it is start with 'hbase:meta', we will use + * {@link MetaCellComparator}, otherwise we will use {@link CellComparatorImpl}. + */ +@InterfaceAudience.Private +public class MasterRegionCellComparator extends CellComparatorImpl { + + /** + * A {@link MasterRegionCellComparator} for {@link MasterRegion} {@link Cell}s. + */ + public static final MasterRegionCellComparator MASTER_REGION_COMPARATOR = + new MasterRegionCellComparator(); + + private static final byte[] CATALOG_ROW_PREFIX = TableName.META_TABLE_NAME.getName(); + + private boolean isCatalogRow(Cell c) { + return PrivateCellUtil.rowsStartWith(c, CATALOG_ROW_PREFIX); + } + + private boolean isCatalogRow(byte[] row, int off, int len) { + if (len < CATALOG_ROW_PREFIX.length) { + return false; + } + return Bytes.equals(row, off, CATALOG_ROW_PREFIX.length, CATALOG_ROW_PREFIX, 0, + CATALOG_ROW_PREFIX.length); + } + + @Override + public int compare(Cell a, Cell b, boolean ignoreSequenceid) { + if (isCatalogRow(a) || isCatalogRow(b)) { + return MetaCellComparator.META_COMPARATOR.compare(a, b, ignoreSequenceid); + } else { + return super.compare(a, b, ignoreSequenceid); + } + } + + @Override + public int compareRows(Cell left, Cell right) { + if (isCatalogRow(left) || isCatalogRow(right)) { + return MetaCellComparator.META_COMPARATOR.compareRows(left, right); + } else { + return super.compareRows(left, right); + } + } + + @Override + public int compareRows(Cell left, byte[] right, int roffset, int rlength) { + if (isCatalogRow(left) || isCatalogRow(right, roffset, rlength)) { + return MetaCellComparator.META_COMPARATOR.compareRows(left, right, roffset, rlength); + } else { + return super.compareRows(left, right, roffset, rlength); + } + } + + @Override + public Comparator getSimpleComparator() { + return this; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java index cfa25f5..d70aef6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFactory.java @@ -107,7 +107,7 @@ public final class MasterRegionFactory { params.ringBufferSlotCount(conf.getInt(RING_BUFFER_SLOT_COUNT, DEFAULT_RING_BUFFER_SLOT_COUNT)); long rollPeriodMs = conf.getLong(ROLL_PERIOD_MS_KEY, DEFAULT_ROLL_PERIOD_MS); params.rollPeriodMs(rollPeriodMs).archivedWalSuffix(ARCHIVED_WAL_SUFFIX) - .archivedHFileSuffix(ARCHIVED_HFILE_SUFFIX).useMetaCellComparator(true); + .archivedHFileSuffix(ARCHIVED_HFILE_SUFFIX); return MasterRegion.create(params); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionParams.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionParams.java index f2a03a4..1bfc84d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionParams.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionParams.java @@ -53,8 +53,6 @@ public class MasterRegionParams { private String archivedHFileSuffix; - private Boolean useMetaCellComparator; - public MasterRegionParams server(Server server) { this.server = server; return this; @@ -120,11 +118,6 @@ public class MasterRegionParams { return this; } - public MasterRegionParams useMetaCellComparator(boolean useMetaCellComparator) { - this.useMetaCellComparator = useMetaCellComparator; - return this; - } - public Server server() { return server; } @@ -176,8 +169,4 @@ public class MasterRegionParams { public String archivedHFileSuffix() { return archivedHFileSuffix; } - - public Boolean useMetaCellComparator() { - return useMetaCellComparator; - } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 9bdc100..e638a8d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -143,6 +143,7 @@ import org.apache.hadoop.hbase.ipc.CallerDisconnectedException; import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; import org.apache.hadoop.hbase.ipc.RpcCall; import org.apache.hadoop.hbase.ipc.RpcServer; +import org.apache.hadoop.hbase.master.region.MasterRegionCellComparator; import org.apache.hadoop.hbase.mob.MobFileCache; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; @@ -262,12 +263,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi "hbase.hregion.special.recovered.edits.dir"; /** - * Whether to use {@link MetaCellComparator} even if we are not meta region. Used when creating - * master local region. + * Whether to use {@link MasterRegionCellComparator}. */ - public static final String USE_META_CELL_COMPARATOR = "hbase.region.use.meta.cell.comparator"; - - public static final boolean DEFAULT_USE_META_CELL_COMPARATOR = false; + public static final String USE_MASTER_REGION_CELL_COMPARATOR = + "hbase.region.use.master.region.cell.comparator"; final AtomicBoolean closed = new AtomicBoolean(false); @@ -750,6 +749,18 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi wal, confParam, htd, rsServices); } + private CellComparator getCellComparator(Configuration conf, TableDescriptor htd) { + boolean useMasterRegionCellComparator = + conf.getBoolean(USE_MASTER_REGION_CELL_COMPARATOR, false); + if (useMasterRegionCellComparator) { + return MasterRegionCellComparator.MASTER_REGION_COMPARATOR; + } else if (htd.isMetaTable()) { + return MetaCellComparator.META_COMPARATOR; + } else { + return CellComparatorImpl.COMPARATOR; + } + } + /** * HRegion constructor. This constructor should only be used for testing and * extensions. Instances of HRegion should be instantiated with the @@ -783,9 +794,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi // 'conf' renamed to 'confParam' b/c we use this.conf in the constructor this.baseConf = confParam; this.conf = new CompoundConfiguration().add(confParam).addBytesMap(htd.getValues()); - this.cellComparator = htd.isMetaTable() || - conf.getBoolean(USE_META_CELL_COMPARATOR, DEFAULT_USE_META_CELL_COMPARATOR) ? - MetaCellComparator.META_COMPARATOR : CellComparatorImpl.COMPARATOR; + this.cellComparator = getCellComparator(this.conf, htd); this.lock = new ReentrantReadWriteLock(conf.getBoolean(FAIR_REENTRANT_CLOSE_LOCK, DEFAULT_FAIR_REENTRANT_CLOSE_LOCK)); this.flushCheckInterval = conf.getInt(MEMSTORE_PERIODIC_FLUSH_INTERVAL,