[jira] [Commented] (HBASE-24508) Why ProtobufUtil does not set scan's limit
[
https://issues.apache.org/jira/browse/HBASE-24508?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17127213#comment-17127213
]
yukunpeng commented on HBASE-24508:
---
{code:java}
org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil{code}
> Why ProtobufUtil does not set scan's limit
>
>
> Key: HBASE-24508
> URL: https://issues.apache.org/jira/browse/HBASE-24508
> Project: HBase
> Issue Type: Bug
>Affects Versions: 2.2.5
>Reporter: yukunpeng
>Priority: Trivial
>
> {code:java}
> //ProtobufUtil
> /**
> * Convert a client Scan to a protocol buffer Scan
> *
> * @param scan the client Scan to convert
> * @return the converted protocol buffer Scan
> * @throws IOException
> */
> public static ClientProtos.Scan toScan(
> final Scan scan) throws IOException {
> ClientProtos.Scan.Builder scanBuilder =
> ClientProtos.Scan.newBuilder();
> scanBuilder.setCacheBlocks(scan.getCacheBlocks());
> if (scan.getBatch() > 0) {
> scanBuilder.setBatchSize(scan.getBatch());
> }
> if (scan.getMaxResultSize() > 0) {
> scanBuilder.setMaxResultSize(scan.getMaxResultSize());
> }
> if (scan.isSmall()) {
> scanBuilder.setSmall(scan.isSmall());
> }
> if (scan.getAllowPartialResults()) {
> scanBuilder.setAllowPartialResults(scan.getAllowPartialResults());
> }
> Boolean loadColumnFamiliesOnDemand =
> scan.getLoadColumnFamiliesOnDemandValue();
> if (loadColumnFamiliesOnDemand != null) {
> scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand);
> }
> scanBuilder.setMaxVersions(scan.getMaxVersions());
> scan.getColumnFamilyTimeRange().forEach((cf, timeRange) -> {
> scanBuilder.addCfTimeRange(HBaseProtos.ColumnFamilyTimeRange.newBuilder()
> .setColumnFamily(UnsafeByteOperations.unsafeWrap(cf))
> .setTimeRange(toTimeRange(timeRange))
> .build());
> });
> scanBuilder.setTimeRange(ProtobufUtil.toTimeRange(scan.getTimeRange()));
> Map attributes = scan.getAttributesMap();
> if (!attributes.isEmpty()) {
> NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
> for (Map.Entry attribute: attributes.entrySet()) {
> attributeBuilder.setName(attribute.getKey());
>
> attributeBuilder.setValue(UnsafeByteOperations.unsafeWrap(attribute.getValue()));
> scanBuilder.addAttribute(attributeBuilder.build());
> }
> }
> byte[] startRow = scan.getStartRow();
> if (startRow != null && startRow.length > 0) {
> scanBuilder.setStartRow(UnsafeByteOperations.unsafeWrap(startRow));
> }
> byte[] stopRow = scan.getStopRow();
> if (stopRow != null && stopRow.length > 0) {
> scanBuilder.setStopRow(UnsafeByteOperations.unsafeWrap(stopRow));
> }
> if (scan.hasFilter()) {
> scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
> }
> if (scan.hasFamilies()) {
> Column.Builder columnBuilder = Column.newBuilder();
> for (Map.Entry>
> family: scan.getFamilyMap().entrySet()) {
>
> columnBuilder.setFamily(UnsafeByteOperations.unsafeWrap(family.getKey()));
> NavigableSet qualifiers = family.getValue();
> columnBuilder.clearQualifier();
> if (qualifiers != null && qualifiers.size() > 0) {
> for (byte [] qualifier: qualifiers) {
>
> columnBuilder.addQualifier(UnsafeByteOperations.unsafeWrap(qualifier));
> }
> }
> scanBuilder.addColumn(columnBuilder.build());
> }
> }
> if (scan.getMaxResultsPerColumnFamily() >= 0) {
> scanBuilder.setStoreLimit(scan.getMaxResultsPerColumnFamily());
> }
> if (scan.getRowOffsetPerColumnFamily() > 0) {
> scanBuilder.setStoreOffset(scan.getRowOffsetPerColumnFamily());
> }
> if (scan.isReversed()) {
> scanBuilder.setReversed(scan.isReversed());
> }
> if (scan.getConsistency() == Consistency.TIMELINE) {
> scanBuilder.setConsistency(toConsistency(scan.getConsistency()));
> }
> if (scan.getCaching() > 0) {
> scanBuilder.setCaching(scan.getCaching());
> }
> long mvccReadPoint = PackagePrivateFieldAccessor.getMvccReadPoint(scan);
> if (mvccReadPoint > 0) {
> scanBuilder.setMvccReadPoint(mvccReadPoint);
> }
> if (!scan.includeStartRow()) {
> scanBuilder.setIncludeStartRow(false);
> }
> scanBuilder.setIncludeStopRow(scan.includeStopRow());
> if (scan.getReadType() != Scan.ReadType.DEFAULT) {
> scanBuilder.setReadType(toReadType(scan.getReadType()));
> }
> if (scan.isNeedCursorResult()) {
> scanBuilder.setNeedCursorResult(true);
> }
> return scanBuilder.build();
> }
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Commented] (HBASE-24508) Why ProtobufUtil does not set scan's limit
[
https://issues.apache.org/jira/browse/HBASE-24508?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17127219#comment-17127219
]
niuyulin commented on HBASE-24508:
--
ScanRequest has limit, we use this one for calls
> Why ProtobufUtil does not set scan's limit
>
>
> Key: HBASE-24508
> URL: https://issues.apache.org/jira/browse/HBASE-24508
> Project: HBase
> Issue Type: Bug
>Affects Versions: 2.2.5
>Reporter: yukunpeng
>Priority: Trivial
>
> {code:java}
> //ProtobufUtil
> /**
> * Convert a client Scan to a protocol buffer Scan
> *
> * @param scan the client Scan to convert
> * @return the converted protocol buffer Scan
> * @throws IOException
> */
> public static ClientProtos.Scan toScan(
> final Scan scan) throws IOException {
> ClientProtos.Scan.Builder scanBuilder =
> ClientProtos.Scan.newBuilder();
> scanBuilder.setCacheBlocks(scan.getCacheBlocks());
> if (scan.getBatch() > 0) {
> scanBuilder.setBatchSize(scan.getBatch());
> }
> if (scan.getMaxResultSize() > 0) {
> scanBuilder.setMaxResultSize(scan.getMaxResultSize());
> }
> if (scan.isSmall()) {
> scanBuilder.setSmall(scan.isSmall());
> }
> if (scan.getAllowPartialResults()) {
> scanBuilder.setAllowPartialResults(scan.getAllowPartialResults());
> }
> Boolean loadColumnFamiliesOnDemand =
> scan.getLoadColumnFamiliesOnDemandValue();
> if (loadColumnFamiliesOnDemand != null) {
> scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand);
> }
> scanBuilder.setMaxVersions(scan.getMaxVersions());
> scan.getColumnFamilyTimeRange().forEach((cf, timeRange) -> {
> scanBuilder.addCfTimeRange(HBaseProtos.ColumnFamilyTimeRange.newBuilder()
> .setColumnFamily(UnsafeByteOperations.unsafeWrap(cf))
> .setTimeRange(toTimeRange(timeRange))
> .build());
> });
> scanBuilder.setTimeRange(ProtobufUtil.toTimeRange(scan.getTimeRange()));
> Map attributes = scan.getAttributesMap();
> if (!attributes.isEmpty()) {
> NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
> for (Map.Entry attribute: attributes.entrySet()) {
> attributeBuilder.setName(attribute.getKey());
>
> attributeBuilder.setValue(UnsafeByteOperations.unsafeWrap(attribute.getValue()));
> scanBuilder.addAttribute(attributeBuilder.build());
> }
> }
> byte[] startRow = scan.getStartRow();
> if (startRow != null && startRow.length > 0) {
> scanBuilder.setStartRow(UnsafeByteOperations.unsafeWrap(startRow));
> }
> byte[] stopRow = scan.getStopRow();
> if (stopRow != null && stopRow.length > 0) {
> scanBuilder.setStopRow(UnsafeByteOperations.unsafeWrap(stopRow));
> }
> if (scan.hasFilter()) {
> scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
> }
> if (scan.hasFamilies()) {
> Column.Builder columnBuilder = Column.newBuilder();
> for (Map.Entry>
> family: scan.getFamilyMap().entrySet()) {
>
> columnBuilder.setFamily(UnsafeByteOperations.unsafeWrap(family.getKey()));
> NavigableSet qualifiers = family.getValue();
> columnBuilder.clearQualifier();
> if (qualifiers != null && qualifiers.size() > 0) {
> for (byte [] qualifier: qualifiers) {
>
> columnBuilder.addQualifier(UnsafeByteOperations.unsafeWrap(qualifier));
> }
> }
> scanBuilder.addColumn(columnBuilder.build());
> }
> }
> if (scan.getMaxResultsPerColumnFamily() >= 0) {
> scanBuilder.setStoreLimit(scan.getMaxResultsPerColumnFamily());
> }
> if (scan.getRowOffsetPerColumnFamily() > 0) {
> scanBuilder.setStoreOffset(scan.getRowOffsetPerColumnFamily());
> }
> if (scan.isReversed()) {
> scanBuilder.setReversed(scan.isReversed());
> }
> if (scan.getConsistency() == Consistency.TIMELINE) {
> scanBuilder.setConsistency(toConsistency(scan.getConsistency()));
> }
> if (scan.getCaching() > 0) {
> scanBuilder.setCaching(scan.getCaching());
> }
> long mvccReadPoint = PackagePrivateFieldAccessor.getMvccReadPoint(scan);
> if (mvccReadPoint > 0) {
> scanBuilder.setMvccReadPoint(mvccReadPoint);
> }
> if (!scan.includeStartRow()) {
> scanBuilder.setIncludeStartRow(false);
> }
> scanBuilder.setIncludeStopRow(scan.includeStopRow());
> if (scan.getReadType() != Scan.ReadType.DEFAULT) {
> scanBuilder.setReadType(toReadType(scan.getReadType()));
> }
> if (scan.isNeedCursorResult()) {
> scanBuilder.setNeedCursorResult(true);
> }
> return scanBuilder.build();
> }
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Commented] (HBASE-24508) Why ProtobufUtil does not set scan's limit
[
https://issues.apache.org/jira/browse/HBASE-24508?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17126851#comment-17126851
]
Anoop Sam John commented on HBASE-24508:
We use only the shaded one for calls from client to RS
> Why ProtobufUtil does not set scan's limit
>
>
> Key: HBASE-24508
> URL: https://issues.apache.org/jira/browse/HBASE-24508
> Project: HBase
> Issue Type: Bug
>Affects Versions: 2.2.5
>Reporter: yukunpeng
>Priority: Trivial
>
> {code:java}
> //ProtobufUtil
> /**
> * Convert a client Scan to a protocol buffer Scan
> *
> * @param scan the client Scan to convert
> * @return the converted protocol buffer Scan
> * @throws IOException
> */
> public static ClientProtos.Scan toScan(
> final Scan scan) throws IOException {
> ClientProtos.Scan.Builder scanBuilder =
> ClientProtos.Scan.newBuilder();
> scanBuilder.setCacheBlocks(scan.getCacheBlocks());
> if (scan.getBatch() > 0) {
> scanBuilder.setBatchSize(scan.getBatch());
> }
> if (scan.getMaxResultSize() > 0) {
> scanBuilder.setMaxResultSize(scan.getMaxResultSize());
> }
> if (scan.isSmall()) {
> scanBuilder.setSmall(scan.isSmall());
> }
> if (scan.getAllowPartialResults()) {
> scanBuilder.setAllowPartialResults(scan.getAllowPartialResults());
> }
> Boolean loadColumnFamiliesOnDemand =
> scan.getLoadColumnFamiliesOnDemandValue();
> if (loadColumnFamiliesOnDemand != null) {
> scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand);
> }
> scanBuilder.setMaxVersions(scan.getMaxVersions());
> scan.getColumnFamilyTimeRange().forEach((cf, timeRange) -> {
> scanBuilder.addCfTimeRange(HBaseProtos.ColumnFamilyTimeRange.newBuilder()
> .setColumnFamily(UnsafeByteOperations.unsafeWrap(cf))
> .setTimeRange(toTimeRange(timeRange))
> .build());
> });
> scanBuilder.setTimeRange(ProtobufUtil.toTimeRange(scan.getTimeRange()));
> Map attributes = scan.getAttributesMap();
> if (!attributes.isEmpty()) {
> NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
> for (Map.Entry attribute: attributes.entrySet()) {
> attributeBuilder.setName(attribute.getKey());
>
> attributeBuilder.setValue(UnsafeByteOperations.unsafeWrap(attribute.getValue()));
> scanBuilder.addAttribute(attributeBuilder.build());
> }
> }
> byte[] startRow = scan.getStartRow();
> if (startRow != null && startRow.length > 0) {
> scanBuilder.setStartRow(UnsafeByteOperations.unsafeWrap(startRow));
> }
> byte[] stopRow = scan.getStopRow();
> if (stopRow != null && stopRow.length > 0) {
> scanBuilder.setStopRow(UnsafeByteOperations.unsafeWrap(stopRow));
> }
> if (scan.hasFilter()) {
> scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
> }
> if (scan.hasFamilies()) {
> Column.Builder columnBuilder = Column.newBuilder();
> for (Map.Entry>
> family: scan.getFamilyMap().entrySet()) {
>
> columnBuilder.setFamily(UnsafeByteOperations.unsafeWrap(family.getKey()));
> NavigableSet qualifiers = family.getValue();
> columnBuilder.clearQualifier();
> if (qualifiers != null && qualifiers.size() > 0) {
> for (byte [] qualifier: qualifiers) {
>
> columnBuilder.addQualifier(UnsafeByteOperations.unsafeWrap(qualifier));
> }
> }
> scanBuilder.addColumn(columnBuilder.build());
> }
> }
> if (scan.getMaxResultsPerColumnFamily() >= 0) {
> scanBuilder.setStoreLimit(scan.getMaxResultsPerColumnFamily());
> }
> if (scan.getRowOffsetPerColumnFamily() > 0) {
> scanBuilder.setStoreOffset(scan.getRowOffsetPerColumnFamily());
> }
> if (scan.isReversed()) {
> scanBuilder.setReversed(scan.isReversed());
> }
> if (scan.getConsistency() == Consistency.TIMELINE) {
> scanBuilder.setConsistency(toConsistency(scan.getConsistency()));
> }
> if (scan.getCaching() > 0) {
> scanBuilder.setCaching(scan.getCaching());
> }
> long mvccReadPoint = PackagePrivateFieldAccessor.getMvccReadPoint(scan);
> if (mvccReadPoint > 0) {
> scanBuilder.setMvccReadPoint(mvccReadPoint);
> }
> if (!scan.includeStartRow()) {
> scanBuilder.setIncludeStartRow(false);
> }
> scanBuilder.setIncludeStopRow(scan.includeStopRow());
> if (scan.getReadType() != Scan.ReadType.DEFAULT) {
> scanBuilder.setReadType(toReadType(scan.getReadType()));
> }
> if (scan.isNeedCursorResult()) {
> scanBuilder.setNeedCursorResult(true);
> }
> return scanBuilder.build();
> }
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
[jira] [Commented] (HBASE-24508) Why ProtobufUtil does not set scan's limit
[
https://issues.apache.org/jira/browse/HBASE-24508?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17126720#comment-17126720
]
Duo Zhang commented on HBASE-24508:
---
Which ProtobufUtil? The shaded one or the non shaded one?
> Why ProtobufUtil does not set scan's limit
>
>
> Key: HBASE-24508
> URL: https://issues.apache.org/jira/browse/HBASE-24508
> Project: HBase
> Issue Type: Bug
>Affects Versions: 2.2.5
>Reporter: yukunpeng
>Priority: Trivial
>
> {code:java}
> //ProtobufUtil
> /**
> * Convert a client Scan to a protocol buffer Scan
> *
> * @param scan the client Scan to convert
> * @return the converted protocol buffer Scan
> * @throws IOException
> */
> public static ClientProtos.Scan toScan(
> final Scan scan) throws IOException {
> ClientProtos.Scan.Builder scanBuilder =
> ClientProtos.Scan.newBuilder();
> scanBuilder.setCacheBlocks(scan.getCacheBlocks());
> if (scan.getBatch() > 0) {
> scanBuilder.setBatchSize(scan.getBatch());
> }
> if (scan.getMaxResultSize() > 0) {
> scanBuilder.setMaxResultSize(scan.getMaxResultSize());
> }
> if (scan.isSmall()) {
> scanBuilder.setSmall(scan.isSmall());
> }
> if (scan.getAllowPartialResults()) {
> scanBuilder.setAllowPartialResults(scan.getAllowPartialResults());
> }
> Boolean loadColumnFamiliesOnDemand =
> scan.getLoadColumnFamiliesOnDemandValue();
> if (loadColumnFamiliesOnDemand != null) {
> scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand);
> }
> scanBuilder.setMaxVersions(scan.getMaxVersions());
> scan.getColumnFamilyTimeRange().forEach((cf, timeRange) -> {
> scanBuilder.addCfTimeRange(HBaseProtos.ColumnFamilyTimeRange.newBuilder()
> .setColumnFamily(UnsafeByteOperations.unsafeWrap(cf))
> .setTimeRange(toTimeRange(timeRange))
> .build());
> });
> scanBuilder.setTimeRange(ProtobufUtil.toTimeRange(scan.getTimeRange()));
> Map attributes = scan.getAttributesMap();
> if (!attributes.isEmpty()) {
> NameBytesPair.Builder attributeBuilder = NameBytesPair.newBuilder();
> for (Map.Entry attribute: attributes.entrySet()) {
> attributeBuilder.setName(attribute.getKey());
>
> attributeBuilder.setValue(UnsafeByteOperations.unsafeWrap(attribute.getValue()));
> scanBuilder.addAttribute(attributeBuilder.build());
> }
> }
> byte[] startRow = scan.getStartRow();
> if (startRow != null && startRow.length > 0) {
> scanBuilder.setStartRow(UnsafeByteOperations.unsafeWrap(startRow));
> }
> byte[] stopRow = scan.getStopRow();
> if (stopRow != null && stopRow.length > 0) {
> scanBuilder.setStopRow(UnsafeByteOperations.unsafeWrap(stopRow));
> }
> if (scan.hasFilter()) {
> scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
> }
> if (scan.hasFamilies()) {
> Column.Builder columnBuilder = Column.newBuilder();
> for (Map.Entry>
> family: scan.getFamilyMap().entrySet()) {
>
> columnBuilder.setFamily(UnsafeByteOperations.unsafeWrap(family.getKey()));
> NavigableSet qualifiers = family.getValue();
> columnBuilder.clearQualifier();
> if (qualifiers != null && qualifiers.size() > 0) {
> for (byte [] qualifier: qualifiers) {
>
> columnBuilder.addQualifier(UnsafeByteOperations.unsafeWrap(qualifier));
> }
> }
> scanBuilder.addColumn(columnBuilder.build());
> }
> }
> if (scan.getMaxResultsPerColumnFamily() >= 0) {
> scanBuilder.setStoreLimit(scan.getMaxResultsPerColumnFamily());
> }
> if (scan.getRowOffsetPerColumnFamily() > 0) {
> scanBuilder.setStoreOffset(scan.getRowOffsetPerColumnFamily());
> }
> if (scan.isReversed()) {
> scanBuilder.setReversed(scan.isReversed());
> }
> if (scan.getConsistency() == Consistency.TIMELINE) {
> scanBuilder.setConsistency(toConsistency(scan.getConsistency()));
> }
> if (scan.getCaching() > 0) {
> scanBuilder.setCaching(scan.getCaching());
> }
> long mvccReadPoint = PackagePrivateFieldAccessor.getMvccReadPoint(scan);
> if (mvccReadPoint > 0) {
> scanBuilder.setMvccReadPoint(mvccReadPoint);
> }
> if (!scan.includeStartRow()) {
> scanBuilder.setIncludeStartRow(false);
> }
> scanBuilder.setIncludeStopRow(scan.includeStopRow());
> if (scan.getReadType() != Scan.ReadType.DEFAULT) {
> scanBuilder.setReadType(toReadType(scan.getReadType()));
> }
> if (scan.isNeedCursorResult()) {
> scanBuilder.setNeedCursorResult(true);
> }
> return scanBuilder.build();
> }
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
