This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 7808c651b75 HBASE-28613 Use streaming when marshalling protobuf REST
output (#5943)
7808c651b75 is described below
commit 7808c651b75435262c8bbffac65b33abfab662b3
Author: Istvan Toth <st...@apache.org>
AuthorDate: Tue May 28 10:21:24 2024 +0200
HBASE-28613 Use streaming when marshalling protobuf REST output (#5943)
Signed-off-by: Ankit Singhal <an...@apache.org>
(cherry picked from commit d1d8b4d64591de6cee302d648be257087c0beb48)
---
.../hadoop/hbase/rest/ProtobufMessageHandler.java | 43 +++++++++++++++++++++-
.../apache/hadoop/hbase/rest/model/CellModel.java | 6 ++-
.../hadoop/hbase/rest/model/CellSetModel.java | 6 ++-
.../hbase/rest/model/NamespacesInstanceModel.java | 6 ++-
.../hadoop/hbase/rest/model/NamespacesModel.java | 6 ++-
.../apache/hadoop/hbase/rest/model/RowModel.java | 4 +-
.../hadoop/hbase/rest/model/ScannerModel.java | 6 ++-
.../rest/model/StorageClusterStatusModel.java | 6 ++-
.../hadoop/hbase/rest/model/TableInfoModel.java | 6 ++-
.../hadoop/hbase/rest/model/TableListModel.java | 6 ++-
.../hadoop/hbase/rest/model/TableSchemaModel.java | 6 ++-
.../hadoop/hbase/rest/model/VersionModel.java | 6 ++-
.../producer/ProtobufMessageBodyProducer.java | 2 +-
13 files changed, 85 insertions(+), 24 deletions(-)
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java
index 2e01ff24d47..175af227407 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/ProtobufMessageHandler.java
@@ -18,21 +18,60 @@
package org.apache.hadoop.hbase.rest;
import java.io.IOException;
+import java.io.OutputStream;
import org.apache.yetus.audience.InterfaceAudience;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.Message;
+
/**
* Common interface for models capable of supporting protobuf marshalling and
unmarshalling. Hooks
* up to the ProtobufMessageBodyConsumer and ProtobufMessageBodyProducer
adapters.
*/
@InterfaceAudience.Private
public interface ProtobufMessageHandler {
- /** Returns the protobuf represention of the model */
- byte[] createProtobufOutput();
+
+ // The Jetty 9.4 HttpOutput default commit size is 32K/4 = 8K. We use that
size to avoid
+ // double buffering (and copying) in HttpOutput. If we ever increase the
HttpOutput commit size,
+ // we need to adjust this accordingly. We should also revisit this when
Jetty is upgraded.
+ int BUFFER_SIZE = 8 * 1024;
+
+ /** Writes the protobuf represention of the model to os */
+ default void writeProtobufOutput(OutputStream os) throws IOException {
+ // Creating an explicit CodedOutputStream for the following reasons :
+ // 1. This avoids the cost of pre-computing the message size
+ // 2. This lets us set the buffer size explicitly
+ CodedOutputStream cos = CodedOutputStream.newInstance(os, BUFFER_SIZE);
+ messageFromObject().writeTo(cos);
+ cos.flush();
+ }
+
+ /**
+ * Returns the protobuf represention of the model in a byte array Use
+ * {@link
org.apache.hadoop.hbase.rest.ProtobufMessageHandler#writeProtobufOutput(OutputStream)}
+ * for better performance
+ * @return the protobuf encoded object in a byte array
+ */
+ default byte[] createProtobufOutput() {
+ return messageFromObject().toByteArray();
+ }
+
+ /**
+ * Convert to model to a protobuf Message object
+ * @return the protobuf Message object
+ */
+ Message messageFromObject();
/**
* Initialize the model from a protobuf representation.
* @param message the raw bytes of the protobuf message
* @return reference to self for convenience
*/
+ // TODO implement proper stream handling for unmarshalling.
+ // Using byte array here lets us use ProtobufUtil.mergeFrom in the
implementations to
+ // avoid the CodedOutputStream size limitation, but is slow
+ // and memory intensive. We could use the ProtobufUtil.mergeFrom() variant
that takes
+ // an inputStream and sets the size limit to maxInt.
+ // This would help both on the client side, and when processing large Puts
on the server.
ProtobufMessageHandler getObjectFromMessage(byte[] message) throws
IOException;
}
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java
index fdfc0f38847..504366fdd01 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellModel.java
@@ -21,6 +21,8 @@ import static
org.apache.hadoop.hbase.KeyValue.COLUMN_FAMILY_DELIMITER;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.protobuf.Message;
+
import java.io.IOException;
import java.io.Serializable;
import javax.xml.bind.annotation.XmlAccessType;
@@ -200,7 +202,7 @@ public class CellModel implements ProtobufMessageHandler,
Serializable {
}
@Override
- public byte[] createProtobufOutput() {
+ public Message messageFromObject() {
Cell.Builder builder = Cell.newBuilder();
builder.setColumn(ByteStringer.wrap(getColumn()));
if (valueLength == MAGIC_LENGTH) {
@@ -211,7 +213,7 @@ public class CellModel implements ProtobufMessageHandler,
Serializable {
if (hasUserTimestamp()) {
builder.setTimestamp(getTimestamp());
}
- return builder.build().toByteArray();
+ return builder.build();
}
@Override
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java
index 832bd241c3b..c79732cee1b 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/CellSetModel.java
@@ -35,6 +35,8 @@ import
org.apache.hadoop.hbase.rest.protobuf.generated.CellSetMessage.CellSet;
import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.yetus.audience.InterfaceAudience;
+import com.google.protobuf.Message;
+
/**
* Representation of a grouping of cells. May contain cells from more than one
row. Encapsulates
* RowModel and CellModel models.
@@ -106,7 +108,7 @@ public class CellSetModel implements Serializable,
ProtobufMessageHandler {
}
@Override
- public byte[] createProtobufOutput() {
+ public Message messageFromObject() {
CellSet.Builder builder = CellSet.newBuilder();
for (RowModel row : getRows()) {
CellSet.Row.Builder rowBuilder = CellSet.Row.newBuilder();
@@ -132,7 +134,7 @@ public class CellSetModel implements Serializable,
ProtobufMessageHandler {
}
builder.addRows(rowBuilder);
}
- return builder.build().toByteArray();
+ return builder.build();
}
@Override
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java
index 39ea0b7c39b..afa91064c44 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesInstanceModel.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.hbase.rest.ProtobufMessageHandler;
import
org.apache.hadoop.hbase.rest.protobuf.generated.NamespacePropertiesMessage.NamespaceProperties;
import org.apache.yetus.audience.InterfaceAudience;
+import com.google.protobuf.Message;
+
/**
* List a HBase namespace's key/value properties.
* <ul>
@@ -139,7 +141,7 @@ public class NamespacesInstanceModel implements
Serializable, ProtobufMessageHan
}
@Override
- public byte[] createProtobufOutput() {
+ public Message messageFromObject() {
NamespaceProperties.Builder builder = NamespaceProperties.newBuilder();
if (properties != null) {
for (Map.Entry<String, String> entry : properties.entrySet()) {
@@ -150,7 +152,7 @@ public class NamespacesInstanceModel implements
Serializable, ProtobufMessageHan
builder.addProps(property);
}
}
- return builder.build().toByteArray();
+ return builder.build();
}
@Override
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesModel.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesModel.java
index 76a7b32e137..abaed1b6627 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesModel.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/NamespacesModel.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hbase.rest.model;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.protobuf.Message;
+
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
@@ -94,10 +96,10 @@ public class NamespacesModel implements Serializable,
ProtobufMessageHandler {
}
@Override
- public byte[] createProtobufOutput() {
+ public Message messageFromObject() {
Namespaces.Builder builder = Namespaces.newBuilder();
builder.addAllNamespace(namespaces);
- return builder.build().toByteArray();
+ return builder.build();
}
@Override
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java
index 8b660ac362f..0de08b76566 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/RowModel.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hbase.rest.model;
import static org.apache.hadoop.hbase.rest.model.CellModel.MAGIC_LENGTH;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.protobuf.Message;
+
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
@@ -179,7 +181,7 @@ public class RowModel implements ProtobufMessageHandler,
Serializable {
}
@Override
- public byte[] createProtobufOutput() {
+ public Message messageFromObject() {
// there is no standalone row protobuf message
throw new UnsupportedOperationException("no protobuf equivalent to
RowModel");
}
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
index 831c7849abb..bb5fb24da9a 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/ScannerModel.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hbase.rest.model;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.google.protobuf.ByteString;
+import com.google.protobuf.Message;
+
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
@@ -791,7 +793,7 @@ public class ScannerModel implements
ProtobufMessageHandler, Serializable {
}
@Override
- public byte[] createProtobufOutput() {
+ public Message messageFromObject() {
Scanner.Builder builder = Scanner.newBuilder();
if (!Bytes.equals(startRow, HConstants.EMPTY_START_ROW)) {
builder.setStartRow(ByteStringer.wrap(startRow));
@@ -821,7 +823,7 @@ public class ScannerModel implements
ProtobufMessageHandler, Serializable {
builder.addLabels(label);
}
builder.setCacheBlocks(cacheBlocks);
- return builder.build().toByteArray();
+ return builder.build();
}
@Override
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java
index 027534c2888..dd1a2bcbb49 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/StorageClusterStatusModel.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hbase.rest.model;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.protobuf.Message;
+
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
@@ -651,7 +653,7 @@ public class StorageClusterStatusModel implements
Serializable, ProtobufMessageH
}
@Override
- public byte[] createProtobufOutput() {
+ public Message messageFromObject() {
StorageClusterStatus.Builder builder = StorageClusterStatus.newBuilder();
builder.setRegions(regions);
builder.setRequests(requests);
@@ -686,7 +688,7 @@ public class StorageClusterStatusModel implements
Serializable, ProtobufMessageH
for (String node : deadNodes) {
builder.addDeadNodes(node);
}
- return builder.build().toByteArray();
+ return builder.build();
}
@Override
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java
index 6b39daaacd6..e86f355852c 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableInfoModel.java
@@ -30,6 +30,8 @@ import
org.apache.hadoop.hbase.rest.protobuf.generated.TableInfoMessage.TableInf
import org.apache.hadoop.hbase.util.ByteStringer;
import org.apache.yetus.audience.InterfaceAudience;
+import com.google.protobuf.Message;
+
/**
* Representation of a list of table regions.
*
@@ -121,7 +123,7 @@ public class TableInfoModel implements Serializable,
ProtobufMessageHandler {
}
@Override
- public byte[] createProtobufOutput() {
+ public Message messageFromObject() {
TableInfo.Builder builder = TableInfo.newBuilder();
builder.setName(name);
for (TableRegionModel aRegion : regions) {
@@ -133,7 +135,7 @@ public class TableInfoModel implements Serializable,
ProtobufMessageHandler {
regionBuilder.setLocation(aRegion.getLocation());
builder.addRegions(regionBuilder);
}
- return builder.build().toByteArray();
+ return builder.build();
}
@Override
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java
index 0b7ea10ab40..01909d72b0d 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableListModel.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.hbase.rest.ProtobufMessageHandler;
import
org.apache.hadoop.hbase.rest.protobuf.generated.TableListMessage.TableList;
import org.apache.yetus.audience.InterfaceAudience;
+import com.google.protobuf.Message;
+
/**
* Simple representation of a list of table names.
*/
@@ -89,12 +91,12 @@ public class TableListModel implements Serializable,
ProtobufMessageHandler {
}
@Override
- public byte[] createProtobufOutput() {
+ public Message messageFromObject() {
TableList.Builder builder = TableList.newBuilder();
for (TableModel aTable : tables) {
builder.addName(aTable.getName());
}
- return builder.build().toByteArray();
+ return builder.build();
}
@Override
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java
index 1756db20d02..85a169fe819 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/TableSchemaModel.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hbase.rest.model;
import com.fasterxml.jackson.annotation.JsonAnyGetter;
import com.fasterxml.jackson.annotation.JsonAnySetter;
import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.google.protobuf.Message;
+
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
@@ -245,7 +247,7 @@ public class TableSchemaModel implements Serializable,
ProtobufMessageHandler {
}
@Override
- public byte[] createProtobufOutput() {
+ public Message messageFromObject() {
TableSchema.Builder builder = TableSchema.newBuilder();
builder.setName(name);
for (Map.Entry<QName, Object> e : attrs.entrySet()) {
@@ -278,7 +280,7 @@ public class TableSchemaModel implements Serializable,
ProtobufMessageHandler {
if (attrs.containsKey(READONLY)) {
builder.setReadOnly(Boolean.parseBoolean(attrs.get(READONLY).toString()));
}
- return builder.build().toByteArray();
+ return builder.build();
}
@Override
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java
index 74bd722d36f..ed97c215c1e 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/model/VersionModel.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.hbase.rest.RESTServlet;
import org.apache.hadoop.hbase.rest.protobuf.generated.VersionMessage.Version;
import org.apache.yetus.audience.InterfaceAudience;
+import com.google.protobuf.Message;
+
import
org.apache.hbase.thirdparty.org.glassfish.jersey.servlet.ServletContainer;
/**
@@ -161,14 +163,14 @@ public class VersionModel implements Serializable,
ProtobufMessageHandler {
}
@Override
- public byte[] createProtobufOutput() {
+ public Message messageFromObject() {
Version.Builder builder = Version.newBuilder();
builder.setRestVersion(restVersion);
builder.setJvmVersion(jvmVersion);
builder.setOsVersion(osVersion);
builder.setServerVersion(serverVersion);
builder.setJerseyVersion(jerseyVersion);
- return builder.build().toByteArray();
+ return builder.build();
}
@Override
diff --git
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java
index 1d95e6f343e..4a7806e652b 100644
---
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java
+++
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/provider/producer/ProtobufMessageBodyProducer.java
@@ -59,6 +59,6 @@ public class ProtobufMessageBodyProducer implements
MessageBodyWriter<ProtobufMe
public void writeTo(ProtobufMessageHandler m, Class<?> type, Type
genericType,
Annotation[] annotations, MediaType mediaType, MultivaluedMap<String,
Object> httpHeaders,
OutputStream entityStream) throws IOException, WebApplicationException {
- entityStream.write(m.createProtobufOutput());
+ m.writeProtobufOutput(entityStream);
}
}
