This is an automated email from the ASF dual-hosted git repository. stoty pushed a commit to branch branch-2.5 in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push: new fcae9c74c3b HBASE-28174 (DELETE endpoint in REST API does not support deleting binary row keys/columns fcae9c74c3b is described below commit fcae9c74c3b1f692716a4fba668226fa924014e8 Author: judilsteve <4328790+judilst...@users.noreply.github.com> AuthorDate: Sat Nov 18 02:23:11 2023 +0800 HBASE-28174 (DELETE endpoint in REST API does not support deleting binary row keys/columns Signed-off-by: Wellington Chevreuil <wchevre...@apache.org> Change-Id: I58c12cabc292415b3310cc3cebf71da98694c1b0 --- .../org/apache/hadoop/hbase/rest/Constants.java | 1 + .../apache/hadoop/hbase/rest/MultiRowResource.java | 9 ++- .../org/apache/hadoop/hbase/rest/RowResource.java | 4 +- .../java/org/apache/hadoop/hbase/rest/RowSpec.java | 33 ++++++++++ .../apache/hadoop/hbase/rest/TableResource.java | 16 +++-- .../apache/hadoop/hbase/rest/RowResourceBase.java | 54 ++++++++++++++++ .../apache/hadoop/hbase/rest/TestDeleteRow.java | 34 +++++++++++ .../hadoop/hbase/rest/TestGetAndPutResource.java | 68 +++++++++++++++++++++ .../hadoop/hbase/rest/TestMultiRowResource.java | 71 ++++++++++++++++++++++ 9 files changed, 282 insertions(+), 8 deletions(-) diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java index af8b9e303bd..f0d1edc986a 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java @@ -85,6 +85,7 @@ public interface Constants { String CUSTOM_FILTERS = "hbase.rest.custom.filters"; String ROW_KEYS_PARAM_NAME = "row"; + String KEY_ENCODING_QUERY_PARAM_NAME = "e"; /** * If this query parameter is present when processing row or scanner resources, it disables server * side block caching diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java index cc5fb22265c..82900135dc4 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java @@ -29,6 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.javax.ws.rs.GET; +import org.apache.hbase.thirdparty.javax.ws.rs.HeaderParam; import org.apache.hbase.thirdparty.javax.ws.rs.Produces; import org.apache.hbase.thirdparty.javax.ws.rs.core.Context; import org.apache.hbase.thirdparty.javax.ws.rs.core.MultivaluedMap; @@ -63,14 +64,18 @@ public class MultiRowResource extends ResourceBase implements Constants { @GET @Produces({ MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, MIMETYPE_PROTOBUF_IETF }) - public Response get(final @Context UriInfo uriInfo) { + public Response get(final @Context UriInfo uriInfo, + final @HeaderParam("Encoding") String keyEncodingHeader) { MultivaluedMap<String, String> params = uriInfo.getQueryParameters(); + String keyEncoding = (keyEncodingHeader != null) + ? keyEncodingHeader + : params.getFirst(KEY_ENCODING_QUERY_PARAM_NAME); servlet.getMetrics().incrementRequests(1); try { CellSetModel model = new CellSetModel(); for (String rk : params.get(ROW_KEYS_PARAM_NAME)) { - RowSpec rowSpec = new RowSpec(rk); + RowSpec rowSpec = new RowSpec(rk, keyEncoding); if (this.versions != null) { rowSpec.setMaxVersions(this.versions); diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java index df4664b76c5..e708657963d 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowResource.java @@ -72,10 +72,10 @@ public class RowResource extends ResourceBase { * Constructor */ public RowResource(TableResource tableResource, String rowspec, String versions, String check, - String returnResult) throws IOException { + String returnResult, String keyEncoding) throws IOException { super(); this.tableResource = tableResource; - this.rowspec = new RowSpec(rowspec); + this.rowspec = new RowSpec(rowspec, keyEncoding); if (versions != null) { this.rowspec.setMaxVersions(Integer.parseInt(versions)); } diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java index c9993336fa1..e1559dd6723 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/RowSpec.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.rest; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.util.ArrayList; +import java.util.Base64; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -47,6 +48,10 @@ public class RowSpec { private int maxValues = Integer.MAX_VALUE; public RowSpec(String path) throws IllegalArgumentException { + this(path, null); + } + + public RowSpec(String path, String keyEncoding) throws IllegalArgumentException { int i = 0; while (path.charAt(i) == '/') { i++; @@ -55,6 +60,34 @@ public class RowSpec { i = parseColumns(path, i); i = parseTimestamp(path, i); i = parseQueryParams(path, i); + + if (keyEncoding != null) { + // See https://en.wikipedia.org/wiki/Base64#Variants_summary_table + Base64.Decoder decoder; + switch (keyEncoding) { + case "b64": + case "base64": + case "b64url": + case "base64url": + decoder = Base64.getUrlDecoder(); + break; + case "b64basic": + case "base64basic": + decoder = Base64.getDecoder(); + break; + default: + throw new IllegalArgumentException("unknown key encoding '" + keyEncoding + "'"); + } + this.row = decoder.decode(this.row); + if (this.endRow != null) { + this.endRow = decoder.decode(this.endRow); + } + TreeSet<byte[]> decodedColumns = new TreeSet<>(Bytes.BYTES_COMPARATOR); + for (byte[] encodedColumn : this.columns) { + decodedColumns.add(decoder.decode(encodedColumn)); + } + this.columns = decodedColumns; + } } private int parseRowKeys(final String path, int i) throws IllegalArgumentException { diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java index 24ced31d36a..0fe71a26513 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java @@ -35,6 +35,7 @@ import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.javax.ws.rs.DefaultValue; import org.apache.hbase.thirdparty.javax.ws.rs.Encoded; +import org.apache.hbase.thirdparty.javax.ws.rs.HeaderParam; import org.apache.hbase.thirdparty.javax.ws.rs.Path; import org.apache.hbase.thirdparty.javax.ws.rs.PathParam; import org.apache.hbase.thirdparty.javax.ws.rs.QueryParam; @@ -94,9 +95,12 @@ public class TableResource extends ResourceBase { // We need the @Encoded decorator so Jersey won't urldecode before // the RowSpec constructor has a chance to parse final @PathParam("rowspec") @Encoded String rowspec, final @QueryParam("v") String versions, - final @QueryParam("check") String check, final @QueryParam("rr") String returnResult) + final @QueryParam("check") String check, final @QueryParam("rr") String returnResult, + final @HeaderParam("Encoding") String keyEncodingHeader, + final @QueryParam(Constants.KEY_ENCODING_QUERY_PARAM_NAME) String keyEncodingQuery) throws IOException { - return new RowResource(this, rowspec, versions, check, returnResult); + String keyEncoding = (keyEncodingHeader != null) ? keyEncodingHeader : keyEncodingQuery; + return new RowResource(this, rowspec, versions, check, returnResult, keyEncoding); } @Path("{suffixglobbingspec: .*\\*/.+}") @@ -105,8 +109,12 @@ public class TableResource extends ResourceBase { // the RowSpec constructor has a chance to parse final @PathParam("suffixglobbingspec") @Encoded String suffixglobbingspec, final @QueryParam("v") String versions, final @QueryParam("check") String check, - final @QueryParam("rr") String returnResult) throws IOException { - return new RowResource(this, suffixglobbingspec, versions, check, returnResult); + final @QueryParam("rr") String returnResult, + final @HeaderParam("Encoding") String keyEncodingHeader, + final @QueryParam(Constants.KEY_ENCODING_QUERY_PARAM_NAME) String keyEncodingQuery) + throws IOException { + String keyEncoding = (keyEncodingHeader != null) ? keyEncodingHeader : keyEncodingQuery; + return new RowResource(this, suffixglobbingspec, versions, check, returnResult, keyEncoding); } @Path("{scanspec: .*[*]$}") diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/RowResourceBase.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/RowResourceBase.java index 774d7eaba29..1e563512b62 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/RowResourceBase.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/RowResourceBase.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.StringWriter; +import java.util.Base64; import java.util.HashMap; import java.util.Map; import javax.xml.bind.JAXBContext; @@ -42,6 +43,8 @@ import org.apache.hadoop.hbase.rest.model.CellModel; import org.apache.hadoop.hbase.rest.model.CellSetModel; import org.apache.hadoop.hbase.rest.model.RowModel; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -459,6 +462,15 @@ public class RowResourceBase { return response; } + protected static Response getValueXML(String url, Header[] headers) throws IOException { + Header[] fullHeaders = new Header[headers.length + 1]; + for (int i = 0; i < headers.length; i++) + fullHeaders[i] = headers[i]; + fullHeaders[headers.length] = new BasicHeader("Accept", Constants.MIMETYPE_XML); + Response response = client.get(url, fullHeaders); + return response; + } + protected static Response getValueJson(String url) throws IOException { Response response = client.get(url, Constants.MIMETYPE_JSON); return response; @@ -478,6 +490,28 @@ public class RowResourceBase { return response; } + protected static Response deleteValueB64(String table, String row, String column, + boolean useQueryString) throws IOException { + StringBuilder path = new StringBuilder(); + Base64.Encoder encoder = Base64.getUrlEncoder(); + path.append('/'); + path.append(table); + path.append('/'); + path.append(encoder.encodeToString(row.getBytes("UTF-8"))); + path.append('/'); + path.append(encoder.encodeToString(column.getBytes("UTF-8"))); + + Response response; + if (useQueryString) { + path.append("?e=b64"); + response = client.delete(path.toString()); + } else { + response = client.delete(path.toString(), new BasicHeader("Encoding", "b64")); + } + Thread.yield(); + return response; + } + protected static Response getValueXML(String table, String row, String column) throws IOException { StringBuilder path = new StringBuilder(); @@ -501,6 +535,26 @@ public class RowResourceBase { return response; } + protected static Response deleteRowB64(String table, String row, boolean useQueryString) + throws IOException { + StringBuilder path = new StringBuilder(); + Base64.Encoder encoder = Base64.getUrlEncoder(); + path.append('/'); + path.append(table); + path.append('/'); + path.append(encoder.encodeToString(row.getBytes("UTF-8"))); + + Response response; + if (useQueryString) { + path.append("?e=b64"); + response = client.delete(path.toString()); + } else { + response = client.delete(path.toString(), new BasicHeader("Encoding", "b64")); + } + Thread.yield(); + return response; + } + protected static Response getValueJson(String table, String row, String column) throws IOException { StringBuilder path = new StringBuilder(); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestDeleteRow.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestDeleteRow.java index 9d9d2f33769..9893a9ef67d 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestDeleteRow.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestDeleteRow.java @@ -100,4 +100,38 @@ public class TestDeleteRow extends RowResourceBase { assertEquals(404, response.getCode()); } + private void testDeleteB64XML(boolean useQueryString) throws IOException, JAXBException { + Response response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(200, response.getCode()); + response = putValueXML(TABLE, ROW_1, COLUMN_2, VALUE_2); + assertEquals(200, response.getCode()); + checkValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1); + checkValueXML(TABLE, ROW_1, COLUMN_2, VALUE_2); + + response = deleteValueB64(TABLE, ROW_1, COLUMN_1, useQueryString); + assertEquals(200, response.getCode()); + response = getValueXML(TABLE, ROW_1, COLUMN_1); + assertEquals(404, response.getCode()); + checkValueXML(TABLE, ROW_1, COLUMN_2, VALUE_2); + + response = putValueXML(TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(200, response.getCode()); + response = checkAndDeletePB(TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(200, response.getCode()); + response = getValueXML(TABLE, ROW_1, COLUMN_1); + assertEquals(404, response.getCode()); + + response = deleteRowB64(TABLE, ROW_1, useQueryString); + assertEquals(200, response.getCode()); + response = getValueXML(TABLE, ROW_1, COLUMN_1); + assertEquals(404, response.getCode()); + response = getValueXML(TABLE, ROW_1, COLUMN_2); + assertEquals(404, response.getCode()); + } + + @Test + public void testDeleteB64XML() throws IOException, JAXBException { + testDeleteB64XML(/* useQueryString: */false); + testDeleteB64XML(/* useQueryString: */true); + } } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java index b2c45e8cbd7..d14c45e0532 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestGetAndPutResource.java @@ -24,6 +24,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.StringWriter; import java.net.URLEncoder; +import java.util.Base64; import java.util.HashMap; import java.util.List; import javax.xml.bind.JAXBException; @@ -40,6 +41,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RestTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.http.Header; +import org.apache.http.message.BasicHeader; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -333,6 +335,72 @@ public class TestGetAndPutResource extends RowResourceBase { checkValueXML(path.toString(), TABLE, urlKey, COLUMN_1, VALUE_1); } + private void setupValue1() throws IOException, JAXBException { + StringBuilder path = new StringBuilder(); + path.append('/'); + path.append(TABLE); + path.append('/'); + path.append(ROW_1); + path.append('/'); + path.append(COLUMN_1); + Response response = putValueXML(path.toString(), TABLE, ROW_1, COLUMN_1, VALUE_1); + assertEquals(200, response.getCode()); + } + + private void checkValue1(Response getResponse) throws JAXBException { + assertEquals(Constants.MIMETYPE_XML, getResponse.getHeader("content-type")); + + CellSetModel cellSet = + (CellSetModel) xmlUnmarshaller.unmarshal(new ByteArrayInputStream(getResponse.getBody())); + assertEquals(1, cellSet.getRows().size()); + RowModel rowModel = cellSet.getRows().get(0); + assertEquals(ROW_1, new String(rowModel.getKey())); + assertEquals(1, rowModel.getCells().size()); + CellModel cell = rowModel.getCells().get(0); + assertEquals(COLUMN_1, new String(cell.getColumn())); + assertEquals(VALUE_1, new String(cell.getValue())); + } + + // See https://issues.apache.org/jira/browse/HBASE-28174 + @Test + public void testUrlB64EncodedKeyQueryParam() throws IOException, JAXBException { + setupValue1(); + + StringBuilder path = new StringBuilder(); + Base64.Encoder encoder = Base64.getUrlEncoder(); + path.append('/'); + path.append(TABLE); + path.append('/'); + path.append(encoder.encodeToString(ROW_1.getBytes("UTF-8"))); + path.append('/'); + path.append(encoder.encodeToString(COLUMN_1.getBytes("UTF-8"))); + path.append("?e=b64"); + Response response = getValueXML(path.toString()); + assertEquals(200, response.getCode()); + + checkValue1(response); + } + + // See https://issues.apache.org/jira/browse/HBASE-28174 + @Test + public void testUrlB64EncodedKeyHeader() throws IOException, JAXBException { + setupValue1(); + + StringBuilder path = new StringBuilder(); + Base64.Encoder encoder = Base64.getUrlEncoder(); + path.append('/'); + path.append(TABLE); + path.append('/'); + path.append(encoder.encodeToString(ROW_1.getBytes("UTF-8"))); + path.append('/'); + path.append(encoder.encodeToString(COLUMN_1.getBytes("UTF-8"))); + Response response = + getValueXML(path.toString(), new Header[] { new BasicHeader("Encoding", "b64") }); + assertEquals(200, response.getCode()); + + checkValue1(response); + } + @Test public void testNoSuchCF() throws IOException { final String goodPath = "/" + TABLE + "/" + ROW_1 + "/" + CFA + ":"; diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java index 61734734871..9a72f3d7032 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; +import java.util.Base64; import java.util.Collection; import javax.xml.bind.JAXBContext; import javax.xml.bind.Marshaller; @@ -152,6 +153,76 @@ public class TestMultiRowResource { client.delete(row_6_url, extraHdr); } + private void checkMultiCellGetJSON(Response response) throws IOException { + assertEquals(200, response.getCode()); + assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type")); + + ObjectMapper mapper = new JacksonJaxbJsonProvider().locateMapper(CellSetModel.class, + MediaType.APPLICATION_JSON_TYPE); + CellSetModel cellSet = mapper.readValue(response.getBody(), CellSetModel.class); + + RowModel rowModel = cellSet.getRows().get(0); + assertEquals(ROW_1, new String(rowModel.getKey())); + assertEquals(1, rowModel.getCells().size()); + CellModel cell = rowModel.getCells().get(0); + assertEquals(COLUMN_1, new String(cell.getColumn())); + assertEquals(VALUE_1, new String(cell.getValue())); + + rowModel = cellSet.getRows().get(1); + assertEquals(ROW_2, new String(rowModel.getKey())); + assertEquals(1, rowModel.getCells().size()); + cell = rowModel.getCells().get(0); + assertEquals(COLUMN_2, new String(cell.getColumn())); + assertEquals(VALUE_2, new String(cell.getValue())); + } + + // See https://issues.apache.org/jira/browse/HBASE-28174 + @Test + public void testMultiCellGetJSONB64() throws IOException { + String row_5_url = "/" + TABLE + "/" + ROW_1 + "/" + COLUMN_1; + String row_6_url = "/" + TABLE + "/" + ROW_2 + "/" + COLUMN_2; + + if (csrfEnabled) { + Response response = client.post(row_5_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_1)); + assertEquals(400, response.getCode()); + } + + client.post(row_5_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_1), extraHdr); + client.post(row_6_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_2), extraHdr); + + StringBuilder path = new StringBuilder(); + Base64.Encoder encoder = Base64.getUrlEncoder(); + path.append("/"); + path.append(TABLE); + path.append("/multiget/?row="); + path.append(encoder.encodeToString(ROW_1.getBytes("UTF-8"))); + path.append("&row="); + path.append(encoder.encodeToString(ROW_2.getBytes("UTF-8"))); + path.append("&e=b64"); // Specify encoding via query string + + Response response = client.get(path.toString(), Constants.MIMETYPE_JSON); + + checkMultiCellGetJSON(response); + + path = new StringBuilder(); + path.append("/"); + path.append(TABLE); + path.append("/multiget/?row="); + path.append(encoder.encodeToString(ROW_1.getBytes("UTF-8"))); + path.append("&row="); + path.append(encoder.encodeToString(ROW_2.getBytes("UTF-8"))); + + Header[] headers = new Header[] { new BasicHeader("Accept", Constants.MIMETYPE_JSON), + new BasicHeader("Encoding", "b64") // Specify encoding via header + }; + response = client.get(path.toString(), headers); + + checkMultiCellGetJSON(response); + + client.delete(row_5_url, extraHdr); + client.delete(row_6_url, extraHdr); + } + @Test public void testMultiCellGetXML() throws IOException { String row_5_url = "/" + TABLE + "/" + ROW_1 + "/" + COLUMN_1;