This is an automated email from the ASF dual-hosted git repository. stoty pushed a commit to branch branch-2.5 in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push: new 4d3a8b8dfd6 HBASE-28518 Allow specifying a filter for the REST multiget endpoint 4d3a8b8dfd6 is described below commit 4d3a8b8dfd6274a26bf72876e93d4e2f3fd164de Author: Istvan Toth <st...@apache.org> AuthorDate: Fri Apr 12 17:29:42 2024 +0200 HBASE-28518 Allow specifying a filter for the REST multiget endpoint Signed-off-by: Ankit Singhal <an...@apache.org> --- .../apache/hadoop/hbase/filter/ParseFilter.java | 2 +- .../org/apache/hadoop/hbase/rest/Constants.java | 3 +- .../apache/hadoop/hbase/rest/MultiRowResource.java | 31 ++++++++- .../apache/hadoop/hbase/rest/TableResource.java | 29 ++++++-- .../hadoop/hbase/rest/TestMultiRowResource.java | 81 +++++++++++++++++++--- .../apache/hadoop/hbase/rest/TestTableScan.java | 40 +++++++++-- 6 files changed, 160 insertions(+), 26 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java index b4f24ef97d0..18b97bb4eff 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/ParseFilter.java @@ -42,7 +42,7 @@ import org.slf4j.LoggerFactory; * of this class and a filter object is constructed. This filter object is then wrapped in a scanner * object which is then returned * <p> - * This class addresses the HBASE-4168 JIRA. More documentation on this Filter Language can be found + * This class addresses the HBASE-4176 JIRA. More documentation on this Filter Language can be found * at: https://issues.apache.org/jira/browse/HBASE-4176 */ @InterfaceAudience.Public diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java index e924be3d7fe..71080de07dd 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/Constants.java @@ -85,7 +85,8 @@ public interface Constants { String SCAN_BATCH_SIZE = "batchsize"; String SCAN_LIMIT = "limit"; String SCAN_FETCH_SIZE = "hbase.rest.scan.fetchsize"; - String SCAN_FILTER = "filter"; + String FILTER = "filter"; + String FILTER_B64 = "filter_b64"; String SCAN_REVERSED = "reversed"; String SCAN_CACHE_BLOCKS = "cacheblocks"; String CUSTOM_FILTERS = "hbase.rest.custom.filters"; diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java index 82900135dc4..47b3c22a7c9 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java @@ -18,8 +18,12 @@ package org.apache.hadoop.hbase.rest; import java.io.IOException; +import java.util.Base64; +import java.util.Base64.Decoder; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.ParseFilter; import org.apache.hadoop.hbase.rest.model.CellModel; import org.apache.hadoop.hbase.rest.model.CellSetModel; import org.apache.hadoop.hbase.rest.model.RowModel; @@ -28,9 +32,11 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.javax.ws.rs.Encoded; import org.apache.hbase.thirdparty.javax.ws.rs.GET; import org.apache.hbase.thirdparty.javax.ws.rs.HeaderParam; import org.apache.hbase.thirdparty.javax.ws.rs.Produces; +import org.apache.hbase.thirdparty.javax.ws.rs.QueryParam; import org.apache.hbase.thirdparty.javax.ws.rs.core.Context; import org.apache.hbase.thirdparty.javax.ws.rs.core.MultivaluedMap; import org.apache.hbase.thirdparty.javax.ws.rs.core.Response; @@ -40,6 +46,8 @@ import org.apache.hbase.thirdparty.javax.ws.rs.core.UriInfo; public class MultiRowResource extends ResourceBase implements Constants { private static final Logger LOG = LoggerFactory.getLogger(MultiRowResource.class); + private static final Decoder base64Urldecoder = Base64.getUrlDecoder(); + TableResource tableResource; Integer versions = null; String[] columns = null; @@ -65,15 +73,34 @@ public class MultiRowResource extends ResourceBase implements Constants { @GET @Produces({ MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF, MIMETYPE_PROTOBUF_IETF }) public Response get(final @Context UriInfo uriInfo, - final @HeaderParam("Encoding") String keyEncodingHeader) { + final @HeaderParam("Encoding") String keyEncodingHeader, + @QueryParam(Constants.FILTER_B64) @Encoded String paramFilterB64, + @QueryParam(Constants.FILTER) String paramFilter) { MultivaluedMap<String, String> params = uriInfo.getQueryParameters(); String keyEncoding = (keyEncodingHeader != null) ? keyEncodingHeader : params.getFirst(KEY_ENCODING_QUERY_PARAM_NAME); servlet.getMetrics().incrementRequests(1); + + byte[] filterBytes = null; + if (paramFilterB64 != null) { + filterBytes = base64Urldecoder.decode(paramFilterB64); + } else if (paramFilter != null) { + // Not binary clean + filterBytes = paramFilter.getBytes(); + } + try { + Filter parsedParamFilter = null; + if (filterBytes != null) { + // Note that this is a completely different representation of the filters + // than the JSON one used in the /table/scanner endpoint + ParseFilter pf = new ParseFilter(); + parsedParamFilter = pf.parseFilterString(filterBytes); + } CellSetModel model = new CellSetModel(); + // TODO map this to a Table.get(List<Get> gets) call instead of multiple get calls for (String rk : params.get(ROW_KEYS_PARAM_NAME)) { RowSpec rowSpec = new RowSpec(rk, keyEncoding); @@ -88,7 +115,7 @@ public class MultiRowResource extends ResourceBase implements Constants { } ResultGenerator generator = ResultGenerator.fromRowSpec(this.tableResource.getName(), - rowSpec, null, !params.containsKey(NOCACHE_PARAM_NAME)); + rowSpec, parsedParamFilter, !params.containsKey(NOCACHE_PARAM_NAME)); Cell value = null; RowModel rowModel = new RowModel(rowSpec.getRow()); if (generator.hasNext()) { diff --git a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java index 0fe71a26513..8bfc0455303 100644 --- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java +++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hbase.rest; import java.io.IOException; +import java.util.Base64.Decoder; import java.util.List; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Scan; @@ -46,6 +46,8 @@ public class TableResource extends ResourceBase { String table; private static final Logger LOG = LoggerFactory.getLogger(TableResource.class); + private static final Decoder base64Urldecoder = java.util.Base64.getUrlDecoder(); + /** * Constructor */ @@ -103,6 +105,7 @@ public class TableResource extends ResourceBase { return new RowResource(this, rowspec, versions, check, returnResult, keyEncoding); } + // TODO document @Path("{suffixglobbingspec: .*\\*/.+}") public RowResource getRowResourceWithSuffixGlobbing( // We need the @Encoded decorator so Jersey won't urldecode before @@ -117,6 +120,8 @@ public class TableResource extends ResourceBase { return new RowResource(this, suffixglobbingspec, versions, check, returnResult, keyEncoding); } + // TODO document + // FIXME handle binary rowkeys (like put and delete does) @Path("{scanspec: .*[*]$}") public TableScanResource getScanResource(final @PathParam("scanspec") String scanSpec, @DefaultValue(Integer.MAX_VALUE + "") @QueryParam(Constants.SCAN_LIMIT) int userRequestedLimit, @@ -129,7 +134,8 @@ public class TableResource extends ResourceBase { @DefaultValue(Long.MAX_VALUE + "") @QueryParam(Constants.SCAN_END_TIME) long endTime, @DefaultValue("true") @QueryParam(Constants.SCAN_CACHE_BLOCKS) boolean cacheBlocks, @DefaultValue("false") @QueryParam(Constants.SCAN_REVERSED) boolean reversed, - @DefaultValue("") @QueryParam(Constants.SCAN_FILTER) String paramFilter) { + @QueryParam(Constants.FILTER) String paramFilter, + @QueryParam(Constants.FILTER_B64) @Encoded String paramFilterB64) { try { Filter prefixFilter = null; Scan tableScan = new Scan(); @@ -173,15 +179,24 @@ public class TableResource extends ResourceBase { } } FilterList filterList = new FilterList(); - if (StringUtils.isNotEmpty(paramFilter)) { + byte[] filterBytes = null; + if (paramFilterB64 != null) { + filterBytes = base64Urldecoder.decode(paramFilterB64); + } else if (paramFilter != null) { + // Not binary clean + filterBytes = paramFilter.getBytes(); + } + if (filterBytes != null) { + // Note that this is a completely different representation of the filters + // than the JSON one used in the /table/scanner endpoint ParseFilter pf = new ParseFilter(); - Filter parsedParamFilter = pf.parseFilterString(paramFilter); + Filter parsedParamFilter = pf.parseFilterString(filterBytes); if (parsedParamFilter != null) { filterList.addFilter(parsedParamFilter); } - if (prefixFilter != null) { - filterList.addFilter(prefixFilter); - } + } + if (prefixFilter != null) { + filterList.addFilter(prefixFilter); } if (filterList.size() > 0) { tableScan.setFilter(filterList); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java index 9a72f3d7032..ee8976ddfcf 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java @@ -21,11 +21,11 @@ import static org.junit.Assert.assertEquals; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.Base64.Encoder; import java.util.Collection; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.Marshaller; -import javax.xml.bind.Unmarshaller; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; @@ -76,10 +76,9 @@ public class TestMultiRowResource { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility(); + private static final Encoder base64UrlEncoder = java.util.Base64.getUrlEncoder(); + private static Client client; - private static JAXBContext context; - private static Marshaller marshaller; - private static Unmarshaller unmarshaller; private static Configuration conf; private static Header extraHdr = null; @@ -104,9 +103,6 @@ public class TestMultiRowResource { extraHdr = new BasicHeader(RESTServer.REST_CSRF_CUSTOM_HEADER_DEFAULT, ""); TEST_UTIL.startMiniCluster(); REST_TEST_UTIL.startServletContainer(conf); - context = JAXBContext.newInstance(CellModel.class, CellSetModel.class, RowModel.class); - marshaller = context.createMarshaller(); - unmarshaller = context.createUnmarshaller(); client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())); Admin admin = TEST_UTIL.getAdmin(); if (admin.tableExists(TABLE)) { @@ -336,4 +332,71 @@ public class TestMultiRowResource { client.delete(row_5_url, extraHdr); client.delete(row_6_url, extraHdr); } + + @Test + public void testMultiCellGetFilterJSON() throws IOException { + String row_5_url = "/" + TABLE + "/" + ROW_1 + "/" + COLUMN_1; + String row_6_url = "/" + TABLE + "/" + ROW_2 + "/" + COLUMN_2; + + StringBuilder path = new StringBuilder(); + path.append("/"); + path.append(TABLE); + path.append("/multiget/?row="); + path.append(ROW_1); + path.append("&row="); + path.append(ROW_2); + + if (csrfEnabled) { + Response response = client.post(row_5_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_1)); + assertEquals(400, response.getCode()); + } + + client.post(row_5_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_1), extraHdr); + client.post(row_6_url, Constants.MIMETYPE_BINARY, Bytes.toBytes(VALUE_2), extraHdr); + + Response response = client.get(path.toString(), Constants.MIMETYPE_JSON); + assertEquals(200, response.getCode()); + assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type")); + + // If the filter is used, then we get the same result + String positivePath = path.toString() + ("&" + Constants.FILTER_B64 + "=" + base64UrlEncoder + .encodeToString("PrefixFilter('testrow')".getBytes(StandardCharsets.UTF_8.toString()))); + response = client.get(positivePath, Constants.MIMETYPE_JSON); + checkMultiCellGetJSON(response); + + // Same with non binary clean param + positivePath = path.toString() + ("&" + Constants.FILTER + "=" + + URLEncoder.encode("PrefixFilter('testrow')", StandardCharsets.UTF_8.name())); + response = client.get(positivePath, Constants.MIMETYPE_JSON); + checkMultiCellGetJSON(response); + + // This filter doesn't match the found rows + String negativePath = path.toString() + ("&" + Constants.FILTER_B64 + "=" + base64UrlEncoder + .encodeToString("PrefixFilter('notfound')".getBytes(StandardCharsets.UTF_8.toString()))); + response = client.get(negativePath, Constants.MIMETYPE_JSON); + assertEquals(404, response.getCode()); + + // Same with non binary clean param + negativePath = path.toString() + ("&" + Constants.FILTER + "=" + + URLEncoder.encode("PrefixFilter('notfound')", StandardCharsets.UTF_8.name())); + response = client.get(negativePath, Constants.MIMETYPE_JSON); + assertEquals(404, response.getCode()); + + // Check with binary parameters + // positive case + positivePath = path.toString() + ("&" + Constants.FILTER_B64 + "=" + base64UrlEncoder + .encodeToString(Bytes.toBytesBinary("ColumnRangeFilter ('\\x00', true, '\\xff', true)"))); + response = client.get(positivePath, Constants.MIMETYPE_JSON); + checkMultiCellGetJSON(response); + + // negative case + negativePath = path.toString() + ("&" + Constants.FILTER_B64 + "=" + base64UrlEncoder + .encodeToString(Bytes.toBytesBinary("ColumnRangeFilter ('\\x00', true, '1', false)"))); + response = client.get(negativePath, Constants.MIMETYPE_JSON); + assertEquals(404, response.getCode()); + + client.delete(row_5_url, extraHdr); + client.delete(row_6_url, extraHdr); + } + } diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java index 2e7a4af13b7..a4cbc8d5f2b 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java @@ -34,6 +34,7 @@ import java.io.Serializable; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Base64.Encoder; import java.util.Collections; import java.util.List; import javax.xml.bind.JAXBContext; @@ -93,6 +94,7 @@ public class TestTableScan { private static Configuration conf; private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final Encoder base64UrlEncoder = java.util.Base64.getUrlEncoder(); private static final HBaseRESTTestingUtility REST_TEST_UTIL = new HBaseRESTTestingUtility(); @BeforeClass @@ -442,7 +444,33 @@ public class TestTableScan { builder.append("&"); builder.append(Constants.SCAN_END_ROW + "=aay"); builder.append("&"); - builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("PrefixFilter('aab')", "UTF-8")); + builder.append(Constants.FILTER + "=" + URLEncoder.encode("PrefixFilter('aab')", "UTF-8")); + Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML); + assertEquals(200, response.getCode()); + JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class); + Unmarshaller ush = ctx.createUnmarshaller(); + CellSetModel model = (CellSetModel) ush.unmarshal(response.getStream()); + int count = TestScannerResource.countCellSet(model); + assertEquals(1, count); + assertEquals("aab", + new String(model.getRows().get(0).getCells().get(0).getValue(), StandardCharsets.UTF_8)); + } + + // This only tests the Base64Url encoded filter definition. + // base64 encoded row values are not implemented for this endpoint + @Test + public void testSimpleFilterBase64() throws IOException, JAXBException { + StringBuilder builder = new StringBuilder(); + builder.append("/*"); + builder.append("?"); + builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1); + builder.append("&"); + builder.append(Constants.SCAN_START_ROW + "=aaa"); + builder.append("&"); + builder.append(Constants.SCAN_END_ROW + "=aay"); + builder.append("&"); + builder.append(Constants.FILTER_B64 + "=" + base64UrlEncoder + .encodeToString("PrefixFilter('aab')".getBytes(StandardCharsets.UTF_8.toString()))); Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML); assertEquals(200, response.getCode()); JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class); @@ -459,8 +487,8 @@ public class TestTableScan { StringBuilder builder = new StringBuilder(); builder.append("/abc*"); builder.append("?"); - builder.append( - Constants.SCAN_FILTER + "=" + URLEncoder.encode("QualifierFilter(=,'binary:1')", "UTF-8")); + builder + .append(Constants.FILTER + "=" + URLEncoder.encode("QualifierFilter(=,'binary:1')", "UTF-8")); Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML); assertEquals(200, response.getCode()); JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class); @@ -477,7 +505,7 @@ public class TestTableScan { StringBuilder builder = new StringBuilder(); builder.append("/*"); builder.append("?"); - builder.append(Constants.SCAN_FILTER + "=" + builder.append(Constants.FILTER + "=" + URLEncoder.encode("PrefixFilter('abc') AND QualifierFilter(=,'binary:1')", "UTF-8")); Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML); assertEquals(200, response.getCode()); @@ -497,7 +525,7 @@ public class TestTableScan { builder.append("?"); builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1); builder.append("&"); - builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8")); + builder.append(Constants.FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8")); Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML); assertEquals(200, response.getCode()); JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class); @@ -516,7 +544,7 @@ public class TestTableScan { builder.append("?"); builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1); builder.append("&"); - builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8")); + builder.append(Constants.FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8")); Response response = client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML); assertEquals(200, response.getCode()); JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);