This is an automated email from the ASF dual-hosted git repository. hulee pushed a commit to branch zooscalability in repository https://gitbox.apache.org/repos/asf/helix.git
commit 3e28ad641303f3aad48cd6bfe7601445718490aa Author: Huizhi Lu <[email protected]> AuthorDate: Sat Feb 22 18:12:53 2020 -0800 Add REST read endpoints to helix-rest for metadata store directory (#761) We need restful metadata store directory service to help scale out zookeeper. This commit adds REST read endpoints to helix-rest to get sharding keys, realms and namespaces. --- .../datamodel/MetadataStoreShardingKey.java | 61 +++++ .../MetadataStoreDirectoryAccessor.java | 172 +++++++++++--- .../server/TestMetadataStoreDirectoryAccessor.java | 255 +++++++++++++++++---- .../constant/MetadataStoreRoutingConstants.java | 33 ++- 4 files changed, 442 insertions(+), 79 deletions(-) diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/datamodel/MetadataStoreShardingKey.java b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/datamodel/MetadataStoreShardingKey.java new file mode 100644 index 0000000..ac5ca59 --- /dev/null +++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/datamodel/MetadataStoreShardingKey.java @@ -0,0 +1,61 @@ +package org.apache.helix.rest.metadatastore.datamodel; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + + +/** + * A POJO class that represents a sharding key can be easily converted to JSON + * in REST API response. The JSON object for a sharding key looks like: + * { + * "shardingKey": "/sharding/key/10/abc", + * "realm": "realm.github.com" + * } + */ +@JsonAutoDetect +public class MetadataStoreShardingKey { + private String shardingKey; + private String realm; + + @JsonCreator + public MetadataStoreShardingKey(@JsonProperty String shardingKey, @JsonProperty String realm) { + this.shardingKey = shardingKey; + this.realm = realm; + } + + @JsonProperty + public String getShardingKey() { + return shardingKey; + } + + @JsonProperty + public String getRealm() { + return realm; + } + + @Override + public String toString() { + return "MetadataStoreShardingKey{" + "shardingKey='" + shardingKey + '\'' + ", realm='" + realm + + '\'' + '}'; + } +} diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java index 5d84d8a..bc6571a 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/metadatastore/MetadataStoreDirectoryAccessor.java @@ -20,11 +20,12 @@ package org.apache.helix.rest.server.resources.metadatastore; */ import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.stream.Collectors; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.PUT; @@ -41,6 +42,7 @@ import org.apache.helix.rest.common.HelixRestNamespace; import org.apache.helix.rest.common.HelixRestUtils; import org.apache.helix.rest.metadatastore.MetadataStoreDirectory; import org.apache.helix.rest.metadatastore.ZkMetadataStoreDirectory; +import org.apache.helix.rest.metadatastore.datamodel.MetadataStoreShardingKey; import org.apache.helix.rest.server.resources.AbstractResource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,25 +67,53 @@ public class MetadataStoreDirectoryAccessor extends AbstractResource { buildMetadataStoreDirectory(_namespace, helixRestNamespace.getMetadataStoreAddress()); } + @PreDestroy + private void preDestroy() { + _metadataStoreDirectory.close(); + } + /** - * Gets all metadata store realms in a namespace with the endpoint. + * Gets all existing namespaces in the routing metadata store at endpoint: + * "GET /metadata-store-namespaces" + * + * @return Json response of all namespaces. + */ + @GET + @Path("/metadata-store-namespaces") + public Response getAllNamespaces() { + Collection<String> namespaces = _metadataStoreDirectory.getAllNamespaces(); + Map<String, Collection<String>> responseMap = + ImmutableMap.of(MetadataStoreRoutingConstants.METADATA_STORE_NAMESPACES, namespaces); + + return JSONRepresentation(responseMap); + } + + /** + * Gets all metadata store realms in a namespace at path: "GET /metadata-store-realms", + * or gets a metadata store realm with the sharding key at path: + * "GET /metadata-store-realms?sharding-key={sharding-key}" * * @return Json representation of all realms. */ @GET @Path("/metadata-store-realms") - public Response getAllMetadataStoreRealms() { - Map<String, Collection<String>> responseMap; + public Response getAllMetadataStoreRealms(@QueryParam("sharding-key") String shardingKey) { try { - Collection<String> realms = _metadataStoreDirectory.getAllMetadataStoreRealms(_namespace); + if (shardingKey == null) { + // Get all realms: "GET /metadata-store-realms" + Collection<String> realms = _metadataStoreDirectory.getAllMetadataStoreRealms(_namespace); + Map<String, Collection<String>> responseMap = + ImmutableMap.of(MetadataStoreRoutingConstants.METADATA_STORE_REALMS, realms); + return JSONRepresentation(responseMap); + } - responseMap = new HashMap<>(1); - responseMap.put(MetadataStoreRoutingConstants.METADATA_STORE_REALMS, realms); + // Get a single realm filtered by sharding key: + // "GET /metadata-store-realms?sharding-key={sharding-key}" + String realm = _metadataStoreDirectory.getMetadataStoreRealm(_namespace, shardingKey); + return JSONRepresentation(new MetadataStoreShardingKey(shardingKey, realm)); } catch (NoSuchElementException ex) { return notFound(ex.getMessage()); } - - return JSONRepresentation(responseMap); } @PUT @@ -111,41 +141,70 @@ public class MetadataStoreDirectoryAccessor extends AbstractResource { } /** - * Gets sharding keys mapped at path "HTTP GET /sharding-keys" which returns all sharding keys in - * a namespace, or path "HTTP GET /sharding-keys?realm={realmName}" which returns sharding keys in - * a realm. + * Gets all sharding keys for following requests: + * - "HTTP GET /sharding-keys" which returns all sharding keys in a namespace. + * - "HTTP GET /sharding-keys?prefix={prefix}" which returns sharding keys that have the prefix. + * -- JSON response example for this path: + * { + * "prefix": "/sharding/key", + * "shardingKeys": [{ + * "realm": "testRealm2", + * "shardingKey": "/sharding/key/1/f" + * }, { + * "realm": "testRealm2", + * "shardingKey": "/sharding/key/1/e" + * }, { + * "realm": "testRealm1", + * "shardingKey": "/sharding/key/1/b" + * }, { + * "realm": "testRealm1", + * "shardingKey": "/sharding/key/1/a" + * }] + * } * - * @param realm Query param in endpoint path - * @return Json representation of a map: shardingKeys -> collection of sharding keys. + * @param prefix Query param in endpoint path: prefix substring of sharding key. + * @return Json representation for the sharding keys. */ @GET @Path("/sharding-keys") - public Response getShardingKeys(@QueryParam("realm") String realm) { - Map<String, Object> responseMap; - Collection<String> shardingKeys; + public Response getShardingKeys(@QueryParam("prefix") String prefix) { try { - // If realm is not set in query param, the endpoint is: "/sharding-keys" - // to get all sharding keys in a namespace. - if (realm == null) { - shardingKeys = _metadataStoreDirectory.getAllShardingKeys(_namespace); - // To avoid allocating unnecessary resource, limit the map's capacity only for - // SHARDING_KEYS. - responseMap = new HashMap<>(1); - } else { - // For endpoint: "/sharding-keys?realm={realmName}" - shardingKeys = _metadataStoreDirectory.getAllShardingKeysInRealm(_namespace, realm); - // To avoid allocating unnecessary resource, limit the map's capacity only for - // SHARDING_KEYS and SINGLE_METADATA_STORE_REALM. - responseMap = new HashMap<>(2); - responseMap.put(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, realm); + if (prefix == null) { + // For endpoint: "/sharding-keys" to get all sharding keys in a namespace. + return getAllShardingKeys(); } + // For endpoint: "/sharding-keys?prefix={prefix}" + return getAllShardingKeysUnderPath(prefix); } catch (NoSuchElementException ex) { return notFound(ex.getMessage()); } + } - responseMap.put(MetadataStoreRoutingConstants.SHARDING_KEYS, shardingKeys); + /** + * Gets all path-based sharding keys for a queried realm at endpoint: + * "GET /metadata-store-realms/{realm}/sharding-keys" + * <p> + * "GET /metadata-store-realms/{realm}/sharding-keys?prefix={prefix}" is also supported, + * which is helpful when you want to check what sharding keys have the prefix substring. + * + * @param realm Queried metadata store realm to get sharding keys. + * @param prefix Query param in endpoint path: prefix substring of sharding key. + * @return All path-based sharding keys in the queried realm. + */ + @GET + @Path("/metadata-store-realms/{realm}/sharding-keys") + public Response getRealmShardingKeys(@PathParam("realm") String realm, + @QueryParam("prefix") String prefix) { + try { + if (prefix == null) { + return getAllShardingKeysInRealm(realm); + } - return JSONRepresentation(responseMap); + // For "GET /metadata-store-realms/{realm}/sharding-keys?prefix={prefix}" + return getRealmShardingKeysUnderPath(realm, prefix); + } catch (NoSuchElementException ex) { + return notFound(ex.getMessage()); + } } @PUT @@ -209,4 +268,51 @@ public class MetadataStoreDirectoryAccessor extends AbstractResource { routingZkAddressMap, ex); } } + + private Response getAllShardingKeys() { + Collection<String> shardingKeys = _metadataStoreDirectory.getAllShardingKeys(_namespace); + Map<String, Object> responseMap = ImmutableMap + .of(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_NAMESPACE, _namespace, + MetadataStoreRoutingConstants.SHARDING_KEYS, shardingKeys); + + return JSONRepresentation(responseMap); + } + + private Response getAllShardingKeysInRealm(String realm) { + Collection<String> shardingKeys = + _metadataStoreDirectory.getAllShardingKeysInRealm(_namespace, realm); + + Map<String, Object> responseMap = ImmutableMap + .of(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, realm, + MetadataStoreRoutingConstants.SHARDING_KEYS, shardingKeys); + + return JSONRepresentation(responseMap); + } + + private Response getAllShardingKeysUnderPath(String prefix) { + List<MetadataStoreShardingKey> shardingKeyList = + _metadataStoreDirectory.getAllMappingUnderPath(_namespace, prefix).entrySet().stream() + .map(entry -> new MetadataStoreShardingKey(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + + Map<String, Object> responseMap = ImmutableMap + .of(MetadataStoreRoutingConstants.SHARDING_KEY_PATH_PREFIX, prefix, + MetadataStoreRoutingConstants.SHARDING_KEYS, shardingKeyList); + + return JSONRepresentation(responseMap); + } + + private Response getRealmShardingKeysUnderPath(String realm, String prefix) { + List<String> shardingKeyList = + _metadataStoreDirectory.getAllMappingUnderPath(_namespace, prefix).entrySet().stream() + .filter(entry -> entry.getValue().equals(realm)).map(Map.Entry::getKey) + .collect(Collectors.toList()); + + Map<String, Object> responseMap = ImmutableMap + .of(MetadataStoreRoutingConstants.SHARDING_KEY_PATH_PREFIX, prefix, + MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, realm, + MetadataStoreRoutingConstants.SHARDING_KEYS, shardingKeyList); + + return JSONRepresentation(responseMap); + } } diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestMetadataStoreDirectoryAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestMetadataStoreDirectoryAccessor.java index 27e2b10..6a9c598 100644 --- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestMetadataStoreDirectoryAccessor.java +++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestMetadataStoreDirectoryAccessor.java @@ -47,6 +47,7 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +// TODO: enable asserts and add verify for refreshed MSD once write operations are ready. public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass { /* * The following are constants to be used for testing. @@ -61,15 +62,14 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass { private static final List<String> TEST_SHARDING_KEYS_2 = Arrays.asList("/sharding/key/1/d", "/sharding/key/1/e", "/sharding/key/1/f"); private static final String TEST_REALM_3 = "testRealm3"; - private static final String TEST_SHARDING_KEY = "/sharding/key/1/x"; + private static final String TEST_SHARDING_KEY = "/sharding/key/3/x"; // List of all ZK addresses, each of which corresponds to a namespace/routing ZK private List<String> _zkList; private MetadataStoreDirectory _metadataStoreDirectory; @BeforeClass - public void beforeClass() - throws Exception { + public void beforeClass() throws Exception { _zkList = new ArrayList<>(ZK_SERVER_MAP.keySet()); deleteRoutingDataPath(); @@ -111,15 +111,39 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass { } @AfterClass - public void afterClass() - throws Exception { + public void afterClass() throws Exception { _metadataStoreDirectory.close(); deleteRoutingDataPath(); } + /* + * Tests REST endpoint: "GET /namespaces/{namespace}/metadata-store-namespaces" + */ @Test - public void testGetAllMetadataStoreRealms() - throws IOException { + public void testGetAllNamespaces() throws IOException { + String responseBody = get(TEST_NAMESPACE_URI_PREFIX + "/metadata-store-namespaces", null, + Response.Status.OK.getStatusCode(), true); + + // It is safe to cast the object and suppress warnings. + @SuppressWarnings("unchecked") + Map<String, Collection<String>> queriedNamespacesMap = + OBJECT_MAPPER.readValue(responseBody, Map.class); + + Assert.assertEquals(queriedNamespacesMap.keySet(), + ImmutableSet.of(MetadataStoreRoutingConstants.METADATA_STORE_NAMESPACES)); + + Set<String> queriedNamespacesSet = new HashSet<>( + queriedNamespacesMap.get(MetadataStoreRoutingConstants.METADATA_STORE_NAMESPACES)); + Set<String> expectedNamespaces = ImmutableSet.of(TEST_NAMESPACE); + + Assert.assertEquals(queriedNamespacesSet, expectedNamespaces); + } + + /* + * Tests REST endpoint: "GET /metadata-store-realms" + */ + @Test(dependsOnMethods = "testGetAllNamespaces") + public void testGetAllMetadataStoreRealms() throws IOException { get(NON_EXISTING_NAMESPACE_URI_PREFIX + "metadata-store-realms", null, Response.Status.NOT_FOUND.getStatusCode(), false); @@ -130,8 +154,8 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass { Map<String, Collection<String>> queriedRealmsMap = OBJECT_MAPPER.readValue(responseBody, Map.class); - Assert.assertTrue( - queriedRealmsMap.containsKey(MetadataStoreRoutingConstants.METADATA_STORE_REALMS)); + Assert.assertEquals(queriedRealmsMap.keySet(), + ImmutableSet.of(MetadataStoreRoutingConstants.METADATA_STORE_REALMS)); Set<String> queriedRealmsSet = new HashSet<>(queriedRealmsMap.get(MetadataStoreRoutingConstants.METADATA_STORE_REALMS)); @@ -140,7 +164,36 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass { Assert.assertEquals(queriedRealmsSet, expectedRealms); } + /* + * Tests REST endpoint: "GET /metadata-store-realms?sharding-key={sharding-key}" + */ @Test(dependsOnMethods = "testGetAllMetadataStoreRealms") + public void testGetMetadataStoreRealmWithShardingKey() throws IOException { + String shardingKey = TEST_SHARDING_KEYS_1.get(0); + + new JerseyUriRequestBuilder( + NON_EXISTING_NAMESPACE_URI_PREFIX + "metadata-store-realms?sharding-key=" + shardingKey) + .expectedReturnStatusCode(Response.Status.NOT_FOUND.getStatusCode()).get(this); + + String responseBody = new JerseyUriRequestBuilder( + TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms?sharding-key=" + shardingKey) + .isBodyReturnExpected(true).get(this); + + // It is safe to cast the object and suppress warnings. + @SuppressWarnings("unchecked") + Map<String, String> queriedRealmMap = OBJECT_MAPPER.readValue(responseBody, Map.class); + + Map<String, String> expectedRealm = ImmutableMap + .of(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, TEST_REALM_1, + MetadataStoreRoutingConstants.SINGLE_SHARDING_KEY, shardingKey); + + Assert.assertEquals(queriedRealmMap, expectedRealm); + } + + /* + * Tests REST endpoint: "PUT /metadata-store-realms/{realm}" + */ + @Test(dependsOnMethods = "testGetMetadataStoreRealmWithShardingKey") public void testAddMetadataStoreRealm() { Collection<String> previousRealms = _metadataStoreDirectory.getAllMetadataStoreRealms(TEST_NAMESPACE); @@ -164,10 +217,12 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass { Set<String> updateRealmsSet = new HashSet<>(updatedRealms); expectedRealmsSet.add(TEST_REALM_3); - // TODO: enable asserts and add verify for refreshed MSD once write operations are ready. // Assert.assertEquals(updateRealmsSet, previousRealms); } + /* + * Tests REST endpoint: "DELETE /metadata-store-realms/{realm}" + */ @Test(dependsOnMethods = "testAddMetadataStoreRealm") public void testDeleteMetadataStoreRealm() { Collection<String> previousRealms = @@ -194,11 +249,10 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass { } /* - * Tests REST endpoints: "/sharding-keys" + * Tests REST endpoint: "GET /sharding-keys" */ @Test(dependsOnMethods = "testDeleteMetadataStoreRealm") - public void testGetShardingKeysInNamespace() - throws IOException { + public void testGetShardingKeysInNamespace() throws IOException { get(NON_EXISTING_NAMESPACE_URI_PREFIX + "sharding-keys", null, Response.Status.NOT_FOUND.getStatusCode(), true); @@ -207,14 +261,19 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass { true); // It is safe to cast the object and suppress warnings. @SuppressWarnings("unchecked") - Map<String, Collection<String>> queriedShardingKeysMap = - OBJECT_MAPPER.readValue(responseBody, Map.class); + Map<String, Object> queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class); + + Assert.assertEquals(queriedShardingKeysMap.keySet(), ImmutableSet + .of(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_NAMESPACE, + MetadataStoreRoutingConstants.SHARDING_KEYS)); - Assert.assertTrue( - queriedShardingKeysMap.containsKey(MetadataStoreRoutingConstants.SHARDING_KEYS)); + Assert.assertEquals( + queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_NAMESPACE), + TEST_NAMESPACE); - Set<String> queriedShardingKeys = - new HashSet<>(queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SHARDING_KEYS)); + @SuppressWarnings("unchecked") + Set<String> queriedShardingKeys = new HashSet<>((Collection<String>) queriedShardingKeysMap + .get(MetadataStoreRoutingConstants.SHARDING_KEYS)); Set<String> expectedShardingKeys = new HashSet<>(); expectedShardingKeys.addAll(TEST_SHARDING_KEYS_1); expectedShardingKeys.addAll(TEST_SHARDING_KEYS_2); @@ -223,38 +282,125 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass { } /* - * Tests REST endpoint: "/sharding-keys?realm={realmName}" + * Tests REST endpoint: "GET /metadata-store-realms/{realm}/sharding-keys" */ @Test(dependsOnMethods = "testGetShardingKeysInNamespace") - public void testGetShardingKeysInRealm() - throws IOException { + public void testGetShardingKeysInRealm() throws IOException { // Test NOT_FOUND response for a non existed realm. - new JerseyUriRequestBuilder(TEST_NAMESPACE_URI_PREFIX + "/sharding-keys?realm=nonExistedRealm") + new JerseyUriRequestBuilder( + TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms/nonExistedRealm/sharding-keys") .expectedReturnStatusCode(Response.Status.NOT_FOUND.getStatusCode()).get(this); - // Query param realm is set to empty, so NOT_FOUND response is returned. - new JerseyUriRequestBuilder(TEST_NAMESPACE_URI_PREFIX + "/sharding-keys?realm=") - .expectedReturnStatusCode(Response.Status.NOT_FOUND.getStatusCode()).get(this); + // Success response for "GET /metadata-store-realms/{realm}/sharding-keys" + String responseBody = new JerseyUriRequestBuilder( + TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms/" + TEST_REALM_1 + "/sharding-keys") + .isBodyReturnExpected(true).get(this); + + verifyRealmShardingKeys(responseBody); + } - // Success response. + /* + * Tests REST endpoint: "GET /sharding-keys?prefix={prefix}" + */ + @SuppressWarnings("unchecked") + @Test(dependsOnMethods = "testGetShardingKeysInRealm") + public void testGetShardingKeysUnderPath() throws IOException { + // Test non existed prefix and empty sharding keys in response. String responseBody = new JerseyUriRequestBuilder( - TEST_NAMESPACE_URI_PREFIX + "/sharding-keys?realm=" + TEST_REALM_1) + TEST_NAMESPACE_URI_PREFIX + "/sharding-keys?prefix=/non/Existed/Prefix") .isBodyReturnExpected(true).get(this); - // It is safe to cast the object and suppress warnings. - @SuppressWarnings("unchecked") + Map<String, Object> queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class); + Collection<Map<String, String>> emptyKeysList = + (Collection<Map<String, String>>) queriedShardingKeysMap + .get(MetadataStoreRoutingConstants.SHARDING_KEYS); + Assert.assertTrue(emptyKeysList.isEmpty()); + + // Success response with non empty sharding keys. + String shardingKeyPrefix = "/sharding/key"; + responseBody = new JerseyUriRequestBuilder( + TEST_NAMESPACE_URI_PREFIX + "/sharding-keys?prefix=" + shardingKeyPrefix) + .isBodyReturnExpected(true).get(this); + + queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class); + + // Check fields. + Assert.assertEquals(queriedShardingKeysMap.keySet(), ImmutableSet + .of(MetadataStoreRoutingConstants.SHARDING_KEY_PATH_PREFIX, + MetadataStoreRoutingConstants.SHARDING_KEYS)); + + // Check sharding key prefix in json response. + Assert.assertEquals( + queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SHARDING_KEY_PATH_PREFIX), + shardingKeyPrefix); + + Collection<Map<String, String>> queriedShardingKeys = + (Collection<Map<String, String>>) queriedShardingKeysMap + .get(MetadataStoreRoutingConstants.SHARDING_KEYS); + Set<Map<String, String>> queriedShardingKeysSet = new HashSet<>(queriedShardingKeys); + Set<Map<String, String>> expectedShardingKeysSet = new HashSet<>(); + + TEST_SHARDING_KEYS_1.forEach(key -> expectedShardingKeysSet.add(ImmutableMap + .of(MetadataStoreRoutingConstants.SINGLE_SHARDING_KEY, key, + MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, TEST_REALM_1))); + + TEST_SHARDING_KEYS_2.forEach(key -> expectedShardingKeysSet.add(ImmutableMap + .of(MetadataStoreRoutingConstants.SINGLE_SHARDING_KEY, key, + MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, TEST_REALM_2))); + + Assert.assertEquals(queriedShardingKeysSet, expectedShardingKeysSet); + } + + /* + * Tests REST endpoint: "GET /metadata-store-realms/{realm}/sharding-keys?prefix={prefix}" + */ + @SuppressWarnings("unchecked") + @Test(dependsOnMethods = "testGetShardingKeysUnderPath") + public void testGetRealmShardingKeysUnderPath() throws IOException { + // Test non existed prefix and empty sharding keys in response. + String responseBody = new JerseyUriRequestBuilder( + TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms/" + TEST_REALM_1 + + "/sharding-keys?prefix=/non/Existed/Prefix").isBodyReturnExpected(true).get(this); + + Map<String, Object> queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class); + Collection<Map<String, String>> emptyKeysList = + (Collection<Map<String, String>>) queriedShardingKeysMap + .get(MetadataStoreRoutingConstants.SHARDING_KEYS); + Assert.assertTrue(emptyKeysList.isEmpty()); + + // Test non existed realm and empty sharding keys in response. + responseBody = new JerseyUriRequestBuilder(TEST_NAMESPACE_URI_PREFIX + + "/metadata-store-realms/nonExistedRealm/sharding-keys?prefix=/sharding/key") + .isBodyReturnExpected(true).get(this); + + queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class); + emptyKeysList = (Collection<Map<String, String>>) queriedShardingKeysMap + .get(MetadataStoreRoutingConstants.SHARDING_KEYS); + Assert.assertTrue(emptyKeysList.isEmpty()); + + // Valid query params and non empty sharding keys. + String shardingKeyPrefix = "/sharding/key/1"; + responseBody = new JerseyUriRequestBuilder( + TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms/" + TEST_REALM_1 + + "/sharding-keys?prefix=" + shardingKeyPrefix).isBodyReturnExpected(true).get(this); + + queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class); + + // Check fields. + Assert.assertEquals(queriedShardingKeysMap.keySet(), ImmutableSet + .of(MetadataStoreRoutingConstants.SHARDING_KEY_PATH_PREFIX, + MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, + MetadataStoreRoutingConstants.SHARDING_KEYS)); + + // Check sharding key prefix in json response. + Assert.assertEquals( + queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SHARDING_KEY_PATH_PREFIX), + shardingKeyPrefix); - // Check realm name in json response. - Assert.assertTrue(queriedShardingKeysMap - .containsKey(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM)); Assert.assertEquals( queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM), TEST_REALM_1); - Assert.assertTrue( - queriedShardingKeysMap.containsKey(MetadataStoreRoutingConstants.SHARDING_KEYS)); - // It is safe to cast the object and suppress warnings. - @SuppressWarnings("unchecked") Set<String> queriedShardingKeys = new HashSet<>((Collection<String>) queriedShardingKeysMap .get(MetadataStoreRoutingConstants.SHARDING_KEYS)); Set<String> expectedShardingKeys = new HashSet<>(TEST_SHARDING_KEYS_1); @@ -262,7 +408,10 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass { Assert.assertEquals(queriedShardingKeys, expectedShardingKeys); } - @Test(dependsOnMethods = "testGetShardingKeysInRealm") + /* + * Tests REST endpoint: "PUT /metadata-store-realms/{realm}/sharding-keys/{sharding-key}" + */ + @Test(dependsOnMethods = "testGetRealmShardingKeysUnderPath") public void testAddShardingKey() { Set<String> expectedShardingKeysSet = new HashSet<>( _metadataStoreDirectory.getAllShardingKeysInRealm(TEST_NAMESPACE, TEST_REALM_1)); @@ -287,6 +436,9 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass { // Assert.assertEquals(updatedShardingKeysSet, expectedShardingKeysSet); } + /* + * Tests REST endpoint: "PUT /metadata-store-realms/{realm}/sharding-keys/{sharding-key}" + */ @Test(dependsOnMethods = "testAddShardingKey") public void testDeleteShardingKey() { Set<String> expectedShardingKeysSet = new HashSet<>( @@ -310,8 +462,31 @@ public class TestMetadataStoreDirectoryAccessor extends AbstractTestClass { // Assert.assertEquals(updatedShardingKeysSet, expectedShardingKeysSet); } - private void deleteRoutingDataPath() - throws Exception { + private void verifyRealmShardingKeys(String responseBody) throws IOException { + // It is safe to cast the object and suppress warnings. + @SuppressWarnings("unchecked") + Map<String, Object> queriedShardingKeysMap = OBJECT_MAPPER.readValue(responseBody, Map.class); + + // Check fields in JSON response. + Assert.assertEquals(queriedShardingKeysMap.keySet(), ImmutableSet + .of(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM, + MetadataStoreRoutingConstants.SHARDING_KEYS)); + + // Check realm name in json response. + Assert.assertEquals( + queriedShardingKeysMap.get(MetadataStoreRoutingConstants.SINGLE_METADATA_STORE_REALM), + TEST_REALM_1); + + // It is safe to cast the object and suppress warnings. + @SuppressWarnings("unchecked") + Set<String> queriedShardingKeys = new HashSet<>((Collection<String>) queriedShardingKeysMap + .get(MetadataStoreRoutingConstants.SHARDING_KEYS)); + Set<String> expectedShardingKeys = new HashSet<>(TEST_SHARDING_KEYS_1); + + Assert.assertEquals(queriedShardingKeys, expectedShardingKeys); + } + + private void deleteRoutingDataPath() throws Exception { Assert.assertTrue(TestHelper.verify(() -> { _zkList.forEach(zk -> ZK_SERVER_MAP.get(zk).getZkClient() .deleteRecursively(MetadataStoreRoutingConstants.ROUTING_DATA_PATH)); diff --git a/metadata-store-directory-common/src/main/java/org/apache/helix/msdcommon/constant/MetadataStoreRoutingConstants.java b/metadata-store-directory-common/src/main/java/org/apache/helix/msdcommon/constant/MetadataStoreRoutingConstants.java index 009e7f3..13e78b0 100644 --- a/metadata-store-directory-common/src/main/java/org/apache/helix/msdcommon/constant/MetadataStoreRoutingConstants.java +++ b/metadata-store-directory-common/src/main/java/org/apache/helix/msdcommon/constant/MetadataStoreRoutingConstants.java @@ -28,14 +28,35 @@ public class MetadataStoreRoutingConstants { // Leader election ZNode for ZkRoutingDataWriter public static final String LEADER_ELECTION_ZNODE = "/_ZK_ROUTING_DATA_WRITER_LEADER"; - // Field name in JSON REST response of getting metadata store realms in one namespace. - public static final String METADATA_STORE_REALMS = "metadataStoreRealms"; + /** Field name in JSON REST response of getting all metadata store namespaces. */ + public static final String METADATA_STORE_NAMESPACES = "namespaces"; - // Field name in JSON REST response of getting sharding keys in one realm. - public static final String SINGLE_METADATA_STORE_REALM = "metadataStoreRealm"; + /** Field name in JSON REST response of getting all sharding keys in a single namespace. */ + public static final String SINGLE_METADATA_STORE_NAMESPACE = "namespace"; - // Field name in JSON REST response of getting sharding keys. - public static final String SHARDING_KEYS = "shardingKeys"; + /** Field name in JSON REST response of getting metadata store realms in one namespace. */ + public static final String METADATA_STORE_REALMS = "realms"; + + /** Field name in JSON REST response of getting sharding keys in one realm. */ + public static final String SINGLE_METADATA_STORE_REALM = "realm"; + /** Field name in JSON REST response of getting sharding keys. */ + public static final String SHARDING_KEYS = "shardingKeys"; + /** Field name in JSON REST response related to one single sharding key. */ + public static final String SINGLE_SHARDING_KEY = "shardingKey"; + + /** + * Field name in JSON response of the REST endpoint getting sharding keys with prefix: + * "GET /sharding-keys?prefix={prefix}" + * It is used in below response as an example: + * { + * "prefix": "/sharding/key", + * "shardingKeys": [{ + * "realm": "testRealm2", + * "shardingKey": "/sharding/key/1/f" + * }] + * } + */ + public static final String SHARDING_KEY_PATH_PREFIX = "prefix"; }
