This is an automated email from the ASF dual-hosted git repository. hulee pushed a commit to branch zooscalability in repository https://gitbox.apache.org/repos/asf/helix.git
commit 33962f1067d2a2f4a17a4f8a9ab3b0c2bc0f86d5 Author: Neal Sun <[email protected]> AuthorDate: Wed Feb 12 15:16:28 2020 -0800 Implement getAllMappingUnderPath and getMetadataStoreRealm in ZkMetadataStoreDirectory This PR implements getAllMappingUnderPath and getMetadataStoreRealm in ZkMetadataStoreDirectory. Also, getRoutingData in ZkRoutingDataReader is updated : not having realm addresses and not having sharding keys for some realm addresses no longer raise exceptions. We now allow realms to not have sharding keys and return them as a part of routing data. --- .../helix/rest/metadatastore/TrieRoutingData.java | 6 +--- .../metadatastore/ZkMetadataStoreDirectory.java | 16 +++++++--- .../accessor/ZkRoutingDataReader.java | 27 ++++++----------- .../rest/metadatastore/TestTrieRoutingData.java | 13 -------- .../TestZkMetadataStoreDirectory.java | 35 ++++++++++++++++++++++ .../accessor/TestZkRoutingDataReader.java | 18 +++++------ 6 files changed, 64 insertions(+), 51 deletions(-) diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/TrieRoutingData.java b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/TrieRoutingData.java index 28add4c..923f818 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/TrieRoutingData.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/TrieRoutingData.java @@ -162,15 +162,11 @@ public class TrieRoutingData implements MetadataStoreRoutingData { * @throws InvalidRoutingDataException - when there is an empty sharding key (edge case that * always renders the routing data invalid); when there is a sharding key which already * contains a sharding key (invalid); when there is a sharding key that is a part of - * another sharding key (invalid) + * another sharding key (invalid); when a sharding key doesn't have a leading delimiter */ private void constructTrie(Map<String, List<String>> routingData) throws InvalidRoutingDataException { for (Map.Entry<String, List<String>> entry : routingData.entrySet()) { - if (entry.getValue().isEmpty()) { - throw new InvalidRoutingDataException( - "Realm address does not have associating sharding keys: " + entry.getKey()); - } for (String shardingKey : entry.getValue()) { // Missing leading delimiter is invalid if (shardingKey.isEmpty() || !shardingKey.substring(0, 1).equals(DELIMITER)) { diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java index 536d058..3be9b72 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/ZkMetadataStoreDirectory.java @@ -79,6 +79,8 @@ public class ZkMetadataStoreDirectory implements MetadataStoreDirectory, Routing // Populate realmToShardingKeys with ZkRoutingDataReader _realmToShardingKeysMap.put(routingEntry.getKey(), _routingDataReaderMap.get(routingEntry.getKey()).getRoutingData()); + _routingDataMap.put(routingEntry.getKey(), + new TrieRoutingData(_realmToShardingKeysMap.get(routingEntry.getKey()))); } } @@ -119,14 +121,20 @@ public class ZkMetadataStoreDirectory implements MetadataStoreDirectory, Routing @Override public Map<String, String> getAllMappingUnderPath(String namespace, String path) { - // TODO: get it from routingData - throw new UnsupportedOperationException(); + if (!_routingDataMap.containsKey(namespace)) { + throw new NoSuchElementException( + "Failed to get all mapping under path: Namespace " + namespace + " is not found!"); + } + return _routingDataMap.get(namespace).getAllMappingUnderPath(path); } @Override public String getMetadataStoreRealm(String namespace, String shardingKey) { - // TODO: get it from routingData - throw new UnsupportedOperationException(); + if (!_routingDataMap.containsKey(namespace)) { + throw new NoSuchElementException( + "Failed to get metadata store realm: Namespace " + namespace + " is not found!"); + } + return _routingDataMap.get(namespace).getMetadataStoreRealm(shardingKey); } @Override diff --git a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/accessor/ZkRoutingDataReader.java b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/accessor/ZkRoutingDataReader.java index 9decf23..44ce110 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/accessor/ZkRoutingDataReader.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/metadatastore/accessor/ZkRoutingDataReader.java @@ -72,36 +72,27 @@ public class ZkRoutingDataReader implements MetadataStoreRoutingDataReader, IZkD /** * Returns (realm, list of ZK path sharding keys) mappings. * @return Map <realm, list of ZK path sharding keys> - * @throws InvalidRoutingDataException + * @throws InvalidRoutingDataException - when the node on + * MetadataStoreRoutingConstants.ROUTING_DATA_PATH is missing */ - public Map<String, List<String>> getRoutingData() - throws InvalidRoutingDataException { + public Map<String, List<String>> getRoutingData() throws InvalidRoutingDataException { Map<String, List<String>> routingData = new HashMap<>(); - List<String> children; + List<String> allRealmAddresses; try { - children = _zkClient.getChildren(MetadataStoreRoutingConstants.ROUTING_DATA_PATH); + allRealmAddresses = _zkClient.getChildren(MetadataStoreRoutingConstants.ROUTING_DATA_PATH); } catch (ZkNoNodeException e) { throw new InvalidRoutingDataException( "Routing data directory ZNode " + MetadataStoreRoutingConstants.ROUTING_DATA_PATH + " does not exist. Routing ZooKeeper address: " + _zkAddress); } - if (children == null || children.isEmpty()) { - throw new InvalidRoutingDataException( - "There are no metadata store realms defined. Routing ZooKeeper address: " + _zkAddress); - } - for (String child : children) { + for (String realmAddress : allRealmAddresses) { ZNRecord record = - _zkClient.readData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + child); + _zkClient.readData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + realmAddress); List<String> shardingKeys = record.getListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY); - if (shardingKeys == null || shardingKeys.isEmpty()) { - throw new InvalidRoutingDataException( - "Realm address ZNode " + MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + child - + " does not have a value for key " - + MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY - + ". Routing ZooKeeper address: " + _zkAddress); + if (shardingKeys != null) { + routingData.put(realmAddress, shardingKeys); } - routingData.put(child, shardingKeys); } return routingData; } diff --git a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestTrieRoutingData.java b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestTrieRoutingData.java index bf71456..4de68a6 100644 --- a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestTrieRoutingData.java +++ b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestTrieRoutingData.java @@ -67,19 +67,6 @@ public class TestTrieRoutingData { } @Test - public void testConstructionEmptyShardingKeys() { - Map<String, List<String>> routingData = new HashMap<>(); - routingData.put("realmAddress1", Collections.emptyList()); - try { - new TrieRoutingData(routingData); - Assert.fail("Expecting InvalidRoutingDataException"); - } catch (InvalidRoutingDataException e) { - Assert.assertTrue(e.getMessage() - .contains("Realm address does not have associating sharding keys: realmAddress1")); - } - } - - @Test public void testConstructionShardingKeyNoLeadingSlash() { Map<String, List<String>> routingData = new HashMap<>(); routingData.put("realmAddress1", Arrays.asList("/g", "/h/i", "/h/j")); diff --git a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestZkMetadataStoreDirectory.java b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestZkMetadataStoreDirectory.java index c0741ee..68b93b3 100644 --- a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestZkMetadataStoreDirectory.java +++ b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/TestZkMetadataStoreDirectory.java @@ -21,6 +21,7 @@ package org.apache.helix.rest.metadatastore; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -152,6 +153,40 @@ public class TestZkMetadataStoreDirectory extends AbstractTestClass { } @Test(dependsOnMethods = "testGetAllShardingKeysInRealm") + public void testGetAllMappingUnderPath() { + Map<String, String> mappingFromRoot = new HashMap<>(); + TEST_SHARDING_KEYS_1.forEach(shardingKey -> { + mappingFromRoot.put(shardingKey, TEST_REALM_1); + }); + TEST_SHARDING_KEYS_2.forEach(shardingKey -> { + mappingFromRoot.put(shardingKey, TEST_REALM_2); + }); + + Map<String, String> mappingFromLeaf = new HashMap<>(1); + mappingFromLeaf.put("/sharding/key/1/a", TEST_REALM_1); + + for (String namespace : _routingZkAddrMap.keySet()) { + Assert.assertEquals(_metadataStoreDirectory.getAllMappingUnderPath(namespace, "/"), + mappingFromRoot); + Assert.assertEquals( + _metadataStoreDirectory.getAllMappingUnderPath(namespace, "/sharding/key/1/a"), + mappingFromLeaf); + } + } + + @Test(dependsOnMethods = "testGetAllMappingUnderPath") + public void testGetMetadataStoreRealm() { + for (String namespace : _routingZkAddrMap.keySet()) { + Assert.assertEquals( + _metadataStoreDirectory.getMetadataStoreRealm(namespace, "/sharding/key/1/a/x/y/z"), + TEST_REALM_1); + Assert.assertEquals( + _metadataStoreDirectory.getMetadataStoreRealm(namespace, "/sharding/key/1/d/x/y/z"), + TEST_REALM_2); + } + } + + @Test(dependsOnMethods = "testGetMetadataStoreRealm") public void testDataChangeCallback() throws Exception { // For all namespaces (Routing ZKs), add an extra sharding key to TEST_REALM_1 diff --git a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java index aa46429..5781a85 100644 --- a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java +++ b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/accessor/TestZkRoutingDataReader.java @@ -104,11 +104,10 @@ public class TestZkRoutingDataReader extends AbstractTestClass { _baseAccessor.create(MetadataStoreRoutingConstants.ROUTING_DATA_PATH, new ZNRecord("test"), AccessOption.PERSISTENT); try { - _zkRoutingDataReader.getRoutingData(); - Assert.fail("Expecting InvalidRoutingDataException"); + Map<String, List<String>> routingData = _zkRoutingDataReader.getRoutingData(); + Assert.assertEquals(routingData.size(), 0); } catch (InvalidRoutingDataException e) { - Assert.assertTrue(e.getMessage().contains( - "There are no metadata store realms defined. Routing ZooKeeper address: " + ZK_ADDR)); + Assert.fail("Not expecting InvalidRoutingDataException"); } } @@ -120,14 +119,11 @@ public class TestZkRoutingDataReader extends AbstractTestClass { _baseAccessor.create(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/testRealmAddress1", testZnRecord1, AccessOption.PERSISTENT); try { - _zkRoutingDataReader.getRoutingData(); - Assert.fail("Expecting InvalidRoutingDataException"); + Map<String, List<String>> routingData = _zkRoutingDataReader.getRoutingData(); + Assert.assertEquals(routingData.size(), 1); + Assert.assertTrue(routingData.get("testRealmAddress1").isEmpty()); } catch (InvalidRoutingDataException e) { - Assert.assertTrue(e.getMessage().contains( - "Realm address ZNode " + MetadataStoreRoutingConstants.ROUTING_DATA_PATH - + "/testRealmAddress1 does not have a value for key " - + MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY - + ". Routing ZooKeeper address: " + ZK_ADDR)); + Assert.fail("Not expecting InvalidRoutingDataException"); } } }
