This is an automated email from the ASF dual-hosted git repository. hulee pushed a commit to branch zooscalability in repository https://gitbox.apache.org/repos/asf/helix.git
commit cc7846c4b5597563e08a9ce6b5cd224b4277199d Author: Hunter Lee <[email protected]> AuthorDate: Tue Feb 4 15:20:52 2020 -0800 Add MockMetadataStoreDirectoryServer (#719) * Add MockMetadataStoreDirectoryServer In order to support ZK horizontal scalability, we need to have Metadata Store Directory Service, which is a feature provided by Helix REST. Helix APIs that talk to ZooKeeper will query against this service at initialization to fetch all metadata store routing keys. For Helix application developers, this means that there's potentially a lot to do for setting up a testing environment assuming multiple ZKs. This MockMetadataStoreDirectoryServer makes it easy to test by abstracting out the work of having to set up and write metadata store routing information to the routing ZK. Changelist: 1. Implement Mock MSDS 2. Write a test in main() --- .../mock/MockMetadataStoreDirectoryServer.java | 127 +++++++++++++++++++++ .../mock/TestMockMetadataStoreDirectoryServer.java | 84 ++++++++++++++ 2 files changed, 211 insertions(+) diff --git a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/mock/MockMetadataStoreDirectoryServer.java b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/mock/MockMetadataStoreDirectoryServer.java new file mode 100644 index 0000000..ae0f85d --- /dev/null +++ b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/mock/MockMetadataStoreDirectoryServer.java @@ -0,0 +1,127 @@ +package org.apache.helix.rest.metadatastore.mock; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.HttpServer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Mock HTTP server that serves GET of metadata store routing data only. + * Helix applications may use this to write unit/integration tests without having to set up the routing ZooKeeper and creating routing data ZNodes. + */ +public class MockMetadataStoreDirectoryServer { + private static final Logger LOG = LoggerFactory.getLogger(MockMetadataStoreDirectoryServer.class); + + protected static final String REST_PREFIX = "/admin/v2/namespaces/"; + protected static final String ZK_REALM_ENDPOINT = "/METADATA_STORE_ROUTING_DATA/"; + protected static final int NOT_IMPLEMENTED = 501; + protected static final int OK = 200; + protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + protected final String _hostname; + protected final int _mockServerPort; + protected final Map<String, List<String>> _routingDataMap; + protected final String _namespace; + protected HttpServer _server; + protected final ThreadPoolExecutor _executor = + (ThreadPoolExecutor) Executors.newFixedThreadPool(10); + + protected enum SupportedHttpVerbs { + GET + } + + /** + * Constructs a Mock MSDS. + * A sample GET might look like the following: + * curl localhost:11000/admin/v2/namespaces/MY-HELIX-NAMESPACE/METADATA_STORE_ROUTING_DATA/zk-1 + * @param hostname hostname for the REST server. E.g.) "localhost" + * @param port port to use. E.g.) 11000 + * @param namespace the Helix REST namespace to mock. E.g.) "MY-HELIX-NAMESPACE" + * @param routingData <ZK realm, List of ZK path sharding keys> + */ + public MockMetadataStoreDirectoryServer(String hostname, int port, String namespace, + Map<String, List<String>> routingData) { + if (hostname == null || hostname.isEmpty()) { + throw new IllegalArgumentException("hostname cannot be null or empty!"); + } + if (port < 0 || port > 65535) { + throw new IllegalArgumentException("port is not a valid port!"); + } + if (namespace == null || namespace.isEmpty()) { + throw new IllegalArgumentException("namespace cannot be null or empty!"); + } + if (routingData == null || routingData.isEmpty()) { + throw new IllegalArgumentException("routingData cannot be null or empty!"); + } + _hostname = hostname; + _mockServerPort = port; + _namespace = namespace; + _routingDataMap = routingData; + } + + public void startServer() + throws IOException { + _server = HttpServer.create(new InetSocketAddress(_hostname, _mockServerPort), 0); + generateContexts(); + _server.setExecutor(_executor); + _server.start(); + LOG.info( + "Started MockMetadataStoreDirectoryServer at " + _hostname + ":" + _mockServerPort + "!"); + } + + public void stopServer() { + _server.stop(0); + _executor.shutdown(); + LOG.info( + "Stopped MockMetadataStoreDirectoryServer at " + _hostname + ":" + _mockServerPort + "!"); + } + + /** + * Dynamically generates HTTP server contexts based on the routing data given. + */ + private void generateContexts() { + _routingDataMap.forEach((zkRealm, shardingKeyList) -> _server + .createContext(REST_PREFIX + _namespace + ZK_REALM_ENDPOINT + zkRealm, httpExchange -> { + OutputStream outputStream = httpExchange.getResponseBody(); + String htmlResponse; + if (SupportedHttpVerbs.GET.name().equals(httpExchange.getRequestMethod())) { + htmlResponse = OBJECT_MAPPER.writeValueAsString(shardingKeyList); + httpExchange.sendResponseHeaders(OK, htmlResponse.length()); + } else { + htmlResponse = httpExchange.getRequestMethod() + " is not supported!\n"; + httpExchange.sendResponseHeaders(NOT_IMPLEMENTED, htmlResponse.length()); + } + outputStream.write(htmlResponse.getBytes()); + outputStream.flush(); + outputStream.close(); + })); + } +} diff --git a/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/mock/TestMockMetadataStoreDirectoryServer.java b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/mock/TestMockMetadataStoreDirectoryServer.java new file mode 100644 index 0000000..5e71089 --- /dev/null +++ b/helix-rest/src/test/java/org/apache/helix/rest/metadatastore/mock/TestMockMetadataStoreDirectoryServer.java @@ -0,0 +1,84 @@ +package org.apache.helix.rest.metadatastore.mock; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.google.common.collect.ImmutableList; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.junit.Test; +import org.testng.Assert; + + +public class TestMockMetadataStoreDirectoryServer { + @Test + public void testMockMetadataStoreDirectoryServer() + throws IOException { + // Create fake routing data + Map<String, List<String>> routingData = new HashMap<>(); + routingData.put("zk-0", ImmutableList.of("sharding-key-0", "sharding-key-1", "sharding-key-2")); + routingData.put("zk-1", ImmutableList.of("sharding-key-3", "sharding-key-4", "sharding-key-5")); + routingData.put("zk-2", ImmutableList.of("sharding-key-6", "sharding-key-7", "sharding-key-8")); + + // Start MockMSDS + String host = "localhost"; + int port = 11000; + String endpoint = "http://"; + host + ":" + port; + String namespace = "MY-HELIX-NAMESPACE"; + MockMetadataStoreDirectoryServer server = + new MockMetadataStoreDirectoryServer(host, port, namespace, routingData); + server.startServer(); + CloseableHttpClient httpClient = HttpClients.createDefault(); + + // Send a GET request + String testZkRealm = "zk-0"; + HttpGet getRequest = new HttpGet( + endpoint + MockMetadataStoreDirectoryServer.REST_PREFIX + namespace + + MockMetadataStoreDirectoryServer.ZK_REALM_ENDPOINT + testZkRealm); + try { + CloseableHttpResponse getResponse = httpClient.execute(getRequest); + List<String> shardingKeyList = MockMetadataStoreDirectoryServer.OBJECT_MAPPER + .readValue(getResponse.getEntity().getContent(), List.class); + Assert.assertEquals(shardingKeyList, routingData.get(testZkRealm)); + } catch (IOException e) { + e.printStackTrace(); + } + + // Try sending a POST request (not supported) + HttpPost postRequest = new HttpPost( + endpoint + MockMetadataStoreDirectoryServer.REST_PREFIX + namespace + + MockMetadataStoreDirectoryServer.ZK_REALM_ENDPOINT + testZkRealm); + try { + CloseableHttpResponse postResponse = httpClient.execute(postRequest); + } catch (IOException e) { + e.printStackTrace(); + } + + // Shutdown + server.stopServer(); + } +}
