http://git-wip-us.apache.org/repos/asf/hadoop/blob/a129e3e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java new file mode 100644 index 0000000..7c5b6db --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper; + +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.w3c.dom.Element; +import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringMatch; +import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlBoolean; +import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlFloat; +import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlInt; +import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong; +import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Contains all value verifications that are needed to verify {@link AppInfo} + * XML documents. + */ +public final class AppInfoXmlVerifications { + + private AppInfoXmlVerifications() { + //utility class + } + + /** + * Tests whether {@link AppInfo} representation object contains the required + * values as per defined in the specified app parameter. + * @param info + * @param app an RMApp instance that contains the required values + */ + public static void verify(Element info, RMApp app) { + checkStringMatch("id", app.getApplicationId() + .toString(), getXmlString(info, "id")); + checkStringMatch("user", app.getUser(), + getXmlString(info, "user")); + checkStringMatch("name", app.getName(), + getXmlString(info, "name")); + checkStringMatch("applicationType", + app.getApplicationType(), getXmlString(info, "applicationType")); + checkStringMatch("queue", app.getQueue(), + getXmlString(info, "queue")); + assertEquals("priority doesn't match", 0, getXmlInt(info, "priority")); + checkStringMatch("state", app.getState().toString(), + getXmlString(info, "state")); + checkStringMatch("finalStatus", app + .getFinalApplicationStatus().toString(), + getXmlString(info, "finalStatus")); + assertEquals("progress doesn't match", 0, getXmlFloat(info, "progress"), + 0.0); + if ("UNASSIGNED".equals(getXmlString(info, "trackingUI"))) { + checkStringMatch("trackingUI", "UNASSIGNED", + getXmlString(info, "trackingUI")); + } + WebServicesTestUtils.checkStringEqual("diagnostics", + app.getDiagnostics().toString(), getXmlString(info, "diagnostics")); + assertEquals("clusterId doesn't match", + ResourceManager.getClusterTimeStamp(), + getXmlLong(info, "clusterId")); + assertEquals("startedTime doesn't match", app.getStartTime(), + getXmlLong(info, "startedTime")); + assertEquals("finishedTime doesn't match", app.getFinishTime(), + getXmlLong(info, "finishedTime")); + assertTrue("elapsed time not greater than 0", + getXmlLong(info, "elapsedTime") > 0); + checkStringMatch("amHostHttpAddress", app + .getCurrentAppAttempt().getMasterContainer() + .getNodeHttpAddress(), + getXmlString(info, "amHostHttpAddress")); + assertTrue("amContainerLogs doesn't match", + getXmlString(info, "amContainerLogs").startsWith("http://";)); + assertTrue("amContainerLogs doesn't contain user info", + getXmlString(info, "amContainerLogs").endsWith("/" + app.getUser())); + assertEquals("allocatedMB doesn't match", 1024, + getXmlInt(info, "allocatedMB")); + assertEquals("allocatedVCores doesn't match", 1, + getXmlInt(info, "allocatedVCores")); + assertEquals("queueUsagePerc doesn't match", 50.0f, + getXmlFloat(info, "queueUsagePercentage"), 0.01f); + assertEquals("clusterUsagePerc doesn't match", 50.0f, + getXmlFloat(info, "clusterUsagePercentage"), 0.01f); + assertEquals("numContainers doesn't match", 1, + getXmlInt(info, "runningContainers")); + assertNotNull("preemptedResourceSecondsMap should not be null", + info.getElementsByTagName("preemptedResourceSecondsMap")); + assertEquals("preemptedResourceMB doesn't match", app + .getRMAppMetrics().getResourcePreempted().getMemorySize(), + getXmlInt(info, "preemptedResourceMB")); + assertEquals("preemptedResourceVCores doesn't match", app + .getRMAppMetrics().getResourcePreempted().getVirtualCores(), + getXmlInt(info, "preemptedResourceVCores")); + assertEquals("numNonAMContainerPreempted doesn't match", app + .getRMAppMetrics().getNumNonAMContainersPreempted(), + getXmlInt(info, "numNonAMContainerPreempted")); + assertEquals("numAMContainerPreempted doesn't match", app + .getRMAppMetrics().getNumAMContainersPreempted(), + getXmlInt(info, "numAMContainerPreempted")); + assertEquals("Log aggregation Status doesn't match", app + .getLogAggregationStatusForAppReport().toString(), + getXmlString(info, "logAggregationStatus")); + assertEquals("unmanagedApplication doesn't match", app + .getApplicationSubmissionContext().getUnmanagedAM(), + getXmlBoolean(info, "unmanagedApplication")); + assertEquals("unmanagedApplication doesn't match", + app.getApplicationSubmissionContext().getNodeLabelExpression(), + getXmlString(info, "appNodeLabelExpression")); + assertEquals("unmanagedApplication doesn't match", + app.getAMResourceRequests().get(0).getNodeLabelExpression(), + getXmlString(info, "amNodeLabelExpression")); + assertEquals("amRPCAddress", + AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()), + getXmlString(info, "amRPCAddress")); + } +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a129e3e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java new file mode 100644 index 0000000..a8990ca --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper; + + +import com.sun.jersey.api.client.ClientHandlerException; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.UniformInterfaceException; + +import javax.ws.rs.core.MediaType; +import java.io.IOException; + +/** + * This class is merely a wrapper for {@link ClientResponse}. Given that the + * entity input stream of {@link ClientResponse} can be read only once by + * default and for some tests it is convenient to read the input stream many + * times, this class hides the details of how to do that and prevents + * unnecessary code duplication in tests. + */ +public class BufferedClientResponse { + private ClientResponse response; + + public BufferedClientResponse(ClientResponse response) { + response.bufferEntity(); + this.response = response; + } + + public <T> T getEntity(Class<T> clazz) + throws ClientHandlerException, UniformInterfaceException { + try { + response.getEntityInputStream().reset(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return response.getEntity(clazz); + } + + public MediaType getType() { + return response.getType(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a129e3e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java new file mode 100644 index 0000000..9d6a111 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper; + +import com.sun.jersey.api.client.WebResource; +import org.apache.hadoop.http.JettyUtils; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.MediaType; + +import java.util.function.Consumer; + +import static org.junit.Assert.*; + +/** + * This class hides the implementation details of how to verify the structure of + * JSON responses. Tests should only provide the path of the + * {@link WebResource}, the response from the resource and + * the verifier Consumer to + * {@link JsonCustomResourceTypeTestcase#verify(Consumer)}. An instance of + * {@link JSONObject} will be passed to that consumer to be able to + * verify the response. + */ +public class JsonCustomResourceTypeTestcase { + private static final Logger LOG = + LoggerFactory.getLogger(JsonCustomResourceTypeTestcase.class); + + private final WebResource path; + private final BufferedClientResponse response; + private final JSONObject parsedResponse; + + public JsonCustomResourceTypeTestcase(WebResource path, + BufferedClientResponse response) { + this.path = path; + this.response = response; + this.parsedResponse = response.getEntity(JSONObject.class); + } + + public void verify(Consumer<JSONObject> verifier) { + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + logResponse(); + + String responseStr = response.getEntity(String.class); + if (responseStr == null || responseStr.isEmpty()) { + throw new IllegalStateException("Response is null or empty!"); + } + verifier.accept(parsedResponse); + } + + private void logResponse() { + String responseStr = response.getEntity(String.class); + LOG.info("Raw response from service URL {}: {}", path.toString(), + responseStr); + LOG.info("Parsed response from service URL {}: {}", path.toString(), + parsedResponse); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/a129e3e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java new file mode 100644 index 0000000..6e58a89 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; + +import java.util.List; +import java.util.Map; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Performs value verifications on + * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo} + * objects against the values of {@link ResourceRequest}. With the help of the + * {@link Builder}, users can also make verifications of the custom resource + * types and its values. + */ +public class ResourceRequestsJsonVerifications { + private final ResourceRequest resourceRequest; + private final JSONObject requestInfo; + private final Map<String, Long> customResourceTypes; + private final List<String> expectedCustomResourceTypes; + + ResourceRequestsJsonVerifications(Builder builder) { + this.resourceRequest = builder.resourceRequest; + this.requestInfo = builder.requestInfo; + this.customResourceTypes = builder.customResourceTypes; + this.expectedCustomResourceTypes = builder.expectedCustomResourceTypes; + } + + public static void verify(JSONObject requestInfo, ResourceRequest rr) + throws JSONException { + createDefaultBuilder(requestInfo, rr).build().verify(); + } + + public static void verifyWithCustomResourceTypes(JSONObject requestInfo, + ResourceRequest resourceRequest, List<String> expectedResourceTypes) + throws JSONException { + + createDefaultBuilder(requestInfo, resourceRequest) + .withExpectedCustomResourceTypes(expectedResourceTypes) + .withCustomResourceTypes( + extractActualCustomResourceTypes(requestInfo, expectedResourceTypes)) + .build().verify(); + } + + private static Builder createDefaultBuilder(JSONObject requestInfo, + ResourceRequest resourceRequest) { + return new ResourceRequestsJsonVerifications.Builder() + .withRequest(resourceRequest) + .withRequestInfoJson(requestInfo); + } + + private static Map<String, Long> extractActualCustomResourceTypes( + JSONObject requestInfo, List<String> expectedResourceTypes) + throws JSONException { + JSONObject capability = requestInfo.getJSONObject("capability"); + Map<String, Long> resourceAndValue = + extractCustomResorceTypeValues(capability, expectedResourceTypes); + Map.Entry<String, Long> resourceEntry = + resourceAndValue.entrySet().iterator().next(); + + assertTrue( + "Found resource type: " + resourceEntry.getKey() + + " is not in expected resource types: " + expectedResourceTypes, + expectedResourceTypes.contains(resourceEntry.getKey())); + + return resourceAndValue; + } + + private static Map<String, Long> extractCustomResorceTypeValues( + JSONObject capability, List<String> expectedResourceTypes) + throws JSONException { + assertTrue( + "resourceCategory does not have resourceInformations: " + capability, + capability.has("resourceInformations")); + + JSONObject resourceInformations = + capability.getJSONObject("resourceInformations"); + assertTrue( + "resourceInformations does not have resourceInformation object: " + + resourceInformations, + resourceInformations.has("resourceInformation")); + JSONArray customResources = + resourceInformations.getJSONArray("resourceInformation"); + + // customResources will include vcores / memory as well + assertEquals( + "Different number of custom resource types found than expected", + expectedResourceTypes.size(), customResources.length() - 2); + + Map<String, Long> resourceValues = Maps.newHashMap(); + for (int i = 0; i < customResources.length(); i++) { + JSONObject customResource = customResources.getJSONObject(i); + assertTrue("Resource type does not have name field: " + customResource, + customResource.has("name")); + assertTrue("Resource type does not have name resourceType field: " + + customResource, customResource.has("resourceType")); + assertTrue( + "Resource type does not have name units field: " + customResource, + customResource.has("units")); + assertTrue( + "Resource type does not have name value field: " + customResource, + customResource.has("value")); + + String name = customResource.getString("name"); + String unit = customResource.getString("units"); + String resourceType = customResource.getString("resourceType"); + Long value = customResource.getLong("value"); + + if (ResourceInformation.MEMORY_URI.equals(name) + || ResourceInformation.VCORES_URI.equals(name)) { + continue; + } + + assertTrue("Custom resource type " + name + " not found", + expectedResourceTypes.contains(name)); + assertEquals("k", unit); + assertEquals(ResourceTypes.COUNTABLE, + ResourceTypes.valueOf(resourceType)); + assertNotNull("Custom resource value " + value + " is null!", value); + resourceValues.put(name, value); + } + + return resourceValues; + } + + private void verify() throws JSONException { + assertEquals("nodeLabelExpression doesn't match", + resourceRequest.getNodeLabelExpression(), + requestInfo.getString("nodeLabelExpression")); + assertEquals("numContainers doesn't match", + resourceRequest.getNumContainers(), + requestInfo.getInt("numContainers")); + assertEquals("relaxLocality doesn't match", + resourceRequest.getRelaxLocality(), + requestInfo.getBoolean("relaxLocality")); + assertEquals("priority does not match", + resourceRequest.getPriority().getPriority(), + requestInfo.getInt("priority")); + assertEquals("resourceName does not match", + resourceRequest.getResourceName(), + requestInfo.getString("resourceName")); + assertEquals("memory does not match", + resourceRequest.getCapability().getMemorySize(), + requestInfo.getJSONObject("capability").getLong("memory")); + assertEquals("vCores does not match", + resourceRequest.getCapability().getVirtualCores(), + requestInfo.getJSONObject("capability").getLong("vCores")); + + verifyAtLeastOneCustomResourceIsSerialized(); + + JSONObject executionTypeRequest = + requestInfo.getJSONObject("executionTypeRequest"); + assertEquals("executionType does not match", + resourceRequest.getExecutionTypeRequest().getExecutionType().name(), + executionTypeRequest.getString("executionType")); + assertEquals("enforceExecutionType does not match", + resourceRequest.getExecutionTypeRequest().getEnforceExecutionType(), + executionTypeRequest.getBoolean("enforceExecutionType")); + } + + /** + * JSON serialization produces "invalid JSON" by default as maps are + * serialized like this: + * "customResources":{"entry":{"key":"customResource-1","value":"0"}} + * If the map has multiple keys then multiple entries will be serialized. + * Our json parser in tests cannot handle duplicates therefore only one + * custom resource will be in the parsed json. See: + * https://issues.apache.org/jira/browse/YARN-7505 + */ + private void verifyAtLeastOneCustomResourceIsSerialized() { + boolean resourceFound = false; + for (String expectedCustomResourceType : expectedCustomResourceTypes) { + if (customResourceTypes.containsKey(expectedCustomResourceType)) { + resourceFound = true; + Long resourceValue = + customResourceTypes.get(expectedCustomResourceType); + assertNotNull("Resource value should not be null!", resourceValue); + } + } + assertTrue("No custom resource type can be found in the response!", + resourceFound); + } + + /** + * Builder class for {@link ResourceRequestsJsonVerifications}. + */ + public static final class Builder { + private List<String> expectedCustomResourceTypes = Lists.newArrayList(); + private Map<String, Long> customResourceTypes; + private ResourceRequest resourceRequest; + private JSONObject requestInfo; + + Builder() { + } + + public static Builder create() { + return new Builder(); + } + + Builder withExpectedCustomResourceTypes( + List<String> expectedCustomResourceTypes) { + this.expectedCustomResourceTypes = expectedCustomResourceTypes; + return this; + } + + Builder withCustomResourceTypes( + Map<String, Long> customResourceTypes) { + this.customResourceTypes = customResourceTypes; + return this; + } + + Builder withRequest(ResourceRequest resourceRequest) { + this.resourceRequest = resourceRequest; + return this; + } + + Builder withRequestInfoJson(JSONObject requestInfo) { + this.requestInfo = requestInfo; + return this; + } + + public ResourceRequestsJsonVerifications build() { + return new ResourceRequestsJsonVerifications(this); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a129e3e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java new file mode 100644 index 0000000..af9b0f3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static junit.framework.TestCase.assertTrue; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase.toXml; +import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlBoolean; +import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlInt; +import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong; +import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Performs value verifications on + * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo} + * objects against the values of {@link ResourceRequest}. With the help of the + * {@link Builder}, users can also make verifications of the custom resource + * types and its values. + */ +public class ResourceRequestsXmlVerifications { + private final ResourceRequest resourceRequest; + private final Element requestInfo; + private final Map<String, Long> customResourceTypes; + private final List<String> expectedCustomResourceTypes; + + ResourceRequestsXmlVerifications(Builder builder) { + this.resourceRequest = builder.resourceRequest; + this.requestInfo = builder.requestInfo; + this.customResourceTypes = builder.customResourceTypes; + this.expectedCustomResourceTypes = builder.expectedCustomResourceTypes; + } + + public static void verifyWithCustomResourceTypes(Element requestInfo, + ResourceRequest resourceRequest, List<String> expectedResourceTypes) { + + createDefaultBuilder(requestInfo, resourceRequest) + .withExpectedCustomResourceTypes(expectedResourceTypes) + .withCustomResourceTypes(extractActualCustomResourceType(requestInfo, + expectedResourceTypes)) + .build().verify(); + } + + private static Builder createDefaultBuilder(Element requestInfo, + ResourceRequest resourceRequest) { + return new ResourceRequestsXmlVerifications.Builder() + .withRequest(resourceRequest).withRequestInfo(requestInfo); + } + + private static Map<String, Long> extractActualCustomResourceType( + Element requestInfo, List<String> expectedResourceTypes) { + Element capability = + (Element) requestInfo.getElementsByTagName("capability").item(0); + + return extractCustomResorceTypes(capability, + Sets.newHashSet(expectedResourceTypes)); + } + + private static Map<String, Long> extractCustomResorceTypes(Element capability, + Set<String> expectedResourceTypes) { + assertEquals( + toXml(capability) + " should have only one resourceInformations child!", + 1, capability.getElementsByTagName("resourceInformations").getLength()); + Element resourceInformations = (Element) capability + .getElementsByTagName("resourceInformations").item(0); + + NodeList customResources = + resourceInformations.getElementsByTagName("resourceInformation"); + + // customResources will include vcores / memory as well + assertEquals( + "Different number of custom resource types found than expected", + expectedResourceTypes.size(), customResources.getLength() - 2); + + Map<String, Long> resourceTypesAndValues = Maps.newHashMap(); + for (int i = 0; i < customResources.getLength(); i++) { + Element customResource = (Element) customResources.item(i); + String name = getXmlString(customResource, "name"); + String unit = getXmlString(customResource, "units"); + String resourceType = getXmlString(customResource, "resourceType"); + Long value = getXmlLong(customResource, "value"); + + if (ResourceInformation.MEMORY_URI.equals(name) + || ResourceInformation.VCORES_URI.equals(name)) { + continue; + } + + assertTrue("Custom resource type " + name + " not found", + expectedResourceTypes.contains(name)); + assertEquals("k", unit); + assertEquals(ResourceTypes.COUNTABLE, + ResourceTypes.valueOf(resourceType)); + assertNotNull("Resource value should not be null for resource type " + + resourceType + ", listing xml contents: " + toXml(customResource), + value); + resourceTypesAndValues.put(name, value); + } + + return resourceTypesAndValues; + } + + private void verify() { + assertEquals("nodeLabelExpression doesn't match", + resourceRequest.getNodeLabelExpression(), + getXmlString(requestInfo, "nodeLabelExpression")); + assertEquals("numContainers doesn't match", + resourceRequest.getNumContainers(), + getXmlInt(requestInfo, "numContainers")); + assertEquals("relaxLocality doesn't match", + resourceRequest.getRelaxLocality(), + getXmlBoolean(requestInfo, "relaxLocality")); + assertEquals("priority does not match", + resourceRequest.getPriority().getPriority(), + getXmlInt(requestInfo, "priority")); + assertEquals("resourceName does not match", + resourceRequest.getResourceName(), + getXmlString(requestInfo, "resourceName")); + Element capability = (Element) requestInfo + .getElementsByTagName("capability").item(0); + assertEquals("memory does not match", + resourceRequest.getCapability().getMemorySize(), + getXmlLong(capability, "memory")); + assertEquals("vCores does not match", + resourceRequest.getCapability().getVirtualCores(), + getXmlLong(capability, "vCores")); + + for (String expectedCustomResourceType : expectedCustomResourceTypes) { + assertTrue( + "Custom resource type " + expectedCustomResourceType + + " cannot be found!", + customResourceTypes.containsKey(expectedCustomResourceType)); + + Long resourceValue = customResourceTypes.get(expectedCustomResourceType); + assertNotNull("Resource value should not be null!", resourceValue); + } + + Element executionTypeRequest = (Element) requestInfo + .getElementsByTagName("executionTypeRequest").item(0); + assertEquals("executionType does not match", + resourceRequest.getExecutionTypeRequest().getExecutionType().name(), + getXmlString(executionTypeRequest, "executionType")); + assertEquals("enforceExecutionType does not match", + resourceRequest.getExecutionTypeRequest().getEnforceExecutionType(), + getXmlBoolean(executionTypeRequest, "enforceExecutionType")); + } + + /** + * Builder class for {@link ResourceRequestsXmlVerifications}. + */ + public static final class Builder { + private List<String> expectedCustomResourceTypes = Lists.newArrayList(); + private Map<String, Long> customResourceTypes; + private ResourceRequest resourceRequest; + private Element requestInfo; + + Builder() { + } + + public static Builder create() { + return new Builder(); + } + + Builder withExpectedCustomResourceTypes( + List<String> expectedCustomResourceTypes) { + this.expectedCustomResourceTypes = expectedCustomResourceTypes; + return this; + } + + Builder withCustomResourceTypes(Map<String, Long> customResourceTypes) { + this.customResourceTypes = customResourceTypes; + return this; + } + + Builder withRequest(ResourceRequest resourceRequest) { + this.resourceRequest = resourceRequest; + return this; + } + + Builder withRequestInfo(Element requestInfo) { + this.requestInfo = requestInfo; + return this; + } + + public ResourceRequestsXmlVerifications build() { + return new ResourceRequestsXmlVerifications(this); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a129e3e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java new file mode 100644 index 0000000..29260aa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper; + +import com.sun.jersey.api.client.WebResource; +import org.apache.hadoop.http.JettyUtils; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.xml.sax.InputSource; + +import javax.ws.rs.core.MediaType; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.*; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.function.Consumer; + +import static org.junit.Assert.assertEquals; + +/** + * This class hides the implementation details of how to verify the structure of + * XML responses. Tests should only provide the path of the + * {@link WebResource}, the response from the resource and + * the verifier Consumer to + * {@link XmlCustomResourceTypeTestCase#verify(Consumer)}. An instance of + * {@link JSONObject} will be passed to that consumer to be able to + * verify the response. + */ +public class XmlCustomResourceTypeTestCase { + private static final Logger LOG = + LoggerFactory.getLogger(XmlCustomResourceTypeTestCase.class); + + private WebResource path; + private BufferedClientResponse response; + private Document parsedResponse; + + public XmlCustomResourceTypeTestCase(WebResource path, + BufferedClientResponse response) { + this.path = path; + this.response = response; + } + + public void verify(Consumer<Document> verifier) { + assertEquals(MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + parsedResponse = parseXml(response); + logResponse(parsedResponse); + verifier.accept(parsedResponse); + } + + private Document parseXml(BufferedClientResponse response) { + try { + String xml = response.getEntity(String.class); + DocumentBuilder db = + DocumentBuilderFactory.newInstance().newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + + return db.parse(is); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void logResponse(Document doc) { + String responseStr = response.getEntity(String.class); + LOG.info("Raw response from service URL {}: {}", path.toString(), + responseStr); + LOG.info("Parsed response from service URL {}: {}", path.toString(), + toXml(doc)); + } + + public static String toXml(Node node) { + StringWriter writer; + try { + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.setOutputProperty( + "{http://xml.apache.org/xslt}indent"; + "-amount", "2"); + writer = new StringWriter(); + transformer.transform(new DOMSource(node), new StreamResult(writer)); + } catch (TransformerException e) { + throw new RuntimeException(e); + } + + return writer.getBuffer().toString(); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
