YARN-2804. Fixed Timeline service to not fill the logs with JAXB bindings exceptions. Contributed by Zhijie Shen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7617989 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7617989 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7617989 Branch: refs/heads/HDFS-EC Commit: b76179895dd2ef4d56e8de31e9f673375faa2afa Parents: d794f78 Author: Vinod Kumar Vavilapalli <vino...@apache.org> Authored: Tue Nov 4 17:45:46 2014 -0800 Committer: Vinod Kumar Vavilapalli <vino...@apache.org> Committed: Tue Nov 4 18:00:06 2014 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../api/records/timeline/TimelineEntity.java | 49 +++++-- .../api/records/timeline/TimelineEvent.java | 17 ++- .../records/timeline/TestTimelineRecords.java | 128 ++++++++++++++++++- .../timeline/webapp/TimelineWebServices.java | 9 +- 5 files changed, 182 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7617989/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f9fcf5b..b5aec42 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -859,6 +859,9 @@ Release 2.6.0 - UNRELEASED YARN-2010. Handle app-recovery failures gracefully. (Jian He and Karthik Kambatla via kasha) + YARN-2804. Fixed Timeline service to not fill the logs with JAXB bindings + exceptions. (Zhijie Shen via vinodkv) + Release 2.5.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7617989/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java index d66c253..03ed4af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java @@ -31,6 +31,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -58,11 +59,11 @@ public class TimelineEntity implements Comparable<TimelineEntity> { private String entityId; private Long startTime; private List<TimelineEvent> events = new ArrayList<TimelineEvent>(); - private Map<String, Set<String>> relatedEntities = + private HashMap<String, Set<String>> relatedEntities = new HashMap<String, Set<String>>(); - private Map<String, Set<Object>> primaryFilters = + private HashMap<String, Set<Object>> primaryFilters = new HashMap<String, Set<Object>>(); - private Map<String, Object> otherInfo = + private HashMap<String, Object> otherInfo = new HashMap<String, Object>(); private String domainId; @@ -175,11 +176,17 @@ public class TimelineEntity implements Comparable<TimelineEntity> { * * @return the related entities */ - @XmlElement(name = "relatedentities") public Map<String, Set<String>> getRelatedEntities() { return relatedEntities; } + // Required by JAXB + @Private + @XmlElement(name = "relatedentities") + public HashMap<String, Set<String>> getRelatedEntitiesJAXB() { + return relatedEntities; + } + /** * Add an entity to the existing related entity map * @@ -224,7 +231,11 @@ public class TimelineEntity implements Comparable<TimelineEntity> { */ public void setRelatedEntities( Map<String, Set<String>> relatedEntities) { - this.relatedEntities = relatedEntities; + if (relatedEntities != null && !(relatedEntities instanceof HashMap)) { + this.relatedEntities = new HashMap<String, Set<String>>(relatedEntities); + } else { + this.relatedEntities = (HashMap<String, Set<String>>) relatedEntities; + } } /** @@ -232,11 +243,17 @@ public class TimelineEntity implements Comparable<TimelineEntity> { * * @return the primary filters */ - @XmlElement(name = "primaryfilters") public Map<String, Set<Object>> getPrimaryFilters() { return primaryFilters; } + // Required by JAXB + @Private + @XmlElement(name = "primaryfilters") + public HashMap<String, Set<Object>> getPrimaryFiltersJAXB() { + return primaryFilters; + } + /** * Add a single piece of primary filter to the existing primary filter map * @@ -280,7 +297,11 @@ public class TimelineEntity implements Comparable<TimelineEntity> { * a map of primary filters */ public void setPrimaryFilters(Map<String, Set<Object>> primaryFilters) { - this.primaryFilters = primaryFilters; + if (primaryFilters != null && !(primaryFilters instanceof HashMap)) { + this.primaryFilters = new HashMap<String, Set<Object>>(primaryFilters); + } else { + this.primaryFilters = (HashMap<String, Set<Object>>) primaryFilters; + } } /** @@ -288,11 +309,17 @@ public class TimelineEntity implements Comparable<TimelineEntity> { * * @return the other information of the entity */ - @XmlElement(name = "otherinfo") public Map<String, Object> getOtherInfo() { return otherInfo; } + // Required by JAXB + @Private + @XmlElement(name = "otherinfo") + public HashMap<String, Object> getOtherInfoJAXB() { + return otherInfo; + } + /** * Add one piece of other information of the entity to the existing other info * map @@ -323,7 +350,11 @@ public class TimelineEntity implements Comparable<TimelineEntity> { * a map of other information */ public void setOtherInfo(Map<String, Object> otherInfo) { - this.otherInfo = otherInfo; + if (otherInfo != null && !(otherInfo instanceof HashMap)) { + this.otherInfo = new HashMap<String, Object>(otherInfo); + } else { + this.otherInfo = (HashMap<String, Object>) otherInfo; + } } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7617989/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java index aa49538..0a09e78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java @@ -26,6 +26,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -43,7 +44,7 @@ public class TimelineEvent implements Comparable<TimelineEvent> { private long timestamp; private String eventType; - private Map<String, Object> eventInfo = new HashMap<String, Object>(); + private HashMap<String, Object> eventInfo = new HashMap<String, Object>(); public TimelineEvent() { } @@ -93,11 +94,17 @@ public class TimelineEvent implements Comparable<TimelineEvent> { * * @return the information of the event */ - @XmlElement(name = "eventinfo") public Map<String, Object> getEventInfo() { return eventInfo; } + // Required by JAXB + @Private + @XmlElement(name = "eventinfo") + public HashMap<String, Object> getEventInfoJAXB() { + return eventInfo; + } + /** * Add one piece of the information of the event to the existing information * map @@ -128,7 +135,11 @@ public class TimelineEvent implements Comparable<TimelineEvent> { * a map of of the information of the event */ public void setEventInfo(Map<String, Object> eventInfo) { - this.eventInfo = eventInfo; + if (eventInfo != null && !(eventInfo instanceof HashMap)) { + this.eventInfo = new HashMap<String, Object>(eventInfo); + } else { + this.eventInfo = (HashMap<String, Object>) eventInfo; + } } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7617989/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java index 2b59ff5..9d16edb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java @@ -19,19 +19,19 @@ package org.apache.hadoop.yarn.api.records.timeline; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; - -import org.junit.Assert; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.WeakHashMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; -import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; +import org.junit.Assert; import org.junit.Test; public class TestTimelineRecords { @@ -202,4 +202,118 @@ public class TestTimelineRecords { } } + @Test + public void testMapInterfaceOrTimelineRecords() throws Exception { + TimelineEntity entity = new TimelineEntity(); + List<Map<String, Set<Object>>> primaryFiltersList = + new ArrayList<Map<String, Set<Object>>>(); + primaryFiltersList.add( + Collections.singletonMap("pkey", Collections.singleton((Object) "pval"))); + Map<String, Set<Object>> primaryFilters = new TreeMap<String, Set<Object>>(); + primaryFilters.put("pkey1", Collections.singleton((Object) "pval1")); + primaryFilters.put("pkey2", Collections.singleton((Object) "pval2")); + primaryFiltersList.add(primaryFilters); + entity.setPrimaryFilters(null); + for (Map<String, Set<Object>> primaryFiltersToSet : primaryFiltersList) { + entity.setPrimaryFilters(primaryFiltersToSet); + assertPrimaryFilters(entity); + + Map<String, Set<Object>> primaryFiltersToAdd = + new WeakHashMap<String, Set<Object>>(); + primaryFiltersToAdd.put("pkey3", Collections.singleton((Object) "pval3")); + entity.addPrimaryFilters(primaryFiltersToAdd); + assertPrimaryFilters(entity); + } + + List<Map<String, Set<String>>> relatedEntitiesList = + new ArrayList<Map<String, Set<String>>>(); + relatedEntitiesList.add( + Collections.singletonMap("rkey", Collections.singleton("rval"))); + Map<String, Set<String>> relatedEntities = new TreeMap<String, Set<String>>(); + relatedEntities.put("rkey1", Collections.singleton("rval1")); + relatedEntities.put("rkey2", Collections.singleton("rval2")); + relatedEntitiesList.add(relatedEntities); + entity.setRelatedEntities(null); + for (Map<String, Set<String>> relatedEntitiesToSet : relatedEntitiesList) { + entity.setRelatedEntities(relatedEntitiesToSet); + assertRelatedEntities(entity); + + Map<String, Set<String>> relatedEntitiesToAdd = + new WeakHashMap<String, Set<String>>(); + relatedEntitiesToAdd.put("rkey3", Collections.singleton("rval3")); + entity.addRelatedEntities(relatedEntitiesToAdd); + assertRelatedEntities(entity); + } + + List<Map<String, Object>> otherInfoList = + new ArrayList<Map<String, Object>>(); + otherInfoList.add(Collections.singletonMap("okey", (Object) "oval")); + Map<String, Object> otherInfo = new TreeMap<String, Object>(); + otherInfo.put("okey1", "oval1"); + otherInfo.put("okey2", "oval2"); + otherInfoList.add(otherInfo); + entity.setOtherInfo(null); + for (Map<String, Object> otherInfoToSet : otherInfoList) { + entity.setOtherInfo(otherInfoToSet); + assertOtherInfo(entity); + + Map<String, Object> otherInfoToAdd = new WeakHashMap<String, Object>(); + otherInfoToAdd.put("okey3", "oval3"); + entity.addOtherInfo(otherInfoToAdd); + assertOtherInfo(entity); + } + + TimelineEvent event = new TimelineEvent(); + List<Map<String, Object>> eventInfoList = + new ArrayList<Map<String, Object>>(); + eventInfoList.add(Collections.singletonMap("ekey", (Object) "eval")); + Map<String, Object> eventInfo = new TreeMap<String, Object>(); + eventInfo.put("ekey1", "eval1"); + eventInfo.put("ekey2", "eval2"); + eventInfoList.add(eventInfo); + event.setEventInfo(null); + for (Map<String, Object> eventInfoToSet : eventInfoList) { + event.setEventInfo(eventInfoToSet); + assertEventInfo(event); + + Map<String, Object> eventInfoToAdd = new WeakHashMap<String, Object>(); + eventInfoToAdd.put("ekey3", "eval3"); + event.addEventInfo(eventInfoToAdd); + assertEventInfo(event); + } + } + + private static void assertPrimaryFilters(TimelineEntity entity) { + Assert.assertNotNull(entity.getPrimaryFilters()); + Assert.assertNotNull(entity.getPrimaryFiltersJAXB()); + Assert.assertTrue(entity.getPrimaryFilters() instanceof HashMap); + Assert.assertTrue(entity.getPrimaryFiltersJAXB() instanceof HashMap); + Assert.assertEquals( + entity.getPrimaryFilters(), entity.getPrimaryFiltersJAXB()); + } + + private static void assertRelatedEntities(TimelineEntity entity) { + Assert.assertNotNull(entity.getRelatedEntities()); + Assert.assertNotNull(entity.getRelatedEntitiesJAXB()); + Assert.assertTrue(entity.getRelatedEntities() instanceof HashMap); + Assert.assertTrue(entity.getRelatedEntitiesJAXB() instanceof HashMap); + Assert.assertEquals( + entity.getRelatedEntities(), entity.getRelatedEntitiesJAXB()); + } + + private static void assertOtherInfo(TimelineEntity entity) { + Assert.assertNotNull(entity.getOtherInfo()); + Assert.assertNotNull(entity.getOtherInfoJAXB()); + Assert.assertTrue(entity.getOtherInfo() instanceof HashMap); + Assert.assertTrue(entity.getOtherInfoJAXB() instanceof HashMap); + Assert.assertEquals(entity.getOtherInfo(), entity.getOtherInfoJAXB()); + } + + private static void assertEventInfo(TimelineEvent event) { + Assert.assertNotNull(event); + Assert.assertNotNull(event.getEventInfoJAXB()); + Assert.assertTrue(event.getEventInfo() instanceof HashMap); + Assert.assertTrue(event.getEventInfoJAXB() instanceof HashMap); + Assert.assertEquals(event.getEventInfo(), event.getEventInfoJAXB()); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7617989/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java index f290930..0907f2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java @@ -42,7 +42,6 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.Status; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; @@ -53,11 +52,11 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; -import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; -import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.timeline.EntityIdentifier; @@ -272,7 +271,7 @@ public class TimelineWebServices { @PUT @Path("/domain") @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) - public Response putDomain( + public TimelinePutResponse putDomain( @Context HttpServletRequest req, @Context HttpServletResponse res, TimelineDomain domain) { @@ -295,7 +294,7 @@ public class TimelineWebServices { throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); } - return Response.status(Status.OK).build(); + return new TimelinePutResponse(); } /**