YARN-5093. created time shows 0 in most REST output (Varun Saxena via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/050846f5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/050846f5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/050846f5 Branch: refs/heads/YARN-2928 Commit: 050846f597fccd68b9b445abcd62fb30c3d49262 Parents: c8100fd Author: Sangjin Lee <sj...@apache.org> Authored: Tue May 24 10:33:04 2016 -0700 Committer: Sangjin Lee <sj...@apache.org> Committed: Fri Jul 8 10:19:04 2016 -0700 ---------------------------------------------------------------------- .../records/timelineservice/TimelineEntity.java | 16 +++++++++-- .../storage/TestHBaseTimelineStorage.java | 30 +++++++++++++------- .../storage/FileSystemTimelineReaderImpl.java | 2 +- .../storage/reader/ApplicationEntityReader.java | 5 ++-- .../storage/reader/GenericEntityReader.java | 4 +-- .../reader/TestTimelineReaderWebServices.java | 4 +-- .../TestFileSystemTimelineReaderImpl.java | 10 +++---- 7 files changed, 45 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/050846f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java index 7ce8279..9c0a983 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java @@ -144,7 +144,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> { private NavigableSet<TimelineEvent> events = new TreeSet<>(); private HashMap<String, Set<String>> isRelatedToEntities = new HashMap<>(); private HashMap<String, Set<String>> relatesToEntities = new HashMap<>(); - private long createdTime; + private Long createdTime; public TimelineEntity() { identifier = new Identifier(); @@ -490,7 +490,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> { } @XmlElement(name = "createdtime") - public long getCreatedTime() { + public Long getCreatedTime() { if (real == null) { return createdTime; } else { @@ -499,7 +499,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> { } @JsonSetter("createdtime") - public void setCreatedTime(long createdTs) { + public void setCreatedTime(Long createdTs) { if (real == null) { this.createdTime = createdTs; } else { @@ -547,6 +547,16 @@ public class TimelineEntity implements Comparable<TimelineEntity> { public int compareTo(TimelineEntity other) { int comparison = getType().compareTo(other.getType()); if (comparison == 0) { + if (getCreatedTime() == null) { + if (other.getCreatedTime() == null) { + return getId().compareTo(other.getId()); + } else { + return 1; + } + } + if (other.getCreatedTime() == null) { + return -1; + } if (getCreatedTime() > other.getCreatedTime()) { // Order by created time desc return -1; http://git-wip-us.apache.org/repos/asf/hadoop/blob/050846f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java index 8ab54bc..aebd936 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java @@ -488,7 +488,7 @@ public class TestHBaseTimelineStorage { ApplicationEntity entity = new ApplicationEntity(); String appId = "application_1000178881110_2002"; entity.setId(appId); - long cTime = 1425016501000L; + Long cTime = 1425016501000L; entity.setCreatedTime(cTime); // add the info map in Timeline Entity @@ -546,7 +546,6 @@ public class TestHBaseTimelineStorage { aggEntity.setId(appId); aggEntity.setType(type); long cTime2 = 1425016502000L; - long mTime2 = 1425026902000L; aggEntity.setCreatedTime(cTime2); TimelineMetric aggMetric = new TimelineMetric(); @@ -574,8 +573,21 @@ public class TestHBaseTimelineStorage { String flowVersion = "AB7822C10F1111"; long runid = 1002345678919L; hbi.write(cluster, user, flow, flowVersion, runid, appId, te); + + // Write entity again, this time without created time. + entity = new ApplicationEntity(); + appId = "application_1000178881110_2002"; + entity.setId(appId); + // add the info map in Timeline Entity + Map<String, Object> infoMap1 = new HashMap<>(); + infoMap1.put("infoMapKey3", "infoMapValue1"); + entity.addInfo(infoMap1); + te = new TimelineEntities(); + te.addEntity(entity); + hbi.write(cluster, user, flow, flowVersion, runid, appId, te); hbi.stop(); + infoMap.putAll(infoMap1); // retrieve the row byte[] rowKey = ApplicationRowKey.getRowKey(cluster, user, flow, runid, appId); @@ -585,7 +597,7 @@ public class TestHBaseTimelineStorage { Result result = new ApplicationTable().getResult(c1, conn, get); assertTrue(result != null); - assertEquals(16, result.size()); + assertEquals(17, result.size()); // check the row key byte[] row1 = result.getRow(); @@ -596,10 +608,9 @@ public class TestHBaseTimelineStorage { String id1 = ApplicationColumn.ID.readResult(result).toString(); assertEquals(appId, id1); - Number val = - (Number) ApplicationColumn.CREATED_TIME.readResult(result); - long cTime1 = val.longValue(); - assertEquals(cTime1, cTime); + Long cTime1 = + (Long) ApplicationColumn.CREATED_TIME.readResult(result); + assertEquals(cTime, cTime1); Map<String, Object> infoColumns = ApplicationColumnPrefix.INFO.readResults(result); @@ -701,7 +712,7 @@ public class TestHBaseTimelineStorage { String type = "world"; entity.setId(id); entity.setType(type); - long cTime = 1425016501000L; + Long cTime = 1425016501000L; entity.setCreatedTime(cTime); // add the info map in Timeline Entity @@ -796,8 +807,7 @@ public class TestHBaseTimelineStorage { String type1 = EntityColumn.TYPE.readResult(result).toString(); assertEquals(type, type1); - Number val = (Number) EntityColumn.CREATED_TIME.readResult(result); - long cTime1 = val.longValue(); + Long cTime1 = (Long) EntityColumn.CREATED_TIME.readResult(result); assertEquals(cTime1, cTime); Map<String, Object> infoColumns = http://git-wip-us.apache.org/repos/asf/hadoop/blob/050846f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java index bdddd7e..00aa686 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java @@ -204,7 +204,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService private static void mergeEntities(TimelineEntity entity1, TimelineEntity entity2) { // Ideally created time wont change except in the case of issue from client. - if (entity2.getCreatedTime() > 0) { + if (entity2.getCreatedTime() != null && entity2.getCreatedTime() > 0) { entity1.setCreatedTime(entity2.getCreatedTime()); } for (Entry<String, String> configEntry : entity2.getConfigs().entrySet()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/050846f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java index 53210f8..71c90fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java @@ -397,9 +397,8 @@ class ApplicationEntityReader extends GenericEntityReader { TimelineEntityFilters filters = getFilters(); // fetch created time - Number createdTime = - (Number)ApplicationColumn.CREATED_TIME.readResult(result); - entity.setCreatedTime(createdTime.longValue()); + Long createdTime = (Long) ApplicationColumn.CREATED_TIME.readResult(result); + entity.setCreatedTime(createdTime); EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve(); // fetch is related to entities and match isRelatedTo filter. If isRelatedTo http://git-wip-us.apache.org/repos/asf/hadoop/blob/050846f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java index 6696ac5..878695c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java @@ -526,8 +526,8 @@ class GenericEntityReader extends TimelineEntityReader { TimelineEntityFilters filters = getFilters(); // fetch created time - Number createdTime = (Number)EntityColumn.CREATED_TIME.readResult(result); - entity.setCreatedTime(createdTime.longValue()); + Long createdTime = (Long) EntityColumn.CREATED_TIME.readResult(result); + entity.setCreatedTime(createdTime); EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve(); // fetch is related to entities and match isRelatedTo filter. If isRelatedTo http://git-wip-us.apache.org/repos/asf/hadoop/blob/050846f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java index 6c3f3e3..0bddf1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java @@ -173,7 +173,7 @@ public class TestTimelineReaderWebServices { assertNotNull(entity); assertEquals("id_1", entity.getId()); assertEquals("app", entity.getType()); - assertEquals(1425016502000L, entity.getCreatedTime()); + assertEquals((Long)1425016502000L, entity.getCreatedTime()); // Default view i.e. when no fields are specified, entity contains only // entity id, entity type and created time. assertEquals(0, entity.getConfigs().size()); @@ -196,7 +196,7 @@ public class TestTimelineReaderWebServices { assertNotNull(entity); assertEquals("id_1", entity.getId()); assertEquals("app", entity.getType()); - assertEquals(1425016502000L, entity.getCreatedTime()); + assertEquals((Long)1425016502000L, entity.getCreatedTime()); } finally { client.destroy(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/050846f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java index 23d64e0..ad2122c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java @@ -278,7 +278,7 @@ public class TestFileSystemTimelineReaderImpl { Assert.assertEquals( (new TimelineEntity.Identifier("app", "id_1")).toString(), result.getIdentifier().toString()); - Assert.assertEquals(1425016502000L, result.getCreatedTime()); + Assert.assertEquals((Long)1425016502000L, result.getCreatedTime()); Assert.assertEquals(0, result.getConfigs().size()); Assert.assertEquals(0, result.getMetrics().size()); } @@ -293,7 +293,7 @@ public class TestFileSystemTimelineReaderImpl { Assert.assertEquals( (new TimelineEntity.Identifier("app", "id_1")).toString(), result.getIdentifier().toString()); - Assert.assertEquals(1425016502000L, result.getCreatedTime()); + Assert.assertEquals((Long)1425016502000L, result.getCreatedTime()); Assert.assertEquals(0, result.getConfigs().size()); Assert.assertEquals(0, result.getMetrics().size()); } @@ -310,7 +310,7 @@ public class TestFileSystemTimelineReaderImpl { Assert.assertEquals( (new TimelineEntity.Identifier("app", "id_5")).toString(), result.getIdentifier().toString()); - Assert.assertEquals(1425016502050L, result.getCreatedTime()); + Assert.assertEquals((Long)1425016502050L, result.getCreatedTime()); } @Test @@ -324,7 +324,7 @@ public class TestFileSystemTimelineReaderImpl { Assert.assertEquals( (new TimelineEntity.Identifier("app", "id_1")).toString(), result.getIdentifier().toString()); - Assert.assertEquals(1425016502000L, result.getCreatedTime()); + Assert.assertEquals((Long)1425016502000L, result.getCreatedTime()); Assert.assertEquals(3, result.getConfigs().size()); Assert.assertEquals(3, result.getMetrics().size()); Assert.assertEquals(2, result.getInfo().size()); @@ -342,7 +342,7 @@ public class TestFileSystemTimelineReaderImpl { Assert.assertEquals( (new TimelineEntity.Identifier("app", "id_1")).toString(), result.getIdentifier().toString()); - Assert.assertEquals(1425016502000L, result.getCreatedTime()); + Assert.assertEquals((Long)1425016502000L, result.getCreatedTime()); Assert.assertEquals(3, result.getConfigs().size()); Assert.assertEquals(3, result.getMetrics().size()); // All fields including events will be returned. --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org