YARN-4210. HBase reader throws NPE if Get returns no rows (Varun Saxena via vrushali)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7a885e3f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7a885e3f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7a885e3f Branch: refs/heads/YARN-2928-rebase Commit: 7a885e3f020b9733bf838391fbe606a126596b67 Parents: 6e0ae24 Author: Vrushali Channapattan <vrush...@apache.org> Authored: Wed Sep 30 13:56:07 2015 -0700 Committer: Sangjin Lee <sj...@apache.org> Committed: Mon Nov 9 16:13:15 2015 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../storage/ApplicationEntityReader.java | 21 ++-------- .../storage/FlowActivityEntityReader.java | 27 +++++++------ .../storage/FlowRunEntityReader.java | 11 +++--- .../storage/GenericEntityReader.java | 3 +- .../storage/TimelineEntityReader.java | 37 ++++++++++++------ .../TestTimelineReaderWebServicesFlowRun.java | 40 ++++++++++++++++++++ .../storage/TestHBaseTimelineStorage.java | 13 ------- 8 files changed, 96 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a885e3f/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6756fb8..f57d92a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -115,6 +115,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1 YARN-4203. Add request/response logging & timing for each REST endpoint call (Varun Saxena via vrushali) + YARN-4210. HBase reader throws NPE if Get returns no rows (Varun Saxena + via vrushali) + IMPROVEMENTS YARN-3276. Code cleanup for timeline service API records. (Junping Du via http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a885e3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java index dfbc31d..d5b5d63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field; @@ -85,24 +86,10 @@ class ApplicationEntityReader extends GenericEntityReader { } @Override - protected Iterable<Result> getResults(Configuration hbaseConf, + protected ResultScanner getResults(Configuration hbaseConf, Connection conn) throws IOException { - // If getEntities() is called for an application, there can be at most - // one entity. If the entity passes the filter, it is returned. Otherwise, - // an empty set is returned. - byte[] rowKey = ApplicationRowKey.getRowKey(clusterId, userId, flowId, - flowRunId, appId); - Get get = new Get(rowKey); - get.setMaxVersions(Integer.MAX_VALUE); - Result result = table.getResult(hbaseConf, conn, get); - TimelineEntity entity = parseEntity(result); - Set<Result> set; - if (entity != null) { - set = Collections.singleton(result); - } else { - set = Collections.emptySet(); - } - return set; + throw new UnsupportedOperationException( + "we don't support multiple apps query"); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a885e3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java index d5ece2e..e68ca17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java @@ -27,6 +27,7 @@ import java.util.TreeSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.PageFilter; import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity; @@ -88,18 +89,22 @@ class FlowActivityEntityReader extends TimelineEntityReader { augmentParams(hbaseConf, conn); NavigableSet<TimelineEntity> entities = new TreeSet<>(); - Iterable<Result> results = getResults(hbaseConf, conn); - for (Result result : results) { - TimelineEntity entity = parseEntity(result); - if (entity == null) { - continue; - } - entities.add(entity); - if (entities.size() == limit) { - break; + ResultScanner results = getResults(hbaseConf, conn); + try { + for (Result result : results) { + TimelineEntity entity = parseEntity(result); + if (entity == null) { + continue; + } + entities.add(entity); + if (entities.size() == limit) { + break; + } } + return entities; + } finally { + results.close(); } - return entities; } @Override @@ -123,7 +128,7 @@ class FlowActivityEntityReader extends TimelineEntityReader { } @Override - protected Iterable<Result> getResults(Configuration hbaseConf, + protected ResultScanner getResults(Configuration hbaseConf, Connection conn) throws IOException { Scan scan = new Scan(); scan.setRowPrefixFilter(FlowActivityRowKey.getRowKeyPrefix(clusterId)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a885e3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java index ced795d..b5d7ae5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field; @@ -96,7 +97,7 @@ class FlowRunEntityReader extends TimelineEntityReader { } @Override - protected Iterable<Result> getResults(Configuration hbaseConf, + protected ResultScanner getResults(Configuration hbaseConf, Connection conn) throws IOException { throw new UnsupportedOperationException( "multiple entity query is not supported"); @@ -110,14 +111,14 @@ class FlowRunEntityReader extends TimelineEntityReader { flowRun.setRunId(flowRunId); // read the start time - Long startTime = (Long)FlowRunColumn.MIN_START_TIME.readResult(result); + Number startTime = (Number)FlowRunColumn.MIN_START_TIME.readResult(result); if (startTime != null) { - flowRun.setStartTime(startTime); + flowRun.setStartTime(startTime.longValue()); } // read the end time if available - Long endTime = (Long)FlowRunColumn.MAX_END_TIME.readResult(result); + Number endTime = (Number)FlowRunColumn.MAX_END_TIME.readResult(result); if (endTime != null) { - flowRun.setMaxEndTime(endTime); + flowRun.setMaxEndTime(endTime.longValue()); } // read the flow version http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a885e3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java index 466914b..396a02b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java @@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; @@ -176,7 +177,7 @@ class GenericEntityReader extends TimelineEntityReader { } @Override - protected Iterable<Result> getResults(Configuration hbaseConf, + protected ResultScanner getResults(Configuration hbaseConf, Connection conn) throws IOException { // Scan through part of the table to find the entities belong to one app // and one type http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a885e3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java index 0d1134c..93be2db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java @@ -25,9 +25,12 @@ import java.util.NavigableSet; import java.util.Set; import java.util.TreeSet; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field; @@ -40,6 +43,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix * entities that are being requested. */ abstract class TimelineEntityReader { + private static final Log LOG = LogFactory.getLog(TimelineEntityReader.class); protected final boolean singleEntityRead; protected String userId; @@ -131,6 +135,11 @@ abstract class TimelineEntityReader { augmentParams(hbaseConf, conn); Result result = getResult(hbaseConf, conn); + if (result == null || result.isEmpty()) { + // Could not find a matching row. + LOG.info("Cannot find matching entity of type " + entityType); + return null; + } return parseEntity(result); } @@ -145,18 +154,22 @@ abstract class TimelineEntityReader { augmentParams(hbaseConf, conn); NavigableSet<TimelineEntity> entities = new TreeSet<>(); - Iterable<Result> results = getResults(hbaseConf, conn); - for (Result result : results) { - TimelineEntity entity = parseEntity(result); - if (entity == null) { - continue; - } - entities.add(entity); - if (entities.size() > limit) { - entities.pollLast(); + ResultScanner results = getResults(hbaseConf, conn); + try { + for (Result result : results) { + TimelineEntity entity = parseEntity(result); + if (entity == null) { + continue; + } + entities.add(entity); + if (entities.size() > limit) { + entities.pollLast(); + } } + return entities; + } finally { + results.close(); } - return entities; } /** @@ -184,9 +197,9 @@ abstract class TimelineEntityReader { throws IOException; /** - * Fetches an iterator for {@link Result} instances for a multi-entity read. + * Fetches a {@link ResultScanner} for a multi-entity read. */ - protected abstract Iterable<Result> getResults(Configuration hbaseConf, + protected abstract ResultScanner getResults(Configuration hbaseConf, Connection conn) throws IOException; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a885e3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java index ae71e2c..e359f78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesFlowRun.java @@ -59,6 +59,7 @@ import org.junit.Test; import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.GenericType; +import com.sun.jersey.api.client.ClientResponse.Status; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; @@ -281,6 +282,16 @@ public class TestTimelineReaderWebServicesFlowRun { return false; } + private static void verifyHttpResponse(Client client, URI uri, + Status status) { + ClientResponse resp = + client.resource(uri).accept(MediaType.APPLICATION_JSON) + .type(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertNotNull(resp); + assertTrue("Response from server should have been " + status, + resp.getClientResponseStatus().equals(status)); + } + @Test public void testGetFlowRun() throws Exception { Client client = createClient(); @@ -354,6 +365,35 @@ public class TestTimelineReaderWebServicesFlowRun { } } + @Test + public void testGetFlowRunNotPresent() throws Exception { + Client client = createClient(); + try { + URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + "timeline/flowrun/cluster1/flow_name/1002345678929?userid=user1"); + verifyHttpResponse(client, uri, Status.NOT_FOUND); + } finally { + client.destroy(); + } + } + + @Test + public void testGetFlowsNotPresent() throws Exception { + Client client = createClient(); + try { + URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + "timeline/flows/cluster2"); + ClientResponse resp = getResponse(client, uri); + Set<FlowActivityEntity> entities = + resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){}); + assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType()); + assertNotNull(entities); + assertEquals(0, entities.size()); + } finally { + client.destroy(); + } + } + @After public void stop() throws Exception { if (server != null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a885e3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java index 01920b3..3b0921b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java @@ -249,11 +249,7 @@ public class TestHBaseTimelineStorage { TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appId, entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL)); - Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid, - appId, entity.getType(), null, null, null, null, null, null, null, - null, null, null, null, EnumSet.of(TimelineReader.Field.ALL)); assertNotNull(e1); - assertEquals(1, es1.size()); // verify attributes assertEquals(appId, e1.getId()); @@ -610,18 +606,9 @@ public class TestHBaseTimelineStorage { TimelineEntity e2 = hbr.getEntity(user, cluster, null, null, appName, entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL)); - Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid, - appName, entity.getType(), null, null, null, null, null, null, null, - null, null, null, null, EnumSet.of(TimelineReader.Field.ALL)); - Set<TimelineEntity> es2 = hbr.getEntities(user, cluster, null, null, - appName, entity.getType(), null, null, null, null, null, null, null, - null, null, null, null, EnumSet.of(TimelineReader.Field.ALL)); assertNotNull(e1); assertNotNull(e2); assertEquals(e1, e2); - assertEquals(1, es1.size()); - assertEquals(1, es2.size()); - assertEquals(es1, es2); // check the events NavigableSet<TimelineEvent> events = e1.getEvents();