YARN-4446. Refactor reader API for better extensibility (Varun Saxena via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10a4f8ae Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10a4f8ae Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10a4f8ae Branch: refs/heads/YARN-2928 Commit: 10a4f8ae63db4e256404f81a79ecd17f7eafc054 Parents: e2e5a9a Author: Sangjin Lee <sj...@apache.org> Authored: Wed Feb 3 16:03:55 2016 -0800 Committer: Sangjin Lee <sj...@apache.org> Committed: Wed Feb 3 16:03:55 2016 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../reader/TimelineDataToRetrieve.java | 119 ++++++ .../reader/TimelineEntityFilters.java | 187 +++++++++ .../reader/TimelineReaderContext.java | 10 + .../reader/TimelineReaderManager.java | 44 +- .../reader/TimelineReaderUtils.java | 4 +- .../reader/TimelineReaderWebServices.java | 417 ++++++++----------- .../reader/TimelineReaderWebServicesUtils.java | 68 +++ .../storage/FileSystemTimelineReaderImpl.java | 116 +++--- .../storage/HBaseTimelineReaderImpl.java | 36 +- .../timelineservice/storage/TimelineReader.java | 234 +++++------ .../storage/reader/ApplicationEntityReader.java | 204 ++++----- .../reader/FlowActivityEntityReader.java | 59 +-- .../storage/reader/FlowRunEntityReader.java | 101 ++--- .../storage/reader/GenericEntityReader.java | 192 ++++----- .../storage/reader/TimelineEntityReader.java | 101 ++--- .../reader/TimelineEntityReaderFactory.java | 74 ++-- .../TestFileSystemTimelineReaderImpl.java | 156 ++++--- .../storage/TestHBaseTimelineStorage.java | 252 ++++++----- .../flow/TestHBaseStorageFlowActivity.java | 33 +- .../storage/flow/TestHBaseStorageFlowRun.java | 48 ++- 21 files changed, 1370 insertions(+), 1088 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a4f8ae/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 03dc60f..ae5d907 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -164,6 +164,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1 YARN-3586. RM to only get back addresses of Collectors that NM needs to know. (Junping Du via varunsaxena) + YARN-4446. Refactor reader API for better extensibility (Varun Saxena via + sjlee) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a4f8ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java new file mode 100644 index 0000000..0cc83d7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader; + +import java.util.EnumSet; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter; +import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field; + +/** + * Encapsulates information regarding which data to retrieve for each entity + * while querying.<br> + * Data to retrieve contains the following :<br> + * <ul> + * <li><b>confsToRetrieve</b> - Used for deciding which configs to return + * in response. This is represented as a {@link TimelineFilterList} object + * containing {@link TimelinePrefixFilter} objects. These can either be + * exact config keys' or prefixes which are then compared against config + * keys' to decide configs(inside entities) to return in response. If null + * or empty, all configurations will be fetched if fieldsToRetrieve + * contains {@link Field#CONFIGS} or {@link Field#ALL}. This should not be + * confused with configFilters which is used to decide which entities to + * return instead.</li> + * <li><b>metricsToRetrieve</b> - Used for deciding which metrics to return + * in response. This is represented as a {@link TimelineFilterList} object + * containing {@link TimelinePrefixFilter} objects. These can either be + * exact metric ids' or prefixes which are then compared against metric + * ids' to decide metrics(inside entities) to return in response. If null + * or empty, all metrics will be fetched if fieldsToRetrieve contains + * {@link Field#METRICS} or {@link Field#ALL}. This should not be confused + * with metricFilters which is used to decide which entities to return + * instead.</li> + * <li><b>fieldsToRetrieve</b> - Specifies which fields of the entity + * object to retrieve, see {@link Field}. If null, retrieves 3 fields, + * namely entity id, entity type and entity created time. All fields will + * be returned if {@link Field#ALL} is specified.</li> + * </ul> + */ +@Private +@Unstable +public class TimelineDataToRetrieve { + private TimelineFilterList confsToRetrieve; + private TimelineFilterList metricsToRetrieve; + private EnumSet<Field> fieldsToRetrieve; + + public TimelineDataToRetrieve() { + this(null, null, null); + } + + public TimelineDataToRetrieve(TimelineFilterList confs, + TimelineFilterList metrics, EnumSet<Field> fields) { + this.confsToRetrieve = confs; + this.metricsToRetrieve = metrics; + this.fieldsToRetrieve = fields; + + if (this.fieldsToRetrieve == null) { + this.fieldsToRetrieve = EnumSet.noneOf(Field.class); + } + } + + public TimelineFilterList getConfsToRetrieve() { + return confsToRetrieve; + } + + public void setConfsToRetrieve(TimelineFilterList confs) { + this.confsToRetrieve = confs; + } + + public TimelineFilterList getMetricsToRetrieve() { + return metricsToRetrieve; + } + + public void setMetricsToRetrieve(TimelineFilterList metrics) { + this.metricsToRetrieve = metrics; + } + + public EnumSet<Field> getFieldsToRetrieve() { + return fieldsToRetrieve; + } + + public void setFieldsToRetrieve(EnumSet<Field> fields) { + this.fieldsToRetrieve = fields; + } + + /** + * Adds configs and metrics fields to fieldsToRetrieve(if they are not + * present) if confsToRetrieve and metricsToRetrieve are specified. + */ + public void addFieldsBasedOnConfsAndMetricsToRetrieve() { + if (!fieldsToRetrieve.contains(Field.CONFIGS) && confsToRetrieve != null && + !confsToRetrieve.getFilterList().isEmpty()) { + fieldsToRetrieve.add(Field.CONFIGS); + } + if (!fieldsToRetrieve.contains(Field.METRICS) && + metricsToRetrieve != null && + !metricsToRetrieve.getFilterList().isEmpty()) { + fieldsToRetrieve.add(Field.METRICS); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a4f8ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java new file mode 100644 index 0000000..5b2c300 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java @@ -0,0 +1,187 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader; + +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * Encapsulates information regarding the filters to apply while querying. These + * filters restrict the number of entities to return.<br> + * Filters contain the following :<br> + * <ul> + * <li><b>limit</b> - A limit on the number of entities to return. If null + * or {@literal <=0}, defaults to {@link #DEFAULT_LIMIT}.</li> + * <li><b>createdTimeBegin</b> - Matched entities should not be created + * before this timestamp. If null or {@literal <=0}, defaults to 0.</li> + * <li><b>createdTimeEnd</b> - Matched entities should not be created after + * this timestamp. If null or {@literal <=0}, defaults to + * {@link Long#MAX_VALUE}.</li> + * <li><b>relatesTo</b> - Matched entities should relate to given entities. + * If null or empty, the relations are not matched.</li> + * <li><b>isRelatedTo</b> - Matched entities should be related to given + * entities. If null or empty, the relations are not matched.</li> + * <li><b>infoFilters</b> - Matched entities should have exact matches to + * the given info represented as key-value pairs. If null or empty, the + * filter is not applied.</li> + * <li><b>configFilters</b> - Matched entities should have exact matches to + * the given configs represented as key-value pairs. If null or empty, the + * filter is not applied.</li> + * <li><b>metricFilters</b> - Matched entities should contain the given + * metrics. If null or empty, the filter is not applied.</li> + * <li><b>eventFilters</b> - Matched entities should contain the given + * events. If null or empty, the filter is not applied.</li> + * </ul> + */ +@Private +@Unstable +public class TimelineEntityFilters { + private Long limit; + private Long createdTimeBegin; + private Long createdTimeEnd; + private Map<String, Set<String>> relatesTo; + private Map<String, Set<String>> isRelatedTo; + private Map<String, Object> infoFilters; + private Map<String, String> configFilters; + private Set<String> metricFilters; + private Set<String> eventFilters; + private static final Long DEFAULT_BEGIN_TIME = 0L; + private static final Long DEFAULT_END_TIME = Long.MAX_VALUE; + + /** + * Default limit of number of entities to return for getEntities API. + */ + public static final long DEFAULT_LIMIT = 100; + + public TimelineEntityFilters() { + this(null, null, null, null, null, null, null, null, null); + } + + public TimelineEntityFilters( + Long entityLimit, Long timeBegin, Long timeEnd, + Map<String, Set<String>> entityRelatesTo, + Map<String, Set<String>> entityIsRelatedTo, + Map<String, Object> entityInfoFilters, + Map<String, String> entityConfigFilters, + Set<String> entityMetricFilters, + Set<String> entityEventFilters) { + this.limit = entityLimit; + if (this.limit == null || this.limit < 0) { + this.limit = DEFAULT_LIMIT; + } + this.createdTimeBegin = timeBegin; + if (this.createdTimeBegin == null || this.createdTimeBegin < 0) { + this.createdTimeBegin = DEFAULT_BEGIN_TIME; + } + this.createdTimeEnd = timeEnd; + if (this.createdTimeEnd == null || this.createdTimeEnd < 0) { + this.createdTimeEnd = DEFAULT_END_TIME; + } + this.relatesTo = entityRelatesTo; + this.isRelatedTo = entityIsRelatedTo; + this.infoFilters = entityInfoFilters; + this.configFilters = entityConfigFilters; + this.metricFilters = entityMetricFilters; + this.eventFilters = entityEventFilters; + } + + public Long getLimit() { + return limit; + } + + public void setLimit(Long entityLimit) { + this.limit = entityLimit; + if (this.limit == null || this.limit < 0) { + this.limit = DEFAULT_LIMIT; + } + } + + public Long getCreatedTimeBegin() { + return createdTimeBegin; + } + + public void setCreatedTimeBegin(Long timeBegin) { + this.createdTimeBegin = timeBegin; + if (this.createdTimeBegin == null || this.createdTimeBegin < 0) { + this.createdTimeBegin = DEFAULT_BEGIN_TIME; + } + } + + public Long getCreatedTimeEnd() { + return createdTimeEnd; + } + + public void setCreatedTimeEnd(Long timeEnd) { + this.createdTimeEnd = timeEnd; + if (this.createdTimeEnd == null || this.createdTimeEnd < 0) { + this.createdTimeEnd = DEFAULT_END_TIME; + } + } + + public Map<String, Set<String>> getRelatesTo() { + return relatesTo; + } + + public void setRelatesTo(Map<String, Set<String>> relations) { + this.relatesTo = relations; + } + + public Map<String, Set<String>> getIsRelatedTo() { + return isRelatedTo; + } + + public void setIsRelatedTo(Map<String, Set<String>> relations) { + this.isRelatedTo = relations; + } + + public Map<String, Object> getInfoFilters() { + return infoFilters; + } + + public void setInfoFilters(Map<String, Object> filters) { + this.infoFilters = filters; + } + + public Map<String, String> getConfigFilters() { + return configFilters; + } + + public void setConfigFilters(Map<String, String> filters) { + this.configFilters = filters; + } + + public Set<String> getMetricFilters() { + return metricFilters; + } + + public void setMetricFilters(Set<String> filters) { + this.metricFilters = filters; + } + + public Set<String> getEventFilters() { + return eventFilters; + } + + public void setEventFilters(Set<String> filters) { + this.eventFilters = filters; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a4f8ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java index d0ad54a..633bb23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java @@ -18,11 +18,15 @@ package org.apache.hadoop.yarn.server.timelineservice.reader; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.server.timelineservice.TimelineContext; /** * Encapsulates fields necessary to make a query in timeline reader. */ +@Private +@Unstable public class TimelineReaderContext extends TimelineContext { private String entityType; @@ -34,6 +38,12 @@ public class TimelineReaderContext extends TimelineContext { this.entityId = entityId; } + public TimelineReaderContext(TimelineReaderContext other) { + this(other.getClusterId(), other.getUserId(), other.getFlowName(), + other.getFlowRunId(), other.getAppId(), other.getEntityType(), + other.getEntityId()); + } + @Override public int hashCode() { final int prime = 31; http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a4f8ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java index a7d864e..84a97ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java @@ -19,8 +19,6 @@ package org.apache.hadoop.yarn.server.timelineservice.reader; import java.io.IOException; -import java.util.EnumSet; -import java.util.Map; import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -33,7 +31,6 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader; -import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field; import com.google.common.annotations.VisibleForTesting; @@ -122,23 +119,14 @@ public class TimelineReaderManager extends AbstractService { * * @see TimelineReader#getEntities */ - public Set<TimelineEntity> getEntities(String userId, String clusterId, - String flowName, Long flowRunId, String appId, String entityType, - Long limit, Long createdTimeBegin, Long createdTimeEnd, - Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, - Map<String, Object> infoFilters, Map<String, String> configFilters, - Set<String> metricFilters, Set<String> eventFilters, - EnumSet<Field> fieldsToRetrieve) throws IOException { - String cluster = getClusterID(clusterId, getConfig()); - Set<TimelineEntity> entities = - reader.getEntities(userId, cluster, flowName, flowRunId, appId, - entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo, - isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters, - null, null, fieldsToRetrieve); + public Set<TimelineEntity> getEntities(TimelineReaderContext context, + TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve) + throws IOException { + context.setClusterId(getClusterID(context.getClusterId(), getConfig())); + Set<TimelineEntity> entities = reader.getEntities( + new TimelineReaderContext(context), filters, dataToRetrieve); if (entities != null) { - TimelineEntityType type = getTimelineEntityType(entityType); - TimelineReaderContext context = new TimelineReaderContext(cluster, userId, - flowName, flowRunId, appId, entityType, null); + TimelineEntityType type = getTimelineEntityType(context.getEntityType()); for (TimelineEntity entity : entities) { fillUID(type, entity, context); } @@ -152,18 +140,14 @@ public class TimelineReaderManager extends AbstractService { * * @see TimelineReader#getEntity */ - public TimelineEntity getEntity(String userId, String clusterId, - String flowName, Long flowRunId, String appId, String entityType, - String entityId, EnumSet<Field> fields) throws IOException { - String cluster = getClusterID(clusterId, getConfig()); - TimelineEntity entity = - reader.getEntity(userId, cluster, flowName, flowRunId, appId, - entityType, entityId, null, null, fields); - + public TimelineEntity getEntity(TimelineReaderContext context, + TimelineDataToRetrieve dataToRetrieve) throws IOException { + context.setClusterId( + getClusterID(context.getClusterId(), getConfig())); + TimelineEntity entity = reader.getEntity( + new TimelineReaderContext(context), dataToRetrieve); if (entity != null) { - TimelineEntityType type = getTimelineEntityType(entityType); - TimelineReaderContext context = new TimelineReaderContext(cluster, userId, - flowName, flowRunId, appId, entityType, null); + TimelineEntityType type = getTimelineEntityType(context.getEntityType()); fillUID(type, entity, context); } return entity; http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a4f8ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java index 66abbfc..c93c631 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java @@ -90,7 +90,7 @@ final class TimelineReaderUtils { // String needs to be split here. Copy remaining chars and add the // string to list. builder.append(str.substring(startOffset, offset)); - list.add(builder.toString()); + list.add(builder.toString().trim()); // Reset the start offset as a delimiter has been encountered. startOffset = ++offset; builder = new StringBuilder(len - offset); @@ -103,7 +103,7 @@ final class TimelineReaderUtils { builder.append(str.substring(startOffset)); } // Add the last part of delimited string to list. - list.add(builder.toString()); + list.add(builder.toString().trim()); return list; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a4f8ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java index 7a70de8..d68199a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java @@ -46,8 +46,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; -import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity; -import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; @@ -70,8 +68,6 @@ public class TimelineReaderWebServices { @Context private ServletContext ctxt; - private static final String COMMA_DELIMITER = ","; - private static final String COLON_DELIMITER = ":"; private static final String QUERY_STRING_SEP = "?"; private static final String RANGE_DELIMITER = "-"; private static final String DATE_PATTERN = "yyyyMMdd"; @@ -243,15 +239,15 @@ public class TimelineReaderWebServices { * events. This is represented as eventfilters=eventid1, eventid2... * @param fields Specifies which fields of the entity object to retrieve, see * {@link Field}. All fields will be retrieved if fields=ALL. If not - * specified, 4 fields i.e. entity type, id and created time is returned + * specified, 3 fields i.e. entity type, id and created time is returned * (Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing - * a set of {@link TimelineEntity} instances of the given entity type is - * returned. - * On failures, + * a set of <cite>TimelineEntity</cite> instances of the given entity type + * is returned.<br> + * On failures,<br> * If any problem occurs in parsing request or UID is incorrect, - * HTTP 400(Bad Request) is returned. + * HTTP 400(Bad Request) is returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -290,30 +286,14 @@ public class TimelineReaderWebServices { if (context == null) { throw new BadRequestException("Incorrect UID " + uId); } - entities = timelineReaderManager.getEntities( - TimelineReaderWebServicesUtils.parseStr(context.getUserId()), - TimelineReaderWebServicesUtils.parseStr(context.getClusterId()), - TimelineReaderWebServicesUtils.parseStr(context.getFlowName()), - context.getFlowRunId(), - TimelineReaderWebServicesUtils.parseStr(context.getAppId()), - TimelineReaderWebServicesUtils.parseStr(entityType), - TimelineReaderWebServicesUtils.parseLongStr(limit), - TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart), - TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd), - TimelineReaderWebServicesUtils.parseKeyStrValuesStr( - relatesTo, COMMA_DELIMITER, COLON_DELIMITER), - TimelineReaderWebServicesUtils.parseKeyStrValuesStr( - isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER), - TimelineReaderWebServicesUtils.parseKeyStrValueObj( - infofilters, COMMA_DELIMITER, COLON_DELIMITER), - TimelineReaderWebServicesUtils.parseKeyStrValueStr( - conffilters, COMMA_DELIMITER, COLON_DELIMITER), - TimelineReaderWebServicesUtils.parseValuesStr( - metricfilters, COMMA_DELIMITER), - TimelineReaderWebServicesUtils.parseValuesStr( - eventfilters, COMMA_DELIMITER), - TimelineReaderWebServicesUtils.parseFieldsStr( - fields, COMMA_DELIMITER)); + context.setEntityType( + TimelineReaderWebServicesUtils.parseStr(entityType)); + entities = timelineReaderManager.getEntities(context, + TimelineReaderWebServicesUtils.createTimelineEntityFilters( + limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, + infofilters, conffilters, metricfilters, eventfilters), + TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( + null, null, fields)); } catch (Exception e) { handleException(e, url, startTime, "createdTime start/end or limit or flowrunid"); @@ -374,17 +354,17 @@ public class TimelineReaderWebServices { * events. This is represented as eventfilters=eventid1, eventid2... * @param fields Specifies which fields of the entity object to retrieve, see * {@link Field}. All fields will be retrieved if fields=ALL. If not - * specified, 4 fields i.e. entity type, id, created time is returned + * specified, 3 fields i.e. entity type, id, created time is returned * (Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing - * a set of {@link TimelineEntity} instances of the given entity type is - * returned. - * On failures, + * a set of <cite>TimelineEntity</cite> instances of the given entity type + * is returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * If flow context information cannot be retrieved, HTTP 404(Not Found) - * is returned. + * is returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -463,17 +443,17 @@ public class TimelineReaderWebServices { * events. This is represented as eventfilters=eventid1, eventid2... * @param fields Specifies which fields of the entity object to retrieve, see * {@link Field}. All fields will be retrieved if fields=ALL. If not - * specified, 4 fields i.e. entity type, id, created time is returned + * specified, 3 fields i.e. entity type, id, created time is returned * (Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing - * a set of {@link TimelineEntity} instances of the given entity type is - * returned. - * On failures, + * a set of <cite>TimelineEntity</cite> instances of the given entity type + * is returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * If flow context information cannot be retrieved, HTTP 404(Not Found) - * is returned. + * is returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -512,29 +492,14 @@ public class TimelineReaderWebServices { Set<TimelineEntity> entities = null; try { entities = timelineReaderManager.getEntities( - TimelineReaderWebServicesUtils.parseStr(userId), - TimelineReaderWebServicesUtils.parseStr(clusterId), - TimelineReaderWebServicesUtils.parseStr(flowName), - TimelineReaderWebServicesUtils.parseLongStr(flowRunId), - TimelineReaderWebServicesUtils.parseStr(appId), - TimelineReaderWebServicesUtils.parseStr(entityType), - TimelineReaderWebServicesUtils.parseLongStr(limit), - TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart), - TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd), - TimelineReaderWebServicesUtils.parseKeyStrValuesStr( - relatesTo, COMMA_DELIMITER, COLON_DELIMITER), - TimelineReaderWebServicesUtils.parseKeyStrValuesStr( - isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER), - TimelineReaderWebServicesUtils.parseKeyStrValueObj( - infofilters, COMMA_DELIMITER, COLON_DELIMITER), - TimelineReaderWebServicesUtils.parseKeyStrValueStr( - conffilters, COMMA_DELIMITER, COLON_DELIMITER), - TimelineReaderWebServicesUtils.parseValuesStr( - metricfilters, COMMA_DELIMITER), - TimelineReaderWebServicesUtils.parseValuesStr( - eventfilters, COMMA_DELIMITER), - TimelineReaderWebServicesUtils.parseFieldsStr( - fields, COMMA_DELIMITER)); + TimelineReaderWebServicesUtils.createTimelineReaderContext( + clusterId, userId, flowName, flowRunId, appId, entityType, null), + + TimelineReaderWebServicesUtils.createTimelineEntityFilters( + limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, + infofilters, conffilters, metricfilters, eventfilters), + TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( + null, null, fields)); } catch (Exception e) { handleException(e, url, startTime, "createdTime start/end or limit or flowrunid"); @@ -559,16 +524,16 @@ public class TimelineReaderWebServices { * UID and then used to query backend(Mandatory path param). * @param fields Specifies which fields of the entity object to retrieve, see * {@link Field}. All fields will be retrieved if fields=ALL. If not - * specified, 4 fields i.e. entity type, id, created time is returned + * specified, 3 fields i.e. entity type, id, created time is returned * (Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a - * {@link TimelineEntity} instance is returned. - * On failures, + * <cite>TimelineEntity</cite> instance is returned.<br> + * On failures,<br> * If any problem occurs in parsing request or UID is incorrect, - * HTTP 400(Bad Request) is returned. + * HTTP 400(Bad Request) is returned.<br> * If entity for the given entity id cannot be found, HTTP 404(Not Found) - * is returned. + * is returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -597,11 +562,9 @@ public class TimelineReaderWebServices { if (context == null) { throw new BadRequestException("Incorrect UID " + uId); } - entity = timelineReaderManager.getEntity(context.getUserId(), - context.getClusterId(), context.getFlowName(), context.getFlowRunId(), - context.getAppId(), context.getEntityType(), context.getEntityId(), - TimelineReaderWebServicesUtils.parseFieldsStr( - fields, COMMA_DELIMITER)); + entity = timelineReaderManager.getEntity(context, + TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( + null, null, fields)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -638,16 +601,16 @@ public class TimelineReaderWebServices { * param). * @param fields Specifies which fields of the entity object to retrieve, see * {@link Field}. All fields will be retrieved if fields=ALL. If not - * specified, 4 fields i.e. entity type, id, created time is returned + * specified, 3 fields i.e. entity type, id, created time is returned * (Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a - * {@link TimelineEntity} instance is returned. - * On failures, + * <cite>TimelineEntity</cite> instance is returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * If flow context information cannot be retrieved or entity for the given - * entity id cannot be found, HTTP 404(Not Found) is returned. + * entity id cannot be found, HTTP 404(Not Found) is returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -690,16 +653,16 @@ public class TimelineReaderWebServices { * param). * @param fields Specifies which fields of the entity object to retrieve, see * {@link Field}. All fields will be retrieved if fields=ALL. If not - * specified, 4 fields i.e. entity type, id and created time is returned + * specified, 3 fields i.e. entity type, id and created time is returned * (Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a - * {@link TimelineEntity} instance is returned. - * On failures, + * <cite>TimelineEntity</cite> instance is returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * If flow context information cannot be retrieved or entity for the given - * entity id cannot be found, HTTP 404(Not Found) is returned. + * entity id cannot be found, HTTP 404(Not Found) is returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -728,17 +691,12 @@ public class TimelineReaderWebServices { init(res); TimelineReaderManager timelineReaderManager = getTimelineReaderManager(); TimelineEntity entity = null; - String type = TimelineReaderWebServicesUtils.parseStr(entityType); - String id = TimelineReaderWebServicesUtils.parseStr(entityId); try { entity = timelineReaderManager.getEntity( - TimelineReaderWebServicesUtils.parseStr(userId), - TimelineReaderWebServicesUtils.parseStr(clusterId), - TimelineReaderWebServicesUtils.parseStr(flowName), - TimelineReaderWebServicesUtils.parseLongStr(flowRunId), - TimelineReaderWebServicesUtils.parseStr(appId), type, id, - TimelineReaderWebServicesUtils.parseFieldsStr( - fields, COMMA_DELIMITER)); + TimelineReaderWebServicesUtils.createTimelineReaderContext( + clusterId, userId, flowName, flowRunId, appId, entityType, entityId), + TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( + null, null, fields)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -746,8 +704,8 @@ public class TimelineReaderWebServices { if (entity == null) { LOG.info("Processed URL " + url + " but entity not found" + " (Took " + (endTime - startTime) + " ms.)"); - throw new NotFoundException("Timeline entity {id: " + id + ", type: " + - type + " } is not found"); + throw new NotFoundException("Timeline entity {id: " + entityId + + ", type: " + entityType + " } is not found"); } LOG.info("Processed URL " + url + " (Took " + (endTime - startTime) + " ms.)"); @@ -765,13 +723,13 @@ public class TimelineReaderWebServices { * (Mandatory path param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a - * {@link FlowRunEntity} instance is returned. By default, all metrics for - * the flow run will be returned. - * On failures, + * <cite>FlowRunEntity</cite> instance is returned. By default, all + * metrics for the flow run will be returned.<br> + * On failures,<br> * If any problem occurs in parsing request or UID is incorrect, - * HTTP 400(Bad Request) is returned. + * HTTP 400(Bad Request) is returned.<br> * If flow run for the given flow run id cannot be found, HTTP 404 - * (Not Found) is returned. + * (Not Found) is returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -799,9 +757,9 @@ public class TimelineReaderWebServices { if (context == null) { throw new BadRequestException("Incorrect UID " + uId); } - entity = timelineReaderManager.getEntity(context.getUserId(), - context.getClusterId(), context.getFlowName(), context.getFlowRunId(), - null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null); + context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString()); + entity = timelineReaderManager.getEntity(context, + new TimelineDataToRetrieve()); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -829,13 +787,13 @@ public class TimelineReaderWebServices { * @param flowRunId Id of the flow run to be queried(Mandatory path param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a - * {@link FlowRunEntity} instance is returned. By default, all metrics for - * the flow run will be returned. - * On failures, + * <cite>FlowRunEntity</cite> instance is returned. By default, all + * metrics for the flow run will be returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * If flow run for the given flow run id cannot be found, HTTP 404 - * (Not Found) is returned. + * (Not Found) is returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -865,13 +823,13 @@ public class TimelineReaderWebServices { * @param flowRunId Id of the flow run to be queried(Mandatory path param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a - * {@link FlowRunEntity} instance is returned. By default, all metrics for - * the flow run will be returned. - * On failures, + * <cite>FlowRunEntity</cite> instance is returned. By default, all + * metrics for the flow run will be returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * If flow run for the given flow run id cannot be found, HTTP 404 - * (Not Found) is returned. + * (Not Found) is returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -899,11 +857,10 @@ public class TimelineReaderWebServices { TimelineEntity entity = null; try { entity = timelineReaderManager.getEntity( - TimelineReaderWebServicesUtils.parseStr(userId), - TimelineReaderWebServicesUtils.parseStr(clusterId), - TimelineReaderWebServicesUtils.parseStr(flowName), - TimelineReaderWebServicesUtils.parseLongStr(flowRunId), - null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null); + TimelineReaderWebServicesUtils.createTimelineReaderContext( + clusterId, userId, flowName, flowRunId, null, + TimelineEntityType.YARN_FLOW_RUN.toString(), null), + new TimelineDataToRetrieve()); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -930,8 +887,6 @@ public class TimelineReaderWebServices { * @param uId a delimited string containing clusterid, userid, and flow name * which are extracted from UID and then used to query backend(Mandatory * path param). - * @param flowName Flow name to which the flow runs to be queried belongs to( - * Mandatory path param). * @param limit Number of flow runs to return(Optional query param). * @param createdTimeStart If specified, matched flow runs should not be * created before this timestamp(Optional query param). @@ -943,10 +898,11 @@ public class TimelineReaderWebServices { * other than metrics are returned(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a - * set of {@link FlowRunEntity} instances for the given flow are returned. - * On failures, + * set of <cite>FlowRunEntity</cite> instances for the given flow are + * returned.<br> + * On failures,<br> * If any problem occurs in parsing request or UID is incorrect, - * HTTP 400(Bad Request) is returned. + * HTTP 400(Bad Request) is returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -978,14 +934,13 @@ public class TimelineReaderWebServices { if (context == null) { throw new BadRequestException("Incorrect UID " + uId); } - entities = timelineReaderManager.getEntities(context.getUserId(), - context.getClusterId(), context.getFlowName(), null, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), - TimelineReaderWebServicesUtils.parseLongStr(limit), - TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart), - TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd), - null, null, null, null, null, null, TimelineReaderWebServicesUtils. - parseFieldsStr(fields, COMMA_DELIMITER)); + context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString()); + entities = timelineReaderManager.getEntities(context, + TimelineReaderWebServicesUtils.createTimelineEntityFilters( + limit, createdTimeStart, createdTimeEnd, null, null, null, + null, null, null), + TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( + null, null, fields)); } catch (Exception e) { handleException(e, url, startTime, "createdTime start/end or limit"); } @@ -1019,10 +974,11 @@ public class TimelineReaderWebServices { * other than metrics are returned(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a - * set of {@link FlowRunEntity} instances for the given flow are returned. - * On failures, + * set of <cite>FlowRunEntity</cite> instances for the given flow are + * returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -1064,10 +1020,11 @@ public class TimelineReaderWebServices { * other than metrics are returned(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a - * set of {@link FlowRunEntity} instances for the given flow are returned. - * On failures, + * set of <cite>FlowRunEntity</cite> instances for the given flow are + * returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -1097,15 +1054,14 @@ public class TimelineReaderWebServices { Set<TimelineEntity> entities = null; try { entities = timelineReaderManager.getEntities( - TimelineReaderWebServicesUtils.parseStr(userId), - TimelineReaderWebServicesUtils.parseStr(clusterId), - TimelineReaderWebServicesUtils.parseStr(flowName), null, null, - TimelineEntityType.YARN_FLOW_RUN.toString(), - TimelineReaderWebServicesUtils.parseLongStr(limit), - TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart), - TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd), - null, null, null, null, null, null, TimelineReaderWebServicesUtils. - parseFieldsStr(fields, COMMA_DELIMITER)); + TimelineReaderWebServicesUtils.createTimelineReaderContext( + clusterId, userId, flowName, null, null, + TimelineEntityType.YARN_FLOW_RUN.toString(), null), + TimelineReaderWebServicesUtils.createTimelineEntityFilters( + limit, createdTimeStart, createdTimeEnd, null, null, null, + null, null, null), + TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( + null, null, fields)); } catch (Exception e) { handleException(e, url, startTime, "createdTime start/end or limit"); } @@ -1142,12 +1098,12 @@ public class TimelineReaderWebServices { * "daterange=-20150711" returns flows active on and before 20150711. * * @return If successful, a HTTP 200(OK) response having a JSON representing a - * set of {@link FlowActivityEntity} instances are returned. - * On failures, + * set of <cite>FlowActivityEntity</cite> instances are returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server - * Error) is returned. + * Error) is returned.<br> */ @GET @Path("/flows/") @@ -1185,10 +1141,10 @@ public class TimelineReaderWebServices { * "daterange=-20150711" returns flows active on and before 20150711. * * @return If successful, a HTTP 200(OK) response having a JSON representing a - * set of {@link FlowActivityEntity} instances are returned. - * On failures, + * set of <cite>FlowActivityEntity</cite> instances are returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -1214,11 +1170,17 @@ public class TimelineReaderWebServices { Set<TimelineEntity> entities = null; try { DateRange range = parseDateRange(dateRange); + TimelineEntityFilters entityFilters = + TimelineReaderWebServicesUtils.createTimelineEntityFilters( + limit, null, null, null, null, null, null, null, null); + entityFilters.setCreatedTimeBegin(range.dateStart); + entityFilters.setCreatedTimeEnd(range.dateEnd); entities = timelineReaderManager.getEntities( - null, TimelineReaderWebServicesUtils.parseStr(clusterId), null, null, - null, TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), - TimelineReaderWebServicesUtils.parseLongStr(limit), range.dateStart, - range.dateEnd, null, null, null, null, null, null, null); + TimelineReaderWebServicesUtils.createTimelineReaderContext( + clusterId, null, null, null, null, + TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null), + entityFilters, TimelineReaderWebServicesUtils. + createTimelineDataToRetrieve(null, null, null)); } catch (Exception e) { handleException(e, url, startTime, "limit"); } @@ -1242,16 +1204,16 @@ public class TimelineReaderWebServices { * backend(Mandatory path param). * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not - * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION), + * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), * app id and app created time is returned(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a - * {@link TimelineEntity} instance is returned. - * On failures, + * <cite>TimelineEntity</cite> instance is returned.<br> + * On failures,<br> * If any problem occurs in parsing request or UID is incorrect, - * HTTP 400(Bad Request) is returned. + * HTTP 400(Bad Request) is returned.<br> * If app for the given app id cannot be found, HTTP 404(Not Found) is - * returned. + * returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -1280,11 +1242,10 @@ public class TimelineReaderWebServices { if (context == null) { throw new BadRequestException("Incorrect UID " + uId); } - entity = timelineReaderManager.getEntity(context.getUserId(), - context.getClusterId(), context.getFlowName(), context.getFlowRunId(), - context.getAppId(), TimelineEntityType.YARN_APPLICATION.toString(), - null, TimelineReaderWebServicesUtils.parseFieldsStr( - fields, COMMA_DELIMITER)); + context.setEntityType(TimelineEntityType.YARN_APPLICATION.toString()); + entity = timelineReaderManager.getEntity(context, + TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( + null, null, fields)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -1316,16 +1277,16 @@ public class TimelineReaderWebServices { * @param userId User id which should match for the app(Optional query param). * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not - * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION), + * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), * app id and app created time is returned(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a - * {@link TimelineEntity} instance is returned. - * On failures, + * <cite>TimelineEntity</cite> instance is returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * If flow context information cannot be retrieved or app for the given - * app id cannot be found, HTTP 404(Not Found) is returned. + * app id cannot be found, HTTP 404(Not Found) is returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -1361,16 +1322,16 @@ public class TimelineReaderWebServices { * @param userId User id which should match for the app(Optional query param). * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not - * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION), + * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), * app id and app created time is returned(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing a - * {@link TimelineEntity} instance is returned. - * On failures, + * <cite>TimelineEntity</cite> instance is returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * If flow context information cannot be retrieved or app for the given - * app id cannot be found, HTTP 404(Not Found) is returned. + * app id cannot be found, HTTP 404(Not Found) is returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -1399,14 +1360,11 @@ public class TimelineReaderWebServices { TimelineEntity entity = null; try { entity = timelineReaderManager.getEntity( - TimelineReaderWebServicesUtils.parseStr(userId), - TimelineReaderWebServicesUtils.parseStr(clusterId), - TimelineReaderWebServicesUtils.parseStr(flowName), - TimelineReaderWebServicesUtils.parseLongStr(flowRunId), - TimelineReaderWebServicesUtils.parseStr(appId), - TimelineEntityType.YARN_APPLICATION.toString(), null, - TimelineReaderWebServicesUtils.parseFieldsStr( - fields, COMMA_DELIMITER)); + TimelineReaderWebServicesUtils.createTimelineReaderContext( + clusterId, userId, flowName, flowRunId, appId, + TimelineEntityType.YARN_APPLICATION.toString(), null), + TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( + null, null, fields)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -1459,15 +1417,15 @@ public class TimelineReaderWebServices { * events. This is represented as eventfilters=eventid1, eventid2... * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not - * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION), + * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), * app id and app created time is returned(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing - * a set of {@link TimelineEntity} instances representing apps is - * returned. - * On failures, + * a set of <cite>TimelineEntity</cite> instances representing apps is + * returned.<br> + * On failures,<br> * If any problem occurs in parsing request or UID is incorrect, - * HTTP 400(Bad Request) is returned. + * HTTP 400(Bad Request) is returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -1505,30 +1463,13 @@ public class TimelineReaderWebServices { if (context == null) { throw new BadRequestException("Incorrect UID " + uId); } - entities = timelineReaderManager.getEntities( - TimelineReaderWebServicesUtils.parseStr(context.getUserId()), - TimelineReaderWebServicesUtils.parseStr(context.getClusterId()), - TimelineReaderWebServicesUtils.parseStr(context.getFlowName()), - context.getFlowRunId(), - TimelineReaderWebServicesUtils.parseStr(context.getAppId()), - TimelineEntityType.YARN_APPLICATION.toString(), - TimelineReaderWebServicesUtils.parseLongStr(limit), - TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart), - TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd), - TimelineReaderWebServicesUtils.parseKeyStrValuesStr( - relatesTo, COMMA_DELIMITER, COLON_DELIMITER), - TimelineReaderWebServicesUtils.parseKeyStrValuesStr( - isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER), - TimelineReaderWebServicesUtils.parseKeyStrValueObj( - infofilters, COMMA_DELIMITER, COLON_DELIMITER), - TimelineReaderWebServicesUtils.parseKeyStrValueStr( - conffilters, COMMA_DELIMITER, COLON_DELIMITER), - TimelineReaderWebServicesUtils.parseValuesStr( - metricfilters, COMMA_DELIMITER), - TimelineReaderWebServicesUtils.parseValuesStr( - eventfilters, COMMA_DELIMITER), - TimelineReaderWebServicesUtils.parseFieldsStr( - fields, COMMA_DELIMITER)); + context.setEntityType(TimelineEntityType.YARN_APPLICATION.toString()); + entities = timelineReaderManager.getEntities(context, + TimelineReaderWebServicesUtils.createTimelineEntityFilters( + limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, + infofilters, conffilters, metricfilters, eventfilters), + TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( + null, null, fields)); } catch (Exception e) { handleException(e, url, startTime, "createdTime start/end or limit or flowrunid"); @@ -1582,15 +1523,15 @@ public class TimelineReaderWebServices { * events. This is represented as eventfilters=eventid1, eventid2... * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not - * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION), + * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), * app id and app created time is returned(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing - * a set of {@link TimelineEntity} instances representing apps is - * returned. - * On failures, + * a set of <cite>TimelineEntity</cite> instances representing apps is + * returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -1661,15 +1602,15 @@ public class TimelineReaderWebServices { * events. This is represented as eventfilters=eventid1, eventid2... * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not - * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION), + * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), * app id and app created time is returned(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing - * a set of {@link TimelineEntity} instances representing apps is - * returned. - * On failures, + * a set of <cite>TimelineEntity</cite> instances representing apps is + * returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -1739,15 +1680,15 @@ public class TimelineReaderWebServices { * events. This is represented as eventfilters=eventid1, eventid2... * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not - * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION), + * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), * app id and app created time is returned(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing - * a set of {@link TimelineEntity} instances representing apps is - * returned. - * On failures, + * a set of <cite>TimelineEntity</cite> instances representing apps is + * returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ @@ -1815,15 +1756,15 @@ public class TimelineReaderWebServices { * events. This is represented as eventfilters=eventid1, eventid2... * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not - * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION), + * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), * app id and app created time is returned(Optional query param). * * @return If successful, a HTTP 200(OK) response having a JSON representing - * a set of {@link TimelineEntity} instances representing apps is - * returned. - * On failures, + * a set of <cite>TimelineEntity</cite> instances representing apps is + * returned.<br> + * On failures,<br> * If any problem occurs in parsing request, HTTP 400(Bad Request) is - * returned. + * returned.<br> * For all other errors while retrieving data, HTTP 500(Internal Server * Error) is returned. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a4f8ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java index d052d51..fc07e51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java @@ -35,10 +35,78 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Fiel * Set of utility methods to be used by timeline reader web services. */ final class TimelineReaderWebServicesUtils { + private static final String COMMA_DELIMITER = ","; + private static final String COLON_DELIMITER = ":"; + private TimelineReaderWebServicesUtils() { } /** + * Parse the passed context information represented as strings and convert + * into a {@link TimelineReaderContext} object. + * @param clusterId + * @param userId + * @param flowName + * @param flowRunId + * @param appId + * @param entityType + * @param entityId + * @return a {@link TimelineReaderContext} object. + * @throws Exception + */ + static TimelineReaderContext createTimelineReaderContext(String clusterId, + String userId, String flowName, String flowRunId, String appId, + String entityType, String entityId) throws Exception { + return new TimelineReaderContext(parseStr(clusterId), parseStr(userId), + parseStr(flowName), parseLongStr(flowRunId), parseStr(appId), + parseStr(entityType), parseStr(entityId)); + } + + /** + * Parse the passed filters represented as strings and convert them into a + * {@link TimelineEntityFilters} object. + * @param limit + * @param createdTimeStart + * @param createdTimeEnd + * @param relatesTo + * @param isRelatedTo + * @param infofilters + * @param conffilters + * @param metricfilters + * @param eventfilters + * @return a {@link TimelineEntityFilters} object. + * @throws Exception + */ + static TimelineEntityFilters createTimelineEntityFilters(String limit, + String createdTimeStart, String createdTimeEnd, String relatesTo, + String isRelatedTo, String infofilters, String conffilters, + String metricfilters, String eventfilters) throws Exception { + return new TimelineEntityFilters(parseLongStr(limit), + parseLongStr(createdTimeStart), parseLongStr(createdTimeEnd), + parseKeyStrValuesStr(relatesTo, COMMA_DELIMITER, COLON_DELIMITER), + parseKeyStrValuesStr(isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER), + parseKeyStrValueObj(infofilters, COMMA_DELIMITER, COLON_DELIMITER), + parseKeyStrValueStr(conffilters, COMMA_DELIMITER, COLON_DELIMITER), + parseValuesStr(metricfilters, COMMA_DELIMITER), + parseValuesStr(eventfilters, COMMA_DELIMITER)); + } + + /** + * Parse the passed fields represented as strings and convert them into a + * {@link TimelineDataToRetrieve} object. + * @param confs + * @param metrics + * @param fields + * @return a {@link TimelineDataToRetrieve} object. + * @throws Exception + */ + static TimelineDataToRetrieve createTimelineDataToRetrieve(String confs, + String metrics, String fields) throws Exception { + return new TimelineDataToRetrieve( + null, null, parseFieldsStr(fields, COMMA_DELIMITER)); + } + + /** * Parse a delimited string and convert it into a set of strings. For * instance, if delimiter is ",", then the string should be represented as * "value1,value2,value3". http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a4f8ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java index 72c9249..aa0071f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java @@ -44,7 +44,9 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.codehaus.jackson.JsonGenerationException; @@ -264,22 +266,8 @@ public class FileSystemTimelineReaderImpl extends AbstractService } private Set<TimelineEntity> getEntities(File dir, String entityType, - Long limit, Long createdTimeBegin, Long createdTimeEnd, - Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, - Map<String, Object> infoFilters, Map<String, String> configFilters, - Set<String> metricFilters, Set<String> eventFilters, - TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve, - EnumSet<Field> fieldsToRetrieve) throws IOException { - if (limit == null || limit <= 0) { - limit = DEFAULT_LIMIT; - } - if (createdTimeBegin == null || createdTimeBegin <= 0) { - createdTimeBegin = 0L; - } - if (createdTimeEnd == null || createdTimeEnd <= 0) { - createdTimeEnd = Long.MAX_VALUE; - } - + TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve) + throws IOException { // First sort the selected entities based on created/start time. Map<Long, Set<TimelineEntity>> sortedEntities = new TreeMap<>( @@ -303,41 +291,48 @@ public class FileSystemTimelineReaderImpl extends AbstractService if (!entity.getType().equals(entityType)) { continue; } - if (!isTimeInRange(entity.getCreatedTime(), createdTimeBegin, - createdTimeEnd)) { + if (!isTimeInRange(entity.getCreatedTime(), + filters.getCreatedTimeBegin(), filters.getCreatedTimeEnd())) { continue; } - if (relatesTo != null && !relatesTo.isEmpty() && - !TimelineStorageUtils - .matchRelations(entity.getRelatesToEntities(), relatesTo)) { + if (filters.getRelatesTo() != null && + !filters.getRelatesTo().isEmpty() && + !TimelineStorageUtils.matchRelations( + entity.getRelatesToEntities(), filters.getRelatesTo())) { continue; } - if (isRelatedTo != null && !isRelatedTo.isEmpty() && - !TimelineStorageUtils - .matchRelations(entity.getIsRelatedToEntities(), isRelatedTo)) { + if (filters.getIsRelatedTo() != null && + !filters.getIsRelatedTo().isEmpty() && + !TimelineStorageUtils.matchRelations( + entity.getIsRelatedToEntities(), filters.getIsRelatedTo())) { continue; } - if (infoFilters != null && !infoFilters.isEmpty() && - !TimelineStorageUtils.matchFilters(entity.getInfo(), infoFilters)) { + if (filters.getInfoFilters() != null && + !filters.getInfoFilters().isEmpty() && + !TimelineStorageUtils.matchFilters( + entity.getInfo(), filters.getInfoFilters())) { continue; } - if (configFilters != null && !configFilters.isEmpty() && + if (filters.getConfigFilters() != null && + !filters.getConfigFilters().isEmpty() && !TimelineStorageUtils.matchFilters( - entity.getConfigs(), configFilters)) { + entity.getConfigs(), filters.getConfigFilters())) { continue; } - if (metricFilters != null && !metricFilters.isEmpty() && + if (filters.getMetricFilters() != null && + !filters.getMetricFilters().isEmpty() && !TimelineStorageUtils.matchMetricFilters( - entity.getMetrics(), metricFilters)) { + entity.getMetrics(), filters.getMetricFilters())) { continue; } - if (eventFilters != null && !eventFilters.isEmpty() && + if (filters.getEventFilters() != null && + !filters.getEventFilters().isEmpty() && !TimelineStorageUtils.matchEventFilters( - entity.getEvents(), eventFilters)) { + entity.getEvents(), filters.getEventFilters())) { continue; } - TimelineEntity entityToBeReturned = - createEntityToBeReturned(entity, fieldsToRetrieve); + TimelineEntity entityToBeReturned = createEntityToBeReturned( + entity, dataToRetrieve.getFieldsToRetrieve()); Set<TimelineEntity> entitiesCreatedAtSameTime = sortedEntities.get(entityToBeReturned.getCreatedTime()); if (entitiesCreatedAtSameTime == null) { @@ -355,7 +350,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService for (TimelineEntity entity : entitySet) { entities.add(entity); ++entitiesAdded; - if (entitiesAdded >= limit) { + if (entitiesAdded >= filters.getLimit()) { return entities; } } @@ -371,45 +366,40 @@ public class FileSystemTimelineReaderImpl extends AbstractService } @Override - public TimelineEntity getEntity(String userId, String clusterId, - String flowName, Long flowRunId, String appId, String entityType, - String entityId, TimelineFilterList confsToRetrieve, - TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) - throws IOException { - String flowRunPath = getFlowRunPath(userId, clusterId, flowName, - flowRunId, appId); + public TimelineEntity getEntity(TimelineReaderContext context, + TimelineDataToRetrieve dataToRetrieve) throws IOException { + String flowRunPath = getFlowRunPath(context.getUserId(), + context.getClusterId(), context.getFlowName(), context.getFlowRunId(), + context.getAppId()); File dir = new File(new File(rootPath, ENTITIES_DIR), - clusterId + "/" + flowRunPath + "/" + appId + "/" + entityType); - File entityFile = - new File(dir, entityId + TIMELINE_SERVICE_STORAGE_EXTENSION); + context.getClusterId() + "/" + flowRunPath + "/" + context.getAppId() + + "/" + context.getEntityType()); + File entityFile = new File( + dir, context.getEntityId() + TIMELINE_SERVICE_STORAGE_EXTENSION); try (BufferedReader reader = new BufferedReader(new InputStreamReader( new FileInputStream(entityFile), Charset.forName("UTF-8")))) { TimelineEntity entity = readEntityFromFile(reader); - return createEntityToBeReturned(entity, fieldsToRetrieve); + return createEntityToBeReturned( + entity, dataToRetrieve.getFieldsToRetrieve()); } catch (FileNotFoundException e) { - LOG.info("Cannot find entity {id:" + entityId + " , type:" + entityType + - "}. Will send HTTP 404 in response."); + LOG.info("Cannot find entity {id:" + context.getEntityId() + " , type:" + + context.getEntityType() + "}. Will send HTTP 404 in response."); return null; } } @Override - public Set<TimelineEntity> getEntities(String userId, String clusterId, - String flowName, Long flowRunId, String appId, String entityType, - Long limit, Long createdTimeBegin, Long createdTimeEnd, - Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, - Map<String, Object> infoFilters, Map<String, String> configFilters, - Set<String> metricFilters, Set<String> eventFilters, - TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve, - EnumSet<Field> fieldsToRetrieve) throws IOException { - String flowRunPath = - getFlowRunPath(userId, clusterId, flowName, flowRunId, appId); + public Set<TimelineEntity> getEntities(TimelineReaderContext context, + TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve) + throws IOException { + String flowRunPath = getFlowRunPath(context.getUserId(), + context.getClusterId(), context.getFlowName(), context.getFlowRunId(), + context.getAppId()); File dir = new File(new File(rootPath, ENTITIES_DIR), - clusterId + "/" + flowRunPath + "/" + appId + "/" + entityType); - return getEntities(dir, entityType, limit, createdTimeBegin, createdTimeEnd, - relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters, - eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve); + context.getClusterId() + "/" + flowRunPath + "/" + + context.getAppId() + "/" + context.getEntityType()); + return getEntities(dir, context.getEntityType(), filters, dataToRetrieve); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a4f8ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java index 0ce9a94..9f8257f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java @@ -19,8 +19,6 @@ package org.apache.hadoop.yarn.server.timelineservice.storage; import java.io.IOException; -import java.util.EnumSet; -import java.util.Map; import java.util.Set; import org.apache.commons.logging.Log; @@ -31,7 +29,9 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; -import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters; +import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext; import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReader; import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReaderFactory; @@ -65,33 +65,21 @@ public class HBaseTimelineReaderImpl } @Override - public TimelineEntity getEntity(String userId, String clusterId, - String flowName, Long flowRunId, String appId, String entityType, - String entityId, TimelineFilterList confsToRetrieve, - TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) - throws IOException { + public TimelineEntity getEntity(TimelineReaderContext context, + TimelineDataToRetrieve dataToRetrieve) throws IOException { TimelineEntityReader reader = - TimelineEntityReaderFactory.createSingleEntityReader(userId, clusterId, - flowName, flowRunId, appId, entityType, entityId, confsToRetrieve, - metricsToRetrieve, fieldsToRetrieve); + TimelineEntityReaderFactory.createSingleEntityReader(context, + dataToRetrieve); return reader.readEntity(hbaseConf, conn); } @Override - public Set<TimelineEntity> getEntities(String userId, String clusterId, - String flowName, Long flowRunId, String appId, String entityType, - Long limit, Long createdTimeBegin, Long createdTimeEnd, - Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo, - Map<String, Object> infoFilters, Map<String, String> configFilters, - Set<String> metricFilters, Set<String> eventFilters, - TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve, - EnumSet<Field> fieldsToRetrieve) throws IOException { + public Set<TimelineEntity> getEntities(TimelineReaderContext context, + TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve) + throws IOException { TimelineEntityReader reader = - TimelineEntityReaderFactory.createMultipleEntitiesReader(userId, - clusterId, flowName, flowRunId, appId, entityType, limit, - createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, - infoFilters, configFilters, metricFilters, eventFilters, - confsToRetrieve, metricsToRetrieve, fieldsToRetrieve); + TimelineEntityReaderFactory.createMultipleEntitiesReader(context, + filters, dataToRetrieve); return reader.readEntities(hbaseConf, conn); } }