YARN-4447. Provide a mechanism to represent complex filters and parse them at the REST layer (Varun Saxena via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79008c15 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79008c15 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79008c15 Branch: refs/heads/YARN-2928 Commit: 79008c15703ece4bcac9116c4b44c3ab872e14a2 Parents: cdd9efb Author: Sangjin Lee <sj...@apache.org> Authored: Mon May 2 14:06:19 2016 -0700 Committer: Sangjin Lee <sj...@apache.org> Committed: Mon May 2 14:06:19 2016 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../reader/TimelineParseConstants.java | 34 + .../reader/TimelineParseException.java | 36 + .../timelineservice/reader/TimelineParser.java | 37 + .../reader/TimelineParserForCompareExpr.java | 300 ++++++ .../reader/TimelineParserForDataToRetrieve.java | 95 ++ .../reader/TimelineParserForEqualityExpr.java | 343 +++++++ .../reader/TimelineParserForExistFilters.java | 51 + .../reader/TimelineParserForKVFilters.java | 78 ++ .../reader/TimelineParserForNumericFilters.java | 72 ++ .../TimelineParserForRelationFilters.java | 71 ++ .../reader/TimelineReaderWebServices.java | 220 ++++- .../reader/TimelineReaderWebServicesUtils.java | 196 ++-- .../reader/filter/TimelineCompareFilter.java | 73 +- .../reader/filter/TimelineExistsFilter.java | 49 +- .../reader/filter/TimelineFilterList.java | 36 + .../reader/filter/TimelineKeyValueFilter.java | 13 + .../reader/filter/TimelineKeyValuesFilter.java | 61 +- .../reader/filter/TimelinePrefixFilter.java | 37 + .../reader/TestTimelineReaderWebServices.java | 14 +- ...stTimelineReaderWebServicesHBaseStorage.java | 900 +++++++++++++++++- .../TestTimelineReaderWebServicesUtils.java | 923 +++++++++++++++++++ 22 files changed, 3445 insertions(+), 197 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/79008c15/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 61a9bc1..2e6996c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -136,6 +136,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1 YARN-3816. [Aggregation] App-level aggregation and accumulation for YARN system metrics (Li Lu via sjlee) + YARN-4447. Provide a mechanism to represent complex filters and parse them + at the REST layer (Varun Saxena via sjlee) + IMPROVEMENTS YARN-4224. Support fetching entities by UID and change the REST interface http://git-wip-us.apache.org/repos/asf/hadoop/blob/79008c15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseConstants.java new file mode 100644 index 0000000..662a102 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseConstants.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader; + +/** + * Set of constants used while parsing filter expressions. + */ +final class TimelineParseConstants { + private TimelineParseConstants() { + } + static final String COMMA_DELIMITER = ","; + static final String COLON_DELIMITER = ":"; + static final char NOT_CHAR = '!'; + static final char SPACE_CHAR = ' '; + static final char OPENING_BRACKET_CHAR = '('; + static final char CLOSING_BRACKET_CHAR = ')'; + static final char COMMA_CHAR = ','; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/79008c15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseException.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseException.java new file mode 100644 index 0000000..8d4a5dc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseException.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader; + +/** + * Exception thrown to indicate that a timeline filter expression cannot be + * parsed. + */ +class TimelineParseException extends Exception { + + private static final long serialVersionUID = 1L; + + public TimelineParseException() { + super(); + } + + public TimelineParseException(String message) { + super(message); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/79008c15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParser.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParser.java new file mode 100644 index 0000000..6b461a0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParser.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader; + +import java.io.Closeable; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; + +@Private +@Unstable +interface TimelineParser extends Closeable { + /** + * Method used for parsing. + * + * @return a {@link TimelineFilterList} object. + * @throws TimelineParseException if any problem occurs while parsing. + */ + TimelineFilterList parse() throws TimelineParseException; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/79008c15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForCompareExpr.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForCompareExpr.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForCompareExpr.java new file mode 100644 index 0000000..1b020d9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForCompareExpr.java @@ -0,0 +1,300 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader; + +import java.util.Deque; +import java.util.LinkedList; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator; + +/** + * Abstract class for parsing compare expressions. + * Compare expressions are of the form : + * (<key> <compareop> <value>) <op> (<key + * > <compareop> <value>) + * compareop is used to compare value of a the specified key in the backend + * storage. compareop can be : + * 1. eq - Equals + * 2. ne - Not equals (matches if key does not exist) + * 3. ene - Exists and not equals (key must exist for match to occur) + * 4. lt - Less than + * 5. gt - Greater than + * 6. le - Less than or equals + * 7. ge - Greater than or equals + * compareop's supported would depend on implementation. For instance, all + * the above compareops' will be supported for metric filters but only eq,ne and + * ene would be supported for KV filters like config/info filters. + * + * op is a logical operator and can be either AND or OR. + * + * The way values will be interpreted would also depend on implementation + * + * A typical compare expression would look as under: + * ((key1 eq val1 OR key2 ne val2) AND (key5 gt val45)) + */ +@Private +@Unstable +abstract class TimelineParserForCompareExpr implements TimelineParser { + private enum ParseState { + PARSING_KEY, + PARSING_VALUE, + PARSING_OP, + PARSING_COMPAREOP + } + // Main expression. + private final String expr; + // Expression in lower case. + private final String exprInLowerCase; + private final String exprName; + private int offset = 0; + private int kvStartOffset = 0; + private final int exprLength; + private ParseState currentParseState = ParseState.PARSING_KEY; + // Linked list implemented as a stack. + private Deque<TimelineFilterList> filterListStack = new LinkedList<>(); + private TimelineFilter currentFilter = null; + private TimelineFilterList filterList = null; + public TimelineParserForCompareExpr(String expression, String name) { + if (expression != null) { + expr = expression.trim(); + exprLength = expr.length(); + exprInLowerCase = expr.toLowerCase(); + } else { + expr = null; + exprInLowerCase = null; + exprLength = 0; + } + this.exprName = name; + } + + protected TimelineFilter getCurrentFilter() { + return currentFilter; + } + + protected TimelineFilter getFilterList() { + return filterList; + } + + protected abstract TimelineFilter createFilter(); + + protected abstract Object parseValue(String strValue) + throws TimelineParseException; + + protected abstract void setCompareOpToCurrentFilter( + TimelineCompareOp compareOp, boolean keyMustExistFlag) + throws TimelineParseException; + + protected abstract void setValueToCurrentFilter(Object value); + + private void handleSpaceChar() throws TimelineParseException { + if (currentParseState == ParseState.PARSING_KEY || + currentParseState == ParseState.PARSING_VALUE) { + if (kvStartOffset == offset) { + kvStartOffset++; + offset++; + return; + } + String str = expr.substring(kvStartOffset, offset); + if (currentParseState == ParseState.PARSING_KEY) { + if (currentFilter == null) { + currentFilter = createFilter(); + } + ((TimelineCompareFilter)currentFilter).setKey(str); + currentParseState = ParseState.PARSING_COMPAREOP; + } else if (currentParseState == ParseState.PARSING_VALUE) { + if (currentFilter != null) { + setValueToCurrentFilter(parseValue(str)); + } + currentParseState = ParseState.PARSING_OP; + } + } + offset++; + } + + private void handleOpeningBracketChar() throws TimelineParseException { + if (currentParseState != ParseState.PARSING_KEY) { + throw new TimelineParseException("Encountered unexpected opening " + + "bracket while parsing " + exprName + "."); + } + offset++; + kvStartOffset = offset; + filterListStack.push(filterList); + filterList = null; + } + + private void handleClosingBracketChar() throws TimelineParseException { + if (currentParseState != ParseState.PARSING_VALUE && + currentParseState != ParseState.PARSING_OP) { + throw new TimelineParseException("Encountered unexpected closing " + + "bracket while parsing " + exprName + "."); + } + if (!filterListStack.isEmpty()) { + if (currentParseState == ParseState.PARSING_VALUE) { + setValueToCurrentFilter( + parseValue(expr.substring(kvStartOffset, offset))); + currentParseState = ParseState.PARSING_OP; + } + if (currentFilter != null) { + filterList.addFilter(currentFilter); + } + // As bracket is closing, pop the filter list from top of the stack and + // combine it with current filter list. + TimelineFilterList fList = filterListStack.pop(); + if (fList != null) { + fList.addFilter(filterList); + filterList = fList; + } + currentFilter = null; + offset++; + kvStartOffset = offset; + } else { + throw new TimelineParseException("Encountered unexpected closing " + + "bracket while parsing " + exprName + "."); + } + } + + private void parseCompareOp() throws TimelineParseException { + if (offset + 2 >= exprLength) { + throw new TimelineParseException("Compare op cannot be parsed for " + + exprName + "."); + } + TimelineCompareOp compareOp = null; + boolean keyExistFlag = true; + if (expr.charAt(offset + 2) == TimelineParseConstants.SPACE_CHAR) { + if (exprInLowerCase.startsWith("eq", offset)) { + compareOp = TimelineCompareOp.EQUAL; + } else if (exprInLowerCase.startsWith("ne", offset)) { + compareOp = TimelineCompareOp.NOT_EQUAL; + keyExistFlag = false; + } else if (exprInLowerCase.startsWith("lt", offset)) { + compareOp = TimelineCompareOp.LESS_THAN; + } else if (exprInLowerCase.startsWith("le", offset)) { + compareOp = TimelineCompareOp.LESS_OR_EQUAL; + } else if (exprInLowerCase.startsWith("gt", offset)) { + compareOp = TimelineCompareOp.GREATER_THAN; + } else if (exprInLowerCase.startsWith("ge", offset)) { + compareOp = TimelineCompareOp.GREATER_OR_EQUAL; + } + offset = offset + 3; + } else if (exprInLowerCase.startsWith("ene ", offset)) { + // Not equal but key should be present. + compareOp = TimelineCompareOp.NOT_EQUAL; + offset = offset + 4; + } + if (compareOp == null) { + throw new TimelineParseException("Compare op cannot be parsed for " + + exprName + "."); + } + setCompareOpToCurrentFilter(compareOp, keyExistFlag); + kvStartOffset = offset; + currentParseState = ParseState.PARSING_VALUE; + } + + private void parseOp(boolean closingBracket) throws TimelineParseException { + Operator operator = null; + if (exprInLowerCase.startsWith("or ", offset)) { + operator = Operator.OR; + offset = offset + 3; + } else if (exprInLowerCase.startsWith("and ", offset)) { + operator = Operator.AND; + offset = offset + 4; + } + if (operator == null) { + throw new TimelineParseException("Operator cannot be parsed for " + + exprName + "."); + } + if (filterList == null) { + filterList = new TimelineFilterList(operator); + } + if (currentFilter != null) { + filterList.addFilter(currentFilter); + } + if (closingBracket || filterList.getOperator() != operator) { + filterList = new TimelineFilterList(operator, filterList); + } + currentFilter = null; + kvStartOffset = offset; + currentParseState = ParseState.PARSING_KEY; + } + + @Override + public TimelineFilterList parse() throws TimelineParseException { + if (expr == null || exprLength == 0) { + return null; + } + boolean closingBracket = false; + while (offset < exprLength) { + char offsetChar = expr.charAt(offset); + switch(offsetChar) { + case TimelineParseConstants.SPACE_CHAR: + handleSpaceChar(); + break; + case TimelineParseConstants.OPENING_BRACKET_CHAR: + handleOpeningBracketChar(); + break; + case TimelineParseConstants.CLOSING_BRACKET_CHAR: + handleClosingBracketChar(); + closingBracket = true; + break; + default: // other characters. + // Parse based on state. + if (currentParseState == ParseState.PARSING_COMPAREOP) { + parseCompareOp(); + } else if (currentParseState == ParseState.PARSING_OP) { + parseOp(closingBracket); + closingBracket = false; + } else { + // Might be a key or value. Move ahead. + offset++; + } + break; + } + } + if (!filterListStack.isEmpty()) { + filterListStack.clear(); + throw new TimelineParseException("Encountered improper brackets while " + + "parsing " + exprName + "."); + } + if (currentParseState == ParseState.PARSING_VALUE) { + setValueToCurrentFilter( + parseValue(expr.substring(kvStartOffset, offset))); + } + if (filterList == null || filterList.getFilterList().isEmpty()) { + filterList = new TimelineFilterList(currentFilter); + } else if (currentFilter != null) { + filterList.addFilter(currentFilter); + } + return filterList; + } + + @Override + public void close() { + if (filterListStack != null) { + filterListStack.clear(); + } + filterList = null; + currentFilter = null; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/79008c15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForDataToRetrieve.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForDataToRetrieve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForDataToRetrieve.java new file mode 100644 index 0000000..1e6039d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForDataToRetrieve.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator; + +/** + * Used for parsing metrics or configs to retrieve. + */ +@Private +@Unstable +public class TimelineParserForDataToRetrieve implements TimelineParser { + private String expr; + private final int exprLength; + public TimelineParserForDataToRetrieve(String expression) { + this.expr = expression; + if (expression != null) { + this.expr = expr.trim(); + exprLength = expr.length(); + } else { + exprLength = 0; + } + } + + @Override + public TimelineFilterList parse() throws TimelineParseException { + if (expr == null || exprLength == 0) { + return null; + } + TimelineCompareOp compareOp = null; + int openingBracketIndex = + expr.indexOf(TimelineParseConstants.OPENING_BRACKET_CHAR); + if (expr.charAt(0) == TimelineParseConstants.NOT_CHAR) { + if (openingBracketIndex == -1) { + throw new TimelineParseException("Invalid config/metric to retrieve " + + "expression"); + } + if (openingBracketIndex != 1 && + expr.substring(1, openingBracketIndex + 1).trim().length() != 1) { + throw new TimelineParseException("Invalid config/metric to retrieve " + + "expression"); + } + compareOp = TimelineCompareOp.NOT_EQUAL; + } else if (openingBracketIndex <= 0) { + compareOp = TimelineCompareOp.EQUAL; + } + char lastChar = expr.charAt(exprLength - 1); + if (compareOp == TimelineCompareOp.NOT_EQUAL && + lastChar != TimelineParseConstants.CLOSING_BRACKET_CHAR) { + throw new TimelineParseException("Invalid config/metric to retrieve " + + "expression"); + } + if (openingBracketIndex != -1 && + expr.charAt(exprLength - 1) == + TimelineParseConstants.CLOSING_BRACKET_CHAR) { + expr = expr.substring(openingBracketIndex + 1, exprLength - 1).trim(); + } + if (expr.isEmpty()) { + return null; + } + Operator op = + (compareOp == TimelineCompareOp.NOT_EQUAL) ? Operator.AND : Operator.OR; + TimelineFilterList list = new TimelineFilterList(op); + String[] splits = expr.split(TimelineParseConstants.COMMA_DELIMITER); + for (String split : splits) { + list.addFilter(new TimelinePrefixFilter(compareOp, split.trim())); + } + return list; + } + + @Override + public void close() { + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/79008c15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForEqualityExpr.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForEqualityExpr.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForEqualityExpr.java new file mode 100644 index 0000000..7451713 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForEqualityExpr.java @@ -0,0 +1,343 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader; + +import java.util.Deque; +import java.util.LinkedList; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator; + +/** + * Abstract class for parsing equality expressions. This means the values in + * expression would either be equal or not equal. + * Equality expressions are of the form : + * (<value>,<value>,<value>) <op> !(<value>, + * <value>) + * + * Here, "!" means all the values should not exist/should not be equal. + * If not specified, they should exist/be equal. + * + * op is a logical operator and can be either AND or OR. + * + * The way values will be interpreted would also depend on implementation. + * + * For instance for event filters this expression may look like, + * (event1,event2) AND !(event3,event4) + * This means for an entity to match, event1 and event2 should exist. But event3 + * and event4 should not exist. + */ +@Private +@Unstable +abstract class TimelineParserForEqualityExpr implements TimelineParser { + private enum ParseState { + PARSING_VALUE, + PARSING_OP, + PARSING_COMPAREOP + } + private final String expr; + // Expression in lower case. + private final String exprInLowerCase; + // Expression name. + private final String exprName; + // Expression offset. + private int offset = 0; + // Offset used to parse values in the expression. + private int startOffset = 0; + private final int exprLength; + private ParseState currentParseState = ParseState.PARSING_COMPAREOP; + private TimelineCompareOp currentCompareOp = null; + // Used to store filter lists which can then be combined as brackets are + // closed. + private Deque<TimelineFilterList> filterListStack = new LinkedList<>(); + private TimelineFilter currentFilter = null; + private TimelineFilterList filterList = null; + // Delimiter used to separate values. + private final char delimiter; + public TimelineParserForEqualityExpr(String expression, String name, + char delim) { + if (expression != null) { + expr = expression.trim(); + exprLength = expr.length(); + exprInLowerCase = expr.toLowerCase(); + } else { + exprLength = 0; + expr = null; + exprInLowerCase = null; + } + exprName = name; + delimiter = delim; + } + + protected TimelineFilter getCurrentFilter() { + return currentFilter; + } + + protected TimelineFilter getFilterList() { + return filterList; + } + + /** + * Creates filter as per implementation. + * + * @return a {@link TimelineFilter} implementation. + */ + protected abstract TimelineFilter createFilter(); + + /** + * Sets compare op to the current filter as per filter implementation. + * + * @param compareOp compare op to be set. + * @throws Exception if any problem occurs. + */ + protected abstract void setCompareOpToCurrentFilter( + TimelineCompareOp compareOp) throws TimelineParseException; + + /** + * Sets value to the current filter as per filter implementation. + * + * @param value value to be set. + * @throws Exception if any problem occurs. + */ + protected abstract void setValueToCurrentFilter(String value) + throws TimelineParseException; + + private void createAndSetFilter(boolean checkIfNull) + throws TimelineParseException { + if (!checkIfNull || currentFilter == null) { + currentFilter = createFilter(); + setCompareOpToCurrentFilter(currentCompareOp); + } + setValueToCurrentFilter(expr.substring(startOffset, offset).trim()); + } + + private void handleSpaceChar() throws TimelineParseException { + if (currentParseState == ParseState.PARSING_VALUE) { + if (startOffset == offset) { + startOffset++; + } else { + createAndSetFilter(true); + currentParseState = ParseState.PARSING_OP; + } + } + offset++; + } + + private void handleDelimiter() throws TimelineParseException { + if (currentParseState == ParseState.PARSING_OP || + currentParseState == ParseState.PARSING_VALUE) { + if (currentParseState == ParseState.PARSING_VALUE) { + createAndSetFilter(false); + } + if (filterList == null) { + filterList = new TimelineFilterList(); + } + // Add parsed filter into filterlist and make it null to move on to next + // filter. + filterList.addFilter(currentFilter); + currentFilter = null; + offset++; + startOffset = offset; + currentParseState = ParseState.PARSING_VALUE; + } else { + throw new TimelineParseException("Invalid " + exprName + "expression."); + } + } + + private void handleOpeningBracketChar(boolean encounteredNot) + throws TimelineParseException { + if (currentParseState == ParseState.PARSING_COMPAREOP || + currentParseState == ParseState.PARSING_VALUE) { + offset++; + startOffset = offset; + filterListStack.push(filterList); + filterList = null; + if (currentFilter == null) { + currentFilter = createFilter(); + } + currentCompareOp = encounteredNot ? + TimelineCompareOp.NOT_EQUAL : TimelineCompareOp.EQUAL; + setCompareOpToCurrentFilter(currentCompareOp); + currentParseState = ParseState.PARSING_VALUE; + } else { + throw new TimelineParseException("Encountered unexpected opening " + + "bracket while parsing " + exprName + "."); + } + } + + private void handleNotChar() throws TimelineParseException { + if (currentParseState == ParseState.PARSING_COMPAREOP || + currentParseState == ParseState.PARSING_VALUE) { + offset++; + while (offset < exprLength && + expr.charAt(offset) == TimelineParseConstants.SPACE_CHAR) { + offset++; + } + if (offset == exprLength) { + throw new TimelineParseException("Invalid " + exprName + "expression"); + } + if (expr.charAt(offset) == TimelineParseConstants.OPENING_BRACKET_CHAR) { + handleOpeningBracketChar(true); + } else { + throw new TimelineParseException("Invalid " + exprName + "expression"); + } + } else { + throw new TimelineParseException("Encountered unexpected not(!) char " + + "while parsing " + exprName + "."); + } + } + + private void handleClosingBracketChar() throws TimelineParseException { + if (currentParseState != ParseState.PARSING_VALUE && + currentParseState != ParseState.PARSING_OP) { + throw new TimelineParseException("Encountered unexpected closing " + + "bracket while parsing " + exprName + "."); + } + if (!filterListStack.isEmpty()) { + if (currentParseState == ParseState.PARSING_VALUE) { + if (startOffset != offset) { + createAndSetFilter(true); + currentParseState = ParseState.PARSING_OP; + } + } + if (filterList == null) { + filterList = new TimelineFilterList(); + } + if (currentFilter != null) { + filterList.addFilter(currentFilter); + } + // As bracket is closing, pop the filter list from top of the stack and + // combine it with current filter list. + TimelineFilterList fList = filterListStack.pop(); + if (fList != null) { + fList.addFilter(filterList); + filterList = fList; + } + currentFilter = null; + offset++; + startOffset = offset; + } else { + throw new TimelineParseException("Encountered unexpected closing " + + "bracket while parsing " + exprName + "."); + } + } + + private void parseOp(boolean closingBracket) throws TimelineParseException { + Operator operator = null; + if (exprInLowerCase.startsWith("or ", offset)) { + operator = Operator.OR; + offset = offset + 3; + } else if (exprInLowerCase.startsWith("and ", offset)) { + operator = Operator.AND; + offset = offset + 4; + } + if (operator == null) { + throw new TimelineParseException("Operator cannot be parsed for " + + exprName + "."); + } + if (filterList == null) { + filterList = new TimelineFilterList(operator); + } + if (currentFilter != null) { + filterList.addFilter(currentFilter); + } + if (closingBracket || filterList.getOperator() != operator) { + filterList = new TimelineFilterList(operator, filterList); + } + currentFilter = null; + startOffset = offset; + currentParseState = ParseState.PARSING_COMPAREOP; + } + + private void parseCompareOp() throws TimelineParseException { + if (currentFilter == null) { + currentFilter = createFilter(); + } + currentCompareOp = TimelineCompareOp.EQUAL; + setCompareOpToCurrentFilter(currentCompareOp); + currentParseState = ParseState.PARSING_VALUE; + } + + @Override + public TimelineFilterList parse() throws TimelineParseException { + if (expr == null || exprLength == 0) { + return null; + } + boolean closingBracket = false; + while (offset < exprLength) { + char offsetChar = expr.charAt(offset); + switch(offsetChar) { + case TimelineParseConstants.NOT_CHAR: + handleNotChar(); + break; + case TimelineParseConstants.SPACE_CHAR: + handleSpaceChar(); + break; + case TimelineParseConstants.OPENING_BRACKET_CHAR: + handleOpeningBracketChar(false); + break; + case TimelineParseConstants.CLOSING_BRACKET_CHAR: + handleClosingBracketChar(); + closingBracket = true; + break; + default: // other characters. + if (offsetChar == delimiter) { + handleDelimiter(); + } else if (currentParseState == ParseState.PARSING_COMPAREOP) { + parseCompareOp(); + } else if (currentParseState == ParseState.PARSING_OP) { + parseOp(closingBracket); + closingBracket = false; + } else { + offset++; + } + break; + } + } + if (!filterListStack.isEmpty()) { + filterListStack.clear(); + throw new TimelineParseException("Encountered improper brackets while " + + "parsing " + exprName + "."); + } + if (currentParseState == ParseState.PARSING_VALUE) { + if (startOffset != offset) { + createAndSetFilter(true); + } + } + if (filterList == null || filterList.getFilterList().isEmpty()) { + filterList = new TimelineFilterList(currentFilter); + } else if (currentFilter != null) { + filterList.addFilter(currentFilter); + } + return filterList; + } + + @Override + public void close() { + if (filterListStack != null) { + filterListStack.clear(); + } + currentFilter = null; + filterList = null; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/79008c15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForExistFilters.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForExistFilters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForExistFilters.java new file mode 100644 index 0000000..8048c6e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForExistFilters.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter; + +/** + * Used for parsing existence filters such as event filters. These filters + * check for existence of a value. For example, in case of event filters, they + * check if an event exists or not and accordingly return an entity. + */ +@Private +@Unstable +class TimelineParserForExistFilters extends TimelineParserForEqualityExpr { + + public TimelineParserForExistFilters(String expression, char delimiter) { + super(expression, "Event Filter", delimiter); + } + + protected TimelineFilter createFilter() { + return new TimelineExistsFilter(); + } + + protected void setValueToCurrentFilter(String value) { + ((TimelineExistsFilter)getCurrentFilter()).setValue(value); + } + + protected void setCompareOpToCurrentFilter(TimelineCompareOp compareOp) { + ((TimelineExistsFilter)getCurrentFilter()).setCompareOp(compareOp); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/79008c15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForKVFilters.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForKVFilters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForKVFilters.java new file mode 100644 index 0000000..ec68bec --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForKVFilters.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter; + +/** + * Used for parsing key-value filters such as config and info filters. + */ +@Private +@Unstable +class TimelineParserForKVFilters extends TimelineParserForCompareExpr { + // Indicates if value has to be interpreted as a string. + private final boolean valueAsString; + public TimelineParserForKVFilters(String expression, boolean valAsStr) { + super(expression, "Config/Info Filter"); + this.valueAsString = valAsStr; + } + + protected TimelineFilter createFilter() { + return new TimelineKeyValueFilter(); + } + + protected Object parseValue(String strValue) { + if (!valueAsString) { + try { + return GenericObjectMapper.OBJECT_READER.readValue(strValue); + } catch (IOException e) { + return strValue; + } + } else { + return strValue; + } + } + + @Override + protected void setCompareOpToCurrentFilter(TimelineCompareOp compareOp, + boolean keyMustExistFlag) throws TimelineParseException { + if (compareOp != TimelineCompareOp.EQUAL && + compareOp != TimelineCompareOp.NOT_EQUAL) { + throw new TimelineParseException("TimelineCompareOp for kv-filter " + + "should be EQUAL or NOT_EQUAL"); + } + ((TimelineKeyValueFilter)getCurrentFilter()).setCompareOp( + compareOp, keyMustExistFlag); + } + + @Override + protected void setValueToCurrentFilter(Object value) { + TimelineFilter currentFilter = getCurrentFilter(); + if (currentFilter != null) { + ((TimelineKeyValueFilter)currentFilter).setValue(value); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/79008c15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForNumericFilters.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForNumericFilters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForNumericFilters.java new file mode 100644 index 0000000..7c14a9f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForNumericFilters.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter; +import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils; + +/** + * Used for parsing numerical filters such as metric filters. + */ +@Private +@Unstable +class TimelineParserForNumericFilters extends TimelineParserForCompareExpr { + + public TimelineParserForNumericFilters(String expression) { + super(expression, "Metric Filter"); + } + + protected TimelineFilter createFilter() { + return new TimelineCompareFilter(); + } + + @Override + protected void setCompareOpToCurrentFilter(TimelineCompareOp compareOp, + boolean keyMustExistFlag) { + ((TimelineCompareFilter)getCurrentFilter()).setCompareOp( + compareOp, keyMustExistFlag); + } + + protected Object parseValue(String strValue) throws TimelineParseException { + Object value = null; + try { + value = GenericObjectMapper.OBJECT_READER.readValue(strValue); + } catch (IOException e) { + throw new TimelineParseException("Value cannot be parsed."); + } + if (value == null || !(TimelineStorageUtils.isIntegralValue(value))) { + throw new TimelineParseException("Value is not a number."); + } + return value; + } + + protected void setValueToCurrentFilter(Object value) { + TimelineFilter currentFilter = getCurrentFilter(); + if (currentFilter != null) { + ((TimelineCompareFilter)currentFilter).setValue(value); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/79008c15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForRelationFilters.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForRelationFilters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForRelationFilters.java new file mode 100644 index 0000000..cde11e4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForRelationFilters.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter; +import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter; + +/** + * Used for parsing relation filters. + */ +@Private +@Unstable +class TimelineParserForRelationFilters extends + TimelineParserForEqualityExpr { + private final String valueDelimiter; + public TimelineParserForRelationFilters(String expression, char valuesDelim, + String valueDelim) { + super(expression, "Relation Filter", valuesDelim); + valueDelimiter = valueDelim; + } + + @Override + protected TimelineFilter createFilter() { + return new TimelineKeyValuesFilter(); + } + + @Override + protected void setCompareOpToCurrentFilter(TimelineCompareOp compareOp) { + ((TimelineKeyValuesFilter)getCurrentFilter()).setCompareOp(compareOp); + } + + @Override + protected void setValueToCurrentFilter(String value) + throws TimelineParseException { + if (value != null) { + String[] pairStrs = value.split(valueDelimiter); + if (pairStrs.length < 2) { + throw new TimelineParseException("Invalid relation filter expression"); + } + String key = pairStrs[0].trim(); + Set<Object> values = new HashSet<Object>(); + for (int i = 1; i < pairStrs.length; i++) { + values.add(pairStrs[i].trim()); + } + ((TimelineKeyValuesFilter)getCurrentFilter()). + setKeyAndValues(key, values); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/79008c15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java index 0b9549b..6ec59ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java @@ -178,6 +178,9 @@ public class TimelineReaderWebServices { "Requested Invalid Field." : e.getMessage()); } else if (e instanceof NotFoundException) { throw (NotFoundException)e; + } else if (e instanceof TimelineParseException) { + throw new BadRequestException(e.getMessage() == null ? + "Filter Parsing failed." : e.getMessage()); } else if (e instanceof BadRequestException) { throw (BadRequestException)e; } else { @@ -239,6 +242,14 @@ public class TimelineReaderWebServices { * metricfilters=metricid1, metricid2... (Optional query param). * @param eventfilters If specified, matched entities should contain the given * events. This is represented as eventfilters=eventid1, eventid2... + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be retrieved + * irrespective of whether configs are specified in fields to retrieve or + * not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields of the entity object to retrieve, see * {@link Field}. All fields will be retrieved if fields=ALL. If not * specified, 3 fields i.e. entity type, id and created time is returned @@ -270,6 +281,8 @@ public class TimelineReaderWebServices { @QueryParam("conffilters") String conffilters, @QueryParam("metricfilters") String metricfilters, @QueryParam("eventfilters") String eventfilters, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : @@ -295,7 +308,7 @@ public class TimelineReaderWebServices { limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - null, null, fields)); + confsToRetrieve, metricsToRetrieve, fields)); } catch (Exception e) { handleException(e, url, startTime, "createdTime start/end or limit or flowrunid"); @@ -354,6 +367,14 @@ public class TimelineReaderWebServices { * metricfilters=metricid1, metricid2... (Optional query param). * @param eventfilters If specified, matched entities should contain the given * events. This is represented as eventfilters=eventid1, eventid2... + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be retrieved + * irrespective of whether configs are specified in fields to retrieve or + * not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields of the entity object to retrieve, see * {@link Field}. All fields will be retrieved if fields=ALL. If not * specified, 3 fields i.e. entity type, id, created time is returned @@ -390,11 +411,13 @@ public class TimelineReaderWebServices { @QueryParam("conffilters") String conffilters, @QueryParam("metricfilters") String metricfilters, @QueryParam("eventfilters") String eventfilters, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { return getEntities(req, res, null, appId, entityType, userId, flowName, flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, - fields); + confsToRetrieve, metricsToRetrieve, fields); } /** @@ -443,6 +466,14 @@ public class TimelineReaderWebServices { * metricfilters=metricid1, metricid2... (Optional query param). * @param eventfilters If specified, matched entities should contain the given * events. This is represented as eventfilters=eventid1, eventid2... + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be retrieved + * irrespective of whether configs are specified in fields to retrieve or + * not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields of the entity object to retrieve, see * {@link Field}. All fields will be retrieved if fields=ALL. If not * specified, 3 fields i.e. entity type, id, created time is returned @@ -480,6 +511,8 @@ public class TimelineReaderWebServices { @QueryParam("conffilters") String conffilters, @QueryParam("metricfilters") String metricfilters, @QueryParam("eventfilters") String eventfilters, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : @@ -496,12 +529,11 @@ public class TimelineReaderWebServices { entities = timelineReaderManager.getEntities( TimelineReaderWebServicesUtils.createTimelineReaderContext( clusterId, userId, flowName, flowRunId, appId, entityType, null), - TimelineReaderWebServicesUtils.createTimelineEntityFilters( limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - null, null, fields)); + confsToRetrieve, metricsToRetrieve, fields)); } catch (Exception e) { handleException(e, url, startTime, "createdTime start/end or limit or flowrunid"); @@ -524,6 +556,14 @@ public class TimelineReaderWebServices { * @param uId a delimited string containing clusterid, userid, flow name, * flowrun id, app id, entity type and entity id which are extracted from * UID and then used to query backend(Mandatory path param). + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be retrieved + * irrespective of whether configs are specified in fields to retrieve or + * not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields of the entity object to retrieve, see * {@link Field}. All fields will be retrieved if fields=ALL. If not * specified, 3 fields i.e. entity type, id, created time is returned @@ -546,6 +586,8 @@ public class TimelineReaderWebServices { @Context HttpServletRequest req, @Context HttpServletResponse res, @PathParam("uid") String uId, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : @@ -566,7 +608,7 @@ public class TimelineReaderWebServices { } entity = timelineReaderManager.getEntity(context, TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - null, null, fields)); + confsToRetrieve, metricsToRetrieve, fields)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -601,6 +643,14 @@ public class TimelineReaderWebServices { * param). * @param flowRunId Run id which should match for the entity(Optional query * param). + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be retrieved + * irrespective of whether configs are specified in fields to retrieve or + * not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields of the entity object to retrieve, see * {@link Field}. All fields will be retrieved if fields=ALL. If not * specified, 3 fields i.e. entity type, id, created time is returned @@ -628,9 +678,11 @@ public class TimelineReaderWebServices { @QueryParam("userid") String userId, @QueryParam("flowname") String flowName, @QueryParam("flowrunid") String flowRunId, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { return getEntity(req, res, null, appId, entityType, entityId, userId, - flowName, flowRunId, fields); + flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields); } /** @@ -653,6 +705,14 @@ public class TimelineReaderWebServices { * param). * @param flowRunId Run id which should match for the entity(Optional query * param). + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be retrieved + * irrespective of whether configs are specified in fields to retrieve or + * not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields of the entity object to retrieve, see * {@link Field}. All fields will be retrieved if fields=ALL. If not * specified, 3 fields i.e. entity type, id and created time is returned @@ -681,6 +741,8 @@ public class TimelineReaderWebServices { @QueryParam("userid") String userId, @QueryParam("flowname") String flowName, @QueryParam("flowrunid") String flowRunId, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : @@ -698,7 +760,7 @@ public class TimelineReaderWebServices { TimelineReaderWebServicesUtils.createTimelineReaderContext( clusterId, userId, flowName, flowRunId, appId, entityType, entityId), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - null, null, fields)); + confsToRetrieve, metricsToRetrieve, fields)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -723,6 +785,8 @@ public class TimelineReaderWebServices { * @param uId a delimited string containing clusterid, userid, flow name and * flowrun id which are extracted from UID and then used to query backend * (Mandatory path param). + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * <cite>FlowRunEntity</cite> instance is returned. By default, all @@ -741,7 +805,8 @@ public class TimelineReaderWebServices { public TimelineEntity getFlowRun( @Context HttpServletRequest req, @Context HttpServletResponse res, - @PathParam("uid") String uId) { + @PathParam("uid") String uId, + @QueryParam("metricstoretrieve") String metricsToRetrieve) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -761,7 +826,8 @@ public class TimelineReaderWebServices { } context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString()); entity = timelineReaderManager.getEntity(context, - new TimelineDataToRetrieve()); + TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( + null, metricsToRetrieve, null)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -787,6 +853,8 @@ public class TimelineReaderWebServices { * @param flowName Flow name to which the flow run to be queried belongs to( * Mandatory path param). * @param flowRunId Id of the flow run to be queried(Mandatory path param). + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * <cite>FlowRunEntity</cite> instance is returned. By default, all @@ -807,8 +875,10 @@ public class TimelineReaderWebServices { @Context HttpServletResponse res, @PathParam("userid") String userId, @PathParam("flowname") String flowName, - @PathParam("flowrunid") String flowRunId) { - return getFlowRun(req, res, null, userId, flowName, flowRunId); + @PathParam("flowrunid") String flowRunId, + @QueryParam("metricstoretrieve") String metricsToRetrieve) { + return getFlowRun(req, res, null, userId, flowName, flowRunId, + metricsToRetrieve); } /** @@ -823,6 +893,8 @@ public class TimelineReaderWebServices { * @param flowName Flow name to which the flow run to be queried belongs to( * Mandatory path param). * @param flowRunId Id of the flow run to be queried(Mandatory path param). + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. * * @return If successful, a HTTP 200(OK) response having a JSON representing a * <cite>FlowRunEntity</cite> instance is returned. By default, all @@ -845,7 +917,8 @@ public class TimelineReaderWebServices { @PathParam("clusterid") String clusterId, @PathParam("userid") String userId, @PathParam("flowname") String flowName, - @PathParam("flowrunid") String flowRunId) { + @PathParam("flowrunid") String flowRunId, + @QueryParam("metricstoretrieve") String metricsToRetrieve) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString()); @@ -862,7 +935,8 @@ public class TimelineReaderWebServices { TimelineReaderWebServicesUtils.createTimelineReaderContext( clusterId, userId, flowName, flowRunId, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null), - new TimelineDataToRetrieve()); + TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( + null, metricsToRetrieve, null)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -894,6 +968,10 @@ public class TimelineReaderWebServices { * created before this timestamp(Optional query param). * @param createdTimeEnd If specified, matched flow runs should not be created * after this timestamp(Optional query param). + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields to retrieve, see {@link Field}. * All fields will be retrieved if fields=ALL. Fields other than METRICS * have no meaning for this REST endpoint. If not specified, all fields @@ -918,6 +996,7 @@ public class TimelineReaderWebServices { @QueryParam("limit") String limit, @QueryParam("createdtimestart") String createdTimeStart, @QueryParam("createdtimeend") String createdTimeEnd, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : @@ -942,7 +1021,7 @@ public class TimelineReaderWebServices { limit, createdTimeStart, createdTimeEnd, null, null, null, null, null, null), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - null, null, fields)); + null, metricsToRetrieve, fields)); } catch (Exception e) { handleException(e, url, startTime, "createdTime start/end or limit"); } @@ -970,6 +1049,10 @@ public class TimelineReaderWebServices { * created before this timestamp(Optional query param). * @param createdTimeEnd If specified, matched flow runs should not be created * after this timestamp(Optional query param). + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields to retrieve, see {@link Field}. * All fields will be retrieved if fields=ALL. Fields other than METRICS * have no meaning for this REST endpoint. If not specified, all fields @@ -995,9 +1078,10 @@ public class TimelineReaderWebServices { @QueryParam("limit") String limit, @QueryParam("createdtimestart") String createdTimeStart, @QueryParam("createdtimeend") String createdTimeEnd, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { return getFlowRuns(req, res, null, userId, flowName, limit, - createdTimeStart, createdTimeEnd, fields); + createdTimeStart, createdTimeEnd, metricsToRetrieve, fields); } /** @@ -1016,6 +1100,10 @@ public class TimelineReaderWebServices { * created before this timestamp(Optional query param). * @param createdTimeEnd If specified, matched flow runs should not be created * after this timestamp(Optional query param). + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields to retrieve, see {@link Field}. * All fields will be retrieved if fields=ALL. Fields other than METRICS * have no meaning for this REST endpoint. If not specified, all fields @@ -1042,6 +1130,7 @@ public class TimelineReaderWebServices { @QueryParam("limit") String limit, @QueryParam("createdtimestart") String createdTimeStart, @QueryParam("createdtimeend") String createdTimeEnd, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : @@ -1063,7 +1152,7 @@ public class TimelineReaderWebServices { limit, createdTimeStart, createdTimeEnd, null, null, null, null, null, null), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - null, null, fields)); + null, metricsToRetrieve, fields)); } catch (Exception e) { handleException(e, url, startTime, "createdTime start/end or limit"); } @@ -1204,6 +1293,14 @@ public class TimelineReaderWebServices { * @param uId a delimited string containing clusterid, userid, flow name, flow * run id and app id which are extracted from UID and then used to query * backend(Mandatory path param). + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be retrieved + * irrespective of whether configs are specified in fields to retrieve or + * not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), @@ -1226,6 +1323,8 @@ public class TimelineReaderWebServices { @Context HttpServletRequest req, @Context HttpServletResponse res, @PathParam("uid") String uId, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : @@ -1247,7 +1346,7 @@ public class TimelineReaderWebServices { context.setEntityType(TimelineEntityType.YARN_APPLICATION.toString()); entity = timelineReaderManager.getEntity(context, TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - null, null, fields)); + confsToRetrieve, metricsToRetrieve, fields)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -1277,6 +1376,14 @@ public class TimelineReaderWebServices { * @param flowRunId Run id which should match for the app(Optional query * param). * @param userId User id which should match for the app(Optional query param). + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be retrieved + * irrespective of whether configs are specified in fields to retrieve or + * not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), @@ -1302,8 +1409,11 @@ public class TimelineReaderWebServices { @QueryParam("flowname") String flowName, @QueryParam("flowrunid") String flowRunId, @QueryParam("userid") String userId, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { - return getApp(req, res, null, appId, flowName, flowRunId, userId, fields); + return getApp(req, res, null, appId, flowName, flowRunId, userId, + confsToRetrieve, metricsToRetrieve, fields); } /** @@ -1322,6 +1432,14 @@ public class TimelineReaderWebServices { * @param flowRunId Run id which should match for the app(Optional query * param). * @param userId User id which should match for the app(Optional query param). + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be retrieved + * irrespective of whether configs are specified in fields to retrieve or + * not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), @@ -1348,6 +1466,8 @@ public class TimelineReaderWebServices { @QueryParam("flowname") String flowName, @QueryParam("flowrunid") String flowRunId, @QueryParam("userid") String userId, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : @@ -1366,7 +1486,7 @@ public class TimelineReaderWebServices { clusterId, userId, flowName, flowRunId, appId, TimelineEntityType.YARN_APPLICATION.toString(), null), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - null, null, fields)); + confsToRetrieve, metricsToRetrieve, fields)); } catch (Exception e) { handleException(e, url, startTime, "flowrunid"); } @@ -1417,6 +1537,14 @@ public class TimelineReaderWebServices { * (Optional query param). * @param eventfilters If specified, matched apps should contain the given * events. This is represented as eventfilters=eventid1, eventid2... + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be retrieved + * irrespective of whether configs are specified in fields to retrieve or + * not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), @@ -1447,6 +1575,8 @@ public class TimelineReaderWebServices { @QueryParam("conffilters") String conffilters, @QueryParam("metricfilters") String metricfilters, @QueryParam("eventfilters") String eventfilters, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { String url = req.getRequestURI() + (req.getQueryString() == null ? "" : @@ -1471,7 +1601,7 @@ public class TimelineReaderWebServices { limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve( - null, null, fields)); + confsToRetrieve, metricsToRetrieve, fields)); } catch (Exception e) { handleException(e, url, startTime, "createdTime start/end or limit or flowrunid"); @@ -1523,6 +1653,14 @@ public class TimelineReaderWebServices { * (Optional query param). * @param eventfilters If specified, matched apps should contain the given * events. This is represented as eventfilters=eventid1, eventid2... + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be retrieved + * irrespective of whether configs are specified in fields to retrieve or + * not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), @@ -1555,12 +1693,14 @@ public class TimelineReaderWebServices { @QueryParam("conffilters") String conffilters, @QueryParam("metricfilters") String metricfilters, @QueryParam("eventfilters") String eventfilters, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { return getEntities(req, res, null, null, TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName, flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, - fields); + confsToRetrieve, metricsToRetrieve, fields); } /** @@ -1602,6 +1742,14 @@ public class TimelineReaderWebServices { * (Optional query param). * @param eventfilters If specified, matched apps should contain the given * events. This is represented as eventfilters=eventid1, eventid2... + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be retrieved + * irrespective of whether configs are specified in fields to retrieve or + * not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), @@ -1636,12 +1784,14 @@ public class TimelineReaderWebServices { @QueryParam("conffilters") String conffilters, @QueryParam("metricfilters") String metricfilters, @QueryParam("eventfilters") String eventfilters, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { return getEntities(req, res, clusterId, null, TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName, flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, - fields); + confsToRetrieve, metricsToRetrieve, fields); } /** @@ -1680,6 +1830,14 @@ public class TimelineReaderWebServices { * (Optional query param). * @param eventfilters If specified, matched apps should contain the given * events. This is represented as eventfilters=eventid1, eventid2... + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be retrieved + * irrespective of whether configs are specified in fields to retrieve or + * not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), @@ -1711,11 +1869,14 @@ public class TimelineReaderWebServices { @QueryParam("conffilters") String conffilters, @QueryParam("metricfilters") String metricfilters, @QueryParam("eventfilters") String eventfilters, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { return getEntities(req, res, null, null, TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName, null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, - infofilters, conffilters, metricfilters, eventfilters, fields); + infofilters, conffilters, metricfilters, eventfilters, + confsToRetrieve, metricsToRetrieve, fields); } /** @@ -1756,6 +1917,14 @@ public class TimelineReaderWebServices { * (Optional query param). * @param eventfilters If specified, matched apps should contain the given * events. This is represented as eventfilters=eventid1, eventid2... + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be retrieved + * irrespective of whether configs are specified in fields to retrieve or + * not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to retrieve or + * not. * @param fields Specifies which fields of the app entity object to retrieve, * see {@link Field}. All fields will be retrieved if fields=ALL. If not * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION), @@ -1788,10 +1957,13 @@ public class TimelineReaderWebServices { @QueryParam("conffilters") String conffilters, @QueryParam("metricfilters") String metricfilters, @QueryParam("eventfilters") String eventfilters, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields) { return getEntities(req, res, clusterId, null, TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName, null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, - infofilters, conffilters, metricfilters, eventfilters, fields); + infofilters, conffilters, metricfilters, eventfilters, + confsToRetrieve, metricsToRetrieve, fields); } } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
