LENS-630: Using Duration and Fact Weight Based Query Cost Calculator for Hive Driver
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/2e5748a8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/2e5748a8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/2e5748a8 Branch: refs/heads/LENS-581 Commit: 2e5748a8c5dcff9627b8d125820371f1d4667d61 Parents: 8ebe7f4 Author: Rajat Khandelwal <[email protected]> Authored: Thu Jul 9 13:05:53 2015 +0530 Committer: Rajat Khandelwal <[email protected]> Committed: Thu Jul 9 13:05:53 2015 +0530 ---------------------------------------------------------------------- .../main/java/org/apache/lens/api/Addable.java | 32 ++++ .../org/apache/lens/api/query/QueryCost.java | 80 ---------- .../apache/lens/api/query/QueryCostType.java | 26 ++++ .../org/apache/lens/api/query/QueryPlan.java | 4 +- .../lens/api/query/QuerySubmitResult.java | 4 +- .../org/apache/lens/api/result/QueryCostTO.java | 75 ++++++++++ .../lens/cube/metadata/FactPartition.java | 16 +- .../apache/lens/cube/metadata/UpdatePeriod.java | 36 +++-- .../FactPartitionBasedQueryCostCalculator.java | 98 ++++++++++++ .../apache/lens/driver/cube/RewriterPlan.java | 2 +- .../lens/cube/metadata/TestFactPartition.java | 57 +++++++ ...stFactPartitionBasedQueryCostCalculator.java | 96 ++++++++++++ .../org/apache/lens/driver/hive/HiveDriver.java | 75 ++++++---- .../apache/lens/driver/hive/HiveQueryPlan.java | 25 +--- .../DurationBasedQueryPriorityDecider.java | 139 ----------------- .../apache/lens/driver/hive/TestHiveDriver.java | 149 +++++++++---------- .../org/apache/lens/driver/jdbc/JDBCDriver.java | 8 +- .../apache/lens/driver/jdbc/TestJdbcDriver.java | 31 ++-- .../regression/core/helpers/QueryHelper.java | 13 +- .../lens/server/api/driver/DriverQueryPlan.java | 12 +- .../lens/server/api/driver/LensDriver.java | 8 +- .../server/api/driver/MinQueryCostSelector.java | 2 +- .../api/priority/CostToPriorityRangeConf.java | 74 --------- .../api/priority/QueryPriorityDecider.java | 39 ----- .../lens/server/api/priority/RangeConf.java | 111 -------------- .../server/api/query/AbstractQueryContext.java | 35 +++-- .../api/query/DriverSelectorQueryContext.java | 10 +- .../server/api/query/QueryExecutionService.java | 1 + .../query/cost/FactPartitionBasedQueryCost.java | 64 ++++++++ .../lens/server/api/query/cost/QueryCost.java | 38 +++++ .../api/query/cost/QueryCostCalculator.java | 32 ++++ .../api/query/cost/QueryCostTOBuilder.java | 50 +++++++ .../priority/CostRangePriorityDecider.java | 42 ++++++ .../query/priority/CostToPriorityRangeConf.java | 74 +++++++++ .../query/priority/QueryPriorityDecider.java | 33 ++++ .../server/api/query/priority/RangeConf.java | 111 ++++++++++++++ .../lens/server/api/driver/MockDriver.java | 9 +- .../lens/server/api/driver/MockFailDriver.java | 2 +- .../cost/TestFactPartitionBasedQueryCost.java | 77 ++++++++++ .../server/query/QueryExecutionServiceImpl.java | 63 ++++---- .../lens/server/query/QueryServiceResource.java | 5 +- .../lens/server/common/RestAPITestUtil.java | 6 +- .../lens/server/query/TestQueryService.java | 18 +-- 43 files changed, 1198 insertions(+), 684 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-api/src/main/java/org/apache/lens/api/Addable.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/Addable.java b/lens-api/src/main/java/org/apache/lens/api/Addable.java new file mode 100644 index 0000000..5a55691 --- /dev/null +++ b/lens-api/src/main/java/org/apache/lens/api/Addable.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * + */ +package org.apache.lens.api; + +public interface Addable<T extends Addable<T>> { + /** + * Add another object of same type object of the same type. + * Implement this if add operation makes sense for the class + * @param other + * @return new T object after logical addition + */ + T add(T other); +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-api/src/main/java/org/apache/lens/api/query/QueryCost.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryCost.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryCost.java deleted file mode 100644 index 1a37c20..0000000 --- a/lens-api/src/main/java/org/apache/lens/api/query/QueryCost.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * - */ -package org.apache.lens.api.query; - -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; - -import lombok.*; - -/** - * The Class QueryCost. - */ -@XmlRootElement -/** - * Instantiates a new query cost. - * - * @param estimatedExecTimeMillis - * the estimated exec time millis - * @param estimatedResourceUsage - * the estimated resource usage - */ -@AllArgsConstructor -/** - * Instantiates a new query cost. - */ -@NoArgsConstructor(access = AccessLevel.PROTECTED) -@ToString -public class QueryCost extends QuerySubmitResult implements Comparable<QueryCost> { - - /** - * The estimated exec time millis. - */ - @XmlElement - @Getter - private long estimatedExecTimeMillis; - - /** - * The estimated resource usage. - */ - @XmlElement - @Getter - private double estimatedResourceUsage; - - /* - * (non-Javadoc) - * - * @see java.lang.Comparable#compareTo(java.lang.Object) - */ - @Override - public int compareTo(QueryCost other) { - if (estimatedExecTimeMillis == other.estimatedExecTimeMillis) { - if (estimatedResourceUsage == other.estimatedResourceUsage) { - return 0; - } else { - return (int) (estimatedResourceUsage - other.estimatedResourceUsage); - } - } else { - return (int) (estimatedExecTimeMillis - other.estimatedExecTimeMillis); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java new file mode 100644 index 0000000..1485e1d --- /dev/null +++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * + */ +package org.apache.lens.api.query; + +public enum QueryCostType { + LOW, MEDIUM, HIGH +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-api/src/main/java/org/apache/lens/api/query/QueryPlan.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryPlan.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryPlan.java index a836b1e..164195b 100644 --- a/lens-api/src/main/java/org/apache/lens/api/query/QueryPlan.java +++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryPlan.java @@ -29,6 +29,8 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.lens.api.result.QueryCostTO; + import lombok.*; /** @@ -110,7 +112,7 @@ public class QueryPlan extends QuerySubmitResult { */ @XmlElement @Getter - private QueryCost queryCost; + private QueryCostTO queryCost; /** * The error. http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-api/src/main/java/org/apache/lens/api/query/QuerySubmitResult.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QuerySubmitResult.java b/lens-api/src/main/java/org/apache/lens/api/query/QuerySubmitResult.java index 2177c78..f3fb6db 100644 --- a/lens-api/src/main/java/org/apache/lens/api/query/QuerySubmitResult.java +++ b/lens-api/src/main/java/org/apache/lens/api/query/QuerySubmitResult.java @@ -24,12 +24,14 @@ package org.apache.lens.api.query; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlSeeAlso; +import org.apache.lens.api.result.QueryCostTO; + /** * The Class QuerySubmitResult. */ @XmlRootElement @XmlSeeAlso({QueryHandle.class, QueryPrepareHandle.class, QueryHandleWithResultSet.class, - org.apache.lens.api.query.QueryPlan.class, QueryCost.class}) + QueryPlan.class, QueryCostTO.class}) public abstract class QuerySubmitResult { } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-api/src/main/java/org/apache/lens/api/result/QueryCostTO.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/result/QueryCostTO.java b/lens-api/src/main/java/org/apache/lens/api/result/QueryCostTO.java new file mode 100644 index 0000000..a247e3c --- /dev/null +++ b/lens-api/src/main/java/org/apache/lens/api/result/QueryCostTO.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * + */ +package org.apache.lens.api.result; + +import java.io.Serializable; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.lens.api.query.QueryCostType; +import org.apache.lens.api.query.QuerySubmitResult; + +import lombok.*; + +/** + * The Class QueryCostTO. + */ +@XmlRootElement +/** + * Instantiates a new query cost. + * + * @param estimatedExecTimeMillis + * the estimated exec time millis + * @param estimatedResourceUsage + * the estimated resource usage + */ +@AllArgsConstructor +/** + * Instantiates a new query cost. + */ +@NoArgsConstructor(access = AccessLevel.PROTECTED) +@ToString +public class QueryCostTO extends QuerySubmitResult implements Serializable { + + /** + * The estimated exec time millis. + */ + @XmlElement + @Getter + private Long estimatedExecTimeMillis; + + /** + * The estimated resource usage. + */ + @XmlElement + @Getter + private Double estimatedResourceUsage; + + /** + * Normalized query cost, calculated by driver + */ + @XmlElement + @Getter + private QueryCostType type; + +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java index 3465513..fc2d85b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java @@ -20,12 +20,11 @@ package org.apache.lens.cube.metadata; import java.text.DateFormat; -import java.util.Date; -import java.util.LinkedHashSet; -import java.util.Set; +import java.util.*; import org.apache.lens.server.api.error.LensException; +import com.google.common.collect.ImmutableMap; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; @@ -166,4 +165,15 @@ public class FactPartition implements Comparable<FactPartition> { public TimePartition getTimePartition() throws LensException { return TimePartition.of(getPeriod(), getPartSpec()); } + + public double getAllTableWeights(ImmutableMap<String, Double> tableWeights) { + float weight = 0; + for (String tblName : getStorageTables()) { + Double tblWeight = tableWeights.get(tblName); + if (tblWeight != null) { + weight += tblWeight; + } + } + return weight; + } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java index 3c9268b..966dbdf 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java @@ -19,24 +19,35 @@ package org.apache.lens.cube.metadata; +import static java.util.Calendar.*; + import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.util.Calendar; import java.util.Comparator; import java.util.Date; import org.apache.lens.cube.parse.DateUtil; public enum UpdatePeriod implements Named { - SECONDLY(Calendar.SECOND, 1000, "yyyy-MM-dd-HH-mm-ss"), MINUTELY(Calendar.MINUTE, 60 * SECONDLY.weight(), - "yyyy-MM-dd-HH-mm"), HOURLY(Calendar.HOUR_OF_DAY, 60 * MINUTELY.weight(), "yyyy-MM-dd-HH"), DAILY( - Calendar.DAY_OF_MONTH, 24 * HOURLY.weight(), "yyyy-MM-dd"), WEEKLY(Calendar.WEEK_OF_YEAR, 7 * DAILY.weight(), - "yyyy-'W'ww"), MONTHLY(Calendar.MONTH, 30 * DAILY.weight(), "yyyy-MM"), QUARTERLY(Calendar.MONTH, 3 * MONTHLY - .weight(), "yyyy-MM"), YEARLY(Calendar.YEAR, 12 * MONTHLY.weight(), "yyyy"); - - public static final long MIN_INTERVAL = SECONDLY.weight(); + SECONDLY(SECOND, 1000, 1.4f, "yyyy-MM-dd-HH-mm-ss"), + MINUTELY(MINUTE, 60 * SECONDLY.weight(), 1.35f, "yyyy-MM-dd-HH-mm"), + HOURLY(HOUR_OF_DAY, 60 * MINUTELY.weight(), 1.3f, "yyyy-MM-dd-HH"), + DAILY(DAY_OF_MONTH, 24 * HOURLY.weight(), 1f, "yyyy-MM-dd"), + WEEKLY(WEEK_OF_YEAR, 7 * DAILY.weight(), 0.7f, "yyyy-'W'ww"), + MONTHLY(MONTH, 30 * DAILY.weight(), 0.6f, "yyyy-MM"), + QUARTERLY(MONTH, 3 * MONTHLY.weight(), 0.55f, "yyyy-MM"), + YEARLY(YEAR, 12 * MONTHLY.weight(), 0.52f, "yyyy"); + + public static final long MIN_INTERVAL = values()[0].weight(); private final int calendarField; private final long weight; + /** + * Normalization factor is calculated in comparison with daily update period. What it means is that + * for a fixed time range, reading partitions of this update period is expensive/cheap as compared to + * reading partitions of daily update period by this factor. Values are tentatively picked based on + * similar logic in an existing system at InMobi. + */ + private final float normalizationFactor; private final String format; private static DateFormat getSecondlyFormat() { @@ -144,9 +155,10 @@ public enum UpdatePeriod implements Named { private static ThreadLocal<DateFormat> quarterlyFormat; private static ThreadLocal<DateFormat> yearlyFormat; - UpdatePeriod(int calendarField, long diff, String format) { + UpdatePeriod(int calendarField, long diff, float normalizationFactor, String format) { this.calendarField = calendarField; this.weight = diff; + this.normalizationFactor = normalizationFactor; this.format = format; } @@ -181,7 +193,7 @@ public enum UpdatePeriod implements Named { case YEARLY: return getYearlyFormat(); default: - return null; + throw new IllegalArgumentException("Update period illegal, or doesn't have defined format"); } } @@ -198,6 +210,10 @@ public enum UpdatePeriod implements Named { return formatStr().replaceAll("'", "").length() == dateString.length(); } + public float getNormalizationFactor() { + return normalizationFactor; + } + public static class UpdatePeriodComparator implements Comparator<UpdatePeriod> { @Override public int compare(UpdatePeriod o1, UpdatePeriod o2) { http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java new file mode 100644 index 0000000..d56e1c7 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.query.cost; + +import java.util.Map; +import java.util.Set; + +import org.apache.lens.cube.metadata.FactPartition; +import org.apache.lens.cube.metadata.UpdatePeriod; +import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.AbstractQueryContext; +import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost; +import org.apache.lens.server.api.query.cost.QueryCost; +import org.apache.lens.server.api.query.cost.QueryCostCalculator; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; + +public class FactPartitionBasedQueryCostCalculator implements QueryCostCalculator { + + public static final String UPDATE_PERIOD_WEIGHT_PREFIX = "update.period.weight."; + + /** + * Calculates total cost based on weights of selected tables and their selected partitions + * + * @param queryContext + * @param driver + * @return Query Cost + * @throws LensException + */ + + @SuppressWarnings("unchecked") // required for (Set<FactPartition>) casting + private double getTotalPartitionCost(final AbstractQueryContext queryContext, LensDriver driver) + throws LensException { + double cost = 0; + for (Map.Entry<String, Set<?>> entry : getAllPartitions(queryContext, driver).entrySet()) { + // Have to do instanceof check, since it can't be handled by polymorphism. + // The '?' is either a FactPartition or a String. When we decide to write a + // DimtablePartition, we can probably think about using polymorphism. + if (!entry.getValue().isEmpty() && entry.getValue().iterator().next() instanceof FactPartition) { + Set<FactPartition> factParts = (Set<FactPartition>) entry.getValue(); + for (FactPartition partition : factParts) { + double allTableWeights = + partition.getAllTableWeights(ImmutableMap.copyOf(queryContext.getTableWeights(driver))); + if (allTableWeights == 0) { + allTableWeights = 1; + } + cost += allTableWeights * getNormalizedUpdatePeriodCost(partition.getPeriod(), driver); + } + } + } + return cost; + } + + /** + * Normalized cost of an update period. update period weight multiplied by number of partitions of that update period. + * + * @param updatePeriod + * @param driver + * @return normalized cost. + * @throws LensException + */ + private double getNormalizedUpdatePeriodCost(final UpdatePeriod updatePeriod, LensDriver driver) + throws LensException { + double weight = driver.getConf().getDouble(UPDATE_PERIOD_WEIGHT_PREFIX + updatePeriod.name().toLowerCase(), + updatePeriod.getNormalizationFactor()); + return weight * updatePeriod.weight() / UpdatePeriod.DAILY.weight(); + } + + @Override + public QueryCost calculateCost(final AbstractQueryContext queryContext, LensDriver driver) throws LensException { + return new FactPartitionBasedQueryCost(getTotalPartitionCost(queryContext, driver)); + } + + public Map<String, Set<?>> getAllPartitions(AbstractQueryContext queryContext, LensDriver driver) { + if (queryContext.getDriverRewriterPlan(driver) != null) { + return queryContext.getDriverRewriterPlan(driver).getPartitions(); + } + return Maps.newHashMap(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java index 841946d..7725043 100644 --- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java +++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java @@ -22,12 +22,12 @@ import java.util.Collection; import java.util.HashSet; import java.util.Set; -import org.apache.lens.api.query.QueryCost; import org.apache.lens.cube.metadata.FactPartition; import org.apache.lens.cube.parse.CandidateTable; import org.apache.lens.cube.parse.CubeQueryContext; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.driver.DriverQueryPlan; +import org.apache.lens.server.api.query.cost.QueryCost; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestFactPartition.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestFactPartition.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestFactPartition.java new file mode 100644 index 0000000..3f2ae6f --- /dev/null +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestFactPartition.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.metadata; + +import static org.apache.lens.cube.metadata.UpdatePeriod.DAILY; +import static org.apache.lens.cube.metadata.UpdatePeriod.HOURLY; + +import static org.testng.Assert.assertEquals; + +import java.util.Date; +import java.util.Map; + +import org.testng.annotations.Test; + +import com.beust.jcommander.internal.Maps; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; + +public class TestFactPartition { + public static final Date DATE = new Date(); + FactPartition fp1 = new FactPartition("p", DATE, DAILY, null, null, Sets.newHashSet("st1", "st2")); + FactPartition fp2 = new FactPartition("q", DATE, HOURLY, fp1, null, Sets.newHashSet("st3", "st4")); + + @Test + public void testGetFormattedFilter() throws Exception { + String dailyFormat = DAILY.format().format(DATE); + String hourlyFormat = HOURLY.format().format(DATE); + assertEquals(fp1.getFormattedFilter("table"), "table.p = '" + dailyFormat + "'"); + assertEquals(fp2.getFormattedFilter("table2"), + "table2.p = '" + dailyFormat + "' AND table2.q = '" + hourlyFormat + "'"); + } + + @Test + public void testGetAllTableWeights() throws Exception { + Map<String, Double> weights = Maps.newHashMap(); + assertEquals(fp1.getAllTableWeights(ImmutableMap.copyOf(weights)), 0.0); + weights.put("st1", 0.2); + weights.put("st2", 0.3); + assertEquals(fp1.getAllTableWeights(ImmutableMap.copyOf(weights)), 0.5); + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java b/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java new file mode 100644 index 0000000..8a60224 --- /dev/null +++ b/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * + */ +package org.apache.lens.cube.query.cost; + +import static org.apache.lens.cube.metadata.UpdatePeriod.*; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.HashMap; +import java.util.Set; + +import org.apache.lens.cube.metadata.FactPartition; +import org.apache.lens.cube.metadata.UpdatePeriod; +import org.apache.lens.server.api.driver.DriverQueryPlan; +import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.query.AbstractQueryContext; +import org.apache.lens.server.api.query.cost.QueryCost; + +import org.apache.hadoop.conf.Configuration; + +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import junit.framework.Assert; + +public class TestFactPartitionBasedQueryCostCalculator { + AbstractQueryContext queryContext; + FactPartitionBasedQueryCostCalculator calculator = new FactPartitionBasedQueryCostCalculator(); + LensDriver driver; + + @BeforeTest + public void setUp() { + driver = mock(LensDriver.class); + when(driver.getConf()).thenReturn(new Configuration()); + queryContext = mock(AbstractQueryContext.class); + + ImmutableMap<String, Double> tableWeights = mock(ImmutableMap.class); + + FactPartition fp1 = mockFactPartition(DAILY, tableWeights, 0.7); + FactPartition fp2 = mockFactPartition(HOURLY, tableWeights, 0.8); + FactPartition fp3 = mockFactPartition(SECONDLY, tableWeights, 0.4); + FactPartition fp4 = mockFactPartition(MONTHLY, tableWeights, 0); + + when(queryContext.getTableWeights(driver)).thenReturn(tableWeights); + + HashMap<String, Set<?>> partitions = new HashMap<>(); + partitions.put("st1", Sets.newHashSet(fp1, fp2)); + partitions.put("st2", Sets.newHashSet(fp3, fp4)); + DriverQueryPlan plan = mock(DriverQueryPlan.class); + when(queryContext.getDriverRewriterPlan(driver)).thenReturn(plan); + when(plan.getPartitions()).thenReturn(partitions); + when(calculator.getAllPartitions(queryContext, driver)).thenReturn(partitions); + } + + private FactPartition mockFactPartition(UpdatePeriod mockPeriod, ImmutableMap<String, Double> tableWeights, + double mockAllTableWeight) { + FactPartition fp = mock(FactPartition.class); + when(fp.getPeriod()).thenReturn(mockPeriod); + when(fp.getAllTableWeights(tableWeights)).thenReturn(mockAllTableWeight); + return fp; + } + + @Test + public void testCalculateCost() throws Exception { + QueryCost cost = calculator.calculateCost(queryContext, driver); + Assert.assertTrue(cost.getEstimatedResourceUsage() > 18.0); + Assert.assertTrue(cost.getEstimatedResourceUsage() < 19.0); + } + + @Test + public void testGetAllPartitions() throws Exception { + + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java index b7a3be7..e87ed23 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java @@ -27,19 +27,23 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.lens.api.LensConf; import org.apache.lens.api.LensSessionHandle; -import org.apache.lens.api.query.QueryCost; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.api.query.QueryPrepareHandle; -import org.apache.lens.driver.hive.priority.DurationBasedQueryPriorityDecider; +import org.apache.lens.cube.query.cost.FactPartitionBasedQueryCostCalculator; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.driver.*; import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.events.LensEventListener; -import org.apache.lens.server.api.priority.QueryPriorityDecider; import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost; +import org.apache.lens.server.api.query.cost.QueryCost; +import org.apache.lens.server.api.query.cost.QueryCostCalculator; +import org.apache.lens.server.api.query.priority.CostRangePriorityDecider; +import org.apache.lens.server.api.query.priority.CostToPriorityRangeConf; +import org.apache.lens.server.api.query.priority.QueryPriorityDecider; import org.apache.lens.server.api.user.UserConfigLoader; import org.apache.commons.lang.StringUtils; @@ -72,21 +76,16 @@ public class HiveDriver implements LensDriver { public static final String HS2_CONNECTION_EXPIRY_DELAY = "lens.driver.hive.hs2.connection.expiry.delay"; public static final String HS2_CALCULATE_PRIORITY = "lens.driver.hive.calculate.priority"; + public static final String HS2_COST_CALCULATOR = "lens.driver.hive.cost.calculator.class"; /** * Config param for defining priority ranges. */ public static final String HS2_PRIORITY_RANGES = "lens.driver.hive.priority.ranges"; - public static final String HS2_PARTITION_WEIGHT_MONTHLY = "lens.driver.hive.priority.partition.weight.monthly"; - public static final String HS2_PARTITION_WEIGHT_DAILY = "lens.driver.hive.priority.partition.weight.daily"; - public static final String HS2_PARTITION_WEIGHT_HOURLY = "lens.driver.hive.priority.partition.weight.hourly"; // Default values of conf params public static final long DEFAULT_EXPIRY_DELAY = 600 * 1000; public static final String HS2_PRIORITY_DEFAULT_RANGES = "VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW"; - public static final float MONTHLY_PARTITION_WEIGHT_DEFAULT = 0.5f; - public static final float DAILY_PARTITION_WEIGHT_DEFAULT = 0.75f; - public static final float HOURLY_PARTITION_WEIGHT_DEFAULT = 1.0f; public static final String SESSION_KEY_DELIMITER = "."; /** The driver conf- which will merged with query conf */ @@ -123,8 +122,9 @@ public class HiveDriver implements LensDriver { /** The driver listeners. */ private List<LensEventListener<DriverEvent>> driverListeners; - QueryPriorityDecider queryPriorityDecider; + QueryCostCalculator queryCostCalculator; + QueryPriorityDecider queryPriorityDecider; // package-local. Test case can change. boolean whetherCalculatePriority; private UserConfigLoader userConfigLoader; @@ -331,15 +331,25 @@ public class HiveDriver implements LensDriver { isEmbedded = (connectionClass.getName().equals(EmbeddedThriftConnection.class.getName())); connectionExpiryTimeout = this.driverConf.getLong(HS2_CONNECTION_EXPIRY_DELAY, DEFAULT_EXPIRY_DELAY); whetherCalculatePriority = this.driverConf.getBoolean(HS2_CALCULATE_PRIORITY, true); - queryPriorityDecider = new DurationBasedQueryPriorityDecider( - this, - this.driverConf.get(HS2_PRIORITY_RANGES, HS2_PRIORITY_DEFAULT_RANGES), - this.driverConf.getFloat(HS2_PARTITION_WEIGHT_MONTHLY, MONTHLY_PARTITION_WEIGHT_DEFAULT), - this.driverConf.getFloat(HS2_PARTITION_WEIGHT_DAILY, DAILY_PARTITION_WEIGHT_DEFAULT), - this.driverConf.getFloat(HS2_PARTITION_WEIGHT_HOURLY, HOURLY_PARTITION_WEIGHT_DEFAULT) + Class<? extends QueryCostCalculator> queryCostCalculatorClass = this.driverConf.getClass(HS2_COST_CALCULATOR, + FactPartitionBasedQueryCostCalculator.class, QueryCostCalculator.class); + try { + queryCostCalculator = queryCostCalculatorClass.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new LensException("Can't instantiate query cost calculator of class: " + queryCostCalculatorClass, e); + } + queryPriorityDecider = new CostRangePriorityDecider( + new CostToPriorityRangeConf(driverConf.get(HS2_PRIORITY_RANGES, HS2_PRIORITY_DEFAULT_RANGES)) ); } + private QueryCost calculateQueryCost(AbstractQueryContext qctx) throws LensException { + if (qctx.isOlapQuery()) { + return queryCostCalculator.calculateCost(qctx, this); + } else { + return new FactPartitionBasedQueryCost(Double.MAX_VALUE); + } + } @Override public QueryCost estimate(AbstractQueryContext qctx) throws LensException { log.info("Estimate: " + qctx.getDriverQuery(this)); @@ -352,14 +362,17 @@ public class HiveDriver implements LensDriver { } if (qctx.isOlapQuery()) { // if query is olap query and rewriting takes care of semantic validation - // estimate is not doing anything as of now - return HiveQueryPlan.HIVE_DRIVER_COST; + // estimate is calculating cost of the query + // the calculation is done only for cube queries + // for all other native table queries, the cost will be maximum + return calculateQueryCost(qctx); } else { - // its native table query. Do explain and return cost + // its native table query. validate and return cost return explain(qctx).getCost(); } } + /* * (non-Javadoc) * @@ -385,14 +398,14 @@ public class HiveDriver implements LensDriver { // Get result set of explain HiveInMemoryResultSet inMemoryResultSet = (HiveInMemoryResultSet) execute(explainQueryCtx); - List<String> explainOutput = new ArrayList<String>(); + List<String> explainOutput = new ArrayList<>(); while (inMemoryResultSet.hasNext()) { explainOutput.add((String) inMemoryResultSet.next().getValues().get(0)); } closeQuery(explainQueryCtx.getQueryHandle()); try { hiveConf.setClassLoader(explainCtx.getConf().getClassLoader()); - HiveQueryPlan hqp = new HiveQueryPlan(explainOutput, null, hiveConf); + HiveQueryPlan hqp = new HiveQueryPlan(explainOutput, null, hiveConf, calculateQueryCost(explainCtx)); explainCtx.getDriverContext().setDriverQueryPlan(this, hqp); return hqp; } catch (HiveException e) { @@ -494,7 +507,10 @@ public class HiveDriver implements LensDriver { if (whetherCalculatePriority) { try { // Inside try since non-data fetching queries can also be executed by async method. - String priority = queryPriorityDecider.decidePriority(ctx).toString(); + if (ctx.getDriverQueryCost(this) == null) { + ctx.setDriverCost(this, queryCostCalculator.calculateCost(ctx, this)); + } + String priority = queryPriorityDecider.decidePriority(ctx.getDriverQueryCost(this)).toString(); qdconf.set("mapred.job.priority", priority); log.info("set priority to " + priority); } catch (Exception e) { @@ -942,7 +958,7 @@ public class HiveDriver implements LensDriver { */ QueryCompletionNotifier(QueryHandle handle, long timeoutMillis, QueryCompletionListener listener) throws LensException { - hiveHandle = getHiveHandle(handle); + this.handle = handle; this.timeoutMillis = timeoutMillis; this.listener = listener; this.pollInterval = timeoutMillis / 10; @@ -957,12 +973,17 @@ public class HiveDriver implements LensDriver { public void run() { // till query is complete or timeout has reached long timeSpent = 0; - String error = null; + String error; try { while (timeSpent <= timeoutMillis) { - if (isFinished(hiveHandle)) { - listener.onCompletion(handle); - return; + try { + hiveHandle = getHiveHandle(handle); + if (isFinished(hiveHandle)) { + listener.onCompletion(handle); + return; + } + } catch(LensException e) { + log.debug("query handle: {} Not yet launched on driver", handle); } Thread.sleep(pollInterval); timeSpent += pollInterval; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java index b0a04df..f38b1da 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java @@ -18,15 +18,12 @@ */ package org.apache.lens.driver.hive; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; -import org.apache.lens.api.query.QueryCost; import org.apache.lens.api.query.QueryPrepareHandle; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.driver.DriverQueryPlan; +import org.apache.lens.server.api.query.cost.QueryCost; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.conf.HiveConf; @@ -34,6 +31,7 @@ import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; /** @@ -42,11 +40,11 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class HiveQueryPlan extends DriverQueryPlan { + @Getter + private final QueryCost cost; /** The explain output. */ private String explainOutput; - static final QueryCost HIVE_DRIVER_COST = new QueryCost(1, 1.0); - /** * The Enum ParserState. */ @@ -100,8 +98,9 @@ public class HiveQueryPlan extends DriverQueryPlan { * @param metastoreConf the metastore conf * @throws HiveException the hive exception */ - public HiveQueryPlan(List<String> explainOutput, QueryPrepareHandle prepared, HiveConf metastoreConf) + public HiveQueryPlan(List<String> explainOutput, QueryPrepareHandle prepared, HiveConf metastoreConf, QueryCost cost) throws HiveException { + this.cost = cost; setPrepareHandle(prepared); setExecMode(ExecMode.BATCH); setScanMode(ScanMode.PARTIAL_SCAN); @@ -196,7 +195,7 @@ public class HiveQueryPlan extends DriverQueryPlan { } } break; - default : + default: break; } } @@ -242,12 +241,4 @@ public class HiveQueryPlan extends DriverQueryPlan { public String getPlan() { return explainOutput; } - - @Override - public QueryCost getCost() { - /* - * Return query cost as 1 so that if JDBC storage and other storage is present, JDBC is given preference. - */ - return HIVE_DRIVER_COST; - } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java deleted file mode 100644 index bc1ec1c..0000000 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java +++ /dev/null @@ -1,139 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.driver.hive.priority; - -import java.util.Map; -import java.util.Set; - -import org.apache.lens.api.Priority; -import org.apache.lens.cube.metadata.FactPartition; -import org.apache.lens.cube.metadata.UpdatePeriod; -import org.apache.lens.server.api.driver.LensDriver; -import org.apache.lens.server.api.error.LensException; -import org.apache.lens.server.api.priority.CostToPriorityRangeConf; -import org.apache.lens.server.api.priority.QueryPriorityDecider; -import org.apache.lens.server.api.query.AbstractQueryContext; - -public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider { - - - private final CostToPriorityRangeConf costToPriorityRangeMap; - - /** Partition Weights for priority calculation based on selected partitions **/ - - /** weight of monthly partition * */ - private final float monthlyPartitionWeight; - /** weight of daily partition * */ - private final float dailyPartitionWeight; - /** weight of hourly partition * */ - private final float hourlyPartitionWeight; - private final LensDriver driver; - - - /** - * Constructor. Takes three weights for partitions. - * @param driver - * @param ranges - * @param monthlyPartitoinWeight - * @param dailyPartitionWeight - * @param hourlyPartitionWeight - */ - public DurationBasedQueryPriorityDecider(LensDriver driver, - String ranges, float monthlyPartitoinWeight, float dailyPartitionWeight, float hourlyPartitionWeight) { - this.driver = driver; - this.costToPriorityRangeMap = new CostToPriorityRangeConf(ranges); - this.monthlyPartitionWeight = monthlyPartitoinWeight; - this.dailyPartitionWeight = dailyPartitionWeight; - this.hourlyPartitionWeight = hourlyPartitionWeight; - } - - /** - * The Implementation - * - * @param abstractQueryContext - * @return decided Priority - * @throws LensException Exception occurs mostly when one of drivers/explained queries/plans is null - */ - public Priority decidePriority(AbstractQueryContext abstractQueryContext) throws LensException { - float cost = getDurationCost(abstractQueryContext); - Priority priority = costToPriorityRangeMap.get(cost); - LOG.info("Deciding Priority " + priority + " since cost = " + cost); - return priority; - } - - /** - * Calculates total cost based on weights of selected tables and their selected partitions - * - * @param queryContext - * @return Query Cost - * @throws LensException - */ - - @SuppressWarnings("unchecked") // required for (Set<FactPartition>) casting - private float getDurationCost(AbstractQueryContext queryContext) throws LensException { - float cost = 0; - if (queryContext.getDriverContext().getDriverRewriterPlan(driver) != null) { - // the calculation is done only for cube queries involving fact tables - // for all other native table queries and dimension only queries, the cost will be zero and priority will - // be the highest one associated with zero cost - for (Map.Entry<String, Set<?>> entry : queryContext.getDriverContext().getDriverRewriterPlan(driver) - .getPartitions().entrySet()) { - if (!entry.getValue().isEmpty() && entry.getValue().iterator().next() instanceof FactPartition) { - Set<FactPartition> factParts = (Set<FactPartition>)entry.getValue(); - for (FactPartition partition : factParts) { - cost += getTableWeights(partition.getStorageTables(), queryContext) * getNormalizedPartitionCost( - partition.getPeriod()); - } - } - } - } - return cost; - } - - private float getTableWeights(Set<String> tables, AbstractQueryContext queryContext) { - float weight = 0; - for (String tblName : tables) { - Double tblWeight = queryContext.getDriverContext().getDriverRewriterPlan(driver).getTableWeight(tblName); - if (tblWeight != null) { - weight += tblWeight; - } - } - return weight == 0 ? 1 : weight; - } - - /** - * Normalized cost of a partition. PartitionWeight multiplied by number of days in that partition. - * - * @param partition - * @return normalized cost. - * @throws LensException - */ - private float getNormalizedPartitionCost(UpdatePeriod updatePeriod) throws LensException { - switch (updatePeriod) { - case MONTHLY: //monthly - return 30 * monthlyPartitionWeight; - case DAILY: // daily - return 1 * dailyPartitionWeight; - case HOURLY: // hourly - return (1.0f / 24) * hourlyPartitionWeight; - default: - throw new LensException("Weight not defined for " + updatePeriod); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java index 8355f29..67a4667 100644 --- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java +++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java @@ -18,7 +18,8 @@ */ package org.apache.lens.driver.hive; -import static org.apache.lens.server.api.user.MockUserConfigLoader.*; +import static org.apache.lens.server.api.user.MockUserConfigLoader.KEY; +import static org.apache.lens.server.api.user.MockUserConfigLoader.VALUE; import static org.testng.Assert.*; @@ -28,30 +29,23 @@ import java.util.*; import org.apache.lens.api.LensConf; import org.apache.lens.api.Priority; -import org.apache.lens.api.query.QueryCost; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.cube.metadata.FactPartition; import org.apache.lens.cube.metadata.UpdatePeriod; -import org.apache.lens.driver.hive.priority.DurationBasedQueryPriorityDecider; import org.apache.lens.server.api.LensConfConstants; -import org.apache.lens.server.api.driver.DriverQueryPlan; +import org.apache.lens.server.api.driver.*; import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState; -import org.apache.lens.server.api.driver.LensDriver; -import org.apache.lens.server.api.driver.LensResultSet; -import org.apache.lens.server.api.driver.LensResultSetMetadata; import org.apache.lens.server.api.error.LensException; -import org.apache.lens.server.api.query.AbstractQueryContext; -import org.apache.lens.server.api.query.ExplainQueryContext; -import org.apache.lens.server.api.query.PreparedQueryContext; -import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.query.*; +import org.apache.lens.server.api.query.cost.QueryCost; +import org.apache.lens.server.api.query.priority.CostRangePriorityDecider; +import org.apache.lens.server.api.query.priority.CostToPriorityRangeConf; import org.apache.lens.server.api.user.MockUserConfigLoader; import org.apache.lens.server.api.util.LensUtil; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.ql.HiveDriverRunHook; @@ -60,12 +54,8 @@ import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.service.cli.ColumnDescriptor; -import org.testng.annotations.AfterTest; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Test; +import org.testng.annotations.*; -import junit.framework.Assert; /** * The Class TestHiveDriver. @@ -165,7 +155,7 @@ public class TestHiveDriver { protected ExplainQueryContext createExplainContext(final String query, Configuration conf) { ExplainQueryContext ectx = new ExplainQueryContext(UUID.randomUUID().toString(), query, "testuser", null, conf, - drivers); + drivers); ectx.setLensSessionIdentifier(sessionid); return ectx; } @@ -201,7 +191,7 @@ public class TestHiveDriver { context = createContext(dataLoad, conf); resultSet = driver.execute(context); assertNull(resultSet); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); } /** @@ -227,7 +217,7 @@ public class TestHiveDriver { context = createContext(dataLoad, conf); resultSet = driver.execute(context); assertNull(resultSet); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); } // Tests @@ -263,15 +253,15 @@ public class TestHiveDriver { QueryContext context = createContext(query, conf); LensResultSet resultSet = driver.execute(context); assertNull(resultSet); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); // fetch results from temp table String select = "SELECT * FROM test_temp_output"; context = createContext(select, conf); resultSet = driver.execute(context); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); validateInMemoryResult(resultSet, "test_temp_output"); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); } /** @@ -288,7 +278,7 @@ public class TestHiveDriver { String select = "SELECT ID FROM test_execute"; QueryContext context = createContext(select, conf); resultSet = driver.execute(context); - Assert.assertNotNull(context.getDriverConf(driver).get("mapred.job.name")); + assertNotNull(context.getDriverConf(driver).get("mapred.job.name")); validateInMemoryResult(resultSet); conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true); context = createContext(select, conf); @@ -302,7 +292,7 @@ public class TestHiveDriver { context = createContext(select, conf); resultSet = driver.execute(context); validatePersistentResult(resultSet, TEST_DATA_FILE, context.getHDFSResultDir(), true); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); } /** @@ -407,30 +397,30 @@ public class TestHiveDriver { failConf.set("hive.exec.driver.run.hooks", FailHook.class.getCanonicalName()); QueryContext context = createContext(expectFail, failConf); driver.executeAsync(context); - Assert.assertEquals(1, driver.getHiveHandleSize()); + assertEquals(1, driver.getHiveHandleSize()); validateExecuteAsync(context, DriverQueryState.FAILED, true, false); - Assert.assertEquals(1, driver.getHiveHandleSize()); + assertEquals(1, driver.getHiveHandleSize()); driver.closeQuery(context.getQueryHandle()); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); // Async select query String select = "SELECT ID FROM test_execute_sync"; conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false); context = createContext(select, conf); driver.executeAsync(context); - Assert.assertNotNull(context.getDriverConf(driver).get("mapred.job.name")); - Assert.assertNotNull(context.getDriverConf(driver).get("mapred.job.priority")); - Assert.assertEquals(1, driver.getHiveHandleSize()); + assertNotNull(context.getDriverConf(driver).get("mapred.job.name")); + assertNotNull(context.getDriverConf(driver).get("mapred.job.priority")); + assertEquals(1, driver.getHiveHandleSize()); validateExecuteAsync(context, DriverQueryState.SUCCESSFUL, false, false); driver.closeQuery(context.getQueryHandle()); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true); context = createContext(select, conf); driver.executeAsync(context); - Assert.assertEquals(1, driver.getHiveHandleSize()); + assertEquals(1, driver.getHiveHandleSize()); validateExecuteAsync(context, DriverQueryState.SUCCESSFUL, true, false); driver.closeQuery(context.getQueryHandle()); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); conf.set(LensConfConstants.QUERY_OUTPUT_DIRECTORY_FORMAT, "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'" @@ -439,10 +429,10 @@ public class TestHiveDriver { select = "SELECT ID, null, ID FROM test_execute_sync"; context = createContext(select, conf); driver.executeAsync(context); - Assert.assertEquals(1, driver.getHiveHandleSize()); + assertEquals(1, driver.getHiveHandleSize()); validateExecuteAsync(context, DriverQueryState.SUCCESSFUL, true, true); driver.closeQuery(context.getQueryHandle()); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); } /** @@ -506,7 +496,7 @@ public class TestHiveDriver { driver.updateStatus(context); assertEquals(context.getDriverStatus().getState(), DriverQueryState.CANCELED, "Expecting query to be cancelled"); driver.closeQuery(context.getQueryHandle()); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); try { driver.cancelQuery(context.getQueryHandle()); @@ -589,14 +579,14 @@ public class TestHiveDriver { QueryContext ctx = createContext("SELECT ID FROM test_persistent_result_set", conf); LensResultSet resultSet = driver.execute(ctx); validatePersistentResult(resultSet, TEST_DATA_FILE, ctx.getHDFSResultDir(), false); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); ctx = createContext("SELECT ID FROM test_persistent_result_set", conf); driver.executeAsync(ctx); - Assert.assertEquals(1, driver.getHiveHandleSize()); + assertEquals(1, driver.getHiveHandleSize()); validateExecuteAsync(ctx, DriverQueryState.SUCCESSFUL, true, false); driver.closeQuery(ctx.getQueryHandle()); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); conf.set(LensConfConstants.QUERY_OUTPUT_DIRECTORY_FORMAT, "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'" @@ -604,17 +594,17 @@ public class TestHiveDriver { + " 'field.delim'=',' ) STORED AS TEXTFILE "); ctx = createContext("SELECT ID, null, ID FROM test_persistent_result_set", conf); resultSet = driver.execute(ctx); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); validatePersistentResult(resultSet, TEST_DATA_FILE, ctx.getHDFSResultDir(), true); driver.closeQuery(ctx.getQueryHandle()); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); ctx = createContext("SELECT ID, null, ID FROM test_persistent_result_set", conf); driver.executeAsync(ctx); - Assert.assertEquals(1, driver.getHiveHandleSize()); + assertEquals(1, driver.getHiveHandleSize()); validateExecuteAsync(ctx, DriverQueryState.SUCCESSFUL, true, true); driver.closeQuery(ctx.getQueryHandle()); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); } /** @@ -640,38 +630,40 @@ public class TestHiveDriver { } } - @Test + @Test(expectedExceptions = {UnsupportedOperationException.class}) public void testEstimateNativeQuery() throws Exception { createTestTable("test_estimate"); SessionState.setCurrentSessionState(ss); QueryCost cost = driver.estimate(createExplainContext("SELECT ID FROM test_estimate", conf)); - Assert.assertEquals(cost.getEstimatedExecTimeMillis(), 1); - Assert.assertEquals(cost.getEstimatedResourceUsage(), 1.0); + assertEquals(cost.getEstimatedResourceUsage(), Double.MAX_VALUE); + cost.getEstimatedExecTimeMillis(); + } - @Test + @Test(expectedExceptions = {UnsupportedOperationException.class}) public void testEstimateOlapQuery() throws Exception { SessionState.setCurrentSessionState(ss); ExplainQueryContext ctx = createExplainContext("cube SELECT ID FROM test_cube", conf); ctx.setOlapQuery(true); QueryCost cost = driver.estimate(ctx); - Assert.assertEquals(cost.getEstimatedExecTimeMillis(), 1); - Assert.assertEquals(cost.getEstimatedResourceUsage(), 1.0); + assertEquals(cost.getEstimatedResourceUsage(), 0.0); + cost.getEstimatedExecTimeMillis(); } @Test - public void testEstimateNativeFailingQuery() throws Exception { + public void testExplainNativeFailingQuery() throws Exception { SessionState.setCurrentSessionState(ss); try { driver.estimate(createExplainContext("SELECT ID FROM nonexist", conf)); - Assert.fail("Should not reach here"); + fail("Should not reach here"); } catch (LensException e) { - Assert.assertEquals(LensUtil.getCauseMessage(e), "Error while" + assertEquals(LensUtil.getCauseMessage(e), "Error while" + " compiling statement: FAILED: SemanticException [Error 10001]: Line 1:32 Table not found 'nonexist'"); } } // explain + /** * Test explain. * @@ -686,7 +678,7 @@ public class TestHiveDriver { DriverQueryPlan plan = driver.explain(createExplainContext("SELECT ID FROM test_explain", conf)); assertTrue(plan instanceof HiveQueryPlan); assertEquals(plan.getTableWeight(dataBase + ".test_explain"), 500.0); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); // test execute prepare PreparedQueryContext pctx = new PreparedQueryContext("SELECT ID FROM test_explain", null, conf, drivers); @@ -699,7 +691,7 @@ public class TestHiveDriver { plan = driver.explainAndPrepare(pctx); QueryContext qctx = createContext(pctx, inConf); LensResultSet result = driver.execute(qctx); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); validateInMemoryResult(result); // test execute prepare async @@ -707,28 +699,28 @@ public class TestHiveDriver { driver.executeAsync(qctx); assertNotNull(qctx.getDriverOpHandle()); validateExecuteAsync(qctx, DriverQueryState.SUCCESSFUL, true, false); - Assert.assertEquals(1, driver.getHiveHandleSize()); + assertEquals(1, driver.getHiveHandleSize()); driver.closeQuery(qctx.getQueryHandle()); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); // for backward compatibility qctx = createContext(pctx, inConf); qctx.setQueryHandle(new QueryHandle(pctx.getPrepareHandle().getPrepareHandleId())); result = driver.execute(qctx); assertNotNull(qctx.getDriverOpHandle()); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); validateInMemoryResult(result); // test execute prepare async qctx = createContext(pctx, conf); qctx.setQueryHandle(new QueryHandle(pctx.getPrepareHandle().getPrepareHandleId())); driver.executeAsync(qctx); - Assert.assertEquals(1, driver.getHiveHandleSize()); + assertEquals(1, driver.getHiveHandleSize()); validateExecuteAsync(qctx, DriverQueryState.SUCCESSFUL, true, false); driver.closeQuery(qctx.getQueryHandle()); driver.closePreparedQuery(pctx.getPrepareHandle()); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); } /** @@ -742,7 +734,7 @@ public class TestHiveDriver { // acquire SessionState.setCurrentSessionState(ss); DriverQueryPlan plan = driver.explain(createExplainContext("SELECT ID FROM test_part_table", conf)); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); assertTrue(plan instanceof HiveQueryPlan); assertNotNull(plan.getTablesQueried()); assertEquals(plan.getTablesQueried().size(), 1); @@ -751,8 +743,8 @@ public class TestHiveDriver { System.out.println("Parts:" + plan.getPartitions()); assertFalse(plan.getPartitions().isEmpty()); assertEquals(plan.getPartitions().size(), 1); - assertTrue(((String)plan.getPartitions().get(dataBase + ".test_part_table").iterator().next()).contains("today")); - assertTrue(((String)plan.getPartitions().get(dataBase + ".test_part_table").iterator().next()).contains("dt")); + assertTrue(((String) plan.getPartitions().get(dataBase + ".test_part_table").iterator().next()).contains("today")); + assertTrue(((String) plan.getPartitions().get(dataBase + ".test_part_table").iterator().next()).contains("dt")); } /** @@ -770,7 +762,7 @@ public class TestHiveDriver { + " explain_test_1 join explain_test_2 on explain_test_1.ID = explain_test_2.ID" + " WHERE explain_test_1.ID = 'foo' or explain_test_2.ID = 'bar'" + " GROUP BY explain_test_1.ID", conf)); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); assertTrue(plan instanceof HiveQueryPlan); assertNotNull(plan.getTablesQueried()); assertEquals(plan.getTablesQueried().size(), 2); @@ -797,13 +789,13 @@ public class TestHiveDriver { pctx.setLensSessionIdentifier(sessionid); DriverQueryPlan plan2 = driver.explainAndPrepare(pctx); // assertNotNull(plan2.getResultDestination()); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); assertNotNull(plan2.getTablesQueried()); assertEquals(plan2.getTablesQueried().size(), 1); assertTrue(plan2.getTableWeights().containsKey(dataBase + ".explain_test_1")); QueryContext ctx = createContext(pctx, conf); LensResultSet resultSet = driver.execute(ctx); - Assert.assertEquals(0, driver.getHiveHandleSize()); + assertEquals(0, driver.getHiveHandleSize()); HivePersistentResultSet persistentResultSet = (HivePersistentResultSet) resultSet; String path = persistentResultSet.getOutputPath(); assertEquals(ctx.getHdfsoutPath(), path); @@ -820,10 +812,8 @@ public class TestHiveDriver { @Test public void testPriority() throws IOException, LensException, ParseException { Configuration conf = new Configuration(); - DurationBasedQueryPriorityDecider alwaysNormalPriorityDecider = new DurationBasedQueryPriorityDecider(driver, - "", - HiveDriver.MONTHLY_PARTITION_WEIGHT_DEFAULT, - HiveDriver.DAILY_PARTITION_WEIGHT_DEFAULT, HiveDriver.HOURLY_PARTITION_WEIGHT_DEFAULT); + CostRangePriorityDecider alwaysNormalPriorityDecider = + new CostRangePriorityDecider(new CostToPriorityRangeConf("")); BufferedReader br = new BufferedReader(new InputStreamReader( TestHiveDriver.class.getResourceAsStream("/priority_tests.data"))); String line; @@ -862,8 +852,9 @@ public class TestHiveDriver { } }); } - Assert.assertEquals(expected, driver.queryPriorityDecider.decidePriority(ctx)); - Assert.assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx)); + ctx.setDriverCost(driver, driver.queryCostCalculator.calculateCost(ctx, driver)); + assertEquals(expected, driver.queryPriorityDecider.decidePriority(ctx.getDriverQueryCost(driver))); + assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx.getDriverQueryCost(driver))); i++; } // test priority without fact partitions @@ -893,13 +884,15 @@ public class TestHiveDriver { put("table1", 1.0); } }); - Assert.assertEquals(Priority.VERY_HIGH, driver.queryPriorityDecider.decidePriority(ctx)); - Assert.assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx)); + ctx.setDriverCost(driver, driver.queryCostCalculator.calculateCost(ctx, driver)); + assertEquals(Priority.VERY_HIGH, driver.queryPriorityDecider.decidePriority(ctx.getDriverQueryCost(driver))); + assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx.getDriverQueryCost(driver))); // test priority without rewriter plan ctx = createContext("test priority query", conf); - Assert.assertEquals(Priority.VERY_HIGH, driver.queryPriorityDecider.decidePriority(ctx)); - Assert.assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx)); + ctx.setDriverCost(driver, driver.queryCostCalculator.calculateCost(ctx, driver)); + assertEquals(Priority.VERY_HIGH, driver.queryPriorityDecider.decidePriority(ctx.getDriverQueryCost(driver))); + assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx.getDriverQueryCost(driver))); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index bda6572..0f2b740 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -33,9 +33,7 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import org.apache.lens.api.LensConf; -import org.apache.lens.api.query.QueryCost; -import org.apache.lens.api.query.QueryHandle; -import org.apache.lens.api.query.QueryPrepareHandle; +import org.apache.lens.api.query.*; import org.apache.lens.cube.parse.HQLParser; import org.apache.lens.server.api.driver.*; import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState; @@ -44,6 +42,8 @@ import org.apache.lens.server.api.events.LensEventListener; import org.apache.lens.server.api.metrics.MethodMetricsContext; import org.apache.lens.server.api.metrics.MethodMetricsFactory; import org.apache.lens.server.api.query.*; +import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost; +import org.apache.lens.server.api.query.cost.QueryCost; import org.apache.lens.server.api.user.UserConfigLoader; import org.apache.lens.server.model.LogSegregationContext; import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; @@ -543,7 +543,7 @@ public class JDBCDriver implements LensDriver { return rewrittenQuery; } - private static final QueryCost JDBC_DRIVER_COST = new QueryCost(0, 0); + static final QueryCost JDBC_DRIVER_COST = new FactPartitionBasedQueryCost(0); /** * Dummy JDBC query Plan class to get min cost selector working. http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java index be34164..a6c97ae 100644 --- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java +++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java @@ -18,31 +18,28 @@ */ package org.apache.lens.driver.jdbc; -import static org.apache.lens.server.api.user.MockUserConfigLoader.*; +import static org.apache.lens.server.api.user.MockUserConfigLoader.KEY; +import static org.apache.lens.server.api.user.MockUserConfigLoader.VALUE; import static org.testng.Assert.*; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.apache.lens.api.LensConf; -import org.apache.lens.api.query.QueryCost; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.api.query.ResultRow; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.driver.*; import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState; -import org.apache.lens.server.api.driver.LensDriver; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metrics.LensMetricsRegistry; import org.apache.lens.server.api.query.ExplainQueryContext; import org.apache.lens.server.api.query.PreparedQueryContext; import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.query.cost.QueryCost; import org.apache.lens.server.api.user.MockUserConfigLoader; import org.apache.lens.server.api.util.LensUtil; @@ -52,14 +49,10 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.service.cli.ColumnDescriptor; import org.testng.Assert; -import org.testng.annotations.AfterTest; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.BeforeTest; -import org.testng.annotations.Test; +import org.testng.annotations.*; import com.codahale.metrics.MetricRegistry; import com.mchange.v2.c3p0.ComboPooledDataSource; - import lombok.extern.slf4j.Slf4j; /** @@ -133,7 +126,7 @@ public class TestJdbcDriver { protected ExplainQueryContext createExplainContext(final String query, Configuration conf) { ExplainQueryContext ectx = new ExplainQueryContext(UUID.randomUUID().toString(), query, "testuser", null, conf, - drivers); + drivers); return ectx; } @@ -265,8 +258,7 @@ public class TestJdbcDriver { ExplainQueryContext ctx = createExplainContext(query1, baseConf); Assert.assertNull(ctx.getFinalDriverQuery(driver)); QueryCost cost = driver.estimate(ctx); - Assert.assertEquals(cost.getEstimatedExecTimeMillis(), 0); - Assert.assertEquals(cost.getEstimatedResourceUsage(), 0.0); + Assert.assertEquals(cost, JDBCDriver.JDBC_DRIVER_COST); Assert.assertNotNull(ctx.getFinalDriverQuery(driver)); // Test connection leak for estimate @@ -339,8 +331,7 @@ public class TestJdbcDriver { // run estimate and execute - because server would first run estimate and then execute with same context QueryContext ctx = createQueryContext(query1, metricConf); QueryCost cost = driver.estimate(ctx); - Assert.assertEquals(cost.getEstimatedExecTimeMillis(), 0); - Assert.assertEquals(cost.getEstimatedResourceUsage(), 0.0); + Assert.assertEquals(cost, JDBCDriver.JDBC_DRIVER_COST); LensResultSet result = driver.execute(ctx); Assert.assertNotNull(result); @@ -348,15 +339,13 @@ public class TestJdbcDriver { // run estimate and prepare - because server would first run estimate and then prepare with same context PreparedQueryContext pContext = new PreparedQueryContext(query1, "SA", metricConf, drivers); cost = driver.estimate(pContext); - Assert.assertEquals(cost.getEstimatedExecTimeMillis(), 0); - Assert.assertEquals(cost.getEstimatedResourceUsage(), 0.0); + Assert.assertEquals(cost, JDBCDriver.JDBC_DRIVER_COST); driver.prepare(pContext); // test explain and prepare PreparedQueryContext pContext2 = new PreparedQueryContext(query1, "SA", metricConf, drivers); cost = driver.estimate(pContext2); - Assert.assertEquals(cost.getEstimatedExecTimeMillis(), 0); - Assert.assertEquals(cost.getEstimatedResourceUsage(), 0.0); + Assert.assertEquals(cost, JDBCDriver.JDBC_DRIVER_COST); driver.prepare(pContext2); driver.explainAndPrepare(pContext2); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java ---------------------------------------------------------------------- diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java index 8957aef..5d58ef2 100644 --- a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java +++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java @@ -30,6 +30,7 @@ import javax.xml.bind.JAXBException; import org.apache.lens.api.LensConf; import org.apache.lens.api.query.*; import org.apache.lens.api.result.LensAPIResult; +import org.apache.lens.api.result.QueryCostTO; import org.apache.lens.regression.core.constants.QueryURL; import org.apache.lens.regression.core.type.FormBuilder; import org.apache.lens.regression.core.type.MapBuilder; @@ -253,7 +254,7 @@ public class QueryHelper extends ServiceManagerHelper { * @return the Estimate result */ - public QueryCost estimateQuery(String queryString, String sessionHandleString, String conf) throws + public QueryCostTO estimateQuery(String queryString, String sessionHandleString, String conf) throws InstantiationException, IllegalAccessException, JAXBException, LensException { FormBuilder formData = new FormBuilder(); formData.add("sessionid", sessionHandleString); @@ -266,20 +267,20 @@ public class QueryHelper extends ServiceManagerHelper { String queryCostString = response.readEntity(String.class); logger.info(queryCostString); LensAPIResult successResponse = (LensAPIResult) Util.getObject(queryCostString, LensAPIResult.class); - QueryCost queryCost = (QueryCost) successResponse.getData(); - if (queryCost == null) { + QueryCostTO queryCostTO = (QueryCostTO) successResponse.getData(); + if (queryCostTO == null) { throw new LensException("Estimate Failed"); } - return queryCost; + return queryCostTO; } - public QueryCost estimateQuery(String queryString, String sessionHandleString) throws + public QueryCostTO estimateQuery(String queryString, String sessionHandleString) throws InstantiationException, IllegalAccessException, JAXBException, LensException { return estimateQuery(queryString, sessionHandleString, "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><conf />"); } - public QueryCost estimateQuery(String queryString) throws + public QueryCostTO estimateQuery(String queryString) throws InstantiationException, IllegalAccessException, JAXBException, LensException { return estimateQuery(queryString, sessionHandleString); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java index 51b1217..c3ec099 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java @@ -22,14 +22,18 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.*; -import org.apache.lens.api.query.QueryCost; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.api.query.QueryPlan; import org.apache.lens.api.query.QueryPrepareHandle; +import org.apache.lens.server.api.query.cost.QueryCost; +import org.apache.lens.server.api.query.cost.QueryCostTOBuilder; + +import lombok.extern.slf4j.Slf4j; /** * The Class DriverQueryPlan. */ +@Slf4j public abstract class DriverQueryPlan { /** @@ -126,7 +130,7 @@ public abstract class DriverQueryPlan { /** * Get the cost associated with the plan * - * @return QueryCost object + * @return QueryCostTO object */ public abstract QueryCost getCost(); @@ -299,8 +303,8 @@ public abstract class DriverQueryPlan { * @throws UnsupportedEncodingException the unsupported encoding exception */ public QueryPlan toQueryPlan() throws UnsupportedEncodingException { - return new QueryPlan(new ArrayList<String>(tablesQueried), hasSubQuery, execMode != null ? execMode.name() : null, + return new QueryPlan(new ArrayList<>(tablesQueried), hasSubQuery, execMode != null ? execMode.name() : null, scanMode != null ? scanMode.name() : null, handle, - URLEncoder.encode(getPlan(), "UTF-8"), getCost(), false, null); + URLEncoder.encode(getPlan(), "UTF-8"), new QueryCostTOBuilder(getCost()).build(), false, null); } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java index 3dc4e73..b56c67c 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java @@ -20,14 +20,12 @@ package org.apache.lens.server.api.driver; import java.io.Externalizable; -import org.apache.lens.api.query.QueryCost; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.api.query.QueryPrepareHandle; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.events.LensEventListener; -import org.apache.lens.server.api.query.AbstractQueryContext; -import org.apache.lens.server.api.query.PreparedQueryContext; -import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.query.*; +import org.apache.lens.server.api.query.cost.QueryCost; import org.apache.lens.server.api.user.UserConfigLoader; import org.apache.hadoop.conf.Configuration; @@ -57,7 +55,7 @@ public interface LensDriver extends Externalizable { * * @param qctx The query context * - * @return The QueryCost object + * @return The QueryCostTO object * * @throws LensException the lens exception if driver cannot estimate */ http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java index 43237f7..c3fe8eb 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java @@ -21,8 +21,8 @@ package org.apache.lens.server.api.driver; import java.util.Collections; import java.util.Comparator; -import org.apache.lens.api.query.QueryCost; import org.apache.lens.server.api.query.AbstractQueryContext; +import org.apache.lens.server.api.query.cost.QueryCost; import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java deleted file mode 100644 index 97574b8..0000000 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.server.api.priority; - - -import org.apache.lens.api.Priority; - -/** - * Cost to priority range conf class. - * - * @see RangeConf - */ - -public class CostToPriorityRangeConf extends RangeConf<Float, Priority> { - /** - * Super constructor - * - * @param confValue - * @see RangeConf#RangeConf(String) - */ - public CostToPriorityRangeConf(String confValue) { - super(confValue); - } - - /** - * Parse key method - * - * @param s - * @return parsed float from string s - * @see RangeConf#parseKey(String) - */ - @Override - protected Float parseKey(String s) { - return Float.parseFloat(s); - } - - /** - * Parse value method - * - * @param s - * @return parsed Priority from String s - * @see RangeConf#parseValue(String) - */ - @Override - protected Priority parseValue(String s) { - return Priority.valueOf(s); - } - - /** - * Default value is "NORMAL". Which implies {@link #get(Comparable)} will always return Priotity.NORMAL - * - * @return "NORMAL" - */ - @Override - protected String getDefaultConf() { - return Priority.NORMAL.toString(); - } -}
