Repository: incubator-lens Updated Branches: refs/heads/master c6c593c66 -> 42ffb4e1a
LENS-412 : Fix missing partitions message propagation across storages (Rajat Khandelwal via amareshwari) Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/42ffb4e1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/42ffb4e1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/42ffb4e1 Branch: refs/heads/master Commit: 42ffb4e1a837320c0014fae7ce0665fb182d97d6 Parents: c6c593c Author: Rajat Khandelwal <pro...@apache.org> Authored: Fri Mar 27 14:19:09 2015 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Fri Mar 27 14:19:09 2015 +0530 ---------------------------------------------------------------------- .../lens/cube/metadata/CubeMetastoreClient.java | 24 +- .../lens/cube/metadata/StorageConstants.java | 10 +- .../lens/cube/metadata/TimePartition.java | 76 ++++++ .../timeline/EndsAndHolesPartitionTimeline.java | 22 +- .../metadata/timeline/PartitionTimeline.java | 32 +-- .../timeline/PartitionTimelineFactory.java | 6 +- .../timeline/RangesPartitionTimeline.java | 243 +++++++++++++++++++ .../timeline/StoreAllPartitionTimeline.java | 8 +- .../cube/parse/CandidateTablePruneCause.java | 7 +- .../lens/cube/parse/CubeQueryConfUtil.java | 2 - .../lens/cube/parse/CubeQueryContext.java | 2 +- .../org/apache/lens/cube/parse/PruneCauses.java | 40 +-- .../lens/cube/parse/StorageTableResolver.java | 70 +++--- .../org/apache/lens/cube/parse/StorageUtil.java | 6 +- .../timeline/TestPartitionTimelines.java | 75 +++++- .../lens/cube/parse/TestCubeRewriter.java | 32 +-- 16 files changed, 508 insertions(+), 147 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 42bf98f..1835d2f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -44,7 +44,6 @@ import org.apache.thrift.TException; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; - import lombok.extern.apachecommons.CommonsLog; /** @@ -330,14 +329,19 @@ public class CubeMetastoreClient { } /** update partition timeline cache for deletion of time partition */ - public void updateForDeletion(String cubeTableName, String storageName, UpdatePeriod updatePeriod, + public boolean updateForDeletion(String cubeTableName, String storageName, UpdatePeriod updatePeriod, Map<String, Date> timePartSpec) throws HiveException, LensException { + boolean updated = false; for (Map.Entry<String, Date> entry : timePartSpec.entrySet()) { - get(cubeTableName, storageName, updatePeriod, entry.getKey()).drop(TimePartition.of( - updatePeriod, entry.getValue())); + TimePartition part = TimePartition.of(updatePeriod, entry.getValue()); + if (!partitionExistsByFilter(cubeTableName, storageName, StorageConstants.getPartFilter(entry.getKey(), + part.getDateString()))) { + get(cubeTableName, storageName, updatePeriod, entry.getKey()).drop(part); + updated = true; + } } + return updated; } - } @@ -937,8 +941,9 @@ public class CubeMetastoreClient { } else { // dropping fact partition getStorage(storageName).dropPartition(getClient(), storageTableName, partVals, null); - partitionTimelineCache.updateForDeletion(cubeTableName, storageName, updatePeriod, timePartSpec); - this.alterTablePartitionCache(storageTableName); + if (partitionTimelineCache.updateForDeletion(cubeTableName, storageName, updatePeriod, timePartSpec)) { + this.alterTablePartitionCache(storageTableName); + } } } @@ -978,6 +983,11 @@ public class CubeMetastoreClient { return partitionExists(storageTableName, getPartitionSpec(updatePeriod, partitionTimestamps)); } + public boolean partitionExistsByFilter(String cubeTableName, String storageName, String filter) throws HiveException { + return partitionExistsByFilter(MetastoreUtil.getStorageTableName(cubeTableName, Storage.getPrefix(storageName)), + filter); + } + public boolean partitionExistsByFilter(String storageTableName, String filter) throws HiveException { int parts; Table tbl = null; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java index aee9f7b..610d168 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/StorageConstants.java @@ -19,10 +19,10 @@ package org.apache.lens.cube.metadata; -import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; +import java.util.Set; public final class StorageConstants { private StorageConstants() { @@ -52,10 +52,8 @@ public final class StorageConstants { * * @return List */ - public static List<String> getPartitionsForLatest() { - List<String> parts = new ArrayList<String>(); - parts.add(LATEST_PARTITION_VALUE); - return parts; + public static Set<String> getPartitionsForLatest() { + return Collections.singleton(LATEST_PARTITION_VALUE); } /** http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java index 6a5b31d..b948467 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java @@ -21,6 +21,7 @@ package org.apache.lens.cube.metadata; import java.text.ParseException; import java.util.Calendar; import java.util.Date; +import java.util.Iterator; import org.apache.lens.api.LensException; @@ -123,4 +124,79 @@ public class TimePartition implements Comparable<TimePartition> { protected static String getWrongUpdatePeriodMessage(UpdatePeriod up, String dateString) { return String.format(UPDATE_PERIOD_WRONG_ERROR_MESSAGE, up, dateString); } + + public TimePartitionRange rangeUpto(TimePartition to) { + return new TimePartitionRange(this, to); + } + + public TimePartitionRange rangeFrom(TimePartition from) { + return new TimePartitionRange(from, this); + } + + public TimePartitionRange singletonRange() { + return rangeUpto(next()); + } + + /** + * Range of time partition. [begin,end). i.e. inclusive begin and exclusive end. + */ + @Data + public static class TimePartitionRange implements Iterable<TimePartition> { + private TimePartition begin; + private TimePartition end; + + public TimePartitionRange(TimePartition from, TimePartition to) { + this.begin = from; + this.end = to; + } + + @Override + public String toString() { + return "[" + begin.getDateString() + ", " + end.getDateString() + ")"; + } + + /** + * returns TimePartition objects starting from begin and upto(excluding) end. interval of iteration is the update + * period of the partitions. Assumes both partitions have same update period. + */ + @Override + public Iterator<TimePartition> iterator() { + + return new Iterator<TimePartition>() { + TimePartition current = begin; + + @Override + public boolean hasNext() { + return current.before(end); + } + + @Override + public TimePartition next() { + TimePartition ret = current; + current = current.next(); + return ret; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("remove not supported"); + } + }; + } + + /** + * @param partition + * @return begin <= partition < end + */ + public boolean contains(TimePartition partition) { + return !partition.before(begin) && partition.before(end); + } + + /** + * @return if range is empty range. + */ + public boolean isEmpty() { + return begin.equals(end); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java index 3e323e1..79e8a62 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java @@ -22,7 +22,6 @@ package org.apache.lens.cube.metadata.timeline; import java.util.*; import org.apache.lens.api.LensException; -import org.apache.lens.cube.metadata.CubeMetastoreClient; import org.apache.lens.cube.metadata.TimePartition; import org.apache.lens.cube.metadata.UpdatePeriod; import org.apache.lens.cube.parse.TimeRange; @@ -48,13 +47,13 @@ public class EndsAndHolesPartitionTimeline extends PartitionTimeline { private TreeSet<TimePartition> holes = Sets.newTreeSet(); private TimePartition latest; - public EndsAndHolesPartitionTimeline(CubeMetastoreClient client, String storageTableName, UpdatePeriod updatePeriod, + public EndsAndHolesPartitionTimeline(String storageTableName, UpdatePeriod updatePeriod, String partCol) { - super(client, storageTableName, updatePeriod, partCol); + super(storageTableName, updatePeriod, partCol); } @Override - public boolean add(TimePartition partition) throws LensException { + public boolean add(@NonNull TimePartition partition) throws LensException { if (isEmpty()) { // First partition being added first = partition; @@ -75,20 +74,7 @@ public class EndsAndHolesPartitionTimeline extends PartitionTimeline { } @Override - public boolean add(@NonNull Collection<TimePartition> partitions) throws LensException { - boolean result = true; - for (TimePartition partition : partitions) { - result &= add(partition); - } - // Can also return the failed to add items. - return result; - } - - @Override - public boolean drop(TimePartition toDrop) throws LensException { - if (morePartitionsExist(toDrop.getDateString())) { - return true; - } + public boolean drop(@NonNull TimePartition toDrop) throws LensException { if (first.equals(latest) && first.equals(toDrop)) { this.first = null; this.latest = null; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java index 7eda58a..237164f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java @@ -22,9 +22,10 @@ package org.apache.lens.cube.metadata.timeline; import java.util.*; import org.apache.lens.api.LensException; -import org.apache.lens.cube.metadata.*; +import org.apache.lens.cube.metadata.MetastoreUtil; +import org.apache.lens.cube.metadata.TimePartition; +import org.apache.lens.cube.metadata.UpdatePeriod; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import com.google.common.collect.Maps; @@ -45,7 +46,6 @@ import lombok.extern.apachecommons.CommonsLog; @ToString(exclude = {"client"}) @CommonsLog public abstract class PartitionTimeline implements Iterable<TimePartition> { - private final CubeMetastoreClient client; private final String storageTableName; private final UpdatePeriod updatePeriod; private final String partCol; @@ -121,23 +121,6 @@ public abstract class PartitionTimeline implements Iterable<TimePartition> { return result; } - /** - * goes to metastore and queries if more partitions exist associated with (partCol = value) in storage table - * #getStorageTableName for update period #getUpdatePeriod. This might be useful for implementations while - * implementing drop. - * - * @param value - * @return - * @throws LensException - */ - public boolean morePartitionsExist(String value) throws LensException { - try { - return getClient().partitionExistsByFilter(getStorageTableName(), StorageConstants.getPartFilter(getPartCol(), - value)); - } catch (HiveException e) { - throw new LensException(e); - } - } /** * Add partition to timeline @@ -155,7 +138,14 @@ public abstract class PartitionTimeline implements Iterable<TimePartition> { * @return whether add was successful * @throws LensException */ - public abstract boolean add(@NonNull Collection<TimePartition> partitions) throws LensException; + public boolean add(@NonNull Collection<TimePartition> partitions) throws LensException { + boolean result = true; + for (TimePartition partition : partitions) { + result &= add(partition); + } + // Can also return the failed to add items. + return result; + } /** * drop partition. http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java index 5626a03..b018a1a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimelineFactory.java @@ -49,11 +49,11 @@ public final class PartitionTimelineFactory { updatePeriod, partitionColumn)); Class<? extends PartitionTimeline> clz = (Class<? extends PartitionTimeline>) Class.forName(storageClassName); Constructor<? extends PartitionTimeline> constructor = clz.getConstructor( - CubeMetastoreClient.class, String.class, UpdatePeriod.class, String.class); + String.class, UpdatePeriod.class, String.class); return constructor.newInstance( - client, storageTable, updatePeriod, partitionColumn); + storageTable, updatePeriod, partitionColumn); } catch (Exception e) { - return new EndsAndHolesPartitionTimeline(client, storageTable, updatePeriod, partitionColumn); + return new EndsAndHolesPartitionTimeline(storageTable, updatePeriod, partitionColumn); } } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java new file mode 100644 index 0000000..fb2d0a8 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java @@ -0,0 +1,243 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.metadata.timeline; + + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.lens.api.LensException; +import org.apache.lens.cube.metadata.TimePartition; +import org.apache.lens.cube.metadata.UpdatePeriod; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import lombok.Data; +import lombok.ToString; + +/** + * One implementation of PartitionTimeline that stores ranges of partition presence, Basically a list of tuples each + * tuple represents a range of presence. range is of the form [from, end) i.e. including the first element and excluding + * the second element of the tuple + */ +@Data +@ToString(callSuper = true) +public class RangesPartitionTimeline extends PartitionTimeline { + private List<TimePartition.TimePartitionRange> ranges = Lists.newArrayList(); + + public RangesPartitionTimeline(String storageTableName, UpdatePeriod updatePeriod, + String partCol) { + super(storageTableName, updatePeriod, partCol); + } + + @Override + public boolean add(TimePartition partition) throws LensException { + int ind = getStrictlyAfterIndex(partition); + int added = 0; + if (ind > 0) { + if (ranges.get(ind - 1).contains(partition)) { + return true; + } + if (ranges.get(ind - 1).getEnd().equals(partition)) { + added++; + ranges.get(ind - 1).setEnd(partition.next()); + } + } + if (ind < ranges.size()) { + if (partition.equals(ranges.get(ind).getBegin().previous())) { + added++; + ranges.get(ind).setBegin(partition); + } + } + switch (added) { + case 0: + ranges.add(ind, partition.singletonRange()); + break; + case 2: + ranges.get(ind - 1).setEnd(ranges.get(ind).getEnd()); + ranges.remove(ind); + break; + case 1: + // Nothing needs to be done. + default: + break; + + } + return true; + } + + private int getStrictlyAfterIndex(TimePartition part) { + int start = 0; + int end = getRanges().size(); + int mid; + while (end - start > 0) { + mid = (start + end) / 2; + if (ranges.get(mid).getBegin().after(part)) { + end = mid; + } else { + start = mid + 1; + } + } + return end; + } + + private void mergeRanges() { + for (int i = 0; i < ranges.size() - 1; i++) { + if (ranges.get(i).getEnd().equals(ranges.get(i + 1).getBegin())) { + TimePartition.TimePartitionRange removed = ranges.remove(i + 1); + ranges.get(i).setEnd(removed.getEnd()); + i--; // check again at same index + } + } + } + + @Override + public boolean drop(TimePartition toDrop) throws LensException { + int ind = getStrictlyAfterIndex(toDrop); + if (ind == 0) { + return true; // nothing to do + } + if (ranges.get(ind - 1).getBegin().equals(toDrop)) { + ranges.get(ind - 1).setBegin(toDrop.next()); + } else if (ranges.get(ind - 1).getEnd().previous().equals(toDrop)) { + ranges.get(ind - 1).setEnd(toDrop); + } else { + TimePartition end = ranges.get(ind - 1).getEnd(); + ranges.get(ind - 1).setEnd(toDrop); + ranges.add(ind, toDrop.next().rangeUpto(end)); + } + if (ranges.get(ind - 1).isEmpty()) { + ranges.remove(ind - 1); + } + return true; + } + + + @Override + public TimePartition latest() { + if (isEmpty()) { + return null; + } + return ranges.get(ranges.size() - 1).getEnd().previous(); + } + + @Override + public Map<String, String> toProperties() { + HashMap<String, String> ret = Maps.newHashMap(); + if (isEmpty()) { + return ret; + } + StringBuilder sb = new StringBuilder(); + String sep = ""; + for (TimePartition.TimePartitionRange range : ranges) { + sb.append(sep); + sep = ","; + sb.append(range.getBegin()).append(sep).append(range.getEnd()); + } + ret.put("ranges", sb.toString()); + return ret; + } + + @Override + public boolean initFromProperties(Map<String, String> properties) throws LensException { + ranges.clear(); + String rangesStr = properties.get("ranges"); + if (!Strings.isNullOrEmpty(rangesStr)) { + String[] split = rangesStr.split("\\s*,\\s*"); + if (split.length % 2 == 1) { + throw new LensException("Ranges incomplete"); + } + for (int i = 0; i < split.length; i += 2) { + ranges.add(TimePartition.of(getUpdatePeriod(), split[i]).rangeUpto(TimePartition.of(getUpdatePeriod(), + split[i + 1]))); + } + } + return isConsistent(); + } + + + public boolean isEmpty() { + return ranges.isEmpty(); + } + + @Override + public boolean isConsistent() { + if (isEmpty()) { + return true; + } + if (!ranges.get(0).getBegin().before(ranges.get(0).getEnd())) { + return false; + } + for (int i = 0; i < ranges.size() - 1; i++) { + if (!ranges.get(i).getEnd().before(ranges.get(i + 1).getBegin())) { + return false; + } + if (!ranges.get(i + 1).getBegin().before(ranges.get(i + 1).getEnd())) { + return false; + } + } + return true; + } + + @Override + public boolean exists(TimePartition toCheck) { + if (isEmpty()) { + return false; + } + for (TimePartition.TimePartitionRange range : ranges) { + if (range.contains(toCheck)) { + return true; + } + } + return false; + } + + @Override + public Iterator<TimePartition> iterator() { + + return new Iterator<TimePartition>() { + Iterator<TimePartition.TimePartitionRange> uber = ranges.iterator(); + Iterator<TimePartition> cur = null; + + @Override + public boolean hasNext() { + if (cur == null || !cur.hasNext()) { + if (!uber.hasNext()) { + return false; + } + cur = uber.next().iterator(); + } + return cur.hasNext(); + } + + @Override + public TimePartition next() { + return cur.next(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } +} http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java index 8f8b03a..d6ee0a1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java @@ -21,7 +21,6 @@ package org.apache.lens.cube.metadata.timeline; import java.util.*; import org.apache.lens.api.LensException; -import org.apache.lens.cube.metadata.CubeMetastoreClient; import org.apache.lens.cube.metadata.TimePartition; import org.apache.lens.cube.metadata.UpdatePeriod; @@ -42,9 +41,9 @@ import lombok.ToString; public class StoreAllPartitionTimeline extends PartitionTimeline { TreeSet<TimePartition> allPartitions; - public StoreAllPartitionTimeline(CubeMetastoreClient client, String storageTableName, + public StoreAllPartitionTimeline(String storageTableName, UpdatePeriod updatePeriod, String partCol) { - super(client, storageTableName, updatePeriod, partCol); + super(storageTableName, updatePeriod, partCol); allPartitions = Sets.newTreeSet(); } @@ -60,9 +59,6 @@ public class StoreAllPartitionTimeline extends PartitionTimeline { @Override public boolean drop(@NonNull TimePartition toDrop) throws LensException { - if (morePartitionsExist(toDrop.getDateString())) { - return true; - } return allPartitions.remove(toDrop); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java index bfeff4f..bc9ef93 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java @@ -22,6 +22,7 @@ import java.util.*; import org.codehaus.jackson.annotate.JsonWriteNullProperties; +import com.google.common.collect.Sets; import lombok.Data; import lombok.NoArgsConstructor; @@ -95,7 +96,7 @@ public class CandidateTablePruneCause { // missing partitions for cube table MISSING_PARTITIONS("Missing partitions for the cube table: %s") { Object[] getFormatPlaceholders(Set<CandidateTablePruneCause> causes) { - List<List<String>> missingPartitions = new ArrayList<List<String>>(); + Set<Set<String>> missingPartitions = Sets.newHashSet(); for (CandidateTablePruneCause cause : causes) { missingPartitions.add(cause.getMissingPartitions()); } @@ -183,7 +184,7 @@ public class CandidateTablePruneCause { private Map<String, SkipStorageCause> storageCauses; // populated only incase of missing partitions cause - private List<String> missingPartitions; + private Set<String> missingPartitions; // populated only incase of missing update periods cause private List<String> missingUpdatePeriods; // populated in case of missing columns @@ -215,7 +216,7 @@ public class CandidateTablePruneCause { return columnNotFound(colList); } - public static CandidateTablePruneCause missingPartitions(List<String> nonExistingParts) { + public static CandidateTablePruneCause missingPartitions(Set<String> nonExistingParts) { CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.MISSING_PARTITIONS); cause.setMissingPartitions(nonExistingParts); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java index c3142cd..a6374f6 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java @@ -44,7 +44,6 @@ public final class CubeQueryConfUtil { public static final String DRIVER_SUPPORTED_STORAGES = "lens.cube.query.driver." + "supported.storages"; public static final String FAIL_QUERY_ON_PARTIAL_DATA = "lens.cube.query.fail.if.data.partial"; public static final String NON_EXISTING_PARTITIONS = "lens.cube.query.nonexisting.partitions"; - public static final String ADD_NON_EXISTING_PARTITIONS = "lens.cube.query.add.nonexisting.partitions"; public static final String ENABLE_MULTI_TABLE_SELECT = "lens.cube.query.enable.multi.table.select"; public static final String QUERY_MAX_INTERVAL = "lens.cube.query.max.interval"; public static final String PROCESS_TIME_PART_COL = "lens.cube.query.process.time" + ".partition.column"; @@ -59,7 +58,6 @@ public final class CubeQueryConfUtil { public static final int DEFAULT_LOOK_AHEAD_PT_PARTS = 1; public static final boolean DEFAULT_ENABLE_GROUP_BY_TO_SELECT = false; public static final boolean DEFAULT_ENABLE_SELECT_TO_GROUPBY = false; - public static final boolean DEFAULT_ADD_NON_EXISTING_PARTITIONS = false; public static final boolean DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL = true; public static String getLookAheadPTPartsKey(UpdatePeriod interval) { http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index e06022c..7ea67f4 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -641,7 +641,7 @@ public class CubeQueryContext { } } - void setNonexistingParts(Map<String, List<String>> nonExistingParts) throws SemanticException { + void setNonexistingParts(Map<String, Set<String>> nonExistingParts) throws SemanticException { if (!nonExistingParts.isEmpty()) { ByteArrayOutputStream out = null; String partsStr; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java index ae9e013..32ef421 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java @@ -18,10 +18,7 @@ */ package org.apache.lens.cube.parse; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.lens.cube.metadata.AbstractCubeTable; import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; @@ -30,6 +27,7 @@ import org.apache.commons.lang.StringUtils; import org.codehaus.jackson.annotate.JsonWriteNullProperties; +import com.google.common.collect.Maps; import lombok.AllArgsConstructor; import lombok.Data; import lombok.Getter; @@ -39,6 +37,21 @@ public class PruneCauses<T extends AbstractCubeTable> extends HashMap<T, List<Ca @Getter(lazy = true) private final HashMap<CandidateTablePruneCause, List<T>> reversed = reverse(); @Getter(lazy = true) + private final HashMap<String, List<CandidateTablePruneCause>> compact = computeCompact(); + + private HashMap<String, List<CandidateTablePruneCause>> computeCompact() { + HashMap<String, List<CandidateTablePruneCause>> detailedMessage = Maps.newHashMap(); + for (Map.Entry<CandidateTablePruneCause, List<T>> entry : getReversed().entrySet()) { + String key = StringUtils.join(entry.getValue(), ","); + if (detailedMessage.get(key) == null) { + detailedMessage.put(key, new ArrayList<CandidateTablePruneCause>()); + } + detailedMessage.get(key).add(entry.getKey()); + } + return detailedMessage; + } + + @Getter(lazy = true) private final BriefAndDetailedError jsonObject = toJsonObject(); public void addPruningMsg(T table, CandidateTablePruneCause msg) { @@ -62,16 +75,7 @@ public class PruneCauses<T extends AbstractCubeTable> extends HashMap<T, List<Ca } public BriefAndDetailedError toJsonObject() { - final HashMap<String, List<CandidateTablePruneCause>> detailedMessage - = new HashMap<String, List<CandidateTablePruneCause>>(); - for (Map.Entry<CandidateTablePruneCause, List<T>> entry : getReversed().entrySet()) { - String key = StringUtils.join(entry.getValue(), ","); - if (detailedMessage.get(key) == null) { - detailedMessage.put(key, new ArrayList<CandidateTablePruneCause>()); - } - detailedMessage.get(key).add(entry.getKey()); - } - return new BriefAndDetailedError(getBriefCause(), detailedMessage); + return new BriefAndDetailedError(getBriefCause(), getCompact()); } public String getBriefCause() { @@ -81,10 +85,10 @@ public class PruneCauses<T extends AbstractCubeTable> extends HashMap<T, List<Ca maxCause = cause.getCause(); } } - Map<CandidateTablePruneCause, List<T>> maxCauseMap = new HashMap<CandidateTablePruneCause, List<T>>(); - for (Map.Entry<CandidateTablePruneCause, List<T>> entry : getReversed().entrySet()) { - if (entry.getKey().getCause().compareTo(maxCause) == 0) { - maxCauseMap.put(entry.getKey(), entry.getValue()); + Map<CandidateTablePruneCause, String> maxCauseMap = Maps.newHashMap(); + for (Map.Entry<CandidateTablePruneCause, List<T>> entry: getReversed().entrySet()) { + if (entry.getKey().getCause().equals(maxCause)) { + maxCauseMap.put(entry.getKey(), StringUtils.join(entry.getValue(), ",")); } } return maxCause.getBriefError(maxCauseMap.keySet()); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index 23fd5a6..6e63483 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -25,6 +25,7 @@ import java.util.*; import org.apache.lens.api.LensException; import org.apache.lens.cube.metadata.*; +import org.apache.lens.cube.metadata.timeline.RangesPartitionTimeline; import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCause; import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode; @@ -38,6 +39,9 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.util.ReflectionUtils; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + /** * Resolve storages and partitions of all candidate tables and prunes candidate tables with missing storages or * partitions. @@ -55,8 +59,7 @@ class StorageTableResolver implements ContextRewriter { new HashMap<CubeFactTable, Map<UpdatePeriod, Set<String>>>(); private String processTimePartCol = null; private final UpdatePeriod maxInterval; - private final boolean populateNonExistingParts; - private final Map<String, List<String>> nonExistingPartitions = new HashMap<String, List<String>>(); + private final Map<String, Set<String>> nonExistingPartitions = new HashMap<String, Set<String>>(); private TimeRangeWriter rangeWriter; private DateFormat partWhereClauseFormat = null; private PHASE phase; @@ -82,13 +85,6 @@ class StorageTableResolver implements ContextRewriter { this.supportedStorages = getSupportedStorages(conf); this.allStoragesSupported = (supportedStorages == null); this.failOnPartialData = conf.getBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false); - if (!failOnPartialData) { - this.populateNonExistingParts = true; - } else { - this.populateNonExistingParts = - conf.getBoolean(CubeQueryConfUtil.ADD_NON_EXISTING_PARTITIONS, - CubeQueryConfUtil.DEFAULT_ADD_NON_EXISTING_PARTITIONS); - } String str = conf.get(CubeQueryConfUtil.VALID_STORAGE_DIM_TABLES); validDimTables = StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ",")); this.processTimePartCol = conf.get(CubeQueryConfUtil.PROCESS_TIME_PART_COL); @@ -320,10 +316,10 @@ class StorageTableResolver implements ContextRewriter { CandidateFact cfact = i.next(); List<FactPartition> answeringParts = new ArrayList<FactPartition>(); HashMap<String, SkipStorageCause> skipStorageCauses = new HashMap<String, SkipStorageCause>(); - List<String> nonExistingParts = new ArrayList<String>(); + Map<UpdatePeriod, RangesPartitionTimeline> missingPartitionRanges = Maps.newHashMap(); boolean noPartsForRange = false; for (TimeRange range : cubeql.getTimeRanges()) { - Set<FactPartition> rangeParts = getPartitions(cfact.fact, range, skipStorageCauses, nonExistingParts); + Set<FactPartition> rangeParts = getPartitions(cfact.fact, range, skipStorageCauses, missingPartitionRanges); if (rangeParts == null || rangeParts.isEmpty()) { LOG.info("No partitions for range:" + range); noPartsForRange = true; @@ -335,6 +331,14 @@ class StorageTableResolver implements ContextRewriter { cfact.getRangeToWhereClause().put(range, rangeWriter.getTimeRangeWhereClause(cubeql, cubeql.getAliasForTabName(cubeql.getCube().getName()), rangeParts)); } + Set<String> nonExistingParts = Sets.newHashSet(); + if (!missingPartitionRanges.isEmpty()) { + for (UpdatePeriod period : missingPartitionRanges.keySet()) { + for (TimePartition.TimePartitionRange range : missingPartitionRanges.get(period).getRanges()) { + nonExistingParts.add(range.toString()); + } + } + } if (!nonExistingParts.isEmpty()) { addNonExistingParts(cfact.fact.getName(), nonExistingParts); } @@ -343,24 +347,19 @@ class StorageTableResolver implements ContextRewriter { + cubeql.getTimeRanges()); /* * This fact is getting discarded because of any of following reasons: - * 1. Storage tables are not partitioned by timedim partition column - * 2. Has missing partitions, and CubeQueryConfUtil.ADD_NON_EXISTING_PARTITIONS is true - which can populate - * all missing partitions - * 3. Has missing partitions, and CubeQueryConfUtil.ADD_NON_EXISTING_PARTITIONS is false - will populate only - * the first missing partition. - * 4. Storage tables do not have the update period for the timerange queried. + * 1. Has missing partitions + * 2. All Storage tables were skipped for some reasons. + * 3. Storage tables do not have the update period for the timerange queried. */ - if (!skipStorageCauses.isEmpty()) { + if (!nonExistingParts.isEmpty()) { + cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.missingPartitions(nonExistingParts)); + } else if (!skipStorageCauses.isEmpty()) { CandidateTablePruneCause cause = CandidateTablePruneCause.noCandidateStorages(skipStorageCauses); cubeql.addFactPruningMsgs(cfact.fact, cause); } else { - if (!nonExistingParts.isEmpty()) { - cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.missingPartitions(nonExistingParts)); - } else { - CandidateTablePruneCause cause = - new CandidateTablePruneCause(CandidateTablePruneCode.NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE); - cubeql.addFactPruningMsgs(cfact.fact, cause); - } + CandidateTablePruneCause cause = + new CandidateTablePruneCause(CandidateTablePruneCode.NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE); + cubeql.addFactPruningMsgs(cfact.fact, cause); } i.remove(); continue; @@ -385,14 +384,16 @@ class StorageTableResolver implements ContextRewriter { } } - void addNonExistingParts(String name, List<String> nonExistingParts) { + + void addNonExistingParts(String name, Set<String> nonExistingParts) { nonExistingPartitions.put(name, nonExistingParts); } private Set<FactPartition> getPartitions(CubeFactTable fact, TimeRange range, - HashMap<String, SkipStorageCause> skipStorageCauses, List<String> nonExistingParts) throws SemanticException { + HashMap<String, SkipStorageCause> skipStorageCauses, + Map<UpdatePeriod, RangesPartitionTimeline> nonExistingParts) throws SemanticException { try { - return getPartitions(fact, range, getValidUpdatePeriods(fact), this.populateNonExistingParts, skipStorageCauses, + return getPartitions(fact, range, getValidUpdatePeriods(fact), true, skipStorageCauses, nonExistingParts); } catch (Exception e) { throw new SemanticException(e); @@ -400,7 +401,8 @@ class StorageTableResolver implements ContextRewriter { } private Set<FactPartition> getPartitions(CubeFactTable fact, TimeRange range, TreeSet<UpdatePeriod> updatePeriods, - boolean addNonExistingParts, Map<String, SkipStorageCause> skipStorageCauses, List<String> nonExistingParts) + boolean addNonExistingParts, Map<String, SkipStorageCause> skipStorageCauses, + Map<UpdatePeriod, RangesPartitionTimeline> nonExistingParts) throws Exception { Set<FactPartition> partitions = new TreeSet<FactPartition>(); if (getPartitions(fact, range.getFromDate(), range.getToDate(), range.getPartitionColumn(), partitions, @@ -413,7 +415,8 @@ class StorageTableResolver implements ContextRewriter { private boolean getPartitions(CubeFactTable fact, Date fromDate, Date toDate, String partCol, Set<FactPartition> partitions, TreeSet<UpdatePeriod> updatePeriods, - boolean addNonExistingParts, Map<String, SkipStorageCause> skipStorageCauses, List<String> nonExistingParts) + boolean addNonExistingParts, Map<String, SkipStorageCause> skipStorageCauses, + Map<UpdatePeriod, RangesPartitionTimeline> nonExistingParts) throws Exception { LOG.info("getPartitions for " + fact + " from fromDate:" + fromDate + " toDate:" + toDate); if (fromDate.equals(toDate) || fromDate.after(toDate)) { @@ -526,10 +529,13 @@ class StorageTableResolver implements ContextRewriter { if (!getPartitions(fact, dt, nextDt, partCol, partitions, newset, false, skipStorageCauses, nonExistingParts)) { - // Add non existing partitions for all cases of whether we populate all non existing or not. LOG.info("Adding non existing partition" + part); - nonExistingParts.add(part.getPartString()); if (addNonExistingParts) { + // Add non existing partitions for all cases of whether we populate all non existing or not. + if (!nonExistingParts.containsKey(part.getPeriod())) { + nonExistingParts.put(part.getPeriod(), new RangesPartitionTimeline(null, null, null)); + } + nonExistingParts.get(part.getPeriod()).add(TimePartition.of(part.getPeriod(), dt)); if (!failOnPartialData) { partitions.add(part); // add all storage tables as the answering tables http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java index 24d9340..0704171 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java @@ -33,20 +33,20 @@ public final class StorageUtil { private static final Log LOG = LogFactory.getLog(StorageUtil.class.getName()); - public static String getWherePartClause(String timeDimName, String tableName, List<String> parts) { + public static String getWherePartClause(String timeDimName, String tableName, Collection<String> parts) { if (parts.size() == 0) { return ""; } StringBuilder partStr = new StringBuilder(); String sep = ""; - for (int i = 0; i < parts.size(); i++) { + for (String part : parts) { partStr.append(sep); partStr.append("("); partStr.append(tableName != null ? tableName : "%s"); partStr.append("."); partStr.append(timeDimName); partStr.append(" = '"); - partStr.append(parts.get(i)); + partStr.append(part); partStr.append("'"); partStr.append(")"); sep = " OR "; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java index 0027e64..50b75e3 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java @@ -18,8 +18,7 @@ */ package org.apache.lens.cube.metadata.timeline; -import java.util.Date; -import java.util.Map; +import java.util.*; import org.apache.lens.api.LensException; import org.apache.lens.cube.metadata.CubeMetastoreClient; @@ -29,23 +28,87 @@ import org.apache.lens.cube.metadata.UpdatePeriod; import org.testng.Assert; import org.testng.annotations.Test; +import com.beust.jcommander.internal.Lists; + public class TestPartitionTimelines { CubeMetastoreClient client = null; private static final String TABLE_NAME = "storage_fact"; private static final UpdatePeriod PERIOD = UpdatePeriod.HOURLY; private static final String PART_COL = "pt"; private static final Date DATE = new Date(); + private static final List<Class<? extends PartitionTimeline>> TIMELINE_IMPLEMENTATIONS = Arrays.asList( + StoreAllPartitionTimeline.class, + EndsAndHolesPartitionTimeline.class, + RangesPartitionTimeline.class + ); @Test public void testPropertiesContractsForAllSubclasses() throws LensException { - testPropertiesContract(StoreAllPartitionTimeline.class); - testPropertiesContract(EndsAndHolesPartitionTimeline.class); + for (Class<? extends PartitionTimeline> clazz : TIMELINE_IMPLEMENTATIONS) { + testPropertiesContract(clazz); + } + } + + @Test + public void testEquivalence() throws LensException { + for (int j = 0; j < 10; j++) { + Random randomGenerator = new Random(); + List<PartitionTimeline> timelines = Lists.newArrayList(); + for (Class<? extends PartitionTimeline> clazz : TIMELINE_IMPLEMENTATIONS) { + timelines.add(getInstance(clazz)); + } + final List<TimePartition> addedPartitions = Lists.newArrayList(); + for (int i = 0; i < 200; i++) { + int randomInt = randomGenerator.nextInt(100) - 50; + TimePartition part = TimePartition.of(PERIOD, timeAtHourDiff(randomInt)); + addedPartitions.add(part); + for (PartitionTimeline timeline : timelines) { + timeline.add(part); + } + } + Iterator<TimePartition> sourceOfTruth = timelines.get(0).iterator(); + List<Iterator<TimePartition>> otherIterators = Lists.newArrayList(); + for (int i = 1; i < TIMELINE_IMPLEMENTATIONS.size() - 1; i++) { + otherIterators.add(timelines.get(i).iterator()); + } + while (sourceOfTruth.hasNext()) { + TimePartition cur = sourceOfTruth.next(); + for (Iterator<TimePartition> iterator : otherIterators) { + Assert.assertTrue(iterator.hasNext()); + Assert.assertEquals(iterator.next(), cur); + } + } + for (Iterator<TimePartition> iterator : otherIterators) { + Assert.assertFalse(iterator.hasNext()); + } + Collections.shuffle(addedPartitions); + Iterator<TimePartition> iter = addedPartitions.iterator(); + while (iter.hasNext()) { + TimePartition part = iter.next(); + iter.remove(); + if (!addedPartitions.contains(part)) { + for (PartitionTimeline timeline : timelines) { + timeline.drop(part); + } + } + } + for (PartitionTimeline timeline : timelines) { + Assert.assertTrue(timeline.isEmpty()); + } + } + } + + private Date timeAtHourDiff(int d) { + Calendar cal = Calendar.getInstance(); + cal.setTime(DATE); + cal.add(PERIOD.calendarField(), d); + return cal.getTime(); } private <T extends PartitionTimeline> T getInstance(Class<T> clz) { try { - return clz.getConstructor(CubeMetastoreClient.class, String.class, UpdatePeriod.class, String.class) - .newInstance(client, TABLE_NAME, PERIOD, PART_COL); + return clz.getConstructor(String.class, UpdatePeriod.class, String.class) + .newInstance(TABLE_NAME, PERIOD, PART_COL); } catch (Exception e) { e.printStackTrace(); } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/42ffb4e1/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 4278229..f9a3762 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -109,7 +109,7 @@ public class TestCubeRewriter extends TestQueryRewrite { compareQueries(expected, hqlQuery); conf.setBoolean(CubeQueryConfUtil.LIGHTEST_FACT_FIRST, true); - conf.setBoolean(CubeQueryConfUtil.ADD_NON_EXISTING_PARTITIONS, true); + SemanticException th = getSemanticExceptionInRewrite( "select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); Assert.assertEquals(th.getCanonicalErrorMsg().getErrorCode(), ErrorMsg.NO_CANDIDATE_FACT_AVAILABLE.getErrorCode()); @@ -122,24 +122,6 @@ public class TestCubeRewriter extends TestQueryRewrite { Assert.assertEquals(pruneCauses.getDetails().get("testfact").size(), 1); Assert.assertEquals(pruneCauses.getDetails().get("testfact").iterator().next().getCause(), CandidateTablePruneCode.MISSING_PARTITIONS); - - // Error should be no missing partitions with first missing partition populated for each update period - conf.setBoolean(CubeQueryConfUtil.ADD_NON_EXISTING_PARTITIONS, false); - th = getSemanticExceptionInRewrite( - "select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf); - Assert.assertEquals(th.getCanonicalErrorMsg().getErrorCode(), ErrorMsg.NO_CANDIDATE_FACT_AVAILABLE.getErrorCode()); - pruneCauses = extractPruneCause(th); - Assert.assertEquals( - pruneCauses.getBrief().substring(0, CandidateTablePruneCode.MISSING_PARTITIONS.errorFormat.length() - 3), - CandidateTablePruneCode.MISSING_PARTITIONS.errorFormat.substring(0, - CandidateTablePruneCode.MISSING_PARTITIONS.errorFormat.length() - 3) - ); - Assert.assertEquals(pruneCauses.getDetails().get("testfact").size(), 1); - Assert.assertEquals(pruneCauses.getDetails().get("testfact").iterator().next().getCause(), - CandidateTablePruneCode.MISSING_PARTITIONS); - Assert.assertEquals(pruneCauses.getDetails().get("testfactmonthly").size(), 1); - Assert.assertEquals(pruneCauses.getDetails().get("testfactmonthly").iterator().next().getCause(), - CandidateTablePruneCode.NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE); } @Test @@ -858,12 +840,20 @@ public class TestCubeRewriter extends TestQueryRewrite { CandidateTablePruneCode.MISSING_PARTITIONS); Assert.assertEquals(pruneCauses.getDetails().get("testfactmonthly").iterator().next().getCause(), CandidateTablePruneCode.MISSING_PARTITIONS); - Assert.assertEquals(pruneCauses.getDetails().get("testfact2_raw,testfact2").iterator().next().getCause(), + Assert.assertEquals(pruneCauses.getDetails().get("testfact2").iterator().next().getCause(), CandidateTablePruneCode.MISSING_PARTITIONS); + Assert.assertEquals(pruneCauses.getDetails().get("testfact2_raw").iterator().next().getCause(), + CandidateTablePruneCode.MISSING_PARTITIONS); Assert.assertEquals(pruneCauses.getDetails().get("cheapfact").iterator().next().getCause(), CandidateTablePruneCode.NO_CANDIDATE_STORAGES); - Assert.assertEquals(pruneCauses.getDetails().get("summary1,summary2,summary3,summary4").iterator().next() + Assert.assertEquals(pruneCauses.getDetails().get("summary1,summary2,summary3").iterator().next().getCause(), + CandidateTablePruneCode.MISSING_PARTITIONS); + Assert.assertEquals(pruneCauses.getDetails().get("summary4").iterator().next() .getCause(), CandidateTablePruneCode.NO_CANDIDATE_STORAGES); + Assert.assertEquals(pruneCauses.getDetails().get("summary4").iterator().next() + .getStorageCauses().values().iterator().next().getCause(), SkipStorageCode.PART_COL_DOES_NOT_EXIST); + Assert.assertEquals(pruneCauses.getDetails().get("summary4").iterator().next() + .getStorageCauses().values().iterator().next().getNonExistantPartCols(), Arrays.asList("dt")); } @Test
