LENS-885: Cleanup of Cube test cases
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7c7c86da Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7c7c86da Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7c7c86da Branch: refs/heads/current-release-line Commit: 7c7c86daed2e9907bda92f5ed29e73ed99a9a726 Parents: 7e9e47e Author: Rajat Khandelwal <pro...@apache.org> Authored: Fri Dec 11 18:40:59 2015 +0530 Committer: Rajat Khandelwal <rajatgupt...@gmail.com> Committed: Fri Dec 11 18:41:00 2015 +0530 ---------------------------------------------------------------------- .../apache/lens/cube/metadata/CubeColumn.java | 1 - .../lens/cube/metadata/CubeFactTable.java | 1 - .../org/apache/lens/cube/metadata/DateUtil.java | 396 ++++++++++++++++ .../lens/cube/metadata/TimePartitionRange.java | 1 - .../apache/lens/cube/metadata/TimeRange.java | 219 +++++++++ .../apache/lens/cube/metadata/UpdatePeriod.java | 84 +++- .../timeline/EndsAndHolesPartitionTimeline.java | 2 +- .../apache/lens/cube/parse/CandidateFact.java | 5 +- .../cube/parse/CandidateTablePruneCause.java | 2 + .../org/apache/lens/cube/parse/DateUtil.java | 456 ------------------ .../lens/cube/parse/ExpressionResolver.java | 11 +- .../lens/cube/parse/SingleFactHQLContext.java | 2 +- .../lens/cube/parse/StorageTableResolver.java | 2 +- .../org/apache/lens/cube/parse/TimeRange.java | 220 --------- .../lens/cube/parse/TimerangeResolver.java | 5 +- .../lens/cube/metadata/CubeFactTableTest.java | 1 - .../apache/lens/cube/metadata/DateFactory.java | 196 ++++++++ .../cube/metadata/TestCubeMetastoreClient.java | 115 ++--- .../apache/lens/cube/metadata/TestDateUtil.java | 297 ++++++++++++ .../apache/lens/cube/parse/CubeTestSetup.java | 191 ++------ .../FieldsCannotBeQueriedTogetherTest.java | 8 +- .../lens/cube/parse/TestAggregateResolver.java | 1 + .../lens/cube/parse/TestBaseCubeQueries.java | 5 +- .../cube/parse/TestBetweenTimeRangeWriter.java | 25 +- .../lens/cube/parse/TestCubeRewriter.java | 459 +++++++++---------- .../apache/lens/cube/parse/TestDateUtil.java | 299 ------------ .../cube/parse/TestDenormalizationResolver.java | 28 +- .../lens/cube/parse/TestExpressionContext.java | 4 +- .../lens/cube/parse/TestExpressionResolver.java | 1 + .../lens/cube/parse/TestJoinResolver.java | 1 + .../lens/cube/parse/TestORTimeRangeWriter.java | 40 +- .../lens/cube/parse/TestQueryMetrics.java | 2 +- .../lens/cube/parse/TestRewriterPlan.java | 2 +- .../apache/lens/cube/parse/TestStorageUtil.java | 98 ++-- .../lens/cube/parse/TestTimeRangeExtractor.java | 33 +- .../lens/cube/parse/TestTimeRangeResolver.java | 2 +- .../lens/cube/parse/TestTimeRangeWriter.java | 48 +- .../parse/TestTimeRangeWriterWithQuery.java | 134 +++--- .../lens/server/query/QueryResultPurger.java | 2 +- 39 files changed, 1693 insertions(+), 1706 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java index a2a00d2..b04532f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeColumn.java @@ -24,7 +24,6 @@ import java.util.Date; import java.util.Map; import java.util.TimeZone; -import org.apache.lens.cube.parse.TimeRange; import com.google.common.base.Optional; http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index d6bfb79..dd0adb7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -21,7 +21,6 @@ package org.apache.lens.cube.metadata; import java.util.*; import org.apache.lens.cube.metadata.UpdatePeriod.UpdatePeriodComparator; -import org.apache.lens.cube.parse.DateUtil; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java new file mode 100644 index 0000000..b76c567 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java @@ -0,0 +1,396 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.metadata; + +import static java.util.Calendar.MONTH; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.lens.cube.error.LensCubeErrorCode; +import org.apache.lens.server.api.error.LensException; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.time.DateUtils; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public final class DateUtil { + private DateUtil() { + + } + + /* + * NOW -> new java.util.Date() NOW-7DAY -> a date one week earlier NOW (+-) + * <NUM>UNIT or Hardcoded dates in DD-MM-YYYY hh:mm:ss,sss + */ + public static final String UNIT; + + static { + StringBuilder sb = new StringBuilder(); + String sep = ""; + for (UpdatePeriod up : UpdatePeriod.values()) { + sb.append(sep).append(up.getUnitName()); + sep = "|"; + } + UNIT = sb.toString(); + } + + public static final String GRANULARITY = "\\.(" + UNIT + ")"; + public static final String RELATIVE = "(now)(" + GRANULARITY + ")?"; + public static final Pattern P_RELATIVE = Pattern.compile(RELATIVE, Pattern.CASE_INSENSITIVE); + + public static final String WSPACE = "\\s+"; + public static final String OPTIONAL_WSPACE = "\\s*"; + + public static final String SIGNAGE = "\\+|\\-"; + public static final Pattern P_SIGNAGE = Pattern.compile(SIGNAGE); + + public static final String QUANTITY = "\\d+"; + public static final Pattern P_QUANTITY = Pattern.compile(QUANTITY); + + public static final Pattern P_UNIT = Pattern.compile(UNIT, Pattern.CASE_INSENSITIVE); + + public static final String RELDATE_VALIDATOR_STR = RELATIVE + OPTIONAL_WSPACE + "((" + SIGNAGE + ")" + "(" + + WSPACE + ")?" + "(" + QUANTITY + ")" + OPTIONAL_WSPACE + "(" + UNIT + "))?" + "(s?)"; + + public static final Pattern RELDATE_VALIDATOR = Pattern.compile(RELDATE_VALIDATOR_STR, Pattern.CASE_INSENSITIVE); + + public static final String YEAR_FMT = "[0-9]{4}"; + public static final String MONTH_FMT = YEAR_FMT + "-[0-9]{2}"; + public static final String DAY_FMT = MONTH_FMT + "-[0-9]{2}"; + public static final String HOUR_FMT = DAY_FMT + "-[0-9]{2}"; + public static final String MINUTE_FMT = HOUR_FMT + ":[0-9]{2}"; + public static final String SECOND_FMT = MINUTE_FMT + ":[0-9]{2}"; + public static final String MILLISECOND_FMT = SECOND_FMT + ",[0-9]{3}"; + public static final String ABSDATE_FMT = "yyyy-MM-dd-HH:mm:ss,SSS"; + public static final String HIVE_QUERY_DATE_FMT = "yyyy-MM-dd HH:mm:ss"; + + public static final ThreadLocal<DateFormat> ABSDATE_PARSER = + new ThreadLocal<DateFormat>() { + @Override + protected SimpleDateFormat initialValue() { + return new SimpleDateFormat(ABSDATE_FMT); + } + }; + public static final ThreadLocal<DateFormat> HIVE_QUERY_DATE_PARSER = + new ThreadLocal<DateFormat>() { + @Override + protected SimpleDateFormat initialValue() { + return new SimpleDateFormat(HIVE_QUERY_DATE_FMT); + } + }; + + public static String getAbsDateFormatString(String str) { + if (str.matches(YEAR_FMT)) { + return str + "-01-01-00:00:00,000"; + } else if (str.matches(MONTH_FMT)) { + return str + "-01-00:00:00,000"; + } else if (str.matches(DAY_FMT)) { + return str + "-00:00:00,000"; + } else if (str.matches(HOUR_FMT)) { + return str + ":00:00,000"; + } else if (str.matches(MINUTE_FMT)) { + return str + ":00,000"; + } else if (str.matches(SECOND_FMT)) { + return str + ",000"; + } else if (str.matches(MILLISECOND_FMT)) { + return str; + } + throw new IllegalArgumentException("Unsupported formatting for date" + str); + } + + public static Date resolveDate(String str, Date now) throws LensException { + if (RELDATE_VALIDATOR.matcher(str).matches()) { + return resolveRelativeDate(str, now); + } else { + return resolveAbsoluteDate(str); + } + } + + public static String relativeToAbsolute(String relative) throws LensException { + return relativeToAbsolute(relative, new Date()); + } + + public static String relativeToAbsolute(String relative, Date now) throws LensException { + if (RELDATE_VALIDATOR.matcher(relative).matches()) { + return ABSDATE_PARSER.get().format(resolveRelativeDate(relative, now)); + } else { + return relative; + } + } + + static Cache<String, Date> stringToDateCache = CacheBuilder.newBuilder() + .expireAfterWrite(2, TimeUnit.HOURS).maximumSize(100).build(); + + public static Date resolveAbsoluteDate(final String str) throws LensException { + try { + return stringToDateCache.get(str, new Callable<Date>() { + @Override + public Date call() throws ParseException { + return ABSDATE_PARSER.get().parse(getAbsDateFormatString(str)); + } + }); + } catch (Exception e) { + log.error("Invalid date format. expected only {} date provided:{}", ABSDATE_FMT, str, e); + throw new LensException(LensCubeErrorCode.WRONG_TIME_RANGE_FORMAT.getLensErrorInfo(), ABSDATE_FMT, str); + } + } + + public static Date resolveRelativeDate(String str, Date now) throws LensException { + if (StringUtils.isBlank(str)) { + throw new LensException(LensCubeErrorCode.NULL_DATE_VALUE.getLensErrorInfo()); + } + + // Resolve NOW with proper granularity + Calendar calendar = Calendar.getInstance(); + calendar.setTime(now); + + str = str.toLowerCase(); + Matcher relativeMatcher = P_RELATIVE.matcher(str); + if (relativeMatcher.find()) { + String nowWithGranularity = relativeMatcher.group(); + nowWithGranularity = nowWithGranularity.replaceAll("now", ""); + nowWithGranularity = nowWithGranularity.replaceAll("\\.", ""); + + Matcher granularityMatcher = P_UNIT.matcher(nowWithGranularity); + if (granularityMatcher.find()) { + calendar = UpdatePeriod.fromUnitName(granularityMatcher.group().toLowerCase()).truncate(calendar); + } + } + + // Get rid of 'now' part and whitespace + String diffStr = str.replaceAll(RELATIVE, "").replace(WSPACE, ""); + TimeDiff diff = TimeDiff.parseFrom(diffStr); + return diff.offsetFrom(calendar.getTime()); + } + + public static Date getCeilDate(Date date, UpdatePeriod interval) { + return interval.getCeilDate(date); + } + + public static Date getFloorDate(Date date, UpdatePeriod interval) { + return interval.getFloorDate(date); + } + + public static CoveringInfo getMonthlyCoveringInfo(Date from, Date to) { + // Move 'from' to end of month, unless its the first day of month + boolean coverable = true; + if (!from.equals(DateUtils.truncate(from, MONTH))) { + from = DateUtils.addMonths(DateUtils.truncate(from, MONTH), 1); + coverable = false; + } + + // Move 'to' to beginning of next month, unless its the first day of the month + if (!to.equals(DateUtils.truncate(to, MONTH))) { + to = DateUtils.truncate(to, MONTH); + coverable = false; + } + + int months = 0; + while (from.before(to)) { + from = DateUtils.addMonths(from, 1); + months++; + } + return new CoveringInfo(months, coverable); + } + + public static CoveringInfo getQuarterlyCoveringInfo(Date from, Date to) { + CoveringInfo monthlyCoveringInfo = getMonthlyCoveringInfo(from, to); + if (monthlyCoveringInfo.getCountBetween() < 3) { + return new CoveringInfo(0, false); + } + boolean coverable = monthlyCoveringInfo.isCoverable(); + if (!from.equals(DateUtils.truncate(from, MONTH))) { + from = DateUtils.addMonths(DateUtils.truncate(from, MONTH), 1); + coverable = false; + } + Calendar cal = Calendar.getInstance(); + cal.setTime(from); + int fromMonth = cal.get(MONTH); + + // Get the start date of the quarter + int beginOffset = (3 - fromMonth % 3) % 3; + int endOffset = (monthlyCoveringInfo.getCountBetween() - beginOffset) % 3; + if (beginOffset > 0 || endOffset > 0) { + coverable = false; + } + return new CoveringInfo((monthlyCoveringInfo.getCountBetween() - beginOffset - endOffset) / 3, coverable); + } + + + public static CoveringInfo getYearlyCoveringInfo(Date from, Date to) { + CoveringInfo monthlyCoveringInfo = getMonthlyCoveringInfo(from, to); + if (monthlyCoveringInfo.getCountBetween() < 12) { + return new CoveringInfo(0, false); + } + boolean coverable = monthlyCoveringInfo.isCoverable(); + if (!from.equals(DateUtils.truncate(from, MONTH))) { + from = DateUtils.addMonths(DateUtils.truncate(from, MONTH), 1); + coverable = false; + } + Calendar cal = Calendar.getInstance(); + cal.setTime(from); + int fromMonth = cal.get(MONTH); + int beginOffset = (12 - fromMonth % 12) % 12; + int endOffset = (monthlyCoveringInfo.getCountBetween() - beginOffset) % 12; + if (beginOffset > 0 || endOffset > 0) { + coverable = false; + } + return new CoveringInfo((monthlyCoveringInfo.getCountBetween() - beginOffset - endOffset) / 12, coverable); + } + + public static CoveringInfo getWeeklyCoveringInfo(Date from, Date to) { + int dayDiff = 0; + Date tmpFrom = from; + while (tmpFrom.before(to)) { + tmpFrom = DateUtils.addDays(tmpFrom, 1); + dayDiff++; + } + + if (dayDiff < 7) { + return new CoveringInfo(0, false); + } + + Calendar cal = Calendar.getInstance(); + cal.setTime(from); + int fromDay = cal.get(Calendar.DAY_OF_WEEK); + cal.set(Calendar.DAY_OF_WEEK, Calendar.SUNDAY); + Date fromWeekStartDate = cal.getTime(); + boolean coverable = dayDiff % 7 == 0; + if (fromWeekStartDate.before(from)) { + // Count from the start of next week + dayDiff -= (cal.getActualMaximum(Calendar.DAY_OF_WEEK) - (fromDay - Calendar.SUNDAY)); + coverable = false; + } + + return new CoveringInfo(dayDiff / 7, coverable); + } + + static CoveringInfo getCoveringInfo(Date from, Date to, UpdatePeriod interval) { + switch (interval) { + case SECONDLY: + case CONTINUOUS: + return getMilliSecondCoveringInfo(from, to, 1000); + case MINUTELY: + case HOURLY: + case DAILY: + return getMilliSecondCoveringInfo(from, to, interval.weight()); + case WEEKLY: + return getWeeklyCoveringInfo(from, to); + case MONTHLY: + return getMonthlyCoveringInfo(from, to); + case QUARTERLY: + return getQuarterlyCoveringInfo(from, to); + case YEARLY: + return getYearlyCoveringInfo(from, to); + default: + return new CoveringInfo(0, false); + } + } + + private static CoveringInfo getMilliSecondCoveringInfo(Date from, Date to, long millisInInterval) { + long diff = to.getTime() - from.getTime(); + return new CoveringInfo((int) (diff / millisInInterval), diff % millisInInterval == 0); + } + + static boolean isCoverableBy(Date from, Date to, Set<UpdatePeriod> intervals) { + for (UpdatePeriod period : intervals) { + if (getCoveringInfo(from, to, period).isCoverable()) { + return true; + } + } + return false; + } + + public static int getTimeDiff(Date fromDate, Date toDate, UpdatePeriod updatePeriod) { + if (fromDate.before(toDate)) { + return getCoveringInfo(fromDate, toDate, updatePeriod).getCountBetween(); + } else { + return -getCoveringInfo(toDate, fromDate, updatePeriod).getCountBetween(); + } + } + + @Data + public static class CoveringInfo { + int countBetween; + boolean coverable; + + public CoveringInfo(int countBetween, boolean coverable) { + this.countBetween = countBetween; + this.coverable = coverable; + } + } + + @EqualsAndHashCode + public static class TimeDiff { + int quantity; + UpdatePeriod updatePeriod; + + private TimeDiff(int quantity, UpdatePeriod updatePeriod) { + this.quantity = quantity; + this.updatePeriod = updatePeriod; + } + + public static TimeDiff parseFrom(String diffStr) throws LensException { + // Get the relative diff part to get eventual date based on now. + Matcher qtyMatcher = P_QUANTITY.matcher(diffStr); + int qty = 1; + if (qtyMatcher.find()) { + qty = Integer.parseInt(qtyMatcher.group()); + } + + Matcher signageMatcher = P_SIGNAGE.matcher(diffStr); + if (signageMatcher.find()) { + String sign = signageMatcher.group(); + if ("-".equals(sign)) { + qty = -qty; + } + } + + Matcher unitMatcher = P_UNIT.matcher(diffStr); + if (unitMatcher.find()) { + return new TimeDiff(qty, UpdatePeriod.fromUnitName(unitMatcher.group().toLowerCase())); + } + return new TimeDiff(0, UpdatePeriod.CONTINUOUS); + } + + public Date offsetFrom(Date time) { + return DateUtils.add(time, updatePeriod.calendarField(), quantity); + } + + public Date negativeOffsetFrom(Date time) { + return DateUtils.add(time, updatePeriod.calendarField(), -quantity); + } + } + +} http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java index 01069a5..2e85111 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java @@ -21,7 +21,6 @@ package org.apache.lens.cube.metadata; import java.util.Date; import java.util.Iterator; -import org.apache.lens.cube.parse.DateUtil; import org.apache.lens.server.api.error.LensException; import lombok.Data; http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java new file mode 100644 index 0000000..bf6cc5c --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.metadata; + +import static org.apache.lens.cube.metadata.DateUtil.ABSDATE_PARSER; + +import java.util.Calendar; +import java.util.Date; +import java.util.TreeSet; + +import org.apache.lens.cube.error.LensCubeErrorCode; +import org.apache.lens.server.api.error.LensException; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.ql.parse.ASTNode; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; + +import lombok.Data; +import lombok.Getter; + +/** + * Timerange data structure + */ +@JsonIgnoreProperties({"astNode", "parent"}) +@Data +public class TimeRange { + private String partitionColumn; + private Date toDate; + private Date fromDate; + private ASTNode astNode; + private ASTNode parent; + private int childIndex; + + public boolean isCoverableBy(TreeSet<UpdatePeriod> updatePeriods) { + return DateUtil.isCoverableBy(fromDate, toDate, updatePeriods); + } + + + public static class TimeRangeBuilder { + private final TimeRange range; + + public TimeRangeBuilder() { + this.range = new TimeRange(); + } + + public TimeRangeBuilder partitionColumn(String col) { + range.partitionColumn = col; + return this; + } + + public TimeRangeBuilder toDate(Date to) { + range.toDate = to; + return this; + } + + public TimeRangeBuilder fromDate(Date from) { + range.fromDate = from; + return this; + } + + public TimeRangeBuilder astNode(ASTNode node) { + range.astNode = node; + return this; + } + + public TimeRangeBuilder parent(ASTNode parent) { + range.parent = parent; + return this; + } + + public TimeRangeBuilder childIndex(int childIndex) { + range.childIndex = childIndex; + return this; + } + + public TimeRange build() { + return range; + } + } + + public static TimeRangeBuilder getBuilder() { + return new TimeRangeBuilder(); + } + + private TimeRange() { + + } + + public void validate() throws LensException { + if (partitionColumn == null || fromDate == null || toDate == null || fromDate.equals(toDate)) { + throw new LensException(LensCubeErrorCode.INVALID_TIME_RANGE.getLensErrorInfo()); + } + + if (fromDate.after(toDate)) { + throw new LensException(LensCubeErrorCode.FROM_AFTER_TO.getLensErrorInfo(), + fromDate.toString(), toDate.toString()); + } + } + + public String toTimeDimWhereClause() { + return toTimeDimWhereClause(null, partitionColumn); + } + + public String toTimeDimWhereClause(String prefix, String column) { + if (StringUtils.isNotBlank(column)) { + column = prefix + "." + column; + } + return new StringBuilder() + .append(column).append(" >= '").append(DateUtil.HIVE_QUERY_DATE_PARSER.get().format(fromDate)).append("'") + .append(" AND ") + .append(column).append(" < '").append(DateUtil.HIVE_QUERY_DATE_PARSER.get().format(toDate)).append("'") + .toString(); + } + + @Override + public String toString() { + return partitionColumn + " [" + ABSDATE_PARSER.get().format(fromDate) + " to " + + ABSDATE_PARSER.get().format(toDate) + ")"; + } + + /** iterable from fromDate(including) to toDate(excluding) incrementing increment units of updatePeriod */ + public static Iterable iterable(Date fromDate, Date toDate, UpdatePeriod updatePeriod, int increment) { + return TimeRange.getBuilder().fromDate(fromDate).toDate(toDate).build().iterable(updatePeriod, increment); + } + + /** iterable from fromDate(including) incrementing increment units of updatePeriod. Do this numIters times */ + public static Iterable iterable(Date fromDate, int numIters, UpdatePeriod updatePeriod, int increment) { + return TimeRange.getBuilder().fromDate(fromDate).build().iterable(updatePeriod, numIters, increment); + } + + private Iterable iterable(UpdatePeriod updatePeriod, int numIters, int increment) { + return new Iterable(updatePeriod, numIters, increment); + } + + public Iterable iterable(UpdatePeriod updatePeriod, int increment) { + if (increment == 0) { + throw new UnsupportedOperationException("Can't iterate if iteration increment is zero"); + } + long numIters = DateUtil.getTimeDiff(fromDate, toDate, updatePeriod) / increment; + return new Iterable(updatePeriod, numIters, increment); + } + + /** Iterable so that foreach is supported */ + public class Iterable implements java.lang.Iterable<Date> { + private UpdatePeriod updatePeriod; + private long numIters; + private int increment; + + public Iterable(UpdatePeriod updatePeriod, long numIters, int increment) { + this.updatePeriod = updatePeriod; + this.numIters = numIters; + if (this.numIters < 0) { + this.numIters = 0; + } + this.increment = increment; + } + + @Override + public Iterator iterator() { + return new Iterator(); + } + + public class Iterator implements java.util.Iterator<Date> { + Calendar calendar; + // Tracks the index of the item returned after the last next() call. + // Index here refers to the index if the iterator were iterated and converted into a list. + @Getter + int counter = -1; + + public Iterator() { + calendar = Calendar.getInstance(); + calendar.setTime(fromDate); + } + + @Override + public boolean hasNext() { + return counter < numIters - 1; + } + + @Override + public Date next() { + Date cur = calendar.getTime(); + updatePeriod.increment(calendar, increment); + counter++; + return cur; + } + + public Date peekNext() { + return calendar.getTime(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException("remove from timerange iterator"); + } + + public long getNumIters() { + return numIters; + } + } + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java index 4c76a69..4238066 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java @@ -278,7 +278,7 @@ public enum UpdatePeriod implements Named { return cal.getTime(); case QUARTERLY: Date dt = DateUtils.truncate(date, this.calendarField()); - dt.setMonth(dt.getMonth() - dt.getMonth() % 3); + dt.setMonth(dt.getMonth() - (dt.getMonth() % 3)); return dt; default: return DateUtils.truncate(date, this.calendarField()); @@ -299,6 +299,86 @@ public enum UpdatePeriod implements Named { calendar.add(calendarField(), increment); } + public Date getCeilDate(Date date) { + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + boolean hasFraction = false; + switch (this) { + case YEARLY: + if (cal.get(MONTH) != 0) { + hasFraction = true; + break; + } + case MONTHLY: + if (cal.get(DAY_OF_MONTH) != 1) { + hasFraction = true; + break; + } + case DAILY: + if (cal.get(Calendar.HOUR_OF_DAY) != 0) { + hasFraction = true; + break; + } + case HOURLY: + if (cal.get(Calendar.MINUTE) != 0) { + hasFraction = true; + break; + } + case MINUTELY: + if (cal.get(Calendar.SECOND) != 0) { + hasFraction = true; + break; + } + case SECONDLY: + case CONTINUOUS: + if (cal.get(Calendar.MILLISECOND) != 0) { + hasFraction = true; + } + break; + case WEEKLY: + if (cal.get(Calendar.DAY_OF_WEEK) != 1) { + hasFraction = true; + break; + } + } + + if (hasFraction) { + cal.add(this.calendarField(), 1); + return getFloorDate(cal.getTime()); + } else { + return date; + } + } + + public Date getFloorDate(Date date) { + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + switch(this) { + case WEEKLY: + cal.set(Calendar.DAY_OF_WEEK, 1); + break; + } + switch (this) { + case YEARLY: + cal.set(MONTH, 0); + case MONTHLY: + cal.set(DAY_OF_MONTH, 1); + case WEEKLY: + // Already covered, only here for fall through cases + case DAILY: + cal.set(Calendar.HOUR_OF_DAY, 0); + case HOURLY: + cal.set(Calendar.MINUTE, 0); + case MINUTELY: + cal.set(Calendar.SECOND, 0); + case SECONDLY: + case CONTINUOUS: + cal.set(Calendar.MILLISECOND, 0); + break; + } + return cal.getTime(); + } + public static class UpdatePeriodComparator implements Comparator<UpdatePeriod> { @Override public int compare(UpdatePeriod o1, UpdatePeriod o2) { @@ -306,7 +386,7 @@ public enum UpdatePeriod implements Named { return -1; } else if (o1 != null && o2 == null) { return 1; - } else if (o1 == null && o2 == null) { + } else if (o1 == null) { return 0; } else { if (o1.weight > o2.weight) { http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java index 9d5e264..c588dc7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java @@ -23,8 +23,8 @@ import java.util.*; import org.apache.lens.cube.metadata.MetastoreUtil; import org.apache.lens.cube.metadata.TimePartition; +import org.apache.lens.cube.metadata.TimeRange; import org.apache.lens.cube.metadata.UpdatePeriod; -import org.apache.lens.cube.parse.TimeRange; import org.apache.lens.server.api.error.LensException; import com.google.common.base.Strings; http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java index 7f81461..1884bde 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java @@ -22,10 +22,7 @@ import static org.apache.hadoop.hive.ql.parse.HiveParser.*; import java.util.*; -import org.apache.lens.cube.metadata.AbstractCubeTable; -import org.apache.lens.cube.metadata.CubeFactTable; -import org.apache.lens.cube.metadata.CubeInterface; -import org.apache.lens.cube.metadata.FactPartition; +import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.parse.HQLParser.ASTNodeVisitor; import org.apache.lens.cube.parse.HQLParser.TreeNode; import org.apache.lens.server.api.error.LensException; http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java index 9c8b5b9..78fb21d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java @@ -22,6 +22,8 @@ import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTable import java.util.*; +import org.apache.lens.cube.metadata.TimeRange; + import org.codehaus.jackson.annotate.JsonWriteNullProperties; import com.google.common.collect.Lists; http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java deleted file mode 100644 index cd05c68..0000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java +++ /dev/null @@ -1,456 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.cube.parse; - -import static java.util.Calendar.*; - -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.lens.cube.error.LensCubeErrorCode; -import org.apache.lens.cube.metadata.UpdatePeriod; -import org.apache.lens.server.api.error.LensException; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.time.DateUtils; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public final class DateUtil { - private DateUtil() { - - } - /* - * NOW -> new java.util.Date() NOW-7DAY -> a date one week earlier NOW (+-) - * <NUM>UNIT or Hardcoded dates in DD-MM-YYYY hh:mm:ss,sss - */ - public static final String UNIT; - - static { - StringBuilder sb = new StringBuilder(); - String sep = ""; - for (UpdatePeriod up : UpdatePeriod.values()) { - sb.append(sep).append(up.getUnitName()); - sep = "|"; - } - UNIT = sb.toString(); - } - - public static final String GRANULARITY = "\\.(" + UNIT + ")"; - public static final String RELATIVE = "(now)(" + GRANULARITY + ")?"; - public static final Pattern P_RELATIVE = Pattern.compile(RELATIVE, Pattern.CASE_INSENSITIVE); - - public static final String WSPACE = "\\s+"; - public static final String OPTIONAL_WSPACE = "\\s*"; - - public static final String SIGNAGE = "\\+|\\-"; - public static final Pattern P_SIGNAGE = Pattern.compile(SIGNAGE); - - public static final String QUANTITY = "\\d+"; - public static final Pattern P_QUANTITY = Pattern.compile(QUANTITY); - - public static final Pattern P_UNIT = Pattern.compile(UNIT, Pattern.CASE_INSENSITIVE); - - public static final String RELDATE_VALIDATOR_STR = RELATIVE + OPTIONAL_WSPACE + "((" + SIGNAGE + ")" + "(" - + WSPACE + ")?" + "(" + QUANTITY + ")" + OPTIONAL_WSPACE + "(" + UNIT + "))?" + "(s?)"; - - public static final Pattern RELDATE_VALIDATOR = Pattern.compile(RELDATE_VALIDATOR_STR, Pattern.CASE_INSENSITIVE); - - public static final String YEAR_FMT = "[0-9]{4}"; - public static final String MONTH_FMT = YEAR_FMT + "-[0-9]{2}"; - public static final String DAY_FMT = MONTH_FMT + "-[0-9]{2}"; - public static final String HOUR_FMT = DAY_FMT + "-[0-9]{2}"; - public static final String MINUTE_FMT = HOUR_FMT + ":[0-9]{2}"; - public static final String SECOND_FMT = MINUTE_FMT + ":[0-9]{2}"; - public static final String MILLISECOND_FMT = SECOND_FMT + ",[0-9]{3}"; - public static final String ABSDATE_FMT = "yyyy-MM-dd-HH:mm:ss,SSS"; - public static final String HIVE_QUERY_DATE_FMT = "yyyy-MM-dd HH:mm:ss"; - - public static final ThreadLocal<DateFormat> ABSDATE_PARSER = - new ThreadLocal<DateFormat>() { - @Override - protected SimpleDateFormat initialValue() { - return new SimpleDateFormat(ABSDATE_FMT); - } - }; - public static final ThreadLocal<DateFormat> HIVE_QUERY_DATE_PARSER = - new ThreadLocal<DateFormat>() { - @Override - protected SimpleDateFormat initialValue() { - return new SimpleDateFormat(HIVE_QUERY_DATE_FMT); - } - }; - - public static String getAbsDateFormatString(String str) { - if (str.matches(YEAR_FMT)) { - return str + "-01-01-00:00:00,000"; - } else if (str.matches(MONTH_FMT)) { - return str + "-01-00:00:00,000"; - } else if (str.matches(DAY_FMT)) { - return str + "-00:00:00,000"; - } else if (str.matches(HOUR_FMT)) { - return str + ":00:00,000"; - } else if (str.matches(MINUTE_FMT)) { - return str + ":00,000"; - } else if (str.matches(SECOND_FMT)) { - return str + ",000"; - } else if (str.matches(MILLISECOND_FMT)) { - return str; - } - throw new IllegalArgumentException("Unsupported formatting for date" + str); - } - - public static Date resolveDate(String str, Date now) throws LensException { - if (RELDATE_VALIDATOR.matcher(str).matches()) { - return resolveRelativeDate(str, now); - } else { - return resolveAbsoluteDate(str); - } - } - - public static String relativeToAbsolute(String relative) throws LensException { - return relativeToAbsolute(relative, new Date()); - } - - public static String relativeToAbsolute(String relative, Date now) throws LensException { - if (RELDATE_VALIDATOR.matcher(relative).matches()) { - return ABSDATE_PARSER.get().format(resolveRelativeDate(relative, now)); - } else { - return relative; - } - } - - public static Date resolveAbsoluteDate(String str) throws LensException { - try { - return ABSDATE_PARSER.get().parse(getAbsDateFormatString(str)); - } catch (ParseException e) { - log.error("Invalid date format. expected only {} date provided:{}", ABSDATE_FMT, str, e); - throw new LensException(LensCubeErrorCode.WRONG_TIME_RANGE_FORMAT.getLensErrorInfo(), ABSDATE_FMT, str); - } - } - - public static Date resolveRelativeDate(String str, Date now) throws LensException { - if (StringUtils.isBlank(str)) { - throw new LensException(LensCubeErrorCode.NULL_DATE_VALUE.getLensErrorInfo()); - } - - // Resolve NOW with proper granularity - Calendar calendar = Calendar.getInstance(); - calendar.setTime(now); - - str = str.toLowerCase(); - Matcher relativeMatcher = P_RELATIVE.matcher(str); - if (relativeMatcher.find()) { - String nowWithGranularity = relativeMatcher.group(); - nowWithGranularity = nowWithGranularity.replaceAll("now", ""); - nowWithGranularity = nowWithGranularity.replaceAll("\\.", ""); - - Matcher granularityMatcher = P_UNIT.matcher(nowWithGranularity); - if (granularityMatcher.find()) { - calendar = UpdatePeriod.fromUnitName(granularityMatcher.group().toLowerCase()).truncate(calendar); - } - } - - // Get rid of 'now' part and whitespace - String diffStr = str.replaceAll(RELATIVE, "").replace(WSPACE, ""); - TimeDiff diff = TimeDiff.parseFrom(diffStr); - return diff.offsetFrom(calendar.getTime()); - } - - public static Date getCeilDate(Date fromDate, UpdatePeriod interval) { - Calendar cal = Calendar.getInstance(); - cal.setTime(fromDate); - boolean hasFraction = false; - switch (interval) { - case YEARLY: - if (cal.get(MONTH) != 0) { - hasFraction = true; - break; - } - case MONTHLY: - if (cal.get(DAY_OF_MONTH) != 1) { - hasFraction = true; - break; - } - case DAILY: - if (cal.get(Calendar.HOUR_OF_DAY) != 0) { - hasFraction = true; - break; - } - case HOURLY: - if (cal.get(Calendar.MINUTE) != 0) { - hasFraction = true; - break; - } - case MINUTELY: - if (cal.get(Calendar.SECOND) != 0) { - hasFraction = true; - break; - } - case SECONDLY: - case CONTINUOUS: - if (cal.get(Calendar.MILLISECOND) != 0) { - hasFraction = true; - } - break; - case WEEKLY: - if (cal.get(Calendar.DAY_OF_WEEK) != 1) { - hasFraction = true; - break; - } - } - - if (hasFraction) { - cal.add(interval.calendarField(), 1); - return getFloorDate(cal.getTime(), interval); - } else { - return fromDate; - } - } - - public static Date getFloorDate(Date toDate, UpdatePeriod interval) { - Calendar cal = Calendar.getInstance(); - cal.setTime(toDate); - switch (interval) { - case YEARLY: - cal.set(MONTH, 0); - case MONTHLY: - cal.set(DAY_OF_MONTH, 1); - case DAILY: - cal.set(Calendar.HOUR_OF_DAY, 0); - case HOURLY: - cal.set(Calendar.MINUTE, 0); - case MINUTELY: - cal.set(Calendar.SECOND, 0); - case SECONDLY: - case CONTINUOUS: - cal.set(Calendar.MILLISECOND, 0); - break; - case WEEKLY: - cal.set(Calendar.DAY_OF_WEEK, 1); - cal.set(Calendar.HOUR_OF_DAY, 0); - cal.set(Calendar.MINUTE, 0); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - break; - } - return cal.getTime(); - } - - public static CoveringInfo getMonthlyCoveringInfo(Date from, Date to) { - // Move 'from' to end of month, unless its the first day of month - boolean coverable = true; - if (!from.equals(DateUtils.truncate(from, MONTH))) { - from = DateUtils.addMonths(DateUtils.truncate(from, MONTH), 1); - coverable = false; - } - - // Move 'to' to beginning of next month, unless its the first day of the month - if (!to.equals(DateUtils.truncate(to, MONTH))) { - to = DateUtils.truncate(to, MONTH); - coverable = false; - } - - int months = 0; - while (from.before(to)) { - from = DateUtils.addMonths(from, 1); - months++; - } - return new CoveringInfo(months, coverable); - } - - public static CoveringInfo getQuarterlyCoveringInfo(Date from, Date to) { - CoveringInfo monthlyCoveringInfo = getMonthlyCoveringInfo(from, to); - if (monthlyCoveringInfo.getCountBetween() < 3) { - return new CoveringInfo(0, false); - } - boolean coverable = monthlyCoveringInfo.isCoverable(); - if (!from.equals(DateUtils.truncate(from, MONTH))) { - from = DateUtils.addMonths(DateUtils.truncate(from, MONTH), 1); - coverable = false; - } - Calendar cal = Calendar.getInstance(); - cal.setTime(from); - int fromMonth = cal.get(MONTH); - - // Get the start date of the quarter - int beginOffset = (3 - fromMonth % 3) % 3; - int endOffset = (monthlyCoveringInfo.getCountBetween() - beginOffset) % 3; - if (beginOffset > 0 || endOffset > 0) { - coverable = false; - } - return new CoveringInfo((monthlyCoveringInfo.getCountBetween() - beginOffset - endOffset) / 3, coverable); - } - - - public static CoveringInfo getYearlyCoveringInfo(Date from, Date to) { - CoveringInfo monthlyCoveringInfo = getMonthlyCoveringInfo(from, to); - if (monthlyCoveringInfo.getCountBetween() < 12) { - return new CoveringInfo(0, false); - } - boolean coverable = monthlyCoveringInfo.isCoverable(); - if (!from.equals(DateUtils.truncate(from, MONTH))) { - from = DateUtils.addMonths(DateUtils.truncate(from, MONTH), 1); - coverable = false; - } - Calendar cal = Calendar.getInstance(); - cal.setTime(from); - int fromMonth = cal.get(MONTH); - int beginOffset = (12 - fromMonth % 12) % 12; - int endOffset = (monthlyCoveringInfo.getCountBetween() - beginOffset) % 12; - if (beginOffset > 0 || endOffset > 0) { - coverable = false; - } - return new CoveringInfo((monthlyCoveringInfo.getCountBetween() - beginOffset - endOffset) / 12, coverable); - } - - public static CoveringInfo getWeeklyCoveringInfo(Date from, Date to) { - int dayDiff = 0; - Date tmpFrom = from; - while (tmpFrom.before(to)) { - tmpFrom = DateUtils.addDays(tmpFrom, 1); - dayDiff++; - } - - if (dayDiff < 7) { - return new CoveringInfo(0, false); - } - - Calendar cal = Calendar.getInstance(); - cal.setTime(from); - int fromDay = cal.get(Calendar.DAY_OF_WEEK); - cal.set(Calendar.DAY_OF_WEEK, Calendar.SUNDAY); - Date fromWeekStartDate = cal.getTime(); - boolean coverable = dayDiff % 7 == 0; - if (fromWeekStartDate.before(from)) { - // Count from the start of next week - dayDiff -= (cal.getActualMaximum(Calendar.DAY_OF_WEEK) - (fromDay - Calendar.SUNDAY)); - coverable = false; - } - - return new CoveringInfo(dayDiff / 7, coverable); - } - - static CoveringInfo getCoveringInfo(Date from, Date to, UpdatePeriod interval) { - switch (interval) { - case SECONDLY: - case CONTINUOUS: - return getMilliSecondCoveringInfo(from, to, 1000); - case MINUTELY: - case HOURLY: - case DAILY: - return getMilliSecondCoveringInfo(from, to, interval.weight()); - case WEEKLY: - return getWeeklyCoveringInfo(from, to); - case MONTHLY: - return getMonthlyCoveringInfo(from, to); - case QUARTERLY: - return getQuarterlyCoveringInfo(from, to); - case YEARLY: - return getYearlyCoveringInfo(from, to); - default: - return new CoveringInfo(0, false); - } - } - - private static CoveringInfo getMilliSecondCoveringInfo(Date from, Date to, long millisInInterval) { - long diff = to.getTime() - from.getTime(); - return new CoveringInfo((int) (diff / millisInInterval), diff % millisInInterval == 0); - } - - static boolean isCoverableBy(Date from, Date to, Set<UpdatePeriod> intervals) { - for (UpdatePeriod period : intervals) { - if (getCoveringInfo(from, to, period).isCoverable()) { - return true; - } - } - return false; - } - - public static int getTimeDiff(Date fromDate, Date toDate, UpdatePeriod updatePeriod) { - if (fromDate.before(toDate)) { - return getCoveringInfo(fromDate, toDate, updatePeriod).getCountBetween(); - } else { - return -getCoveringInfo(toDate, fromDate, updatePeriod).getCountBetween(); - } - } - - @Data - public static class CoveringInfo { - int countBetween; - boolean coverable; - - public CoveringInfo(int countBetween, boolean coverable) { - this.countBetween = countBetween; - this.coverable = coverable; - } - } - - @EqualsAndHashCode - public static class TimeDiff { - int quantity; - UpdatePeriod updatePeriod; - - private TimeDiff(int quantity, UpdatePeriod updatePeriod) { - this.quantity = quantity; - this.updatePeriod = updatePeriod; - } - - public static TimeDiff parseFrom(String diffStr) throws LensException { - // Get the relative diff part to get eventual date based on now. - Matcher qtyMatcher = P_QUANTITY.matcher(diffStr); - int qty = 1; - if (qtyMatcher.find()) { - qty = Integer.parseInt(qtyMatcher.group()); - } - - Matcher signageMatcher = P_SIGNAGE.matcher(diffStr); - if (signageMatcher.find()) { - String sign = signageMatcher.group(); - if ("-".equals(sign)) { - qty = -qty; - } - } - - Matcher unitMatcher = P_UNIT.matcher(diffStr); - if (unitMatcher.find()) { - return new TimeDiff(qty, UpdatePeriod.fromUnitName(unitMatcher.group().toLowerCase())); - } - return new TimeDiff(0, UpdatePeriod.CONTINUOUS); - } - - public Date offsetFrom(Date time) { - return DateUtils.add(time, updatePeriod.calendarField(), quantity); - } - - public Date negativeOffsetFrom(Date time) { - return DateUtils.add(time, updatePeriod.calendarField(), -quantity); - } - } - -} http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index 200a48c..776021d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -19,18 +19,11 @@ package org.apache.lens.cube.parse; -import static org.apache.hadoop.hive.ql.parse.HiveParser.DOT; -import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier; -import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL; +import static org.apache.hadoop.hive.ql.parse.HiveParser.*; import java.util.*; -import org.apache.lens.cube.metadata.AbstractBaseTable; -import org.apache.lens.cube.metadata.AbstractCubeTable; -import org.apache.lens.cube.metadata.CubeColumn; -import org.apache.lens.cube.metadata.CubeInterface; -import org.apache.lens.cube.metadata.Dimension; -import org.apache.lens.cube.metadata.ExprColumn; +import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.cube.parse.HQLParser.ASTNodeVisitor; http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java index 60b2dde..f7271e5 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java @@ -21,7 +21,7 @@ package org.apache.lens.cube.parse; import java.util.Map; import org.apache.lens.cube.metadata.Dimension; - +import org.apache.lens.cube.metadata.TimeRange; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index cc8e68c..62cc071 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -18,6 +18,7 @@ */ package org.apache.lens.cube.parse; +import static org.apache.lens.cube.metadata.DateUtil.WSPACE; import static org.apache.lens.cube.metadata.MetastoreUtil.getFactOrDimtableStorageTableName; import static org.apache.lens.cube.metadata.MetastoreUtil.getStoragetableEndTimesKey; import static org.apache.lens.cube.metadata.MetastoreUtil.getStoragetableStartTimesKey; @@ -25,7 +26,6 @@ import static org.apache.lens.cube.parse.CandidateTablePruneCause.*; import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*; import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.PART_COL_DOES_NOT_EXIST; import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.RANGE_NOT_ANSWERABLE; -import static org.apache.lens.cube.parse.DateUtil.WSPACE; import static org.apache.lens.cube.parse.StorageUtil.joinWithAnd; import java.text.DateFormat; http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRange.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRange.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRange.java deleted file mode 100644 index 7be7ace..0000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRange.java +++ /dev/null @@ -1,220 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.cube.parse; - -import static org.apache.lens.cube.parse.DateUtil.ABSDATE_PARSER; - -import java.util.Calendar; -import java.util.Date; -import java.util.TreeSet; - -import org.apache.lens.cube.error.LensCubeErrorCode; -import org.apache.lens.cube.metadata.UpdatePeriod; -import org.apache.lens.server.api.error.LensException; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hive.ql.parse.ASTNode; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; - -import lombok.Data; -import lombok.Getter; - -/** - * Timerange data structure - */ -@JsonIgnoreProperties({"astNode", "parent"}) -@Data -public class TimeRange { - private String partitionColumn; - private Date toDate; - private Date fromDate; - private ASTNode astNode; - private ASTNode parent; - private int childIndex; - - public boolean isCoverableBy(TreeSet<UpdatePeriod> updatePeriods) { - return DateUtil.isCoverableBy(fromDate, toDate, updatePeriods); - } - - - public static class TimeRangeBuilder { - private final TimeRange range; - - public TimeRangeBuilder() { - this.range = new TimeRange(); - } - - public TimeRangeBuilder partitionColumn(String col) { - range.partitionColumn = col; - return this; - } - - public TimeRangeBuilder toDate(Date to) { - range.toDate = to; - return this; - } - - public TimeRangeBuilder fromDate(Date from) { - range.fromDate = from; - return this; - } - - public TimeRangeBuilder astNode(ASTNode node) { - range.astNode = node; - return this; - } - - public TimeRangeBuilder parent(ASTNode parent) { - range.parent = parent; - return this; - } - - public TimeRangeBuilder childIndex(int childIndex) { - range.childIndex = childIndex; - return this; - } - - public TimeRange build() { - return range; - } - } - - public static TimeRangeBuilder getBuilder() { - return new TimeRangeBuilder(); - } - - private TimeRange() { - - } - - public void validate() throws LensException { - if (partitionColumn == null || fromDate == null || toDate == null || fromDate.equals(toDate)) { - throw new LensException(LensCubeErrorCode.INVALID_TIME_RANGE.getLensErrorInfo()); - } - - if (fromDate.after(toDate)) { - throw new LensException(LensCubeErrorCode.FROM_AFTER_TO.getLensErrorInfo(), - fromDate.toString(), toDate.toString()); - } - } - - public String toTimeDimWhereClause() { - return toTimeDimWhereClause(null, partitionColumn); - } - - public String toTimeDimWhereClause(String prefix, String column) { - if (StringUtils.isNotBlank(column)) { - column = prefix + "." + column; - } - return new StringBuilder() - .append(column).append(" >= '").append(DateUtil.HIVE_QUERY_DATE_PARSER.get().format(fromDate)).append("'") - .append(" AND ") - .append(column).append(" < '").append(DateUtil.HIVE_QUERY_DATE_PARSER.get().format(toDate)).append("'") - .toString(); - } - - @Override - public String toString() { - return partitionColumn + " [" + ABSDATE_PARSER.get().format(fromDate) + " to " - + ABSDATE_PARSER.get().format(toDate) + ")"; - } - - /** iterable from fromDate(including) to toDate(excluding) incrementing increment units of updatePeriod */ - public static Iterable iterable(Date fromDate, Date toDate, UpdatePeriod updatePeriod, int increment) { - return TimeRange.getBuilder().fromDate(fromDate).toDate(toDate).build().iterable(updatePeriod, increment); - } - - /** iterable from fromDate(including) incrementing increment units of updatePeriod. Do this numIters times */ - public static Iterable iterable(Date fromDate, int numIters, UpdatePeriod updatePeriod, int increment) { - return TimeRange.getBuilder().fromDate(fromDate).build().iterable(updatePeriod, numIters, increment); - } - - private Iterable iterable(UpdatePeriod updatePeriod, int numIters, int increment) { - return new Iterable(updatePeriod, numIters, increment); - } - - public Iterable iterable(UpdatePeriod updatePeriod, int increment) { - if (increment == 0) { - throw new UnsupportedOperationException("Can't iterate if iteration increment is zero"); - } - long numIters = DateUtil.getTimeDiff(fromDate, toDate, updatePeriod) / increment; - return new Iterable(updatePeriod, numIters, increment); - } - - /** Iterable so that foreach is supported */ - public class Iterable implements java.lang.Iterable<Date> { - private UpdatePeriod updatePeriod; - private long numIters; - private int increment; - - public Iterable(UpdatePeriod updatePeriod, long numIters, int increment) { - this.updatePeriod = updatePeriod; - this.numIters = numIters; - if (this.numIters < 0) { - this.numIters = 0; - } - this.increment = increment; - } - - @Override - public Iterator iterator() { - return new Iterator(); - } - - public class Iterator implements java.util.Iterator<Date> { - Calendar calendar; - // Tracks the index of the item returned after the last next() call. - // Index here refers to the index if the iterator were iterated and converted into a list. - @Getter - int counter = -1; - - public Iterator() { - calendar = Calendar.getInstance(); - calendar.setTime(fromDate); - } - - @Override - public boolean hasNext() { - return counter < numIters - 1; - } - - @Override - public Date next() { - Date cur = calendar.getTime(); - updatePeriod.increment(calendar, increment); - counter++; - return cur; - } - - public Date peekNext() { - return calendar.getTime(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException("remove from timerange iterator"); - } - - public long getNumIters() { - return numIters; - } - } - } -} http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java index f772279..1a83d09 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimerangeResolver.java @@ -27,10 +27,7 @@ import java.util.*; import org.apache.lens.cube.error.ColUnAvailableInTimeRange; import org.apache.lens.cube.error.ColUnAvailableInTimeRangeException; import org.apache.lens.cube.error.LensCubeErrorCode; -import org.apache.lens.cube.metadata.AbstractCubeTable; -import org.apache.lens.cube.metadata.CubeColumn; -import org.apache.lens.cube.metadata.Dimension; -import org.apache.lens.cube.metadata.SchemaGraph; +import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.parse.DenormalizationResolver.ReferencedQueriedColumn; import org.apache.lens.server.api.error.LensException; http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java index 2abc6d0..25eaaef 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java @@ -26,7 +26,6 @@ import java.util.Date; import java.util.HashMap; import java.util.Map; -import org.apache.lens.cube.parse.DateUtil; import org.apache.lens.server.api.error.LensException; import org.testng.annotations.DataProvider; http://git-wip-us.apache.org/repos/asf/lens/blob/7c7c86da/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java new file mode 100644 index 0000000..87e4ce3 --- /dev/null +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.metadata; + + +import static org.apache.lens.cube.metadata.UpdatePeriod.*; + +import java.text.DateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; + +public class DateFactory { + private DateFactory() { + + } + + public static class DateOffsetProvider extends HashMap<Integer, Date> { + private final UpdatePeriod updatePeriod; + Calendar calendar = Calendar.getInstance(); + + public DateOffsetProvider(UpdatePeriod updatePeriod) { + this(updatePeriod, false); + } + + public DateOffsetProvider(UpdatePeriod updatePeriod, boolean truncate) { + this.updatePeriod = updatePeriod; + Date date = calendar.getTime(); + if (truncate) { + date = updatePeriod.truncate(date); + calendar.setTime(date); + } + put(0, date); + } + + @Override + public Date get(Object key) { + if (!containsKey(key) && key instanceof Integer) { + calendar.setTime(super.get(0)); + calendar.add(updatePeriod.calendarField(), (Integer) key); + put((Integer) key, calendar.getTime()); + } + return super.get(key); + } + } + + public static class GeneralDateOffsetProvider extends HashMap<UpdatePeriod, DateOffsetProvider> { + @Override + public DateOffsetProvider get(Object key) { + if (!containsKey(key) && key instanceof UpdatePeriod) { + UpdatePeriod up = (UpdatePeriod) key; + put(up, new DateOffsetProvider(up)); + } + return super.get(key); + } + + public Date get(UpdatePeriod updatePeriod, int offset) { + return get(updatePeriod).get(offset); + } + } + + public static final GeneralDateOffsetProvider GENERAL_DATE_OFFSET_PROVIDER = new GeneralDateOffsetProvider(); + + + public static Date getDateWithOffset(UpdatePeriod up, int offset) { + return GENERAL_DATE_OFFSET_PROVIDER.get(up, offset); + } + + public static String getDateStringWithOffset(UpdatePeriod up, int offset) { + return getDateStringWithOffset(up, offset, up); + } + + public static String getDateStringWithOffset(UpdatePeriod up, int offset, UpdatePeriod formatWith) { + return formatWith.format(GENERAL_DATE_OFFSET_PROVIDER.get(up, offset)); + } + + public static String getTimeRangeString(final String timeDim, final String startDate, final String endDate) { + return "time_range_in(" + timeDim + ", '" + startDate + "','" + endDate + "')"; + } + + public static String getTimeRangeString(final String timeDim, final UpdatePeriod updatePeriod, + final int startOffset, final int endOffset) { + return getTimeRangeString(timeDim, + getDateStringWithOffset(updatePeriod, startOffset), getDateStringWithOffset(updatePeriod, endOffset)); + } + + public static String getTimeRangeString(final String startDate, final String endDate) { + return getTimeRangeString("d_time", startDate, endDate); + } + + public static String getTimeRangeString(final UpdatePeriod updatePeriod, + final int startOffset, final int endOffset) { + return getTimeRangeString("d_time", updatePeriod, startOffset, endOffset); + } + + public static String getTimeRangeString(String partCol, UpdatePeriod updatePeriod, int startOffset, int endOffset, + UpdatePeriod formatWith) { + return getTimeRangeString(partCol, + formatWith.format(getDateWithOffset(updatePeriod, startOffset)), + formatWith.format(getDateWithOffset(updatePeriod, endOffset))); + } + + public static String getTimeRangeString(String partCol, UpdatePeriod updatePeriod, int startOffset, int endOffset, + DateFormat formatWith) { + return getTimeRangeString(partCol, + formatWith.format(getDateWithOffset(updatePeriod, startOffset)), + formatWith.format(getDateWithOffset(updatePeriod, endOffset))); + } + + public static String getTimeRangeString(UpdatePeriod updatePeriod, int startOffset, int endOffset, + UpdatePeriod formatWith) { + return getTimeRangeString("d_time", updatePeriod, startOffset, endOffset, formatWith); + } + + public static String getTimeRangeString(UpdatePeriod updatePeriod, int startOffset, int endOffset, + DateFormat formatWith) { + return getTimeRangeString("d_time", updatePeriod, startOffset, endOffset, formatWith); + } + + // Time Instances as Date Type + public static final Date NOW; + public static final Date TWODAYS_BACK; + public static final Date TWO_MONTHS_BACK; + public static final Date BEFORE_6_DAYS; + public static final Date BEFORE_4_DAYS; + + // Time Ranges + public static final String LAST_HOUR_TIME_RANGE; + public static final String TWO_DAYS_RANGE; + public static final String TWO_DAYS_RANGE_TTD; + public static final String TWO_DAYS_RANGE_TTD_BEFORE_4_DAYS; + public static final String TWO_DAYS_RANGE_TTD2; + public static final String TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS; + public static final String TWO_DAYS_RANGE_IT; + public static final String THIS_YEAR_RANGE; + public static final String LAST_YEAR_RANGE; + public static final String TWO_MONTHS_RANGE_UPTO_MONTH; + public static final String TWO_MONTHS_RANGE_UPTO_HOURS; + public static final String TWO_DAYS_RANGE_BEFORE_4_DAYS; + private static boolean zerothHour; + + + public static boolean isZerothHour() { + return zerothHour; + } + + static { + NOW = getDateWithOffset(HOURLY, 0); + + // Figure out if current hour is 0th hour + zerothHour = getDateStringWithOffset(HOURLY, 0).endsWith("-00"); + + TWODAYS_BACK = getDateWithOffset(DAILY, -2); + System.out.println("Test TWODAYS_BACK:" + TWODAYS_BACK); + + // two months back + TWO_MONTHS_BACK = getDateWithOffset(MONTHLY, -2); + System.out.println("Test TWO_MONTHS_BACK:" + TWO_MONTHS_BACK); + + // Before 4days + BEFORE_4_DAYS = getDateWithOffset(DAILY, -4); + BEFORE_6_DAYS = getDateWithOffset(DAILY, -6); + + TWO_DAYS_RANGE_BEFORE_4_DAYS = getTimeRangeString(DAILY, -6, -4, HOURLY); + + TWO_DAYS_RANGE = getTimeRangeString(HOURLY, -48, 0); + TWO_DAYS_RANGE_TTD = getTimeRangeString("test_time_dim", DAILY, -2, 0, HOURLY); + TWO_DAYS_RANGE_TTD_BEFORE_4_DAYS = getTimeRangeString("test_time_dim", DAILY, -6, -4, HOURLY); + TWO_DAYS_RANGE_TTD2 = getTimeRangeString("test_time_dim2", DAILY, -2, 0, HOURLY); + TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS = getTimeRangeString("test_time_dim2", DAILY, -6, -4, HOURLY); + TWO_DAYS_RANGE_IT = getTimeRangeString("it", DAILY, -2, 0, HOURLY); + THIS_YEAR_RANGE = getTimeRangeString(YEARLY, 0, 1); + LAST_YEAR_RANGE = getTimeRangeString(YEARLY, -1, 0); + TWO_MONTHS_RANGE_UPTO_MONTH = getTimeRangeString(MONTHLY, -2, 0); + TWO_MONTHS_RANGE_UPTO_HOURS = getTimeRangeString(MONTHLY, -2, 0, HOURLY); + + // calculate LAST_HOUR_TIME_RANGE + LAST_HOUR_TIME_RANGE = getTimeRangeString(HOURLY, -1, 0); + } +}
