http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java new file mode 100644 index 0000000..f87cf44 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.metadata; + +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.hive.metastore.api.FieldSchema; + +/** + * The Fact table interface + */ +public interface FactTable extends Named { + + /** + * Get map of storage to update period mapping + * + * @return Map of storage to set of update periods + */ + public Map<String, Set<UpdatePeriod>> getUpdatePeriods(); + + /** + * Cube to which this fact belongs to + * + * @return the cube string + */ + public String getCubeName(); + + /** + * The set of Storage names + * + * @return set of strings + */ + public Set<String> getStorages(); + + /** + *The type of the fact + * + * @return table type {@link CubeTableType} + */ + public CubeTableType getTableType(); + + /** + * Config properties + * + * @return map of string, string + */ + public Map<String, String> getProperties(); + + /** + * Valid columns of the fact + * + * @return list of column names + */ + public Set<String> getValidColumns(); + + /** + * Weight of the fact + * + * @return weight of the fact in double + */ + public double weight(); + + /** + * Set of all the columns names of the fact + * + * @return set of column names + */ + public Set<String> getAllFieldNames(); + + /** + *tag for checking data completeness + * + * @return Tag String + */ + public String getDataCompletenessTag(); + + /** + * List of columns of the fact + * + * @return set of {@link FieldSchema} + */ + public List<FieldSchema> getColumns(); + + /** + * Is Aggregated Fact + * + * @return true if fact is Aggregated , false otherwise + */ + public boolean isAggregated(); + + /** + * Absolute start time of the fact + * + * @return Absolute Start time of the fact {@link Date} + */ + public Date getAbsoluteStartTime(); + + /** + * Relative start time of the fact + * + * @return Relative Start time of the fact {@link Date} + */ + public Date getRelativeStartTime(); + + /** + * Start time of the fact + * + * @return Start time of the fact {@link Date} + */ + public Date getStartTime(); + + /** + * Absolute end time of the fact + * + * @return Absolute End time of the fact {@link Date} + */ + public Date getAbsoluteEndTime(); + + /** + * Relative End time of the Fact + * + * @return Relative end time of the fact {@link Date} + */ + public Date getRelativeEndTime(); + + /** + * End time of the fact + * + * @return End time of the fact {@link Date} + */ + public Date getEndTime(); + + /** + * Is Virtual Fact + * + * @return true if fact is a virtual fact, false otherwise + */ + public boolean isVirtualFact(); + + /** + * Storage name of the fact + * + * @return Storage name of the fact + */ + public String getSourceFactName(); + +}
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java index e1e3d16..7dfb166 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java @@ -718,6 +718,15 @@ public final class JAXBUtils { mapFromXProperties(fact.getProperties()), storageTablePrefixMap); } + public static CubeVirtualFactTable cubeVirtualFactFromFactTable(XVirtualFactTable fact, FactTable sourceFactTable) + throws LensException { + + Optional<Double> optionalWeight = Optional.fromNullable(fact.getWeight()); + + return new CubeVirtualFactTable(fact.getCubeName(), fact.getName(), + optionalWeight, mapFromXProperties(fact.getProperties()), sourceFactTable); + } + public static Segmentation segmentationFromXSegmentation(XSegmentation seg) throws LensException { Map<String, String> props = new HashMap<>(); @@ -736,7 +745,7 @@ public final class JAXBUtils { } - public static XFactTable factTableFromCubeFactTable(CubeFactTable cFact) { + public static XFactTable factTableFromCubeFactTable(FactTable cFact) { XFactTable fact = XCF.createXFactTable(); fact.setName(cFact.getName()); fact.setColumns(new XColumns()); @@ -749,6 +758,19 @@ public final class JAXBUtils { return fact; } + public static XVirtualFactTable virtualFactTableFromVirtualCubeFactTable(CubeVirtualFactTable vFact) { + XVirtualFactTable fact = XCF.createXVirtualFactTable(); + fact.setName(vFact.getName()); + fact.setProperties(new XProperties()); + + fact.getProperties().getProperty().addAll(xPropertiesFromMap(vFact.getProperties())); + + fact.setWeight(vFact.weight()); + fact.setSourceFactName(vFact.getSourceCubeFactTable().getName()); + fact.setCubeName(vFact.getCubeName()); + return fact; + } + public static XSegmentation xsegmentationFromSegmentation(Segmentation cSeg) { XSegmentation seg = XCF.createXSegmentation(); seg.setName(cSeg.getName()); http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java index 88500fd..945fe26 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java @@ -49,6 +49,7 @@ public final class MetastoreConstants { public static final String FACT_KEY_PFX = "cube.fact."; public static final String UPDATE_PERIOD_SFX = ".updateperiods"; public static final String CUBE_NAME_SFX = ".cubename"; + public static final String SOURCE_NAME_SFX = ".source"; public static final String VALID_COLUMNS_SFX = ".valid.columns"; public static final String FACT_AGGREGATED_PROPERTY = "cube.fact.is.aggregated"; public static final String FACT_ABSOLUTE_START_TIME = "cube.fact.absolute.start.time"; @@ -58,6 +59,7 @@ public final class MetastoreConstants { public static final String FACT_COL_START_TIME_PFX = "cube.fact.col.start.time."; public static final String FACT_COL_END_TIME_PFX = "cube.fact.col.end.time."; public static final String FACT_DATA_COMPLETENESS_TAG = "cube.fact.datacompleteness.tag"; + public static final String VIRTUAL_FACT_FILTER = "cube.fact.query.where.filter"; // Segmentation constants public static final String SEGMENTATION_KEY_PFX = "cube.segmentation.internal."; http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java index 40f766b..fbc37ac 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java @@ -49,6 +49,9 @@ import org.antlr.runtime.CommonToken; import com.google.common.collect.Sets; +import lombok.extern.slf4j.Slf4j; + +@Slf4j public class MetastoreUtil { private MetastoreUtil() { @@ -345,7 +348,7 @@ public class MetastoreUtil { } // ////////////////////////// - // Fact propertes /// + // Fact properties /// // ///////////////////////// public static String getFactStorageListKey(String name) { return getFactKeyPrefix(name) + STORAGE_LIST_SFX; @@ -363,6 +366,10 @@ public class MetastoreUtil { return getFactKeyPrefix(name) + CUBE_NAME_SFX; } + public static String getSourceFactNameKey(String name) { + return getFactKeyPrefix(name) + SOURCE_NAME_SFX; + } + public static String getValidColumnsKey(String name) { return getFactKeyPrefix(name) + VALID_COLUMNS_SFX; } @@ -398,6 +405,23 @@ public class MetastoreUtil { // ////////////////////////// // Utils /// // ///////////////////////// + + public static Date getDateFromProperty(String prop, boolean relative, boolean start) { + try { + if (StringUtils.isNotBlank(prop)) { + if (relative) { + return DateUtil.resolveRelativeDate(prop, new Date()); + } else { + return DateUtil.resolveAbsoluteDate(prop); + } + } + } catch (LensException e) { + log.error("unable to parse {} {} date: {}", relative ? "relative" : "absolute", start ? "start" : "end", prop); + } + return start ? DateUtil.MIN_DATE : DateUtil.MAX_DATE; + } + + public static <E extends Named> String getNamedStr(Collection<E> set) { if (set == null) { return ""; http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/Segmentation.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Segmentation.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Segmentation.java index 1e3cef3..54059d1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Segmentation.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Segmentation.java @@ -160,11 +160,13 @@ public class Segmentation extends AbstractCubeTable { public Date getAbsoluteStartTime() { - return getDateFromProperty(MetastoreConstants.SEGMENTATION_ABSOLUTE_START_TIME, false, true); + return MetastoreUtil.getDateFromProperty(this.getProperties() + .get(MetastoreConstants.SEGMENTATION_ABSOLUTE_START_TIME), false, true); } public Date getRelativeStartTime() { - return getDateFromProperty(MetastoreConstants.SEGMENTATION_RELATIVE_START_TIME, true, true); + return MetastoreUtil.getDateFromProperty(this.getProperties() + .get(MetastoreConstants.SEGMENTATION_RELATIVE_START_TIME), true, true); } public Date getStartTime() { @@ -172,16 +174,19 @@ public class Segmentation extends AbstractCubeTable { } public Date getAbsoluteEndTime() { - return getDateFromProperty(MetastoreConstants.SEGMENTATION_ABSOLUTE_END_TIME, false, false); + return MetastoreUtil.getDateFromProperty(this.getProperties() + .get(MetastoreConstants.SEGMENTATION_ABSOLUTE_END_TIME), false, false); } public Date getRelativeEndTime() { - return getDateFromProperty(MetastoreConstants.SEGMENTATION_RELATIVE_END_TIME, true, false); + return MetastoreUtil.getDateFromProperty(this.getProperties() + .get(MetastoreConstants.SEGMENTATION_RELATIVE_END_TIME), true, false); } public Date getEndTime() { return Collections.min(Lists.newArrayList(getRelativeEndTime(), getAbsoluteEndTime())); } + static String getCubeName(String segName, Map<String, String> props) { return props.get(MetastoreUtil.getSegmentationCubeNameKey(segName)); } http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java index 9f07336..2edeb41 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java @@ -18,13 +18,7 @@ */ package org.apache.lens.cube.parse; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; import org.apache.lens.cube.metadata.CubeInterface; import org.apache.lens.cube.metadata.CubeMetastoreClient; http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index a8d6fbd..93736cd 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -84,12 +84,12 @@ class CandidateTableResolver implements ContextRewriter { private void populateCandidateTables(CubeQueryContext cubeql) throws LensException { if (cubeql.getCube() != null) { - List<CubeFactTable> factTables = cubeql.getMetastoreClient().getAllFacts(cubeql.getCube()); + List<FactTable> factTables = cubeql.getMetastoreClient().getAllFacts(cubeql.getCube()); if (factTables.isEmpty()) { throw new LensException(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(), cubeql.getCube().getName() + " does not have any facts"); } - for (CubeFactTable fact : factTables) { + for (FactTable fact : factTables) { if (fact.getUpdatePeriods().isEmpty()) { log.info("Not considering fact: {} as it has no update periods", fact.getName()); } else { @@ -99,6 +99,7 @@ class CandidateTableResolver implements ContextRewriter { } } } + log.info("Populated storage candidates: {}", cubeql.getCandidates()); List<SegmentationCandidate> segmentationCandidates = Lists.newArrayList(); for (Segmentation segmentation : cubeql.getMetastoreClient().getAllSegmentations(cubeql.getCube())) { @@ -211,7 +212,8 @@ class CandidateTableResolver implements ContextRewriter { if (key.contains(MetastoreConstants.FACT_COL_START_TIME_PFX)) { String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_START_TIME_PFX); if (factCol.equals(propCol)) { - startTime = ((StorageCandidate) table).getFact().getDateFromProperty(key, false, true); + startTime = MetastoreUtil.getDateFromProperty(((StorageCandidate) table).getFact().getProperties().get(key), + false, true); } } } @@ -226,7 +228,8 @@ class CandidateTableResolver implements ContextRewriter { if (key.contains(MetastoreConstants.FACT_COL_END_TIME_PFX)) { String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_END_TIME_PFX); if (factCol.equals(propCol)) { - endTime = ((StorageCandidate) table).getFact().getDateFromProperty(key, false, true); + endTime = MetastoreUtil.getDateFromProperty(((StorageCandidate) table).getFact().getProperties().get(key), + false, true); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 1e54f13..99c6cf6 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -46,16 +46,7 @@ import java.util.TimeZone; import java.util.TreeSet; import java.util.stream.Stream; -import org.apache.lens.cube.metadata.AbstractCubeTable; -import org.apache.lens.cube.metadata.CubeFactTable; -import org.apache.lens.cube.metadata.CubeInterface; -import org.apache.lens.cube.metadata.DateUtil; -import org.apache.lens.cube.metadata.Dimension; -import org.apache.lens.cube.metadata.FactPartition; -import org.apache.lens.cube.metadata.MetastoreConstants; -import org.apache.lens.cube.metadata.MetastoreUtil; -import org.apache.lens.cube.metadata.TimeRange; -import org.apache.lens.cube.metadata.UpdatePeriod; +import org.apache.lens.cube.metadata.*; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.DataCompletenessChecker; @@ -126,7 +117,7 @@ public class StorageCandidate implements Candidate, CandidateTable { * Participating fact, storage and dimensions for this StorageCandidate */ @Getter - private CubeFactTable fact; + private FactTable fact; @Getter private String storageName; @Getter @@ -180,7 +171,7 @@ public class StorageCandidate implements Candidate, CandidateTable { this.answerableMeasurePhraseIndices = sc.answerableMeasurePhraseIndices; } - public StorageCandidate(CubeInterface cube, CubeFactTable fact, String storageName, CubeQueryContext cubeQueryContext) + public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, CubeQueryContext cubeQueryContext) throws LensException { this.cube = cube; this.fact = fact; @@ -189,7 +180,7 @@ public class StorageCandidate implements Candidate, CandidateTable { throw new IllegalArgumentException("Cube,fact and storageName should be non null"); } this.storageName = storageName; - this.storageTable = MetastoreUtil.getFactOrDimtableStorageTableName(fact.getName(), storageName); + this.storageTable = MetastoreUtil.getFactOrDimtableStorageTableName(fact.getSourceFactName(), storageName); this.name = getFact().getName(); this.processTimePartCol = getConf().get(CubeQueryConfUtil.PROCESS_TIME_PART_COL); String formatStr = getConf().get(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT); @@ -199,7 +190,8 @@ public class StorageCandidate implements Candidate, CandidateTable { completenessPartCol = getConf().get(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL); completenessThreshold = getConf() .getFloat(CubeQueryConfUtil.COMPLETENESS_THRESHOLD, CubeQueryConfUtil.DEFAULT_COMPLETENESS_THRESHOLD); - Set<String> storageTblNames = getCubeMetastoreClient().getStorageTables(fact.getName(), storageName); + + Set<String> storageTblNames = getCubeMetastoreClient().getStorageTables(fact, storageName); isStorageTblsAtUpdatePeriodLevel = storageTblNames.size() > 1 || !storageTblNames.iterator().next().equalsIgnoreCase(storageTable); setStorageStartAndEndDate(); @@ -259,8 +251,8 @@ public class StorageCandidate implements Candidate, CandidateTable { List<Date> startDates = new ArrayList<>(); List<Date> endDates = new ArrayList<>(); for (String storageTablePrefix : getValidStorageTableNames()) { - startDates.add(getCubeMetastoreClient().getStorageTableStartDate(storageTablePrefix, fact.getName())); - endDates.add(getCubeMetastoreClient().getStorageTableEndDate(storageTablePrefix, fact.getName())); + startDates.add(getCubeMetastoreClient().getStorageTableStartDate(storageTablePrefix, fact.getSourceFactName())); + endDates.add(getCubeMetastoreClient().getStorageTableEndDate(storageTablePrefix, fact.getSourceFactName())); } this.startTime = Collections.min(startDates); this.endTime = Collections.max(endDates); @@ -278,7 +270,7 @@ public class StorageCandidate implements Candidate, CandidateTable { return uniqueStorageTables; } else { //Get all storage tables. - return getCubeMetastoreClient().getStorageTables(fact.getName(), storageName); + return getCubeMetastoreClient().getStorageTables(fact, storageName); } } @@ -302,11 +294,6 @@ public class StorageCandidate implements Candidate, CandidateTable { } @Override - public AbstractCubeTable getTable() { - return fact; - } - - @Override public AbstractCubeTable getBaseTable() { return (AbstractCubeTable) cube; } @@ -320,14 +307,17 @@ public class StorageCandidate implements Candidate, CandidateTable { return phrase.isEvaluable(this); } - @Override + public AbstractCubeTable getTable() { + return (AbstractCubeTable) fact; + } + public Optional<Date> getColumnStartTime(String column) { Date startTime = null; for (String key : getTable().getProperties().keySet()) { if (key.contains(MetastoreConstants.FACT_COL_START_TIME_PFX)) { String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_START_TIME_PFX); if (column.equals(propCol)) { - startTime = getTable().getDateFromProperty(key, false, true); + startTime = MetastoreUtil.getDateFromProperty(getTable().getProperties().get(key), false, true); } } } @@ -341,7 +331,7 @@ public class StorageCandidate implements Candidate, CandidateTable { if (key.contains(MetastoreConstants.FACT_COL_END_TIME_PFX)) { String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_END_TIME_PFX); if (column.equals(propCol)) { - endTime = getTable().getDateFromProperty(key, false, true); + endTime = MetastoreUtil.getDateFromProperty(getTable().getProperties().get(key), false, true); } } } @@ -436,7 +426,7 @@ public class StorageCandidate implements Candidate, CandidateTable { return true; } - if (!getCubeMetastoreClient().partColExists(this.getFact().getName(), storageName, partCol)) { + if (!getCubeMetastoreClient().partColExists(this.getFact(), storageName, partCol)) { log.info("{} does not exist in {}", partCol, name); return false; } @@ -587,7 +577,6 @@ public class StorageCandidate implements Candidate, CandidateTable { Set<FactPartition> rangeParts = getPartitions(timeRange, validUpdatePeriods, true, failOnPartialData, missingParts); String partCol = timeRange.getPartitionColumn(); boolean partColNotSupported = rangeParts.isEmpty(); - String storageTableName = getStorageTable(); if (storagePruningMsgs.containsKey(this)) { List<CandidateTablePruneCause> causes = storagePruningMsgs.get(this); @@ -604,11 +593,13 @@ public class StorageCandidate implements Candidate, CandidateTable { String sep = ""; while (rangeParts.isEmpty()) { String timeDim = cubeQueryContext.getBaseCube().getTimeDimOfPartitionColumn(partCol); - if (partColNotSupported && !getFact().hasColumn(timeDim)) { - unsupportedTimeDims.add( - cubeQueryContext.getBaseCube().getTimeDimOfPartitionColumn(timeRange.getPartitionColumn()) - ); - break; + if (getFact() instanceof CubeFactTable) { + if (partColNotSupported && !((CubeFactTable) getFact()).hasColumn(timeDim)) { + unsupportedTimeDims.add( + cubeQueryContext.getBaseCube().getTimeDimOfPartitionColumn(timeRange.getPartitionColumn()) + ); + break; + } } TimeRange fallBackRange = getFallbackRange(prevRange, this.getFact().getName(), cubeQueryContext); log.info("No partitions for range:{}. fallback range: {}", timeRange, fallBackRange); http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java index 730b802..c535196 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java @@ -19,6 +19,8 @@ package org.apache.lens.cube.parse; +import static org.apache.lens.cube.metadata.MetastoreConstants.VIRTUAL_FACT_FILTER; + import java.util.Map; import java.util.Objects; import java.util.Set; @@ -128,7 +130,15 @@ public class StorageCandidateHQLContext extends DimHQLContext { @Override protected void setMissingExpressions() throws LensException { setFrom(getFromTable()); - setWhere(genWhereClauseWithDimPartitions(getWhere())); + String whereString = genWhereClauseWithDimPartitions(getWhere()); + StringBuilder whereStringBuilder = (whereString != null) ? new StringBuilder(whereString) : new StringBuilder(); + + if (this.storageCandidate.getFact().getProperties().get(VIRTUAL_FACT_FILTER) != null) { + appendWhereClause(whereStringBuilder, + this.storageCandidate.getFact().getProperties().get(VIRTUAL_FACT_FILTER), whereString != null); + } + setWhere(whereStringBuilder.length() == 0 ? null : whereStringBuilder.toString()); + if (isRoot()) { if (Objects.equals(getStorageCandidate(), getCubeQueryContext().getPickedCandidate())) { updateAnswerableSelectColumns(); http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index 291712b..3acd754 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -305,6 +305,7 @@ class StorageTableResolver implements ContextRewriter { if (!skipUpdatePeriodCauses.isEmpty()) { sc.setUpdatePeriodRejectionCause(skipUpdatePeriodCauses); } + // if no update periods were added in previous section, we skip this storage candidate if (!isUpdatePeriodForStorageAdded) { if (skipUpdatePeriodCauses.values().stream().allMatch( @@ -327,12 +328,12 @@ class StorageTableResolver implements ContextRewriter { pruningCauseForThisTimeRange = new CandidateTablePruneCause(CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE); } else if (!sc.getValidUpdatePeriods().contains(UpdatePeriod.CONTINUOUS)) { - if (!client.partColExists(sc.getFact().getName(), sc.getStorageName(), range.getPartitionColumn())) { + if (!client.partColExists(sc.getFact(), sc.getStorageName(), range.getPartitionColumn())) { pruningCauseForThisTimeRange = partitionColumnsMissing(range.getPartitionColumn()); TimeRange fallBackRange = StorageUtil.getFallbackRange(range, sc.getFact().getName(), cubeql); while (fallBackRange != null) { pruningCauseForThisTimeRange = null; - if (!client.partColExists(sc.getFact().getName(), sc.getStorageName(), + if (!client.partColExists(sc.getFact(), sc.getStorageName(), fallBackRange.getPartitionColumn())) { pruningCauseForThisTimeRange = partitionColumnsMissing(fallBackRange.getPartitionColumn()); fallBackRange = StorageUtil.getFallbackRange(fallBackRange, sc.getFact().getName(), cubeql); http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java index 42e32cb..dd3fae9 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/CubeFactTableTest.java @@ -18,8 +18,6 @@ */ package org.apache.lens.cube.metadata; -import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; @@ -38,10 +36,10 @@ public class CubeFactTableTest { @DataProvider(name = "properties") public Object[][] factProperties() throws LensException { - String minus1DaysRelative = "now -1 days"; - String minus2DaysRelative = "now -2 days"; - String plus1DaysRelative = "now +1 days"; - String plus2DaysRelative = "now +2 days"; + String minus1DaysRelative = "now.day -1 days"; + String minus2DaysRelative = "now.day -2 days"; + String plus1DaysRelative = "now.day +1 days"; + String plus2DaysRelative = "now.day +2 days"; String minus1DaysAbsolute = DateUtil.relativeToAbsolute(minus1DaysRelative, now); String minus2DaysAbsolute = DateUtil.relativeToAbsolute(minus2DaysRelative, now); @@ -69,8 +67,6 @@ public class CubeFactTableTest { when(cubeFactTable.getProperties()).thenReturn(properties); - when(cubeFactTable.getDateFromProperty(anyString(), anyBoolean(), anyBoolean())).thenCallRealMethod(); - when(cubeFactTable.getRelativeStartTime()).thenCallRealMethod(); when(cubeFactTable.getAbsoluteStartTime()).thenCallRealMethod(); http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index a76ef13..6f054c4 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -82,6 +82,7 @@ public class TestCubeMetastoreClient { private static Set<CubeDimAttribute> moreCubeDimensions = Sets.newHashSet(); private static Set<UpdatePeriod> hourlyAndDaily = Sets.newHashSet(HOURLY, DAILY); private static final String CUBE_NAME = "testMetastoreCube"; + private static final String VIRTUAL_CUBE_NAME = "testMetastoreVirtualCube"; private static final String CUBE_NAME_WITH_PROPS = "testMetastoreCubeWithProps"; private static final String DERIVED_CUBE_NAME = "derivedTestMetastoreCube"; private static final String DERIVED_CUBE_NAME_WITH_PROPS = "derivedTestMetastoreCubeWithProps"; @@ -136,6 +137,7 @@ public class TestCubeMetastoreClient { @BeforeClass public static void setup() throws HiveException, AlreadyExistsException, LensException { SessionState.start(conf); + Database database = new Database(); database.setName(TestCubeMetastoreClient.class.getSimpleName()); Hive.get(conf).createDatabase(database); @@ -149,6 +151,7 @@ public class TestCubeMetastoreClient { public static void teardown() throws Exception { // Drop the cube client.dropCube(CUBE_NAME); + client.dropCube(VIRTUAL_CUBE_NAME); client = CubeMetastoreClient.getInstance(conf); assertFalse(client.tableExists(CUBE_NAME)); @@ -1223,6 +1226,93 @@ public class TestCubeMetastoreClient { assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); } + @Test(priority = 2) + public void testVirtualCubeFact() throws Exception { + + client.createCube(VIRTUAL_CUBE_NAME, cubeMeasures, cubeDimensions); + String sourceFactName = "testMetastoreFact1"; + List<FieldSchema> factColumns = new ArrayList<>(cubeMeasures.size()); + for (CubeMeasure measure : cubeMeasures) { + factColumns.add(measure.getColumn()); + } + + // add one dimension of the cube + factColumns.add(new FieldSchema("zipcode", "int", "zip")); + + StorageTableDesc s1 = new StorageTableDesc(TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class, + datePartSingleton, datePartKeySingleton); + + s1.getTblProps().put(MetastoreUtil.getStoragetableStartTimesKey(), "2015, now.day -10 days"); + s1.getTblProps().put(MetastoreUtil.getStoragetableEndTimesKey(), "now.day - 1 day"); + + Map<String, Set<UpdatePeriod>> updatePeriods = getHashMap(c1, Sets.newHashSet(HOURLY, DAILY)); + Map<String, StorageTableDesc> storageTables = getHashMap(c1, s1); + + Map<String, String> sourceFactPropertiesMap = getHashMap("name1", "value1", "name2", "value2"); + CubeFactTable sourceFact = new CubeFactTable(CUBE_NAME, sourceFactName, factColumns, updatePeriods, 10.0, + sourceFactPropertiesMap); + + // create cube fact + client.createCubeFactTable(CUBE_NAME, sourceFactName, factColumns, updatePeriods, 10.0, + sourceFactPropertiesMap, storageTables); + + String virtualFactName = "testMetastoreVirtualFact"; + + Map<String, String> virtualFactPropertiesMap = getHashMap("name1", "newvalue1"); + + CubeVirtualFactTable cubeVirtualFact = new CubeVirtualFactTable(VIRTUAL_CUBE_NAME, virtualFactName, + com.google.common.base.Optional.fromNullable(null), virtualFactPropertiesMap, sourceFact); + + // create virtual cube fact + client.createVirtualFactTable(VIRTUAL_CUBE_NAME, virtualFactName, sourceFactName, null, + virtualFactPropertiesMap); + assertTrue(client.tableExists(virtualFactName)); + Table virtualTbl = client.getHiveTable(virtualFactName); + assertTrue(client.isVirtualFactTable(virtualTbl)); + assertTrue(client.isVirtualFactTableForCube(virtualTbl, VIRTUAL_CUBE_NAME)); + + //get virtual fact + assertTrue(client.getAllFacts(client.getCube(VIRTUAL_CUBE_NAME)).get(0).getName().equals(virtualFactName.trim() + .toLowerCase())); + + CubeVirtualFactTable actualcubeVirtualFact = (CubeVirtualFactTable) (client.getFactTable(virtualFactName)); + assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); + + //alter virtual fact + Map<String, String> alterVirtualFactPropertiesMap = getHashMap("name1", "newvalue2", "name3", "value3"); + cubeVirtualFact = new CubeVirtualFactTable(VIRTUAL_CUBE_NAME, virtualFactName, + com.google.common.base.Optional.fromNullable(null), alterVirtualFactPropertiesMap, + sourceFact); + client.alterVirtualCubeFactTable(cubeVirtualFact); + actualcubeVirtualFact = (CubeVirtualFactTable) client.getFactTable(virtualFactName); + assertEquals(actualcubeVirtualFact.getProperties().get("name1"), "newvalue2"); + assertEquals(actualcubeVirtualFact.getProperties().get("name3"), "value3"); + assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); + + //alter source fact + FieldSchema newcol = new FieldSchema("newcol", "int", "new col for part"); + sourceFact.alterColumn(newcol); + sourceFact.alterWeight(100); + client.alterCubeFactTable(sourceFact.getName(), sourceFact, storageTables, new HashMap<String, String>()); + actualcubeVirtualFact = (CubeVirtualFactTable) client.getFactTable(virtualFactName); + assertTrue(actualcubeVirtualFact.getColumns().contains(newcol)); + assertEquals(actualcubeVirtualFact.weight(), 100.0); + assertTrue(cubeVirtualFact.equals(actualcubeVirtualFact)); + + //drop source fact + client.dropFact(sourceFactName, true); + + //drop virtual fact + try { + client.dropVirtualFact(virtualFactName); + fail("Expected 404"); + } catch (LensException nfe) { + // PASS since virtual fact is already dropped when source fact was dropped + } + + assertFalse(client.tableExists(virtualFactName)); + } + @Test(priority = 1) public void testSegmentation() throws Exception { String segmentName = "testMetastoreSegmentation"; @@ -1372,7 +1462,7 @@ public class TestCubeMetastoreClient { updatePeriodStoragePrefix.put(DAILY, c4); factTable.addStorage(c4, hourlyAndDaily, updatePeriodStoragePrefix); client.alterCubeFactTable(factName, factTable, storageTables, new HashMap<String, String>()); - CubeFactTable altered2 = client.getCubeFact(factName); + CubeFactTable altered2 = client.getCubeFactTable(factName); assertTrue(client.tableExists(c1TableName)); Table alteredC1Table = client.getTable(c1TableName); assertEquals(alteredC1Table.getInputFormatClass(), SequenceFileInputFormat.class); @@ -1399,7 +1489,7 @@ public class TestCubeMetastoreClient { Map<String, StorageTableDesc> storageTableDescMap = new HashMap<>(); storageTableDescMap.put(c3, s1); client.addStorage(altered2, c3, hourlyAndDaily, storageTableDescMap, updatePeriodStoragePrefix); - CubeFactTable altered3 = client.getCubeFact(factName); + CubeFactTable altered3 = client.getCubeFactTable(factName); assertTrue(altered3.getStorages().contains("C3")); assertTrue(altered3.getUpdatePeriods().get("C3").equals(hourlyAndDaily)); String storageTableName = getFactOrDimtableStorageTableName(factName, c3); @@ -1407,15 +1497,15 @@ public class TestCubeMetastoreClient { client.dropStorageFromFact(factName, c2); storageTableName = getFactOrDimtableStorageTableName(factName, c2); assertFalse(client.tableExists(storageTableName)); - List<CubeFactTable> cubeFacts = client.getAllFacts(client.getCube(CUBE_NAME)); + List<FactTable> cubeFacts = client.getAllFacts(client.getCube(CUBE_NAME)); List<String> cubeFactNames = new ArrayList<>(); - for (CubeFactTable cfact : cubeFacts) { + for (FactTable cfact : cubeFacts) { cubeFactNames.add(cfact.getName()); } assertTrue(cubeFactNames.contains(factName.toLowerCase())); cubeFacts = client.getAllFacts(client.getCube(DERIVED_CUBE_NAME)); cubeFactNames = new ArrayList<>(); - for (CubeFactTable cfact : cubeFacts) { + for (FactTable cfact : cubeFacts) { cubeFactNames.add(cfact.getName()); } assertTrue(cubeFactNames.contains(factName.toLowerCase())); @@ -1425,7 +1515,7 @@ public class TestCubeMetastoreClient { assertFalse(client.tableExists(factName)); cubeFacts = client.getAllFacts(cube); cubeFactNames = new ArrayList<>(); - for (CubeFactTable cfact : cubeFacts) { + for (FactTable cfact : cubeFacts) { cubeFactNames.add(cfact.getName()); } assertFalse(cubeFactNames.contains(factName.toLowerCase())); @@ -1784,6 +1874,7 @@ public class TestCubeMetastoreClient { assertTimeline(endsAndHolesPartitionTimeline, storeAllPartitionTimeline, updatePeriod, getDateWithOffset(HOURLY, firstOffset), getDateWithOffset(HOURLY, latestOffset), holeDates); } + private void assertTimeline(EndsAndHolesPartitionTimeline endsAndHolesPartitionTimeline, StoreAllPartitionTimeline storeAllPartitionTimeline, UpdatePeriod updatePeriod, Date first, Date latest, Date... holes) throws LensException { @@ -2856,8 +2947,8 @@ public class TestCubeMetastoreClient { public void testCaching() throws HiveException, LensException { client = CubeMetastoreClient.getInstance(conf); CubeMetastoreClient client2 = CubeMetastoreClient.getInstance(new HiveConf(TestCubeMetastoreClient.class)); - assertEquals(5, client.getAllCubes().size()); - assertEquals(5, client2.getAllCubes().size()); + assertEquals(6, client.getAllCubes().size()); + assertEquals(6, client2.getAllCubes().size()); defineCube("testcache1", "testcache2", "derived1", "derived2"); client.createCube("testcache1", cubeMeasures, cubeDimensions); @@ -2866,18 +2957,18 @@ public class TestCubeMetastoreClient { client.createDerivedCube("testcache2", "derived2", measures, dimensions, CUBE_PROPERTIES, 0L); assertNotNull(client.getCube("testcache1")); assertNotNull(client2.getCube("testcache1")); - assertEquals(9, client.getAllCubes().size()); - assertEquals(9, client2.getAllCubes().size()); + assertEquals(10, client.getAllCubes().size()); + assertEquals(10, client2.getAllCubes().size()); client2 = CubeMetastoreClient.getInstance(conf); - assertEquals(9, client.getAllCubes().size()); - assertEquals(9, client2.getAllCubes().size()); + assertEquals(10, client.getAllCubes().size()); + assertEquals(10, client2.getAllCubes().size()); conf.setBoolean(MetastoreConstants.METASTORE_ENABLE_CACHING, false); client = CubeMetastoreClient.getInstance(conf); client2 = CubeMetastoreClient.getInstance(conf); - assertEquals(9, client.getAllCubes().size()); - assertEquals(9, client2.getAllCubes().size()); + assertEquals(10, client.getAllCubes().size()); + assertEquals(10, client2.getAllCubes().size()); defineCube("testcache3", "testcache4", "dervied3", "derived4"); client.createCube("testcache3", cubeMeasures, cubeDimensions); client.createCube("testcache4", cubeMeasures, cubeDimensions, CUBE_PROPERTIES); @@ -2885,8 +2976,8 @@ public class TestCubeMetastoreClient { client.createDerivedCube("testcache4", "derived4", measures, dimensions, CUBE_PROPERTIES, 0L); assertNotNull(client.getCube("testcache3")); assertNotNull(client2.getCube("testcache3")); - assertEquals(13, client.getAllCubes().size()); - assertEquals(13, client2.getAllCubes().size()); + assertEquals(14, client.getAllCubes().size()); + assertEquals(14, client2.getAllCubes().size()); conf.setBoolean(MetastoreConstants.METASTORE_ENABLE_CACHING, true); client = CubeMetastoreClient.getInstance(conf); } http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 2f300d3..48869c2 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -67,23 +67,7 @@ import javax.xml.bind.JAXBException; import org.apache.lens.api.ToXMLString; import org.apache.lens.api.jaxb.LensJAXBContext; import org.apache.lens.api.metastore.SchemaTraverser; -import org.apache.lens.cube.metadata.CubeDimAttribute; -import org.apache.lens.cube.metadata.CubeDimensionTable; -import org.apache.lens.cube.metadata.CubeFactTable; -import org.apache.lens.cube.metadata.CubeMeasure; -import org.apache.lens.cube.metadata.CubeMetastoreClient; -import org.apache.lens.cube.metadata.CubeTableType; -import org.apache.lens.cube.metadata.DateUtil; -import org.apache.lens.cube.metadata.JAXBUtils; -import org.apache.lens.cube.metadata.MetastoreConstants; -import org.apache.lens.cube.metadata.MetastoreUtil; -import org.apache.lens.cube.metadata.Storage; -import org.apache.lens.cube.metadata.StorageConstants; -import org.apache.lens.cube.metadata.StoragePartitionDesc; -import org.apache.lens.cube.metadata.TestCubeMetastoreClient; -import org.apache.lens.cube.metadata.TimePartition; -import org.apache.lens.cube.metadata.TimePartitionRange; -import org.apache.lens.cube.metadata.UpdatePeriod; +import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline; import org.apache.lens.cube.metadata.timeline.PartitionTimeline; import org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline; @@ -140,6 +124,7 @@ public class CubeTestSetup { public static final String TEST_CUBE_NAME = "testCube"; public static final String DERIVED_CUBE_NAME = "derivedCube"; public static final String BASE_CUBE_NAME = "baseCube"; + public static final String VIRTUAL_CUBE_NAME = "virtualCube"; private static String c0 = "C0"; private static String c1 = "C1"; @@ -598,7 +583,7 @@ public class CubeTestSetup { client.clearHiveTableCache(); - CubeFactTable fact = client.getFactTable(factName); + CubeFactTable fact = client.getCubeFactTable(factName); Table table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c1))); assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true"); for (UpdatePeriod period : Lists.newArrayList(MINUTELY, HOURLY, DAILY, MONTHLY, YEARLY, QUARTERLY)) { @@ -647,7 +632,7 @@ public class CubeTestSetup { private void createCubeCheapFactPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "cheapFact"; - CubeFactTable fact = client.getFactTable(factName); + CubeFactTable fact = client.getCubeFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); @@ -679,7 +664,7 @@ public class CubeTestSetup { private void createTestFact2Partitions(CubeMetastoreClient client) throws Exception { String factName = "testFact2"; - CubeFactTable fact = client.getFactTable(factName); + CubeFactTable fact = client.getCubeFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); @@ -767,7 +752,7 @@ public class CubeTestSetup { private void createTestFact2RawPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "testFact2_raw"; - CubeFactTable fact2 = client.getFactTable(factName); + CubeFactTable fact2 = client.getCubeFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); @@ -850,7 +835,8 @@ public class CubeTestSetup { // bw.write(ToXMLString.toString(JAXBUtils.xCubeFromHiveCube(cubeInterface))); // } // } - for (CubeFactTable cubeFactTable : client.getAllFacts()) { + for (FactTable factTable : client.getAllFacts(false)) { + CubeFactTable cubeFactTable = (CubeFactTable) factTable; try(BufferedWriter bw = new BufferedWriter(new FileWriter(getClass() .getResource("/schema/facts").getPath()+"/"+cubeFactTable.getName()+".xml"))) { bw.write(ToXMLString.toString(client.getXFactTable(cubeFactTable))); @@ -884,26 +870,26 @@ public class CubeTestSetup { private void createSummaryPartitions(CubeMetastoreClient client) throws Exception { String factName = "summary1"; - CubeFactTable fact1 = client.getFactTable(factName); + CubeFactTable fact1 = client.getCubeFactTable(factName); createPIEParts(client, fact1, c2); factName = "summary2"; - CubeFactTable fact2 = client.getFactTable(factName); + CubeFactTable fact2 = client.getCubeFactTable(factName); createPIEParts(client, fact2, c2); factName = "summary3"; - CubeFactTable fact3 = client.getFactTable(factName); + CubeFactTable fact3 = client.getCubeFactTable(factName); createPIEParts(client, fact3, c2); factName = "summary4"; - CubeFactTable fact4 = client.getFactTable(factName); + CubeFactTable fact4 = client.getCubeFactTable(factName); createPIEParts(client, fact4, c2); } private void createBaseCubeFactPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "testFact5_RAW_BASE"; - CubeFactTable fact = client.getFactTable(factName); + CubeFactTable fact = client.getCubeFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 7d1f80d..65ca861 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -145,6 +145,22 @@ public class TestCubeRewriter extends TestQueryRewrite { // assertNotNull(rewrittenQuery.getNonExistingParts()); } + + @Test + public void testVirtualFactCubeSimpleQuery() throws Exception { + Configuration conf = getConf(); + conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); + String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", + null, "AND ( dim1 = 10 )", getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, + "C1_summary1")); + String hql = rewrittenQuery.toHQL(); + compareQueries(hql, expected); + System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); + } + + @Test public void testMaxCoveringFact() throws Exception { Configuration conf = getConf(); http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java index fe52600..9fa31dc 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java @@ -21,7 +21,6 @@ package org.apache.lens.cube.parse; import static org.apache.lens.cube.metadata.DateFactory.NOW; import static org.apache.lens.cube.metadata.DateFactory.TWO_DAYS_RANGE; -import static org.apache.lens.cube.metadata.DateFactory.TWO_MONTHS_RANGE_UPTO_DAYS; import static org.apache.lens.cube.metadata.DateFactory.getDateWithOffset; import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.SEGMENTATION_PRUNED; import static org.apache.lens.cube.parse.CubeQueryConfUtil.DISABLE_AGGREGATE_RESOLVER; @@ -38,8 +37,6 @@ import static org.apache.lens.cube.parse.CubeTestSetup.getWhereForHourly2days; import static org.apache.lens.cube.parse.CubeTestSetup.getWhereForUpdatePeriods; import static org.apache.lens.cube.parse.TestCubeRewriter.compareQueries; -import static org.apache.commons.lang3.time.DateUtils.addDays; - import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -126,13 +123,13 @@ public class TestCubeSegmentationRewriter extends TestQueryRewrite { */ @Test public void testFactUnionSegmentWithInnerUnion() throws Exception { - CubeQueryContext ctx = rewriteCtx("select cityid, segmsr1 from testcube where " + TWO_MONTHS_RANGE_UPTO_DAYS, - getConf()); + CubeQueryContext ctx = rewriteCtx("select cityid, segmsr1 from testcube where " + + "time_range_in(d_time, 'now.day-40day', 'now.day')", getConf()); String query1, query2, query3, query4; query1 = getExpectedQuery("testcube", "select testcube.cityid as alias0, sum(testcube.segmsr1) as alias1 from ", null, "group by testcube.cityid", getWhereForUpdatePeriods("testcube", "c0_b1fact1", - addDays(getDateWithOffset(UpdatePeriod.MONTHLY, -1), -1), getDateWithOffset(UpdatePeriod.DAILY, -10), + getDateWithOffset(UpdatePeriod.DAILY, -31), getDateWithOffset(UpdatePeriod.DAILY, -10), Sets.newHashSet(UpdatePeriod.MONTHLY, UpdatePeriod.DAILY))); query2 = getExpectedQuery("testcube", "select testcube.cityid as alias0, sum(testcube.segmsr1) as alias1 from ", null, "group by testcube.cityid", @@ -142,13 +139,13 @@ public class TestCubeSegmentationRewriter extends TestQueryRewrite { query3 = getExpectedQuery("testcube", "select testcube.cityid as alias0, sum(testcube.segmsr1) as alias1 from ", null, "group by testcube.cityid", getWhereForUpdatePeriods("testcube", "c0_b2fact1", - addDays(getDateWithOffset(UpdatePeriod.MONTHLY, -1), -1), NOW, + getDateWithOffset(UpdatePeriod.DAILY, -31), NOW, Sets.newHashSet(UpdatePeriod.MONTHLY, UpdatePeriod.DAILY))); query4 = getExpectedQuery("testcube", "select testcube.cityid as alias0, sum(testcube.segmsr1) as alias1 from ", null, "group by testcube.cityid", getWhereForUpdatePeriods("testcube", "c0_b1b2fact1", - addDays(getDateWithOffset(UpdatePeriod.MONTHLY, -2), -1), - addDays(getDateWithOffset(UpdatePeriod.MONTHLY, -1), 0), + getDateWithOffset(UpdatePeriod.DAILY, -41), + getDateWithOffset(UpdatePeriod.DAILY, -30), Sets.newHashSet(UpdatePeriod.MONTHLY, UpdatePeriod.DAILY))); compareUnionQuery(ctx, "select testcube.alias0 as cityid, sum(testcube.alias1) as segmsr1 from (", ") AS testcube GROUP BY (testcube.alias0)", newArrayList(query1, query2, query3, query4)); http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java index 7ec3324..90c6514 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java @@ -372,7 +372,7 @@ public class TestUnionQueries extends TestQueryRewrite { FAIL_QUERY_ON_PARTIAL_DATA, false); //If not beginning of month. Expecting this to pass at beginning of every month (example April 01 00:00) - if (!THREE_MONTHS_RANGE_UPTO_DAYS.equals(THREE_MONTHS_RANGE_UPTO_MONTH)) { + if (Calendar.getInstance().get(Calendar.DAY_OF_MONTH) != 1) { NoCandidateFactAvailableException e = getLensExceptionInRewrite("select count(msr4) from testCube where " + THREE_MONTHS_RANGE_UPTO_DAYS, conf); Set<Map.Entry<Candidate, List<CandidateTablePruneCause>>> causes = http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml b/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml new file mode 100644 index 0000000..2a5e0b0 --- /dev/null +++ b/lens-cube/src/test/resources/schema/cubes/base/virtualcube.xml @@ -0,0 +1,36 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<x_base_cube name="virtualcube" xmlns="uri:lens:cube:0.1"> + <properties> + <property name="cube.timedim.partition.d_time" value="dt"/> + <property name="cube.virtualcube.timed.dimensions.list" value="d_time"/> + </properties> + <measures> + <measure _type="FLOAT" default_aggr="SUM" unit="RS" name="msr2" display_string="Measure2" + description="second measure"/> + + </measures> + <dim_attributes> + <dim_attribute _type="string" name="dim1" description="basedim"> + </dim_attribute> + </dim_attributes> +</x_base_cube> http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/test/resources/schema/facts/virtualfact.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/virtualfact.xml b/lens-cube/src/test/resources/schema/facts/virtualfact.xml new file mode 100644 index 0000000..860965f --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/virtualfact.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<x_virtual_fact_table source_fact_name="summary1" cube_name="virtualcube" name="virtualfact1" xmlns="uri:lens:cube:0.1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> + <properties> + <property name="cube.fact.query.where.filter" value=" dim1 = 10 "/> + </properties> +</x_virtual_fact_table> http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-driver-es/pom.xml ---------------------------------------------------------------------- diff --git a/lens-driver-es/pom.xml b/lens-driver-es/pom.xml index 925020e..7823a62 100644 --- a/lens-driver-es/pom.xml +++ b/lens-driver-es/pom.xml @@ -19,8 +19,7 @@ under the License. --> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.lens</groupId> http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-examples/pom.xml ---------------------------------------------------------------------- diff --git a/lens-examples/pom.xml b/lens-examples/pom.xml index af467ee..abf349f 100644 --- a/lens-examples/pom.xml +++ b/lens-examples/pom.xml @@ -19,8 +19,7 @@ under the License. --> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <name>Lens Examples</name> <parent> http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-examples/src/test/resources/yaml/fact1.yaml ---------------------------------------------------------------------- diff --git a/lens-examples/src/test/resources/yaml/fact1.yaml b/lens-examples/src/test/resources/yaml/fact1.yaml index 9c1c527..1ad539e 100644 --- a/lens-examples/src/test/resources/yaml/fact1.yaml +++ b/lens-examples/src/test/resources/yaml/fact1.yaml @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +name: fact1 +cubeName: sample_cube columns: dim1: type: INT measure2: type: BIGINT @@ -37,6 +39,4 @@ storageTables: external: true tableLocation: /tmp/examples/fact1_local fieldDelimiter: , -name: fact1 -cubeName: sample_cube weight: 100.0 http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-examples/src/test/resources/yaml/fact2.yaml ---------------------------------------------------------------------- diff --git a/lens-examples/src/test/resources/yaml/fact2.yaml b/lens-examples/src/test/resources/yaml/fact2.yaml index 6fadd11..4eed1ae 100644 --- a/lens-examples/src/test/resources/yaml/fact2.yaml +++ b/lens-examples/src/test/resources/yaml/fact2.yaml @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +name: fact2 +cubeName: sample_cube columns: dim1: type: INT dim3: type: INT @@ -32,6 +34,4 @@ storageTables: external: true tableLocation: /tmp/examples/fact2_local fieldDelimiter: , -name: fact2 -cubeName: sample_cube weight: 200.0 http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-examples/src/test/resources/yaml/rawfact.yaml ---------------------------------------------------------------------- diff --git a/lens-examples/src/test/resources/yaml/rawfact.yaml b/lens-examples/src/test/resources/yaml/rawfact.yaml index f7b2d55..4764548 100644 --- a/lens-examples/src/test/resources/yaml/rawfact.yaml +++ b/lens-examples/src/test/resources/yaml/rawfact.yaml @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +name: rawfact +cubeName: sample_cube columns: dim1: type: INT dim2: type: INT @@ -34,6 +36,4 @@ storageTables: external: true tableLocation: /tmp/examples/rawfact fieldDelimiter: , -name: rawfact -cubeName: sample_cube weight: 500.0 http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-examples/src/test/resources/yaml/sales-aggr-continuous-fact.yaml ---------------------------------------------------------------------- diff --git a/lens-examples/src/test/resources/yaml/sales-aggr-continuous-fact.yaml b/lens-examples/src/test/resources/yaml/sales-aggr-continuous-fact.yaml index f95f91b..4fb7165 100644 --- a/lens-examples/src/test/resources/yaml/sales-aggr-continuous-fact.yaml +++ b/lens-examples/src/test/resources/yaml/sales-aggr-continuous-fact.yaml @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +name: sales_aggr_continuous_fact +cubeName: sales columns: order_time: type: TIMESTAMP delivery_time: type: TIMESTAMP @@ -41,6 +43,4 @@ storageTables: tableLocation: /tmp/db-storage.db fieldDelimiter: , storageHandlerName: org.apache.lens.storage.db.DBStorageHandler -name: sales_aggr_continuous_fact -cubeName: sales weight: 50.0 http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-examples/src/test/resources/yaml/sales-aggr-fact1.yaml ---------------------------------------------------------------------- diff --git a/lens-examples/src/test/resources/yaml/sales-aggr-fact1.yaml b/lens-examples/src/test/resources/yaml/sales-aggr-fact1.yaml index 02f65ff..80178f8 100644 --- a/lens-examples/src/test/resources/yaml/sales-aggr-fact1.yaml +++ b/lens-examples/src/test/resources/yaml/sales-aggr-fact1.yaml @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +name: sales_aggr_fact1 +cubeName: sales columns: order_time: type: TIMESTAMP delivery_time: type: TIMESTAMP @@ -57,6 +59,4 @@ storageTables: tableLocation: /tmp/db-storage.db fieldDelimiter: , storageHandlerName: org.apache.lens.storage.db.DBStorageHandler -name: sales_aggr_fact1 -cubeName: sales weight: 300.0 http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-examples/src/test/resources/yaml/sales-aggr-fact2.yaml ---------------------------------------------------------------------- diff --git a/lens-examples/src/test/resources/yaml/sales-aggr-fact2.yaml b/lens-examples/src/test/resources/yaml/sales-aggr-fact2.yaml index ff47d36..ff44dd0 100644 --- a/lens-examples/src/test/resources/yaml/sales-aggr-fact2.yaml +++ b/lens-examples/src/test/resources/yaml/sales-aggr-fact2.yaml @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +name: sales_aggr_fact2 +cubeName: sales columns: order_time: type: TIMESTAMP delivery_time: type: TIMESTAMP @@ -53,6 +55,4 @@ storageTables: tableLocation: /tmp/db-storage.db fieldDelimiter: , storageHandlerName: org.apache.lens.storage.db.DBStorageHandler -name: sales_aggr_fact2 -cubeName: sales weight: 100.0 http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-examples/src/test/resources/yaml/sales-raw-fact.yaml ---------------------------------------------------------------------- diff --git a/lens-examples/src/test/resources/yaml/sales-raw-fact.yaml b/lens-examples/src/test/resources/yaml/sales-raw-fact.yaml index 15becfc..20106b6 100644 --- a/lens-examples/src/test/resources/yaml/sales-raw-fact.yaml +++ b/lens-examples/src/test/resources/yaml/sales-raw-fact.yaml @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +name: sales_raw_fact +cubeName: sales columns: order_time: type: TIMESTAMP delivery_time: type: TIMESTAMP @@ -46,6 +48,4 @@ storageTables: external: true tableLocation: /tmp/examples/rawfact fieldDelimiter: , -name: sales_raw_fact -cubeName: sales weight: 500.0 http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java index 28b9d22..9d16533 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java @@ -378,10 +378,10 @@ public interface CubeMetastoreService extends LensService, SessionValidator { * * @param sessionid The sessionid * @param fact The fact table name - * @return {@link XFactTable} + * @return {@link XFact} * @throws LensException */ - XFactTable getFactTable(LensSessionHandle sessionid, String fact) throws LensException; + XFact getFactTable(LensSessionHandle sessionid, String fact) throws LensException; /** * Create fact table @@ -390,7 +390,7 @@ public interface CubeMetastoreService extends LensService, SessionValidator { * @param fact The fact table definition * @throws LensException */ - void createFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException; + void createFactTable(LensSessionHandle sessionid, XFact fact) throws LensException; /** * Update/Alter fact table @@ -399,7 +399,7 @@ public interface CubeMetastoreService extends LensService, SessionValidator { * @param fact The fact table's new definition * @throws LensException */ - void updateFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException; + void updateFactTable(LensSessionHandle sessionid, XFact fact) throws LensException; /** * Drop fact table. http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-server/pom.xml ---------------------------------------------------------------------- diff --git a/lens-server/pom.xml b/lens-server/pom.xml index 60ed1ae..34a7819 100644 --- a/lens-server/pom.xml +++ b/lens-server/pom.xml @@ -409,4 +409,4 @@ </plugin> </plugins> </build> -</project> \ No newline at end of file +</project> http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index 194b380..0a8635c 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -323,7 +323,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet public void dropAllStoragesOfFact(LensSessionHandle sessionid, String factName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); - CubeFactTable tab = msClient.getFactTable(factName); + CubeFactTable tab = msClient.getCubeFactTable(factName); int total = tab.getStorages().size(); int i = 0; List<String> storageNames = new ArrayList<>(tab.getStorages()); @@ -354,7 +354,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet } @Override - public XFactTable getFactTable(LensSessionHandle sessionid, String fact) throws LensException { + public XFact getFactTable(LensSessionHandle sessionid, String fact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ return getClient(sessionid).getXFactTable(fact); } @@ -371,15 +371,15 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet @Override - public void createFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException { + public void createFactTable(LensSessionHandle sessionid, XFact fact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ - getClient(sessionid).createCubeFactTable(fact); + getClient(sessionid).createFactTable(fact); log.info("Created fact table " + fact.getName()); } } @Override - public void updateFactTable(LensSessionHandle sessionid, XFactTable fact) throws LensException { + public void updateFactTable(LensSessionHandle sessionid, XFact fact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ getClient(sessionid).alterCubeFactTable(fact); log.info("Updated fact table " + fact.getName()); @@ -432,9 +432,9 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet if (cubeName != null && fact == null) { throw new LensException("Could not get table: " + cubeName + " as a cube"); } - Collection<CubeFactTable> facts = client.getAllFacts(fact); + Collection<FactTable> facts = client.getAllFacts(fact); List<String> factNames = new ArrayList<>(facts.size()); - for (CubeFactTable cft : facts) { + for (FactTable cft : facts) { factNames.add(cft.getName()); } return factNames; @@ -468,7 +468,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet throw new NotFoundException("Not a fact table " + fact); } - CubeFactTable cft = msClient.getFactTable(fact); + CubeFactTable cft = msClient.getCubeFactTable(fact); if (cft != null) { return new ArrayList<>(cft.getStorages()); } else { @@ -481,7 +481,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient msClient = getClient(sessionid); - CubeFactTable factTable = msClient.getFactTable(fact); + FactTable factTable = msClient.getFactTable(fact); Set<UpdatePeriod> updatePeriods = factTable.getUpdatePeriods().get(storageName); XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(fact, storageName))); @@ -520,7 +520,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet CubeMetastoreClient msClient = getClient(sessionid); XStorageTables tables = new XStorageTables(); tables.getStorageTable().add(storageTable); - msClient.addStorage(msClient.getFactTable(fact), storageTable.getStorageName(), updatePeriods, + msClient.addStorage(msClient.getCubeFactTable(fact), storageTable.getStorageName(), updatePeriods, JAXBUtils.tableDescPrefixMapFromXStorageTables(tables), JAXBUtils.storageTablePrefixMapOfStorage(tables).get(storageTable.getStorageName())); log.info("Added storage " + storageTable.getStorageName() + ":" + updatePeriods + " for fact " + fact); @@ -541,7 +541,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet private CubeFactTable checkFactStorage(LensSessionHandle sessionid, String fact, String storage) throws HiveException, LensException { CubeMetastoreClient client = getClient(sessionid); - CubeFactTable factTable = client.getCubeFact(fact); + CubeFactTable factTable = client.getCubeFactTable(fact); client.verifyStorageExists(factTable, storage); return factTable; } @@ -550,7 +550,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet throws LensException { Set<String> storageTableNames = new HashSet<>(); if (getClient(sessionHandle).isFactTable(fact)) { - CubeFactTable cft = getClient(sessionHandle).getCubeFact(fact); + CubeFactTable cft = getClient(sessionHandle).getCubeFactTable(fact); Map<UpdatePeriod, String> storageMap = cft.getStoragePrefixUpdatePeriodMap().get(storageName); for (Map.Entry entry : storageMap.entrySet()) { storageTableNames.add(MetastoreUtil.getStorageTableName(fact, Storage.getPrefix((String) entry.getValue()))); http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index 7451aa9..0af6a34 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -710,7 +710,6 @@ public class MetastoreResource { return Entity.FACT.getAll(sessionid); } - /** * Get all segmentations in the current database * @@ -760,15 +759,15 @@ public class MetastoreResource { * * @param sessionid The sessionid in which user is working * @param factName The fact table name - * @return JAXB representation of {@link XFactTable} + * @return JAXB representation of {@link XFact} */ @GET @Path("/facts/{factName}") - public JAXBElement<XFactTable> getFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, + public JAXBElement<XFact> getFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, @PathParam("factName") String factName) throws LensException { checkSessionId(sessionid); - return X_CUBE_OBJECT_FACTORY.createXFactTable(getSvc().getFactTable(sessionid, factName)); + return X_CUBE_OBJECT_FACTORY.createXFact(getSvc().getFactTable(sessionid, factName)); } /** @@ -791,13 +790,13 @@ public class MetastoreResource { * Create a new fact tabble * * @param sessionid The sessionid in which user is working - * @param fact The {@link XFactTable} representation of the fact table definition + * @param fact The {@link XFact} representation of the fact table definition * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if create was successful. {@link APIResult} with * state {@link Status#FAILED}, if create has failed */ @POST @Path("/facts") - public APIResult createFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, XFactTable fact) + public APIResult createFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, XFact fact) throws LensException { checkSessionId(sessionid); log.info("Create fact table"); @@ -823,20 +822,19 @@ public class MetastoreResource { return success(); } - /** * Update fact table definition * * @param sessionid The sessionid in which user is working * @param factName name of the fact table - * @param fact The {@link XFactTable} representation of the updated fact table definition + * @param fact The {@link XFact} representation of the updated fact table definition * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if update was successful. {@link APIResult} with * state {@link Status#FAILED}, if update has failed */ @PUT @Path("/facts/{factName}") public APIResult updateFactTable(@QueryParam("sessionid") LensSessionHandle sessionid, - @PathParam("factName") String factName, XFactTable fact) + @PathParam("factName") String factName, XFact fact) throws LensException { checkSessionId(sessionid); getSvc().updateFactTable(sessionid, fact); @@ -880,7 +878,6 @@ public class MetastoreResource { return Entity.FACT.delete(sessionid, factName, cascade); } - /** * Drop the segmentation, specified by name * http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java index df4e07a..5c82d74 100644 --- a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java +++ b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java @@ -36,10 +36,7 @@ import javax.xml.bind.JAXBElement; import org.apache.lens.api.APIResult; import org.apache.lens.api.LensConf; import org.apache.lens.api.LensSessionHandle; -import org.apache.lens.api.metastore.ObjectFactory; -import org.apache.lens.api.metastore.XCube; -import org.apache.lens.api.metastore.XFactTable; -import org.apache.lens.api.metastore.XStorage; +import org.apache.lens.api.metastore.*; import org.apache.lens.api.query.*; import org.apache.lens.api.result.LensAPIResult; @@ -161,7 +158,7 @@ public class RestAPITestUtil { APIResult result = target.path("metastore").path("facts").queryParam("sessionid", sessionId) .request(mt).post(Entity.entity( - new GenericEntity<JAXBElement<XFactTable>>(cubeObjectFactory.createXFactTable(factTable)) { + new GenericEntity<JAXBElement<XFact>>(cubeObjectFactory.createXFact(factTable)) { }, mt), APIResult.class); checkResponse(result);
