Repository: lens Updated Branches: refs/heads/master 3e7d92e9d -> 2d64f6ca4
LENS-1533 : Adding support for Partition Columns to queried without giving additional xml tag valid.columns Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2d64f6ca Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2d64f6ca Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2d64f6ca Branch: refs/heads/master Commit: 2d64f6ca4167570097bab7cde3604b190ab89edf Parents: 3e7d92e Author: Rajitha R <[email protected]> Authored: Thu Sep 20 16:53:51 2018 +0530 Committer: Rajitha.R <[email protected]> Committed: Thu Sep 20 16:53:51 2018 +0530 ---------------------------------------------------------------------- .../lens/cube/metadata/CubeFactTable.java | 55 +++++++++++++++++--- .../lens/cube/metadata/CubeMetastoreClient.java | 12 ++--- .../cube/metadata/CubeVirtualFactTable.java | 7 ++- .../apache/lens/cube/metadata/FactTable.java | 7 +++ .../apache/lens/cube/metadata/JAXBUtils.java | 28 ++++++++-- .../lens/cube/metadata/MetastoreUtil.java | 4 ++ .../lens/cube/parse/StorageCandidate.java | 13 ++++- .../cube/metadata/TestCubeMetastoreClient.java | 7 +-- .../lens/examples/PopulateSampleMetastore.java | 1 + .../apache/lens/examples/SampleMetastore.java | 1 + .../src/main/resources/cube-queries.sql | 2 +- .../src/main/resources/fact3-local-part.xml | 19 +++++++ .../src/main/resources/fact3-local1.data | 8 +++ .../src/main/resources/fact3-local2.data | 8 +++ lens-examples/src/main/resources/fact3.xml | 51 ++++++++++++++++++ .../src/main/resources/sample-cube.xml | 1 + 16 files changed, 199 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index 8f3f4aa..218e153 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -40,12 +40,14 @@ public class CubeFactTable extends AbstractCubeTable implements FactTable { private final Map<String, Map<UpdatePeriod, String>> storagePrefixUpdatePeriodMap; private String cubeName; private final Map<String, Set<UpdatePeriod>> storageUpdatePeriods; + private Map<String, Set<String>> storageTablePartitionColumns; public CubeFactTable(Table hiveTable) { super(hiveTable); this.storageUpdatePeriods = getUpdatePeriods(getName(), getProperties()); this.cubeName = this.getProperties().get(MetastoreUtil.getFactCubeNameKey(getName())); this.storagePrefixUpdatePeriodMap = getUpdatePeriodMap(getName(), getProperties()); + this.storageTablePartitionColumns = getStorageTablePartitionColumns(getName(), getProperties()); } public CubeFactTable(String cubeName, String factName, List<FieldSchema> columns, @@ -61,17 +63,20 @@ public class CubeFactTable extends AbstractCubeTable implements FactTable { public CubeFactTable(String cubeName, String factName, List<FieldSchema> columns, Map<String, Set<UpdatePeriod>> storageUpdatePeriods, double weight, Map<String, String> properties) { this(cubeName, factName, columns, storageUpdatePeriods, weight, properties, - new HashMap<String, Map<UpdatePeriod, String>>()); + new HashMap<String, Map<UpdatePeriod, String>>(), new HashMap<String, Set<String>>()); } public CubeFactTable(String cubeName, String factName, List<FieldSchema> columns, - Map<String, Set<UpdatePeriod>> storageUpdatePeriods, double weight, Map<String, String> properties, - Map<String, Map<UpdatePeriod, String>> storagePrefixUpdatePeriodMap) { + Map<String, Set<UpdatePeriod>> storageUpdatePeriods, double weight, + Map<String, String> properties, + Map<String, Map<UpdatePeriod, String>> storagePrefixUpdatePeriodMap, + Map<String, Set<String>> storageTablePartitionColumns) { super(factName, columns, properties, weight); this.cubeName = cubeName; this.storageUpdatePeriods = storageUpdatePeriods; this.storagePrefixUpdatePeriodMap = storagePrefixUpdatePeriodMap; + this.storageTablePartitionColumns = storageTablePartitionColumns; addProperties(); } @@ -90,10 +95,20 @@ public class CubeFactTable extends AbstractCubeTable implements FactTable { this.getProperties().put(MetastoreUtil.getFactCubeNameKey(getName()), cubeName); addUpdatePeriodProperies(getName(), getProperties(), storageUpdatePeriods); addStorageTableProperties(getName(), getProperties(), storagePrefixUpdatePeriodMap); + addStorageTableParititionColumns(getName(), getProperties(), storageTablePartitionColumns); } - private void addStorageTableProperties(String name, Map<String, String> properties, - Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap) { + private void addStorageTableParititionColumns(String name, Map<String, String> properties, + Map<String, Set<String>> storageTablePartitionColumns) { + for (String storage : storageTablePartitionColumns.keySet()) { + String partitionColumnKey = MetastoreUtil.getPartitionColumnKey(name, storage); + String partitionColumnValues = StringUtils.join(storageTablePartitionColumns.get(storage), ','); + properties.put(partitionColumnKey, partitionColumnValues); + } + } + + private void addStorageTableProperties(String name, Map<String, String> properties, Map<String, Map<UpdatePeriod, + String>> storageUpdatePeriodMap) { for (String storageName : storageUpdatePeriodMap.keySet()) { String prefix = MetastoreUtil.getFactKeyPrefix(name) + "." + storageName; for (Map.Entry updatePeriodEntry : storageUpdatePeriodMap.get(storageName).entrySet()) { @@ -104,12 +119,12 @@ public class CubeFactTable extends AbstractCubeTable implements FactTable { } private static void addUpdatePeriodProperies(String name, Map<String, String> props, - Map<String, Set<UpdatePeriod>> updatePeriods) { + Map<String, Set<UpdatePeriod>> updatePeriods) { if (updatePeriods != null) { props.put(MetastoreUtil.getFactStorageListKey(name), MetastoreUtil.getStr(updatePeriods.keySet())); for (Map.Entry<String, Set<UpdatePeriod>> entry : updatePeriods.entrySet()) { props.put(MetastoreUtil.getFactUpdatePeriodKey(name, entry.getKey()), - MetastoreUtil.getNamedStr(entry.getValue())); + MetastoreUtil.getNamedStr(entry.getValue())); } } } @@ -151,10 +166,36 @@ public class CubeFactTable extends AbstractCubeTable implements FactTable { return storageUpdatePeriods; } + private Map<String, Set<String>> getStorageTablePartitionColumns(String name, Map<String, String> props) { + Map<String, Set<String>> storageTablePartitionColumns = new HashMap<>(); + String storagesStr = props.get(MetastoreUtil.getFactStorageListKey(name)); + if (!StringUtils.isBlank(storagesStr)) { + String[] storages = storagesStr.split(","); + for (String storage : storages) { + String partitionColumnKey = MetastoreUtil.getPartitionColumnKey(name, storage); + String partitionColumnValues = props.get(partitionColumnKey); + if (StringUtils.isNotEmpty(partitionColumnValues)) { + storageTablePartitionColumns.put(partitionColumnKey, + new HashSet<String>(Arrays.asList(partitionColumnValues.split(",")))); + } + + } + } + return storageTablePartitionColumns; + } + public Map<String, Set<UpdatePeriod>> getUpdatePeriods() { return storageUpdatePeriods; } + public Set<String> getPartitionColumns(String storageName) { + String partitionColumnKey = MetastoreUtil.getPartitionColumnKey(getName(), storageName); + if (storageTablePartitionColumns.containsKey(partitionColumnKey)) { + return storageTablePartitionColumns.get(partitionColumnKey); + } + return new HashSet<String>(); + } + @Override public int hashCode() { return super.hashCode(); http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index c611963..34069d9 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -29,19 +29,16 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; import org.apache.lens.api.metastore.*; - import org.apache.lens.cube.authorization.AuthorizationUtil; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.Storage.LatestInfo; import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo; import org.apache.lens.cube.metadata.timeline.PartitionTimeline; import org.apache.lens.cube.metadata.timeline.PartitionTimelineFactory; - import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.authorization.ActionType; import org.apache.lens.server.api.authorization.LensAuthorizer; import org.apache.lens.server.api.authorization.LensPrivilegeObject; - import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.DataCompletenessChecker; import org.apache.lens.server.api.util.LensUtil; @@ -68,7 +65,6 @@ import com.google.common.base.Optional; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; - import lombok.extern.slf4j.Slf4j; /** @@ -330,11 +326,12 @@ public class CubeMetastoreClient { public void createCubeFactTable(String cubeName, String factName, List<FieldSchema> columns, Map<String, Set<UpdatePeriod>> storageAggregatePeriods, double weight, Map<String, String> properties, - Map<String, StorageTableDesc> storageTableDescs, Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap) + Map<String, StorageTableDesc> storageTableDescs, Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap, + Map<String, Set<String>> storageTablePartitionColumns) throws LensException { checkIfAuthorized(); CubeFactTable factTable = new CubeFactTable(cubeName, factName, columns, storageAggregatePeriods, weight, - properties, storageUpdatePeriodMap); + properties, storageUpdatePeriodMap, storageTablePartitionColumns); createCubeTable(factTable, storageTableDescs); // do a get to update cache getFactTable(factName); @@ -401,7 +398,8 @@ public class CubeMetastoreClient { xf.getWeight(), addFactColStartTimePropertyToFactProperties(xf), JAXBUtils.tableDescPrefixMapFromXStorageTables(xf.getStorageTables()), - JAXBUtils.storageTablePrefixMapOfStorage(xf.getStorageTables())); + JAXBUtils.storageTablePrefixMapOfStorage(xf.getStorageTables()), + JAXBUtils.getStorageTablePartitionColumnsFromTableDescs(xf.getStorageTables())); } } http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java index 96948cc..4881640 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java @@ -25,8 +25,8 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Table; import com.google.common.base.Optional; - import com.google.common.collect.Lists; + import lombok.Getter; import lombok.Setter; @@ -195,4 +195,9 @@ public class CubeVirtualFactTable extends AbstractCubeTable implements FactTable public Map<String, String> getSourceFactProperties() { return getSourceCubeFactTable().getProperties(); } + + @Override + public Set<String> getPartitionColumns(String storage) { + return new HashSet<String>(); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java index e351d7f..ab39e60 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java @@ -87,6 +87,13 @@ public interface FactTable extends Named { Set<String> getAllFieldNames(); /** + * Set of all partition columns for the fact + * @param storage Name of the stroage + * @return Set of all partition columns + */ + Set<String> getPartitionColumns(String storage); + + /** *tag for checking data completeness * * @return Tag String http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java index 7dfb166..623425e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java @@ -710,12 +710,14 @@ public final class JAXBUtils { public static CubeFactTable cubeFactFromFactTable(XFactTable fact) throws LensException { List<FieldSchema> columns = fieldSchemaListFromColumns(fact.getColumns()); - Map<String, Set<UpdatePeriod>> storageUpdatePeriods = getFactUpdatePeriodsFromStorageTables( - fact.getStorageTables()); - Map<String, Map<UpdatePeriod, String>> storageTablePrefixMap = storageTablePrefixMapOfStorage( - fact.getStorageTables()); + Map<String, Set<UpdatePeriod>> storageUpdatePeriods = + getFactUpdatePeriodsFromStorageTables(fact.getStorageTables()); + Map<String, Map<UpdatePeriod, String>> storageTablePrefixMap = + storageTablePrefixMapOfStorage(fact.getStorageTables()); + Map<String, Set<String>> storageTablePartitionColumns = + getStorageTablePartitionColumnsFromTableDescs(fact.getStorageTables()); return new CubeFactTable(fact.getCubeName(), fact.getName(), columns, storageUpdatePeriods, fact.getWeight(), - mapFromXProperties(fact.getProperties()), storageTablePrefixMap); + mapFromXProperties(fact.getProperties()), storageTablePrefixMap, storageTablePartitionColumns); } public static CubeVirtualFactTable cubeVirtualFactFromFactTable(XVirtualFactTable fact, FactTable sourceFactTable) @@ -893,6 +895,22 @@ public final class JAXBUtils { return storageTablePrefixToDescMap; } + public static Map<String, Set<String>> getStorageTablePartitionColumnsFromTableDescs(XStorageTables storageTables) { + Map<String, Set<String>> storageTablePartitionColumns = new HashMap<>(); + if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { + for (XStorageTableElement xStorageTableElement : storageTables.getStorageTable()) { + Set<String> partitionColumns = new HashSet<>(); + if (xStorageTableElement.getTableDesc() != null && xStorageTableElement.getTableDesc().getPartCols() != null) { + for (XColumn xColumn : xStorageTableElement.getTableDesc().getPartCols().getColumn()) { + partitionColumns.add(xColumn.getName()); + } + } + storageTablePartitionColumns.put(xStorageTableElement.getStorageName(), partitionColumns); + } + } + return storageTablePartitionColumns; + } + public static Map<String, Map<UpdatePeriod, String>> storageTablePrefixMapOfStorage(XStorageTables storageTables) { Map<String, Map<UpdatePeriod, String>> storageTableMap = new HashMap<>(); if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java index 44411f2..2c25d61 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java @@ -374,6 +374,10 @@ public class MetastoreUtil { return getFactKeyPrefix(name) + VALID_COLUMNS_SFX; } + public static String getPartitionColumnKey(String name, String storage) { + return getFactKeyPrefix(name) + "." + storage + PARTCOLS_SFX; + } + public static String getRestrictedColumnsKey(String name) { return getCubePrefix(name) + RESTRICTED_COLUMNS_SFX; } http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 193ce6b..ff67b8c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -47,7 +47,17 @@ import java.util.TimeZone; import java.util.TreeSet; import java.util.stream.Stream; -import org.apache.lens.cube.metadata.*; +import org.apache.lens.cube.metadata.AbstractCubeTable; +import org.apache.lens.cube.metadata.CubeFactTable; +import org.apache.lens.cube.metadata.CubeInterface; +import org.apache.lens.cube.metadata.DateUtil; +import org.apache.lens.cube.metadata.Dimension; +import org.apache.lens.cube.metadata.FactPartition; +import org.apache.lens.cube.metadata.FactTable; +import org.apache.lens.cube.metadata.MetastoreConstants; +import org.apache.lens.cube.metadata.MetastoreUtil; +import org.apache.lens.cube.metadata.TimeRange; +import org.apache.lens.cube.metadata.UpdatePeriod; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.DataCompletenessChecker; @@ -348,6 +358,7 @@ public class StorageCandidate implements Candidate, CandidateTable { factColumns = fact.getValidColumns(); if (factColumns == null) { factColumns = fact.getAllFieldNames(); + factColumns.addAll(fact.getPartitionColumns(storageName)); } } return factColumns; http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 9b8a55a..6e76eda 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -28,7 +28,6 @@ import static org.apache.lens.server.api.util.LensUtil.getHashMap; import static org.testng.Assert.*; import java.text.SimpleDateFormat; - import java.util.*; import org.apache.lens.cube.error.LensCubeErrorCode; @@ -1007,10 +1006,12 @@ public class TestCubeMetastoreClient { Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap = getHashMap(c1, getHashMap(HOURLY, HOURLY + "_" + c1, DAILY, DAILY + "_" + c1), c2, getHashMap(HOURLY, c2, DAILY, c2)); + Map<String, Set<String>> storageTablePartitionColumns = getHashMap("pl_pi", new HashSet<String>(Arrays.asList("pt" + , "it")), "pl_p3", new HashSet<String>(Arrays.asList("pt", "et"))); CubeFactTable cubeFact = new CubeFactTable(CUBE_NAME, factName, factColumns, updatePeriods, 0L, null, - storageUpdatePeriodMap); + storageUpdatePeriodMap, storageTablePartitionColumns); client.createCubeFactTable(CUBE_NAME, factName, factColumns, updatePeriods, 0L, null, storageTables, - storageUpdatePeriodMap); + storageUpdatePeriodMap, storageTablePartitionColumns); assertTrue(client.tableExists(factName)); Table cubeTbl = client.getHiveTable(factName); http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java b/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java index 7c8fd2c..5fb57dc 100644 --- a/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java +++ b/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java @@ -190,6 +190,7 @@ public class PopulateSampleMetastore { createFactPartition("fact2-local-part1.xml", "fact2", "local"); createFactPartition("fact2-local-part2.xml", "fact2", "local"); createFactPartition("fact2-local-part3.xml", "fact2", "local"); + createFactPartitions("fact3-local-part.xml", "fact3", "local"); createFactPartition("raw-local-part1.xml", "rawfact", "local"); createFactPartition("raw-local-part2.xml", "rawfact", "local"); createFactPartition("raw-local-part3.xml", "rawfact", "local"); http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java index 1aa993b..ac7470b 100644 --- a/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java +++ b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java @@ -167,6 +167,7 @@ public class SampleMetastore { private void createFacts() throws JAXBException, IOException { createFact("fact1.xml"); createFact("fact2.xml"); + createFact("fact3.xml"); createFact("rawfact.xml"); createFact("sales-raw-fact.xml"); createFact("sales-aggr-fact1.xml"); http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-examples/src/main/resources/cube-queries.sql ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/cube-queries.sql b/lens-examples/src/main/resources/cube-queries.sql index 0169175..f3d6cc7 100644 --- a/lens-examples/src/main/resources/cube-queries.sql +++ b/lens-examples/src/main/resources/cube-queries.sql @@ -16,7 +16,7 @@ -- specific language governing permissions and limitations -- under the License. -- - +cube select dim1, measure2 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') and nontimedim = 'nonTimeDimValue2' cube select measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') cube select measure2 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') cube select measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-examples/src/main/resources/fact3-local-part.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/fact3-local-part.xml b/lens-examples/src/main/resources/fact3-local-part.xml new file mode 100644 index 0000000..05ed815 --- /dev/null +++ b/lens-examples/src/main/resources/fact3-local-part.xml @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="UTF-8"?> +<x_partition_list xmlns="uri:lens:cube:0.1"> + <partition fact_or_dimension_table_name="fact3" location="examples/data/fact3-local1" update_period="DAILY"> + <non_time_partition_spec> + <part_spec_element key="nontimedim" value="nonTimeDimValue1"/> + </non_time_partition_spec> + <time_partition_spec> + <part_spec_element key="dt" value="2014-06-25T00:00:00"/> + </time_partition_spec> + </partition> + <partition fact_or_dimension_table_name="fact3" location="examples/data/fact3-local2" update_period="DAILY"> + <non_time_partition_spec> + <part_spec_element key="nontimedim" value="nonTimeDimValue2"/> + </non_time_partition_spec> + <time_partition_spec> + <part_spec_element key="dt" value="2014-06-25T00:00:00"/> + </time_partition_spec> + </partition> +</x_partition_list> http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-examples/src/main/resources/fact3-local1.data ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/fact3-local1.data b/lens-examples/src/main/resources/fact3-local1.data new file mode 100644 index 0000000..273765d --- /dev/null +++ b/lens-examples/src/main/resources/fact3-local1.data @@ -0,0 +1,8 @@ +21,1,100,100,100 +22,2,200,200,200 +23,3,300,300,300 +24,4,400,400,400 +25,5,500,500,500 +26,6,600,600,600 +27,7,700,700,700 +28,8,800,800,800 http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-examples/src/main/resources/fact3-local2.data ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/fact3-local2.data b/lens-examples/src/main/resources/fact3-local2.data new file mode 100644 index 0000000..2f7f831 --- /dev/null +++ b/lens-examples/src/main/resources/fact3-local2.data @@ -0,0 +1,8 @@ +21,1,2400,100,100 +22,2,4800,200,200 +23,3,7200,300,300 +24,4,9600,400,400 +25,5,12000,500,500 +26,6,14400,600,600 +27,7,16800,700,700 +28,8,19200,800,800 http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-examples/src/main/resources/fact3.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/fact3.xml b/lens-examples/src/main/resources/fact3.xml new file mode 100644 index 0000000..a8a93f8 --- /dev/null +++ b/lens-examples/src/main/resources/fact3.xml @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<x_fact_table cube_name="sample_cube" name="fact3" weight="100.0" xmlns="uri:lens:cube:0.1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd "> + <columns> + <column comment="" name="dim1" _type="INT"/> + <column comment="" name="measure2" _type="BIGINT"/> + <column comment="" name="measure3" _type="INT"/> + <column comment="" name="measure4" _type="FLOAT"/> + <column comment="" name="measure5" _type="FLOAT" start_time="2015-01-01"/> + <column comment="" name="measure6" _type="FLOAT" end_time="2015-01-01"/> + </columns> + <properties> + <property name="cube.fact.is.aggregated" value="true"/> + </properties> + <storage_tables> + <storage_table> + <update_periods> + <update_period>HOURLY</update_period> + <update_period>DAILY</update_period> + </update_periods> + <storage_name>local</storage_name> + <table_desc external="true" field_delimiter="," table_location="/tmp/examples/fact3_local"> + <part_cols> + <column comment="Time column" name="dt" _type="STRING"/> + <column comment="Non Time column" name="nontimedim" _type="STRING"/> + </part_cols> + <time_part_cols>dt</time_part_cols> + </table_desc> + </storage_table> + </storage_tables> +</x_fact_table> http://git-wip-us.apache.org/repos/asf/lens/blob/2d64f6ca/lens-examples/src/main/resources/sample-cube.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sample-cube.xml b/lens-examples/src/main/resources/sample-cube.xml index 6b17da5..464fb0b 100644 --- a/lens-examples/src/main/resources/sample-cube.xml +++ b/lens-examples/src/main/resources/sample-cube.xml @@ -53,6 +53,7 @@ <property name="is_ui_visible" value="true"/> </tags> </dim_attribute> + <dim_attribute name="nonTimeDim" _type="STRING"/> </dim_attributes> <expressions> <expression name="expr_msr5" _type="DOUBLE">
