lens git commit: LENS-903 : No candidate dim available exception should contain only brief error
Repository: lens Updated Branches: refs/heads/master 0b3203d04 -> 0a60ef331 LENS-903 : No candidate dim available exception should contain only brief error Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/0a60ef33 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/0a60ef33 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/0a60ef33 Branch: refs/heads/master Commit: 0a60ef331fa792e96f770e86df91bab36c70027d Parents: 0b3203d Author: Sushil Mohanty Authored: Wed Feb 10 12:22:32 2016 +0530 Committer: Sushil Mohanty Committed: Wed Feb 10 12:22:32 2016 +0530 -- lens-api/src/main/resources/lens-errors.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/0a60ef33/lens-api/src/main/resources/lens-errors.conf -- diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf index 9087fcd..09b221f 100644 --- a/lens-api/src/main/resources/lens-errors.conf +++ b/lens-api/src/main/resources/lens-errors.conf @@ -262,7 +262,7 @@ lensCubeErrorsForQuery = [ { errorCode = 3027 httpStatusCode = ${BAD_REQUEST} -errorMsg = "No candidate dimension table available for %s to answer the query, because %s" +errorMsg = "No candidate dimension table available to answer the query, because %s" } {
lens git commit: LENS-903 : No candidate dim available exception should contain only brief error
Repository: lens Updated Branches: refs/heads/current-release-line 2f1ddd463 -> 991ceb312 LENS-903 : No candidate dim available exception should contain only brief error Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/991ceb31 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/991ceb31 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/991ceb31 Branch: refs/heads/current-release-line Commit: 991ceb31262177e88b6dc6a89f6c498d5893a734 Parents: 2f1ddd4 Author: Sushil Mohanty Authored: Wed Feb 10 12:22:32 2016 +0530 Committer: Sushil Mohanty Committed: Thu Feb 11 15:37:40 2016 +0530 -- lens-api/src/main/resources/lens-errors.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/991ceb31/lens-api/src/main/resources/lens-errors.conf -- diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf index 9087fcd..09b221f 100644 --- a/lens-api/src/main/resources/lens-errors.conf +++ b/lens-api/src/main/resources/lens-errors.conf @@ -262,7 +262,7 @@ lensCubeErrorsForQuery = [ { errorCode = 3027 httpStatusCode = ${BAD_REQUEST} -errorMsg = "No candidate dimension table available for %s to answer the query, because %s" +errorMsg = "No candidate dimension table available to answer the query, because %s" } {
lens git commit: LENS-932 : Skip partition registration for very old and future dates for a fact tables
Repository: lens Updated Branches: refs/heads/master 6be4d48fd -> d81211723 LENS-932 : Skip partition registration for very old and future dates for a fact tables Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d8121172 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d8121172 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d8121172 Branch: refs/heads/master Commit: d812117238962c4d6c11103c221d808034f631e9 Parents: 6be4d48 Author: Sushil Mohanty Authored: Mon Feb 22 16:18:47 2016 +0530 Committer: Sushil Mohanty Committed: Mon Feb 22 16:18:47 2016 +0530 -- .../lens/cube/metadata/CubeMetastoreClient.java | 107 +++--- .../cube/metadata/TestCubeMetastoreClient.java | 145 ++- .../server/metastore/TestMetastoreService.java | 58 +++- 3 files changed, 283 insertions(+), 27 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/d8121172/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 8969d1f..02d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -863,18 +863,36 @@ public class CubeMetastoreClient { latestLookupCache.add(storageTableName); return partsAdded; } else { + List partsAdded = new ArrayList<>(); // first update in memory, then add to hive table's partitions. delete is reverse. partitionTimelineCache.updateForAddition(factOrDimTable, storageName, updatePeriod, -getTimePartSpecs(storagePartitionDescs)); + getTimePartSpecs(storagePartitionDescs, getStorageTableStartDate(storageTableName, factOrDimTable), + getStorageTableEndDate(storageTableName, factOrDimTable))); // Adding partition in fact table. - List partsAdded = -getStorage(storageName).addPartitions(getClient(), factOrDimTable, updatePeriod, storagePartitionDescs, null); + if (storagePartitionDescs.size() > 0) { +partsAdded = getStorage(storageName).addPartitions(getClient(), factOrDimTable, updatePeriod, +storagePartitionDescs, null); + } // update hive table alterTablePartitionCache(getStorageTableName(factOrDimTable, Storage.getPrefix(storageName))); return partsAdded; } } + private Date getStorageTableStartDate(String storageTable, String factTableName) +throws HiveException, LensException { +List startDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableStartTimesKey()); +startDates.add(getFactTable(factTableName).getStartTime()); +return Collections.max(startDates); + } + + private Date getStorageTableEndDate(String storageTable, String factTableName) +throws HiveException, LensException { +List endDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableEndTimesKey()); +endDates.add(getFactTable(factTableName).getEndTime()); +return Collections.min(endDates); + } + private Map> getTimePartSpecs(List storagePartitionDescs) { Map> timeSpecs = Maps.newHashMap(); for (StoragePartitionDesc storagePartitionDesc : storagePartitionDescs) { @@ -888,6 +906,41 @@ public class CubeMetastoreClient { return timeSpecs; } + + private Map> getTimePartSpecs(List storagePartitionDescs, + Date storageStartDate, Date storageEndDate) throws LensException { +Date now = new Date(); +Map> skippedParts = Maps.newHashMap(); +Map> timeSpecs = Maps.newHashMap(); +Iterator itr = storagePartitionDescs.iterator(); +while (itr.hasNext()) { + StoragePartitionDesc storageDesc = itr.next(); + for (Map.Entry entry : storageDesc.getTimePartSpec().entrySet()) { +if (!timeSpecs.containsKey(entry.getKey())) { + timeSpecs.put(entry.getKey(), Sets.newTreeSet()); +} +// check whether partition falls between storage table start_time and +// end_time or d+2, in such case partition is eligible for registration. +if ((entry.getValue().compareTo(storageStartDate) >= 0 && entry.getValue().compareTo(storageEndDate) < 0) +&& entry.getValue().compareTo(DateUtil.resolveRelativeDate("now +2 days", now)) < 0) { + timeSpecs.get(entry.getKey()).add(entry.getValue()); +} else { + if (!skippedParts.containsKey(entry.getKey())) { +skippedParts.put(entry.getKey(), Sets.newHash
lens git commit: LENS-995 : QueryContextPriorityComparator should compare the priorities instead of cost
Repository: lens Updated Branches: refs/heads/master 5faaf11b4 -> b36a02774 LENS-995 : QueryContextPriorityComparator should compare the priorities instead of cost Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b36a0277 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b36a0277 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b36a0277 Branch: refs/heads/master Commit: b36a027748413db07099837bbd7c62207b95c896 Parents: 5faaf11 Author: sushilmohanty Authored: Fri Apr 1 11:33:18 2016 +0530 Committer: sushilmohanty Committed: Fri Apr 1 11:33:18 2016 +0530 -- .../lens/server/api/LensConfConstants.java | 2 + .../lens/server/query/FIFOQueryComparator.java | 33 + .../lens/server/query/QueryComparator.java | 28 + .../query/QueryContextPriorityComparator.java | 48 .../lens/server/query/QueryCostComparator.java | 39 ++ .../server/query/QueryExecutionServiceImpl.java | 31 - .../server/query/QueryPriorityComparator.java | 38 ++ .../query/QueryContextComparatorTest.java | 121 +++ .../QueryContextPriorityComparatorTest.java | 99 --- .../collect/DefaultQueryCollectionTest.java | 34 +- .../server/query/collect/QueryCollectUtil.java | 25 +++- 11 files changed, 345 insertions(+), 153 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/b36a0277/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java -- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java index 724c868..23537cb 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java @@ -91,6 +91,8 @@ public final class LensConfConstants { public static final String MAX_SESSIONS_PER_USER = SERVER_PFX + "max.sessions.per.user"; + public static final String QUERY_COMPARATOR_CLASS = SERVER_PFX + "query.comparator.class"; + public static final Integer DEFAULT_MAX_SESSIONS_PER_USER = 10; /** http://git-wip-us.apache.org/repos/asf/lens/blob/b36a0277/lens-server/src/main/java/org/apache/lens/server/query/FIFOQueryComparator.java -- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/FIFOQueryComparator.java b/lens-server/src/main/java/org/apache/lens/server/query/FIFOQueryComparator.java new file mode 100644 index 000..75c1146 --- /dev/null +++ b/lens-server/src/main/java/org/apache/lens/server/query/FIFOQueryComparator.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.query; + +import org.apache.lens.server.api.query.QueryContext; + +public class FIFOQueryComparator implements QueryComparator { + + @Override + public int compare(QueryContext o1, QueryContext o2) { + +Long submitTimeO1 = o1.getSubmissionTime(); +Long submitTimeO2 = o2.getSubmissionTime(); + +return submitTimeO1.compareTo(submitTimeO2); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/b36a0277/lens-server/src/main/java/org/apache/lens/server/query/QueryComparator.java -- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryComparator.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryComparator.java new file mode 100644 index 000..67dda6b --- /dev/null +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryComparator.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information
lens git commit: LENS-998 : Partition skip is not logging correct start and end time in the logs.
Repository: lens Updated Branches: refs/heads/master b36a02774 -> 125a429ad LENS-998 : Partition skip is not logging correct start and end time in the logs. Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/125a429a Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/125a429a Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/125a429a Branch: refs/heads/master Commit: 125a429ad0ff333e1579c015e138777606339455 Parents: b36a027 Author: sushilmohanty Authored: Fri Apr 1 11:35:33 2016 +0530 Committer: sushilmohanty Committed: Fri Apr 1 11:35:33 2016 +0530 -- .../org/apache/lens/cube/metadata/CubeMetastoreClient.java | 8 +--- 1 file changed, 5 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/125a429a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index a29f42e..42a8eb2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -962,9 +962,11 @@ public class CubeMetastoreClient { } } } -log.info("List of partitions skipped : {}, because they fall before fact start time : {} " -+ "and after fact end time : {}", -skippedParts, storageStartDate, storageEndDate); +if (!skippedParts.isEmpty()) { + log.info("List of partitions skipped : {}, because they fall before fact start time : {} " + + "and after fact end time : {}", + skippedParts, storageStartDate, storageEndDate); +} return timeSpecs; }
lens git commit: LENS-39 : Add tags to cube column
Repository: lens Updated Branches: refs/heads/master 2cf0b51fd -> 90cbd94f4 LENS-39 : Add tags to cube column Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/90cbd94f Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/90cbd94f Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/90cbd94f Branch: refs/heads/master Commit: 90cbd94f439487b7e6da9fc6e085581037099928 Parents: 2cf0b51 Author: sushilmohanty Authored: Wed Apr 20 12:49:28 2016 +0530 Committer: sushilmohanty Committed: Wed Apr 20 12:49:28 2016 +0530 -- lens-api/src/main/resources/cube-0.1.xsd| 10 .../apache/lens/cli/TestLensCubeCommands.java | 3 +- lens-cli/src/test/resources/sample-cube.xml | 26 + .../lens/cube/metadata/BaseDimAttribute.java| 14 +++-- .../lens/cube/metadata/ColumnMeasure.java | 15 +++-- .../apache/lens/cube/metadata/CubeColumn.java | 36 +++- .../lens/cube/metadata/CubeDimAttribute.java| 12 +++- .../apache/lens/cube/metadata/CubeMeasure.java | 11 +++- .../apache/lens/cube/metadata/ExprColumn.java | 10 +++- .../lens/cube/metadata/MetastoreConstants.java | 1 + .../lens/cube/metadata/MetastoreUtil.java | 4 ++ .../cube/metadata/ReferencedDimAttribute.java | 8 ++- .../cube/metadata/TestCubeMetastoreClient.java | 60 .../src/main/resources/sample-cube.xml | 28 - .../examples/ExampleSchemaToStringTest.java | 2 +- .../src/test/resources/yaml/sales-cube.yaml | 14 - .../src/test/resources/yaml/sample-cube.yaml| 17 +- .../apache/lens/server/metastore/JAXBUtils.java | 19 ++- 18 files changed, 261 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-api/src/main/resources/cube-0.1.xsd -- diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index f221a6d..4a6a5d7 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -40,6 +40,16 @@ some documentation + + + + +Tags can be key value pairs associated with a field so that similar +fields can be categorized into certain group. + + + + http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java -- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java index 5a353df..f53ed6b 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java @@ -137,7 +137,8 @@ public class TestLensCubeCommands extends LensCliApplicationTest { private void testFields(LensCubeCommands command) { String fields = command.showQueryableFields("sample_cube", true); for (String field : Arrays - .asList("dim1", "dim2", "dim3", "dimdetail", "measure1", "measure2", "measure3", "measure4", "expr_msr5")) { + .asList("dim1", "dim2", "dim3", "dimdetail", "dim4", "measure1", "measure2", "measure3", "measure4", + "measure5", "measure6", "expr_msr5")) { assertTrue(fields.contains(field), fields + " do not contain " + field); } assertTrue(fields.contains("measure3 + measure4 + 0.01")); http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cli/src/test/resources/sample-cube.xml -- diff --git a/lens-cli/src/test/resources/sample-cube.xml b/lens-cli/src/test/resources/sample-cube.xml index 9bcf177..2d6bc34 100644 --- a/lens-cli/src/test/resources/sample-cube.xml +++ b/lens-cli/src/test/resources/sample-cube.xml @@ -30,6 +30,18 @@ + + + + + + + + + + + + @@ -40,12 +52,26 @@ + + + + + + + + + + + + + + http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cube/src/main/java/org/apache/lens/cube/metadata/BaseDimAttribute.java -- di
lens git commit: LENS-1149 : Union queries with dim-attribute expressions are not written properly
Repository: lens Updated Branches: refs/heads/master 62cafa448 -> a4b4b85d8 LENS-1149 : Union queries with dim-attribute expressions are not written properly Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a4b4b85d Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a4b4b85d Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a4b4b85d Branch: refs/heads/master Commit: a4b4b85d8179f1d7c7f0773f47b7aeebe2a0234d Parents: 62cafa4 Author: Amareshwari Sriramadasu Authored: Wed May 25 16:47:03 2016 +0530 Committer: sushilmohanty Committed: Wed May 25 16:47:03 2016 +0530 -- .../parse/SingleFactMultiStorageHQLContext.java | 169 --- .../apache/lens/cube/parse/CubeTestSetup.java | 2 + .../lens/cube/parse/TestUnionQueries.java | 68 3 files changed, 180 insertions(+), 59 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/a4b4b85d/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java index 1af031a..9b48213 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java @@ -66,7 +66,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { private void processSelectAST() { ASTNode originalSelectAST = MetastoreUtil.copyAST(ast.getSelectAST()); ast.setSelectAST(new ASTNode(originalSelectAST.getToken())); -ASTNode outerSelectAST = processExpression(originalSelectAST); +ASTNode outerSelectAST = processSelectExpression(originalSelectAST); setSelect(getString(outerSelectAST)); } @@ -86,7 +86,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { private void processOrderByAST() { if (ast.getOrderByAST() != null) { - setOrderby(getString(processExpression(ast.getOrderByAST(; + setOrderby(getString(processOrderbyExpression(ast.getOrderByAST(; ast.setOrderByAST(null); } } @@ -96,51 +96,117 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { ast.setLimitValue(null); } + private ASTNode processExpression(ASTNode astNode) { +if (astNode == null) { + return null; +} +ASTNode outerExpression = new ASTNode(astNode); +// iterate over all children of the ast and get outer ast corresponding to it. +for (Node child : astNode.getChildren()) { + outerExpression.addChild(getOuterAST((ASTNode)child)); +} +return outerExpression; + } + + private ASTNode processSelectExpression(ASTNode astNode) { +if (astNode == null) { + return null; +} +ASTNode outerExpression = new ASTNode(astNode); +// iterate over all children of the ast and get outer ast corresponding to it. +for (Node node : astNode.getChildren()) { + ASTNode child = (ASTNode)node; + ASTNode outerSelect = new ASTNode(child); + ASTNode selectExprAST = (ASTNode)child.getChild(0); + ASTNode outerAST = getOuterAST(selectExprAST); + outerSelect.addChild(outerAST); + + // has an alias? add it + if (child.getChildCount() > 1) { +outerSelect.addChild(child.getChild(1)); + } + outerExpression.addChild(outerSelect); +} +return outerExpression; + } + + private ASTNode processOrderbyExpression(ASTNode astNode) { +if (astNode == null) { + return null; +} +ASTNode outerExpression = new ASTNode(astNode); +// sample orderby AST looks the following : +/* +TOK_ORDERBY + TOK_TABSORTCOLNAMEDESC + TOK_NULLS_LAST + . +TOK_TABLE_OR_COL + testcube +cityid + TOK_TABSORTCOLNAMEASC + TOK_NULLS_FIRST + . +TOK_TABLE_OR_COL + testcube +stateid + TOK_TABSORTCOLNAMEASC + TOK_NULLS_FIRST + . +TOK_TABLE_OR_COL + testcube +zipcode + */ +for (Node node : astNode.getChildren()) { + ASTNode child = (ASTNode)node; + ASTNode outerOrderby = new ASTNode(child); + ASTNode tokNullsChild = (ASTNode) child.getChild(0); + ASTNode outerTokNullsChild = new ASTNode(tokNullsChild); + outerTokNullsChild.addChild(getOuterAST((ASTNode)tokNullsChild.getChild(0))); + outerOrderby.addChild(outerTokNullsChild); + outerExpression.addChild(outerOrderby); +} +return outerExpression; + } /* + Perform a DFS on the provid
lens git commit: LENS-1155 : Seeing INTERNAL_SERVER_ERROR for JDBC queries due to connection unavailability
Repository: lens Updated Branches: refs/heads/master a4b4b85d8 -> 5c25e68fe LENS-1155 : Seeing INTERNAL_SERVER_ERROR for JDBC queries due to connection unavailability Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/5c25e68f Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/5c25e68f Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/5c25e68f Branch: refs/heads/master Commit: 5c25e68fe71670df6c72d3b8dc08fcbcad6011c4 Parents: a4b4b85 Author: SushilMohanty Authored: Wed May 25 16:53:22 2016 +0530 Committer: sushilmohanty Committed: Wed May 25 16:53:22 2016 +0530 -- .../org/apache/lens/driver/jdbc/JDBCDriver.java | 3 +- .../jdbc/MaxJDBCConnectionCheckConstraint.java | 48 ...MaxJDBCConnectionCheckConstraintFactory.java | 38 .../src/main/resources/jdbcdriver-default.xml | 3 +- .../apache/lens/driver/jdbc/TestJdbcDriver.java | 37 +++ .../apache/lens/server/api/util/LensUtil.java | 2 +- 6 files changed, 128 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java -- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index 044a19c..8047302 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -94,6 +94,7 @@ public class JDBCDriver extends AbstractLensDriver { private ExecutorService asyncQueryPool; /** The query context map. */ + @Getter private ConcurrentHashMap queryContextMap; /** The conf. */ @@ -458,7 +459,7 @@ public class JDBCDriver extends AbstractLensDriver { final int maxPoolSize = parseInt(this.conf.get(JDBC_POOL_MAX_SIZE.getConfigKey())); final int maxConcurrentQueries = parseInt(this.conf.get(MaxConcurrentDriverQueriesConstraintFactory.MAX_CONCURRENT_QUERIES_KEY)); -checkState(maxPoolSize == maxConcurrentQueries, "maxPoolSize:" + maxPoolSize + " maxConcurrentQueries:" +checkState(maxPoolSize >= maxConcurrentQueries, "maxPoolSize:" + maxPoolSize + " maxConcurrentQueries:" + maxConcurrentQueries); queryContextMap = new ConcurrentHashMap<>(); http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java -- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java new file mode 100644 index 000..82b5647 --- /dev/null +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.driver.jdbc; + +import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection; +import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class MaxJDBCConnectionCheckConstraint implements QueryLaunchingConstraint { + + private final int poolMaxSize; + + public MaxJDBCConnectionCheckConstraint(final int poolMaxSize) { +this.poolMaxSize = poolMaxSize; + } + + @Override + public boolean allowsLaunchOf(final QueryContext candidateQuery, +EstimatedImmutableQueryCollection launchedQueries) { +final LensDriver selectedDriver = candidateQuery.getSelectedDriver
lens git commit: LENS-1289 : Fact column start_time and end_time not getting reflected with update fact command
Repository: lens Updated Branches: refs/heads/master c2100fa2d -> 2e56643b8 LENS-1289 : Fact column start_time and end_time not getting reflected with update fact command Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2e56643b Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2e56643b Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2e56643b Branch: refs/heads/master Commit: 2e56643b81cfb871bd133bbf9df55707e973f3ae Parents: c2100fa Author: Sushil Mohanty Authored: Fri Aug 26 13:17:56 2016 +0530 Committer: sushilmohanty Committed: Fri Aug 26 13:17:56 2016 +0530 -- .../test/java/org/apache/lens/cli/TestLensFactCommands.java | 7 ++- .../org/apache/lens/cube/metadata/CubeMetastoreClient.java | 7 ++- .../apache/lens/cube/metadata/TestCubeMetastoreClient.java | 8 .../lens/server/metastore/CubeMetastoreServiceImpl.java | 3 ++- 4 files changed, 18 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/2e56643b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java -- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java index 9670d8f..b4cbf34 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java @@ -162,6 +162,10 @@ public class TestLensFactCommands extends LensCliApplicationTest { xmlContent = xmlContent.replace("\n", "" + "\n\n"); + xmlContent = xmlContent.replace("", + "" + + "\n\n"); + File newFile = new File("target/local-fact1.xml"); Writer writer = new OutputStreamWriter(new FileOutputStream(newFile)); writer.write(xmlContent); @@ -171,7 +175,7 @@ public class TestLensFactCommands extends LensCliApplicationTest { log.debug(desc); String propString = "fact1.prop: f1"; String propString1 = "fact1.prop1: f2"; - + String propStringColStartTime = "cube.fact.col.start.time.measure4: 2015-01-01"; assertTrue(desc.contains(propString)); command.updateFactTable("fact1", new File("target/local-fact1.xml")); @@ -179,6 +183,7 @@ public class TestLensFactCommands extends LensCliApplicationTest { log.debug(desc); assertTrue(desc.contains(propString), "The sample property value is not set"); assertTrue(desc.contains(propString1), "The sample property value is not set"); + assertTrue(desc.contains(propStringColStartTime), "The sample property value is not set"); newFile.delete(); http://git-wip-us.apache.org/repos/asf/lens/blob/2e56643b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 0dce6c4..e14c43f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -2275,8 +2275,13 @@ public class CubeMetastoreClient { * @throws HiveException */ public void alterCubeFactTable(String factTableName, CubeFactTable cubeFactTable, -Map storageTableDescs) throws HiveException, LensException { + Map storageTableDescs, + Map props) +throws HiveException, LensException { Table factTbl = getTableWithTypeFailFast(factTableName, CubeTableType.FACT); +if (!props.isEmpty()) { + cubeFactTable.getProperties().putAll(props); +} alterCubeTable(factTableName, factTbl, cubeFactTable); if (storageTableDescs != null) { // create/alter tables for each storage http://git-wip-us.apache.org/repos/asf/lens/blob/2e56643b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java -- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 62c7ec9..e21dc2a 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -1062,7 +106
lens git commit: LENS-1360 : org.apache.lens.driver.jdbc.TestJdbcDriver#testCancelQuery failing in pre-commit
Repository: lens Updated Branches: refs/heads/master f274e29d5 -> 547a42f33 LENS-1360 : org.apache.lens.driver.jdbc.TestJdbcDriver#testCancelQuery failing in pre-commit Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/547a42f3 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/547a42f3 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/547a42f3 Branch: refs/heads/master Commit: 547a42f33caa7669fe54509e6e4dac108e904a61 Parents: f274e29 Author: Amareshwari Sriramadasu Authored: Tue Oct 25 14:04:45 2016 +0530 Committer: sushilmohanty Committed: Tue Oct 25 14:04:45 2016 +0530 -- .../org/apache/lens/driver/jdbc/JDBCDriver.java | 2 +- .../apache/lens/driver/jdbc/TestJdbcDriver.java | 3 +- lens-driver-jdbc/src/test/resources/logback.xml | 36 3 files changed, 39 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/547a42f3/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java -- diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java index 82e0231..7ce1a97 100644 --- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java +++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java @@ -290,7 +290,7 @@ public class JDBCDriver extends AbstractLensDriver { } if (queryContext.isClosed()) { log.info("Ignored exception on already closed query : {} - {}", -queryContext.getLensContext().getQueryHandle(), e.getMessage()); +queryContext.getLensContext().getQueryHandle(), e.getMessage(), e); } else { log.error("Error executing SQL query: {} reason: {}", queryContext.getLensContext().getQueryHandle(), e.getMessage(), e); http://git-wip-us.apache.org/repos/asf/lens/blob/547a42f3/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java -- diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java index a402f91..6291021 100644 --- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java +++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java @@ -834,6 +834,7 @@ public class TestJdbcDriver { public static int sleep(int t) { try { + log.info("Sleeping for {} seconds", t); Thread.sleep(t * 1000); } catch (InterruptedException ie) { // ignore @@ -878,7 +879,7 @@ public class TestJdbcDriver { QueryHandle handle = context.getQueryHandle(); // without wait query may not be launched. if (waitBeforeCancel) { - Thread.sleep(100); + Thread.sleep(1000); } boolean isCancelled = driver.cancelQuery(handle); driver.updateStatus(context); http://git-wip-us.apache.org/repos/asf/lens/blob/547a42f3/lens-driver-jdbc/src/test/resources/logback.xml -- diff --git a/lens-driver-jdbc/src/test/resources/logback.xml b/lens-driver-jdbc/src/test/resources/logback.xml new file mode 100644 index 000..eed4f66 --- /dev/null +++ b/lens-driver-jdbc/src/test/resources/logback.xml @@ -0,0 +1,36 @@ + + + + +target/test.log + + %d{dd MMM HH:mm:ss,SSS} [%t] %-5p %c %L - %m%n + + + DEBUG + + + + + + +
[3/4] lens git commit: Addressed final reviews and fixed checkstyle error
http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java -- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java index ba8a5e4..93dbfc3 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java @@ -37,7 +37,6 @@ import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.error.NoCandidateFactAvailableException; import org.apache.lens.cube.metadata.TimeRange; import org.apache.lens.cube.metadata.UpdatePeriod; -import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.time.DateUtils; @@ -68,15 +67,12 @@ public class TestBaseCubeQueries extends TestQueryRewrite { } @Test - public void testNoCandidateFactAvailableExceptionCompareTo() throws Exception { -//maxCause : UNSUPPORTED_STORAGE -NoCandidateFactAvailableException ne1 = - (NoCandidateFactAvailableException)getLensExceptionInRewrite( + public void testNoUnionCandidateAndNoJoinCandidateErrorWeight() throws Exception { +LensException e1 = getLensExceptionInRewrite( "select dim1, test_time_dim, msr3, msr13 from basecube where " + TWO_DAYS_RANGE, conf); -//maxCause : STORAGE_NOT_AVAILABLE_IN_RANGE -NoCandidateFactAvailableException ne2 = (NoCandidateFactAvailableException) - getLensExceptionInRewrite("select dim1 from " + cubeName + " where " + LAST_YEAR_RANGE, getConf()); -assertEquals(ne1.compareTo(ne2), -1); +LensException e2 = getLensExceptionInRewrite("select dim1 from " + cubeName ++ " where " + LAST_YEAR_RANGE, getConf()); +assertEquals(e1.getErrorWeight() - e2.getErrorWeight(), 1); } @Test @@ -92,53 +88,9 @@ public class TestBaseCubeQueries extends TestQueryRewrite { e = getLensExceptionInRewrite("select dim1, test_time_dim, msr3, msr13 from basecube where " + TWO_DAYS_RANGE, conf); assertEquals(e.getErrorCode(), - LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo().getErrorCode()); -NoCandidateFactAvailableException ne = (NoCandidateFactAvailableException) e; -//ne.briefAndDetailedError.getBriefCause() -//ne.getJsonMessage().brief - assertTrue(CandidateTablePruneCode.UNSUPPORTED_STORAGE.errorFormat.equals(ne.getJsonMessage().getBrief())); -//PruneCauses.BriefAndDetailedError pruneCauses = ne.getJsonMessage(); -//String regexp = String.format(CandidateTablePruneCode.UNSUPPORTED_STORAGE.errorFormat, -// "Column Sets: (.*?)", "queriable together"); -//Matcher matcher = Pattern.compile(regexp).matcher(pruneCauses.getBrief()); -//assertTrue(matcher.matches(), pruneCauses.getBrief()); -//assertEquals(matcher.groupCount(), 1); -//String columnSetsStr = matcher.group(1); -//assertNotEquals(columnSetsStr.indexOf("test_time_dim"), -1, columnSetsStr); -//assertNotEquals(columnSetsStr.indexOf("msr3, msr13"), -1); -// -///** -// * Verifying the BriefAndDetailedError: -// * 1. Check for missing columns(COLUMN_NOT_FOUND) -// *and check the respective tables for each COLUMN_NOT_FOUND -// * 2. check for ELEMENT_IN_SET_PRUNED -// * -// */ -//boolean columnNotFound = false; -//List testTimeDimFactTables = Arrays.asList("c1_testfact3_raw_base", -//"c1_testfact5_base", "c1_testfact6_base", "c1_testfact1_raw_base", -//"c1_testfact4_raw_base", "c1_testfact3_base"); -//List factTablesForMeasures = Arrays.asList( -// "c2_testfact2_base","c2_testfact_deprecated","c1_union_join_ctx_fact1","c1_union_join_ctx_fact2", -// "c1_union_join_ctx_fact3","c1_union_join_ctx_fact5","c1_testfact2_base", -// "c1_union_join_ctx_fact6","c1_testfact2_raw_base","c1_testfact5_raw_base", -// "c3_testfact_deprecated","c1_testfact_deprecated","c4_testfact_deprecated", -//"c3_testfact2_base","c4_testfact2_base"); -//for (Map.Entry> entry : pruneCauses.getDetails().entrySet()) { -// if (entry.getValue().contains(CandidateTablePruneCause.columnNotFound( -// CandidateTablePruneCode.COLUMN_NOT_FOUND, "test_time_dim"))) { -//columnNotFound = true; -//compareStrings(testTimeDimFactTables, entry); -// } -// if (entry.getValue().contains(CandidateTablePruneCause.columnNotFound( -// CandidateTablePruneCode.COLUMN_NOT_FOUND, "msr3", "msr13"))) { -//columnNotFound = true; -//compareStrings(factTablesForMeasures, entry); -// } -//} -//Assert.assertTrue(columnNotFound); - // assertEquals(pruneCauses.getDetails().get("testfact1_base"), - // Array
[2/4] lens git commit: Addressed final reviews and fixed checkstyle error
http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java -- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java index 9168d10..bed1e7a 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java @@ -30,14 +30,11 @@ import static org.testng.Assert.*; import java.util.*; import java.util.stream.Collectors; -import org.apache.lens.cube.error.LensCubeErrorCode; -import org.apache.lens.cube.error.NoCandidateDimAvailableException; import org.apache.lens.cube.error.NoCandidateFactAvailableException; import org.apache.lens.server.api.LensServerAPITestUtil; import org.apache.lens.server.api.error.LensException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.parse.ParseException; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; @@ -53,12 +50,12 @@ public class TestUnionQueries extends TestQueryRewrite { @BeforeTest public void setupDriver() throws Exception { testConf = LensServerAPITestUtil.getConfiguration( - DRIVER_SUPPORTED_STORAGES, "C0,C1,C2", - DISABLE_AUTO_JOINS, false, - ENABLE_SELECT_TO_GROUPBY, true, - ENABLE_GROUP_BY_TO_SELECT, true, - DISABLE_AGGREGATE_RESOLVER, false, - ENABLE_STORAGES_UNION, true); +DRIVER_SUPPORTED_STORAGES, "C0,C1,C2", +DISABLE_AUTO_JOINS, false, +ENABLE_SELECT_TO_GROUPBY, true, +ENABLE_GROUP_BY_TO_SELECT, true, +DISABLE_AGGREGATE_RESOLVER, false, +ENABLE_STORAGES_UNION, true); } @Override @@ -66,334 +63,228 @@ public class TestUnionQueries extends TestQueryRewrite { return new Configuration(testConf); } - //TODOD union: @Test + @Test public void testUnionQueries() throws Exception { -Configuration conf = getConf(); -conf.set(getValidStorageTablesKey("testfact"), "C1_testFact,C2_testFact"); -conf.set(getValidUpdatePeriodsKey("testfact", "C1"), "DAILY,HOURLY"); -conf.set(getValidUpdatePeriodsKey("testfact2", "C1"), "YEARLY"); -conf.set(getValidUpdatePeriodsKey("testfact", "C2"), "MONTHLY,DAILY"); -conf.setBoolean(CubeQueryConfUtil.ENABLE_STORAGES_UNION, false); -ArrayList storages = Lists.newArrayList("c1_testfact", "c2_testfact"); +Configuration conf = LensServerAPITestUtil.getConfigurationWithParams(getConf(), +CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C6", +getValidFactTablesKey("basecube"), "testfact", +FAIL_QUERY_ON_PARTIAL_DATA, false); +ArrayList storages = Lists.newArrayList("daily_c6_testfact", "monthly_c6_testfact"); try { - getStorageToUpdatePeriodMap().put("c1_testfact", Lists.newArrayList(HOURLY, DAILY)); - getStorageToUpdatePeriodMap().put("c2_testfact", Lists.newArrayList(MONTHLY)); - // Union query String hqlQuery; String expected; StoragePartitionProvider provider = new StoragePartitionProvider() { @Override public Map providePartitionsForStorage(String storage) { - return getWhereForMonthlyDailyAndHourly2monthsUnionQuery(storage); + if (storage.contains("daily_c6_testfact")) { +return getWhereForDays(storage, ONE_MONTH_BACK_TRUNCATED, getTruncatedDateWithOffset(MONTHLY, 0)); + } else if (storage.contains("monthly_c6_testfact")) { +return getWhereForMonthly(storage, THREE_MONTHS_BACK_TRUNCATED, ONE_MONTH_BACK_TRUNCATED); + } + return null; } }; - try { -rewrite("select cityid as `City ID`, msr8, msr7 as `Third measure` " - + "from testCube where " + TWO_MONTHS_RANGE_UPTO_HOURS, conf); - // fail("Union feature is disabled, should have failed"); - } catch (LensException e) { -assertEquals(e.getErrorCode(), LensCubeErrorCode.STORAGE_UNION_DISABLED.getLensErrorInfo().getErrorCode()); - } - conf.setBoolean(CubeQueryConfUtil.ENABLE_STORAGES_UNION, true); - - hqlQuery = rewrite("select ascii(cityname) as `City Name`, msr8, msr7 as `Third measure` " -+ "from testCube where ascii(cityname) = 'c' and cityname = 'a' and zipcode = 'b' and " -+ TWO_MONTHS_RANGE_UPTO_HOURS, conf); + hqlQuery = rewrite("select cityname1 as `City Name`, msr8, msr7 as `Third measure` " + + "from testCube where cityname1 = 'a' and zipcode = 'b' and " + + THREE_MONTHS_RANGE_UPTO_MONTH, conf); expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, -"SELECT testcube.alias0 as `City Name`, sum(testcube.alias1) + max(testcube.alias2), " - + "case when sum(testcube.alias1) = 0 then 0 else sum(testcube.alias3)/sum(testcube.
[1/4] lens git commit: Addressed final reviews and fixed checkstyle error
Repository: lens Updated Branches: refs/heads/lens-1381 72c20ff83 -> cb5e2a77d http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/statetable.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/statetable.xml b/lens-cube/src/test/resources/schema/dimtables/statetable.xml index 2aab131..58eb948 100644 --- a/lens-cube/src/test/resources/schema/dimtables/statetable.xml +++ b/lens-cube/src/test/resources/schema/dimtables/statetable.xml @@ -1,4 +1,24 @@ + @@ -32,5 +52,23 @@ dt + + +HOURLY + + C6 + + + + + + + + + + +dt + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/statetable_partitioned.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/statetable_partitioned.xml b/lens-cube/src/test/resources/schema/dimtables/statetable_partitioned.xml index e7c808f..826d8dc 100644 --- a/lens-cube/src/test/resources/schema/dimtables/statetable_partitioned.xml +++ b/lens-cube/src/test/resources/schema/dimtables/statetable_partitioned.xml @@ -1,4 +1,24 @@ + http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/testdim2tbl.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl.xml b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl.xml index b320ec2..d04cb5c 100644 --- a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl.xml +++ b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl.xml @@ -1,4 +1,24 @@ + http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/testdim2tbl2.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl2.xml b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl2.xml index 2239997..efc0969 100644 --- a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl2.xml +++ b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl2.xml @@ -1,4 +1,24 @@ + http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/testdim2tbl3.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl3.xml b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl3.xml index 62059ad..d9b39c7 100644 --- a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl3.xml +++ b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl3.xml @@ -1,4 +1,24 @@ + http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/testdim3tbl.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim3tbl.xml b/lens-cube/src/test/resources/schema/dimtables/testdim3tbl.xml index 7d2af60..ffd09c0 100644 --- a/lens-cube/src/test/resources/schema/dimtables/testdim3tbl.xml +++ b/lens-cube/src/test/resources/schema/dimtables/testdim3tbl.xml @@ -1,4 +1,24 @@ + http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/testdim4tbl.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim4tbl.xml b/lens-cube/src/test/resources/schema/dimtables/testdim4tbl.xml index 2044851..d81daa7 100644 --- a/lens-cube/src/test/resources/schema/dimtables/testdim4tbl.xml +++ b/lens-cube/src/test/resources/schema/dimtables/testdim4tbl.xml @@ -1,4 +1,24 @@ + http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/unreachabledimtable.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/unreachabledimtable.xml b/lens-cube/src/test/resources/schema/dimtables/unreachabledimtable.xml index 6f80c27..e940cf7 100644 --- a/lens-cube/src/test/resources/schema/dimtables/unreachabledimtable.xml +++ b/lens-cube/src/test/resources/schema/dimtables/unreachabledimtable.xml @@ -1,4 +1,24 @@ + http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/user_interests_tbl.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/user_interests_tbl.xml b/lens-cube/src/test/resources/sc
[4/4] lens git commit: Addressed final reviews and fixed checkstyle error
Addressed final reviews and fixed checkstyle error Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/cb5e2a77 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/cb5e2a77 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/cb5e2a77 Branch: refs/heads/lens-1381 Commit: cb5e2a77dfd09c6736c9b295f68addba6bf27e6e Parents: 72c20ff Author: sushilmohanty Authored: Wed Apr 12 18:10:36 2017 +0530 Committer: sushilmohanty Committed: Wed Apr 12 18:10:36 2017 +0530 -- lens-api/src/main/resources/lens-errors.conf| 13 + .../lens/cube/error/LensCubeErrorCode.java | 2 + .../NoCandidateFactAvailableException.java | 13 +- .../lens/cube/metadata/CubeMetastoreClient.java | 5 +- .../org/apache/lens/cube/metadata/DateUtil.java | 3 +- .../lens/cube/metadata/FactPartition.java | 7 + .../lens/cube/metadata/MetastoreUtil.java | 2 +- .../apache/lens/cube/metadata/TimeRange.java| 4 + .../cube/parse/AbridgedTimeRangeWriter.java | 3 - .../parse/CandidateCoveringSetsResolver.java| 67 ++- .../apache/lens/cube/parse/CandidateDim.java| 6 +- .../apache/lens/cube/parse/CandidateTable.java | 2 +- .../cube/parse/CandidateTablePruneCause.java| 41 +- .../apache/lens/cube/parse/CandidateUtil.java | 15 +- .../lens/cube/parse/CubeQueryContext.java | 83 ++-- .../lens/cube/parse/CubeQueryRewriter.java | 3 +- .../lens/cube/parse/CubeSemanticAnalyzer.java | 3 +- .../cube/parse/DenormalizationResolver.java | 22 +- .../lens/cube/parse/ExpressionResolver.java | 3 +- .../apache/lens/cube/parse/JoinCandidate.java | 2 +- .../cube/parse/MaxCoveringFactResolver.java | 1 - .../org/apache/lens/cube/parse/PruneCauses.java | 3 +- .../lens/cube/parse/StorageCandidate.java | 36 +- .../lens/cube/parse/StorageTableResolver.java | 20 +- .../apache/lens/cube/parse/UnionCandidate.java | 24 +- .../lens/cube/parse/UnionQueryWriter.java | 22 +- .../apache/lens/driver/cube/RewriterPlan.java | 6 +- .../apache/lens/cube/parse/CubeTestSetup.java | 25 +- .../lens/cube/parse/TestAggregateResolver.java | 42 +- .../lens/cube/parse/TestBaseCubeQueries.java| 226 - .../cube/parse/TestBetweenTimeRangeWriter.java | 6 +- .../lens/cube/parse/TestBridgeTableQueries.java | 40 +- .../lens/cube/parse/TestCubeRewriter.java | 11 +- .../cube/parse/TestDenormalizationResolver.java | 10 +- .../lens/cube/parse/TestTimeRangeResolver.java | 36 +- .../cube/parse/TestUnionAndJoinCandidates.java | 24 +- .../lens/cube/parse/TestUnionQueries.java | 464 +++ .../resources/schema/cubes/base/basecube.xml| 20 + .../resources/schema/cubes/base/testcube.xml| 23 + .../resources/schema/cubes/derived/der1.xml | 20 + .../resources/schema/cubes/derived/der2.xml | 20 + .../resources/schema/cubes/derived/der3.xml | 20 + .../schema/cubes/derived/derivedcube.xml| 20 + .../cubes/derived/union_join_ctx_der1.xml | 20 + .../resources/schema/dimensions/citydim.xml | 20 + .../resources/schema/dimensions/countrydim.xml | 20 + .../resources/schema/dimensions/cycledim1.xml | 20 + .../resources/schema/dimensions/cycledim2.xml | 20 + .../test/resources/schema/dimensions/daydim.xml | 20 + .../resources/schema/dimensions/hourdim.xml | 20 + .../test/resources/schema/dimensions/sports.xml | 20 + .../resources/schema/dimensions/statedim.xml| 20 + .../resources/schema/dimensions/testdim2.xml| 20 + .../resources/schema/dimensions/testdim3.xml| 20 + .../resources/schema/dimensions/testdim4.xml| 20 + .../schema/dimensions/unreachabledim.xml| 20 + .../schema/dimensions/user_interests.xml| 20 + .../resources/schema/dimensions/userdim.xml | 20 + .../test/resources/schema/dimensions/zipdim.xml | 20 + .../resources/schema/dimtables/citytable.xml| 37 ++ .../resources/schema/dimtables/citytable2.xml | 20 + .../resources/schema/dimtables/citytable3.xml | 20 + .../resources/schema/dimtables/citytable4.xml | 20 + .../resources/schema/dimtables/countrytable.xml | 20 + .../dimtables/countrytable_partitioned.xml | 20 + .../resources/schema/dimtables/cycledim1tbl.xml | 20 + .../resources/schema/dimtables/cycledim2tbl.xml | 20 + .../resources/schema/dimtables/daydimtbl.xml| 20 + .../resources/schema/dimtables/hourdimtbl.xml | 20 + .../resources/schema/dimtables/sports_tbl.xml | 20 + .../resources/schema/dimtables/statetable.xml | 38 ++ .../schema/dimtables/statetable_partitioned.xml | 20 + .../resources/schema/dimtables/testdim2tbl.xml | 20 + .../resources/schema/dimtables/testdim2tbl2.xml | 20 + .../resources/schema/dimtables/testdim2tbl3.xml | 20 + .../resources/schema/dimtables/testdim3tbl.xml | 20
lens git commit: LENS-1428 : Common measures are getting repeated in UnionQuery for JoinCandidate
Repository: lens Updated Branches: refs/heads/master 5de45e0f8 -> 159cf71e7 LENS-1428 : Common measures are getting repeated in UnionQuery for JoinCandidate Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/159cf71e Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/159cf71e Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/159cf71e Branch: refs/heads/master Commit: 159cf71e732a0b63bf8c7efde6489f4eea8d943a Parents: 5de45e0 Author: Sushil Mohanty Authored: Tue Jun 6 15:08:38 2017 +0530 Committer: sushilmohanty Committed: Tue Jun 6 15:08:38 2017 +0530 -- .../apache/lens/cube/metadata/CubeMetastoreClient.java | 2 +- .../java/org/apache/lens/cube/parse/Candidate.java | 9 + .../org/apache/lens/cube/parse/CubeQueryContext.java | 5 + .../java/org/apache/lens/cube/parse/JoinCandidate.java | 13 + .../org/apache/lens/cube/parse/StorageCandidate.java | 7 +++ .../org/apache/lens/cube/parse/UnionQueryWriter.java | 6 ++ 6 files changed, 37 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/159cf71e/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 749e44c..5c05e53 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -443,7 +443,7 @@ public class CubeMetastoreClient { if (get(timeLineKey) == null) { loadTimeLines(fact, storage, timeLineKey); } -log.info("timeline for {} is: {}", storage, get(timeLineKey)); +log.debug("timeline for {} is: {}", storage, get(timeLineKey)); // return the final value from memory return get(timeLineKey); // RESUME CHECKSTYLE CHECK DoubleCheckedLockingCheck http://git-wip-us.apache.org/repos/asf/lens/blob/159cf71e/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java index 2edeb41..0855ced 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java @@ -355,4 +355,13 @@ public interface Candidate { } throw new IllegalArgumentException("Candidate doesn't have children and no suitable implementation found"); } + + default Set decideMeasurePhrasesToAnswer(Set measurePhraseIndices) throws LensException { +HashSet allCovered = Sets.newHashSet(); +for (Candidate candidate : getChildren()) { + Set covered = candidate.decideMeasurePhrasesToAnswer(measurePhraseIndices); + allCovered.addAll(covered); +} +return allCovered; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/159cf71e/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index 47bce65..b5330a7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -830,6 +830,11 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, cand = iter.next(); log.info("Available Candidates:{}, picking up Candidate: {} for querying", candidates, cand); pickedCandidate = cand; +// Answerable common measures in JoinCandidate should be answered by one of the children, otherwise +// measure numbers will be added multiple times in the final union query. +Set measureIndices = getQueriedPhrases().stream().filter(x -> x.hasMeasures(this)) + .map(QueriedPhraseContext::getPosition).collect(toSet()); +pickedCandidate.decideMeasurePhrasesToAnswer(measureIndices); } if (pickedCandidate == null) { throwNoCandidateFactException(); http://git-wip-us.apache.org/repos/asf/lens/blob/159cf71e/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.j
lens git commit: LENS-1428 : Common measures are getting repeated in UnionQuery for JoinCandidate
Repository: lens Updated Branches: refs/heads/current-release-line 27ad9aa8f -> ac9a56fcd LENS-1428 : Common measures are getting repeated in UnionQuery for JoinCandidate Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/ac9a56fc Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/ac9a56fc Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/ac9a56fc Branch: refs/heads/current-release-line Commit: ac9a56fcd828b0cbb6acee614c72f650a6cb78b8 Parents: 27ad9aa Author: Sushil Mohanty Authored: Tue Jun 6 15:10:24 2017 +0530 Committer: sushilmohanty Committed: Tue Jun 6 15:10:24 2017 +0530 -- .../apache/lens/cube/metadata/CubeMetastoreClient.java | 2 +- .../java/org/apache/lens/cube/parse/Candidate.java | 9 + .../org/apache/lens/cube/parse/CubeQueryContext.java | 5 + .../java/org/apache/lens/cube/parse/JoinCandidate.java | 13 + .../org/apache/lens/cube/parse/StorageCandidate.java | 7 +++ .../org/apache/lens/cube/parse/UnionQueryWriter.java | 6 ++ 6 files changed, 37 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/ac9a56fc/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 749e44c..5c05e53 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -443,7 +443,7 @@ public class CubeMetastoreClient { if (get(timeLineKey) == null) { loadTimeLines(fact, storage, timeLineKey); } -log.info("timeline for {} is: {}", storage, get(timeLineKey)); +log.debug("timeline for {} is: {}", storage, get(timeLineKey)); // return the final value from memory return get(timeLineKey); // RESUME CHECKSTYLE CHECK DoubleCheckedLockingCheck http://git-wip-us.apache.org/repos/asf/lens/blob/ac9a56fc/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java index 2edeb41..0855ced 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java @@ -355,4 +355,13 @@ public interface Candidate { } throw new IllegalArgumentException("Candidate doesn't have children and no suitable implementation found"); } + + default Set decideMeasurePhrasesToAnswer(Set measurePhraseIndices) throws LensException { +HashSet allCovered = Sets.newHashSet(); +for (Candidate candidate : getChildren()) { + Set covered = candidate.decideMeasurePhrasesToAnswer(measurePhraseIndices); + allCovered.addAll(covered); +} +return allCovered; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/ac9a56fc/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index 47bce65..b5330a7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -830,6 +830,11 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, cand = iter.next(); log.info("Available Candidates:{}, picking up Candidate: {} for querying", candidates, cand); pickedCandidate = cand; +// Answerable common measures in JoinCandidate should be answered by one of the children, otherwise +// measure numbers will be added multiple times in the final union query. +Set measureIndices = getQueriedPhrases().stream().filter(x -> x.hasMeasures(this)) + .map(QueriedPhraseContext::getPosition).collect(toSet()); +pickedCandidate.decideMeasurePhrasesToAnswer(measureIndices); } if (pickedCandidate == null) { throwNoCandidateFactException(); http://git-wip-us.apache.org/repos/asf/lens/blob/ac9a56fc/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java -- diff --git a/lens-cube/src/main/java/org/apache/
lens git commit: LENS-1429 : Queries failing intermittently with NO_REF_COL_AVAILABLE
Repository: lens Updated Branches: refs/heads/current-release-line ac9a56fcd -> cc6c907d7 LENS-1429 : Queries failing intermittently with NO_REF_COL_AVAILABLE Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/cc6c907d Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/cc6c907d Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/cc6c907d Branch: refs/heads/current-release-line Commit: cc6c907d783c03565063d51b7bfbac79177262ab Parents: ac9a56f Author: Sushil Mohanty Authored: Tue Jun 6 16:35:32 2017 +0530 Committer: sushilmohanty Committed: Tue Jun 6 16:35:32 2017 +0530 -- .../org/apache/lens/cube/parse/CandidateTableResolver.java | 6 -- 1 file changed, 4 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/cc6c907d/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index 93736cd..f530650 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -674,11 +674,13 @@ class CandidateTableResolver implements ContextRewriter { if (colSet == null || colSet.isEmpty()) { return true; } +boolean isEvaluable = false; for (QueriedPhraseContext qur : colSet) { if (sc.isPhraseAnswerable(qur)) { -return true; +isEvaluable = true; +continue; } } -return false; +return isEvaluable; } }
lens git commit: LENS-1430 : Expressions are not getting resolved correctly in union query
Repository: lens Updated Branches: refs/heads/master cb2529672 -> 89b17e640 LENS-1430 : Expressions are not getting resolved correctly in union query Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/89b17e64 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/89b17e64 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/89b17e64 Branch: refs/heads/master Commit: 89b17e640b47ff9e071f8df53c05ae0feaf25181 Parents: cb25296 Author: Sushil Mohanty Authored: Tue Jun 6 16:42:10 2017 +0530 Committer: sushilmohanty Committed: Tue Jun 6 16:42:10 2017 +0530 -- .../lens/cube/parse/ExpressionResolver.java | 36 +++- .../cube/parse/TestUnionAndJoinCandidates.java | 60 +++- .../resources/schema/cubes/base/basecube.xml| 12 .../resources/schema/cubes/base/testcube.xml| 9 +++ .../cubes/derived/union_join_ctx_der1.xml | 2 + .../schema/facts/union_join_ctx_fact1.xml | 2 +- .../schema/facts/union_join_ctx_fact3.xml | 1 + .../schema/facts/union_join_ctx_fact4.xml | 59 +++ 8 files changed, 177 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/89b17e64/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index f38aa54..ea6d5c7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -285,6 +285,7 @@ class ExpressionResolver implements ContextRewriter { @Getter private Map> allExprsQueried = new HashMap>(); private Map> pickedExpressions = new HashMap>(); +private Map nonPickedExpressionsForCandidate = new HashMap(); private final CubeQueryContext cubeql; ExpressionResolverContext(CubeQueryContext cubeql) { @@ -407,6 +408,7 @@ class ExpressionResolver implements ContextRewriter { } pickedExpressions.clear(); + nonPickedExpressionsForCandidate.clear(); return exprDims; } @@ -426,7 +428,7 @@ class ExpressionResolver implements ContextRewriter { // Having ast is not copied now, it's maintained in cubeQueryContext, each fact processes that serially. if (queryAST.getHavingAST() != null) { replaceAST(cubeql, queryAST.getHavingAST()); - } else if (cubeql.getHavingAST() != null) { + } else if (cubeql.getHavingAST() != null && nonPickedExpressionsForCandidate.isEmpty()) { replaceAST(cubeql, cubeql.getHavingAST()); queryAST.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST())); } @@ -459,6 +461,9 @@ class ExpressionResolver implements ContextRewriter { if (expr != null) { node1.setChild(i, replaceAlias(expr.getRewrittenAST(), cubeql)); } +} else if (nonPickedExpressionsForCandidate.containsKey(column)) { + node1.setChild(i, nonPickedExpressionsForCandidate.get(column)); + } } } @@ -477,7 +482,7 @@ class ExpressionResolver implements ContextRewriter { return null; } -private void pickExpressionsForTable(CandidateTable cTable) { +private void pickExpressionsForTable(CandidateTable cTable) throws LensException { for (Map.Entry> ecEntry : allExprsQueried.entrySet()) { Set ecSet = ecEntry.getValue(); for (ExpressionContext ec : ecSet) { @@ -488,6 +493,8 @@ class ExpressionResolver implements ContextRewriter { // pick first evaluable expression pickedExpressions.computeIfAbsent(ecEntry.getKey(), k -> new HashSet<>()) .add(new PickedExpression(ec.srcAlias, ec.evaluableExpressions.get(cTable).iterator().next())); + } else { +nonPickedExpressionsForCandidate.put(ecEntry.getKey(), getDefaultExpr(getExprAst(ec))); } } } @@ -495,6 +502,31 @@ class ExpressionResolver implements ContextRewriter { } } +private ASTNode getExprAst(ExpressionContext ec) { + Set scSet = CandidateUtil.getStorageCandidates(cubeql.getCandidates()); + Set storageTableNames = new HashSet(); + for (StorageCandidate sc : scSet) { +storageTableNames.add(sc.getStorageTable()); + } + for (CandidateTable table : ec.evaluableExpressions.keySet()) { +if (storageTableNames.contains(table.getStorageTable())) { + return MetastoreUtil.copyAST(ec.evaluableExpressio
lens git commit: LENS-1430 : Expressions are not getting resolved correctly in union query
Repository: lens Updated Branches: refs/heads/current-release-line cc6c907d7 -> be75e1d55 LENS-1430 : Expressions are not getting resolved correctly in union query Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/be75e1d5 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/be75e1d5 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/be75e1d5 Branch: refs/heads/current-release-line Commit: be75e1d5571f33ca1ec51f91a5d113abe0838842 Parents: cc6c907 Author: Sushil Mohanty Authored: Tue Jun 6 16:43:17 2017 +0530 Committer: sushilmohanty Committed: Tue Jun 6 16:43:17 2017 +0530 -- .../lens/cube/parse/ExpressionResolver.java | 36 +++- .../cube/parse/TestUnionAndJoinCandidates.java | 60 +++- .../resources/schema/cubes/base/basecube.xml| 12 .../resources/schema/cubes/base/testcube.xml| 9 +++ .../cubes/derived/union_join_ctx_der1.xml | 2 + .../schema/facts/union_join_ctx_fact1.xml | 2 +- .../schema/facts/union_join_ctx_fact3.xml | 1 + .../schema/facts/union_join_ctx_fact4.xml | 59 +++ 8 files changed, 177 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/be75e1d5/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index 7cad400..a2f4bb6 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -285,6 +285,7 @@ class ExpressionResolver implements ContextRewriter { @Getter private Map> allExprsQueried = new HashMap>(); private Map> pickedExpressions = new HashMap>(); +private Map nonPickedExpressionsForCandidate = new HashMap(); private final CubeQueryContext cubeql; ExpressionResolverContext(CubeQueryContext cubeql) { @@ -407,6 +408,7 @@ class ExpressionResolver implements ContextRewriter { } pickedExpressions.clear(); + nonPickedExpressionsForCandidate.clear(); return exprDims; } @@ -426,7 +428,7 @@ class ExpressionResolver implements ContextRewriter { // Having ast is not copied now, it's maintained in cubeQueryContext, each fact processes that serially. if (queryAST.getHavingAST() != null) { replaceAST(cubeql, queryAST.getHavingAST()); - } else if (cubeql.getHavingAST() != null) { + } else if (cubeql.getHavingAST() != null && nonPickedExpressionsForCandidate.isEmpty()) { replaceAST(cubeql, cubeql.getHavingAST()); queryAST.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST())); } @@ -459,6 +461,9 @@ class ExpressionResolver implements ContextRewriter { if (expr != null) { node1.setChild(i, replaceAlias(expr.getRewrittenAST(), cubeql)); } +} else if (nonPickedExpressionsForCandidate.containsKey(column)) { + node1.setChild(i, nonPickedExpressionsForCandidate.get(column)); + } } } @@ -477,7 +482,7 @@ class ExpressionResolver implements ContextRewriter { return null; } -private void pickExpressionsForTable(CandidateTable cTable) { +private void pickExpressionsForTable(CandidateTable cTable) throws LensException { for (Map.Entry> ecEntry : allExprsQueried.entrySet()) { Set ecSet = ecEntry.getValue(); for (ExpressionContext ec : ecSet) { @@ -488,6 +493,8 @@ class ExpressionResolver implements ContextRewriter { // pick first evaluable expression pickedExpressions.computeIfAbsent(ecEntry.getKey(), k -> new HashSet<>()) .add(new PickedExpression(ec.srcAlias, ec.evaluableExpressions.get(cTable).iterator().next())); + } else { +nonPickedExpressionsForCandidate.put(ecEntry.getKey(), getDefaultExpr(getExprAst(ec))); } } } @@ -495,6 +502,31 @@ class ExpressionResolver implements ContextRewriter { } } +private ASTNode getExprAst(ExpressionContext ec) { + Set scSet = CandidateUtil.getStorageCandidates(cubeql.getCandidates()); + Set storageTableNames = new HashSet(); + for (StorageCandidate sc : scSet) { +storageTableNames.add(sc.getStorageTable()); + } + for (CandidateTable table : ec.evaluableExpressions.keySet()) { +if (storageTableNames.contains(table.getStorageTable())) { + return MetastoreUtil.c
lens git commit: LENS-1437 : Missing unit test cases for virtual fact
Repository: lens Updated Branches: refs/heads/master 70722824b -> 67adf5b84 LENS-1437 : Missing unit test cases for virtual fact Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/67adf5b8 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/67adf5b8 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/67adf5b8 Branch: refs/heads/master Commit: 67adf5b8431cab29d36f8479d8273182cb0e088a Parents: 7072282 Author: Rajitha R Authored: Tue Jun 27 16:12:56 2017 +0530 Committer: sushilmohanty Committed: Tue Jun 27 16:12:56 2017 +0530 -- .../apache/lens/cube/parse/CubeTestSetup.java | 4 +- .../lens/cube/parse/TestCubeRewriter.java | 18 +-- .../lens/cube/parse/TestVirtualFactQueries.java | 123 +++ .../resources/schema/cubes/base/testcube2.xml | 31 + .../resources/schema/cubes/base/virtualcube.xml | 23 +++- .../resources/schema/facts/testfact7_base.xml | 67 ++ .../resources/schema/facts/testfact8_base.xml | 58 + .../resources/schema/facts/testfact9_base.xml | 57 + .../schema/facts/virtual/virtualfact.xml| 2 +- 9 files changed, 361 insertions(+), 22 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/67adf5b8/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java -- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 48869c2..0366e56 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -364,7 +364,7 @@ public class CubeTestSetup { // storageName[0] is hourly // storageName[1] is daily // storageName[2] is monthly - public static Map getWhereForMonthlyDailyAndHourly2months(String... storageTables) { + public static Map getWhereForMonthlyDailyAndHourly2months(String cubeName, String... storageTables) { Map storageTableToWhereClause = new LinkedHashMap(); List hourlyparts = new ArrayList(); List dailyparts = new ArrayList(); @@ -411,7 +411,7 @@ public class CubeTestSetup { tables.append(storageTables[0]); } Collections.sort(parts); -storageTableToWhereClause.put(tables.toString(), StorageUtil.getWherePartClause("dt", TEST_CUBE_NAME, parts)); +storageTableToWhereClause.put(tables.toString(), StorageUtil.getWherePartClause("dt", cubeName, parts)); return storageTableToWhereClause; } http://git-wip-us.apache.org/repos/asf/lens/blob/67adf5b8/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java -- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index d8c7335..1c75e6c 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -145,22 +145,6 @@ public class TestCubeRewriter extends TestQueryRewrite { //assertNotNull(rewrittenQuery.getNonExistingParts()); } - - @Test - public void testVirtualFactCubeSimpleQuery() throws Exception { -Configuration conf = getConf(); -conf.set(DRIVER_SUPPORTED_STORAGES, "C1"); -CubeQueryContext rewrittenQuery = - rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, getConfWithStorages("C1")); -String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select sum(virtualcube.msr2) as `sum(msr2)` FROM ", - null, "AND ( dim1 = 10 )", getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME, -"C1_summary1")); -String hql = rewrittenQuery.toHQL(); -compareQueries(hql, expected); -System.out.println("Non existing parts:" + rewrittenQuery.getNonExistingParts()); - } - - @Test public void testMaxCoveringFact() throws Exception { Configuration conf = getConf(); @@ -954,7 +938,7 @@ public class TestCubeRewriter extends TestQueryRewrite { rewrite("select SUM(msr2) from testCube" + " where " + TWO_MONTHS_RANGE_UPTO_HOURS, getConfWithStorages("C2")); String expected = getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as `sum(msr2)` FROM ", null, null, -getWhereForMonthlyDailyAndHourly2months("C2_testfact")); +getWhereForMonthlyDailyAndHourly2months(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, ex
lens git commit: LENS-1458 : RESULT_SET_PARENT_DIR set at driver level getting overwritten for queued queries after server restart
Repository: lens Updated Branches: refs/heads/master 6c40b24ff -> 0ea84729a LENS-1458 : RESULT_SET_PARENT_DIR set at driver level getting overwritten for queued queries after server restart Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/0ea84729 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/0ea84729 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/0ea84729 Branch: refs/heads/master Commit: 0ea84729ad0a1ff0ee7869f43600bcd38bdf Parents: 6c40b24 Author: Sushil Mohanty Authored: Mon Aug 7 18:38:15 2017 +0530 Committer: sushilmohanty Committed: Mon Aug 7 18:38:15 2017 +0530 -- .../apache/lens/server/api/query/DriverSelectorQueryContext.java | 2 +- .../src/test/java/org/apache/lens/server/TestServerRestart.java| 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/0ea84729/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java -- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java index 41e533e..9cec42a 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java @@ -254,7 +254,7 @@ public class DriverSelectorQueryContext { } public void setDriverConf(LensDriver driver, Configuration conf) { -driverQueryContextMap.get(driver).setDriverSpecificConf(conf); +driverQueryContextMap.get(driver).setDriverSpecificConf(mergeConf(driver, conf)); } public void setDriverCost(LensDriver driver, QueryCost cost) { http://git-wip-us.apache.org/repos/asf/lens/blob/0ea84729/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java -- diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java index 20aa8cb..8b7728e 100644 --- a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java +++ b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java @@ -39,6 +39,7 @@ import org.apache.lens.api.APIResult.Status; import org.apache.lens.api.query.*; import org.apache.lens.api.result.LensAPIResult; import org.apache.lens.driver.hive.TestRemoteHiveDriver; +import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.QueryContext; import org.apache.lens.server.api.query.QueryExecutionService; @@ -245,6 +246,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { assertEquals(lensQueryConf.getProperty(KEY_POST_SELECT), VALUE_POST_SELECT); if (afterRestart) { + assertEquals(driverConf.get(LensConfConstants.RESULT_SET_PARENT_DIR), "target/hive-lens-results"); //This will be unavailable since if was not updated in LensConf by MockDriverQueryHook assertNull(driverConf.get(UNSAVED_KEY_POST_SELECT)); } else {
lens git commit: LENS-1475 : Override equals(Object obj) and hashcode() methods of SegmentationCandidate
Repository: lens Updated Branches: refs/heads/master 24d4f1531 -> dbbd0c446 LENS-1475 : Override equals(Object obj) and hashcode() methods of SegmentationCandidate Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/dbbd0c44 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/dbbd0c44 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/dbbd0c44 Branch: refs/heads/master Commit: dbbd0c44670d5b15022bb93f05f526783f29a4a4 Parents: 24d4f15 Author: Sushil Mohanty Authored: Thu Sep 14 12:42:57 2017 +0530 Committer: sushilmohanty Committed: Thu Sep 14 12:42:57 2017 +0530 -- .../lens/cube/parse/SegmentationCandidate.java | 21 1 file changed, 21 insertions(+) -- http://git-wip-us.apache.org/repos/asf/lens/blob/dbbd0c44/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java index 055d6ef..9378009 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java @@ -404,4 +404,25 @@ public class SegmentationCandidate implements Candidate { return cubeQueryContextMap.entrySet().stream().filter(entry -> entry.getValue().getPickedCandidate() == null) .collect(toMap(Map.Entry::getKey, entry -> entry.getValue().getStoragePruningMsgs())); } + + @Override + public boolean equals(Object obj) { +if (super.equals(obj)) { + return true; +} + +if (obj == null || !(obj instanceof SegmentationCandidate)) { + return false; +} + +SegmentationCandidate segmantationCandidate = (SegmentationCandidate) obj; +return (segmantationCandidate.segmentation.getSegments().equals(this.segmentation.getSegments()) +&& segmantationCandidate.segmentation.getBaseCube().equals(this.segmentation.getBaseCube())); + } + + @Override + public int hashCode() { +return segmentation.hashCode(); + } + }
lens git commit: LENS-1476 : Order By clause is not rewritten correctly when column is wrapped with function
Repository: lens Updated Branches: refs/heads/master dbbd0c446 -> 9a40b6455 LENS-1476 : Order By clause is not rewritten correctly when column is wrapped with function Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/9a40b645 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/9a40b645 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/9a40b645 Branch: refs/heads/master Commit: 9a40b6455b50fe0e0930d1542a1ec9745025ccc7 Parents: dbbd0c4 Author: Sushil Mohanty Authored: Thu Sep 14 12:43:55 2017 +0530 Committer: sushilmohanty Committed: Thu Sep 14 12:43:55 2017 +0530 -- .../main/java/org/apache/lens/cube/parse/UnionQueryWriter.java| 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/9a40b645/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java index 4eb086b..0d512e4 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java @@ -204,7 +204,8 @@ public class UnionQueryWriter extends SimpleHQLContext { ASTNode outerOrderby = new ASTNode(child); ASTNode tokNullsChild = (ASTNode) child.getChild(0); ASTNode outerTokNullsChild = new ASTNode(tokNullsChild); - if (((ASTNode) tokNullsChild.getChild(0)).getToken().getType() == HiveParser.DOT) { + if (((ASTNode) tokNullsChild.getChild(0)).getToken().getType() == HiveParser.DOT || + ((ASTNode) tokNullsChild.getChild(0)).getToken().getType() == HiveParser.TOK_FUNCTION) { outerTokNullsChild.addChild(innerToOuterSelectASTs.get(new HQLParser.HashableASTNode((ASTNode) tokNullsChild))); } else { outerTokNullsChild.addChild(tokNullsChild);
lens git commit: LENS-1477 : Change default measure value to 0 instead of 0.0 in union queries
Repository: lens Updated Branches: refs/heads/master 9a40b6455 -> a7ec47ed3 LENS-1477 : Change default measure value to 0 instead of 0.0 in union queries Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a7ec47ed Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a7ec47ed Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a7ec47ed Branch: refs/heads/master Commit: a7ec47ed33ad6aa906b8c62dd68a62b7b85fab1a Parents: 9a40b64 Author: Sushil Mohanty Authored: Mon Sep 18 16:33:28 2017 +0530 Committer: sushilmohanty Committed: Mon Sep 18 16:33:28 2017 +0530 -- .../lens/cube/parse/ExpressionResolver.java | 2 +- .../lens/cube/parse/UnionQueryWriter.java | 14 +- .../lens/cube/parse/TestBaseCubeQueries.java| 166 +-- .../lens/cube/parse/TestBridgeTableQueries.java | 40 ++--- .../parse/TestCubeSegmentationRewriter.java | 24 +-- .../cube/parse/TestUnionAndJoinCandidates.java | 34 ++-- .../lens/cube/parse/TestVirtualFactQueries.java | 4 +- 7 files changed, 142 insertions(+), 142 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/a7ec47ed/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index 553468f..e0a0d17 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -560,7 +560,7 @@ class ExpressionResolver implements ContextRewriter { private ASTNode getDefaultExpr(ASTNode node) { if (HQLParser.isAggregateAST(node)) { -node.setChild(1, new ASTNode(new CommonToken(HiveParser.Identifier, "0.0"))); +node.setChild(1, new ASTNode(new CommonToken(HiveParser.Identifier, "0"))); } for (int i = 0; i < node.getChildCount(); i++) { ASTNode child = (ASTNode) node.getChild(i); http://git-wip-us.apache.org/repos/asf/lens/blob/a7ec47ed/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java index 0d512e4..9d0d0d2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java @@ -54,7 +54,7 @@ public class UnionQueryWriter extends SimpleHQLContext { private Map storageCandidateToSelectAstMap = new HashMap<>(); private CubeQueryContext cubeql; static final ASTNode DEFAULT_MEASURE_AST; - private static final String DEFAULT_MEASURE = "0.0"; + private static final String DEFAULT_MEASURE = "0"; static { try { DEFAULT_MEASURE_AST = HQLParser.parseExpr(DEFAULT_MEASURE); @@ -204,8 +204,8 @@ public class UnionQueryWriter extends SimpleHQLContext { ASTNode outerOrderby = new ASTNode(child); ASTNode tokNullsChild = (ASTNode) child.getChild(0); ASTNode outerTokNullsChild = new ASTNode(tokNullsChild); - if (((ASTNode) tokNullsChild.getChild(0)).getToken().getType() == HiveParser.DOT || - ((ASTNode) tokNullsChild.getChild(0)).getToken().getType() == HiveParser.TOK_FUNCTION) { + if (((ASTNode) tokNullsChild.getChild(0)).getToken().getType() == HiveParser.DOT + || ((ASTNode) tokNullsChild.getChild(0)).getToken().getType() == HiveParser.TOK_FUNCTION) { outerTokNullsChild.addChild(innerToOuterSelectASTs.get(new HQLParser.HashableASTNode((ASTNode) tokNullsChild))); } else { outerTokNullsChild.addChild(tokNullsChild); @@ -217,7 +217,7 @@ public class UnionQueryWriter extends SimpleHQLContext { } /** - * Get the select expression. In case of node is default retunrs "0.0" with alias + * Get the select expression. In case of node is default retunrs "0" with alias * otherwise the select phrase with alias. * * @param nodeWithoutAlias @@ -247,7 +247,7 @@ public class UnionQueryWriter extends SimpleHQLContext { /** * Get the aggregate node for the SelectPhrase index. A given measure might not be answerable - * for a StorageCanddate. In that case get the non default aggregate node wcich ideally not "0.0", + * for a StorageCanddate. In that case get the non default aggregate node wcich ideally not "0", * from otherStorage candidate. * * @param position @@ -267,
lens git commit: LENS-1478 : Queries with partial data set as false lookahead not working
Repository: lens Updated Branches: refs/heads/master a7ec47ed3 -> a86cb883f LENS-1478 : Queries with partial data set as false lookahead not working Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a86cb883 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a86cb883 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a86cb883 Branch: refs/heads/master Commit: a86cb883ffef8699454a215be9996df30c9988fd Parents: a7ec47e Author: Sushil Mohanty Authored: Tue Sep 19 14:26:22 2017 +0530 Committer: sushilmohanty Committed: Tue Sep 19 14:26:22 2017 +0530 -- .../main/java/org/apache/lens/cube/parse/StorageCandidate.java| 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/a86cb883/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 140b2c1..fd07bf2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -515,7 +515,8 @@ public class StorageCandidate implements Candidate, CandidateTable { partWhereClauseFormat); updatePartitionStorage(innerPart); innerPart.setFound(pPart.isFound()); -if (innerPart.isFound()) { +if (innerPart.isFound() || !failOnPartialData) { + this.participatingUpdatePeriods.add(maxInterval); partitions.add(innerPart); } }
lens git commit: LENS-1461 : Lens services state are not persisted correctly
Repository: lens Updated Branches: refs/heads/master cdac40874 -> 3b657dee4 LENS-1461 : Lens services state are not persisted correctly Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3b657dee Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3b657dee Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3b657dee Branch: refs/heads/master Commit: 3b657dee40ef70d06e6ed589467bd455b77a5066 Parents: cdac408 Author: Ankit Kailaswar Authored: Fri Nov 3 13:25:53 2017 +0530 Committer: sushilmohanty Committed: Fri Nov 3 13:25:53 2017 +0530 -- .../lens/server/session/HiveSessionService.java| 17 ++--- 1 file changed, 14 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/3b657dee/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java -- diff --git a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java index b480d14..a9fd233 100644 --- a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java +++ b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java @@ -420,12 +420,23 @@ public class HiveSessionService extends BaseLensService implements SessionServic @Override public void writeExternal(ObjectOutput out) throws IOException { // Write out all the sessions -out.writeInt(SESSION_MAP.size()); +List sessions = new ArrayList<>(); for (LensSessionHandle sessionHandle : SESSION_MAP.values()) { - LensSessionImpl session = getSession(sessionHandle); + try { +sessions.add(getSession(sessionHandle)); + } catch (ClientErrorException e) { +// warn for invalid/null session and continue. +log.warn("Cannot persist " + (sessionHandle != null ? sessionHandle.getPublicId() : "null ") ++ " session. {}", e); + } +} + +out.writeInt(sessions.size()); +for (LensSessionImpl session : sessions) { session.getLensSessionPersistInfo().writeExternal(out); } -log.info("Session service pesristed " + SESSION_MAP.size() + " sessions"); + +log.info("Session service pesristed " + sessions.size() + " sessions out of " + SESSION_MAP.size()); } /**
lens git commit: LENS-1483 : Implementation for Jdbc driver cost
Repository: lens Updated Branches: refs/heads/master 3b657dee4 -> 231121425 LENS-1483 : Implementation for Jdbc driver cost Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/23112142 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/23112142 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/23112142 Branch: refs/heads/master Commit: 231121425f156bd572fb847fe87847ee093bbb9a Parents: 3b657de Author: Rajitha R Authored: Fri Nov 17 13:55:44 2017 +0530 Committer: sushilmohanty Committed: Fri Nov 17 13:55:44 2017 +0530 -- .../apache/lens/api/query/QueryCostType.java| 2 +- .../FactPartitionBasedQueryCostCalculator.java | 21 -- .../cube/query/cost/StaticCostCalculator.java | 48 + ...stFactPartitionBasedQueryCostCalculator.java | 4 +- .../org/apache/lens/driver/hive/HiveDriver.java | 9 +-- .../src/main/resources/hivedriver-default.xml | 12 .../apache/lens/driver/hive/TestHiveDriver.java | 1 - .../org/apache/lens/driver/jdbc/JDBCDriver.java | 41 +++ .../driver/jdbc/JDBCDriverConfConstants.java| 4 ++ .../src/main/resources/jdbcdriver-default.xml | 25 +++ .../apache/lens/driver/jdbc/TestJdbcDriver.java | 15 ++-- .../lens/server/api/LensConfConstants.java | 22 ++ .../query/cost/FactPartitionBasedQueryCost.java | 12 ++-- .../lens/server/api/query/cost/QueryCost.java | 9 ++- .../api/query/cost/QueryCostCalculator.java | 2 + .../api/query/cost/QueryCostTypeDecider.java| 33 + .../api/query/cost/QueryCostTypeRangeConf.java | 71 ++ .../cost/RangeBasedQueryCostTypeDecider.java| 45 .../server/api/query/cost/StaticQueryCost.java | 76 .../server/api/query/priority/RangeConf.java| 2 +- .../api/query/cost/MockQueryCostCalculator.java | 5 ++ .../cost/TestFactPartitionBasedQueryCost.java | 27 +++ .../query/QueryContextComparatorTest.java | 11 +-- src/site/apt/admin/hivedriver-config.apt| 20 +++--- src/site/apt/admin/jdbcdriver-config.apt| 66 + 25 files changed, 507 insertions(+), 76 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/23112142/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java -- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java index 37eda42..f65578e 100644 --- a/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java +++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java @@ -27,5 +27,5 @@ import javax.xml.bind.annotation.XmlType; @XmlType @XmlEnum public enum QueryCostType { - LOW, MEDIUM, HIGH + VERY_LOW, LOW, MEDIUM, HIGH } http://git-wip-us.apache.org/repos/asf/lens/blob/23112142/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java index 3c157ee..6d845f8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java @@ -18,6 +18,9 @@ */ package org.apache.lens.cube.query.cost; +import static org.apache.lens.server.api.LensConfConstants.DRIVER_COST_TYPE_RANGES; +import static org.apache.lens.server.api.LensConfConstants.DRIVER_QUERY_COST_TYPE_DEFAULT_RANGES; + import java.util.Map; import java.util.Set; @@ -26,9 +29,7 @@ import org.apache.lens.cube.metadata.UpdatePeriod; import org.apache.lens.server.api.driver.LensDriver; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.AbstractQueryContext; -import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost; -import org.apache.lens.server.api.query.cost.QueryCost; -import org.apache.lens.server.api.query.cost.QueryCostCalculator; +import org.apache.lens.server.api.query.cost.*; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; @@ -37,6 +38,8 @@ public class FactPartitionBasedQueryCostCalculator implements QueryCostCalculato public static final String UPDATE_PERIOD_WEIGHT_PREFIX = "update.period.weight."; + protected QueryCostTypeDecider queryCostTypeDecider; + /** * Calculates total cost based on weights of selected tables and their selected partitions * @@ -91,9 +94,19 @@
lens git commit: LENS-1486 : Disable test case testQueryAliveOnSessionClose
Repository: lens Updated Branches: refs/heads/master 231121425 -> 26125ac67 LENS-1486 : Disable test case testQueryAliveOnSessionClose Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/26125ac6 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/26125ac6 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/26125ac6 Branch: refs/heads/master Commit: 26125ac67080477219930945c61f6093a22ec45a Parents: 2311214 Author: Sushil Mohanty Authored: Tue Nov 21 15:19:31 2017 +0530 Committer: sushilmohanty Committed: Tue Nov 21 15:19:31 2017 +0530 -- .../lens/server/query/TestQueryIndependenceFromSessionClose.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/26125ac6/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java -- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java index 068d46c..5d37f41 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java @@ -217,7 +217,7 @@ public class TestQueryIndependenceFromSessionClose extends LensJerseyTest { }; } - @Test(dataProvider = "restartDataProvider") + @Test(dataProvider = "restartDataProvider", enabled = false) public void testQueryAliveOnSessionClose(boolean restartBeforeFinish, boolean restartAfterFinish) throws LensException, InterruptedException { int numSessions = getSessionsOfFoo().size();
lens git commit: LENS-1496 : Look ahead Time Partitions to be made configurable
Repository: lens Updated Branches: refs/heads/master ae23e9c97 -> 53c342bd2 LENS-1496 : Look ahead Time Partitions to be made configurable Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/53c342bd Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/53c342bd Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/53c342bd Branch: refs/heads/master Commit: 53c342bd20dc35f75b80da78ac799c48cc2e04b1 Parents: ae23e9c Author: Rajitha R Authored: Wed Dec 27 13:43:09 2017 +0530 Committer: sushilmohanty Committed: Wed Dec 27 13:43:09 2017 +0530 -- .../lens/cube/parse/CubeQueryConfUtil.java | 6 +++ .../lens/cube/parse/StorageCandidate.java | 27 +--- .../src/main/resources/olap-query-conf.xml | 8 .../apache/lens/cube/metadata/DateFactory.java | 2 + .../lens/cube/parse/TestCubeRewriter.java | 43 5 files changed, 80 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/53c342bd/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java index eeaa3af..4768fb8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java @@ -48,6 +48,7 @@ public final class CubeQueryConfUtil { public static final String PROCESS_TIME_PART_COL = "lens.cube.query.process.time" + ".partition.column"; public static final String COMPLETENESS_CHECK_PART_COL = "lens.cube.query.completeness.check.partition.column"; public static final String LOOK_AHEAD_PT_PARTS_PFX = "lens.cube.query.lookahead.ptparts.forinterval."; + public static final String LOOK_AHEAD_TIME_PARTS_PFX = "lens.cube.query.lookahead.timeparts.forinterval."; public static final String ENABLE_GROUP_BY_TO_SELECT = "lens.cube.query.promote.groupby.toselect"; public static final String ENABLE_SELECT_TO_GROUPBY = "lens.cube.query.promote.select.togroupby"; public static final String ENABLE_ATTRFIELDS_ADD_DISTINCT = "lens.cube.query.enable.attrfields.add.distinct"; @@ -58,6 +59,7 @@ public final class CubeQueryConfUtil { public static final String REPLACE_TIMEDIM_WITH_PART_COL = "lens.cube.query.replace.timedim"; public static final boolean DEFAULT_MULTI_TABLE_SELECT = true; public static final int DEFAULT_LOOK_AHEAD_PT_PARTS = 1; + public static final int DEFAULT_LOOK_AHEAD_TIME_PARTS = 1; public static final boolean DEFAULT_ENABLE_GROUP_BY_TO_SELECT = false; public static final boolean DEFAULT_ENABLE_SELECT_TO_GROUPBY = false; public static final boolean DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL = true; @@ -66,6 +68,10 @@ public final class CubeQueryConfUtil { return LOOK_AHEAD_PT_PARTS_PFX + interval.name().toLowerCase(); } + public static String getLookAheadTimePartsKey(UpdatePeriod interval) { +return LOOK_AHEAD_TIME_PARTS_PFX + interval.name().toLowerCase(); + } + private static String getValidKeyCubePFX(String cubeName) { return VALID_PFX + cubeName.toLowerCase(); } http://git-wip-us.apache.org/repos/asf/lens/blob/53c342bd/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 3ee66ee..b48be7f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -456,6 +456,10 @@ public class StorageCandidate implements Candidate, CandidateTable { int lookAheadNumParts = getConf() .getInt(CubeQueryConfUtil.getLookAheadPTPartsKey(maxInterval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_PT_PARTS); + +int lookAheadNumTimeParts = getConf() + .getInt(CubeQueryConfUtil.getLookAheadTimePartsKey(maxInterval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_TIME_PARTS); + TimeRange.Iterable.Iterator iter = TimeRange.iterable(ceilFromDate, floorToDate, maxInterval, 1).iterator(); // add partitions from ceilFrom to floorTo while (iter.hasNext()) { @@ -475,7 +479,7 @@ public class StorageCandidate implements Candidate, CandidateTable { log.debug("part column is process time col"); } else if (updatePeriods.first().equals(maxInterval)) { log.debug
lens git commit: First Commit, added SushilMohanty to developers list
Repository: lens Updated Branches: refs/heads/master e5691d8d6 -> f41c176b4 First Commit, added SushilMohanty to developers list Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/f41c176b Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/f41c176b Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/f41c176b Branch: refs/heads/master Commit: f41c176b473ac014a0c3b24f16e82444a445f28b Parents: e5691d8 Author: Sushil Mohanty Authored: Tue Nov 24 12:42:25 2015 +0530 Committer: Sushil Mohanty Committed: Tue Nov 24 12:42:25 2015 +0530 -- pom.xml | 9 + 1 file changed, 9 insertions(+) -- http://git-wip-us.apache.org/repos/asf/lens/blob/f41c176b/pom.xml -- diff --git a/pom.xml b/pom.xml index 9b417d9..b19857f 100644 --- a/pom.xml +++ b/pom.xml @@ -342,6 +342,15 @@ Committer + + sushilmohanty + sushilmoha...@apache.org + SushilMohanty + +5.5 + +Committer + + scm:git:https://git-wip-us.apache.org/repos/asf/lens.git
lens git commit: LENS-903 : No candidate dim available exception should contain only brief error
Repository: lens Updated Branches: refs/heads/master 4d3d2f82f -> b84cb2cd3 LENS-903 : No candidate dim available exception should contain only brief error Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b84cb2cd Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b84cb2cd Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b84cb2cd Branch: refs/heads/master Commit: b84cb2cd32a53806278bbe5d45028dbaa760bdf5 Parents: 4d3d2f8 Author: Sushil Mohanty Authored: Thu Dec 17 17:31:04 2015 +0530 Committer: Sushil Mohanty Committed: Thu Dec 17 17:31:04 2015 +0530 -- .../lens/cube/parse/CubeQueryContext.java | 7 --- .../lens/cube/parse/TestCubeRewriter.java | 22 +++- 2 files changed, 25 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/b84cb2cd/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index f75a6b9..4034a54 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.util.*; import org.apache.lens.cube.error.LensCubeErrorCode; +import org.apache.lens.cube.error.NoCandidateDimAvailableException; import org.apache.lens.cube.error.NoCandidateFactAvailableException; import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; @@ -810,12 +811,12 @@ public class CubeQueryContext implements TrackQueriedColumns { } } } - throw new LensException(LensCubeErrorCode.NO_CANDIDATE_DIM_AVAILABLE.getLensErrorInfo(), - dim.getName(), reason); + log.error("Query rewrite failed due to NO_CANDIDATE_DIM_AVAILABLE, Cause {}", + dimPruningMsgs.get(dim).toJsonObject()); + throw new NoCandidateDimAvailableException(dimPruningMsgs.get(dim)); } } } - return dimsToQuery; } http://git-wip-us.apache.org/repos/asf/lens/blob/b84cb2cd/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java -- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 802ff42..9a08735 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -32,6 +32,7 @@ import java.text.SimpleDateFormat; import java.util.*; import org.apache.lens.cube.error.LensCubeErrorCode; +import org.apache.lens.cube.error.NoCandidateDimAvailableException; import org.apache.lens.cube.error.NoCandidateFactAvailableException; import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCause; @@ -46,6 +47,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; @@ -1078,6 +1080,23 @@ public class TestCubeRewriter extends TestQueryRewrite { } @Test + public void testNoCandidateDimAvailableExceptionCompare() throws Exception { + +//Max cause COLUMN_NOT_FOUND, Ordinal 9 +PruneCauses pr1 = new PruneCauses(); +pr1.addPruningMsg(new CubeDimensionTable(new Table("test", "citydim")), +CandidateTablePruneCause.columnNotFound("test1", "test2", "test3")); +NoCandidateDimAvailableException ne1 = new NoCandidateDimAvailableException(pr1); + +//Max cause EXPRESSION_NOT_EVALUABLE, Ordinal 6 +PruneCauses pr2 = new PruneCauses(); +pr2.addPruningMsg(new CubeDimensionTable(new Table("test", "citydim")), +CandidateTablePruneCause.expressionNotEvaluable("testexp1", "testexp2")); +NoCandidateDimAvailableException ne2 = new NoCandidateDimAvailableException(pr2); +assertEquals(ne1.compareTo(ne2), 3); + } + + @Test public void testDimensionQueryWithMultipleStorages() throws Exception { String hqlQuery = rewrite("select name, stateid from" + " citydim", getConf()); String expected = @@ -1095,7 +1114,8 @@ public class TestCubeRewriter extends TestQueryR
lens git commit: LENS-903 : No candidate dim available exception should contain only brief error - added missing file
Repository: lens Updated Branches: refs/heads/master b84cb2cd3 -> d6aeecc73 LENS-903 : No candidate dim available exception should contain only brief error - added missing file Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d6aeecc7 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d6aeecc7 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d6aeecc7 Branch: refs/heads/master Commit: d6aeecc7306d9cd37dacd53caa0dfd842ca48bdc Parents: b84cb2c Author: Sushil Mohanty Authored: Thu Dec 17 18:22:26 2015 +0530 Committer: Sushil Mohanty Committed: Thu Dec 17 18:22:26 2015 +0530 -- .../error/NoCandidateDimAvailableException.java | 47 1 file changed, 47 insertions(+) -- http://git-wip-us.apache.org/repos/asf/lens/blob/d6aeecc7/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateDimAvailableException.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateDimAvailableException.java b/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateDimAvailableException.java new file mode 100644 index 000..ef76dc6 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateDimAvailableException.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.error; + +import org.apache.lens.cube.metadata.CubeDimensionTable; +import org.apache.lens.cube.parse.PruneCauses; +import org.apache.lens.server.api.error.LensException; + +public class NoCandidateDimAvailableException extends LensException { + + private final PruneCauses briefAndDetailedError; + + public NoCandidateDimAvailableException(PruneCauses briefAndDetailedError) { +super(LensCubeErrorCode.NO_CANDIDATE_DIM_AVAILABLE.getLensErrorInfo(), briefAndDetailedError.getBriefCause()); +this.briefAndDetailedError = briefAndDetailedError; + } + + public PruneCauses.BriefAndDetailedError getJsonMessage() { +return briefAndDetailedError.toJsonObject(); + } + + @Override + public int compareTo(LensException e) { +//Compare the max CandidateTablePruneCode coming from different instances. +if (e instanceof NoCandidateDimAvailableException) { + return briefAndDetailedError.getMaxCause().compareTo( + ((NoCandidateDimAvailableException) e).briefAndDetailedError.getMaxCause()); +} +return super.compareTo(e); + } +}
lens git commit: LENS-905 : Group by is not promoted when keys projected along with having clause
Repository: lens Updated Branches: refs/heads/master d6aeecc73 -> c17908134 LENS-905 : Group by is not promoted when keys projected along with having clause Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c1790813 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c1790813 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c1790813 Branch: refs/heads/master Commit: c17908134e004c646724d7e464252ec76a43113d Parents: d6aeecc Author: Sushil Mohanty Authored: Fri Dec 18 11:50:33 2015 +0530 Committer: Sushil Mohanty Committed: Fri Dec 18 11:50:33 2015 +0530 -- .../org/apache/lens/cube/parse/AggregateResolver.java| 4 +++- .../apache/lens/cube/parse/TestAggregateResolver.java| 11 +-- 2 files changed, 12 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/c1790813/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java -- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java index b544a67..39bd1cc 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java @@ -97,7 +97,9 @@ class AggregateResolver implements ContextRewriter { Configuration distConf = cubeql.getConf(); boolean isDimOnlyDistinctEnabled = distConf.getBoolean(CubeQueryConfUtil.ENABLE_ATTRFIELDS_ADD_DISTINCT, CubeQueryConfUtil.DEFAULT_ATTR_FIELDS_ADD_DISTINCT); -if (isDimOnlyDistinctEnabled) { +//Having clause will always work with measures, if only keys projected +//query should skip distinct and promote group by. +if (cubeql.getHavingAST() == null && isDimOnlyDistinctEnabled) { // Check if any measure/aggregate columns and distinct clause used in // select tree. If not, update selectAST token "SELECT" to "SELECT DISTINCT" if (!hasMeasures(cubeql, cubeql.getSelectAST()) && !isDistinctClauseUsed(cubeql.getSelectAST()) http://git-wip-us.apache.org/repos/asf/lens/blob/c1790813/lens-cube/src/test/java/org/apache/lens/cube/parse/TestAggregateResolver.java -- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestAggregateResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestAggregateResolver.java index a48d753..35234a1 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestAggregateResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestAggregateResolver.java @@ -93,6 +93,9 @@ public class TestAggregateResolver extends TestQueryRewrite { String q10 = "SELECT cityid, round(testCube.msr2) from testCube where " + TWO_DAYS_RANGE; +//dimension selected with having +String q11 = "SELECT cityid from testCube where " + TWO_DAYS_RANGE + " having (testCube.msr2 > 100)"; + String expectedq1 = getExpectedQuery(cubeName, "SELECT testcube.cityid," + " sum(testCube.msr2) from ", null, "group by testcube.cityid", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); @@ -128,13 +131,17 @@ public class TestAggregateResolver extends TestQueryRewrite { String expectedq10 = getExpectedQuery(cubeName, "SELECT testcube.cityid," + " round(sum(testCube.msr2)) from ", null, "group by testcube.cityid", getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); - +String expectedq11 = + getExpectedQuery(cubeName, "SELECT testcube.cityid from ", null, +"group by testcube.cityid" + "having sum(testCube.msr2) > 100", + getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); String[] tests = { - q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, + q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11, }; String[] expected = { expectedq1, expectedq2, expectedq3, expectedq4, expectedq5, expectedq6, expectedq7, expectedq8, expectedq9, expectedq10, + expectedq11, }; for (int i = 0; i < tests.length; i++) {
lens git commit: LENS-908 : Add error codes for jdbc query execution exception
Repository: lens Updated Branches: refs/heads/master 4d7c8e4db -> fdf04be04 LENS-908 : Add error codes for jdbc query execution exception Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/fdf04be0 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/fdf04be0 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/fdf04be0 Branch: refs/heads/master Commit: fdf04be044e727e9d86708b7b0e24b0aec9ed6a3 Parents: 4d7c8e4 Author: Sushil Mohanty Authored: Wed Dec 30 18:30:46 2015 +0530 Committer: Sushil Mohanty Committed: Wed Dec 30 18:30:46 2015 +0530 -- lens-api/src/main/resources/lens-errors.conf| 6 ++-- .../org/apache/lens/driver/hive/HiveDriver.java | 4 +-- .../lens/driver/hive/LensHiveErrorCode.java | 36 .../apache/lens/driver/hive/TestHiveDriver.java | 4 +-- .../org/apache/lens/driver/jdbc/JDBCDriver.java | 25 +- .../apache/lens/driver/jdbc/TestJdbcDriver.java | 14 .../server/api/error/LensDriverErrorCode.java | 36 .../server/query/QueryAPIErrorResponseTest.java | 24 ++--- .../lens/server/query/TestQueryService.java | 4 +-- 9 files changed, 95 insertions(+), 58 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/fdf04be0/lens-api/src/main/resources/lens-errors.conf -- diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf index 06960a0..c7ccea1 100644 --- a/lens-api/src/main/resources/lens-errors.conf +++ b/lens-api/src/main/resources/lens-errors.conf @@ -317,7 +317,7 @@ lensCubeErrorsForMetastore = [ ] -lensHiveErrors = [ +lensDriverErrors = [ { errorCode = 4001 httpStatusCode = ${BAD_REQUEST} @@ -327,7 +327,7 @@ lensHiveErrors = [ { errorCode = 4002 httpStatusCode = ${INTERNAL_SERVER_ERROR} -errorMsg = "Hive Error : %s" +errorMsg = "Driver Execution Error : %s" } ] @@ -364,4 +364,4 @@ lensCubeErrors = ${lensCubeErrorsForQuery}${lensCubeErrorsForMetastore} # Lens server and Lens client are only aware of errors array. They are not aware of any other array defined in # error configuration files. Hence an errors array is prepared which is a concatenation of all other error arrays. -errors = ${lensCommonErrors}${lensServerErrors}${lensCubeErrors}${lensHiveErrors} +errors = ${lensCommonErrors}${lensServerErrors}${lensCubeErrors}${lensDriverErrors} http://git-wip-us.apache.org/repos/asf/lens/blob/fdf04be0/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java -- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java index c7ef8f1..149c6ab 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java @@ -18,7 +18,7 @@ */ package org.apache.lens.driver.hive; -import static org.apache.lens.driver.hive.LensHiveErrorCode.*; +import static org.apache.lens.server.api.error.LensDriverErrorCode.*; import static org.apache.lens.server.api.util.LensUtil.getImplementations; import java.io.ByteArrayInputStream; @@ -579,7 +579,7 @@ public class HiveDriver extends AbstractLensDriver { if (ex.getMessage().contains("SemanticException")) { throw new LensException(SEMANTIC_ERROR.getLensErrorInfo(), ex, ex.getMessage()); } -throw new LensException(HIVE_ERROR.getLensErrorInfo(), ex, ex.getMessage()); +throw new LensException(DRIVER_ERROR.getLensErrorInfo(), ex, ex.getMessage()); } /* http://git-wip-us.apache.org/repos/asf/lens/blob/fdf04be0/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/LensHiveErrorCode.java -- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/LensHiveErrorCode.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/LensHiveErrorCode.java deleted file mode 100644 index 3bac9e7..000 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/LensHiveErrorCode.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses