lens git commit: LENS-903 : No candidate dim available exception should contain only brief error
Repository: lens
Updated Branches:
refs/heads/master 0b3203d04 -> 0a60ef331
LENS-903 : No candidate dim available exception should contain only brief error
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/0a60ef33
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/0a60ef33
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/0a60ef33
Branch: refs/heads/master
Commit: 0a60ef331fa792e96f770e86df91bab36c70027d
Parents: 0b3203d
Author: Sushil Mohanty
Authored: Wed Feb 10 12:22:32 2016 +0530
Committer: Sushil Mohanty
Committed: Wed Feb 10 12:22:32 2016 +0530
--
lens-api/src/main/resources/lens-errors.conf | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/0a60ef33/lens-api/src/main/resources/lens-errors.conf
--
diff --git a/lens-api/src/main/resources/lens-errors.conf
b/lens-api/src/main/resources/lens-errors.conf
index 9087fcd..09b221f 100644
--- a/lens-api/src/main/resources/lens-errors.conf
+++ b/lens-api/src/main/resources/lens-errors.conf
@@ -262,7 +262,7 @@ lensCubeErrorsForQuery = [
{
errorCode = 3027
httpStatusCode = ${BAD_REQUEST}
-errorMsg = "No candidate dimension table available for %s to answer the
query, because %s"
+errorMsg = "No candidate dimension table available to answer the query,
because %s"
}
{
lens git commit: LENS-903 : No candidate dim available exception should contain only brief error
Repository: lens
Updated Branches:
refs/heads/current-release-line 2f1ddd463 -> 991ceb312
LENS-903 : No candidate dim available exception should contain only brief error
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/991ceb31
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/991ceb31
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/991ceb31
Branch: refs/heads/current-release-line
Commit: 991ceb31262177e88b6dc6a89f6c498d5893a734
Parents: 2f1ddd4
Author: Sushil Mohanty
Authored: Wed Feb 10 12:22:32 2016 +0530
Committer: Sushil Mohanty
Committed: Thu Feb 11 15:37:40 2016 +0530
--
lens-api/src/main/resources/lens-errors.conf | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/991ceb31/lens-api/src/main/resources/lens-errors.conf
--
diff --git a/lens-api/src/main/resources/lens-errors.conf
b/lens-api/src/main/resources/lens-errors.conf
index 9087fcd..09b221f 100644
--- a/lens-api/src/main/resources/lens-errors.conf
+++ b/lens-api/src/main/resources/lens-errors.conf
@@ -262,7 +262,7 @@ lensCubeErrorsForQuery = [
{
errorCode = 3027
httpStatusCode = ${BAD_REQUEST}
-errorMsg = "No candidate dimension table available for %s to answer the
query, because %s"
+errorMsg = "No candidate dimension table available to answer the query,
because %s"
}
{
lens git commit: LENS-932 : Skip partition registration for very old and future dates for a fact tables
Repository: lens
Updated Branches:
refs/heads/master 6be4d48fd -> d81211723
LENS-932 : Skip partition registration for very old and future dates for a fact
tables
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d8121172
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d8121172
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d8121172
Branch: refs/heads/master
Commit: d812117238962c4d6c11103c221d808034f631e9
Parents: 6be4d48
Author: Sushil Mohanty
Authored: Mon Feb 22 16:18:47 2016 +0530
Committer: Sushil Mohanty
Committed: Mon Feb 22 16:18:47 2016 +0530
--
.../lens/cube/metadata/CubeMetastoreClient.java | 107 +++---
.../cube/metadata/TestCubeMetastoreClient.java | 145 ++-
.../server/metastore/TestMetastoreService.java | 58 +++-
3 files changed, 283 insertions(+), 27 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/d8121172/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 8969d1f..02d 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -863,18 +863,36 @@ public class CubeMetastoreClient {
latestLookupCache.add(storageTableName);
return partsAdded;
} else {
+ List partsAdded = new ArrayList<>();
// first update in memory, then add to hive table's partitions. delete
is reverse.
partitionTimelineCache.updateForAddition(factOrDimTable, storageName,
updatePeriod,
-getTimePartSpecs(storagePartitionDescs));
+ getTimePartSpecs(storagePartitionDescs,
getStorageTableStartDate(storageTableName, factOrDimTable),
+ getStorageTableEndDate(storageTableName,
factOrDimTable)));
// Adding partition in fact table.
- List partsAdded =
-getStorage(storageName).addPartitions(getClient(), factOrDimTable,
updatePeriod, storagePartitionDescs, null);
+ if (storagePartitionDescs.size() > 0) {
+partsAdded = getStorage(storageName).addPartitions(getClient(),
factOrDimTable, updatePeriod,
+storagePartitionDescs, null);
+ }
// update hive table
alterTablePartitionCache(getStorageTableName(factOrDimTable,
Storage.getPrefix(storageName)));
return partsAdded;
}
}
+ private Date getStorageTableStartDate(String storageTable, String
factTableName)
+throws HiveException, LensException {
+List startDates = getStorageTimes(storageTable,
MetastoreUtil.getStoragetableStartTimesKey());
+startDates.add(getFactTable(factTableName).getStartTime());
+return Collections.max(startDates);
+ }
+
+ private Date getStorageTableEndDate(String storageTable, String
factTableName)
+throws HiveException, LensException {
+List endDates = getStorageTimes(storageTable,
MetastoreUtil.getStoragetableEndTimesKey());
+endDates.add(getFactTable(factTableName).getEndTime());
+return Collections.min(endDates);
+ }
+
private Map>
getTimePartSpecs(List storagePartitionDescs) {
Map> timeSpecs = Maps.newHashMap();
for (StoragePartitionDesc storagePartitionDesc : storagePartitionDescs) {
@@ -888,6 +906,41 @@ public class CubeMetastoreClient {
return timeSpecs;
}
+
+ private Map>
getTimePartSpecs(List storagePartitionDescs,
+ Date storageStartDate,
Date storageEndDate) throws LensException {
+Date now = new Date();
+Map> skippedParts = Maps.newHashMap();
+Map> timeSpecs = Maps.newHashMap();
+Iterator itr = storagePartitionDescs.iterator();
+while (itr.hasNext()) {
+ StoragePartitionDesc storageDesc = itr.next();
+ for (Map.Entry entry :
storageDesc.getTimePartSpec().entrySet()) {
+if (!timeSpecs.containsKey(entry.getKey())) {
+ timeSpecs.put(entry.getKey(), Sets.newTreeSet());
+}
+// check whether partition falls between storage table start_time and
+// end_time or d+2, in such case partition is eligible for
registration.
+if ((entry.getValue().compareTo(storageStartDate) >= 0 &&
entry.getValue().compareTo(storageEndDate) < 0)
+&&
entry.getValue().compareTo(DateUtil.resolveRelativeDate("now +2 days", now)) <
0) {
+ timeSpecs.get(entry.getKey()).add(entry.getValue());
+} else {
+ if (!skippedParts.containsKey(entry.getKey())) {
+skippedParts.put(entry.getKey(),
Sets.newHash
lens git commit: LENS-995 : QueryContextPriorityComparator should compare the priorities instead of cost
Repository: lens
Updated Branches:
refs/heads/master 5faaf11b4 -> b36a02774
LENS-995 : QueryContextPriorityComparator should compare the priorities instead
of cost
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b36a0277
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b36a0277
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b36a0277
Branch: refs/heads/master
Commit: b36a027748413db07099837bbd7c62207b95c896
Parents: 5faaf11
Author: sushilmohanty
Authored: Fri Apr 1 11:33:18 2016 +0530
Committer: sushilmohanty
Committed: Fri Apr 1 11:33:18 2016 +0530
--
.../lens/server/api/LensConfConstants.java | 2 +
.../lens/server/query/FIFOQueryComparator.java | 33 +
.../lens/server/query/QueryComparator.java | 28 +
.../query/QueryContextPriorityComparator.java | 48
.../lens/server/query/QueryCostComparator.java | 39 ++
.../server/query/QueryExecutionServiceImpl.java | 31 -
.../server/query/QueryPriorityComparator.java | 38 ++
.../query/QueryContextComparatorTest.java | 121 +++
.../QueryContextPriorityComparatorTest.java | 99 ---
.../collect/DefaultQueryCollectionTest.java | 34 +-
.../server/query/collect/QueryCollectUtil.java | 25 +++-
11 files changed, 345 insertions(+), 153 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/b36a0277/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 724c868..23537cb 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -91,6 +91,8 @@ public final class LensConfConstants {
public static final String MAX_SESSIONS_PER_USER = SERVER_PFX +
"max.sessions.per.user";
+ public static final String QUERY_COMPARATOR_CLASS = SERVER_PFX +
"query.comparator.class";
+
public static final Integer DEFAULT_MAX_SESSIONS_PER_USER = 10;
/**
http://git-wip-us.apache.org/repos/asf/lens/blob/b36a0277/lens-server/src/main/java/org/apache/lens/server/query/FIFOQueryComparator.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/FIFOQueryComparator.java
b/lens-server/src/main/java/org/apache/lens/server/query/FIFOQueryComparator.java
new file mode 100644
index 000..75c1146
--- /dev/null
+++
b/lens-server/src/main/java/org/apache/lens/server/query/FIFOQueryComparator.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query;
+
+import org.apache.lens.server.api.query.QueryContext;
+
+public class FIFOQueryComparator implements QueryComparator {
+
+ @Override
+ public int compare(QueryContext o1, QueryContext o2) {
+
+Long submitTimeO1 = o1.getSubmissionTime();
+Long submitTimeO2 = o2.getSubmissionTime();
+
+return submitTimeO1.compareTo(submitTimeO2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/b36a0277/lens-server/src/main/java/org/apache/lens/server/query/QueryComparator.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/QueryComparator.java
b/lens-server/src/main/java/org/apache/lens/server/query/QueryComparator.java
new file mode 100644
index 000..67dda6b
--- /dev/null
+++
b/lens-server/src/main/java/org/apache/lens/server/query/QueryComparator.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
lens git commit: LENS-998 : Partition skip is not logging correct start and end time in the logs.
Repository: lens
Updated Branches:
refs/heads/master b36a02774 -> 125a429ad
LENS-998 : Partition skip is not logging correct start and end time in the logs.
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/125a429a
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/125a429a
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/125a429a
Branch: refs/heads/master
Commit: 125a429ad0ff333e1579c015e138777606339455
Parents: b36a027
Author: sushilmohanty
Authored: Fri Apr 1 11:35:33 2016 +0530
Committer: sushilmohanty
Committed: Fri Apr 1 11:35:33 2016 +0530
--
.../org/apache/lens/cube/metadata/CubeMetastoreClient.java | 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/125a429a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index a29f42e..42a8eb2 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -962,9 +962,11 @@ public class CubeMetastoreClient {
}
}
}
-log.info("List of partitions skipped : {}, because they fall before fact
start time : {} "
-+ "and after fact end time : {}",
-skippedParts, storageStartDate, storageEndDate);
+if (!skippedParts.isEmpty()) {
+ log.info("List of partitions skipped : {}, because they fall before fact
start time : {} "
+ + "and after fact end time : {}",
+ skippedParts, storageStartDate, storageEndDate);
+}
return timeSpecs;
}
lens git commit: LENS-39 : Add tags to cube column
Repository: lens
Updated Branches:
refs/heads/master 2cf0b51fd -> 90cbd94f4
LENS-39 : Add tags to cube column
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/90cbd94f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/90cbd94f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/90cbd94f
Branch: refs/heads/master
Commit: 90cbd94f439487b7e6da9fc6e085581037099928
Parents: 2cf0b51
Author: sushilmohanty
Authored: Wed Apr 20 12:49:28 2016 +0530
Committer: sushilmohanty
Committed: Wed Apr 20 12:49:28 2016 +0530
--
lens-api/src/main/resources/cube-0.1.xsd| 10
.../apache/lens/cli/TestLensCubeCommands.java | 3 +-
lens-cli/src/test/resources/sample-cube.xml | 26 +
.../lens/cube/metadata/BaseDimAttribute.java| 14 +++--
.../lens/cube/metadata/ColumnMeasure.java | 15 +++--
.../apache/lens/cube/metadata/CubeColumn.java | 36 +++-
.../lens/cube/metadata/CubeDimAttribute.java| 12 +++-
.../apache/lens/cube/metadata/CubeMeasure.java | 11 +++-
.../apache/lens/cube/metadata/ExprColumn.java | 10 +++-
.../lens/cube/metadata/MetastoreConstants.java | 1 +
.../lens/cube/metadata/MetastoreUtil.java | 4 ++
.../cube/metadata/ReferencedDimAttribute.java | 8 ++-
.../cube/metadata/TestCubeMetastoreClient.java | 60
.../src/main/resources/sample-cube.xml | 28 -
.../examples/ExampleSchemaToStringTest.java | 2 +-
.../src/test/resources/yaml/sales-cube.yaml | 14 -
.../src/test/resources/yaml/sample-cube.yaml| 17 +-
.../apache/lens/server/metastore/JAXBUtils.java | 19 ++-
18 files changed, 261 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-api/src/main/resources/cube-0.1.xsd
--
diff --git a/lens-api/src/main/resources/cube-0.1.xsd
b/lens-api/src/main/resources/cube-0.1.xsd
index f221a6d..4a6a5d7 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -40,6 +40,16 @@
some documentation
+
+
+
+
+Tags can be key value pairs associated with a field so that similar
+fields can be categorized into certain group.
+
+
+
+
http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
--
diff --git
a/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
index 5a353df..f53ed6b 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
@@ -137,7 +137,8 @@ public class TestLensCubeCommands extends
LensCliApplicationTest {
private void testFields(LensCubeCommands command) {
String fields = command.showQueryableFields("sample_cube", true);
for (String field : Arrays
- .asList("dim1", "dim2", "dim3", "dimdetail", "measure1", "measure2",
"measure3", "measure4", "expr_msr5")) {
+ .asList("dim1", "dim2", "dim3", "dimdetail", "dim4", "measure1",
"measure2", "measure3", "measure4",
+ "measure5", "measure6", "expr_msr5")) {
assertTrue(fields.contains(field), fields + " do not contain " + field);
}
assertTrue(fields.contains("measure3 + measure4 + 0.01"));
http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cli/src/test/resources/sample-cube.xml
--
diff --git a/lens-cli/src/test/resources/sample-cube.xml
b/lens-cli/src/test/resources/sample-cube.xml
index 9bcf177..2d6bc34 100644
--- a/lens-cli/src/test/resources/sample-cube.xml
+++ b/lens-cli/src/test/resources/sample-cube.xml
@@ -30,6 +30,18 @@
+
+
+
+
+
+
+
+
+
+
+
+
@@ -40,12 +52,26 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/lens/blob/90cbd94f/lens-cube/src/main/java/org/apache/lens/cube/metadata/BaseDimAttribute.java
--
di
lens git commit: LENS-1149 : Union queries with dim-attribute expressions are not written properly
Repository: lens
Updated Branches:
refs/heads/master 62cafa448 -> a4b4b85d8
LENS-1149 : Union queries with dim-attribute expressions are not written
properly
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a4b4b85d
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a4b4b85d
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a4b4b85d
Branch: refs/heads/master
Commit: a4b4b85d8179f1d7c7f0773f47b7aeebe2a0234d
Parents: 62cafa4
Author: Amareshwari Sriramadasu
Authored: Wed May 25 16:47:03 2016 +0530
Committer: sushilmohanty
Committed: Wed May 25 16:47:03 2016 +0530
--
.../parse/SingleFactMultiStorageHQLContext.java | 169 ---
.../apache/lens/cube/parse/CubeTestSetup.java | 2 +
.../lens/cube/parse/TestUnionQueries.java | 68
3 files changed, 180 insertions(+), 59 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/a4b4b85d/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
index 1af031a..9b48213 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
@@ -66,7 +66,7 @@ public class SingleFactMultiStorageHQLContext extends
UnionHQLContext {
private void processSelectAST() {
ASTNode originalSelectAST = MetastoreUtil.copyAST(ast.getSelectAST());
ast.setSelectAST(new ASTNode(originalSelectAST.getToken()));
-ASTNode outerSelectAST = processExpression(originalSelectAST);
+ASTNode outerSelectAST = processSelectExpression(originalSelectAST);
setSelect(getString(outerSelectAST));
}
@@ -86,7 +86,7 @@ public class SingleFactMultiStorageHQLContext extends
UnionHQLContext {
private void processOrderByAST() {
if (ast.getOrderByAST() != null) {
- setOrderby(getString(processExpression(ast.getOrderByAST(;
+ setOrderby(getString(processOrderbyExpression(ast.getOrderByAST(;
ast.setOrderByAST(null);
}
}
@@ -96,51 +96,117 @@ public class SingleFactMultiStorageHQLContext extends
UnionHQLContext {
ast.setLimitValue(null);
}
+ private ASTNode processExpression(ASTNode astNode) {
+if (astNode == null) {
+ return null;
+}
+ASTNode outerExpression = new ASTNode(astNode);
+// iterate over all children of the ast and get outer ast corresponding to
it.
+for (Node child : astNode.getChildren()) {
+ outerExpression.addChild(getOuterAST((ASTNode)child));
+}
+return outerExpression;
+ }
+
+ private ASTNode processSelectExpression(ASTNode astNode) {
+if (astNode == null) {
+ return null;
+}
+ASTNode outerExpression = new ASTNode(astNode);
+// iterate over all children of the ast and get outer ast corresponding to
it.
+for (Node node : astNode.getChildren()) {
+ ASTNode child = (ASTNode)node;
+ ASTNode outerSelect = new ASTNode(child);
+ ASTNode selectExprAST = (ASTNode)child.getChild(0);
+ ASTNode outerAST = getOuterAST(selectExprAST);
+ outerSelect.addChild(outerAST);
+
+ // has an alias? add it
+ if (child.getChildCount() > 1) {
+outerSelect.addChild(child.getChild(1));
+ }
+ outerExpression.addChild(outerSelect);
+}
+return outerExpression;
+ }
+
+ private ASTNode processOrderbyExpression(ASTNode astNode) {
+if (astNode == null) {
+ return null;
+}
+ASTNode outerExpression = new ASTNode(astNode);
+// sample orderby AST looks the following :
+/*
+TOK_ORDERBY
+ TOK_TABSORTCOLNAMEDESC
+ TOK_NULLS_LAST
+ .
+TOK_TABLE_OR_COL
+ testcube
+cityid
+ TOK_TABSORTCOLNAMEASC
+ TOK_NULLS_FIRST
+ .
+TOK_TABLE_OR_COL
+ testcube
+stateid
+ TOK_TABSORTCOLNAMEASC
+ TOK_NULLS_FIRST
+ .
+TOK_TABLE_OR_COL
+ testcube
+zipcode
+ */
+for (Node node : astNode.getChildren()) {
+ ASTNode child = (ASTNode)node;
+ ASTNode outerOrderby = new ASTNode(child);
+ ASTNode tokNullsChild = (ASTNode) child.getChild(0);
+ ASTNode outerTokNullsChild = new ASTNode(tokNullsChild);
+
outerTokNullsChild.addChild(getOuterAST((ASTNode)tokNullsChild.getChild(0)));
+ outerOrderby.addChild(outerTokNullsChild);
+ outerExpression.addChild(outerOrderby);
+}
+return outerExpression;
+ }
/*
+
Perform a DFS on the provid
lens git commit: LENS-1155 : Seeing INTERNAL_SERVER_ERROR for JDBC queries due to connection unavailability
Repository: lens
Updated Branches:
refs/heads/master a4b4b85d8 -> 5c25e68fe
LENS-1155 : Seeing INTERNAL_SERVER_ERROR for JDBC queries due to connection
unavailability
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/5c25e68f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/5c25e68f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/5c25e68f
Branch: refs/heads/master
Commit: 5c25e68fe71670df6c72d3b8dc08fcbcad6011c4
Parents: a4b4b85
Author: SushilMohanty
Authored: Wed May 25 16:53:22 2016 +0530
Committer: sushilmohanty
Committed: Wed May 25 16:53:22 2016 +0530
--
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 3 +-
.../jdbc/MaxJDBCConnectionCheckConstraint.java | 48
...MaxJDBCConnectionCheckConstraintFactory.java | 38
.../src/main/resources/jdbcdriver-default.xml | 3 +-
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 37 +++
.../apache/lens/server/api/util/LensUtil.java | 2 +-
6 files changed, 128 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
--
diff --git
a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 044a19c..8047302 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -94,6 +94,7 @@ public class JDBCDriver extends AbstractLensDriver {
private ExecutorService asyncQueryPool;
/** The query context map. */
+ @Getter
private ConcurrentHashMap queryContextMap;
/** The conf. */
@@ -458,7 +459,7 @@ public class JDBCDriver extends AbstractLensDriver {
final int maxPoolSize =
parseInt(this.conf.get(JDBC_POOL_MAX_SIZE.getConfigKey()));
final int maxConcurrentQueries
=
parseInt(this.conf.get(MaxConcurrentDriverQueriesConstraintFactory.MAX_CONCURRENT_QUERIES_KEY));
-checkState(maxPoolSize == maxConcurrentQueries, "maxPoolSize:" +
maxPoolSize + " maxConcurrentQueries:"
+checkState(maxPoolSize >= maxConcurrentQueries, "maxPoolSize:" +
maxPoolSize + " maxConcurrentQueries:"
+ maxConcurrentQueries);
queryContextMap = new ConcurrentHashMap<>();
http://git-wip-us.apache.org/repos/asf/lens/blob/5c25e68f/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
--
diff --git
a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
new file mode 100644
index 000..82b5647
--- /dev/null
+++
b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.driver.jdbc;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.query.QueryContext;
+import
org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class MaxJDBCConnectionCheckConstraint implements
QueryLaunchingConstraint {
+
+ private final int poolMaxSize;
+
+ public MaxJDBCConnectionCheckConstraint(final int poolMaxSize) {
+this.poolMaxSize = poolMaxSize;
+ }
+
+ @Override
+ public boolean allowsLaunchOf(final QueryContext candidateQuery,
+EstimatedImmutableQueryCollection
launchedQueries) {
+final LensDriver selectedDriver = candidateQuery.getSelectedDriver
lens git commit: LENS-1289 : Fact column start_time and end_time not getting reflected with update fact command
Repository: lens
Updated Branches:
refs/heads/master c2100fa2d -> 2e56643b8
LENS-1289 : Fact column start_time and end_time not getting reflected with
update fact command
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2e56643b
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2e56643b
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2e56643b
Branch: refs/heads/master
Commit: 2e56643b81cfb871bd133bbf9df55707e973f3ae
Parents: c2100fa
Author: Sushil Mohanty
Authored: Fri Aug 26 13:17:56 2016 +0530
Committer: sushilmohanty
Committed: Fri Aug 26 13:17:56 2016 +0530
--
.../test/java/org/apache/lens/cli/TestLensFactCommands.java | 7 ++-
.../org/apache/lens/cube/metadata/CubeMetastoreClient.java | 7 ++-
.../apache/lens/cube/metadata/TestCubeMetastoreClient.java | 8
.../lens/server/metastore/CubeMetastoreServiceImpl.java | 3 ++-
4 files changed, 18 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/2e56643b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
--
diff --git
a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
index 9670d8f..b4cbf34 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
@@ -162,6 +162,10 @@ public class TestLensFactCommands extends
LensCliApplicationTest {
xmlContent = xmlContent.replace("\n",
"" + "\n\n");
+ xmlContent = xmlContent.replace("",
+ ""
+ + "\n\n");
+
File newFile = new File("target/local-fact1.xml");
Writer writer = new OutputStreamWriter(new FileOutputStream(newFile));
writer.write(xmlContent);
@@ -171,7 +175,7 @@ public class TestLensFactCommands extends
LensCliApplicationTest {
log.debug(desc);
String propString = "fact1.prop: f1";
String propString1 = "fact1.prop1: f2";
-
+ String propStringColStartTime = "cube.fact.col.start.time.measure4:
2015-01-01";
assertTrue(desc.contains(propString));
command.updateFactTable("fact1", new File("target/local-fact1.xml"));
@@ -179,6 +183,7 @@ public class TestLensFactCommands extends
LensCliApplicationTest {
log.debug(desc);
assertTrue(desc.contains(propString), "The sample property value is not
set");
assertTrue(desc.contains(propString1), "The sample property value is not
set");
+ assertTrue(desc.contains(propStringColStartTime), "The sample property
value is not set");
newFile.delete();
http://git-wip-us.apache.org/repos/asf/lens/blob/2e56643b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 0dce6c4..e14c43f 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -2275,8 +2275,13 @@ public class CubeMetastoreClient {
* @throws HiveException
*/
public void alterCubeFactTable(String factTableName, CubeFactTable
cubeFactTable,
-Map storageTableDescs) throws HiveException,
LensException {
+ Map
storageTableDescs,
+ Map props)
+throws HiveException, LensException {
Table factTbl = getTableWithTypeFailFast(factTableName,
CubeTableType.FACT);
+if (!props.isEmpty()) {
+ cubeFactTable.getProperties().putAll(props);
+}
alterCubeTable(factTableName, factTbl, cubeFactTable);
if (storageTableDescs != null) {
// create/alter tables for each storage
http://git-wip-us.apache.org/repos/asf/lens/blob/2e56643b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index 62c7ec9..e21dc2a 100644
---
a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++
b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -1062,7 +106
lens git commit: LENS-1360 : org.apache.lens.driver.jdbc.TestJdbcDriver#testCancelQuery failing in pre-commit
Repository: lens
Updated Branches:
refs/heads/master f274e29d5 -> 547a42f33
LENS-1360 : org.apache.lens.driver.jdbc.TestJdbcDriver#testCancelQuery failing
in pre-commit
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/547a42f3
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/547a42f3
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/547a42f3
Branch: refs/heads/master
Commit: 547a42f33caa7669fe54509e6e4dac108e904a61
Parents: f274e29
Author: Amareshwari Sriramadasu
Authored: Tue Oct 25 14:04:45 2016 +0530
Committer: sushilmohanty
Committed: Tue Oct 25 14:04:45 2016 +0530
--
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 2 +-
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 3 +-
lens-driver-jdbc/src/test/resources/logback.xml | 36
3 files changed, 39 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/547a42f3/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
--
diff --git
a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 82e0231..7ce1a97 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -290,7 +290,7 @@ public class JDBCDriver extends AbstractLensDriver {
}
if (queryContext.isClosed()) {
log.info("Ignored exception on already closed query : {} - {}",
-queryContext.getLensContext().getQueryHandle(),
e.getMessage());
+queryContext.getLensContext().getQueryHandle(),
e.getMessage(), e);
} else {
log.error("Error executing SQL query: {} reason: {}",
queryContext.getLensContext().getQueryHandle(),
e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/lens/blob/547a42f3/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
--
diff --git
a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index a402f91..6291021 100644
---
a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++
b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -834,6 +834,7 @@ public class TestJdbcDriver {
public static int sleep(int t) {
try {
+ log.info("Sleeping for {} seconds", t);
Thread.sleep(t * 1000);
} catch (InterruptedException ie) {
// ignore
@@ -878,7 +879,7 @@ public class TestJdbcDriver {
QueryHandle handle = context.getQueryHandle();
// without wait query may not be launched.
if (waitBeforeCancel) {
- Thread.sleep(100);
+ Thread.sleep(1000);
}
boolean isCancelled = driver.cancelQuery(handle);
driver.updateStatus(context);
http://git-wip-us.apache.org/repos/asf/lens/blob/547a42f3/lens-driver-jdbc/src/test/resources/logback.xml
--
diff --git a/lens-driver-jdbc/src/test/resources/logback.xml
b/lens-driver-jdbc/src/test/resources/logback.xml
new file mode 100644
index 000..eed4f66
--- /dev/null
+++ b/lens-driver-jdbc/src/test/resources/logback.xml
@@ -0,0 +1,36 @@
+
+
+
+
+target/test.log
+
+ %d{dd MMM HH:mm:ss,SSS} [%t] %-5p %c %L - %m%n
+
+
+ DEBUG
+
+
+
+
+
+
+
[3/4] lens git commit: Addressed final reviews and fixed checkstyle error
http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
index ba8a5e4..93dbfc3 100644
---
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
+++
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
@@ -37,7 +37,6 @@ import org.apache.lens.cube.error.LensCubeErrorCode;
import org.apache.lens.cube.error.NoCandidateFactAvailableException;
import org.apache.lens.cube.metadata.TimeRange;
import org.apache.lens.cube.metadata.UpdatePeriod;
-import
org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.time.DateUtils;
@@ -68,15 +67,12 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
}
@Test
- public void testNoCandidateFactAvailableExceptionCompareTo() throws
Exception {
-//maxCause : UNSUPPORTED_STORAGE
-NoCandidateFactAvailableException ne1 =
- (NoCandidateFactAvailableException)getLensExceptionInRewrite(
+ public void testNoUnionCandidateAndNoJoinCandidateErrorWeight() throws
Exception {
+LensException e1 = getLensExceptionInRewrite(
"select dim1, test_time_dim, msr3, msr13 from basecube where " +
TWO_DAYS_RANGE, conf);
-//maxCause : STORAGE_NOT_AVAILABLE_IN_RANGE
-NoCandidateFactAvailableException ne2 = (NoCandidateFactAvailableException)
- getLensExceptionInRewrite("select dim1 from " + cubeName + " where " +
LAST_YEAR_RANGE, getConf());
-assertEquals(ne1.compareTo(ne2), -1);
+LensException e2 = getLensExceptionInRewrite("select dim1 from " + cubeName
++ " where " + LAST_YEAR_RANGE, getConf());
+assertEquals(e1.getErrorWeight() - e2.getErrorWeight(), 1);
}
@Test
@@ -92,53 +88,9 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
e = getLensExceptionInRewrite("select dim1, test_time_dim, msr3, msr13
from basecube where "
+ TWO_DAYS_RANGE, conf);
assertEquals(e.getErrorCode(),
-
LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo().getErrorCode());
-NoCandidateFactAvailableException ne = (NoCandidateFactAvailableException)
e;
-//ne.briefAndDetailedError.getBriefCause()
-//ne.getJsonMessage().brief
-
assertTrue(CandidateTablePruneCode.UNSUPPORTED_STORAGE.errorFormat.equals(ne.getJsonMessage().getBrief()));
-//PruneCauses.BriefAndDetailedError pruneCauses = ne.getJsonMessage();
-//String regexp =
String.format(CandidateTablePruneCode.UNSUPPORTED_STORAGE.errorFormat,
-// "Column Sets: (.*?)", "queriable together");
-//Matcher matcher =
Pattern.compile(regexp).matcher(pruneCauses.getBrief());
-//assertTrue(matcher.matches(), pruneCauses.getBrief());
-//assertEquals(matcher.groupCount(), 1);
-//String columnSetsStr = matcher.group(1);
-//assertNotEquals(columnSetsStr.indexOf("test_time_dim"), -1,
columnSetsStr);
-//assertNotEquals(columnSetsStr.indexOf("msr3, msr13"), -1);
-//
-///**
-// * Verifying the BriefAndDetailedError:
-// * 1. Check for missing columns(COLUMN_NOT_FOUND)
-// *and check the respective tables for each COLUMN_NOT_FOUND
-// * 2. check for ELEMENT_IN_SET_PRUNED
-// *
-// */
-//boolean columnNotFound = false;
-//List testTimeDimFactTables =
Arrays.asList("c1_testfact3_raw_base",
-//"c1_testfact5_base", "c1_testfact6_base", "c1_testfact1_raw_base",
-//"c1_testfact4_raw_base", "c1_testfact3_base");
-//List factTablesForMeasures = Arrays.asList(
-//
"c2_testfact2_base","c2_testfact_deprecated","c1_union_join_ctx_fact1","c1_union_join_ctx_fact2",
-//
"c1_union_join_ctx_fact3","c1_union_join_ctx_fact5","c1_testfact2_base",
-//
"c1_union_join_ctx_fact6","c1_testfact2_raw_base","c1_testfact5_raw_base",
-//
"c3_testfact_deprecated","c1_testfact_deprecated","c4_testfact_deprecated",
-//"c3_testfact2_base","c4_testfact2_base");
-//for (Map.Entry> entry :
pruneCauses.getDetails().entrySet()) {
-// if (entry.getValue().contains(CandidateTablePruneCause.columnNotFound(
-// CandidateTablePruneCode.COLUMN_NOT_FOUND, "test_time_dim"))) {
-//columnNotFound = true;
-//compareStrings(testTimeDimFactTables, entry);
-// }
-// if (entry.getValue().contains(CandidateTablePruneCause.columnNotFound(
-// CandidateTablePruneCode.COLUMN_NOT_FOUND, "msr3", "msr13"))) {
-//columnNotFound = true;
-//compareStrings(factTablesForMeasures, entry);
-// }
-//}
-//Assert.assertTrue(columnNotFound);
- // assertEquals(pruneCauses.getDetails().get("testfact1_base"),
- // Array
[2/4] lens git commit: Addressed final reviews and fixed checkstyle error
http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java
index 9168d10..bed1e7a 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java
@@ -30,14 +30,11 @@ import static org.testng.Assert.*;
import java.util.*;
import java.util.stream.Collectors;
-import org.apache.lens.cube.error.LensCubeErrorCode;
-import org.apache.lens.cube.error.NoCandidateDimAvailableException;
import org.apache.lens.cube.error.NoCandidateFactAvailableException;
import org.apache.lens.server.api.LensServerAPITestUtil;
import org.apache.lens.server.api.error.LensException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.ql.parse.ParseException;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
@@ -53,12 +50,12 @@ public class TestUnionQueries extends TestQueryRewrite {
@BeforeTest
public void setupDriver() throws Exception {
testConf = LensServerAPITestUtil.getConfiguration(
- DRIVER_SUPPORTED_STORAGES, "C0,C1,C2",
- DISABLE_AUTO_JOINS, false,
- ENABLE_SELECT_TO_GROUPBY, true,
- ENABLE_GROUP_BY_TO_SELECT, true,
- DISABLE_AGGREGATE_RESOLVER, false,
- ENABLE_STORAGES_UNION, true);
+DRIVER_SUPPORTED_STORAGES, "C0,C1,C2",
+DISABLE_AUTO_JOINS, false,
+ENABLE_SELECT_TO_GROUPBY, true,
+ENABLE_GROUP_BY_TO_SELECT, true,
+DISABLE_AGGREGATE_RESOLVER, false,
+ENABLE_STORAGES_UNION, true);
}
@Override
@@ -66,334 +63,228 @@ public class TestUnionQueries extends TestQueryRewrite {
return new Configuration(testConf);
}
- //TODOD union: @Test
+ @Test
public void testUnionQueries() throws Exception {
-Configuration conf = getConf();
-conf.set(getValidStorageTablesKey("testfact"), "C1_testFact,C2_testFact");
-conf.set(getValidUpdatePeriodsKey("testfact", "C1"), "DAILY,HOURLY");
-conf.set(getValidUpdatePeriodsKey("testfact2", "C1"), "YEARLY");
-conf.set(getValidUpdatePeriodsKey("testfact", "C2"), "MONTHLY,DAILY");
-conf.setBoolean(CubeQueryConfUtil.ENABLE_STORAGES_UNION, false);
-ArrayList storages = Lists.newArrayList("c1_testfact",
"c2_testfact");
+Configuration conf =
LensServerAPITestUtil.getConfigurationWithParams(getConf(),
+CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C6",
+getValidFactTablesKey("basecube"), "testfact",
+FAIL_QUERY_ON_PARTIAL_DATA, false);
+ArrayList storages = Lists.newArrayList("daily_c6_testfact",
"monthly_c6_testfact");
try {
- getStorageToUpdatePeriodMap().put("c1_testfact",
Lists.newArrayList(HOURLY, DAILY));
- getStorageToUpdatePeriodMap().put("c2_testfact",
Lists.newArrayList(MONTHLY));
-
// Union query
String hqlQuery;
String expected;
StoragePartitionProvider provider = new StoragePartitionProvider() {
@Override
public Map providePartitionsForStorage(String storage)
{
- return getWhereForMonthlyDailyAndHourly2monthsUnionQuery(storage);
+ if (storage.contains("daily_c6_testfact")) {
+return getWhereForDays(storage, ONE_MONTH_BACK_TRUNCATED,
getTruncatedDateWithOffset(MONTHLY, 0));
+ } else if (storage.contains("monthly_c6_testfact")) {
+return getWhereForMonthly(storage, THREE_MONTHS_BACK_TRUNCATED,
ONE_MONTH_BACK_TRUNCATED);
+ }
+ return null;
}
};
- try {
-rewrite("select cityid as `City ID`, msr8, msr7 as `Third measure` "
- + "from testCube where " + TWO_MONTHS_RANGE_UPTO_HOURS, conf);
- // fail("Union feature is disabled, should have failed");
- } catch (LensException e) {
-assertEquals(e.getErrorCode(),
LensCubeErrorCode.STORAGE_UNION_DISABLED.getLensErrorInfo().getErrorCode());
- }
- conf.setBoolean(CubeQueryConfUtil.ENABLE_STORAGES_UNION, true);
-
- hqlQuery = rewrite("select ascii(cityname) as `City Name`, msr8, msr7 as
`Third measure` "
-+ "from testCube where ascii(cityname) = 'c' and cityname = 'a' and
zipcode = 'b' and "
-+ TWO_MONTHS_RANGE_UPTO_HOURS, conf);
+ hqlQuery = rewrite("select cityname1 as `City Name`, msr8, msr7 as
`Third measure` "
+ + "from testCube where cityname1 = 'a' and zipcode = 'b' and "
+ + THREE_MONTHS_RANGE_UPTO_MONTH, conf);
expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider,
-"SELECT testcube.alias0 as `City Name`, sum(testcube.alias1) +
max(testcube.alias2), "
- + "case when sum(testcube.alias1) = 0 then 0 else
sum(testcube.alias3)/sum(testcube.
[1/4] lens git commit: Addressed final reviews and fixed checkstyle error
Repository: lens Updated Branches: refs/heads/lens-1381 72c20ff83 -> cb5e2a77d http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/statetable.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/statetable.xml b/lens-cube/src/test/resources/schema/dimtables/statetable.xml index 2aab131..58eb948 100644 --- a/lens-cube/src/test/resources/schema/dimtables/statetable.xml +++ b/lens-cube/src/test/resources/schema/dimtables/statetable.xml @@ -1,4 +1,24 @@ + @@ -32,5 +52,23 @@ dt + + +HOURLY + + C6 + + + + + + + + + + +dt + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/statetable_partitioned.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/statetable_partitioned.xml b/lens-cube/src/test/resources/schema/dimtables/statetable_partitioned.xml index e7c808f..826d8dc 100644 --- a/lens-cube/src/test/resources/schema/dimtables/statetable_partitioned.xml +++ b/lens-cube/src/test/resources/schema/dimtables/statetable_partitioned.xml @@ -1,4 +1,24 @@ + http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/testdim2tbl.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl.xml b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl.xml index b320ec2..d04cb5c 100644 --- a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl.xml +++ b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl.xml @@ -1,4 +1,24 @@ + http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/testdim2tbl2.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl2.xml b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl2.xml index 2239997..efc0969 100644 --- a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl2.xml +++ b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl2.xml @@ -1,4 +1,24 @@ + http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/testdim2tbl3.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl3.xml b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl3.xml index 62059ad..d9b39c7 100644 --- a/lens-cube/src/test/resources/schema/dimtables/testdim2tbl3.xml +++ b/lens-cube/src/test/resources/schema/dimtables/testdim2tbl3.xml @@ -1,4 +1,24 @@ + http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/testdim3tbl.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim3tbl.xml b/lens-cube/src/test/resources/schema/dimtables/testdim3tbl.xml index 7d2af60..ffd09c0 100644 --- a/lens-cube/src/test/resources/schema/dimtables/testdim3tbl.xml +++ b/lens-cube/src/test/resources/schema/dimtables/testdim3tbl.xml @@ -1,4 +1,24 @@ + http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/testdim4tbl.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/testdim4tbl.xml b/lens-cube/src/test/resources/schema/dimtables/testdim4tbl.xml index 2044851..d81daa7 100644 --- a/lens-cube/src/test/resources/schema/dimtables/testdim4tbl.xml +++ b/lens-cube/src/test/resources/schema/dimtables/testdim4tbl.xml @@ -1,4 +1,24 @@ + http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/unreachabledimtable.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/unreachabledimtable.xml b/lens-cube/src/test/resources/schema/dimtables/unreachabledimtable.xml index 6f80c27..e940cf7 100644 --- a/lens-cube/src/test/resources/schema/dimtables/unreachabledimtable.xml +++ b/lens-cube/src/test/resources/schema/dimtables/unreachabledimtable.xml @@ -1,4 +1,24 @@ + http://git-wip-us.apache.org/repos/asf/lens/blob/cb5e2a77/lens-cube/src/test/resources/schema/dimtables/user_interests_tbl.xml -- diff --git a/lens-cube/src/test/resources/schema/dimtables/user_interests_tbl.xml b/lens-cube/src/test/resources/sc
[4/4] lens git commit: Addressed final reviews and fixed checkstyle error
Addressed final reviews and fixed checkstyle error Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/cb5e2a77 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/cb5e2a77 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/cb5e2a77 Branch: refs/heads/lens-1381 Commit: cb5e2a77dfd09c6736c9b295f68addba6bf27e6e Parents: 72c20ff Author: sushilmohanty Authored: Wed Apr 12 18:10:36 2017 +0530 Committer: sushilmohanty Committed: Wed Apr 12 18:10:36 2017 +0530 -- lens-api/src/main/resources/lens-errors.conf| 13 + .../lens/cube/error/LensCubeErrorCode.java | 2 + .../NoCandidateFactAvailableException.java | 13 +- .../lens/cube/metadata/CubeMetastoreClient.java | 5 +- .../org/apache/lens/cube/metadata/DateUtil.java | 3 +- .../lens/cube/metadata/FactPartition.java | 7 + .../lens/cube/metadata/MetastoreUtil.java | 2 +- .../apache/lens/cube/metadata/TimeRange.java| 4 + .../cube/parse/AbridgedTimeRangeWriter.java | 3 - .../parse/CandidateCoveringSetsResolver.java| 67 ++- .../apache/lens/cube/parse/CandidateDim.java| 6 +- .../apache/lens/cube/parse/CandidateTable.java | 2 +- .../cube/parse/CandidateTablePruneCause.java| 41 +- .../apache/lens/cube/parse/CandidateUtil.java | 15 +- .../lens/cube/parse/CubeQueryContext.java | 83 ++-- .../lens/cube/parse/CubeQueryRewriter.java | 3 +- .../lens/cube/parse/CubeSemanticAnalyzer.java | 3 +- .../cube/parse/DenormalizationResolver.java | 22 +- .../lens/cube/parse/ExpressionResolver.java | 3 +- .../apache/lens/cube/parse/JoinCandidate.java | 2 +- .../cube/parse/MaxCoveringFactResolver.java | 1 - .../org/apache/lens/cube/parse/PruneCauses.java | 3 +- .../lens/cube/parse/StorageCandidate.java | 36 +- .../lens/cube/parse/StorageTableResolver.java | 20 +- .../apache/lens/cube/parse/UnionCandidate.java | 24 +- .../lens/cube/parse/UnionQueryWriter.java | 22 +- .../apache/lens/driver/cube/RewriterPlan.java | 6 +- .../apache/lens/cube/parse/CubeTestSetup.java | 25 +- .../lens/cube/parse/TestAggregateResolver.java | 42 +- .../lens/cube/parse/TestBaseCubeQueries.java| 226 - .../cube/parse/TestBetweenTimeRangeWriter.java | 6 +- .../lens/cube/parse/TestBridgeTableQueries.java | 40 +- .../lens/cube/parse/TestCubeRewriter.java | 11 +- .../cube/parse/TestDenormalizationResolver.java | 10 +- .../lens/cube/parse/TestTimeRangeResolver.java | 36 +- .../cube/parse/TestUnionAndJoinCandidates.java | 24 +- .../lens/cube/parse/TestUnionQueries.java | 464 +++ .../resources/schema/cubes/base/basecube.xml| 20 + .../resources/schema/cubes/base/testcube.xml| 23 + .../resources/schema/cubes/derived/der1.xml | 20 + .../resources/schema/cubes/derived/der2.xml | 20 + .../resources/schema/cubes/derived/der3.xml | 20 + .../schema/cubes/derived/derivedcube.xml| 20 + .../cubes/derived/union_join_ctx_der1.xml | 20 + .../resources/schema/dimensions/citydim.xml | 20 + .../resources/schema/dimensions/countrydim.xml | 20 + .../resources/schema/dimensions/cycledim1.xml | 20 + .../resources/schema/dimensions/cycledim2.xml | 20 + .../test/resources/schema/dimensions/daydim.xml | 20 + .../resources/schema/dimensions/hourdim.xml | 20 + .../test/resources/schema/dimensions/sports.xml | 20 + .../resources/schema/dimensions/statedim.xml| 20 + .../resources/schema/dimensions/testdim2.xml| 20 + .../resources/schema/dimensions/testdim3.xml| 20 + .../resources/schema/dimensions/testdim4.xml| 20 + .../schema/dimensions/unreachabledim.xml| 20 + .../schema/dimensions/user_interests.xml| 20 + .../resources/schema/dimensions/userdim.xml | 20 + .../test/resources/schema/dimensions/zipdim.xml | 20 + .../resources/schema/dimtables/citytable.xml| 37 ++ .../resources/schema/dimtables/citytable2.xml | 20 + .../resources/schema/dimtables/citytable3.xml | 20 + .../resources/schema/dimtables/citytable4.xml | 20 + .../resources/schema/dimtables/countrytable.xml | 20 + .../dimtables/countrytable_partitioned.xml | 20 + .../resources/schema/dimtables/cycledim1tbl.xml | 20 + .../resources/schema/dimtables/cycledim2tbl.xml | 20 + .../resources/schema/dimtables/daydimtbl.xml| 20 + .../resources/schema/dimtables/hourdimtbl.xml | 20 + .../resources/schema/dimtables/sports_tbl.xml | 20 + .../resources/schema/dimtables/statetable.xml | 38 ++ .../schema/dimtables/statetable_partitioned.xml | 20 + .../resources/schema/dimtables/testdim2tbl.xml | 20 + .../resources/schema/dimtables/testdim2tbl2.xml | 20 + .../resources/schema/dimtables/testdim2tbl3.xml | 20 + .../resources/schema/dimtables/testdim3tbl.xml | 20
lens git commit: LENS-1428 : Common measures are getting repeated in UnionQuery for JoinCandidate
Repository: lens
Updated Branches:
refs/heads/master 5de45e0f8 -> 159cf71e7
LENS-1428 : Common measures are getting repeated in UnionQuery for JoinCandidate
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/159cf71e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/159cf71e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/159cf71e
Branch: refs/heads/master
Commit: 159cf71e732a0b63bf8c7efde6489f4eea8d943a
Parents: 5de45e0
Author: Sushil Mohanty
Authored: Tue Jun 6 15:08:38 2017 +0530
Committer: sushilmohanty
Committed: Tue Jun 6 15:08:38 2017 +0530
--
.../apache/lens/cube/metadata/CubeMetastoreClient.java | 2 +-
.../java/org/apache/lens/cube/parse/Candidate.java | 9 +
.../org/apache/lens/cube/parse/CubeQueryContext.java | 5 +
.../java/org/apache/lens/cube/parse/JoinCandidate.java | 13 +
.../org/apache/lens/cube/parse/StorageCandidate.java | 7 +++
.../org/apache/lens/cube/parse/UnionQueryWriter.java | 6 ++
6 files changed, 37 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/159cf71e/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 749e44c..5c05e53 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -443,7 +443,7 @@ public class CubeMetastoreClient {
if (get(timeLineKey) == null) {
loadTimeLines(fact, storage, timeLineKey);
}
-log.info("timeline for {} is: {}", storage, get(timeLineKey));
+log.debug("timeline for {} is: {}", storage, get(timeLineKey));
// return the final value from memory
return get(timeLineKey);
// RESUME CHECKSTYLE CHECK DoubleCheckedLockingCheck
http://git-wip-us.apache.org/repos/asf/lens/blob/159cf71e/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
--
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
index 2edeb41..0855ced 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
@@ -355,4 +355,13 @@ public interface Candidate {
}
throw new IllegalArgumentException("Candidate doesn't have children and no
suitable implementation found");
}
+
+ default Set decideMeasurePhrasesToAnswer(Set
measurePhraseIndices) throws LensException {
+HashSet allCovered = Sets.newHashSet();
+for (Candidate candidate : getChildren()) {
+ Set covered =
candidate.decideMeasurePhrasesToAnswer(measurePhraseIndices);
+ allCovered.addAll(covered);
+}
+return allCovered;
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/159cf71e/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 47bce65..b5330a7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -830,6 +830,11 @@ public class CubeQueryContext extends TracksQueriedColumns
implements QueryAST,
cand = iter.next();
log.info("Available Candidates:{}, picking up Candidate: {} for
querying", candidates, cand);
pickedCandidate = cand;
+// Answerable common measures in JoinCandidate should be answered by
one of the children, otherwise
+// measure numbers will be added multiple times in the final union
query.
+Set measureIndices = getQueriedPhrases().stream().filter(x ->
x.hasMeasures(this))
+ .map(QueriedPhraseContext::getPosition).collect(toSet());
+pickedCandidate.decideMeasurePhrasesToAnswer(measureIndices);
}
if (pickedCandidate == null) {
throwNoCandidateFactException();
http://git-wip-us.apache.org/repos/asf/lens/blob/159cf71e/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.j
lens git commit: LENS-1428 : Common measures are getting repeated in UnionQuery for JoinCandidate
Repository: lens
Updated Branches:
refs/heads/current-release-line 27ad9aa8f -> ac9a56fcd
LENS-1428 : Common measures are getting repeated in UnionQuery for JoinCandidate
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/ac9a56fc
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/ac9a56fc
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/ac9a56fc
Branch: refs/heads/current-release-line
Commit: ac9a56fcd828b0cbb6acee614c72f650a6cb78b8
Parents: 27ad9aa
Author: Sushil Mohanty
Authored: Tue Jun 6 15:10:24 2017 +0530
Committer: sushilmohanty
Committed: Tue Jun 6 15:10:24 2017 +0530
--
.../apache/lens/cube/metadata/CubeMetastoreClient.java | 2 +-
.../java/org/apache/lens/cube/parse/Candidate.java | 9 +
.../org/apache/lens/cube/parse/CubeQueryContext.java | 5 +
.../java/org/apache/lens/cube/parse/JoinCandidate.java | 13 +
.../org/apache/lens/cube/parse/StorageCandidate.java | 7 +++
.../org/apache/lens/cube/parse/UnionQueryWriter.java | 6 ++
6 files changed, 37 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/ac9a56fc/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 749e44c..5c05e53 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -443,7 +443,7 @@ public class CubeMetastoreClient {
if (get(timeLineKey) == null) {
loadTimeLines(fact, storage, timeLineKey);
}
-log.info("timeline for {} is: {}", storage, get(timeLineKey));
+log.debug("timeline for {} is: {}", storage, get(timeLineKey));
// return the final value from memory
return get(timeLineKey);
// RESUME CHECKSTYLE CHECK DoubleCheckedLockingCheck
http://git-wip-us.apache.org/repos/asf/lens/blob/ac9a56fc/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
--
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
index 2edeb41..0855ced 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
@@ -355,4 +355,13 @@ public interface Candidate {
}
throw new IllegalArgumentException("Candidate doesn't have children and no
suitable implementation found");
}
+
+ default Set decideMeasurePhrasesToAnswer(Set
measurePhraseIndices) throws LensException {
+HashSet allCovered = Sets.newHashSet();
+for (Candidate candidate : getChildren()) {
+ Set covered =
candidate.decideMeasurePhrasesToAnswer(measurePhraseIndices);
+ allCovered.addAll(covered);
+}
+return allCovered;
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/ac9a56fc/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 47bce65..b5330a7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -830,6 +830,11 @@ public class CubeQueryContext extends TracksQueriedColumns
implements QueryAST,
cand = iter.next();
log.info("Available Candidates:{}, picking up Candidate: {} for
querying", candidates, cand);
pickedCandidate = cand;
+// Answerable common measures in JoinCandidate should be answered by
one of the children, otherwise
+// measure numbers will be added multiple times in the final union
query.
+Set measureIndices = getQueriedPhrases().stream().filter(x ->
x.hasMeasures(this))
+ .map(QueriedPhraseContext::getPosition).collect(toSet());
+pickedCandidate.decideMeasurePhrasesToAnswer(measureIndices);
}
if (pickedCandidate == null) {
throwNoCandidateFactException();
http://git-wip-us.apache.org/repos/asf/lens/blob/ac9a56fc/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
--
diff --git
a/lens-cube/src/main/java/org/apache/
lens git commit: LENS-1429 : Queries failing intermittently with NO_REF_COL_AVAILABLE
Repository: lens
Updated Branches:
refs/heads/current-release-line ac9a56fcd -> cc6c907d7
LENS-1429 : Queries failing intermittently with NO_REF_COL_AVAILABLE
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/cc6c907d
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/cc6c907d
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/cc6c907d
Branch: refs/heads/current-release-line
Commit: cc6c907d783c03565063d51b7bfbac79177262ab
Parents: ac9a56f
Author: Sushil Mohanty
Authored: Tue Jun 6 16:35:32 2017 +0530
Committer: sushilmohanty
Committed: Tue Jun 6 16:35:32 2017 +0530
--
.../org/apache/lens/cube/parse/CandidateTableResolver.java | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/cc6c907d/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index 93736cd..f530650 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -674,11 +674,13 @@ class CandidateTableResolver implements ContextRewriter {
if (colSet == null || colSet.isEmpty()) {
return true;
}
+boolean isEvaluable = false;
for (QueriedPhraseContext qur : colSet) {
if (sc.isPhraseAnswerable(qur)) {
-return true;
+isEvaluable = true;
+continue;
}
}
-return false;
+return isEvaluable;
}
}
lens git commit: LENS-1430 : Expressions are not getting resolved correctly in union query
Repository: lens
Updated Branches:
refs/heads/master cb2529672 -> 89b17e640
LENS-1430 : Expressions are not getting resolved correctly in union query
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/89b17e64
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/89b17e64
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/89b17e64
Branch: refs/heads/master
Commit: 89b17e640b47ff9e071f8df53c05ae0feaf25181
Parents: cb25296
Author: Sushil Mohanty
Authored: Tue Jun 6 16:42:10 2017 +0530
Committer: sushilmohanty
Committed: Tue Jun 6 16:42:10 2017 +0530
--
.../lens/cube/parse/ExpressionResolver.java | 36 +++-
.../cube/parse/TestUnionAndJoinCandidates.java | 60 +++-
.../resources/schema/cubes/base/basecube.xml| 12
.../resources/schema/cubes/base/testcube.xml| 9 +++
.../cubes/derived/union_join_ctx_der1.xml | 2 +
.../schema/facts/union_join_ctx_fact1.xml | 2 +-
.../schema/facts/union_join_ctx_fact3.xml | 1 +
.../schema/facts/union_join_ctx_fact4.xml | 59 +++
8 files changed, 177 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/89b17e64/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index f38aa54..ea6d5c7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -285,6 +285,7 @@ class ExpressionResolver implements ContextRewriter {
@Getter
private Map> allExprsQueried = new
HashMap>();
private Map> pickedExpressions = new
HashMap>();
+private Map nonPickedExpressionsForCandidate = new
HashMap();
private final CubeQueryContext cubeql;
ExpressionResolverContext(CubeQueryContext cubeql) {
@@ -407,6 +408,7 @@ class ExpressionResolver implements ContextRewriter {
}
pickedExpressions.clear();
+ nonPickedExpressionsForCandidate.clear();
return exprDims;
}
@@ -426,7 +428,7 @@ class ExpressionResolver implements ContextRewriter {
// Having ast is not copied now, it's maintained in cubeQueryContext,
each fact processes that serially.
if (queryAST.getHavingAST() != null) {
replaceAST(cubeql, queryAST.getHavingAST());
- } else if (cubeql.getHavingAST() != null) {
+ } else if (cubeql.getHavingAST() != null &&
nonPickedExpressionsForCandidate.isEmpty()) {
replaceAST(cubeql, cubeql.getHavingAST());
queryAST.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST()));
}
@@ -459,6 +461,9 @@ class ExpressionResolver implements ContextRewriter {
if (expr != null) {
node1.setChild(i, replaceAlias(expr.getRewrittenAST(),
cubeql));
}
+} else if (nonPickedExpressionsForCandidate.containsKey(column)) {
+ node1.setChild(i, nonPickedExpressionsForCandidate.get(column));
+
}
}
}
@@ -477,7 +482,7 @@ class ExpressionResolver implements ContextRewriter {
return null;
}
-private void pickExpressionsForTable(CandidateTable cTable) {
+private void pickExpressionsForTable(CandidateTable cTable) throws
LensException {
for (Map.Entry> ecEntry :
allExprsQueried.entrySet()) {
Set ecSet = ecEntry.getValue();
for (ExpressionContext ec : ecSet) {
@@ -488,6 +493,8 @@ class ExpressionResolver implements ContextRewriter {
// pick first evaluable expression
pickedExpressions.computeIfAbsent(ecEntry.getKey(), k -> new
HashSet<>())
.add(new PickedExpression(ec.srcAlias,
ec.evaluableExpressions.get(cTable).iterator().next()));
+ } else {
+nonPickedExpressionsForCandidate.put(ecEntry.getKey(),
getDefaultExpr(getExprAst(ec)));
}
}
}
@@ -495,6 +502,31 @@ class ExpressionResolver implements ContextRewriter {
}
}
+private ASTNode getExprAst(ExpressionContext ec) {
+ Set scSet =
CandidateUtil.getStorageCandidates(cubeql.getCandidates());
+ Set storageTableNames = new HashSet();
+ for (StorageCandidate sc : scSet) {
+storageTableNames.add(sc.getStorageTable());
+ }
+ for (CandidateTable table : ec.evaluableExpressions.keySet()) {
+if (storageTableNames.contains(table.getStorageTable())) {
+ return
MetastoreUtil.copyAST(ec.evaluableExpressio
lens git commit: LENS-1430 : Expressions are not getting resolved correctly in union query
Repository: lens
Updated Branches:
refs/heads/current-release-line cc6c907d7 -> be75e1d55
LENS-1430 : Expressions are not getting resolved correctly in union query
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/be75e1d5
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/be75e1d5
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/be75e1d5
Branch: refs/heads/current-release-line
Commit: be75e1d5571f33ca1ec51f91a5d113abe0838842
Parents: cc6c907
Author: Sushil Mohanty
Authored: Tue Jun 6 16:43:17 2017 +0530
Committer: sushilmohanty
Committed: Tue Jun 6 16:43:17 2017 +0530
--
.../lens/cube/parse/ExpressionResolver.java | 36 +++-
.../cube/parse/TestUnionAndJoinCandidates.java | 60 +++-
.../resources/schema/cubes/base/basecube.xml| 12
.../resources/schema/cubes/base/testcube.xml| 9 +++
.../cubes/derived/union_join_ctx_der1.xml | 2 +
.../schema/facts/union_join_ctx_fact1.xml | 2 +-
.../schema/facts/union_join_ctx_fact3.xml | 1 +
.../schema/facts/union_join_ctx_fact4.xml | 59 +++
8 files changed, 177 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/be75e1d5/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 7cad400..a2f4bb6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -285,6 +285,7 @@ class ExpressionResolver implements ContextRewriter {
@Getter
private Map> allExprsQueried = new
HashMap>();
private Map> pickedExpressions = new
HashMap>();
+private Map nonPickedExpressionsForCandidate = new
HashMap();
private final CubeQueryContext cubeql;
ExpressionResolverContext(CubeQueryContext cubeql) {
@@ -407,6 +408,7 @@ class ExpressionResolver implements ContextRewriter {
}
pickedExpressions.clear();
+ nonPickedExpressionsForCandidate.clear();
return exprDims;
}
@@ -426,7 +428,7 @@ class ExpressionResolver implements ContextRewriter {
// Having ast is not copied now, it's maintained in cubeQueryContext,
each fact processes that serially.
if (queryAST.getHavingAST() != null) {
replaceAST(cubeql, queryAST.getHavingAST());
- } else if (cubeql.getHavingAST() != null) {
+ } else if (cubeql.getHavingAST() != null &&
nonPickedExpressionsForCandidate.isEmpty()) {
replaceAST(cubeql, cubeql.getHavingAST());
queryAST.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST()));
}
@@ -459,6 +461,9 @@ class ExpressionResolver implements ContextRewriter {
if (expr != null) {
node1.setChild(i, replaceAlias(expr.getRewrittenAST(),
cubeql));
}
+} else if (nonPickedExpressionsForCandidate.containsKey(column)) {
+ node1.setChild(i, nonPickedExpressionsForCandidate.get(column));
+
}
}
}
@@ -477,7 +482,7 @@ class ExpressionResolver implements ContextRewriter {
return null;
}
-private void pickExpressionsForTable(CandidateTable cTable) {
+private void pickExpressionsForTable(CandidateTable cTable) throws
LensException {
for (Map.Entry> ecEntry :
allExprsQueried.entrySet()) {
Set ecSet = ecEntry.getValue();
for (ExpressionContext ec : ecSet) {
@@ -488,6 +493,8 @@ class ExpressionResolver implements ContextRewriter {
// pick first evaluable expression
pickedExpressions.computeIfAbsent(ecEntry.getKey(), k -> new
HashSet<>())
.add(new PickedExpression(ec.srcAlias,
ec.evaluableExpressions.get(cTable).iterator().next()));
+ } else {
+nonPickedExpressionsForCandidate.put(ecEntry.getKey(),
getDefaultExpr(getExprAst(ec)));
}
}
}
@@ -495,6 +502,31 @@ class ExpressionResolver implements ContextRewriter {
}
}
+private ASTNode getExprAst(ExpressionContext ec) {
+ Set scSet =
CandidateUtil.getStorageCandidates(cubeql.getCandidates());
+ Set storageTableNames = new HashSet();
+ for (StorageCandidate sc : scSet) {
+storageTableNames.add(sc.getStorageTable());
+ }
+ for (CandidateTable table : ec.evaluableExpressions.keySet()) {
+if (storageTableNames.contains(table.getStorageTable())) {
+ return
MetastoreUtil.c
lens git commit: LENS-1437 : Missing unit test cases for virtual fact
Repository: lens
Updated Branches:
refs/heads/master 70722824b -> 67adf5b84
LENS-1437 : Missing unit test cases for virtual fact
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/67adf5b8
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/67adf5b8
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/67adf5b8
Branch: refs/heads/master
Commit: 67adf5b8431cab29d36f8479d8273182cb0e088a
Parents: 7072282
Author: Rajitha R
Authored: Tue Jun 27 16:12:56 2017 +0530
Committer: sushilmohanty
Committed: Tue Jun 27 16:12:56 2017 +0530
--
.../apache/lens/cube/parse/CubeTestSetup.java | 4 +-
.../lens/cube/parse/TestCubeRewriter.java | 18 +--
.../lens/cube/parse/TestVirtualFactQueries.java | 123 +++
.../resources/schema/cubes/base/testcube2.xml | 31 +
.../resources/schema/cubes/base/virtualcube.xml | 23 +++-
.../resources/schema/facts/testfact7_base.xml | 67 ++
.../resources/schema/facts/testfact8_base.xml | 58 +
.../resources/schema/facts/testfact9_base.xml | 57 +
.../schema/facts/virtual/virtualfact.xml| 2 +-
9 files changed, 361 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/67adf5b8/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 48869c2..0366e56 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -364,7 +364,7 @@ public class CubeTestSetup {
// storageName[0] is hourly
// storageName[1] is daily
// storageName[2] is monthly
- public static Map
getWhereForMonthlyDailyAndHourly2months(String... storageTables) {
+ public static Map
getWhereForMonthlyDailyAndHourly2months(String cubeName, String...
storageTables) {
Map storageTableToWhereClause = new LinkedHashMap();
List hourlyparts = new ArrayList();
List dailyparts = new ArrayList();
@@ -411,7 +411,7 @@ public class CubeTestSetup {
tables.append(storageTables[0]);
}
Collections.sort(parts);
-storageTableToWhereClause.put(tables.toString(),
StorageUtil.getWherePartClause("dt", TEST_CUBE_NAME, parts));
+storageTableToWhereClause.put(tables.toString(),
StorageUtil.getWherePartClause("dt", cubeName, parts));
return storageTableToWhereClause;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/67adf5b8/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index d8c7335..1c75e6c 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -145,22 +145,6 @@ public class TestCubeRewriter extends TestQueryRewrite {
//assertNotNull(rewrittenQuery.getNonExistingParts());
}
-
- @Test
- public void testVirtualFactCubeSimpleQuery() throws Exception {
-Configuration conf = getConf();
-conf.set(DRIVER_SUPPORTED_STORAGES, "C1");
-CubeQueryContext rewrittenQuery =
- rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE,
getConfWithStorages("C1"));
-String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select
sum(virtualcube.msr2) as `sum(msr2)` FROM ",
- null, "AND ( dim1 = 10 )",
getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME,
-"C1_summary1"));
-String hql = rewrittenQuery.toHQL();
-compareQueries(hql, expected);
-System.out.println("Non existing parts:" +
rewrittenQuery.getNonExistingParts());
- }
-
-
@Test
public void testMaxCoveringFact() throws Exception {
Configuration conf = getConf();
@@ -954,7 +938,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
rewrite("select SUM(msr2) from testCube" + " where " +
TWO_MONTHS_RANGE_UPTO_HOURS, getConfWithStorages("C2"));
String expected =
getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as
`sum(msr2)` FROM ", null, null,
-getWhereForMonthlyDailyAndHourly2months("C2_testfact"));
+getWhereForMonthlyDailyAndHourly2months(TEST_CUBE_NAME,
"C2_testfact"));
compareQueries(hqlQuery, ex
lens git commit: LENS-1458 : RESULT_SET_PARENT_DIR set at driver level getting overwritten for queued queries after server restart
Repository: lens
Updated Branches:
refs/heads/master 6c40b24ff -> 0ea84729a
LENS-1458 : RESULT_SET_PARENT_DIR set at driver level getting overwritten for
queued queries after server restart
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/0ea84729
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/0ea84729
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/0ea84729
Branch: refs/heads/master
Commit: 0ea84729ad0a1ff0ee7869f43600bcd38bdf
Parents: 6c40b24
Author: Sushil Mohanty
Authored: Mon Aug 7 18:38:15 2017 +0530
Committer: sushilmohanty
Committed: Mon Aug 7 18:38:15 2017 +0530
--
.../apache/lens/server/api/query/DriverSelectorQueryContext.java | 2 +-
.../src/test/java/org/apache/lens/server/TestServerRestart.java| 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/0ea84729/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index 41e533e..9cec42a 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -254,7 +254,7 @@ public class DriverSelectorQueryContext {
}
public void setDriverConf(LensDriver driver, Configuration conf) {
-driverQueryContextMap.get(driver).setDriverSpecificConf(conf);
+driverQueryContextMap.get(driver).setDriverSpecificConf(mergeConf(driver,
conf));
}
public void setDriverCost(LensDriver driver, QueryCost cost) {
http://git-wip-us.apache.org/repos/asf/lens/blob/0ea84729/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
--
diff --git
a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
index 20aa8cb..8b7728e 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
@@ -39,6 +39,7 @@ import org.apache.lens.api.APIResult.Status;
import org.apache.lens.api.query.*;
import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.driver.hive.TestRemoteHiveDriver;
+import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.QueryExecutionService;
@@ -245,6 +246,7 @@ public class TestServerRestart extends
LensAllApplicationJerseyTest {
assertEquals(lensQueryConf.getProperty(KEY_POST_SELECT),
VALUE_POST_SELECT);
if (afterRestart) {
+ assertEquals(driverConf.get(LensConfConstants.RESULT_SET_PARENT_DIR),
"target/hive-lens-results");
//This will be unavailable since if was not updated in LensConf by
MockDriverQueryHook
assertNull(driverConf.get(UNSAVED_KEY_POST_SELECT));
} else {
lens git commit: LENS-1475 : Override equals(Object obj) and hashcode() methods of SegmentationCandidate
Repository: lens
Updated Branches:
refs/heads/master 24d4f1531 -> dbbd0c446
LENS-1475 : Override equals(Object obj) and hashcode() methods of
SegmentationCandidate
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/dbbd0c44
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/dbbd0c44
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/dbbd0c44
Branch: refs/heads/master
Commit: dbbd0c44670d5b15022bb93f05f526783f29a4a4
Parents: 24d4f15
Author: Sushil Mohanty
Authored: Thu Sep 14 12:42:57 2017 +0530
Committer: sushilmohanty
Committed: Thu Sep 14 12:42:57 2017 +0530
--
.../lens/cube/parse/SegmentationCandidate.java | 21
1 file changed, 21 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/dbbd0c44/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
index 055d6ef..9378009 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
@@ -404,4 +404,25 @@ public class SegmentationCandidate implements Candidate {
return cubeQueryContextMap.entrySet().stream().filter(entry ->
entry.getValue().getPickedCandidate() == null)
.collect(toMap(Map.Entry::getKey, entry ->
entry.getValue().getStoragePruningMsgs()));
}
+
+ @Override
+ public boolean equals(Object obj) {
+if (super.equals(obj)) {
+ return true;
+}
+
+if (obj == null || !(obj instanceof SegmentationCandidate)) {
+ return false;
+}
+
+SegmentationCandidate segmantationCandidate = (SegmentationCandidate) obj;
+return
(segmantationCandidate.segmentation.getSegments().equals(this.segmentation.getSegments())
+&&
segmantationCandidate.segmentation.getBaseCube().equals(this.segmentation.getBaseCube()));
+ }
+
+ @Override
+ public int hashCode() {
+return segmentation.hashCode();
+ }
+
}
lens git commit: LENS-1476 : Order By clause is not rewritten correctly when column is wrapped with function
Repository: lens
Updated Branches:
refs/heads/master dbbd0c446 -> 9a40b6455
LENS-1476 : Order By clause is not rewritten correctly when column is wrapped
with function
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/9a40b645
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/9a40b645
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/9a40b645
Branch: refs/heads/master
Commit: 9a40b6455b50fe0e0930d1542a1ec9745025ccc7
Parents: dbbd0c4
Author: Sushil Mohanty
Authored: Thu Sep 14 12:43:55 2017 +0530
Committer: sushilmohanty
Committed: Thu Sep 14 12:43:55 2017 +0530
--
.../main/java/org/apache/lens/cube/parse/UnionQueryWriter.java| 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/9a40b645/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index 4eb086b..0d512e4 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -204,7 +204,8 @@ public class UnionQueryWriter extends SimpleHQLContext {
ASTNode outerOrderby = new ASTNode(child);
ASTNode tokNullsChild = (ASTNode) child.getChild(0);
ASTNode outerTokNullsChild = new ASTNode(tokNullsChild);
- if (((ASTNode) tokNullsChild.getChild(0)).getToken().getType() ==
HiveParser.DOT) {
+ if (((ASTNode) tokNullsChild.getChild(0)).getToken().getType() ==
HiveParser.DOT ||
+ ((ASTNode) tokNullsChild.getChild(0)).getToken().getType() ==
HiveParser.TOK_FUNCTION) {
outerTokNullsChild.addChild(innerToOuterSelectASTs.get(new
HQLParser.HashableASTNode((ASTNode) tokNullsChild)));
} else {
outerTokNullsChild.addChild(tokNullsChild);
lens git commit: LENS-1477 : Change default measure value to 0 instead of 0.0 in union queries
Repository: lens
Updated Branches:
refs/heads/master 9a40b6455 -> a7ec47ed3
LENS-1477 : Change default measure value to 0 instead of 0.0 in union queries
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a7ec47ed
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a7ec47ed
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a7ec47ed
Branch: refs/heads/master
Commit: a7ec47ed33ad6aa906b8c62dd68a62b7b85fab1a
Parents: 9a40b64
Author: Sushil Mohanty
Authored: Mon Sep 18 16:33:28 2017 +0530
Committer: sushilmohanty
Committed: Mon Sep 18 16:33:28 2017 +0530
--
.../lens/cube/parse/ExpressionResolver.java | 2 +-
.../lens/cube/parse/UnionQueryWriter.java | 14 +-
.../lens/cube/parse/TestBaseCubeQueries.java| 166 +--
.../lens/cube/parse/TestBridgeTableQueries.java | 40 ++---
.../parse/TestCubeSegmentationRewriter.java | 24 +--
.../cube/parse/TestUnionAndJoinCandidates.java | 34 ++--
.../lens/cube/parse/TestVirtualFactQueries.java | 4 +-
7 files changed, 142 insertions(+), 142 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/a7ec47ed/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 553468f..e0a0d17 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -560,7 +560,7 @@ class ExpressionResolver implements ContextRewriter {
private ASTNode getDefaultExpr(ASTNode node) {
if (HQLParser.isAggregateAST(node)) {
-node.setChild(1, new ASTNode(new CommonToken(HiveParser.Identifier,
"0.0")));
+node.setChild(1, new ASTNode(new CommonToken(HiveParser.Identifier,
"0")));
}
for (int i = 0; i < node.getChildCount(); i++) {
ASTNode child = (ASTNode) node.getChild(i);
http://git-wip-us.apache.org/repos/asf/lens/blob/a7ec47ed/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index 0d512e4..9d0d0d2 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -54,7 +54,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
private Map
storageCandidateToSelectAstMap = new HashMap<>();
private CubeQueryContext cubeql;
static final ASTNode DEFAULT_MEASURE_AST;
- private static final String DEFAULT_MEASURE = "0.0";
+ private static final String DEFAULT_MEASURE = "0";
static {
try {
DEFAULT_MEASURE_AST = HQLParser.parseExpr(DEFAULT_MEASURE);
@@ -204,8 +204,8 @@ public class UnionQueryWriter extends SimpleHQLContext {
ASTNode outerOrderby = new ASTNode(child);
ASTNode tokNullsChild = (ASTNode) child.getChild(0);
ASTNode outerTokNullsChild = new ASTNode(tokNullsChild);
- if (((ASTNode) tokNullsChild.getChild(0)).getToken().getType() ==
HiveParser.DOT ||
- ((ASTNode) tokNullsChild.getChild(0)).getToken().getType() ==
HiveParser.TOK_FUNCTION) {
+ if (((ASTNode) tokNullsChild.getChild(0)).getToken().getType() ==
HiveParser.DOT
+ || ((ASTNode) tokNullsChild.getChild(0)).getToken().getType() ==
HiveParser.TOK_FUNCTION) {
outerTokNullsChild.addChild(innerToOuterSelectASTs.get(new
HQLParser.HashableASTNode((ASTNode) tokNullsChild)));
} else {
outerTokNullsChild.addChild(tokNullsChild);
@@ -217,7 +217,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
}
/**
- * Get the select expression. In case of node is default retunrs "0.0" with
alias
+ * Get the select expression. In case of node is default retunrs "0" with
alias
* otherwise the select phrase with alias.
*
* @param nodeWithoutAlias
@@ -247,7 +247,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
/**
* Get the aggregate node for the SelectPhrase index. A given measure might
not be answerable
- * for a StorageCanddate. In that case get the non default aggregate node
wcich ideally not "0.0",
+ * for a StorageCanddate. In that case get the non default aggregate node
wcich ideally not "0",
* from otherStorage candidate.
*
* @param position
@@ -267,
lens git commit: LENS-1478 : Queries with partial data set as false lookahead not working
Repository: lens
Updated Branches:
refs/heads/master a7ec47ed3 -> a86cb883f
LENS-1478 : Queries with partial data set as false lookahead not working
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a86cb883
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a86cb883
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a86cb883
Branch: refs/heads/master
Commit: a86cb883ffef8699454a215be9996df30c9988fd
Parents: a7ec47e
Author: Sushil Mohanty
Authored: Tue Sep 19 14:26:22 2017 +0530
Committer: sushilmohanty
Committed: Tue Sep 19 14:26:22 2017 +0530
--
.../main/java/org/apache/lens/cube/parse/StorageCandidate.java| 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/a86cb883/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 140b2c1..fd07bf2 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -515,7 +515,8 @@ public class StorageCandidate implements Candidate,
CandidateTable {
partWhereClauseFormat);
updatePartitionStorage(innerPart);
innerPart.setFound(pPart.isFound());
-if (innerPart.isFound()) {
+if (innerPart.isFound() || !failOnPartialData) {
+ this.participatingUpdatePeriods.add(maxInterval);
partitions.add(innerPart);
}
}
lens git commit: LENS-1461 : Lens services state are not persisted correctly
Repository: lens
Updated Branches:
refs/heads/master cdac40874 -> 3b657dee4
LENS-1461 : Lens services state are not persisted correctly
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3b657dee
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3b657dee
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3b657dee
Branch: refs/heads/master
Commit: 3b657dee40ef70d06e6ed589467bd455b77a5066
Parents: cdac408
Author: Ankit Kailaswar
Authored: Fri Nov 3 13:25:53 2017 +0530
Committer: sushilmohanty
Committed: Fri Nov 3 13:25:53 2017 +0530
--
.../lens/server/session/HiveSessionService.java| 17 ++---
1 file changed, 14 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/3b657dee/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
index b480d14..a9fd233 100644
---
a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
+++
b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
@@ -420,12 +420,23 @@ public class HiveSessionService extends BaseLensService
implements SessionServic
@Override
public void writeExternal(ObjectOutput out) throws IOException {
// Write out all the sessions
-out.writeInt(SESSION_MAP.size());
+List sessions = new ArrayList<>();
for (LensSessionHandle sessionHandle : SESSION_MAP.values()) {
- LensSessionImpl session = getSession(sessionHandle);
+ try {
+sessions.add(getSession(sessionHandle));
+ } catch (ClientErrorException e) {
+// warn for invalid/null session and continue.
+log.warn("Cannot persist " + (sessionHandle != null ?
sessionHandle.getPublicId() : "null ")
++ " session. {}", e);
+ }
+}
+
+out.writeInt(sessions.size());
+for (LensSessionImpl session : sessions) {
session.getLensSessionPersistInfo().writeExternal(out);
}
-log.info("Session service pesristed " + SESSION_MAP.size() + " sessions");
+
+log.info("Session service pesristed " + sessions.size() + " sessions out
of " + SESSION_MAP.size());
}
/**
lens git commit: LENS-1483 : Implementation for Jdbc driver cost
Repository: lens
Updated Branches:
refs/heads/master 3b657dee4 -> 231121425
LENS-1483 : Implementation for Jdbc driver cost
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/23112142
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/23112142
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/23112142
Branch: refs/heads/master
Commit: 231121425f156bd572fb847fe87847ee093bbb9a
Parents: 3b657de
Author: Rajitha R
Authored: Fri Nov 17 13:55:44 2017 +0530
Committer: sushilmohanty
Committed: Fri Nov 17 13:55:44 2017 +0530
--
.../apache/lens/api/query/QueryCostType.java| 2 +-
.../FactPartitionBasedQueryCostCalculator.java | 21 --
.../cube/query/cost/StaticCostCalculator.java | 48 +
...stFactPartitionBasedQueryCostCalculator.java | 4 +-
.../org/apache/lens/driver/hive/HiveDriver.java | 9 +--
.../src/main/resources/hivedriver-default.xml | 12
.../apache/lens/driver/hive/TestHiveDriver.java | 1 -
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 41 +++
.../driver/jdbc/JDBCDriverConfConstants.java| 4 ++
.../src/main/resources/jdbcdriver-default.xml | 25 +++
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 15 ++--
.../lens/server/api/LensConfConstants.java | 22 ++
.../query/cost/FactPartitionBasedQueryCost.java | 12 ++--
.../lens/server/api/query/cost/QueryCost.java | 9 ++-
.../api/query/cost/QueryCostCalculator.java | 2 +
.../api/query/cost/QueryCostTypeDecider.java| 33 +
.../api/query/cost/QueryCostTypeRangeConf.java | 71 ++
.../cost/RangeBasedQueryCostTypeDecider.java| 45
.../server/api/query/cost/StaticQueryCost.java | 76
.../server/api/query/priority/RangeConf.java| 2 +-
.../api/query/cost/MockQueryCostCalculator.java | 5 ++
.../cost/TestFactPartitionBasedQueryCost.java | 27 +++
.../query/QueryContextComparatorTest.java | 11 +--
src/site/apt/admin/hivedriver-config.apt| 20 +++---
src/site/apt/admin/jdbcdriver-config.apt| 66 +
25 files changed, 507 insertions(+), 76 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/23112142/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java
--
diff --git
a/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java
b/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java
index 37eda42..f65578e 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java
@@ -27,5 +27,5 @@ import javax.xml.bind.annotation.XmlType;
@XmlType
@XmlEnum
public enum QueryCostType {
- LOW, MEDIUM, HIGH
+ VERY_LOW, LOW, MEDIUM, HIGH
}
http://git-wip-us.apache.org/repos/asf/lens/blob/23112142/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
index 3c157ee..6d845f8 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
@@ -18,6 +18,9 @@
*/
package org.apache.lens.cube.query.cost;
+import static
org.apache.lens.server.api.LensConfConstants.DRIVER_COST_TYPE_RANGES;
+import static
org.apache.lens.server.api.LensConfConstants.DRIVER_QUERY_COST_TYPE_DEFAULT_RANGES;
+
import java.util.Map;
import java.util.Set;
@@ -26,9 +29,7 @@ import org.apache.lens.cube.metadata.UpdatePeriod;
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.AbstractQueryContext;
-import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
-import org.apache.lens.server.api.query.cost.QueryCost;
-import org.apache.lens.server.api.query.cost.QueryCostCalculator;
+import org.apache.lens.server.api.query.cost.*;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
@@ -37,6 +38,8 @@ public class FactPartitionBasedQueryCostCalculator implements
QueryCostCalculato
public static final String UPDATE_PERIOD_WEIGHT_PREFIX =
"update.period.weight.";
+ protected QueryCostTypeDecider queryCostTypeDecider;
+
/**
* Calculates total cost based on weights of selected tables and their
selected partitions
*
@@ -91,9 +94,19 @@
lens git commit: LENS-1486 : Disable test case testQueryAliveOnSessionClose
Repository: lens
Updated Branches:
refs/heads/master 231121425 -> 26125ac67
LENS-1486 : Disable test case testQueryAliveOnSessionClose
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/26125ac6
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/26125ac6
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/26125ac6
Branch: refs/heads/master
Commit: 26125ac67080477219930945c61f6093a22ec45a
Parents: 2311214
Author: Sushil Mohanty
Authored: Tue Nov 21 15:19:31 2017 +0530
Committer: sushilmohanty
Committed: Tue Nov 21 15:19:31 2017 +0530
--
.../lens/server/query/TestQueryIndependenceFromSessionClose.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/26125ac6/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java
--
diff --git
a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java
b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java
index 068d46c..5d37f41 100644
---
a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java
+++
b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java
@@ -217,7 +217,7 @@ public class TestQueryIndependenceFromSessionClose extends
LensJerseyTest {
};
}
- @Test(dataProvider = "restartDataProvider")
+ @Test(dataProvider = "restartDataProvider", enabled = false)
public void testQueryAliveOnSessionClose(boolean restartBeforeFinish,
boolean restartAfterFinish)
throws LensException, InterruptedException {
int numSessions = getSessionsOfFoo().size();
lens git commit: LENS-1496 : Look ahead Time Partitions to be made configurable
Repository: lens
Updated Branches:
refs/heads/master ae23e9c97 -> 53c342bd2
LENS-1496 : Look ahead Time Partitions to be made configurable
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/53c342bd
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/53c342bd
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/53c342bd
Branch: refs/heads/master
Commit: 53c342bd20dc35f75b80da78ac799c48cc2e04b1
Parents: ae23e9c
Author: Rajitha R
Authored: Wed Dec 27 13:43:09 2017 +0530
Committer: sushilmohanty
Committed: Wed Dec 27 13:43:09 2017 +0530
--
.../lens/cube/parse/CubeQueryConfUtil.java | 6 +++
.../lens/cube/parse/StorageCandidate.java | 27 +---
.../src/main/resources/olap-query-conf.xml | 8
.../apache/lens/cube/metadata/DateFactory.java | 2 +
.../lens/cube/parse/TestCubeRewriter.java | 43
5 files changed, 80 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/53c342bd/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
index eeaa3af..4768fb8 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
@@ -48,6 +48,7 @@ public final class CubeQueryConfUtil {
public static final String PROCESS_TIME_PART_COL =
"lens.cube.query.process.time" + ".partition.column";
public static final String COMPLETENESS_CHECK_PART_COL =
"lens.cube.query.completeness.check.partition.column";
public static final String LOOK_AHEAD_PT_PARTS_PFX =
"lens.cube.query.lookahead.ptparts.forinterval.";
+ public static final String LOOK_AHEAD_TIME_PARTS_PFX =
"lens.cube.query.lookahead.timeparts.forinterval.";
public static final String ENABLE_GROUP_BY_TO_SELECT =
"lens.cube.query.promote.groupby.toselect";
public static final String ENABLE_SELECT_TO_GROUPBY =
"lens.cube.query.promote.select.togroupby";
public static final String ENABLE_ATTRFIELDS_ADD_DISTINCT =
"lens.cube.query.enable.attrfields.add.distinct";
@@ -58,6 +59,7 @@ public final class CubeQueryConfUtil {
public static final String REPLACE_TIMEDIM_WITH_PART_COL =
"lens.cube.query.replace.timedim";
public static final boolean DEFAULT_MULTI_TABLE_SELECT = true;
public static final int DEFAULT_LOOK_AHEAD_PT_PARTS = 1;
+ public static final int DEFAULT_LOOK_AHEAD_TIME_PARTS = 1;
public static final boolean DEFAULT_ENABLE_GROUP_BY_TO_SELECT = false;
public static final boolean DEFAULT_ENABLE_SELECT_TO_GROUPBY = false;
public static final boolean DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL = true;
@@ -66,6 +68,10 @@ public final class CubeQueryConfUtil {
return LOOK_AHEAD_PT_PARTS_PFX + interval.name().toLowerCase();
}
+ public static String getLookAheadTimePartsKey(UpdatePeriod interval) {
+return LOOK_AHEAD_TIME_PARTS_PFX + interval.name().toLowerCase();
+ }
+
private static String getValidKeyCubePFX(String cubeName) {
return VALID_PFX + cubeName.toLowerCase();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/53c342bd/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 3ee66ee..b48be7f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -456,6 +456,10 @@ public class StorageCandidate implements Candidate,
CandidateTable {
int lookAheadNumParts = getConf()
.getInt(CubeQueryConfUtil.getLookAheadPTPartsKey(maxInterval),
CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_PT_PARTS);
+
+int lookAheadNumTimeParts = getConf()
+ .getInt(CubeQueryConfUtil.getLookAheadTimePartsKey(maxInterval),
CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_TIME_PARTS);
+
TimeRange.Iterable.Iterator iter = TimeRange.iterable(ceilFromDate,
floorToDate, maxInterval, 1).iterator();
// add partitions from ceilFrom to floorTo
while (iter.hasNext()) {
@@ -475,7 +479,7 @@ public class StorageCandidate implements Candidate,
CandidateTable {
log.debug("part column is process time col");
} else if (updatePeriods.first().equals(maxInterval)) {
log.debug
lens git commit: First Commit, added SushilMohanty to developers list
Repository: lens Updated Branches: refs/heads/master e5691d8d6 -> f41c176b4 First Commit, added SushilMohanty to developers list Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/f41c176b Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/f41c176b Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/f41c176b Branch: refs/heads/master Commit: f41c176b473ac014a0c3b24f16e82444a445f28b Parents: e5691d8 Author: Sushil Mohanty Authored: Tue Nov 24 12:42:25 2015 +0530 Committer: Sushil Mohanty Committed: Tue Nov 24 12:42:25 2015 +0530 -- pom.xml | 9 + 1 file changed, 9 insertions(+) -- http://git-wip-us.apache.org/repos/asf/lens/blob/f41c176b/pom.xml -- diff --git a/pom.xml b/pom.xml index 9b417d9..b19857f 100644 --- a/pom.xml +++ b/pom.xml @@ -342,6 +342,15 @@ Committer + + sushilmohanty + sushilmoha...@apache.org + SushilMohanty + +5.5 + +Committer + + scm:git:https://git-wip-us.apache.org/repos/asf/lens.git
lens git commit: LENS-903 : No candidate dim available exception should contain only brief error
Repository: lens
Updated Branches:
refs/heads/master 4d3d2f82f -> b84cb2cd3
LENS-903 : No candidate dim available exception should contain only brief error
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b84cb2cd
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b84cb2cd
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b84cb2cd
Branch: refs/heads/master
Commit: b84cb2cd32a53806278bbe5d45028dbaa760bdf5
Parents: 4d3d2f8
Author: Sushil Mohanty
Authored: Thu Dec 17 17:31:04 2015 +0530
Committer: Sushil Mohanty
Committed: Thu Dec 17 17:31:04 2015 +0530
--
.../lens/cube/parse/CubeQueryContext.java | 7 ---
.../lens/cube/parse/TestCubeRewriter.java | 22 +++-
2 files changed, 25 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/b84cb2cd/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index f75a6b9..4034a54 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -32,6 +32,7 @@ import java.io.IOException;
import java.util.*;
import org.apache.lens.cube.error.LensCubeErrorCode;
+import org.apache.lens.cube.error.NoCandidateDimAvailableException;
import org.apache.lens.cube.error.NoCandidateFactAvailableException;
import org.apache.lens.cube.metadata.*;
import
org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
@@ -810,12 +811,12 @@ public class CubeQueryContext implements
TrackQueriedColumns {
}
}
}
- throw new
LensException(LensCubeErrorCode.NO_CANDIDATE_DIM_AVAILABLE.getLensErrorInfo(),
- dim.getName(), reason);
+ log.error("Query rewrite failed due to NO_CANDIDATE_DIM_AVAILABLE,
Cause {}",
+ dimPruningMsgs.get(dim).toJsonObject());
+ throw new NoCandidateDimAvailableException(dimPruningMsgs.get(dim));
}
}
}
-
return dimsToQuery;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/b84cb2cd/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 802ff42..9a08735 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -32,6 +32,7 @@ import java.text.SimpleDateFormat;
import java.util.*;
import org.apache.lens.cube.error.LensCubeErrorCode;
+import org.apache.lens.cube.error.NoCandidateDimAvailableException;
import org.apache.lens.cube.error.NoCandidateFactAvailableException;
import org.apache.lens.cube.metadata.*;
import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCause;
@@ -46,6 +47,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
@@ -1078,6 +1080,23 @@ public class TestCubeRewriter extends TestQueryRewrite {
}
@Test
+ public void testNoCandidateDimAvailableExceptionCompare() throws Exception {
+
+//Max cause COLUMN_NOT_FOUND, Ordinal 9
+PruneCauses pr1 = new
PruneCauses();
+pr1.addPruningMsg(new CubeDimensionTable(new Table("test", "citydim")),
+CandidateTablePruneCause.columnNotFound("test1", "test2",
"test3"));
+NoCandidateDimAvailableException ne1 = new
NoCandidateDimAvailableException(pr1);
+
+//Max cause EXPRESSION_NOT_EVALUABLE, Ordinal 6
+PruneCauses pr2 = new
PruneCauses();
+pr2.addPruningMsg(new CubeDimensionTable(new Table("test", "citydim")),
+CandidateTablePruneCause.expressionNotEvaluable("testexp1",
"testexp2"));
+NoCandidateDimAvailableException ne2 = new
NoCandidateDimAvailableException(pr2);
+assertEquals(ne1.compareTo(ne2), 3);
+ }
+
+ @Test
public void testDimensionQueryWithMultipleStorages() throws Exception {
String hqlQuery = rewrite("select name, stateid from" + " citydim",
getConf());
String expected =
@@ -1095,7 +1114,8 @@ public class TestCubeRewriter extends TestQueryR
lens git commit: LENS-903 : No candidate dim available exception should contain only brief error - added missing file
Repository: lens
Updated Branches:
refs/heads/master b84cb2cd3 -> d6aeecc73
LENS-903 : No candidate dim available exception should contain only brief error
- added missing file
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d6aeecc7
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d6aeecc7
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d6aeecc7
Branch: refs/heads/master
Commit: d6aeecc7306d9cd37dacd53caa0dfd842ca48bdc
Parents: b84cb2c
Author: Sushil Mohanty
Authored: Thu Dec 17 18:22:26 2015 +0530
Committer: Sushil Mohanty
Committed: Thu Dec 17 18:22:26 2015 +0530
--
.../error/NoCandidateDimAvailableException.java | 47
1 file changed, 47 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/d6aeecc7/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateDimAvailableException.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateDimAvailableException.java
b/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateDimAvailableException.java
new file mode 100644
index 000..ef76dc6
--- /dev/null
+++
b/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateDimAvailableException.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.error;
+
+import org.apache.lens.cube.metadata.CubeDimensionTable;
+import org.apache.lens.cube.parse.PruneCauses;
+import org.apache.lens.server.api.error.LensException;
+
+public class NoCandidateDimAvailableException extends LensException {
+
+ private final PruneCauses briefAndDetailedError;
+
+ public NoCandidateDimAvailableException(PruneCauses
briefAndDetailedError) {
+super(LensCubeErrorCode.NO_CANDIDATE_DIM_AVAILABLE.getLensErrorInfo(),
briefAndDetailedError.getBriefCause());
+this.briefAndDetailedError = briefAndDetailedError;
+ }
+
+ public PruneCauses.BriefAndDetailedError getJsonMessage() {
+return briefAndDetailedError.toJsonObject();
+ }
+
+ @Override
+ public int compareTo(LensException e) {
+//Compare the max CandidateTablePruneCode coming from different instances.
+if (e instanceof NoCandidateDimAvailableException) {
+ return briefAndDetailedError.getMaxCause().compareTo(
+ ((NoCandidateDimAvailableException)
e).briefAndDetailedError.getMaxCause());
+}
+return super.compareTo(e);
+ }
+}
lens git commit: LENS-905 : Group by is not promoted when keys projected along with having clause
Repository: lens
Updated Branches:
refs/heads/master d6aeecc73 -> c17908134
LENS-905 : Group by is not promoted when keys projected along with having clause
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c1790813
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c1790813
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c1790813
Branch: refs/heads/master
Commit: c17908134e004c646724d7e464252ec76a43113d
Parents: d6aeecc
Author: Sushil Mohanty
Authored: Fri Dec 18 11:50:33 2015 +0530
Committer: Sushil Mohanty
Committed: Fri Dec 18 11:50:33 2015 +0530
--
.../org/apache/lens/cube/parse/AggregateResolver.java| 4 +++-
.../apache/lens/cube/parse/TestAggregateResolver.java| 11 +--
2 files changed, 12 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/c1790813/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
index b544a67..39bd1cc 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
@@ -97,7 +97,9 @@ class AggregateResolver implements ContextRewriter {
Configuration distConf = cubeql.getConf();
boolean isDimOnlyDistinctEnabled =
distConf.getBoolean(CubeQueryConfUtil.ENABLE_ATTRFIELDS_ADD_DISTINCT,
CubeQueryConfUtil.DEFAULT_ATTR_FIELDS_ADD_DISTINCT);
-if (isDimOnlyDistinctEnabled) {
+//Having clause will always work with measures, if only keys projected
+//query should skip distinct and promote group by.
+if (cubeql.getHavingAST() == null && isDimOnlyDistinctEnabled) {
// Check if any measure/aggregate columns and distinct clause used in
// select tree. If not, update selectAST token "SELECT" to "SELECT
DISTINCT"
if (!hasMeasures(cubeql, cubeql.getSelectAST()) &&
!isDistinctClauseUsed(cubeql.getSelectAST())
http://git-wip-us.apache.org/repos/asf/lens/blob/c1790813/lens-cube/src/test/java/org/apache/lens/cube/parse/TestAggregateResolver.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestAggregateResolver.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestAggregateResolver.java
index a48d753..35234a1 100644
---
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestAggregateResolver.java
+++
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestAggregateResolver.java
@@ -93,6 +93,9 @@ public class TestAggregateResolver extends TestQueryRewrite {
String q10 = "SELECT cityid, round(testCube.msr2) from testCube where " +
TWO_DAYS_RANGE;
+//dimension selected with having
+String q11 = "SELECT cityid from testCube where " + TWO_DAYS_RANGE + "
having (testCube.msr2 > 100)";
+
String expectedq1 =
getExpectedQuery(cubeName, "SELECT testcube.cityid," + "
sum(testCube.msr2) from ", null,
"group by testcube.cityid", getWhereForDailyAndHourly2days(cubeName,
"C2_testfact"));
@@ -128,13 +131,17 @@ public class TestAggregateResolver extends
TestQueryRewrite {
String expectedq10 =
getExpectedQuery(cubeName, "SELECT testcube.cityid," + "
round(sum(testCube.msr2)) from ", null,
"group by testcube.cityid", getWhereForDailyAndHourly2days(cubeName,
"C2_testfact"));
-
+String expectedq11 =
+ getExpectedQuery(cubeName, "SELECT testcube.cityid from ", null,
+"group by testcube.cityid" + "having sum(testCube.msr2) > 100",
+ getWhereForDailyAndHourly2days(cubeName, "C2_testfact"));
String[] tests = {
- q1, q2, q3, q4, q5, q6, q7, q8, q9, q10,
+ q1, q2, q3, q4, q5, q6, q7, q8, q9, q10, q11,
};
String[] expected = {
expectedq1, expectedq2, expectedq3, expectedq4, expectedq5,
expectedq6, expectedq7, expectedq8, expectedq9, expectedq10,
+ expectedq11,
};
for (int i = 0; i < tests.length; i++) {
lens git commit: LENS-908 : Add error codes for jdbc query execution exception
Repository: lens
Updated Branches:
refs/heads/master 4d7c8e4db -> fdf04be04
LENS-908 : Add error codes for jdbc query execution exception
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/fdf04be0
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/fdf04be0
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/fdf04be0
Branch: refs/heads/master
Commit: fdf04be044e727e9d86708b7b0e24b0aec9ed6a3
Parents: 4d7c8e4
Author: Sushil Mohanty
Authored: Wed Dec 30 18:30:46 2015 +0530
Committer: Sushil Mohanty
Committed: Wed Dec 30 18:30:46 2015 +0530
--
lens-api/src/main/resources/lens-errors.conf| 6 ++--
.../org/apache/lens/driver/hive/HiveDriver.java | 4 +--
.../lens/driver/hive/LensHiveErrorCode.java | 36
.../apache/lens/driver/hive/TestHiveDriver.java | 4 +--
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 25 +-
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 14
.../server/api/error/LensDriverErrorCode.java | 36
.../server/query/QueryAPIErrorResponseTest.java | 24 ++---
.../lens/server/query/TestQueryService.java | 4 +--
9 files changed, 95 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/fdf04be0/lens-api/src/main/resources/lens-errors.conf
--
diff --git a/lens-api/src/main/resources/lens-errors.conf
b/lens-api/src/main/resources/lens-errors.conf
index 06960a0..c7ccea1 100644
--- a/lens-api/src/main/resources/lens-errors.conf
+++ b/lens-api/src/main/resources/lens-errors.conf
@@ -317,7 +317,7 @@ lensCubeErrorsForMetastore = [
]
-lensHiveErrors = [
+lensDriverErrors = [
{
errorCode = 4001
httpStatusCode = ${BAD_REQUEST}
@@ -327,7 +327,7 @@ lensHiveErrors = [
{
errorCode = 4002
httpStatusCode = ${INTERNAL_SERVER_ERROR}
-errorMsg = "Hive Error : %s"
+errorMsg = "Driver Execution Error : %s"
}
]
@@ -364,4 +364,4 @@ lensCubeErrors =
${lensCubeErrorsForQuery}${lensCubeErrorsForMetastore}
# Lens server and Lens client are only aware of errors array. They are not
aware of any other array defined in
# error configuration files. Hence an errors array is prepared which is a
concatenation of all other error arrays.
-errors =
${lensCommonErrors}${lensServerErrors}${lensCubeErrors}${lensHiveErrors}
+errors =
${lensCommonErrors}${lensServerErrors}${lensCubeErrors}${lensDriverErrors}
http://git-wip-us.apache.org/repos/asf/lens/blob/fdf04be0/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
--
diff --git
a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index c7ef8f1..149c6ab 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -18,7 +18,7 @@
*/
package org.apache.lens.driver.hive;
-import static org.apache.lens.driver.hive.LensHiveErrorCode.*;
+import static org.apache.lens.server.api.error.LensDriverErrorCode.*;
import static org.apache.lens.server.api.util.LensUtil.getImplementations;
import java.io.ByteArrayInputStream;
@@ -579,7 +579,7 @@ public class HiveDriver extends AbstractLensDriver {
if (ex.getMessage().contains("SemanticException")) {
throw new LensException(SEMANTIC_ERROR.getLensErrorInfo(), ex,
ex.getMessage());
}
-throw new LensException(HIVE_ERROR.getLensErrorInfo(), ex,
ex.getMessage());
+throw new LensException(DRIVER_ERROR.getLensErrorInfo(), ex,
ex.getMessage());
}
/*
http://git-wip-us.apache.org/repos/asf/lens/blob/fdf04be0/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/LensHiveErrorCode.java
--
diff --git
a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/LensHiveErrorCode.java
b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/LensHiveErrorCode.java
deleted file mode 100644
index 3bac9e7..000
---
a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/LensHiveErrorCode.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses
