[3/3] lens git commit: LENS-788 : Push expressions before flattening and convert filters to array_contains filters for bridge tables
LENS-788 : Push expressions before flattening and convert filters to
array_contains filters for bridge tables
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/dba885ca
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/dba885ca
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/dba885ca
Branch: refs/heads/master
Commit: dba885ca972e8a11281f1bc44ceb4b7dd3906ce0
Parents: 79b95f0
Author: Amareshwari Sriramadasu
Authored: Wed Mar 30 10:25:20 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed Mar 30 10:25:20 2016 +0530
--
.../apache/lens/cube/parse/CandidateFact.java | 51 +-
.../lens/cube/parse/CubeQueryConfUtil.java | 5 +
.../lens/cube/parse/CubeQueryContext.java | 21 +-
.../lens/cube/parse/DefaultAliasDecider.java| 13 +-
.../apache/lens/cube/parse/DefaultQueryAST.java | 5 +-
.../apache/lens/cube/parse/DimHQLContext.java | 29 +-
.../lens/cube/parse/DimOnlyHQLContext.java | 10 -
.../org/apache/lens/cube/parse/HQLParser.java | 49 +-
.../apache/lens/cube/parse/JoinResolver.java| 9 +-
.../lens/cube/parse/MultiFactHQLContext.java| 3 -
.../org/apache/lens/cube/parse/QueryAST.java| 6 +-
.../parse/SingleFactMultiStorageHQLContext.java | 8 -
.../SingleFactSingleStorageHQLContext.java | 12 -
.../lens/cube/parse/join/AutoJoinContext.java | 104 +---
.../cube/parse/join/BridgeTableJoinContext.java | 321 ++
.../src/main/resources/olap-query-conf.xml | 18 +
.../apache/lens/cube/parse/CubeTestSetup.java | 19 +-
.../lens/cube/parse/TestBridgeTableQueries.java | 618 +--
.../cube/parse/TestDefaultAliasDecider.java | 53 ++
.../apache/lens/cube/parse/TestHQLParser.java | 82 +++
.../org/apache/lens/cube/parse/TestQuery.java | 18 +-
.../cube/parse/join/TestBridgeTableJoinCtx.java | 122
lens-dist/src/main/assembly/bin-dist.xml| 16 +
.../lens/examples/PopulateSampleMetastore.java | 2 +
.../apache/lens/examples/SampleMetastore.java | 4 +
.../src/main/resources/cube-queries.sql | 6 +-
.../resources/customer-interests-local-part.xml | 30 +
.../resources/customer-interests-local.data | 6 +
.../src/main/resources/customer-interests.xml | 31 +
.../main/resources/customer_interests_table.xml | 52 ++
.../src/main/resources/db-storage-schema.sql| 18 +
.../src/main/resources/interests-local-part.xml | 29 +
.../src/main/resources/interests-local.data | 4 +
lens-examples/src/main/resources/interests.xml | 31 +
.../src/main/resources/interests_table.xml | 51 ++
lens-examples/src/main/resources/sales-cube.xml | 24 +
src/site/apt/user/olap-cube.apt | 8 +
src/site/apt/user/olap-query-conf.apt | 48 +-
tools/conf/client/lens-client-site.xml | 16 +-
39 files changed, 1678 insertions(+), 274 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
index 82ca4f4..3f724b6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
@@ -23,8 +23,6 @@ import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
import java.util.*;
import org.apache.lens.cube.metadata.*;
-import org.apache.lens.cube.parse.HQLParser.ASTNodeVisitor;
-import org.apache.lens.cube.parse.HQLParser.TreeNode;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
@@ -76,6 +74,8 @@ public class CandidateFact implements CandidateTable,
QueryAST {
@Getter
@Setter
private Integer limitValue;
+ @Getter
+ private String fromString;
private final List selectIndices = Lists.newArrayList();
private final List dimFieldIndices = Lists.newArrayList();
private Collection columns;
@@ -166,7 +166,7 @@ public class CandidateFact implements CandidateTable,
QueryAST {
}
public boolean isExpressionAnswerable(ASTNode node, CubeQueryContext
context) throws LensException {
-return getColumns().containsAll(getColsInExpr(context,
context.getCube().getAllFieldNames(), node));
+return
getColumns().containsAll(HQLParser.getColsInExpr(context.getAliasForTableName(context.getCube()),
node));
}
/**
@@ -182,7 +182,7 @@ public class CandidateFact implements CandidateTable,
QueryAST {
int currentChild = 0;
for (int i = 0; i < cubeql.getSelectAST().getChildCount(); i++) {
ASTNode selectExpr = (ASTN
[2/3] lens git commit: LENS-788 : Push expressions before flattening and convert filters to array_contains filters for bridge tables
http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
index e8ad8ff..e8905c1 100644
---
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
+++
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
@@ -48,14 +48,14 @@ public class TestBridgeTableQueries extends
TestQueryRewrite {
public void testBridgeTablesWithoutDimtablePartitioning() throws Exception {
String query = "select usersports.name, sum(msr2) from basecube where " +
TWO_DAYS_RANGE;
String hqlQuery = rewrite(query, hConf);
-String expected = getExpectedQuery("basecube", "select usersports.name,
sum(basecube.msr2) FROM ",
+String expected = getExpectedQuery("basecube", "select usersports.balias0,
sum(basecube.msr2) FROM ",
" join " + getDbName() + "c1_usertable userdim ON basecube.userid =
userdim.id "
-+ " join (select user_interests.user_id as
user_id,collect_set(usersports.name) as name"
++ " join (select user_interests.user_id as
user_id,collect_set(usersports.name) as balias0"
+ " from " + getDbName() + "c1_user_interests_tbl user_interests"
+ " join " + getDbName() + "c1_sports_tbl usersports on
user_interests.sport_id = usersports.id"
+ " group by user_interests.user_id) usersports"
+ " on userdim.id = usersports.user_id ",
- null, "group by usersports.name", null,
+ null, "group by usersports.balias0", null,
getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base"));
TestCubeRewriter.compareQueries(hqlQuery, expected);
// run with chain ref column
@@ -68,14 +68,14 @@ public class TestBridgeTableQueries extends
TestQueryRewrite {
public void testBridgeTablesForExprFieldWithoutDimtablePartitioning() throws
Exception {
String query = "select substr(usersports.name, 10), sum(msr2) from
basecube where " + TWO_DAYS_RANGE;
String hqlQuery = rewrite(query, hConf);
-String expected = getExpectedQuery("basecube", "select
substr(usersports.name, 10), sum(basecube.msr2) FROM ",
+String expected = getExpectedQuery("basecube", "select usersports.balias0,
sum(basecube.msr2) FROM ",
" join " + getDbName() + "c1_usertable userdim ON basecube.userid =
userdim.id "
-+ " join (select user_interests.user_id as
user_id,collect_set(usersports.name) as name"
++ " join (select user_interests.user_id as
user_id,collect_set(substr(usersports.name, 10)) as balias0"
+ " from " + getDbName() + "c1_user_interests_tbl user_interests"
+ " join " + getDbName() + "c1_sports_tbl usersports on
user_interests.sport_id = usersports.id"
+ " group by user_interests.user_id) usersports"
+ " on userdim.id = usersports.user_id ",
- null, "group by substr(( usersports . name ), 10 )", null,
+ null, "group by usersports.balias0", null,
getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base"));
TestCubeRewriter.compareQueries(hqlQuery, expected);
// run with chain ref column
@@ -109,14 +109,14 @@ public class TestBridgeTableQueries extends
TestQueryRewrite {
conf.set(CubeQueryConfUtil.BRIDGE_TABLE_FIELD_AGGREGATOR, "custom_aggr");
String query = "select usersports.name, sum(msr2) from basecube where " +
TWO_DAYS_RANGE;
String hqlQuery = rewrite(query, conf);
-String expected = getExpectedQuery("basecube", "select usersports.name,
sum(basecube.msr2) FROM ",
+String expected = getExpectedQuery("basecube", "select usersports.balias0,
sum(basecube.msr2) FROM ",
" join " + getDbName() + "c1_usertable userdim ON basecube.userid =
userdim.id "
-+ " join (select user_interests.user_id as
user_id,custom_aggr(usersports.name) as name"
++ " join (select user_interests.user_id as
user_id,custom_aggr(usersports.name) as balias0"
+ " from " + getDbName() + "c1_user_interests_tbl user_interests"
+ " join " + getDbName() + "c1_sports_tbl usersports on
user_interests.sport_id = usersports.id"
+ " group by user_interests.user_id) usersports"
+ " on userdim.id = usersports.user_id ",
- null, "group by usersports.name", null,
+ null, "group by usersports.balias0", null,
getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base"));
TestCubeRewriter.compareQueries(hqlQuery, expected);
// run with chain ref column
@@ -129,16 +129,17 @@ public class TestBridgeTableQueries extends
TestQueryRewrite {
public void testBridgeTablesWithMegringChains() throws Exception {
String query = "select userInterestIds.sport_id, usersports.name,
sum(msr2) from basecube where
lens git commit: LENS-918 : Fix setting log seggregation id for rewriteAndSelect
Repository: lens
Updated Branches:
refs/heads/master 09fc9d1a3 -> 33dfee1d6
LENS-918 : Fix setting log seggregation id for rewriteAndSelect
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/33dfee1d
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/33dfee1d
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/33dfee1d
Branch: refs/heads/master
Commit: 33dfee1d6c50b7f90ff835b99c1d4633f6bb9032
Parents: 09fc9d1
Author: Ajay Yadava
Authored: Tue Apr 12 18:03:32 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Apr 12 18:03:32 2016 +0530
--
.../org/apache/lens/server/query/QueryExecutionServiceImpl.java | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/33dfee1d/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index ad39028..42bd4ab 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -1334,6 +1334,7 @@ public class QueryExecutionServiceImpl extends
BaseLensService implements QueryE
* @throws LensException the lens exception
*/
private void rewriteAndSelect(final AbstractQueryContext ctx) throws
LensException {
+logSegregationContext.setLogSegragationAndQueryId(ctx.getLogHandle());
MethodMetricsContext parallelCallGauge =
MethodMetricsFactory.createMethodGauge(ctx.getConf(), false,
PARALLEL_CALL_GAUGE);
try {
lens git commit: LENS-529 : Add doc links to how to add new error codes and new driver
Repository: lens
Updated Branches:
refs/heads/master e269651cf -> 2cf0b51fd
LENS-529 : Add doc links to how to add new error codes and new driver
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2cf0b51f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2cf0b51f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2cf0b51f
Branch: refs/heads/master
Commit: 2cf0b51fd260a559e7d0623c27d579b241fc12da
Parents: e269651
Author: Ajay Yadava
Authored: Thu Apr 14 15:30:03 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Apr 14 15:30:03 2016 +0530
--
src/site/apt/developer/contribute.apt | 9 +
1 file changed, 9 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/2cf0b51f/src/site/apt/developer/contribute.apt
--
diff --git a/src/site/apt/developer/contribute.apt
b/src/site/apt/developer/contribute.apt
index 38c6818..d1fea00 100644
--- a/src/site/apt/developer/contribute.apt
+++ b/src/site/apt/developer/contribute.apt
@@ -695,3 +695,12 @@ Developer Documentation : How to contribute to Apache Lens?
There can be feature branches created from <>, if feature is not
actively developed in <> branch
directly. For a feature branch to be created a contirbutor can start discuss
thread on dev list for consensus on
whether it is required.
+
+** How to add a new Driver in Lens?
+ Please refer to
{{{https://cwiki.apache.org/confluence/display/LENS/Lens+Driver} this wiki
article}}.
+ For examples you can refer to the following review board requests
+ 1. {{{https://reviews.apache.org/r/43649/} Druid driver in Lens}}
+ 2. {{{ https://reviews.apache.org/r/36434/} Elastic Search Driver}}
+
+** How to add a new Error Code in Lens?
+ Please refer to
{{{https://cwiki.apache.org/confluence/display/LENS/How+to+add+new+Error+Code}
this wiki article}}
lens git commit: LENS-906 : Reduce test time for TestCubeMetastoreClient
Repository: lens
Updated Branches:
refs/heads/master f21130992 -> 6e11d780f
LENS-906 : Reduce test time for TestCubeMetastoreClient
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/6e11d780
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/6e11d780
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/6e11d780
Branch: refs/heads/master
Commit: 6e11d780f8ede78745f5247d983c0f63f42fb41d
Parents: f211309
Author: Amareshwari Sriramadasu
Authored: Wed Apr 27 08:45:30 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed Apr 27 08:45:30 2016 +0530
--
.../cube/metadata/TestCubeMetastoreClient.java | 73 ++--
1 file changed, 69 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/6e11d780/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index 0773005..02571ee 100644
---
a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++
b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -72,6 +72,10 @@ public class TestCubeMetastoreClient {
private static Set dimensions;
private static Set cubeMeasures;
private static Set cubeDimensions;
+ private static Set moreMeasures = Sets.newHashSet();
+ private static Set moreDimensions = Sets.newHashSet();
+ private static Set moreCubeMeasures = Sets.newHashSet();
+ private static Set moreCubeDimensions = Sets.newHashSet();
private static Set hourlyAndDaily = Sets.newHashSet(HOURLY,
DAILY);
private static final String CUBE_NAME = "testMetastoreCube";
private static final String CUBE_NAME_WITH_PROPS =
"testMetastoreCubeWithProps";
@@ -340,16 +344,20 @@ public class TestCubeMetastoreClient {
});
cubeDimensions.add(new ReferencedDimAttribute(new
FieldSchema("zipcityname", "string", "zip city name"),
"Zip city name", "cityFromZip", "name", null, null, null));
-cubeMeasures.addAll(dummyMeasure);
-cubeDimensions.addAll(dummyDimAttributes);
+moreCubeMeasures.addAll(cubeMeasures);
+moreCubeMeasures.addAll(dummyMeasure);
+moreCubeDimensions.addAll(cubeDimensions);
+moreCubeDimensions.addAll(dummyDimAttributes);
cube = new Cube(cubeName, cubeMeasures, cubeDimensions, cubeExpressions,
joinChains, emptyHashMap, 0.0);
measures = Sets.newHashSet("msr1", "msr2", "msr3");
+moreMeasures.addAll(measures);
for(CubeMeasure measure: dummyMeasure) {
- measures.add(measure.getName());
+ moreMeasures.add(measure.getName());
}
dimensions = Sets.newHashSet("dim1", "dim2", "dim3");
+moreDimensions.addAll(dimensions);
for(CubeDimAttribute dimAttribute: dummyDimAttributes) {
- dimensions.add(dimAttribute.getName());
+ moreDimensions.add(dimAttribute.getName());
}
derivedCube = new DerivedCube(derivedCubeName, measures, dimensions, cube);
@@ -784,6 +792,63 @@ public class TestCubeMetastoreClient {
}
@Test(priority = 1)
+ public void testCubeWithMoreMeasures() throws Exception {
+String cubeName = "cubeWithMoreMeasures";
+Cube cube = new Cube(cubeName, moreCubeMeasures, moreCubeDimensions,
cubeExpressions, joinChains, emptyHashMap,
+ 0.0);
+client.createCube(cubeName, moreCubeMeasures, moreCubeDimensions,
cubeExpressions, joinChains, emptyHashMap);
+assertTrue(client.tableExists(cubeName));
+Table cubeTbl = client.getHiveTable(cubeName);
+assertTrue(client.isCube(cubeTbl));
+Cube cube2 = new Cube(cubeTbl);
+assertTrue(cube.equals(cube2));
+assertFalse(cube2.isDerivedCube());
+assertTrue(cube2.getTimedDimensions().isEmpty());
+assertEquals(moreCubeMeasures.size(), cube2.getMeasureNames().size());
+// +8 is for hierarchical dimension
+assertEquals(moreCubeDimensions.size() + 8,
cube2.getDimAttributeNames().size());
+assertEquals(moreCubeMeasures.size(), cube2.getMeasures().size());
+assertEquals(cubeExpressions.size(), cube2.getExpressions().size());
+assertEquals(cubeExpressions.size(), cube2.getExpressionNames().size());
+assertEquals(moreCubeDimensions.size(), cube2.getDimAttributes().size());
+assertEquals(moreCubeDimensions.size() + 8 + moreCubeMeasures.size() +
cubeExpressions.size(), cube2
+ .getAllFieldNames().size());
+assertNotNull(cube2.getMeasureByName("msr4"));
+assertN
lens git commit: LENS-730 : Fix issues with column aliases in multifact query
Repository: lens
Updated Branches:
refs/heads/master 6e11d780f -> 3b3f42da5
LENS-730 : Fix issues with column aliases in multifact query
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3b3f42da
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3b3f42da
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3b3f42da
Branch: refs/heads/master
Commit: 3b3f42da51808e1bb0d23357e1bfaecaca978896
Parents: 6e11d78
Author: Amareshwari Sriramadasu
Authored: Wed Apr 27 09:52:06 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed Apr 27 09:52:06 2016 +0530
--
.../lens/cube/parse/AggregateResolver.java | 4 +-
.../apache/lens/cube/parse/ColumnResolver.java | 44 +++--
.../lens/cube/parse/ExpressionResolver.java | 2 +-
.../apache/lens/cube/parse/CubeTestSetup.java | 5 +
.../lens/cube/parse/TestBaseCubeQueries.java| 165 ---
.../lens/cube/parse/TestBridgeTableQueries.java | 46 +++---
.../lens/cube/parse/TestCubeRewriter.java | 10 +-
.../lens/cube/parse/TestExpressionResolver.java | 12 +-
8 files changed, 210 insertions(+), 78 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/3b3f42da/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
index fd7036a..1593a54 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
@@ -170,8 +170,8 @@ class AggregateResolver implements ContextRewriter {
ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL,
Identifier);
ASTNode colIdent = (ASTNode) node.getChild(1);
- colname = colIdent.getText();
- tabname = tabident.getText();
+ colname = colIdent.getText().toLowerCase();
+ tabname = tabident.getText().toLowerCase();
}
String msrname = StringUtils.isBlank(tabname) ? colname : tabname + "." +
colname;
http://git-wip-us.apache.org/repos/asf/lens/blob/3b3f42da/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
index 75aa3f4..58a1d4d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
@@ -61,10 +61,10 @@ class ColumnResolver implements ContextRewriter {
}
getColsForSelectTree(cubeql);
getColsForWhereTree(cubeql);
-getColsForTree(cubeql, cubeql.getJoinAST(), cubeql);
-getColsForTree(cubeql, cubeql.getGroupByAST(), cubeql);
-getColsForTree(cubeql, cubeql.getHavingAST(), cubeql);
-getColsForTree(cubeql, cubeql.getOrderByAST(), cubeql);
+getColsForTree(cubeql, cubeql.getJoinAST(), cubeql, true);
+getColsForTree(cubeql, cubeql.getGroupByAST(), cubeql, true);
+getColsForTree(cubeql, cubeql.getHavingAST(), cubeql, true);
+getColsForTree(cubeql, cubeql.getOrderByAST(), cubeql, true);
// Update join dimension tables
for (String table : cubeql.getTblAliasToColumns().keySet()) {
@@ -77,7 +77,8 @@ class ColumnResolver implements ContextRewriter {
}
// finds columns in AST passed.
- static void getColsForTree(final CubeQueryContext cubeql, ASTNode tree,
final TrackQueriedColumns tqc)
+ static void getColsForTree(final CubeQueryContext cubeql, ASTNode tree,
final TrackQueriedColumns tqc,
+final boolean skipAliases)
throws LensException {
if (tree == null) {
return;
@@ -98,7 +99,7 @@ class ColumnResolver implements ContextRewriter {
// Take child ident.totext
ASTNode ident = (ASTNode) node.getChild(0);
String column = ident.getText().toLowerCase();
- if (cubeql.getExprToAliasMap().values().contains(column)) {
+ if (skipAliases &&
cubeql.getExprToAliasMap().values().contains(column)) {
// column is an existing alias
return;
}
@@ -137,18 +138,12 @@ class ColumnResolver implements ContextRewriter {
// Updates alias for each selected expression.
// Alias is updated as follows:
// Case 1: If select expression does not have an alias
- // ** And the expression has only one column queried, the column name is put
- // as
- // select alias.
- // ** If the expression has more than one column queried, the alias is
- // constructed as
-
lens git commit: LENS-1022 : Reduce time for TestLensQueryCommands
Repository: lens
Updated Branches:
refs/heads/master e26ee5a71 -> d1c58cbcf
LENS-1022 : Reduce time for TestLensQueryCommands
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d1c58cbc
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d1c58cbc
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d1c58cbc
Branch: refs/heads/master
Commit: d1c58cbcf755321ae72d0fb1d9e9994e95f9f189
Parents: e26ee5a
Author: Amareshwari Sriramadasu
Authored: Wed Apr 27 18:18:14 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed Apr 27 18:18:14 2016 +0530
--
.../org/apache/lens/cli/TestLensQueryCommands.java | 16
.../lens/server/api/query/FinishedLensQuery.java| 2 +-
.../lens/server/query/TestResultFormatting.java | 4 +++-
lens-server/src/test/resources/lens-site.xml| 2 +-
4 files changed, 13 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/d1c58cbc/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
--
diff --git
a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
index dbb5e9d..1dd2ffe 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
@@ -175,10 +175,10 @@ public class TestLensQueryCommands extends
LensCliApplicationTest {
int chkCount = 0;
while (!qCom.getAllQueries("SUCCESSFUL", null, "all", null, -1,
Long.MAX_VALUE).contains(
"Total number of queries: " + (noOfSuccQueriesSoFar + 1))) {
- Thread.sleep(2000);
+ Thread.sleep(100);
chkCount++;
- if (chkCount > 30) {
-fail("Unable to get successful status for query even after 30 checks");
+ if (chkCount > 100) {
+fail("Unable to get successful status for query even after 100
checks");
}
}
if (closeConn) {
@@ -215,7 +215,7 @@ public class TestLensQueryCommands extends
LensCliApplicationTest {
String handle = qCom.executePreparedQuery(qh, true, "testPrepQuery2");
log.debug("Perpared query handle is " + handle);
while (!qCom.getClient().getQueryStatus(handle).finished()) {
- Thread.sleep(5000);
+ Thread.sleep(100);
}
String status = qCom.getStatus(handle);
log.debug("Prepared Query Status is " + status);
@@ -225,8 +225,8 @@ public class TestLensQueryCommands extends
LensCliApplicationTest {
result = qCom.getQueryResults(handle, null, true);
log.debug("Prepared Query Result is " + result);
assertTrue(result.contains("1\tfirst"));
-//Wait for query to purge. Purger runs every second
-Thread.sleep(3000);
+//Wait for query to purge. Purger runs every 100 millis
+Thread.sleep(300);
//Fetch again. Should not get resultset
result = qCom.getQueryResults(handle, null, true);
log.debug("Prepared Query Result is " + result);
@@ -335,7 +335,7 @@ public class TestLensQueryCommands extends
LensCliApplicationTest {
String details = qCom.getDetails(null);
assertTrue(details.contains("Driver Query:"));
}
- Thread.sleep(1000);
+ Thread.sleep(100);
queryStatus = qCom.getClient().getQueryStatus(qh);
}
@@ -521,7 +521,7 @@ public class TestLensQueryCommands extends
LensCliApplicationTest {
String query = "cube select id,name from test_dim";
String qh = qCom.executeQuery(query, true, "testQuery");
while (!qCom.getClient().getQueryStatus(qh).finished()) {
- Thread.sleep(5000);
+ Thread.sleep(100);
}
assertTrue(qCom.getStatus(qh).contains("Status: SUCCESSFUL"));
http://git-wip-us.apache.org/repos/asf/lens/blob/d1c58cbc/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
index e1ce415..1d56757 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
@@ -225,7 +225,7 @@ public class FinishedLensQuery {
qctx.setLaunchTime(this.startTime);
qctx.setEndTime(getEndTime());
qctx.setStatusSkippingTransitionTest(new QueryStatus(0.0, null,
QueryStatus.Status.valu
lens git commit: LENS-1008 : Support simple iterator over HTTPResultSet for lens clients
Repository: lens
Updated Branches:
refs/heads/master d1c58cbcf -> 6995962fd
LENS-1008 : Support simple iterator over HTTPResultSet for lens clients
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/6995962f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/6995962f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/6995962f
Branch: refs/heads/master
Commit: 6995962fd9386f54fa69f5a3598714bdf439ee91
Parents: d1c58cb
Author: Puneet Gupta
Authored: Wed Apr 27 19:05:13 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed Apr 27 19:05:13 2016 +0530
--
lens-client/pom.xml | 44 +
.../java/org/apache/lens/client/LensClient.java | 59 ++
.../org/apache/lens/client/LensConnection.java | 15 ++
.../exceptions/LensClientIOException.java | 33
.../client/resultset/AbstractResultSet.java | 102 +++
.../lens/client/resultset/CsvResultSet.java | 38
.../client/resultset/CsvResultSetReader.java| 57 ++
.../apache/lens/client/resultset/ResultSet.java | 46 +
.../lens/client/resultset/ResultSetReader.java | 36
.../client/resultset/ZippedCsvResultSet.java| 38
.../resultset/ZippedCsvResultSetReader.java | 37
.../client/resultset/ZippedResultSetReader.java | 92 ++
.../org/apache/lens/client/TestLensClient.java | 178 +--
.../src/test/resources/dim2-part/data.data | 3 +
lens-client/src/test/resources/dim_table.xml| 48 +
lens-client/src/test/resources/fact1.xml| 51 ++
.../src/test/resources/lens-client-site.xml | 11 ++
.../src/test/resources/local-storage.xml| 27 +++
lens-client/src/test/resources/sample-cube.xml | 76
lens-client/src/test/resources/test-detail.xml | 32
.../src/test/resources/test-dimension.xml | 61 +++
21 files changed, 1070 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/6995962f/lens-client/pom.xml
--
diff --git a/lens-client/pom.xml b/lens-client/pom.xml
index 4fd01fb..2032cf5 100644
--- a/lens-client/pom.xml
+++ b/lens-client/pom.xml
@@ -27,6 +27,10 @@
2.6.0-beta-SNAPSHOT
+
+${pom.basedir}/target/classpath
+
+
lens-client
jar
Lens client
@@ -142,5 +146,45 @@
org.slf4j
jcl-over-slf4j
+
+ net.sf.opencsv
+ opencsv
+
+
+
+
+
+org.apache.maven.plugins
+maven-dependency-plugin
+
+
+build-classpath
+compile
+
+ build-classpath
+
+
+ ${mvn.classpath.file}
+
+
+
+
+
+
+org.apache.maven.plugins
+maven-surefire-plugin
+
+
+
${pom.basedir}/../lens-driver-hive/testutils/hadoop
+
+
+${mvn.classpath.file}
+
+ -Xms256m -Xmx512m -XX:PermSize=256m
-XX:MaxPermSize=256m
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/lens/blob/6995962f/lens-client/src/main/java/org/apache/lens/client/LensClient.java
--
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index 9626820..43da691 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -18,9 +18,12 @@
*/
package org.apache.lens.client;
+import java.io.InputStream;
+import java.nio.charset.Charset;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import javax.ws.rs.core.Response;
@@ -31,9 +34,13 @@ import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.api.util.PathValidator;
import org.apache.lens.client.exceptions.LensAPIException;
import org.apache.lens.client.exceptions.LensBriefErrorException;
+import org.apache.lens.client.exceptions.LensClientIOException;
import org.apache.lens.client.model.BriefError;
import org.apache.lens.client.model.IdBriefErrorTemplate;
import org.apache.lens.client.model.IdBriefErrorTemplateKey;
+import org.apache.lens.client.resultset.CsvResultSet;
+import org.apache.lens.client.resultset.ResultSet;
+import org.apache.lens.client.resultset.ZippedCsvResultSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,6 +71,14 @@ public class LensClient {
@Getter
private PathValidator pathValidator;
+ public static final String QUERY_RESULT_SPLIT_INTO_MULTI
lens git commit: LENS-1013 : Add postDriverSelection to DriverQueryHook
Repository: lens
Updated Branches:
refs/heads/master 6995962fd -> 9b7541bcb
LENS-1013 : Add postDriverSelection to DriverQueryHook
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/9b7541bc
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/9b7541bc
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/9b7541bc
Branch: refs/heads/master
Commit: 9b7541bcbcc33faf3cbde71452f334783795ea27
Parents: 6995962
Author: Puneet Gupta
Authored: Thu Apr 28 09:36:38 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Apr 28 09:36:38 2016 +0530
--
.../org/apache/lens/driver/hive/HiveDriver.java | 6 ++-
.../apache/lens/driver/hive/TestHiveDriver.java | 3 --
.../lens/driver/hive/TestRemoteHiveDriver.java | 3 --
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 6 ++-
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 3 --
.../server/api/driver/AbstractLensDriver.java | 7 +++
.../lens/server/api/driver/DriverQueryHook.java | 39 --
.../lens/server/api/driver/LensDriver.java | 6 +++
.../server/api/driver/NoOpDriverQueryHook.java | 14 +++--
.../server/api/user/MockDriverQueryHook.java| 34 +---
.../server/query/QueryExecutionServiceImpl.java | 18 ---
.../org/apache/lens/server/LensJerseyTest.java | 11 +++-
.../apache/lens/server/TestServerRestart.java | 55 +---
.../lens/server/query/TestQueryService.java | 7 ++-
.../drivers/hive/hive1/hivedriver-site.xml | 7 +++
.../drivers/hive/hive2/hivedriver-site.xml | 7 +++
16 files changed, 176 insertions(+), 50 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
--
diff --git
a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 6eec3f7..19a010e 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -559,7 +559,6 @@ public class HiveDriver extends AbstractLensDriver {
Configuration qdconf = ctx.getDriverConf(this);
qdconf.set("mapred.job.name", ctx.getQueryHandle().toString());
decidePriority(ctx);
- queryHook.preLaunch(ctx);
SessionHandle sessionHandle = getSession(ctx);
OperationHandle op = getClient().executeStatementAsync(sessionHandle,
ctx.getSelectedDriverQuery(),
qdconf.getValByRegex(".*"));
@@ -1370,4 +1369,9 @@ public class HiveDriver extends AbstractLensDriver {
public boolean hasLensSession(LensSessionHandle session) {
return lensToHiveSession.containsKey(session.getPublicId().toString());
}
+
+ @Override
+ public DriverQueryHook getQueryHook() {
+return queryHook;
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/9b7541bc/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
--
diff --git
a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index 3f77b0c..cf51909 100644
---
a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++
b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -39,7 +39,6 @@ import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.query.priority.CostRangePriorityDecider;
import org.apache.lens.server.api.query.priority.CostToPriorityRangeConf;
-import org.apache.lens.server.api.user.MockDriverQueryHook;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.hadoop.conf.Configuration;
@@ -121,7 +120,6 @@ public class TestHiveDriver {
conf = new HiveConf();
conf.addResource("drivers/hive/hive1/hivedriver-site.xml");
conf.setClass(HiveDriver.HIVE_CONNECTION_CLASS,
EmbeddedThriftConnection.class, ThriftConnection.class);
-conf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class,
DriverQueryHook.class);
conf.set("hive.lock.manager",
"org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
driver = new HiveDriver();
@@ -484,7 +482,6 @@ public class TestHiveDriver {
*/
protected void validateExecuteAsync(QueryContext ctx, DriverQueryState
finalState, boolean isPersistent,
boolean formatNulls) throws Exception {
-assertEquals(ct
lens git commit: LENS-383 :Add connection and read timeouts on rest api calls in lens client
Repository: lens
Updated Branches:
refs/heads/master 9b7541bcb -> dce5a812d
LENS-383 :Add connection and read timeouts on rest api calls in lens client
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/dce5a812
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/dce5a812
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/dce5a812
Branch: refs/heads/master
Commit: dce5a812ddef01800bd83e229d6ccbad4ddee18a
Parents: 9b7541b
Author: Puneet Gupta
Authored: Tue May 3 10:06:28 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue May 3 10:06:28 2016 +0530
--
.../apache/lens/client/LensClientConfig.java| 8 ++
.../org/apache/lens/client/LensConnection.java | 13 ++-
.../org/apache/lens/client/LensStatement.java | 41 +++--
.../src/main/resources/lens-client-default.xml | 12 +++
.../org/apache/lens/client/TestLensClient.java | 95 +---
.../server/MockQueryExecutionServiceImpl.java | 74 +++
src/site/apt/admin/config.apt | 4 +-
src/site/apt/user/client-config.apt | 16 ++--
8 files changed, 239 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/dce5a812/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java
--
diff --git
a/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java
b/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java
index 6a35a5e..b703e13 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java
@@ -82,6 +82,14 @@ public class LensClientConfig extends Configuration {
public static final String WS_FILTER_IMPL_SFX = ".ws.filter.impl";
+ public static final String READ_TIMEOUT_MILLIS = CLIENT_PFX +
"read.timeout.millis";
+
+ public static final int DEFAULT_READ_TIMEOUT_MILLIS = 30; //5 mins
+
+ public static final String CONNECTION_TIMEOUT_MILLIS = CLIENT_PFX +
"connection.timeout.millis";
+
+ public static final int DEFAULT_CONNECTION_TIMEOUT_MILLIS = 6; //60 secs
+
/**
* Get the username from config
*
http://git-wip-us.apache.org/repos/asf/lens/blob/dce5a812/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
--
diff --git
a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
index 0c2557c..4951866 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
@@ -18,6 +18,8 @@
*/
package org.apache.lens.client;
+import static org.apache.lens.client.LensClientConfig.*;
+
import java.net.ConnectException;
import java.util.HashMap;
import java.util.Iterator;
@@ -39,6 +41,7 @@ import org.apache.lens.api.StringList;
import org.apache.lens.api.util.MoxyJsonConfigurationContextResolver;
import org.apache.lens.client.exceptions.LensClientServerConnectionException;
+import org.glassfish.jersey.client.ClientProperties;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
@@ -120,7 +123,15 @@ public class LensConnection {
while (itr.hasNext()) {
cb.register(itr.next());
}
-return cb.build();
+Client client = cb.build();
+
+//Set Timeouts
+LensClientConfig config = params.getConf();
+client.property(ClientProperties.CONNECT_TIMEOUT,
config.getInt(CONNECTION_TIMEOUT_MILLIS,
+ DEFAULT_CONNECTION_TIMEOUT_MILLIS));
+client.property(ClientProperties.READ_TIMEOUT,
config.getInt(READ_TIMEOUT_MILLIS, DEFAULT_READ_TIMEOUT_MILLIS));
+
+return client;
}
private WebTarget getSessionWebTarget() {
http://git-wip-us.apache.org/repos/asf/lens/blob/dce5a812/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
--
diff --git
a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
index f06bcd1..3cae717 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.client;
+import java.net.SocketTimeoutException;
import java.util.List;
import javax.ws.rs.client.Client;
@@ -228,16 +229,20 @@ public class LensStatement {
lens git commit: LENS-1033 : Fix HQLParser to honor double spaces in literals
Repository: lens
Updated Branches:
refs/heads/master 34223e880 -> 4e822031f
LENS-1033 : Fix HQLParser to honor double spaces in literals
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/4e822031
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/4e822031
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/4e822031
Branch: refs/heads/master
Commit: 4e822031f4a3d75b45281ec5aadaf8f64cbb7542
Parents: 34223e8
Author: Amareshwari Sriramadasu
Authored: Mon May 9 10:08:31 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Mon May 9 10:08:31 2016 +0530
--
.../org/apache/lens/cube/parse/HQLParser.java | 36
.../lens/cube/parse/TestBaseCubeQueries.java| 36
.../lens/cube/parse/TestBridgeTableQueries.java | 8 +-
.../lens/cube/parse/TestCubeRewriter.java | 24 +++---
.../lens/cube/parse/TestExpressionContext.java | 64 +++---
.../apache/lens/cube/parse/TestHQLParser.java | 87
.../cube/parse/join/TestBridgeTableJoinCtx.java | 42 +-
.../driver/jdbc/TestColumnarSQLRewriter.java| 30 +++
8 files changed, 170 insertions(+), 157 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/4e822031/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
--
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
index 0d00188..1f6b66c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
@@ -372,7 +372,7 @@ public final class HQLParser {
|| KW_FALSE == rootType || KW_FORMATTED == rootType || KW_EXTENDED ==
rootType || KW_DEPENDENCY == rootType) {
// StringLiterals should not be lower cased.
if (StringLiteral == rootType) {
-buf.append(' ').append(rootText).append(' ');
+buf.append(rootText);
} else if (KW_TRUE == rootType) {
buf.append(" true ");
} else if (KW_FALSE == rootType) {
@@ -380,12 +380,9 @@ public final class HQLParser {
} else if (Identifier == rootType && TOK_SELEXPR ==
root.getParent().getType()) {
// back quote column alias in all cases. This is required since some
alias values can match DB keywords
// (example : year as alias) and in such case queries can fail on
certain DBs if the alias in not back quoted
-buf.append(" as `").append(rootText).append("` ");
- } else if (Identifier == rootType && TOK_FUNCTIONSTAR ==
root.getParent().getType()) {
-// count(*) or count(someTab.*): Don't append space after the
identifier
-buf.append(" ").append(rootText == null ? "" : rootText.toLowerCase());
+buf.append(" as `").append(rootText).append("`");
} else {
-buf.append(" ").append(rootText == null ? "" :
rootText.toLowerCase()).append(" ");
+buf.append(rootText == null ? "" : rootText.toLowerCase());
}
} else if (TOK_ALLCOLREF == rootType) {
@@ -395,14 +392,14 @@ public final class HQLParser {
}
buf.append(".");
}
- buf.append(" * ");
+ buf.append("*");
} else if (TOK_FUNCTIONSTAR == rootType) {
if (root.getChildCount() > 0) {
for (int i = 0; i < root.getChildCount(); i++) {
toInfixString((ASTNode) root.getChild(i), buf);
}
}
- buf.append("(*) ");
+ buf.append("(*)");
} else if (UNARY_OPERATORS.contains(rootType)) {
if (KW_NOT == rootType) {
// Check if this is actually NOT IN
@@ -410,7 +407,7 @@ public final class HQLParser {
buf.append(" not ");
}
} else if (TILDE == rootType) {
-buf.append(" ~ ");
+buf.append(" ~");
}
for (int i = 0; i < root.getChildCount(); i++) {
@@ -429,7 +426,7 @@ public final class HQLParser {
if (MINUS == rootType && root.getChildCount() == 1) {
// If minus has only one child, then it's a unary operator.
// Add Operator name first
-buf.append(' ').append(rootText.toLowerCase()).append(' ');
+buf.append(rootText.toLowerCase());
// Operand
toInfixString((ASTNode) root.getChild(0), buf);
} else {
@@ -508,7 +505,7 @@ public final class HQLParser {
// Distinct is a different case.
String fname = root.getChild(0).getText();
-
lens git commit: LENS-1029 : Fix exception handling in lens server snapshotter
Repository: lens
Updated Branches:
refs/heads/master 4e822031f -> d264c9a44
LENS-1029 : Fix exception handling in lens server snapshotter
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d264c9a4
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d264c9a4
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d264c9a4
Branch: refs/heads/master
Commit: d264c9a44b3f0b889e87241a6eb9ec1051f7c6a5
Parents: 4e82203
Author: Puneet Gupta
Authored: Mon May 9 16:43:42 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Mon May 9 16:43:42 2016 +0530
--
.../lens/lib/query/AbstractFileFormatter.java | 12 +-
.../lens/server/api/LensConfConstants.java | 27 +--
.../org/apache/lens/server/LensServices.java| 237 ++-
.../src/main/resources/lensserver-default.xml | 20 +-
.../apache/lens/server/TestServerRestart.java | 16 +-
src/site/apt/admin/config.apt | 106 -
src/site/apt/admin/deployment.apt | 10 +-
7 files changed, 220 insertions(+), 208 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/d264c9a4/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
--
diff --git
a/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
b/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
index 8c06621..2cd6b3f 100644
---
a/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
+++
b/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
@@ -18,9 +18,7 @@
*/
package org.apache.lens.lib.query;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
+import java.io.*;
import org.apache.lens.server.api.driver.LensResultSetMetadata;
import org.apache.lens.server.api.query.QueryContext;
@@ -97,15 +95,15 @@ public abstract class AbstractFileFormatter extends
AbstractOutputFormatter impl
public void writeExternal(ObjectOutput out) throws IOException {
out.writeUTF(metadata.toJson());
out.writeUTF(finalPath.toString());
-out.writeInt(numRows);
-out.writeLong(fileSize);
+out.writeObject(numRows); // writeObject and readObject have handling for
null objects
+out.writeObject(fileSize);
}
@Override
public void readExternal(ObjectInput in) throws IOException,
ClassNotFoundException {
metadata = LensResultSetMetadata.fromJson(in.readUTF());
finalPath = new Path(in.readUTF());
-numRows = in.readInt();
-fileSize = in.readLong();
+numRows = (Integer)in.readObject();
+fileSize = (Long)in.readObject();
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/d264c9a4/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 23537cb..7cf6449 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -141,14 +141,14 @@ public final class LensConfConstants {
public static final String DEFAULT_SERVER_BASE_URL =
"http://0.0.0.0:/lensapi";;
/**
- * The Constant SERVER_RESTART_ENABLED.
+ * The Constant SERVER_STATE_PERSISTENCE_ENABLED.
*/
- public static final String SERVER_RESTART_ENABLED = SERVER_PFX +
"restart.enabled";
+ public static final String SERVER_STATE_PERSISTENCE_ENABLED = SERVER_PFX +
"state.persistence.enabled";
/**
- * The Constant DEFAULT_SERVER_RESTART_ENABLED.
+ * The Constant DEFAULT_SERVER_STATE_PERSISTENCE_ENABLED.
*/
- public static final boolean DEFAULT_SERVER_RESTART_ENABLED = true;
+ public static final boolean DEFAULT_SERVER_STATE_PERSISTENCE_ENABLED = true;
/**
* The Constant SERVER_STATE_PERSIST_LOCATION.
@@ -161,16 +161,6 @@ public final class LensConfConstants {
public static final String DEFAULT_SERVER_STATE_PERSIST_LOCATION =
"file:///tmp/lensserver";
/**
- * The Constant SERVER_RECOVER_ON_RESTART.
- */
- public static final String SERVER_RECOVER_ON_RESTART = SERVER_PFX +
"recover.onrestart";
-
- /**
- * The Constant DEFAULT_SERVER_RECOVER_ON_RESTART.
- */
- public static final boolean DEFAULT_SERVER_RECOVER_ON_RESTART = true;
-
- /**
* The Constant SESSION_TIMEOUT_SECONDS.
*/
public static final String SESSION_TIMEOUT_SECONDS = SERVER_PFX +
"session.
[2/3] lens git commit: LENS-518 : Move Hive dependency to Apache Hive master
http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
--
diff --git
a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index cf51909..daf01ac 100644
---
a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++
b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -39,13 +39,12 @@ import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.query.priority.CostRangePriorityDecider;
import org.apache.lens.server.api.query.priority.CostToPriorityRangeConf;
+import org.apache.lens.server.api.user.MockDriverQueryHook;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.ql.HiveDriverRunHook;
@@ -72,7 +71,9 @@ public class TestHiveDriver {
private final String testOutputDir = "target/" +
this.getClass().getSimpleName() + "/test-output";
/** The conf. */
- protected HiveConf conf;
+ protected Configuration driverConf = new Configuration();
+ protected HiveConf hiveConf = new HiveConf();
+ protected Configuration queryConf = new Configuration();
/** The driver. */
protected HiveDriver driver;
@@ -98,39 +99,37 @@ public class TestHiveDriver {
// Check if hadoop property set
System.out.println("###HADOOP_PATH " +
System.getProperty("hadoop.bin.path"));
assertNotNull(System.getProperty("hadoop.bin.path"));
-
createDriver();
-ss = new SessionState(conf, "testuser");
+ss = new SessionState(hiveConf, "testuser");
SessionState.start(ss);
-Hive client = Hive.get(conf);
+Hive client = Hive.get(hiveConf);
Database database = new Database();
database.setName(dataBase);
client.createDatabase(database, true);
SessionState.get().setCurrentDatabase(dataBase);
sessionid = SessionState.get().getSessionId();
-
-conf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false);
-QueryContext context = createContext("USE " + dataBase, conf);
+driverConf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false);
+QueryContext context = createContext("USE " + dataBase, this.queryConf);
driver.execute(context);
-conf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, true);
-conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
+driverConf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, true);
+driverConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER,
true);
}
protected void createDriver() throws LensException {
-conf = new HiveConf();
-conf.addResource("drivers/hive/hive1/hivedriver-site.xml");
-conf.setClass(HiveDriver.HIVE_CONNECTION_CLASS,
EmbeddedThriftConnection.class, ThriftConnection.class);
-conf.set("hive.lock.manager",
"org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
-conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
+driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml");
+driverConf.setClass(HiveDriver.HIVE_CONNECTION_CLASS,
EmbeddedThriftConnection.class, ThriftConnection.class);
+driverConf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS,
MockDriverQueryHook.class, DriverQueryHook.class);
+driverConf.set("hive.lock.manager",
"org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
+driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
driver = new HiveDriver();
-driver.configure(conf, "hive", "hive1");
+driver.configure(driverConf, "hive", "hive1");
drivers = Lists.newArrayList(driver);
System.out.println("TestHiveDriver created");
}
@BeforeMethod
public void setDB() {
-SessionState.get().setCurrentDatabase(dataBase);
+SessionState.setCurrentSessionState(ss);
}
protected QueryContext createContext(final String query, Configuration conf)
throws LensException {
@@ -167,8 +166,19 @@ public class TestHiveDriver {
*/
@AfterTest
public void afterTest() throws Exception {
+verifyThriftLogs();
driver.close();
-Hive.get(conf).dropDatabase(dataBase, true, true, true);
+Hive.get(hiveConf).dropDatabase(dataBase, true, true, true);
+ }
+
+ private void verifyThriftLogs() throws IOException {
+BufferedReader br = new BufferedReader(new FileReader(new
File("target/test.log")));
+for (String line = br.readLine(); line
[1/3] lens git commit: LENS-518 : Move Hive dependency to Apache Hive master
Repository: lens
Updated Branches:
refs/heads/master d264c9a44 -> d6b121692
http://git-wip-us.apache.org/repos/asf/lens/blob/d6b12169/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
b/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
index 1c0cd35..18c2f2c 100644
--- a/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
+++ b/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
@@ -107,9 +107,9 @@ public final class RewriteUtil {
int childCount = ast.getChildCount();
if (ast.getToken() != null) {
if (log.isDebugEnabled() && ast.getChild(0) != null) {
-log.debug("First child: {} Type:{}", ast.getChild(0), ((ASTNode)
ast.getChild(0)).getToken().getType());
+log.debug("First child: {} Type:{}", ast.getChild(0),
ast.getChild(0).getType());
}
- if (ast.getToken().getType() == HiveParser.TOK_QUERY
+ if (ast.getType() == HiveParser.TOK_QUERY
&& (isCubeKeywordNode((ASTNode) ast.getChild(0)) ||
isFromNodeWithCubeTable((ASTNode) ast.getChild(0), conf))) {
log.debug("Inside cube clause");
CubeQueryInfo cqi = new CubeQueryInfo();
@@ -120,34 +120,34 @@ public final class RewriteUtil {
cqi.startPos = ast.getCharPositionInLine();
} else {
ASTNode selectAST = (ASTNode) ast.getChild(1).getChild(1);
-// Left most child of select AST will have char position just
after select / select distinct
-// Go back one "select[ distinct]"
+// Left most child of select AST will have char position after
select with
+// no selects in between. search for select backward from there.
cqi.startPos = getStartPos(originalQuery,
HQLParser.leftMostChild(selectAST).getCharPositionInLine(),
- "distinct");
-cqi.startPos = getStartPos(originalQuery, cqi.startPos, "select");
+ "select");
}
int ci = ast.getChildIndex();
- if (parent.getToken() == null || parent.getToken().getType() ==
HiveParser.TOK_EXPLAIN
-|| parent.getToken().getType() == HiveParser.TOK_CREATETABLE) {
+ if (parent.getToken() == null || parent.getType() ==
HiveParser.TOK_EXPLAIN
+|| parent.getType() == HiveParser.TOK_CREATETABLE) {
// Not a sub query
cqi.endPos = originalQuery.length();
- } else if (parent.getChildCount() > ci + 1) {
-if (parent.getToken().getType() == HiveParser.TOK_SUBQUERY) {
+ } else if (parent.getChildCount() > ci + 1
+|| (parent.getParent() != null && parent.getType() ==
parent.getParent().getType())) {
+if (parent.getType() == HiveParser.TOK_SUBQUERY) {
// less for the next start and for close parenthesis
cqi.endPos = getEndPos(originalQuery, parent.getChild(ci +
1).getCharPositionInLine(), ")");
-} else if (parent.getToken().getType() == HiveParser.TOK_UNION) {
- // one less for the next start and less the size of string
'UNION ALL'
- ASTNode nextChild = (ASTNode) parent.getChild(ci + 1);
- if (isCubeKeywordNode((ASTNode) nextChild.getChild(0))) {
-cqi.endPos = getEndPos(originalQuery,
nextChild.getCharPositionInLine() - 1, "UNION ALL");
+} else if (parent.getType() == HiveParser.TOK_UNIONALL) {
+ ASTNode nextChild;
+ if (parent.getChildCount() > ci + 1) {
+// top level child
+nextChild = (ASTNode) parent.getChild(ci + 1);
} else {
-// Go back one "union all select[ distinct]"
-cqi.endPos = getEndPos(originalQuery,
nextChild.getChild(1).getChild(1).getCharPositionInLine() - 1,
- "distinct");
-cqi.endPos = getEndPos(originalQuery, cqi.endPos, "select");
-cqi.endPos = getEndPos(originalQuery, cqi.endPos, "union all");
+// middle child, it's left child's right child.
+nextChild = (ASTNode)
parent.getParent().getChild(parent.getChildIndex()+1);
}
-
+ // Go back one select
+ cqi.endPos = getStartPos(originalQuery,
nextChild.getChild(1).getChild(1).getCharPositionInLine() - 1,
+"select");
+ cqi.endPos = getEndPos(originalQuery, cqi.endPos, "union all");
} else {
// Not expected to reach here
log.warn("Unknown query pattern found with AST:{}", ast.dump());
@@ -159,8 +159,12 @@ public final class RewriteUtil {
// and one for the close parenthesis if there are no more unionall
// or one for the string 'UNION ALL' if there are
[3/3] lens git commit: LENS-518 : Move Hive dependency to Apache Hive master
LENS-518 : Move Hive dependency to Apache Hive master Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d6b12169 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d6b12169 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d6b12169 Branch: refs/heads/master Commit: d6b1216922f1438a7fc000dfcdd9121d87c65149 Parents: d264c9a Author: Rajat Khandelwal Authored: Mon May 9 16:45:32 2016 +0530 Committer: Amareshwari Sriramadasu Committed: Mon May 9 16:45:32 2016 +0530 -- lens-cli/pom.xml| 31 +++ .../apache/lens/cli/TestLensQueryCommands.java | 28 ++- lens-client/pom.xml | 21 ++ lens-cube/pom.xml | 27 ++- .../lens/cube/metadata/CubeMetastoreClient.java | 11 +- .../org/apache/lens/cube/metadata/Storage.java | 4 +- .../lens/cube/parse/CubeQueryContext.java | 7 +- .../lens/cube/parse/CubeSemanticAnalyzer.java | 7 +- .../org/apache/lens/cube/parse/HQLParser.java | 35 ++-- .../cube/metadata/TestCubeMetastoreClient.java | 19 +- .../lens/cube/parse/TestBaseCubeQueries.java| 34 ++-- .../apache/lens/cube/parse/TestHQLParser.java | 56 +++--- .../org/apache/lens/cube/parse/TestQuery.java | 9 +- ...stFactPartitionBasedQueryCostCalculator.java | 2 +- lens-cube/src/test/resources/hive-site.xml | 10 + lens-driver-es/pom.xml | 129 +++- .../client/jest/JestResultSetTransformer.java | 2 +- .../org/apache/lens/driver/es/ESDriverTest.java | 5 + .../org/apache/lens/driver/es/MockClientES.java | 2 +- .../driver/es/ResultSetTransformationTest.java | 2 +- lens-driver-es/src/test/resources/hive-site.xml | 53 + lens-driver-es/src/test/resources/logback.xml | 36 lens-driver-hive/pom.xml| 17 ++ .../driver/hive/EmbeddedThriftConnection.java | 200 ++- .../org/apache/lens/driver/hive/HiveDriver.java | 62 +++--- .../lens/driver/hive/HiveInMemoryResultSet.java | 2 +- .../apache/lens/driver/hive/HiveQueryPlan.java | 63 -- .../driver/hive/RemoteThriftConnection.java | 7 +- .../apache/lens/driver/hive/TestHiveDriver.java | 185 + .../lens/driver/hive/TestRemoteHiveDriver.java | 79 .../src/test/resources/hive-site.xml| 10 + lens-driver-jdbc/pom.xml| 24 +++ .../lens/driver/jdbc/ColumnarSQLRewriter.java | 2 +- .../apache/lens/driver/jdbc/JDBCResultSet.java | 6 +- .../driver/jdbc/TestColumnarSQLRewriter.java| 77 +-- .../src/test/resources/hive-site.xml| 15 ++ lens-examples/pom.xml | 1 - lens-ml-lib/pom.xml | 4 + .../java/org/apache/lens/ml/TestMLResource.java | 5 - .../java/org/apache/lens/ml/TestMLRunner.java | 5 - lens-query-lib/pom.xml | 8 + .../lens/lib/query/FilePersistentFormatter.java | 9 +- .../lib/query/MockLensResultSetMetadata.java| 41 ++-- lens-server-api/pom.xml | 12 ++ .../api/driver/LensResultSetMetadata.java | 4 +- .../save/param/ParameterDataTypeEncoder.java| 2 +- .../MappedDiagnosticLogSegregationContext.java | 7 + lens-server/pom.xml | 28 +++ .../org/apache/lens/server/BaseLensService.java | 25 +-- .../org/apache/lens/server/LensServices.java| 2 +- .../metastore/CubeMetastoreServiceImpl.java | 6 +- .../apache/lens/server/metastore/JAXBUtils.java | 4 - .../server/query/QueryExecutionServiceImpl.java | 1 + .../apache/lens/server/rewrite/RewriteUtil.java | 61 +++--- .../lens/server/session/HiveSessionService.java | 27 +-- .../lens/server/session/LensSessionImpl.java| 75 --- .../log/StatisticsLogPartitionHandler.java | 2 +- .../apache/lens/server/TestServerRestart.java | 2 +- .../lens/server/common/RestAPITestUtil.java | 2 +- .../server/metastore/TestMetastoreService.java | 4 +- .../lens/server/query/TestQueryService.java | 8 +- .../lens/server/query/TestResultFormatting.java | 2 +- .../lens/server/rewrite/TestRewriting.java | 198 ++ .../server/session/TestSessionClassLoaders.java | 2 +- .../lens/server/session/TestSessionExpiry.java | 4 +- lens-server/src/test/resources/hive-site.xml| 15 ++ lens-storage-db/pom.xml | 8 + .../src/test/resources/hive-site.xml| 9 + pom.xml | 93 - src/site/apt/admin/deployment.apt | 8 +- src/site/apt/developer/contribute.apt | 2 +- src/site/apt/lenshome/install-and-run.apt | 3 +- tools/conf/server/lens-site.xml | 5 + 73 files changed, 1268 insertions(+), 705
lens git commit: LENS-1034 : Fix picking lens-env.sh from conf passed via --conf directory
Repository: lens Updated Branches: refs/heads/master 59c2f8b11 -> 46e915367 LENS-1034 : Fix picking lens-env.sh from conf passed via --conf directory Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/46e91536 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/46e91536 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/46e91536 Branch: refs/heads/master Commit: 46e915367eaf9c49ce4293a1ac1375301d9c2c3e Parents: 59c2f8b Author: Rajat Khandelwal Authored: Tue May 10 15:34:08 2016 +0530 Committer: Amareshwari Sriramadasu Committed: Tue May 10 15:34:08 2016 +0530 -- tools/scripts/lens-ctl | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/lens/blob/46e91536/tools/scripts/lens-ctl -- diff --git a/tools/scripts/lens-ctl b/tools/scripts/lens-ctl index 822ead8..8523360 100755 --- a/tools/scripts/lens-ctl +++ b/tools/scripts/lens-ctl @@ -336,6 +336,7 @@ done # prepend conf dir to classpath if [ -n "$opt_conf" ]; then + export LENS_CONF=$opt_conf LENSCPPATH="$opt_conf:$LENSCPPATH" fi
lens git commit: LENS-614 : Support single columns as expressions - more like alias
Repository: lens
Updated Branches:
refs/heads/master 46e915367 -> 382881940
LENS-614 : Support single columns as expressions - more like alias
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/38288194
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/38288194
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/38288194
Branch: refs/heads/master
Commit: 3828819403c7c27c25e1d7d2421d0dcc5f506cdf
Parents: 46e9153
Author: Amareshwari Sriramadasu
Authored: Thu May 12 10:06:16 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu May 12 10:06:16 2016 +0530
--
.../apache/lens/cube/metadata/ExprColumn.java | 26 ++--
.../lens/cube/metadata/MetastoreUtil.java | 16 ++-
.../lens/cube/parse/AggregateResolver.java | 15 +-
.../apache/lens/cube/parse/AliasReplacer.java | 15 +-
.../apache/lens/cube/parse/CandidateFact.java | 8 +-
.../apache/lens/cube/parse/ColumnResolver.java | 4 +-
.../lens/cube/parse/ExpressionResolver.java | 31 ++---
.../org/apache/lens/cube/parse/HQLParser.java | 13 --
.../parse/SingleFactMultiStorageHQLContext.java | 7 +-
.../cube/metadata/TestCubeMetastoreClient.java | 36 ++---
.../apache/lens/cube/parse/CubeTestSetup.java | 12 ++
.../lens/cube/parse/TestExpressionResolver.java | 136 +++
12 files changed, 233 insertions(+), 86 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/38288194/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
index 1c9d6d2..52feeb8 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
@@ -36,8 +36,8 @@ public class ExprColumn extends CubeColumn {
public static final char EXPRESSION_DELIMITER = '|';
public static final char EXPRESSION_SPEC_DELIMITER = ':';
private static final String EXPRESSION_ENCODED = "true";
- private final Set expressionSet = new LinkedHashSet();
- private List astNodeList = new ArrayList();
+ private final Set expressionSet = new LinkedHashSet<>();
+ private List astNodeList = new ArrayList<>();
private final String type;
private boolean hasHashCode = false;
private int hashCode;
@@ -98,17 +98,17 @@ public class ExprColumn extends CubeColumn {
String decodedExpr =
isExpressionBase64Encoded ? new
String(Base64.decodeBase64(exprSpecStrs[0]), "UTF-8") : exprSpecStrs[0];
ExprSpec exprSpec = new ExprSpec();
-exprSpec.setExpr(decodedExpr);
+exprSpec.expr = decodedExpr;
if (exprSpecStrs.length > 1) {
// start time and end time serialized
if (StringUtils.isNotBlank(exprSpecStrs[1])) {
// start time available
-exprSpec.setStartTime(getDate(exprSpecStrs[1]));
+exprSpec.startTime = getDate(exprSpecStrs[1]);
}
if (exprSpecStrs.length > 2) {
if (StringUtils.isNotBlank(exprSpecStrs[2])) {
// end time available
- exprSpec.setEndTime(getDate(exprSpecStrs[2]));
+ exprSpec.endTime = getDate(exprSpecStrs[2]);
}
}
}
@@ -126,14 +126,11 @@ public class ExprColumn extends CubeColumn {
@ToString(exclude = {"astNode", "hasHashCode", "hashCode"})
public static class ExprSpec {
@Getter
-@Setter
@NonNull
private String expr;
@Getter
-@Setter
private Date startTime;
@Getter
-@Setter
private Date endTime;
private transient ASTNode astNode;
@@ -145,18 +142,25 @@ public class ExprColumn extends CubeColumn {
this.startTime = startTime;
this.endTime = endTime;
// validation
- getASTNode();
+ initASTNode();
}
-public synchronized ASTNode getASTNode() throws LensException {
+private synchronized void initASTNode() throws LensException {
if (astNode == null) {
if (StringUtils.isNotBlank(expr)) {
astNode = MetastoreUtil.parseExpr(getExpr());
}
}
+}
+
+private ASTNode getASTNode() throws LensException {
+ initASTNode();
return astNode;
}
+public ASTNode copyASTNode() throws LensException {
+ return MetastoreUtil.copyAST(getASTNode());
+}
@Override
public int hashCode() {
if (!hasHashCode) {
@@ -340,7 +344,7 @@ public class ExprColumn extends CubeColumn {
synchronized (expre
lens git commit: LENS-1032 : Add option to kill the query on timeout api upon timeout
Repository: lens
Updated Branches:
refs/heads/master 382881940 -> 389daac9a
LENS-1032 : Add option to kill the query on timeout api upon timeout
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/389daac9
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/389daac9
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/389daac9
Branch: refs/heads/master
Commit: 389daac9a66b0cb69ac729a56b39b3aa903ad36a
Parents: 3828819
Author: Puneet Gupta
Authored: Thu May 12 10:08:11 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu May 12 10:08:11 2016 +0530
--
.../org/apache/lens/api/query/QueryStatus.java | 8 ++
.../src/main/resources/lens-client-default.xml | 7 ++
.../org/apache/lens/driver/hive/HiveDriver.java | 3 +
.../lens/server/api/LensConfConstants.java | 11 ++
.../server/query/QueryExecutionServiceImpl.java | 120 +++
.../lens/server/query/QueryServiceResource.java | 6 +-
.../src/main/resources/lenssession-default.xml | 8 ++
.../lens/server/query/TestQueryService.java | 36 +-
src/site/apt/admin/session-config.apt | 58 -
9 files changed, 201 insertions(+), 56 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/389daac9/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
--
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
index 67d1e79..daedbf9 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
@@ -205,6 +205,14 @@ public class QueryStatus extends ToYAMLString implements
Serializable {
return status.equals(Status.FAILED);
}
+ public boolean cancelled() {
+return status.equals(Status.CANCELED);
+ }
+
+ public boolean executed() {
+return status.equals(Status.EXECUTED);
+ }
+
/**
* Checks if is valid transition.
http://git-wip-us.apache.org/repos/asf/lens/blob/389daac9/lens-client/src/main/resources/lens-client-default.xml
--
diff --git a/lens-client/src/main/resources/lens-client-default.xml
b/lens-client/src/main/resources/lens-client-default.xml
index 96506ac..132e1b1 100644
--- a/lens-client/src/main/resources/lens-client-default.xml
+++ b/lens-client/src/main/resources/lens-client-default.xml
@@ -75,4 +75,11 @@
is used to execute a query. The default value is 10 seconds.
+
+lens.query.cancel.on.timeout
+false
+Specifies whether to attempt cancellation of a query whose
execution takes longer than the timeout
+ value specified while submitting the query for execution.
+
+
http://git-wip-us.apache.org/repos/asf/lens/blob/389daac9/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
--
diff --git
a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 1463cc2..b85a66f 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -1048,6 +1048,9 @@ public class HiveDriver extends AbstractLensDriver {
this.timeoutMillis = timeoutMillis;
this.listener = listener;
this.pollInterval = timeoutMillis / 10;
+ if (pollInterval < 5000) {
+pollInterval = 5000; //minimum poll interval is 5 secs
+ }
}
/*
http://git-wip-us.apache.org/repos/asf/lens/blob/389daac9/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 7cf6449..a6df138 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -1070,4 +1070,15 @@ public final class LensConfConstants {
* Default value of DEFAULT_STATUS_UPDATE_EXPONENTIAL_WAIT_FACTOR is 3
millis (30 seconds)
*/
public static final long DEFAULT_STATUS_UPDATE_EXPONENTIAL_WAIT_FACTOR =
3;
+
+ /**
+ * Specifies whether to attempt cancellation of a query whose execution
takes longer than the timeout value
+ * specified while submitting the query for execution.
+ */
+ public static final
lens git commit: LENS-1028 : Convert dimension filter to fact filters for perfomace improvement on outer join queries
Repository: lens
Updated Branches:
refs/heads/master 797899c31 -> bbf2154ee
LENS-1028 : Convert dimension filter to fact filters for perfomace improvement
on outer join queries
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/bbf2154e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/bbf2154e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/bbf2154e
Branch: refs/heads/master
Commit: bbf2154ee541907db727249ff96b14dbceec4537
Parents: 797899c
Author: Sushil Mohanty
Authored: Mon May 16 12:38:59 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Mon May 16 12:38:59 2016 +0530
--
.../apache/lens/cube/parse/CandidateFact.java | 15 +-
.../lens/cube/parse/CubeQueryConfUtil.java | 2 +
.../lens/cube/parse/CubeQueryContext.java | 146 ++-
.../apache/lens/cube/parse/DefaultQueryAST.java | 22 ++-
.../apache/lens/cube/parse/DimHQLContext.java | 5 +-
.../apache/lens/cube/parse/GroupbyResolver.java | 2 +-
.../lens/cube/parse/MultiFactHQLContext.java| 4 +-
.../org/apache/lens/cube/parse/QueryAST.java| 10 +-
.../lens/cube/parse/join/AutoJoinContext.java | 2 +-
.../lens/cube/parse/TestCubeRewriter.java | 105 +
src/site/apt/user/olap-query-conf.apt | 2 +
11 files changed, 278 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/bbf2154e/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
index cd7a02c..e19e03a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
@@ -82,6 +82,8 @@ public class CandidateFact implements CandidateTable,
QueryAST {
@Getter
private final Map storgeWhereClauseMap = new HashMap<>();
@Getter
+ private final Map storgeWhereStringMap = new HashMap<>();
+ @Getter
private final Map>>
rangeToStoragePartMap = new HashMap<>();
@Getter
private final Map> rangeToStorageWhereMap =
new HashMap<>();
@@ -164,6 +166,9 @@ public class CandidateFact implements CandidateTable,
QueryAST {
public ASTNode getStorageWhereClause(String storageTable) {
return storgeWhereClauseMap.get(storageTable);
}
+ public String getStorageWhereString(String storageTable) {
+return storgeWhereStringMap.get(storageTable);
+ }
public boolean isExpressionAnswerable(ASTNode node, CubeQueryContext
context) throws LensException {
return
getColumns().containsAll(HQLParser.getColsInExpr(context.getAliasForTableName(context.getCube()),
node));
@@ -274,18 +279,18 @@ public class CandidateFact implements CandidateTable,
QueryAST {
return result;
}
- public String getSelectTree() {
+ public String getSelectString() {
return HQLParser.getString(selectAST);
}
- public String getWhereTree() {
+ public String getWhereString() {
if (whereAST != null) {
return HQLParser.getString(whereAST);
}
return null;
}
- public String getHavingTree() {
+ public String getHavingString() {
if (havingAST != null) {
return HQLParser.getString(havingAST);
}
@@ -293,7 +298,7 @@ public class CandidateFact implements CandidateTable,
QueryAST {
}
@Override
- public String getOrderByTree() {
+ public String getOrderByString() {
if (orderByAST != null) {
return HQLParser.getString(orderByAST);
}
@@ -314,7 +319,7 @@ public class CandidateFact implements CandidateTable,
QueryAST {
return dimFieldIndices;
}
- public String getGroupByTree() {
+ public String getGroupByString() {
if (groupByAST != null) {
return HQLParser.getString(groupByAST);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/bbf2154e/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
index a57292c..49ed5ef 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
@@ -115,4 +115,6 @@ public final class CubeQueryConfUtil {
public static final boolean DEFAULT_DO_FLATTENING_OF_BRIDGE_TABLE_EARLY =
false;
public static final String BRIDGE_TABLE_FIELD_ARRAY_FILTER =
"lens.cube.query.bridge.table.field.arr
lens git commit: LENS-1144 : Fix writing multi storage queries when storage start and end time is configured
Repository: lens
Updated Branches:
refs/heads/master bbf2154ee -> 1f12ce55a
LENS-1144 : Fix writing multi storage queries when storage start and end time
is configured
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/1f12ce55
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/1f12ce55
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/1f12ce55
Branch: refs/heads/master
Commit: 1f12ce55a74fd81edfb334f625c214ca370db9c1
Parents: bbf2154
Author: Amareshwari Sriramadasu
Authored: Wed May 18 10:53:21 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed May 18 10:53:21 2016 +0530
--
.../lens/cube/metadata/CubeMetastoreClient.java | 43 ++-
.../lens/cube/parse/StorageTableResolver.java | 26 +-
.../apache/lens/cube/metadata/DateFactory.java | 2 +
.../cube/metadata/TestCubeMetastoreClient.java | 28 +-
.../apache/lens/cube/parse/CubeTestSetup.java | 33 +-
.../lens/cube/parse/TestCubeRewriter.java | 302 +--
.../lens/cube/parse/TestUnionQueries.java | 374 +++
.../lens/server/api/LensServerAPITestUtil.java | 5 +-
8 files changed, 496 insertions(+), 317 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/1f12ce55/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index c8190bc..71f6b39 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -2383,27 +2383,58 @@ public class CubeMetastoreClient {
return storageTimes;
}
+ /*
+ * Storage table is not a candidate for range (t0, tn) :
+ * if start_time is after tn; or end date is before t0.
+ */
public boolean isStorageTableCandidateForRange(String storageTableName, Date
fromDate, Date toDate) throws
HiveException, LensException {
+List storageEndDates = getStorageTimes(storageTableName,
MetastoreUtil.getStoragetableEndTimesKey());
+for(Date endDate : storageEndDates) {
+ // endDate is exclusive
+ if (endDate.before(fromDate) || endDate.equals(fromDate)) {
+log.debug("from date {} is after validity end time: {}, hence
discarding {}",
+ fromDate, endDate, storageTableName);
+return false;
+ }
+}
+
List storageStartDates = getStorageTimes(storageTableName,
MetastoreUtil.getStoragetableStartTimesKey());
for(Date startDate : storageStartDates) {
- if (fromDate.before(startDate)) {
-log.info("from date {} is before validity start time: {}, hence
discarding {}",
-fromDate, startDate, storageTableName);
+ // toDate is exclusive on the range
+ if (startDate.after(toDate) || startDate.equals(toDate)) {
+log.debug("to date {} is before validity start time: {}, hence
discarding {}",
+ toDate, startDate, storageTableName);
+return false;
+ }
+}
+return true;
+ }
+
+ // Check if partition is valid wrt start and end dates
+ public boolean isStorageTablePartitionACandidate(String storageTableName,
Date partDate) throws
+HiveException, LensException {
+List storageStartDates = getStorageTimes(storageTableName,
MetastoreUtil.getStoragetableStartTimesKey());
+for(Date startDate : storageStartDates) {
+ if (partDate.before(startDate)) {
+log.info("part date {} is before validity start time: {}, hence
discarding {}",
+ partDate, startDate, storageTableName);
return false;
}
}
List storageEndDates = getStorageTimes(storageTableName,
MetastoreUtil.getStoragetableEndTimesKey());
for(Date endDate : storageEndDates) {
- if (toDate.after(endDate)) {
-log.info("to date {} is after validity end time: {}, hence discarding
{}",
-toDate, endDate, storageTableName);
+ // end date should be exclusive
+ if (partDate.after(endDate) || partDate.equals(endDate)) {
+log.info("part date {} is after validity end time: {}, hence
discarding {}",
+ partDate, endDate, storageTableName);
return false;
}
}
return true;
}
+
public boolean isStorageTableCandidateForRange(String storageTableName,
String fromDate, String toDate) throws
HiveException, LensException {
Date now = new Date();
http://git-wip-us.apache.org/repos/asf/lens/blob/1f12ce55/lens-cube/src
lens git commit: LENS-1028 : Convert dimension filter to fact filters for perfomace improvement on outer join queries
Repository: lens
Updated Branches:
refs/heads/master 1f12ce55a -> f3e6bc542
LENS-1028 : Convert dimension filter to fact filters for perfomace improvement
on outer join queries
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/f3e6bc54
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/f3e6bc54
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/f3e6bc54
Branch: refs/heads/master
Commit: f3e6bc54263c3d5612449bb9b957b110524a9e50
Parents: 1f12ce5
Author: Sushil Mohanty
Authored: Wed May 18 10:56:14 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed May 18 10:56:14 2016 +0530
--
.../lens/cube/parse/CubeQueryContext.java | 4
.../lens/cube/parse/TestCubeRewriter.java | 23 +++-
2 files changed, 21 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/f3e6bc54/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 0fc8549..cf32746 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -1342,6 +1342,10 @@ public class CubeQueryContext implements
TrackQueriedColumns, QueryAST {
allFilters.add(getFilter(table, cubeAlias, node, joinClause, i,
dimToQuery));
}
}
+} else if (node.getParent() == null
+&& node.getToken().getType() != HiveParser.KW_AND) {
+ // if node is the only child
+ allFilters.add(HQLParser.getString((ASTNode) node));
}
for (int i = 0; i < node.getChildCount(); i++) {
ASTNode child = (ASTNode) node.getChild(i);
http://git-wip-us.apache.org/repos/asf/lens/blob/f3e6bc54/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 87799eb..fff6f24 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -515,7 +515,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
+ TWO_DAYS_RANGE, conf);
String filterSubquery = "testcube.countryid in ( select id from
TestQueryRewrite.c3_countrytable_partitioned "
+ "cubecountry where ((cubecountry.region) = 'asia') and
(cubecountry.dt = 'latest') )";
-//assertTrue(hql.contains(filterSubquery));
+assertTrue(hql.contains(filterSubquery));
// filter with or
hql = rewrite(
@@ -524,7 +524,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
filterSubquery = "testcube.countryid in ( select id from
TestQueryRewrite.c3_countrytable_partitioned "
+ "cubecountry where (((cubecountry.region) = 'asia') or
((cubecountry.region) = 'europe')) "
+ "and (cubecountry.dt = 'latest') )";
-//assertTrue(hql.contains(filterSubquery));
+assertTrue(hql.contains(filterSubquery));
//filter with in
hql = rewrite(
@@ -532,7 +532,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
+ "and " + TWO_DAYS_RANGE , conf);
filterSubquery = "testcube.countryid in ( select id from
TestQueryRewrite.c3_countrytable_partitioned "
+ "cubecountry where (cubecountry.region) in ('asia' , 'europe') and
(cubecountry.dt = 'latest') )";
-//assertTrue(hql.contains(filterSubquery));
+assertTrue(hql.contains(filterSubquery));
//filter with not in
hql = rewrite(
@@ -540,7 +540,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
+ "and " + TWO_DAYS_RANGE , conf);
filterSubquery = "testcube.countryid in ( select id from
TestQueryRewrite.c3_countrytable_partitioned "
+ "cubecountry where (cubecountry.region) not in ('asia' , 'europe')
and (cubecountry.dt = 'latest') )";
-//assertTrue(hql.contains(filterSubquery));
+assertTrue(hql.contains(filterSubquery));
//filter with !=
hql = rewrite(
@@ -548,7 +548,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
+ "and " + TWO_DAYS_RANGE , conf);
filterSubquery = "testcube.countryid in ( select
lens git commit: LENS-1154 : Remove warnings "HiveConf of name ... doesnt exist" in testcases
Repository: lens
Updated Branches:
refs/heads/master bed8e9a83 -> bfd4f00a2
LENS-1154 : Remove warnings "HiveConf of name ... doesnt exist" in testcases
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/bfd4f00a
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/bfd4f00a
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/bfd4f00a
Branch: refs/heads/master
Commit: bfd4f00a2aa6e71e7dd049d6e20081ae5b4e8e42
Parents: bed8e9a
Author: Rajat Khandelwal
Authored: Sat Jun 4 18:29:27 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Sat Jun 4 18:29:27 2016 +0530
--
.../resources/drivers/hive/hive1/hivedriver-site.xml | 7 ---
.../resources/drivers/hive/hive1/hivedriver-site.xml | 7 ---
.../resources/drivers/hive/hive1/hivedriver-site.xml | 7 ---
lens-driver-hive/src/test/resources/hive-site.xml| 15 ---
lens-driver-jdbc/src/test/resources/hive-site.xml| 7 ---
.../resources/drivers/hive/hive1/hivedriver-site.xml | 7 ---
lens-ml-lib/src/test/resources/lens-site.xml | 5 -
.../resources/drivers/hive/hive1/hivedriver-site.xml | 7 ---
.../resources/drivers/hive/hive2/hivedriver-site.xml | 7 ---
.../drivers/mockHive/mockHive1/hivedriver-site.xml | 7 ---
.../drivers/mockHive/mockHive2/hivedriver-site.xml | 7 ---
lens-server/src/test/resources/hive-site.xml | 13 -
lens-server/src/test/resources/lens-site.xml | 7 +--
lens-storage-db/src/test/resources/hive-site.xml | 7 ---
14 files changed, 5 insertions(+), 105 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/bfd4f00a/lens-cli/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
--
diff --git a/lens-cli/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
b/lens-cli/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
index f2aed88..2f35f13 100644
--- a/lens-cli/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
+++ b/lens-cli/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
@@ -30,13 +30,6 @@
-ÂÂ
-hive.metastore.local
-ÂÂ
-true
-
-
-
hive.metastore.warehouse.dir
${project.build.directory}/hive/warehouse
http://git-wip-us.apache.org/repos/asf/lens/blob/bfd4f00a/lens-client/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
--
diff --git
a/lens-client/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
b/lens-client/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
index f2aed88..2f35f13 100644
--- a/lens-client/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
+++ b/lens-client/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
@@ -30,13 +30,6 @@
-ÂÂ
-hive.metastore.local
-ÂÂ
-true
-
-
-
hive.metastore.warehouse.dir
${project.build.directory}/hive/warehouse
http://git-wip-us.apache.org/repos/asf/lens/blob/bfd4f00a/lens-driver-hive/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
--
diff --git
a/lens-driver-hive/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
b/lens-driver-hive/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
index 613938d..ab8a9cf 100644
--- a/lens-driver-hive/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
+++ b/lens-driver-hive/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
@@ -24,13 +24,6 @@
-ÂÂ
-hive.metastore.local
-ÂÂ
-true
-
-
-
hive.metastore.warehouse.dir
${project.build.directory}/hive/warehouse
http://git-wip-us.apache.org/repos/asf/lens/blob/bfd4f00a/lens-driver-hive/src/test/resources/hive-site.xml
--
diff --git a/lens-driver-hive/src/test/resources/hive-site.xml
b/lens-driver-hive/src/test/resources/hive-site.xml
index 67fe744..2f4076a 100644
--- a/lens-driver-hive/src/test/resources/hive-site.xml
+++ b/lens-driver-hive/src/test/resources/hive-site.xml
@@ -24,15 +24,13 @@
-ÂÂ
-hive.metastore.local
-ÂÂ
-true
+hive.metastore.warehouse.dir
+${project.build.directory}/hive/warehouse
-hive.metastore.warehouse.dir
-${project.build.directory}/hive/warehouse
+hive.server2.logging.operation.log.location
+${project.build.directory}/operation_logs/
@@ -57,11 +55,6 @@
-hive.server2.query.log.dir
-target/query_logs
-
-
-
datanucleus.schema.autoCreateTables
true
http://git-wip-us.apache.org/repos/asf/lens/blob/bfd4f00a/lens-driver-jdbc/src/test/r
lens git commit: LENS-1167 : Fix NPE when result formatting fails
Repository: lens
Updated Branches:
refs/heads/master df75a4b83 -> 020b65167
LENS-1167 : Fix NPE when result formatting fails
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/020b6516
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/020b6516
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/020b6516
Branch: refs/heads/master
Commit: 020b651678969289ab12290d292d6043b734b144
Parents: df75a4b
Author: Rajat Khandelwal
Authored: Thu Jun 9 09:57:59 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Jun 9 09:57:59 2016 +0530
--
.../java/org/apache/lens/lib/query/AbstractFileFormatter.java | 6 --
.../java/org/apache/lens/server/query/ResultFormatter.java | 2 ++
2 files changed, 6 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/020b6516/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
--
diff --git
a/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
b/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
index 2cd6b3f..bafb2fb 100644
---
a/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
+++
b/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
@@ -19,6 +19,7 @@
package org.apache.lens.lib.query;
import java.io.*;
+import java.net.URI;
import org.apache.lens.server.api.driver.LensResultSetMetadata;
import org.apache.lens.server.api.query.QueryContext;
@@ -94,7 +95,7 @@ public abstract class AbstractFileFormatter extends
AbstractOutputFormatter impl
@Override
public void writeExternal(ObjectOutput out) throws IOException {
out.writeUTF(metadata.toJson());
-out.writeUTF(finalPath.toString());
+out.writeObject(finalPath == null ? null : finalPath.toUri());
out.writeObject(numRows); // writeObject and readObject have handling for
null objects
out.writeObject(fileSize);
}
@@ -102,7 +103,8 @@ public abstract class AbstractFileFormatter extends
AbstractOutputFormatter impl
@Override
public void readExternal(ObjectInput in) throws IOException,
ClassNotFoundException {
metadata = LensResultSetMetadata.fromJson(in.readUTF());
-finalPath = new Path(in.readUTF());
+URI uri = (URI) in.readObject();
+finalPath = uri == null ? null : new Path(uri);
numRows = (Integer)in.readObject();
fileSize = (Long)in.readObject();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/020b6516/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
index f465e9d..41cf33b 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
+++
b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
@@ -142,6 +142,8 @@ public class ResultFormatter extends
AsyncEventListener {
metricsService.incrCounter(ResultFormatter.class, "formatting-errors");
log.warn("Exception while formatting result for {}", queryHandle, e);
try {
+// set output formatter to null so that server restart is faster in
case this query is not purged.
+ctx.setQueryOutputFormatter(null);
queryService.setFailedStatus(ctx, ERROR_MESSAGE, e);
} catch (LensException e1) {
log.error("Exception while setting failure for {}", queryHandle, e1);
lens git commit: LENS-987 : Add AlarmService for scheduling time based queries
Repository: lens
Updated Branches:
refs/heads/master 68c5267f0 -> 92456effb
LENS-987 : Add AlarmService for scheduling time based queries
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/92456eff
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/92456eff
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/92456eff
Branch: refs/heads/master
Commit: 92456effb9508d4578894a82c1f2a419fe504d97
Parents: 68c5267
Author: Lavkesh Lahngir
Authored: Mon Jun 13 17:17:04 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Mon Jun 13 17:17:04 2016 +0530
--
lens-server-api/pom.xml | 4 +
.../server/api/events/SchedulerAlarmEvent.java | 51 +
lens-server/pom.xml | 4 +
.../org/apache/lens/server/LensServices.java| 7 +-
.../lens/server/metrics/MetricsServiceImpl.java | 2 +
.../notification/services/AlarmService.java | 220 +++
.../notification/services/AlarmServiceTest.java | 180 +++
pom.xml | 6 +
8 files changed, 473 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/92456eff/lens-server-api/pom.xml
--
diff --git a/lens-server-api/pom.xml b/lens-server-api/pom.xml
index 5508fb9..b59c249 100644
--- a/lens-server-api/pom.xml
+++ b/lens-server-api/pom.xml
@@ -105,5 +105,9 @@
org.slf4j
jcl-over-slf4j
+
+ joda-time
+ joda-time
+
http://git-wip-us.apache.org/repos/asf/lens/blob/92456eff/lens-server-api/src/main/java/org/apache/lens/server/api/events/SchedulerAlarmEvent.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/events/SchedulerAlarmEvent.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/events/SchedulerAlarmEvent.java
new file mode 100644
index 000..2bcf0d6
--- /dev/null
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/events/SchedulerAlarmEvent.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lens.server.api.events;
+
+import org.apache.lens.api.query.SchedulerJobHandle;
+
+import org.joda.time.DateTime;
+
+import lombok.Data;
+
+/**
+ * This event is triggered by the AlarmService whenever a scheduled query
needs to be scheduled.
+ */
+@Data
+public class SchedulerAlarmEvent extends LensEvent {
+
+ /**
+ * jobHandle for which the alarm needs to be triggered.
+ */
+ private SchedulerJobHandle jobHandle;
+
+ private DateTime nominalTime;
+
+ public SchedulerAlarmEvent(SchedulerJobHandle jobHandle, DateTime
nominalTime) {
+super(nominalTime.getMillis());
+this.jobHandle = jobHandle;
+this.nominalTime = nominalTime;
+ }
+
+ @Override
+ public String getEventId() {
+return jobHandle.getHandleIdString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/92456eff/lens-server/pom.xml
--
diff --git a/lens-server/pom.xml b/lens-server/pom.xml
index b73cd05..2294b2a 100644
--- a/lens-server/pom.xml
+++ b/lens-server/pom.xml
@@ -242,6 +242,10 @@
guava
+ org.quartz-scheduler
+ quartz
+
+
org.apache.lens
lens-server-api
test-jar
http://git-wip-us.apache.org/repos/asf/lens/blob/92456eff/lens-server/src/main/java/org/apache/lens/server/LensServices.java
--
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
index 10ec8b7..f6f2f36 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
@@ -20,7 +20,10 @@ package org.apache.lens.s
lens git commit: LENS-1151 : Add logs for Queries moving between waiting and queued queries data structures
Repository: lens
Updated Branches:
refs/heads/master 4fea0c4a2 -> 714aae99d
LENS-1151 : Add logs for Queries moving between waiting and queued queries data
structures
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/714aae99
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/714aae99
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/714aae99
Branch: refs/heads/master
Commit: 714aae99d67319a1719b08a3386b5893d2b250ce
Parents: 4fea0c4
Author: Rajat Khandelwal
Authored: Fri Jun 17 10:35:27 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Fri Jun 17 10:35:27 2016 +0530
--
.../lens/server/query/QueryExecutionServiceImpl.java | 13 +++--
.../query/collect/UnioningWaitingQueriesSelector.java | 3 +++
.../DefaultQueryLaunchingConstraintsChecker.java | 3 +++
3 files changed, 9 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/714aae99/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 1f6ec13..2de098d 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -697,18 +697,12 @@ public class QueryExecutionServiceImpl extends
BaseLensService implements QueryE
log.debug("Acquiring lock in QuerySubmitter");
removalFromLaunchedQueriesLock.lock();
try {
-
-boolean isQueryAllowedToLaunch =
this.constraintsChecker.canLaunch(query, launchedQueries);
-
-log.debug("isQueryAllowedToLaunch:{}", isQueryAllowedToLaunch);
-if (isQueryAllowedToLaunch) {
-
+if (this.constraintsChecker.canLaunch(query, launchedQueries))
{
/* Query is not going to be added to waiting queries. No
need to keep the lock.
First release lock, then launch query */
removalFromLaunchedQueriesLock.unlock();
launchQuery(query);
} else {
-
/* Query is going to be added to waiting queries. Keep
holding the lock to avoid any removal from
launched queries. First add to waiting queries, then release
lock */
addToWaitingQueries(query);
@@ -3185,12 +3179,11 @@ public class QueryExecutionServiceImpl extends
BaseLensService implements QueryE
Set eligibleWaitingQueries = this.waitingQueriesSelector
.selectQueries(finishedQuery, this.waitingQueries);
-
+log.info("Eligible queries to pick from waiting queries: {}",
eligibleWaitingQueries);
if (eligibleWaitingQueries.isEmpty()) {
- log.debug("No queries eligible to move out of waiting state.");
+ log.info("No queries eligible to move out of waiting state.");
return;
}
-
waitingQueries.removeAll(eligibleWaitingQueries);
queuedQueries.addAll(eligibleWaitingQueries);
if (log.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/lens/blob/714aae99/lens-server/src/main/java/org/apache/lens/server/query/collect/UnioningWaitingQueriesSelector.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/collect/UnioningWaitingQueriesSelector.java
b/lens-server/src/main/java/org/apache/lens/server/query/collect/UnioningWaitingQueriesSelector.java
index 8db059e..40b837f 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/collect/UnioningWaitingQueriesSelector.java
+++
b/lens-server/src/main/java/org/apache/lens/server/query/collect/UnioningWaitingQueriesSelector.java
@@ -32,11 +32,13 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
/**
* Selects queries eligible by all {@link WaitingQueriesSelectionPolicy} to
move them out of waiting state.
*
*/
+@Slf4j
public class UnioningWaitingQueriesSelector implements WaitingQueriesSelector {
private final ImmutableSet selectionPolicies;
@@ -86,6 +88,7 @@ public class UnioningWaitingQueriesSelector implements
WaitingQueriesSelector {
Set candiateQueries =
selectionPolicy.selectQueries(finishedQuery, waitingQueries);
candidateQueriesSets.add(candiateQueries);
+ log.info("Queries selected by policy: {} are: {
lens git commit: LENS-1200 : Fix error message when MR job failures happen
Repository: lens
Updated Branches:
refs/heads/master 600b9be4c -> 5f79cc6b2
LENS-1200 : Fix error message when MR job failures happen
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/5f79cc6b
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/5f79cc6b
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/5f79cc6b
Branch: refs/heads/master
Commit: 5f79cc6b216b8fc864c1cca34b929b6e9cc868cb
Parents: 600b9be
Author: Rajat Khandelwal
Authored: Tue Jun 28 14:05:59 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Jun 28 14:05:59 2016 +0530
--
.../src/main/java/org/apache/lens/driver/hive/HiveDriver.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/5f79cc6b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
--
diff --git
a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 88de651..28b77c8 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -655,7 +655,7 @@ public class HiveDriver extends AbstractLensDriver {
if (tstate == Task.TaskState.FINISHED) {
completedTasks++;
}
- if (taskStat.getErrorMsg() != null) {
+ if ((taskStat.getReturnValue() != null && taskStat.getReturnValue()
!= 0) || taskStat.getErrorMsg() != null) {
appendTaskIds(errorMessage, taskStat);
errorMessage.append(" has failed! ");
}
lens git commit: LENS-1206 : Add test case for SchedulerDAO
Repository: lens
Updated Branches:
refs/heads/master a33cc53b3 -> 9068e3c17
LENS-1206 : Add test case for SchedulerDAO
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/9068e3c1
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/9068e3c1
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/9068e3c1
Branch: refs/heads/master
Commit: 9068e3c17051192911a2f44f2a3a0e6acfc7285d
Parents: a33cc53
Author: Lavkesh Lahngir
Authored: Fri Jul 1 09:51:06 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Fri Jul 1 09:51:06 2016 +0530
--
.../lens/server/scheduler/SchedulerDAOTest.java | 188 +++
1 file changed, 188 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/9068e3c1/lens-server/src/test/java/org/apache/lens/server/scheduler/SchedulerDAOTest.java
--
diff --git
a/lens-server/src/test/java/org/apache/lens/server/scheduler/SchedulerDAOTest.java
b/lens-server/src/test/java/org/apache/lens/server/scheduler/SchedulerDAOTest.java
new file mode 100644
index 000..41cfc99
--- /dev/null
+++
b/lens-server/src/test/java/org/apache/lens/server/scheduler/SchedulerDAOTest.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.scheduler;
+
+import java.util.*;
+
+import javax.xml.datatype.DatatypeConfigurationException;
+import javax.xml.datatype.DatatypeFactory;
+import javax.xml.datatype.XMLGregorianCalendar;
+
+import org.apache.lens.api.LensSessionHandle;
+import org.apache.lens.api.scheduler.*;
+import org.apache.lens.server.LensServerConf;
+import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.scheduler.util.UtilityMethods;
+
+import org.apache.commons.dbutils.QueryRunner;
+import org.apache.hadoop.conf.Configuration;
+
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Test(groups = "unit-test")
+public class SchedulerDAOTest {
+
+ SchedulerDAO schedulerDAO;
+ Map instances = new
HashMap<>();
+
+ @BeforeClass
+ public void setup() throws Exception {
+System.setProperty(LensConfConstants.CONFIG_LOCATION,
"target/test-classes/");
+Configuration conf = LensServerConf.getHiveConf();
+QueryRunner runner = new
QueryRunner(UtilityMethods.getDataSourceFromConf(conf));
+// Cleanup all tables
+runner.update("DROP TABLE IF EXISTS job_table");
+runner.update("DROP TABLE IF EXISTS job_instance_table");
+this.schedulerDAO = new SchedulerDAO(conf);
+ }
+
+ private XTrigger getTestTrigger() {
+XTrigger trigger = new XTrigger();
+XFrequency frequency = new XFrequency();
+frequency.setCronExpression("0 0 12 * * ?");
+frequency.setTimezone("UTC");
+trigger.setFrequency(frequency);
+return trigger;
+ }
+
+ private XExecution getTestExecution() {
+XExecution execution = new XExecution();
+XJobQuery query = new XJobQuery();
+query.setQuery("select * from test_table");
+execution.setQuery(query);
+XSessionType sessionType = new XSessionType();
+sessionType.setDb("test");
+execution.setSession(sessionType);
+return execution;
+ }
+
+ private XJob getTestJob() throws DatatypeConfigurationException {
+XJob job = new XJob();
+job.setTrigger(getTestTrigger());
+job.setName("Test lens Job");
+GregorianCalendar startTime = new GregorianCalendar();
+startTime.setTimeInMillis(System.currentTimeMillis());
+XMLGregorianCalendar start =
DatatypeFactory.newInstance().newXMLGregorianCalendar(startTime);
+
+GregorianCalendar endTime = new GregorianCalendar();
+endTime.setTimeInMillis(System.currentTimeMillis());
+XMLGregorianCalendar end =
DatatypeFactory.newInstance().newXMLGregorianCalendar(endTime);
+
+job.setStartTime(start
lens git commit: LENS-1184 : Add REST API for the scheduler
Repository: lens
Updated Branches:
refs/heads/master 4bcb7aa96 -> 265c4667a
LENS-1184 : Add REST API for the scheduler
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/265c4667
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/265c4667
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/265c4667
Branch: refs/heads/master
Commit: 265c4667af0dd7850892d0180c9ee72bb40b3e83
Parents: 4bcb7aa
Author: Ajay Yadava
Authored: Tue Jul 5 10:16:07 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Jul 5 10:16:07 2016 +0530
--
.../server/api/scheduler/SchedulerService.java | 25 ++-
.../lens/server/scheduler/LensScheduler.java| 48 +
.../lens/server/scheduler/ScheduleResource.java | 181 ++-
.../server/scheduler/SchedulerServiceImpl.java | 73 ++--
4 files changed, 288 insertions(+), 39 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/265c4667/lens-server-api/src/main/java/org/apache/lens/server/api/scheduler/SchedulerService.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/scheduler/SchedulerService.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/scheduler/SchedulerService.java
index 700a255..d0af876 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/scheduler/SchedulerService.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/scheduler/SchedulerService.java
@@ -19,18 +19,22 @@
package org.apache.lens.server.api.scheduler;
import java.util.Collection;
-import java.util.Date;
import java.util.List;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.scheduler.*;
+import org.apache.lens.server.api.LensService;
+import org.apache.lens.server.api.SessionValidator;
import org.apache.lens.server.api.error.LensException;
/**
* Scheduler interface.
*/
-public interface SchedulerService {
+public interface SchedulerService extends LensService, SessionValidator {
+
+ /** The constant NAME */
+ String NAME = "scheduler";
/**
* Submit a job.
@@ -104,11 +108,9 @@ public interface SchedulerService {
*
* @param sessionHandle handle for the current session.
* @param jobHandle handle for the job
- * @param expiryTimetime after which the job shouldn't execute.
* @throws LensException the lens exception
*/
- void expireJob(LensSessionHandle sessionHandle, SchedulerJobHandle jobHandle,
- Date expiryTime) throws LensException;
+ void expireJob(LensSessionHandle sessionHandle, SchedulerJobHandle
jobHandle) throws LensException;
/**
@@ -131,12 +133,10 @@ public interface SchedulerService {
*
* @param sessionHandle handle for the session.
* @param jobHandle handle for the job
- * @param effectiveTime time from which to resume the instances.
* @return true if the job was resumed successfully, false otherwise.
* @throws LensException the lens exception
*/
- boolean resumeJob(LensSessionHandle sessionHandle, SchedulerJobHandle
jobHandle,
-Date effectiveTime) throws LensException;
+ boolean resumeJob(LensSessionHandle sessionHandle, SchedulerJobHandle
jobHandle) throws LensException;
/**
* Delete a job.
@@ -155,16 +155,13 @@ public interface SchedulerService {
* if null no entries will be removed from result
* @param user filter for user who submitted the job, if specified
only jobs submitted by the given user
* will be returned, if not specified no entries will
be removed from result on basis of userName
- * @param jobName filter for jobName, if specified only the jobs with
name same as given name will be considered
- * , else no jobs will be filtered out on the basis of
name.
* @param startTime if specified only instances with scheduleTime after
this time will be considered.
* @param endTime if specified only instances with scheduleTime before
this time will be considered.
* @return A collection of stats per job
* @throws LensException
*/
Collection getAllJobStats(LensSessionHandle sessionHandle,
- String state, String user,
- String jobName, long startTime, long
endTime) throws LensException;
+ String state, String user, long
startTime, long endTime) throws LensException;
/**
* Returns stats for a job.
@@ -187,10 +184,10 @@ public interface SchedulerService {
* @param sessionHandle handle for the session.
* @param jobHandle handle for the job
* @param numResu
lens git commit: LENS-991 : Remove shutdown hook getting added for each command
Repository: lens
Updated Branches:
refs/heads/master 265c4667a -> 42c4dfcb6
LENS-991 : Remove shutdown hook getting added for each command
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/42c4dfcb
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/42c4dfcb
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/42c4dfcb
Branch: refs/heads/master
Commit: 42c4dfcb64f59c341ad244b96872a921ea9043db
Parents: 265c466
Author: Raju Bairishetti
Authored: Tue Jul 5 11:13:49 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Jul 5 11:13:49 2016 +0530
--
.../lens/cli/commands/BaseLensCommand.java | 12 +-
.../lens/cli/TestLensConnectionCliCommands.java | 146 +--
.../apache/lens/cli/TestLensCubeCommands.java | 6 +-
.../lens/cli/TestLensDatabaseCommands.java | 5 +-
.../lens/cli/TestLensNativeTableCommands.java | 4 +-
.../apache/lens/cli/TestLensQueryCommands.java | 5 -
.../org/apache/lens/client/TestLensClient.java | 56 +++
7 files changed, 107 insertions(+), 127 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/42c4dfcb/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
--
diff --git
a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
index 9eb73b3..479746a 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
@@ -57,8 +57,7 @@ public class BaseLensCommand implements ExecutionProcessor {
protected DefaultPrettyPrinter pp;
/** The is connection active. */
- protected boolean isConnectionActive;
-
+ protected static boolean isConnectionActive;
public static final String DATE_FMT = "-MM-dd'T'HH:mm:ss:SSS";
private LensClient lensClient = null;
@@ -75,7 +74,7 @@ public class BaseLensCommand implements ExecutionProcessor {
return DATE_PARSER.get().format(dt);
}
- private void registerShutDownHook() {
+ static {
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
closeClientConnection();
@@ -86,10 +85,10 @@ public class BaseLensCommand implements ExecutionProcessor {
/**
* Close client connection.
*/
- protected synchronized void closeClientConnection() {
+ protected static synchronized void closeClientConnection() {
if (isConnectionActive) {
log.debug("Request for stopping lens cli received");
- lensClient.closeConnection();
+ getClientWrapper().getClient().closeConnection();
isConnectionActive = false;
}
}
@@ -98,7 +97,6 @@ public class BaseLensCommand implements ExecutionProcessor {
* Instantiates a new base lens command.
*/
public BaseLensCommand() {
-registerShutDownHook();
mapper = new ObjectMapper();
mapper.setSerializationInclusion(Inclusion.NON_NULL);
mapper.setSerializationInclusion(Inclusion.NON_DEFAULT);
@@ -120,13 +118,13 @@ public class BaseLensCommand implements
ExecutionProcessor {
}
public void setClient(LensClient client) {
-isConnectionActive = true;
lensClient = client;
}
public LensClient getClient() {
if (lensClient == null) {
setClient(getClientWrapper().getClient());
+ isConnectionActive = true;
}
return lensClient;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/42c4dfcb/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java
--
diff --git
a/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java
b/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java
index d1ec94e..76d387d 100644
---
a/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java
+++
b/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java
@@ -63,19 +63,19 @@ public class TestLensConnectionCliCommands extends
LensCliApplicationTest {
*/
@Test
public void testConnectionCommand() {
-LensClient client = new LensClient();
-LensConnectionCommands commands = new LensConnectionCommands();
-commands.setClient(client);
-String key = "connectiontest1";
-String value = "connectiontest1val";
-String keyvalList = commands.showParameters();
-
-Assert.assertFalse(keyvalList.contains("connectiontest1"));
-
-commands.setParam(key + "=" + value);
-String val = commands.getParam(key);
-Assert.assertEquals(val, key + "="
lens git commit: LENS-1195 : Add state machine for scheduler service
Repository: lens
Updated Branches:
refs/heads/master a736f4697 -> 9e61e5106
LENS-1195 : Add state machine for scheduler service
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/9e61e510
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/9e61e510
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/9e61e510
Branch: refs/heads/master
Commit: 9e61e51064ad0f15e511fea08a0eba63771715d8
Parents: a736f46
Author: Ajay Yadava
Authored: Thu Jul 7 16:29:32 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Jul 7 16:29:32 2016 +0530
--
.../lens/api/scheduler/SchedulerJobInfo.java| 2 +-
.../api/scheduler/SchedulerJobInstanceInfo.java | 2 +-
.../scheduler/SchedulerJobInstanceState.java| 23 ---
.../scheduler/SchedulerJobInstanceStatus.java | 23 +++
.../lens/api/scheduler/SchedulerJobState.java | 23 ---
.../lens/api/scheduler/SchedulerJobStatus.java | 23 +++
.../error/InvalidStateTransitionException.java | 46 +
.../lens/server/api/scheduler/StateMachine.java | 35
.../lens/server/scheduler/SchedulerDAO.java | 12 +-
.../state/SchedulerJobInstanceState.java| 193 +++
.../scheduler/state/SchedulerJobState.java | 150 ++
.../lens/server/scheduler/SchedulerDAOTest.java | 20 +-
12 files changed, 488 insertions(+), 64 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/9e61e510/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInfo.java
--
diff --git
a/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInfo.java
b/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInfo.java
index 0561834..7d06689 100644
--- a/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInfo.java
+++ b/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInfo.java
@@ -52,7 +52,7 @@ public class SchedulerJobInfo {
* @param state state of this job.
* @return current state of this job
*/
- private SchedulerJobState state;
+ private SchedulerJobStatus state;
/**
* @param createdOn time to be set as createdOn.
http://git-wip-us.apache.org/repos/asf/lens/blob/9e61e510/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceInfo.java
--
diff --git
a/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceInfo.java
b/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceInfo.java
index cb0d938..8158576 100644
---
a/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceInfo.java
+++
b/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceInfo.java
@@ -76,7 +76,7 @@ public class SchedulerJobInstanceInfo {
* @param state state to be set.
* @return state of this instance.
*/
- private SchedulerJobInstanceState state;
+ private SchedulerJobInstanceStatus state;
/**
* @param createdOn time to be set as created_on time for the instance.
http://git-wip-us.apache.org/repos/asf/lens/blob/9e61e510/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceState.java
--
diff --git
a/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceState.java
b/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceState.java
deleted file mode 100644
index c66add6..000
---
a/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceState.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.api.scheduler;
-
-public enum SchedulerJobInstanceState {
- WAITING, LAUNCHED, FAILED, SUCCEEDED, KILLED, TIMEOUT
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/9e61e510/lens-api/src/main/java/org/apache/lens/api/scheduler/Schedule
lens git commit: LENS-1200 : Fix error message on MR job failure
Repository: lens
Updated Branches:
refs/heads/master 9e61e5106 -> eabd4e491
LENS-1200 : Fix error message on MR job failure
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/eabd4e49
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/eabd4e49
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/eabd4e49
Branch: refs/heads/master
Commit: eabd4e4910390dc6970491c4b117973a6971e7e7
Parents: 9e61e51
Author: Rajat Khandelwal
Authored: Thu Jul 7 17:45:35 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Jul 7 17:45:35 2016 +0530
--
.../src/main/java/org/apache/lens/driver/hive/HiveDriver.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/eabd4e49/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
--
diff --git
a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 28b77c8..1326611 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -669,7 +669,7 @@ public class HiveDriver extends AbstractLensDriver {
if (StringUtils.isNotBlank(errorMsg)) {
error = errorMsg;
} else if (opStatus.getState().equals(OperationState.ERROR)) {
-error = context.getDriverStatus().getStatusMessage();
+error = context.getDriverStatus().getErrorMessage();
}
context.getDriverStatus().setErrorMessage(error);
context.getDriverStatus().setProgressMessage(jsonTaskStatus);
lens git commit: LENS-1214 : Fix NPE in await termination for query service
Repository: lens
Updated Branches:
refs/heads/master 0dfdaa9ec -> a89c1a696
LENS-1214 : Fix NPE in await termination for query service
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a89c1a69
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a89c1a69
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a89c1a69
Branch: refs/heads/master
Commit: a89c1a6961d07b7beb80073a695981a9051047b6
Parents: 0dfdaa9
Author: Rajat Khandelwal
Authored: Tue Jul 19 15:40:26 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Jul 19 15:40:26 2016 +0530
--
.../org/apache/lens/server/query/QueryExecutionServiceImpl.java | 3 +++
1 file changed, 3 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/a89c1a69/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 2de098d..924466b 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -1236,6 +1236,9 @@ public class QueryExecutionServiceImpl extends
BaseLensService implements QueryE
}
private void awaitTermination(QueryResultPurger service) {
+if (service == null) {
+ return;
+}
try {
service.awaitTermination(1, TimeUnit.MINUTES);
} catch (InterruptedException e) {
lens git commit: LENS-1212 : Enable saved queries by default
Repository: lens Updated Branches: refs/heads/master a89c1a696 -> b3a9cb9a1 LENS-1212 : Enable saved queries by default Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b3a9cb9a Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b3a9cb9a Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b3a9cb9a Branch: refs/heads/master Commit: b3a9cb9a188f14dffdfcf9c9cf9bacfe223fcebb Parents: a89c1a6 Author: Rajat Khandelwal Authored: Tue Jul 19 15:43:22 2016 +0530 Committer: Amareshwari Sriramadasu Committed: Tue Jul 19 15:43:22 2016 +0530 -- lens-server/src/main/resources/lensserver-default.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/b3a9cb9a/lens-server/src/main/resources/lensserver-default.xml -- diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml index 6dc322e..b80e7ed 100644 --- a/lens-server/src/main/resources/lensserver-default.xml +++ b/lens-server/src/main/resources/lensserver-default.xml @@ -55,7 +55,7 @@ lens.server.servicenames -session,query,metastore,scheduler,quota +session,query,savedquery,metastore,scheduler,quota These services would be started in the specified order when lens-server starts up @@ -395,7 +395,7 @@ lens.server.ws.resourcenames -session,metastore,query,quota,scheduler,index,log +session,metastore,query,savedquery,quota,scheduler,index,log These JAX-RS resources would be started in the specified order when lens-server starts up
lens git commit: LENS-1211 : Fix GenericExceptionMapper to return 405 only when there is no matching resource
Repository: lens
Updated Branches:
refs/heads/master b3a9cb9a1 -> 0190ef567
LENS-1211 : Fix GenericExceptionMapper to return 405 only when there is no
matching resource
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/0190ef56
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/0190ef56
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/0190ef56
Branch: refs/heads/master
Commit: 0190ef5676f160678c8e66c601f8168cf9a42938
Parents: b3a9cb9
Author: Rajat Khandelwal
Authored: Tue Jul 19 15:53:38 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Jul 19 15:53:38 2016 +0530
--
.../org/apache/lens/server/error/GenericExceptionMapper.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/0190ef56/lens-server/src/main/java/org/apache/lens/server/error/GenericExceptionMapper.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/error/GenericExceptionMapper.java
b/lens-server/src/main/java/org/apache/lens/server/error/GenericExceptionMapper.java
index 9aa74ed..d761b6d 100644
---
a/lens-server/src/main/java/org/apache/lens/server/error/GenericExceptionMapper.java
+++
b/lens-server/src/main/java/org/apache/lens/server/error/GenericExceptionMapper.java
@@ -62,11 +62,13 @@ public class GenericExceptionMapper implements
ExceptionMapper {
status = le.getLensAPIResult().getHttpStatusCode();
} else if (exception instanceof WebApplicationException) {
status = Response.Status.fromStatusCode(((WebApplicationException)
exception).getResponse().getStatus());
+} else if (extendedUriInfo.getMatchedResourceMethod() == null) {
+ status = Response.Status.METHOD_NOT_ALLOWED;
} else {
status = Response.Status.INTERNAL_SERVER_ERROR;
}
if (extendedUriInfo.getMatchedResourceMethod() == null) {
- return Response.status(Response.Status.METHOD_NOT_ALLOWED).build();
+ return Response.status(status).entity("No matching resource
method").build();
}
if
(extendedUriInfo.getMatchedResourceMethod().getInvocable().getRawResponseType()
== LensAPIResult.class) {
if (le != null) {
lens git commit: LENS-1030 : Fix logs flowing in query logs
Repository: lens
Updated Branches:
refs/heads/master 0190ef567 -> c0e0b0e3b
LENS-1030 : Fix logs flowing in query logs
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c0e0b0e3
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c0e0b0e3
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c0e0b0e3
Branch: refs/heads/master
Commit: c0e0b0e3b6d4e9f5793041bf3849045d38851826
Parents: 0190ef5
Author: Amareshwari Sriramadasu
Authored: Tue Jul 19 15:58:24 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Jul 19 15:58:24 2016 +0530
--
.../MappedDiagnosticLogSegregationContext.java | 5 +
.../lens/server/AuthenticationFilter.java | 47
.../java/org/apache/lens/server/BaseApp.java| 2 +-
.../org/apache/lens/server/LensApplication.java | 2 -
.../server/LensRequestContextInitFilter.java| 59 --
.../lens/server/LensRequestLoggingFilter.java | 79 +
.../server/query/QueryExecutionServiceImpl.java | 1 -
.../src/main/resources/lensserver-default.xml | 8 +-
.../server/query/QueryAPIErrorResponseTest.java | 6 +-
lens-server/src/test/resources/lens-site.xml| 2 +-
src/site/apt/admin/config.apt | 116 +--
11 files changed, 151 insertions(+), 176 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/c0e0b0e3/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
b/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
index 7780dff..cde153b 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
@@ -41,6 +41,11 @@ public class MappedDiagnosticLogSegregationContext
implements LogSegregationCont
setQueryId(id);
}
+ public static void removeLogSegragationIds() {
+MDC.remove(LOG_SEGREGATION_ID);
+MDC.remove(QUERY_LOG_ID);
+ }
+
@Override
public String getQueryId() {
return MDC.get(QUERY_LOG_ID);
http://git-wip-us.apache.org/repos/asf/lens/blob/c0e0b0e3/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
b/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
deleted file mode 100644
index e982520..000
--- a/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server;
-
-import java.io.IOException;
-
-import javax.ws.rs.container.ContainerRequestContext;
-import javax.ws.rs.container.ContainerRequestFilter;
-import javax.ws.rs.core.SecurityContext;
-
-import lombok.extern.slf4j.Slf4j;
-
-/**
- * The Class AuthenticationFilter.
- */
-@Slf4j
-public class AuthenticationFilter implements ContainerRequestFilter {
-
- /*
- * (non-Javadoc)
- *
- * @see
javax.ws.rs.container.ContainerRequestFilter#filter(javax.ws.rs.container.ContainerRequestContext)
- */
- @Override
- public void filter(ContainerRequestContext requestContext) throws
IOException {
-
-final SecurityContext securityContext =
requestContext.getSecurityContext();
-String user = securityContext.getUserPrincipal() != null ?
securityContext.getUserPrincipal().getName() : null;
-log.info("Request from user: {} , path={}", user,
requestContext.getUriInfo().getPath());
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/c0e0b0e3/lens-server/src/main/java/or
lens git commit: LENS-1209 : Remove toString warnings from debug log
Repository: lens
Updated Branches:
refs/heads/master c0e0b0e3b -> 113aebba1
LENS-1209 : Remove toString warnings from debug log
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/113aebba
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/113aebba
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/113aebba
Branch: refs/heads/master
Commit: 113aebba18a70a3427cbda7ed230e03d9a83223d
Parents: c0e0b0e
Author: Rajat Khandelwal
Authored: Tue Jul 19 16:03:55 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Jul 19 16:03:55 2016 +0530
--
.../main/java/org/apache/lens/api/jaxb/YAMLToStringStrategy.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/113aebba/lens-api/src/main/java/org/apache/lens/api/jaxb/YAMLToStringStrategy.java
--
diff --git
a/lens-api/src/main/java/org/apache/lens/api/jaxb/YAMLToStringStrategy.java
b/lens-api/src/main/java/org/apache/lens/api/jaxb/YAMLToStringStrategy.java
index 3b55dfe..1562394 100644
--- a/lens-api/src/main/java/org/apache/lens/api/jaxb/YAMLToStringStrategy.java
+++ b/lens-api/src/main/java/org/apache/lens/api/jaxb/YAMLToStringStrategy.java
@@ -324,7 +324,7 @@ public class YAMLToStringStrategy extends
JAXBToStringStrategy {
Object wrappedValue =
claz.getDeclaredMethod(getterName).invoke(value);
return appendNewLine(appendInternal(locator, stringBuilder,
wrappedValue));
} catch (NoSuchMethodException | IllegalAccessException |
InvocationTargetException e) {
- log.debug("getter access failed for {}#{}. Going the usual way",
claz.getName(), getterName, e);
+ log.trace("getter access failed for {}#{}. Going the usual way",
claz.getName(), getterName, e);
}
}
return super.appendInternal(locator, stringBuilder, value);
lens git commit: LENS-1207 : Fix gaps in lens ui
Repository: lens
Updated Branches:
refs/heads/master 77e7cc641 -> 2de1b5218
LENS-1207 : Fix gaps in lens ui
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2de1b521
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2de1b521
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2de1b521
Branch: refs/heads/master
Commit: 2de1b5218aa4f1d21606e067c229752e9f2ffbdc
Parents: 77e7cc6
Author: Rajat Khandelwal
Authored: Tue Jul 19 17:23:29 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Jul 19 17:23:29 2016 +0530
--
lens-ui/app/actions/LoginActions.js | 5 +-
lens-ui/app/adapters/AdhocQueryAdapter.js | 9 +-
lens-ui/app/adapters/AuthenticationAdapter.js | 3 +-
lens-ui/app/app.js| 2 +-
lens-ui/app/components/AdhocQueryComponent.js | 4 +-
lens-ui/app/components/CubeSchemaComponent.js | 159 +++--
lens-ui/app/components/CubeTreeComponent.js | 13 +-
lens-ui/app/components/DatabaseComponent.js | 14 +-
lens-ui/app/components/LoginComponent.js | 12 +-
lens-ui/app/components/QueryBoxComponent.js | 2 +-
lens-ui/app/components/TableTreeComponent.js | 17 ++-
lens-ui/app/stores/CubeStore.js | 5 +-
lens-ui/app/stores/DatabaseStore.js | 10 --
lens-ui/app/stores/UserStore.js | 11 ++
lens-ui/app/utils/ErrorParser.js | 31 +---
lens-ui/config.json | 4 +-
lens-ui/package.json | 6 +-
lens-ui/server.js | 14 +-
18 files changed, 191 insertions(+), 130 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/2de1b521/lens-ui/app/actions/LoginActions.js
--
diff --git a/lens-ui/app/actions/LoginActions.js
b/lens-ui/app/actions/LoginActions.js
index 3cb39d0..5c57087 100644
--- a/lens-ui/app/actions/LoginActions.js
+++ b/lens-ui/app/actions/LoginActions.js
@@ -22,8 +22,8 @@ import AppConstants from '../constants/AppConstants';
import AuthenticationAdapter from '../adapters/AuthenticationAdapter';
let LoginActions = {
- authenticate (email, password) {
-AuthenticationAdapter.authenticate(email, password)
+ authenticate (email, password, database) {
+AuthenticationAdapter.authenticate(email, password, database)
.then(function (response) {
// authenticating user right away
@@ -31,6 +31,7 @@ let LoginActions = {
actionType: AppConstants.AUTHENTICATION_SUCCESS,
payload: {
email: email,
+database: database,
secretToken: response
}
});
http://git-wip-us.apache.org/repos/asf/lens/blob/2de1b521/lens-ui/app/adapters/AdhocQueryAdapter.js
--
diff --git a/lens-ui/app/adapters/AdhocQueryAdapter.js
b/lens-ui/app/adapters/AdhocQueryAdapter.js
index 9d4b416..a54274f 100644
--- a/lens-ui/app/adapters/AdhocQueryAdapter.js
+++ b/lens-ui/app/adapters/AdhocQueryAdapter.js
@@ -52,7 +52,12 @@ let AdhocQueryAdapter = {
getCubes (secretToken) {
let url = baseUrl + urls.getCubes;
-return BaseAdapter.get(url + '?sessionid=' + secretToken);
+let postURL = "?";
+if (Config.cubes_type) {
+ postURL += "type=" + Config.cubes_type + "&"
+}
+postURL += "sessionid=" + secretToken;
+return BaseAdapter.get(url + postURL);
},
getCubeDetails (secretToken, cubeName) {
@@ -212,7 +217,7 @@ let AdhocQueryAdapter = {
},
getParams (secretToken, query) {
-let url = baseUrl + urls.parameters;
+let url = baseUrl + urls.parameters + '?sessionid=' + secretToken;
let formData = new FormData();
formData.append('query', query);
http://git-wip-us.apache.org/repos/asf/lens/blob/2de1b521/lens-ui/app/adapters/AuthenticationAdapter.js
--
diff --git a/lens-ui/app/adapters/AuthenticationAdapter.js
b/lens-ui/app/adapters/AuthenticationAdapter.js
index 15c196b..1effb14 100644
--- a/lens-ui/app/adapters/AuthenticationAdapter.js
+++ b/lens-ui/app/adapters/AuthenticationAdapter.js
@@ -38,12 +38,13 @@ let sessionconf = `
`;
let AuthenticationAdapter = {
- authenticate (email, password) {
+ authenticate (email, password, database) {
// preparing data as API accepts multipart/form-data :(
var formData = new FormData();
formData.append('username', email);
formData.append('password', password || "");
+formData.append('database', database|| "default");
lens git commit: LENS-1218 : Document Lens UI in install and run steps
Repository: lens
Updated Branches:
refs/heads/master 66b7f47ff -> c33ef9991
LENS-1218 : Document Lens UI in install and run steps
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c33ef999
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c33ef999
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c33ef999
Branch: refs/heads/master
Commit: c33ef999112a0c784a2b78701af40cb17127d84d
Parents: 66b7f47
Author: Rajat Khandelwal
Authored: Fri Jul 22 07:52:41 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Fri Jul 22 07:52:41 2016 +0530
--
src/site/apt/developer/contribute.apt | 28 --
src/site/apt/lenshome/install-and-run.apt | 24 +-
2 files changed, 32 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/c33ef999/src/site/apt/developer/contribute.apt
--
diff --git a/src/site/apt/developer/contribute.apt
b/src/site/apt/developer/contribute.apt
index a2fd59e..8ea7a6d 100644
--- a/src/site/apt/developer/contribute.apt
+++ b/src/site/apt/developer/contribute.apt
@@ -114,14 +114,30 @@ Developer Documentation : How to contribute to Apache
Lens?
+---+
Once one of the above sets of commands completes successfully, the build
will produce
-
-bin/apache-lens--bin/server>
and
-
-bin/apache-lens--bin/client>.
Former can be used as the Lens server
- installation directory and later can be used as Lens client installation
directory to
- {{{/lenshome/install-and-run.html#Running_Lens} run}} lens server and lens
client.
+
-bin/apache-lens--bin/server>,
+
-bin/apache-lens--bin/client>,
and
+ -bin/apache-lens--bin/ui>.
These can be used as the Lens server
+ installation directory, Lens client installation directory, and Lens UI
installation directory respectively.
+ {{{/lenshome/install-and-run.html#Running_Lens} run}} lens server, lens
client and UI.
- The build will also produce debians for both client and server in
. Client debian uses
+ Lens UI is written in , and can be run directly from the module
itself. In fact, it's recommended to run
+ from there since the npm dependencies are already fetched in
module by step above.
+
++---+
+ cd lens-ui
+ npm start
++---+
+
+ For making changes in lens-ui code, it can be started in dev mode where
it'll watch for your changes and re-bundle
+ whenever changes are made. To run ui server in dev mode, use the following
command:
+
++---+
+ npm run dev
++---+
+
+ The build will also produce debians for server, client and ui in
. Client debian uses
as the Lens client installation directory and
Server debian uses as
- the Lens server installation directory.
+ the Lens server installation directory. UI debian uses
as the installation directory.
Apache Lens depends on Hive. Please {{{#Building_Hive_from_Source} build}}
Hive from Source or install it using the
documentation {{{/lenshome/install-and-run.html#Installing_Hive} here}}.
After installing Lens and Hive,
http://git-wip-us.apache.org/repos/asf/lens/blob/c33ef999/src/site/apt/lenshome/install-and-run.apt
--
diff --git a/src/site/apt/lenshome/install-and-run.apt
b/src/site/apt/lenshome/install-and-run.apt
index f133822..b9727c4 100644
--- a/src/site/apt/lenshome/install-and-run.apt
+++ b/src/site/apt/lenshome/install-and-run.apt
@@ -48,10 +48,11 @@ Installing and Running Lens
├── NOTICE
├── README
├── LICENSE*
- └── server
+ ├── server
+ └── ui
+---+
- -bin/client> can be used as the Lens client
installation directory and -bin/server> can be used as
the Lens server installation directory to {{{#Running_Lens}run}} lens client
and lens server respectively.
+ -bin/client> can be used as the Lens client
installation directory, -bin/server> can be used as the
Lens server installation directory, and -bin/ui> can be
used as the Lens UI installation directory to {{{#Running_Lens}run}} lens
client, lens server and lens ui respectively.
** Installing Hive
@@ -159,9 +160,6 @@ Installing and Running Lens
You can try out examples described below or use the REST api for each
service.
- Lens server also comes with simple UI service which can be used to browse
catalog and submit queries. Once the server
- starts up, you can open the UI service on
http://:1/index.html and browse.
-
The command to stop the server:
+---+
@@ -228,7 +226,7 @@ Installing and Running Lens
Examples folder contains sample resource files, data files and query files
which will be used by <> script.
-*** S
lens git commit: LENS-998 : Fix logging when partitions are skipped while registering
Repository: lens
Updated Branches:
refs/heads/master c33ef9991 -> 7ca35b8a7
LENS-998 : Fix logging when partitions are skipped while registering
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7ca35b8a
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7ca35b8a
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7ca35b8a
Branch: refs/heads/master
Commit: 7ca35b8a75a5839a3e92099c50db95ff193ee093
Parents: c33ef99
Author: Sushil Mohanty
Authored: Sun Jul 24 07:50:15 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Sun Jul 24 07:50:15 2016 +0530
--
.../java/org/apache/lens/cube/metadata/CubeMetastoreClient.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/7ca35b8a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 61f9f99..0dce6c4 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -973,9 +973,8 @@ public class CubeMetastoreClient {
}
}
if (!skippedParts.isEmpty()) {
- log.info("List of partitions skipped : {}, because they fall before fact
start time : {} "
- + "and after fact end time : {}",
- skippedParts, storageStartDate, storageEndDate);
+ log.info("List of partitions skipped : {}, because they fall before fact
start time "
+ + "or after end time or they are future partitions", skippedParts);
}
return timeSpecs;
}
lens git commit: LENS-1217 : Remove old java ui server
Repository: lens
Updated Branches:
refs/heads/master 7ca35b8a7 -> b5763ff06
LENS-1217 : Remove old java ui server
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b5763ff0
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b5763ff0
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b5763ff0
Branch: refs/heads/master
Commit: b5763ff06eecec1bcf5ab08b84c97da8c4d3cd38
Parents: 7ca35b8
Author: Rajat Khandelwal
Authored: Sun Jul 24 08:05:55 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Sun Jul 24 08:05:55 2016 +0530
--
.../lens/server/api/LensConfConstants.java | 40 ---
.../java/org/apache/lens/server/LensServer.java | 17 -
.../org/apache/lens/server/LensServices.java| 2 +-
.../lens/server/ui/MetastoreUIResource.java | 321 ---
.../lens/server/ui/QueryServiceUIResource.java | 269
.../lens/server/ui/SessionUIResource.java | 148 -
.../lens/server/ui/StaticFileResource.java | 123 ---
.../java/org/apache/lens/server/ui/UIApp.java | 46 ---
.../src/main/resources/lensserver-default.xml | 24 --
.../org/apache/lens/server/TestLensServer.java | 45 ---
.../lens/server/TestStaticFileResource.java | 109 ---
.../lens/server/ui/TestSessionUIResource.java | 124 ---
12 files changed, 1 insertion(+), 1267 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/b5763ff0/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index bd9b1ab..9f51565 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -188,46 +188,6 @@ public final class LensConfConstants {
public static final long SESSION_TIMEOUT_SECONDS_DEFAULT = 1440 * 60; //
Default is one day
/**
- * The Constant
- */
- public static final String SERVER_UI_ENABLE = SERVER_PFX + "ui.enable";
-
- /**
- * The Constant
- */
- public static final boolean DEFAULT_SERVER_UI_ENABLE = true;
-
- /**
- * The Constant SERVER_UI_URI.
- */
- public static final String SERVER_UI_URI = SERVER_PFX + "ui.base.uri";
-
- /**
- * The Constant DEFAULT_SERVER_UI_URI.
- */
- public static final String DEFAULT_SERVER_UI_URI = "http://0.0.0.0:1/";;
-
- /**
- * The Constant SERVER_UI_STATIC_DIR.
- */
- public static final String SERVER_UI_STATIC_DIR = SERVER_PFX +
".ui.static.dir";
-
- /**
- * The Constant DEFAULT_SERVER_UI_STATIC_DIR.
- */
- public static final String DEFAULT_SERVER_UI_STATIC_DIR =
"webapp/lens-server/static";
-
- /**
- * The Constant SERVER_UI_ENABLE_CACHING.
- */
- public static final String SERVER_UI_ENABLE_CACHING = SERVER_PFX +
"ui.enable.caching";
-
- /**
- * The Constant DEFAULT_SERVER_UI_ENABLE_CACHING.
- */
- public static final boolean DEFAULT_SERVER_UI_ENABLE_CACHING = true;
-
- /**
* The Constant SERVER_STATE_PERSISTENCE_INTERVAL_MILLIS.
*/
public static final String SERVER_STATE_PERSISTENCE_INTERVAL_MILLIS =
http://git-wip-us.apache.org/repos/asf/lens/blob/b5763ff0/lens-server/src/main/java/org/apache/lens/server/LensServer.java
--
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServer.java
b/lens-server/src/main/java/org/apache/lens/server/LensServer.java
index b6983f4..203c249 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServer.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServer.java
@@ -32,7 +32,6 @@ import org.apache.lens.server.error.GenericExceptionMapper;
import org.apache.lens.server.error.LensJAXBValidationExceptionMapper;
import org.apache.lens.server.metrics.MetricsServiceImpl;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
-import org.apache.lens.server.ui.UIApp;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -93,19 +92,9 @@ public class LensServer {
adminCtx
.setAttribute("com.codahale.metrics.servlets.MetricsServlet.registry",
(metricsService.getMetricRegistry()));
adminCtx.setAttribute("com.codahale.metrics.servlets.HealthCheckServlet.registry",
metricsService.getHealthCheck());
-
final ServletRegistration sgMetrics = adminCtx.addServlet("admin", new
AdminServlet());
sgMetrics.addMapping("/admin/*");
-
adminCtx.deploy(server);
-
-if
lens git commit: LENS-1198 : Update hive dependency to apache hive release 2.1.0
Repository: lens
Updated Branches:
refs/heads/master b5763ff06 -> 7707a25b1
LENS-1198 : Update hive dependency to apache hive release 2.1.0
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7707a25b
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7707a25b
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7707a25b
Branch: refs/heads/master
Commit: 7707a25b1f195b846292adfe679881004d748b31
Parents: b5763ff
Author: Rajat Khandelwal
Authored: Sun Jul 24 08:25:58 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Sun Jul 24 08:25:58 2016 +0530
--
bin-dist-files/README | 14 ++---
pom.xml | 19 +
src/site/apt/developer/contribute.apt | 2 +-
src/site/apt/lenshome/install-and-run.apt | 29 +++---
4 files changed, 29 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/7707a25b/bin-dist-files/README
--
diff --git a/bin-dist-files/README b/bin-dist-files/README
index 84dcdf1..e4e9f9d 100644
--- a/bin-dist-files/README
+++ b/bin-dist-files/README
@@ -14,14 +14,12 @@ Detailed documentation for the project is available at
https://lens.apache.org
Here are quick start steps :
Before starting the server, you need to set HADOOP_HOME and HIVE_HOME
environment variables. These should point to
-the Hadoop and Hive installation directories. Lens depends on a forked version
of Hive which is available at
-https://github.com/inmobi/hive. So, Hive needs to be built and HIVE_HOME
should be set as packaged directory from
-build.
-
-Building hive :
-$ git clone https://github.com/inmobi/hive
-$ git checkout hive-release-0.13.3-inm
-$ mvn clean package -DskipTests -Phadoop-2,dist
+the Hadoop and Hive installation directories. Lens depends on apache Hive 2.1+.
+
+Installing Hive :
+$ # Download hive binary from http://apache.org/download.cgi
+$ # extract
+$ export HIVE_HOME=
Server packaging is server/ folder.
$ cd server
http://git-wip-us.apache.org/repos/asf/lens/blob/7707a25b/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 176fb0e..f5a80df 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,7 +45,7 @@
2.6.0
1.3.0
-2.1.1-inm
+2.1.0
0.1.5
@@ -753,23 +753,6 @@
- inmobi.repo
- https://github.com/InMobi/mvn-repo/raw/master/releases
-
-false
-
-
-
- inmobi.snapshots
- https://github.com/InMobi/mvn-repo/raw/master/snapshots
-
-false
-
-
-true
-
-
-
central
http://repo1.maven.org/maven2
http://git-wip-us.apache.org/repos/asf/lens/blob/7707a25b/src/site/apt/developer/contribute.apt
--
diff --git a/src/site/apt/developer/contribute.apt
b/src/site/apt/developer/contribute.apt
index 8ea7a6d..3358052 100644
--- a/src/site/apt/developer/contribute.apt
+++ b/src/site/apt/developer/contribute.apt
@@ -148,7 +148,7 @@ Developer Documentation : How to contribute to Apache Lens?
+---+
- git clone https://github.com/InMobi/hive.git
+ git clone https://github.com/apache/hive.git
cd hive
http://git-wip-us.apache.org/repos/asf/lens/blob/7707a25b/src/site/apt/lenshome/install-and-run.apt
--
diff --git a/src/site/apt/lenshome/install-and-run.apt
b/src/site/apt/lenshome/install-and-run.apt
index b9727c4..6c9bb09 100644
--- a/src/site/apt/lenshome/install-and-run.apt
+++ b/src/site/apt/lenshome/install-and-run.apt
@@ -26,8 +26,10 @@ Installing and Running Lens
** Downloading
Download a release of Apache Lens from {{{/releases/download.html}here}}.
- Lens depends on Hive forked from Apache Hive. Download a release of hive
from
{{{https://github.com/InMobi/hive/releases}https://github.com/InMobi/hive/releases}}.
- Lens < 2.6 works with hive 0.13.x releases, Lens >= 2.6 works with hive 2.1+
releases.
+ Lens \< 2.6 depend on Hive forked from Apache Hive. Download a release of
hive from
{{{https://github.com/InMobi/hive/releases}https://github.com/InMobi/hive/releases}}.
Compatible versions are 0.13.x.
+
+ In version <2.6>, hive dependency has been migrated to apache hive. Now lens
no longer depends on forked hive.
+ Lens > 2.6 work with apache hive 2.1+ releases. Downloadable from
{{{http://www.apache.org/dyn/closer.cgi/hive/}http://www.apache.org/dyn/closer.cgi/hive/}}
** Installing Lens
@@ -35,14 +37,14 @@ Installing and Running Lens
+---+
- tar zxvf apache-lens--bin.tar.gz
+ tar zxvf ap
lens git commit: LENS-1168 : Add query launcher pool to launch queries in parallel
Repository: lens
Updated Branches:
refs/heads/master 7707a25b1 -> 776a2c331
LENS-1168 : Add query launcher pool to launch queries in parallel
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/776a2c33
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/776a2c33
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/776a2c33
Branch: refs/heads/master
Commit: 776a2c331e8e43e11b188046a1369b1d640355eb
Parents: 7707a25
Author: Amareshwari Sriramadasu
Authored: Mon Jul 25 11:26:38 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Mon Jul 25 11:26:38 2016 +0530
--
.../lens/server/api/LensConfConstants.java | 20 ++
.../lens/server/api/metrics/MetricsService.java | 2 +
.../lens/server/api/query/QueryContext.java | 8 +
.../server/api/query/QueryExecutionService.java | 7 +
.../MaxConcurrentDriverQueriesConstraint.java | 28 ++-
...oncurrentDriverQueriesConstraintFactory.java | 16 +-
...axConcurrentDriverQueriesConstraintTest.java | 32 ++-
.../lens/server/metrics/MetricsServiceImpl.java | 11 ++
.../server/query/QueryExecutionServiceImpl.java | 196 ++-
.../lens/server/session/LensSessionImpl.java| 12 +-
.../src/main/resources/lensserver-default.xml | 22 +++
.../org/apache/lens/server/LensJerseyTest.java | 2 +-
.../lens/server/common/FailingQueryDriver.java | 9 +
.../lens/server/query/TestQueryConstraints.java | 19 +-
.../lens/server/query/TestQueryService.java | 24 +++
.../ThreadSafeEstimatedQueryCollectionTest.java | 2 +-
lens-server/src/test/resources/logback.xml | 2 +-
src/site/apt/admin/config.apt | 174
18 files changed, 416 insertions(+), 170 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 9f51565..43bd7e4 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -886,6 +886,26 @@ public final class LensConfConstants {
public static final String ESTIMATE_POOL_KEEP_ALIVE_MILLIS = SERVER_PFX +
"estimate.pool.keepalive.millis";
public static final int DEFAULT_ESTIMATE_POOL_KEEP_ALIVE_MILLIS = 6; //
1 minute
+ /**
+ * Key used to get minimum number of threads in the launcher thread pool
+ */
+ public static final String LAUNCHER_POOL_MIN_THREADS = SERVER_PFX +
"launcher.pool.min.threads";
+ public static final int DEFAULT_LAUNCHER_POOL_MIN_THREADS = 3;
+
+ /**
+ * Key used to get maximum number of threads in the laucnher thread pool
+ */
+ public static final String LAUNCHER_POOL_MAX_THREADS = SERVER_PFX +
"launcher.pool.max.threads";
+ // keeping the default to hundred, we may never grow till there, it would go
to max for concurrrent queries allowed on
+ // all drivers together.
+ public static final int DEFAULT_LAUNCHER_POOL_MAX_THREADS = 100;
+
+ /**
+ * Key used to get keep alive time for threads in the launcher thread pool
+ */
+ public static final String LAUNCHER_POOL_KEEP_ALIVE_MILLIS = SERVER_PFX +
"launcher.pool.keepalive.millis";
+ public static final int DEFAULT_LAUNCHER_POOL_KEEP_ALIVE_MILLIS = 6; //
1 minute
+
public static final String QUERY_PHASE1_REWRITERS = SERVER_PFX +
"query.phase1.rewriters";
/**
http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
index 7fd2d81..35daa21 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
@@ -124,6 +124,8 @@ public interface MetricsService extends LensService {
*/
String WAITING_QUERIES = "waiting-queries";
+ String LAUNCHING_QUERIES = "launching-queries";
+
/**
* The Constant FINISHED_QUERIES.
*/
http://git-wip-us.apache.org/repos/asf/lens/blob/776a2c33/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
--
dif
lens git commit: LENS-1228 : Fix TestQueryConstraints test failure
Repository: lens
Updated Branches:
refs/heads/master 1a96948ee -> c5dd7de4b
LENS-1228 : Fix TestQueryConstraints test failure
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c5dd7de4
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c5dd7de4
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c5dd7de4
Branch: refs/heads/master
Commit: c5dd7de4b69f7454539cb7147d9d1bfade7e34d5
Parents: 1a96948
Author: Amareshwari Sriramadasu
Authored: Tue Jul 26 09:43:32 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Jul 26 09:43:32 2016 +0530
--
.../lens/server/api/query/QueryContext.java | 3 +++
.../server/query/QueryExecutionServiceImpl.java | 22 -
.../DefaultEstimatedQueryCollection.java| 5
.../apache/lens/server/TestLensApplication.java | 5
.../lens/server/query/TestQueryConstraints.java | 26 +---
.../TestQueryIndependenceFromSessionClose.java | 20 ---
6 files changed, 50 insertions(+), 31 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/c5dd7de4/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 7f323dc..2641b60 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -529,4 +529,7 @@ public class QueryContext extends AbstractQueryContext {
this.driverResult = result;
}
+ public String toString() {
+return queryHandle + ":" + this.status;
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c5dd7de4/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index cf0c168..a582994 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -902,18 +902,16 @@ public class QueryExecutionServiceImpl extends
BaseLensService implements QueryE
private void updateFinishedQuery(QueryContext ctx, QueryStatus before) {
// before would be null in case of server restart
if (before != null) {
- if (before.queued()) {
-/* Seems like query is cancelled, remove it from both queuedQueries
and waitingQueries because we don't know
-* where it is right now. It might happen that when we remove it from
queued, it was in waiting OR
-* when we removed it from waiting, it was in queued. We might just
miss removing it from everywhere due to this
-* hide and seek. Then QuerySubmitter thread will come to rescue, as it
always checks that a query should be in
-* queued state before processing it after deque. If it is in cancelled
state, then it will skip it. */
-queuedQueries.remove(ctx);
-waitingQueries.remove(ctx);
- } else {
-if (removeFromLaunchedQueries(ctx)) {
- processWaitingQueriesAsync(ctx);
-}
+ /* Seems like query is cancelled, remove it from both queuedQueries and
waitingQueries because we don't know
+ * where it is right now. It might happen that when we remove it from
queued, it was in waiting OR
+ * when we removed it from waiting, it was in queued. We might just miss
removing it from everywhere due to this
+ * hide and seek. Then QuerySubmitter thread will come to rescue, as it
always checks that a query should be in
+ * queued state before processing it after deque. If it is in cancelled
state, then it will skip it. */
+ queuedQueries.remove(ctx);
+ waitingQueries.remove(ctx);
+ // Remove it from launched queries as well - the query got cancelled or
failed to launch or failed to execute
+ if (removeFromLaunchedQueries(ctx)) {
+processWaitingQueriesAsync(ctx);
}
}
// Remove from active queries
http://git-wip-us.apache.org/repos/asf/lens/blob/c5dd7de4/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java
b/lens-server/src/ma
lens git commit: LENS-955 : Fix test failure TestQueryEndEmailNotifier.testLaunchFailure
Repository: lens
Updated Branches:
refs/heads/master c5dd7de4b -> 1924bbb01
LENS-955 : Fix test failure TestQueryEndEmailNotifier.testLaunchFailure
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/1924bbb0
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/1924bbb0
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/1924bbb0
Branch: refs/heads/master
Commit: 1924bbb018cbb4aa7cf8c0b2cf11376a053e3b9d
Parents: c5dd7de
Author: Amareshwari Sriramadasu
Authored: Tue Jul 26 16:15:59 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Jul 26 16:15:59 2016 +0530
--
.../server/query/QueryExecutionServiceImpl.java | 23 ++--
1 file changed, 12 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/1924bbb0/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index a582994..4df2b9b 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -2049,17 +2049,18 @@ public class QueryExecutionServiceImpl extends
BaseLensService implements QueryE
}
private QueryHandle submitQuery(final QueryContext ctx) throws LensException
{
-
-QueryStatus before = ctx.getStatus();
-ctx.setStatus(new QueryStatus(0.0, null, QUEUED, "Query is queued", false,
null, null, null));
-queuedQueries.add(ctx);
-log.debug("Added to Queued Queries:{}", ctx.getQueryHandleString());
-allQueries.put(ctx.getQueryHandle(), ctx);
-// Add to session's active query list
-
getSession(SESSION_MAP.get(ctx.getLensSessionIdentifier())).addToActiveQueries(ctx.getQueryHandle());
-fireStatusChangeEvent(ctx, ctx.getStatus(), before);
-log.info("Returning handle {}", ctx.getQueryHandle().getHandleId());
-return ctx.getQueryHandle();
+synchronized (ctx) {
+ QueryStatus before = ctx.getStatus();
+ ctx.setStatus(new QueryStatus(0.0, null, QUEUED, "Query is queued",
false, null, null, null));
+ queuedQueries.add(ctx);
+ log.info("Added to Queued Queries:{}", ctx.getQueryHandleString());
+ allQueries.put(ctx.getQueryHandle(), ctx);
+ // Add to session's active query list
+
getSession(SESSION_MAP.get(ctx.getLensSessionIdentifier())).addToActiveQueries(ctx.getQueryHandle());
+ fireStatusChangeEvent(ctx, ctx.getStatus(), before);
+ log.info("Returning handle {}", ctx.getQueryHandle().getHandleId());
+ return ctx.getQueryHandle();
+}
}
/*
lens git commit: LENS-1156 : Fix TestUnionQueries.testDimAttrExpressionQuery test failure
Repository: lens
Updated Branches:
refs/heads/master e4d8b88f9 -> 3f03d29ba
LENS-1156 : Fix TestUnionQueries.testDimAttrExpressionQuery test failure
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3f03d29b
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3f03d29b
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3f03d29b
Branch: refs/heads/master
Commit: 3f03d29ba47755913e06cd3a397119c679b2f65a
Parents: e4d8b88
Author: Amareshwari Sriramadasu
Authored: Fri Jul 29 10:11:32 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Fri Jul 29 10:11:32 2016 +0530
--
.../lens/cube/parse/TestUnionQueries.java | 176 +++
1 file changed, 100 insertions(+), 76 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/3f03d29b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java
index f5657e5..d5bc81c 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java
@@ -218,96 +218,120 @@ public class TestUnionQueries extends TestQueryRewrite {
conf.set(getValidUpdatePeriodsKey("testfact", "C1"), "DAILY,HOURLY");
conf.set(getValidUpdatePeriodsKey("testfact2", "C1"), "YEARLY");
conf.set(getValidUpdatePeriodsKey("testfact", "C2"), "MONTHLY,DAILY");
-ArrayList storages = Lists.newArrayList("c1_testfact",
"c2_testfact");
-getStorageToUpdatePeriodMap().put("c1_testfact",
Lists.newArrayList(HOURLY, DAILY));
-getStorageToUpdatePeriodMap().put("c2_testfact",
Lists.newArrayList(MONTHLY));
-StoragePartitionProvider provider = new StoragePartitionProvider() {
- @Override
- public Map providePartitionsForStorage(String storage) {
-return getWhereForMonthlyDailyAndHourly2monthsUnionQuery(storage);
- }
-};
+
String hqlQuery = rewrite("select asciicity as `City Name`, cityAndState
as citystate, isIndia as isIndia,"
+ " msr8, msr7 as `Third measure` "
+ "from testCube where asciicity = 'c' and cityname = 'a' and zipcode =
'b' and "
+ TWO_MONTHS_RANGE_UPTO_HOURS, conf);
-String expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider,
- "SELECT testcube.alias0 as `City Name`, testcube.alias1 as citystate,
testcube.alias2 as isIndia, "
-+ "sum(testcube.alias3) + max(testcube.alias4), "
-+ "case when sum(testcube.alias3) = 0 then 0 else
sum(testcube.alias5)/sum(testcube.alias3) end "
-+ "as `Third Measure`",
- null, " group by testcube.alias0, testcube.alias1, testcube.alias2",
- "select ascii(cubecity.name) as `alias0`, concat(cubecity.name, \":\",
cubestate.name) as alias1,"
-+ "cubecity.name == 'DELHI' OR cubestate.name == 'KARNATAKA' OR
cubestate.name == 'MAHARASHTRA' as alias2,"
-+ "sum(testcube.msr2) as `alias3`, max(testcube.msr3) as `alias4`, "
-+ "sum(case when testcube.cityid = 'x' then testcube.msr21 else
testcube.msr22 end) as `alias5`", " join "
-+ getDbName() + "c1_statetable cubestate on testcube.stateid =
cubestate.id and (cubestate.dt = 'latest') join"
-+ getDbName() + "c1_citytable cubecity on testcube.cityid =
cubecity.id and (cubecity.dt = 'latest')",
- "ascii(cubecity.name) = 'c' and cubecity.name = 'a' and testcube.zipcode
= 'b'",
- " group by ascii(cubecity.name)), concat(cubecity.name, \":\",
cubestate.name),"
-+ "cubecity.name == 'DELHI' OR cubestate.name == 'KARNATAKA' OR
cubestate.name == 'MAHARASHTRA'");
-compareQueries(hqlQuery, expected);
+String joinExpr1 = " join "
+ + getDbName() + "c1_statetable cubestate on testcube.stateid =
cubestate.id and (cubestate.dt = 'latest') join"
+ + getDbName() + "c1_citytable cubecity on testcube.cityid = cubecity.id
and (cubecity.dt = 'latest')";
+String joinExpr2 = " join "
+ + getDbName() + "c1_citytable cubecity on testcube.cityid = cubecity.id
and (cubecity.d
lens git commit: LENS-1242: (incremental)JDBCDriver cancel query is taking huge time
Repository: lens
Updated Branches:
refs/heads/current-release-line f330010cd -> 7ae4a5472
LENS-1242: (incremental)JDBCDriver cancel query is taking huge time
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7ae4a547
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7ae4a547
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7ae4a547
Branch: refs/heads/current-release-line
Commit: 7ae4a547294e23ec0b61d89679c9137f86709fb2
Parents: f330010
Author: Amareshwari Sriramadasu
Authored: Mon Aug 1 17:18:21 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Aug 2 09:37:31 2016 +0530
--
.../src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/7ae4a547/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
--
diff --git
a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 15a1826..bebb9ae 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -217,7 +217,7 @@ public class JDBCDriver extends AbstractLensDriver {
log.debug("Done resultFuture cancel!");
// queryResult object would be null if query is not yet launched - since
we did future.cancel, no other cancel is
// required.
- if (queryResult != null && isStatementCancelSupported) {
+ if (queryResult != null && queryResult.stmt != null &&
isStatementCancelSupported) {
log.debug("Cancelling query through statement cancel");
try {
queryResult.stmt.cancel();
lens git commit: LENS-1254 : Fix TestLensQueryCommands.showPersistentResultSet failure
Repository: lens
Updated Branches:
refs/heads/master 90c710018 -> ca372c1dc
LENS-1254 : Fix TestLensQueryCommands.showPersistentResultSet failure
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/ca372c1d
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/ca372c1d
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/ca372c1d
Branch: refs/heads/master
Commit: ca372c1dc28c79677d0f417370bf09fcf1a329e8
Parents: 90c7100
Author: Puneet Gupta
Authored: Thu Aug 4 10:42:11 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Aug 4 10:42:11 2016 +0530
--
.../src/test/java/org/apache/lens/cli/TestLensQueryCommands.java| 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/ca372c1d/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
--
diff --git
a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
index ae30e9f..e1b19de 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
@@ -461,6 +461,7 @@ public class TestLensQueryCommands extends
LensCliApplicationTest {
public void showPersistentResultSet(LensQueryCommands qCom) throws Exception
{
System.out.println("@@PERSISTENT_RESULT_TEST-");
qCom.getClient().setConnectionParam("lens.query.enable.persistent.resultset",
"true");
+
qCom.getClient().setConnectionParam("lens.query.prefetch.inmemory.resultset",
"false");
String query = "cube select id,name from test_dim";
try {
String result = qCom.executeQuery(query, false, "testQuery3");
lens git commit: LENS-1254 : Fix TestLensQueryCommands.showPersistentResultSet failure
Repository: lens
Updated Branches:
refs/heads/current-release-line 06c6f47b3 -> 68e01c2b4
LENS-1254 : Fix TestLensQueryCommands.showPersistentResultSet failure
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/68e01c2b
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/68e01c2b
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/68e01c2b
Branch: refs/heads/current-release-line
Commit: 68e01c2b40b15fc1f14e6071267823d77efd317c
Parents: 06c6f47
Author: Puneet Gupta
Authored: Thu Aug 4 10:42:11 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Aug 4 10:46:47 2016 +0530
--
.../src/test/java/org/apache/lens/cli/TestLensQueryCommands.java| 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/68e01c2b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
--
diff --git
a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
index ae30e9f..e1b19de 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
@@ -461,6 +461,7 @@ public class TestLensQueryCommands extends
LensCliApplicationTest {
public void showPersistentResultSet(LensQueryCommands qCom) throws Exception
{
System.out.println("@@PERSISTENT_RESULT_TEST-");
qCom.getClient().setConnectionParam("lens.query.enable.persistent.resultset",
"true");
+
qCom.getClient().setConnectionParam("lens.query.prefetch.inmemory.resultset",
"false");
String query = "cube select id,name from test_dim";
try {
String result = qCom.executeQuery(query, false, "testQuery3");
lens git commit: LENS-1246 : Fix NPE in explainFailedAttempt in Lens Client
Repository: lens
Updated Branches:
refs/heads/current-release-line 68e01c2b4 -> fbe44ff0b
LENS-1246 : Fix NPE in explainFailedAttempt in Lens Client
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/fbe44ff0
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/fbe44ff0
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/fbe44ff0
Branch: refs/heads/current-release-line
Commit: fbe44ff0be1ceca163a3d74310fa10b5ac5918eb
Parents: 68e01c2
Author: Rajat Khandelwal
Authored: Thu Aug 4 10:51:56 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Aug 4 10:52:35 2016 +0530
--
.../apache/lens/client/LensClientSingletonWrapper.java | 12 +---
1 file changed, 9 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/fbe44ff0/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
--
diff --git
a/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
b/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
index 4629559..3fd754e 100644
---
a/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
+++
b/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
@@ -58,15 +58,21 @@ public class LensClientSingletonWrapper {
log.error("failed login attempt", e);
switch (e.getErrorCode()) {
case 401:
- System.console().printf("username/password combination incorrect.\n");
+ printError("username/password combination incorrect.");
break;
case 500:
- System.console().printf("server unresponsive, Returned error code
500\n");
+ printError("server unresponsive, Returned error code 500");
break;
default:
- System.console().printf("ERROR: " + e.getMessage() + "\n");
+ printError("ERROR: " + e.getMessage());
}
}
+ private void printError(String error) {
+if (System.console() != null) {
+ System.console().printf(error + "\n");
+}
+log.error(error);
+ }
public LensClient getClient() {
if (client == null) {
lens git commit: LENS-1246 : Fix NPE in explainFailedAttempt in Lens Client
Repository: lens
Updated Branches:
refs/heads/master ca372c1dc -> 77bc8c20b
LENS-1246 : Fix NPE in explainFailedAttempt in Lens Client
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/77bc8c20
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/77bc8c20
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/77bc8c20
Branch: refs/heads/master
Commit: 77bc8c20b0a59f0349a9cfd8f51f48e00250b4b0
Parents: ca372c1
Author: Rajat Khandelwal
Authored: Thu Aug 4 10:51:56 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Aug 4 10:51:56 2016 +0530
--
.../apache/lens/client/LensClientSingletonWrapper.java | 12 +---
1 file changed, 9 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/77bc8c20/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
--
diff --git
a/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
b/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
index 4629559..3fd754e 100644
---
a/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
+++
b/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
@@ -58,15 +58,21 @@ public class LensClientSingletonWrapper {
log.error("failed login attempt", e);
switch (e.getErrorCode()) {
case 401:
- System.console().printf("username/password combination incorrect.\n");
+ printError("username/password combination incorrect.");
break;
case 500:
- System.console().printf("server unresponsive, Returned error code
500\n");
+ printError("server unresponsive, Returned error code 500");
break;
default:
- System.console().printf("ERROR: " + e.getMessage() + "\n");
+ printError("ERROR: " + e.getMessage());
}
}
+ private void printError(String error) {
+if (System.console() != null) {
+ System.console().printf(error + "\n");
+}
+log.error(error);
+ }
public LensClient getClient() {
if (client == null) {
lens git commit: LENS-1245 : Fix error message in LensAPIException
Repository: lens
Updated Branches:
refs/heads/master 77bc8c20b -> 9ea1513b4
LENS-1245 : Fix error message in LensAPIException
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/9ea1513b
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/9ea1513b
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/9ea1513b
Branch: refs/heads/master
Commit: 9ea1513b400b9896270d4e28fc47cdff63a7c970
Parents: 77bc8c2
Author: Amareshwari Sriramadasu
Authored: Thu Aug 4 11:04:10 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Aug 4 11:04:10 2016 +0530
--
.../java/org/apache/lens/client/exceptions/LensAPIException.java | 4
1 file changed, 4 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/9ea1513b/lens-client/src/main/java/org/apache/lens/client/exceptions/LensAPIException.java
--
diff --git
a/lens-client/src/main/java/org/apache/lens/client/exceptions/LensAPIException.java
b/lens-client/src/main/java/org/apache/lens/client/exceptions/LensAPIException.java
index fcc53c5..13fc7e8 100644
---
a/lens-client/src/main/java/org/apache/lens/client/exceptions/LensAPIException.java
+++
b/lens-client/src/main/java/org/apache/lens/client/exceptions/LensAPIException.java
@@ -45,4 +45,8 @@ public class LensAPIException extends Exception {
public String getLensAPIRequestId() {
return this.errorResult.getId();
}
+
+ public String getMessage() {
+return getLensAPIErrorMessage();
+ }
}
lens git commit: LENS-1245 : Fix error message in LensAPIException
Repository: lens
Updated Branches:
refs/heads/current-release-line fbe44ff0b -> 1b0326948
LENS-1245 : Fix error message in LensAPIException
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/1b032694
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/1b032694
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/1b032694
Branch: refs/heads/current-release-line
Commit: 1b0326948b43190370d9c32ac14cb521a449479d
Parents: fbe44ff
Author: Amareshwari Sriramadasu
Authored: Thu Aug 4 11:04:10 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Aug 4 11:04:39 2016 +0530
--
.../java/org/apache/lens/client/exceptions/LensAPIException.java | 4
1 file changed, 4 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/1b032694/lens-client/src/main/java/org/apache/lens/client/exceptions/LensAPIException.java
--
diff --git
a/lens-client/src/main/java/org/apache/lens/client/exceptions/LensAPIException.java
b/lens-client/src/main/java/org/apache/lens/client/exceptions/LensAPIException.java
index fcc53c5..13fc7e8 100644
---
a/lens-client/src/main/java/org/apache/lens/client/exceptions/LensAPIException.java
+++
b/lens-client/src/main/java/org/apache/lens/client/exceptions/LensAPIException.java
@@ -45,4 +45,8 @@ public class LensAPIException extends Exception {
public String getLensAPIRequestId() {
return this.errorResult.getId();
}
+
+ public String getMessage() {
+return getLensAPIErrorMessage();
+ }
}
lens git commit: LENS-1264 : Fix aggregate resolution with aggregate around dim attributes
Repository: lens
Updated Branches:
refs/heads/master 9ea1513b4 -> 017c40310
LENS-1264 : Fix aggregate resolution with aggregate around dim attributes
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/017c4031
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/017c4031
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/017c4031
Branch: refs/heads/master
Commit: 017c40310e39af81bb75c6888a2494dd4bd4304a
Parents: 9ea1513
Author: Sushil Mohanty
Authored: Mon Aug 8 10:01:16 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Mon Aug 8 10:01:16 2016 +0530
--
.../lens/cube/parse/AggregateResolver.java | 24 +++-
.../apache/lens/cube/parse/CubeTestSetup.java | 5
.../lens/cube/parse/TestCubeRewriter.java | 18 +++
3 files changed, 46 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/017c4031/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
index ef2ca4e..292868a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
@@ -26,6 +26,7 @@ import java.util.Iterator;
import org.apache.lens.cube.error.LensCubeErrorCode;
import org.apache.lens.cube.metadata.CubeMeasure;
+import org.apache.lens.cube.metadata.ExprColumn;
import
org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext;
import org.apache.lens.server.api.error.LensException;
@@ -102,12 +103,33 @@ class AggregateResolver implements ContextRewriter {
// Check if any measure/aggregate columns and distinct clause used in
// select tree. If not, update selectAST token "SELECT" to "SELECT
DISTINCT"
if (!hasMeasures(cubeql, cubeql.getSelectAST()) &&
!isDistinctClauseUsed(cubeql.getSelectAST())
-&& !HQLParser.hasAggregate(cubeql.getSelectAST())) {
+&& !HQLParser.hasAggregate(cubeql.getSelectAST())
+ && !isAggregateDimExprUsedInSelect(cubeql, cubeql.getSelectAST())) {
cubeql.getSelectAST().getToken().setType(HiveParser.TOK_SELECTDI);
}
}
}
+ private boolean isAggregateDimExprUsedInSelect(CubeQueryContext cubeql,
ASTNode selectAST) throws LensException {
+for (int i = 0; i < selectAST.getChildCount(); i++) {
+ ASTNode child = (ASTNode) selectAST.getChild(i);
+ String expr = HQLParser.getString((ASTNode)
child.getChild(0).getChild(1));
+ if (cubeql.getQueriedExprs().contains(expr)) {
+for (Iterator itrContext =
+ cubeql.getExprCtx().getAllExprsQueried().get(expr).iterator();
itrContext.hasNext();) {
+ for (Iterator itrCol =
+ itrContext.next().getExprCol().getExpressionSpecs().iterator();
itrCol.hasNext();) {
+ASTNode exprAST = HQLParser.parseExpr(itrCol.next().getExpr());
+if (HQLParser.isAggregateAST(exprAST)) {
+ return true;
+}
+ }
+}
+ }
+}
+return false;
+ }
+
// We need to traverse the clause looking for eligible measures which can be
// wrapped inside aggregates
// We have to skip any columns that are already inside an aggregate UDAF
http://git-wip-us.apache.org/repos/asf/lens/blob/017c4031/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 48652f2..2631f40 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -659,6 +659,11 @@ public class CubeTestSetup {
"substr(dim2big1, 5)"));
exprs.add(new ExprColumn(new FieldSchema("asciicity", "String", "ascii
cityname"), "ascii cityname substr",
"ascii(cityname)"));
+exprs.add(new ExprColumn(new FieldSchema("countofdistinctcityid", "int",
"Count of Distinct CityId"),
+"Count of Distinct CityId Expr", "count(distinct(cityid))"));
+exprs.add(new ExprColumn(new FieldSchema("notnullcityid", "int", "Not null
cityid"),
+"Not null cityid Exp
lens git commit: LENS-197 : Allow start and end times for columns in fact tables
Repository: lens
Updated Branches:
refs/heads/master 017c40310 -> 934f84573
LENS-197 : Allow start and end times for columns in fact tables
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/934f8457
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/934f8457
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/934f8457
Branch: refs/heads/master
Commit: 934f845737558349af00ce4de6ffcb16c6fa904b
Parents: 017c403
Author: Sushil Mohanty
Authored: Mon Aug 8 10:04:10 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Mon Aug 8 10:04:10 2016 +0530
--
lens-api/src/main/resources/cube-0.1.xsd| 2 +
.../lens/cube/metadata/MetastoreConstants.java | 2 +
.../lens/cube/parse/CandidateTableResolver.java | 83 +-
.../lens/cube/parse/CubeQueryRewriter.java | 6 +-
.../lens/cube/parse/TimeRangeChecker.java | 258 +++
.../lens/cube/parse/TimerangeResolver.java | 124 -
.../apache/lens/cube/parse/CubeTestSetup.java | 45 +++-
.../lens/cube/parse/TestBaseCubeQueries.java| 2 +-
.../lens/cube/parse/TestCubeRewriter.java | 20 ++
.../lens/cube/parse/TestQueryMetrics.java | 31 +--
lens-examples/src/main/resources/fact1.xml | 2 +
.../src/test/resources/yaml/fact1.yaml | 6 +
.../metastore/CubeMetastoreServiceImpl.java | 8 +-
.../apache/lens/server/metastore/JAXBUtils.java | 16 ++
.../server/metastore/TestMetastoreService.java | 12 +
15 files changed, 459 insertions(+), 158 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/934f8457/lens-api/src/main/resources/cube-0.1.xsd
--
diff --git a/lens-api/src/main/resources/cube-0.1.xsd
b/lens-api/src/main/resources/cube-0.1.xsd
index d6c6bd2..f438f48 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -711,6 +711,8 @@
+
+
http://git-wip-us.apache.org/repos/asf/lens/blob/934f8457/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
index 61675bc..4585ef7 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java
@@ -55,6 +55,8 @@ public final class MetastoreConstants {
public static final String FACT_RELATIVE_START_TIME =
"cube.fact.relative.start.time";
public static final String FACT_ABSOLUTE_END_TIME =
"cube.fact.absolute.end.time";
public static final String FACT_RELATIVE_END_TIME =
"cube.fact.relative.end.time";
+ public static final String FACT_COL_START_TIME_PFX =
"cube.fact.col.start.time.";
+ public static final String FACT_COL_END_TIME_PFX = "cube.fact.col.end.time.";
// Segmentation constants
public static final String SEGMENTATION_KEY_PFX =
"cube.segmentation.internal.";
http://git-wip-us.apache.org/repos/asf/lens/blob/934f8457/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index 12d6e3a..83e5088 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration;
import com.google.common.collect.Sets;
+import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
/**
@@ -175,6 +176,58 @@ class CandidateTableResolver implements ContextRewriter {
}
}
+ public static boolean isColumnAvailableInRange(final TimeRange range, Date
startTime, Date endTime) {
+return (isColumnAvailableFrom(range.getFromDate(), startTime)
+&& isColumnAvailableTill(range.getToDate(), endTime));
+ }
+
+ public static boolean isColumnAvailableFrom(@NonNull final Date date, Date
startTime) {
+return (startTime == null) ? true : date.equals(startTime) ||
date.after(startTime);
+ }
+
+ public static boolean isColumnAvailableTill(@NonNull final Date date, Date
endTime) {
+return (endTime == null) ? true : date.equals(endTime) ||
date.before(endTime);
+ }
+
+ public static boolean isFactColumnValidForRange(CubeQueryContex
lens git commit: LENS-1270 : AlarmService should be inited through configuration
Repository: lens
Updated Branches:
refs/heads/master 37136a577 -> 12d478244
LENS-1270 : AlarmService should be inited through configuration
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/12d47824
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/12d47824
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/12d47824
Branch: refs/heads/master
Commit: 12d4782444b77e6abbb311da0b9b0a2392cda871
Parents: 37136a5
Author: Amareshwari Sriramadasu
Authored: Wed Aug 10 08:19:32 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed Aug 10 08:19:32 2016 +0530
--
.../org/apache/lens/server/LensServices.java| 2 -
.../lens/server/scheduler/AlarmService.java | 7 +-
.../src/main/resources/lensserver-default.xml | 8 +-
lens-server/src/test/resources/lens-site.xml| 2 +-
src/site/apt/admin/config.apt | 224 ++-
5 files changed, 124 insertions(+), 119 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/12d47824/lens-server/src/main/java/org/apache/lens/server/LensServices.java
--
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
index a709f16..7ff5cbb 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
@@ -37,7 +37,6 @@ import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.metrics.MetricsServiceImpl;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
-import org.apache.lens.server.scheduler.AlarmService;
import org.apache.lens.server.session.LensSessionImpl;
import org.apache.lens.server.stats.StatisticsService;
import org.apache.lens.server.user.UserConfigLoaderFactory;
@@ -203,7 +202,6 @@ public class LensServices extends CompositeService
implements ServiceProvider {
UserConfigLoaderFactory.init(conf);
// Add default services
addService(cliService);
- addService(new AlarmService(AlarmService.NAME));
addService(new EventServiceImpl(LensEventService.NAME));
addService(new MetricsServiceImpl(MetricsService.NAME));
addService(new StatisticsService(StatisticsService.STATS_SVC_NAME));
http://git-wip-us.apache.org/repos/asf/lens/blob/12d47824/lens-server/src/main/java/org/apache/lens/server/scheduler/AlarmService.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/scheduler/AlarmService.java
b/lens-server/src/main/java/org/apache/lens/server/scheduler/AlarmService.java
index 217879f..4491261 100644
---
a/lens-server/src/main/java/org/apache/lens/server/scheduler/AlarmService.java
+++
b/lens-server/src/main/java/org/apache/lens/server/scheduler/AlarmService.java
@@ -48,7 +48,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class AlarmService extends AbstractService implements LensService {
- public static final String NAME = "alarm-service";
+ public static final String NAME = "alarm";
public static final String LENS_JOBS = "LensJobs";
public static final String ALARM_SERVICE = "AlarmService";
@@ -68,10 +68,9 @@ public class AlarmService extends AbstractService implements
LensService {
/**
* Creates a new instance of AlarmService.
*
- * @param name the name
*/
- public AlarmService(String name) {
-super(name);
+ public AlarmService() {
+super(NAME);
}
@Override
http://git-wip-us.apache.org/repos/asf/lens/blob/12d47824/lens-server/src/main/resources/lensserver-default.xml
--
diff --git a/lens-server/src/main/resources/lensserver-default.xml
b/lens-server/src/main/resources/lensserver-default.xml
index d672c88..460e3dd 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -55,7 +55,7 @@
lens.server.servicenames
-session,query,savedquery,metastore,scheduler,quota
+session,alarm,query,savedquery,metastore,scheduler,quota
These services would be started in the specified order when
lens-server starts up
@@ -90,6 +90,12 @@
+lens.server.alarm.service.impl
+org.apache.lens.server.scheduler.AlarmService
+Implementation class for alarm service
+
+
+
lens.server.scheduler.service.impl
org.apache.lens.server.scheduler.SchedulerServiceImpl
Implementation class for query scheduler service
http://git-wip-us.apache.o
lens git commit: LENS-1270 : AlarmService should be inited through configuration
Repository: lens
Updated Branches:
refs/heads/current-release-line 1b0326948 -> 05533d1a0
LENS-1270 : AlarmService should be inited through configuration
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/05533d1a
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/05533d1a
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/05533d1a
Branch: refs/heads/current-release-line
Commit: 05533d1a094db317eccbae86fb2c59b4ef368717
Parents: 1b03269
Author: Amareshwari Sriramadasu
Authored: Wed Aug 10 08:19:32 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed Aug 10 08:22:07 2016 +0530
--
.../org/apache/lens/server/LensServices.java| 2 -
.../lens/server/scheduler/AlarmService.java | 7 +-
.../src/main/resources/lensserver-default.xml | 8 +-
lens-server/src/test/resources/lens-site.xml| 2 +-
src/site/apt/admin/config.apt | 220 ++-
5 files changed, 122 insertions(+), 117 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/05533d1a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
--
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
index a709f16..7ff5cbb 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
@@ -37,7 +37,6 @@ import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.metrics.MetricsServiceImpl;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
-import org.apache.lens.server.scheduler.AlarmService;
import org.apache.lens.server.session.LensSessionImpl;
import org.apache.lens.server.stats.StatisticsService;
import org.apache.lens.server.user.UserConfigLoaderFactory;
@@ -203,7 +202,6 @@ public class LensServices extends CompositeService
implements ServiceProvider {
UserConfigLoaderFactory.init(conf);
// Add default services
addService(cliService);
- addService(new AlarmService(AlarmService.NAME));
addService(new EventServiceImpl(LensEventService.NAME));
addService(new MetricsServiceImpl(MetricsService.NAME));
addService(new StatisticsService(StatisticsService.STATS_SVC_NAME));
http://git-wip-us.apache.org/repos/asf/lens/blob/05533d1a/lens-server/src/main/java/org/apache/lens/server/scheduler/AlarmService.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/scheduler/AlarmService.java
b/lens-server/src/main/java/org/apache/lens/server/scheduler/AlarmService.java
index 217879f..4491261 100644
---
a/lens-server/src/main/java/org/apache/lens/server/scheduler/AlarmService.java
+++
b/lens-server/src/main/java/org/apache/lens/server/scheduler/AlarmService.java
@@ -48,7 +48,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class AlarmService extends AbstractService implements LensService {
- public static final String NAME = "alarm-service";
+ public static final String NAME = "alarm";
public static final String LENS_JOBS = "LensJobs";
public static final String ALARM_SERVICE = "AlarmService";
@@ -68,10 +68,9 @@ public class AlarmService extends AbstractService implements
LensService {
/**
* Creates a new instance of AlarmService.
*
- * @param name the name
*/
- public AlarmService(String name) {
-super(name);
+ public AlarmService() {
+super(NAME);
}
@Override
http://git-wip-us.apache.org/repos/asf/lens/blob/05533d1a/lens-server/src/main/resources/lensserver-default.xml
--
diff --git a/lens-server/src/main/resources/lensserver-default.xml
b/lens-server/src/main/resources/lensserver-default.xml
index e9525fd..4e20dea 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -55,7 +55,7 @@
lens.server.servicenames
-session,query,savedquery,metastore,scheduler,quota
+session,alarm,query,savedquery,metastore,scheduler,quota
These services would be started in the specified order when
lens-server starts up
@@ -90,6 +90,12 @@
+lens.server.alarm.service.impl
+org.apache.lens.server.scheduler.AlarmService
+Implementation class for alarm service
+
+
+
lens.server.scheduler.service.impl
org.apache.lens.server.scheduler.SchedulerServiceImpl
Implementation class for query scheduler service
lens git commit: LENS-1272 : Check-style fixes after LENS-1265
Repository: lens
Updated Branches:
refs/heads/master a0b3f818f -> 603cd0fd0
LENS-1272 : Check-style fixes after LENS-1265
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/603cd0fd
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/603cd0fd
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/603cd0fd
Branch: refs/heads/master
Commit: 603cd0fd0d8dd5fa978c9bdd625a6c9ed76d05bc
Parents: a0b3f81
Author: Lavkesh Lahngir
Authored: Wed Aug 10 19:21:43 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed Aug 10 19:21:43 2016 +0530
--
.../main/java/org/apache/lens/server/scheduler/SchedulerDAO.java | 3 ++-
.../org/apache/lens/server/scheduler/SchedulerServiceImpl.java| 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/603cd0fd/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java
b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java
index 6a4c77b..966a64e 100644
---
a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java
+++
b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java
@@ -665,7 +665,8 @@ public class SchedulerDAO {
"CREATE TABLE IF NOT EXISTS " + JOB_INSTANCE_RUN_TABLE + "( " +
COLUMN_ID + " VARCHAR(255) NOT NULL, "
+ COLUMN_RUN_ID + " INT NOT NULL, " + COLUMN_SESSION_HANDLE + "
VARCHAR(255), " + COLUMN_START_TIME
+ " BIGINT, " + COLUMN_END_TIME + " BIGINT, " +
COLUMN_RESULT_PATH + " TEXT, " + COLUMN_QUERY_HANDLE
- + " VARCHAR(255), " + COLUMN_STATUS + " VARCHAR(20), " + "
PRIMARY KEY ( " + COLUMN_ID + ", " + COLUMN_RUN_ID
+ + " VARCHAR(255), " + COLUMN_STATUS + " VARCHAR(20), " + "
PRIMARY KEY ( " + COLUMN_ID + ", "
+ + COLUMN_RUN_ID
+ ")" + ")";
runner.update(createSQL);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/603cd0fd/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
index 28d7e27..74b337c 100644
---
a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
@@ -97,8 +97,9 @@ public class SchedulerServiceImpl extends BaseLensService
implements SchedulerSe
private void doesSessionBelongToUser(LensSessionHandle sessionHandle, String
user) throws LensException {
LensSessionImpl session = getSession(sessionHandle);
-if (!session.getLoggedInUser().equals(user))
+if (!session.getLoggedInUser().equals(user)) {
throw new LensException("Logged in user " + session.getLoggedInUser() +
" is not same as " + user);
+}
}
@Override
lens git commit: LENS-1268: Lens build on dev enviornment is failing
Repository: lens Updated Branches: refs/heads/current-release-line d37e84994 -> baf4e765c LENS-1268: Lens build on dev enviornment is failing Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/baf4e765 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/baf4e765 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/baf4e765 Branch: refs/heads/current-release-line Commit: baf4e765c6ca0690686762767456514b9240b1a0 Parents: d37e849 Author: Praveen Adlakha Authored: Tue Aug 9 13:19:43 2016 +0530 Committer: Amareshwari Sriramadasu Committed: Wed Aug 17 18:45:04 2016 +0530 -- pom.xml | 4 1 file changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/lens/blob/baf4e765/pom.xml -- diff --git a/pom.xml b/pom.xml index c85dc17..97cb612 100644 --- a/pom.xml +++ b/pom.xml @@ -1268,6 +1268,10 @@ org.apache.avro avro-mapred + +org.pentaho +pentaho-aggdesigner-algorithm +
lens git commit: LENS-1274 : Saved query table create failure shouldn't stop lens server from starting
Repository: lens
Updated Branches:
refs/heads/master 603cd0fd0 -> 66bd58e4c
LENS-1274 : Saved query table create failure shouldn't stop lens server from
starting
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/66bd58e4
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/66bd58e4
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/66bd58e4
Branch: refs/heads/master
Commit: 66bd58e4c4354e88cb6af9dcfcb219fdaf9fdf2a
Parents: 603cd0f
Author: Rajat Khandelwal
Authored: Fri Aug 19 09:52:35 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Fri Aug 19 09:52:35 2016 +0530
--
.../org/apache/lens/server/query/save/SavedQueryDao.java | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/66bd58e4/lens-server/src/main/java/org/apache/lens/server/query/save/SavedQueryDao.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/save/SavedQueryDao.java
b/lens-server/src/main/java/org/apache/lens/server/query/save/SavedQueryDao.java
index 8762aff..4f072e2 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/save/SavedQueryDao.java
+++
b/lens-server/src/main/java/org/apache/lens/server/query/save/SavedQueryDao.java
@@ -45,7 +45,9 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import lombok.AllArgsConstructor;
import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
public class SavedQueryDao {
private static final ObjectMapper MAPPER = new ObjectMapper();
@@ -65,12 +67,12 @@ public class SavedQueryDao {
SavedQueryDao(String dialectClass, QueryRunner runner)
throws LensException {
try {
- this.runner = runner;
this.dialect = (Dialect) Class.forName(dialectClass).newInstance();
- createSavedQueryTableIfNotExists();
-} catch (Exception e) {
+} catch (InstantiationException | IllegalAccessException |
ClassNotFoundException e) {
throw new LensException("Error initializing saved query dao", e);
}
+this.runner = runner;
+createSavedQueryTableIfNotExists();
}
/**
@@ -82,7 +84,7 @@ public class SavedQueryDao {
try {
runner.update(dialect.getCreateTableSyntax());
} catch (SQLException e) {
- throw new LensException("Cannot create saved query table!", e);
+ log.warn("Unable to create saved query table.");
}
}
lens git commit: LENS-1278 : Fix too many instances of UDFClassLoader
Repository: lens
Updated Branches:
refs/heads/master 66bd58e4c -> 210e31fc0
LENS-1278 : Fix too many instances of UDFClassLoader
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/210e31fc
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/210e31fc
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/210e31fc
Branch: refs/heads/master
Commit: 210e31fc076a884b664dbbb87f8ab09fb481dc98
Parents: 66bd58e
Author: Amareshwari Sriramadasu
Authored: Fri Aug 19 12:37:02 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Fri Aug 19 12:37:02 2016 +0530
--
.../java/org/apache/lens/server/session/LensSessionImpl.java| 4
.../java/org/apache/lens/server/session/SessionClassLoader.java | 5 +++--
2 files changed, 7 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/210e31fc/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
index 3003ab7..a6013e7 100644
---
a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
@@ -221,6 +221,8 @@ public class LensSessionImpl extends HiveSessionImpl
implements AutoCloseable {
}
sessionDbClassLoaders.clear();
}
+// reset classloader in close
+
Thread.currentThread().setContextClassLoader(LensSessionImpl.class.getClassLoader());
}
public CubeMetastoreClient getCubeMetastoreClient() throws LensException {
@@ -272,6 +274,8 @@ public class LensSessionImpl extends HiveSessionImpl
implements AutoCloseable {
setActive();
if (acquireCount.decrementAndGet() == 0) {
super.release(userAccess);
+ // reset classloader in release
+
Thread.currentThread().setContextClassLoader(LensSessionImpl.class.getClassLoader());
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/210e31fc/lens-server/src/main/java/org/apache/lens/server/session/SessionClassLoader.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/session/SessionClassLoader.java
b/lens-server/src/main/java/org/apache/lens/server/session/SessionClassLoader.java
index f5e2068..0230db6 100644
---
a/lens-server/src/main/java/org/apache/lens/server/session/SessionClassLoader.java
+++
b/lens-server/src/main/java/org/apache/lens/server/session/SessionClassLoader.java
@@ -20,7 +20,8 @@ package org.apache.lens.server.session;
import java.io.IOException;
import java.net.URL;
-import java.net.URLClassLoader;
+
+import org.apache.hadoop.hive.ql.exec.UDFClassLoader;
import lombok.EqualsAndHashCode;
import lombok.Getter;
@@ -29,7 +30,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@EqualsAndHashCode(callSuper = false)
-public class SessionClassLoader extends URLClassLoader {
+public class SessionClassLoader extends UDFClassLoader {
@Getter
@Setter
private boolean closed;
lens git commit: LENS-1278 : Fix too many instances of UDFClassLoader
Repository: lens
Updated Branches:
refs/heads/current-release-line baf4e765c -> 35aba09ab
LENS-1278 : Fix too many instances of UDFClassLoader
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/35aba09a
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/35aba09a
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/35aba09a
Branch: refs/heads/current-release-line
Commit: 35aba09ab0288f67634f70ad65ea41ed89322892
Parents: baf4e76
Author: Amareshwari Sriramadasu
Authored: Fri Aug 19 12:37:02 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Mon Aug 22 10:32:29 2016 +0530
--
.../java/org/apache/lens/server/session/LensSessionImpl.java| 4
.../java/org/apache/lens/server/session/SessionClassLoader.java | 5 +++--
2 files changed, 7 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/35aba09a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
index 3003ab7..a6013e7 100644
---
a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
@@ -221,6 +221,8 @@ public class LensSessionImpl extends HiveSessionImpl
implements AutoCloseable {
}
sessionDbClassLoaders.clear();
}
+// reset classloader in close
+
Thread.currentThread().setContextClassLoader(LensSessionImpl.class.getClassLoader());
}
public CubeMetastoreClient getCubeMetastoreClient() throws LensException {
@@ -272,6 +274,8 @@ public class LensSessionImpl extends HiveSessionImpl
implements AutoCloseable {
setActive();
if (acquireCount.decrementAndGet() == 0) {
super.release(userAccess);
+ // reset classloader in release
+
Thread.currentThread().setContextClassLoader(LensSessionImpl.class.getClassLoader());
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/35aba09a/lens-server/src/main/java/org/apache/lens/server/session/SessionClassLoader.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/session/SessionClassLoader.java
b/lens-server/src/main/java/org/apache/lens/server/session/SessionClassLoader.java
index f5e2068..0230db6 100644
---
a/lens-server/src/main/java/org/apache/lens/server/session/SessionClassLoader.java
+++
b/lens-server/src/main/java/org/apache/lens/server/session/SessionClassLoader.java
@@ -20,7 +20,8 @@ package org.apache.lens.server.session;
import java.io.IOException;
import java.net.URL;
-import java.net.URLClassLoader;
+
+import org.apache.hadoop.hive.ql.exec.UDFClassLoader;
import lombok.EqualsAndHashCode;
import lombok.Getter;
@@ -29,7 +30,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@EqualsAndHashCode(callSuper = false)
-public class SessionClassLoader extends URLClassLoader {
+public class SessionClassLoader extends UDFClassLoader {
@Getter
@Setter
private boolean closed;
lens git commit: LENS-1241 : Fix tests failures with setup and teardown issues
Repository: lens
Updated Branches:
refs/heads/master 210e31fc0 -> 443ec5f5f
LENS-1241 : Fix tests failures with setup and teardown issues
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/443ec5f5
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/443ec5f5
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/443ec5f5
Branch: refs/heads/master
Commit: 443ec5f5fe18c5d60e8de835f3dedd08a0ffab7a
Parents: 210e31f
Author: Sushil Mohanty
Authored: Mon Aug 22 17:20:25 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Mon Aug 22 17:20:25 2016 +0530
--
.../lens/server/metrics/MetricsServiceImpl.java | 6 --
.../org/apache/lens/server/TestServerMode.java | 20 +++-
.../server/healthcheck/TestHealthChecks.java| 4 +---
.../server/metastore/TestMetastoreService.java | 11 +--
.../metrics/TestResourceMethodMetrics.java | 15 ++-
.../lens/server/query/TestEventService.java | 4 ++--
.../server/query/TestQueryEndEmailNotifier.java | 10 +-
.../server/query/TestQueryResultPurger.java | 8 +++-
.../lens/server/query/TestQueryService.java | 10 +-
.../lens/server/query/TestResultFormatting.java | 16 +++-
.../query/save/TestSavedQueryService.java | 14 ++
.../TestUserQueryToCubeQueryRewriter.java | 4 ++--
.../lens/server/scheduler/AlarmServiceTest.java | 4 ++--
.../server/session/TestSessionResource.java | 9 +++--
.../lens/server/user/TestUserConfigLoader.java | 8 +++-
15 files changed, 93 insertions(+), 50 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/443ec5f5/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
index e09180f..b88c717 100644
---
a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
@@ -585,12 +585,6 @@ public class MetricsServiceImpl extends AbstractService
implements MetricsServic
isHealthy = false;
}
-// Also unhealthy if 30% of queries have failed.
-if (getTotalFailedQueries()/(float)getTotalAcceptedQueries() > 0.3) {
- details.append("30% of queries have failed.");
- isHealthy = false;
-}
-
if (!isHealthy) {
log.error(details.toString());
}
http://git-wip-us.apache.org/repos/asf/lens/blob/443ec5f5/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
--
diff --git
a/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
b/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
index caf968a..bc3955d 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
@@ -43,9 +43,7 @@ import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.testng.Assert;
-import org.testng.annotations.AfterTest;
-import org.testng.annotations.BeforeTest;
-import org.testng.annotations.Test;
+import org.testng.annotations.*;
/**
* The Class TestServerMode.
@@ -61,8 +59,12 @@ public class TestServerMode extends
LensAllApplicationJerseyTest {
@BeforeTest
public void setUp() throws Exception {
super.setUp();
-LensServerTestUtil.createTable("test_table", target(),
RestAPITestUtil.openFooBarSession(target(), defaultMT),
- defaultMT);
+ }
+ @BeforeClass
+ public void create() throws Exception {
+LensSessionHandle fooBarSessonHandle =
RestAPITestUtil.openFooBarSession(target(), defaultMT);
+LensServerTestUtil.createTable("test_table", target(), fooBarSessonHandle,
defaultMT);
+RestAPITestUtil.closeSession(target(), fooBarSessonHandle, defaultMT);
}
/*
@@ -75,6 +77,14 @@ public class TestServerMode extends
LensAllApplicationJerseyTest {
super.tearDown();
}
+ @AfterClass
+ public void drop() throws Exception {
+LensSessionHandle fooBarSessonHandle =
RestAPITestUtil.openFooBarSession(target(), defaultMT);
+LensServerTestUtil.dropTable("test_table", target(), fooBarSessonHandle,
+defaultMT);
+RestAPITestUtil.closeSession(target(), fooBarSessonHandle, defaultMT);
+ }
+
/**
* Test read only mode.
*
http://git-wip-u
lens git commit: LENS-1281 : Fix license issue with quartz dependency
Repository: lens
Updated Branches:
refs/heads/master 443ec5f5f -> c6e3abc63
LENS-1281 : Fix license issue with quartz dependency
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c6e3abc6
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c6e3abc6
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c6e3abc6
Branch: refs/heads/master
Commit: c6e3abc63be51ff9953b6d0b5222745264ede49e
Parents: 443ec5f
Author: Ajay Yadava
Authored: Mon Aug 22 17:21:50 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Mon Aug 22 17:21:50 2016 +0530
--
pom.xml | 8
1 file changed, 8 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/c6e3abc6/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 7c03862..871d1ab 100644
--- a/pom.xml
+++ b/pom.xml
@@ -815,6 +815,14 @@
org.quartz-scheduler
quartz
${quartz.version}
+
+
+
+c3p0
+c3p0
+
+
org.projectlombok
[1/2] lens git commit: LENS-1241 : Fix tests failures with setup and teardown issues
Repository: lens
Updated Branches:
refs/heads/current-release-line 35aba09ab -> 15cf0b577
LENS-1241 : Fix tests failures with setup and teardown issues
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/8fcc1a12
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/8fcc1a12
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/8fcc1a12
Branch: refs/heads/current-release-line
Commit: 8fcc1a12b652c667fb153bb0e95e4a8de4f8638a
Parents: 35aba09
Author: Sushil Mohanty
Authored: Mon Aug 22 17:20:25 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Mon Aug 22 17:28:57 2016 +0530
--
.../lens/server/metrics/MetricsServiceImpl.java | 6 --
.../org/apache/lens/server/TestServerMode.java | 20 +++-
.../server/healthcheck/TestHealthChecks.java| 4 +---
.../server/metastore/TestMetastoreService.java | 11 +--
.../metrics/TestResourceMethodMetrics.java | 15 ++-
.../lens/server/query/TestEventService.java | 4 ++--
.../server/query/TestQueryEndEmailNotifier.java | 10 +-
.../server/query/TestQueryResultPurger.java | 8 +++-
.../lens/server/query/TestQueryService.java | 10 +-
.../lens/server/query/TestResultFormatting.java | 16 +++-
.../query/save/TestSavedQueryService.java | 14 ++
.../TestUserQueryToCubeQueryRewriter.java | 4 ++--
.../lens/server/scheduler/AlarmServiceTest.java | 4 ++--
.../server/session/TestSessionResource.java | 9 +++--
.../lens/server/user/TestUserConfigLoader.java | 8 +++-
15 files changed, 93 insertions(+), 50 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/8fcc1a12/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
index e09180f..b88c717 100644
---
a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
@@ -585,12 +585,6 @@ public class MetricsServiceImpl extends AbstractService
implements MetricsServic
isHealthy = false;
}
-// Also unhealthy if 30% of queries have failed.
-if (getTotalFailedQueries()/(float)getTotalAcceptedQueries() > 0.3) {
- details.append("30% of queries have failed.");
- isHealthy = false;
-}
-
if (!isHealthy) {
log.error(details.toString());
}
http://git-wip-us.apache.org/repos/asf/lens/blob/8fcc1a12/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
--
diff --git
a/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
b/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
index caf968a..bc3955d 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
@@ -43,9 +43,7 @@ import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.testng.Assert;
-import org.testng.annotations.AfterTest;
-import org.testng.annotations.BeforeTest;
-import org.testng.annotations.Test;
+import org.testng.annotations.*;
/**
* The Class TestServerMode.
@@ -61,8 +59,12 @@ public class TestServerMode extends
LensAllApplicationJerseyTest {
@BeforeTest
public void setUp() throws Exception {
super.setUp();
-LensServerTestUtil.createTable("test_table", target(),
RestAPITestUtil.openFooBarSession(target(), defaultMT),
- defaultMT);
+ }
+ @BeforeClass
+ public void create() throws Exception {
+LensSessionHandle fooBarSessonHandle =
RestAPITestUtil.openFooBarSession(target(), defaultMT);
+LensServerTestUtil.createTable("test_table", target(), fooBarSessonHandle,
defaultMT);
+RestAPITestUtil.closeSession(target(), fooBarSessonHandle, defaultMT);
}
/*
@@ -75,6 +77,14 @@ public class TestServerMode extends
LensAllApplicationJerseyTest {
super.tearDown();
}
+ @AfterClass
+ public void drop() throws Exception {
+LensSessionHandle fooBarSessonHandle =
RestAPITestUtil.openFooBarSession(target(), defaultMT);
+LensServerTestUtil.dropTable("test_table", target(), fooBarSessonHandle,
+defaultMT);
+RestAPITestUtil.closeSession(target(), fooBarSessonHandle, defaultMT);
+ }
+
/**
* Test read only m
[2/2] lens git commit: LENS-1281 : Fix license issue with quartz dependency
LENS-1281 : Fix license issue with quartz dependency
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/15cf0b57
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/15cf0b57
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/15cf0b57
Branch: refs/heads/current-release-line
Commit: 15cf0b577216d8bd6073391da98fbb5fc0dd01e4
Parents: 8fcc1a1
Author: Ajay Yadava
Authored: Mon Aug 22 17:21:50 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Mon Aug 22 17:29:08 2016 +0530
--
pom.xml | 8
1 file changed, 8 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/15cf0b57/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 97cb612..c181a58 100644
--- a/pom.xml
+++ b/pom.xml
@@ -815,6 +815,14 @@
org.quartz-scheduler
quartz
${quartz.version}
+
+
+
+c3p0
+c3p0
+
+
org.projectlombok
lens git commit: LENS-1269 : Time range fallback in lens examples should not give empty results
Repository: lens Updated Branches: refs/heads/master c6e3abc63 -> 99f8e344e LENS-1269 : Time range fallback in lens examples should not give empty results Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/99f8e344 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/99f8e344 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/99f8e344 Branch: refs/heads/master Commit: 99f8e344ee35a43c302d527d6b4579d9de091f40 Parents: c6e3abc Author: Rajat Khandelwal Authored: Mon Aug 22 17:35:10 2016 +0530 Committer: Amareshwari Sriramadasu Committed: Mon Aug 22 17:35:10 2016 +0530 -- lens-examples/src/main/resources/sales-aggr-fact2-local3.data | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/99f8e344/lens-examples/src/main/resources/sales-aggr-fact2-local3.data -- diff --git a/lens-examples/src/main/resources/sales-aggr-fact2-local3.data b/lens-examples/src/main/resources/sales-aggr-fact2-local3.data index a0cb704..407b097 100644 --- a/lens-examples/src/main/resources/sales-aggr-fact2-local3.data +++ b/lens-examples/src/main/resources/sales-aggr-fact2-local3.data @@ -1,2 +1,2 @@ -2015-04-13 00:00:00,2015-04-13 04:00:00,1,1,1,1,1,1,5,0 -2015-04-13 00:00:00,2015-04-13 04:00:00,2,1,2,2,2,1,8,2 \ No newline at end of file +2015-04-13 03:00:00,2015-04-13 04:00:00,1,1,1,1,1,1,5,0 +2015-04-13 03:00:00,2015-04-13 04:00:00,2,1,2,2,2,1,8,2 \ No newline at end of file
lens git commit: LENS-1269 : Time range fallback in lens examples should not give empty results
Repository: lens Updated Branches: refs/heads/current-release-line 15cf0b577 -> 2748cd9c4 LENS-1269 : Time range fallback in lens examples should not give empty results Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2748cd9c Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2748cd9c Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2748cd9c Branch: refs/heads/current-release-line Commit: 2748cd9c4fdf23b57c3ebd70c75d8311a6538cde Parents: 15cf0b5 Author: Rajat Khandelwal Authored: Mon Aug 22 17:35:10 2016 +0530 Committer: Amareshwari Sriramadasu Committed: Mon Aug 22 17:35:53 2016 +0530 -- lens-examples/src/main/resources/sales-aggr-fact2-local3.data | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/2748cd9c/lens-examples/src/main/resources/sales-aggr-fact2-local3.data -- diff --git a/lens-examples/src/main/resources/sales-aggr-fact2-local3.data b/lens-examples/src/main/resources/sales-aggr-fact2-local3.data index a0cb704..407b097 100644 --- a/lens-examples/src/main/resources/sales-aggr-fact2-local3.data +++ b/lens-examples/src/main/resources/sales-aggr-fact2-local3.data @@ -1,2 +1,2 @@ -2015-04-13 00:00:00,2015-04-13 04:00:00,1,1,1,1,1,1,5,0 -2015-04-13 00:00:00,2015-04-13 04:00:00,2,1,2,2,2,1,8,2 \ No newline at end of file +2015-04-13 03:00:00,2015-04-13 04:00:00,1,1,1,1,1,1,5,0 +2015-04-13 03:00:00,2015-04-13 04:00:00,2,1,2,2,2,1,8,2 \ No newline at end of file
lens git commit: LENS-1282: getUpdatedQueryContext() call is missing from QueryExecutionServiceImpl#executeTimeoutInternal
Repository: lens
Updated Branches:
refs/heads/current-release-line 2748cd9c4 -> 466cee342
LENS-1282: getUpdatedQueryContext() call is missing from
QueryExecutionServiceImpl#executeTimeoutInternal
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/466cee34
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/466cee34
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/466cee34
Branch: refs/heads/current-release-line
Commit: 466cee342d9657d8ca2348efbd7393c47ed0d3d0
Parents: 2748cd9
Author: Puneet Gupta
Authored: Mon Aug 22 18:20:48 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Aug 23 11:05:09 2016 +0530
--
.../apache/lens/server/query/QueryExecutionServiceImpl.java | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/466cee34/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 84dcecd..896ddbb 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -2294,9 +2294,15 @@ public class QueryExecutionServiceImpl extends
BaseLensService implements QueryE
long totalWaitMillisSoFar = 0;
synchronized (listener) {
while (totalWaitMillisSoFar < totalWaitTime
-&& !queryCtx.getStatus().executed() &&
!queryCtx.getStatus().finished()) {
+&& !listener.querySuccessful
+&& !queryCtx.getStatus().executed()
+&& !queryCtx.getStatus().finished()) {
listener.wait(waitMillisPerCheck);
totalWaitMillisSoFar += waitMillisPerCheck;
+if (!listener.querySuccessful) {
+ //update ths status in case query is not successful yet
+ queryCtx = getUpdatedQueryContext(sessionHandle, handle);
+}
}
}
} catch (InterruptedException e) {
lens git commit: LENS-1306 : Fix final rewritten when more than two facts are candidates for a query
Repository: lens
Updated Branches:
refs/heads/master 0013ec206 -> 5db78ce18
LENS-1306 : Fix final rewritten when more than two facts are candidates for a
query
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/5db78ce1
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/5db78ce1
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/5db78ce1
Branch: refs/heads/master
Commit: 5db78ce1826048d7099df50b4c92ff455a1472c0
Parents: 0013ec2
Author: Lavkesh Lahngir
Authored: Tue Sep 6 09:15:43 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Sep 6 09:15:43 2016 +0530
--
.../lens/cube/parse/MultiFactHQLContext.java| 2 +-
.../lens/cube/parse/TestBaseCubeQueries.java| 35
2 files changed, 36 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/5db78ce1/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
index e90da4c..7fbcd7e 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
@@ -152,7 +152,7 @@ class MultiFactHQLContext extends SimpleHQLContext {
fromBuilder.append(" AND ");
}
}
- if (i != facts.size()) {
+ if (i != facts.size() && firstFact.getDimFieldIndices().size() > 0) {
fromBuilder.append(" AND ");
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/5db78ce1/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
index 84e18dc..6fb027a 100644
---
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
+++
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
@@ -201,6 +201,41 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
}
@Test
+ public void testMoreThanTwoFactQueryWithNoDimensionsSelected() throws
Exception {
+CubeQueryContext ctx = rewriteCtx("select roundedmsr2, msr14, msr12 from
basecube" + " where " + TWO_DAYS_RANGE,
+ conf);
+Set candidateFacts = new HashSet();
+for (CandidateFact cfact : ctx.getCandidateFacts()) {
+ candidateFacts.add(cfact.getName().toLowerCase());
+}
+Assert.assertEquals(candidateFacts.size(), 3);
+Assert.assertTrue(candidateFacts.contains("testfact1_base"));
+Assert.assertTrue(candidateFacts.contains("testfact2_base"));
+Assert.assertTrue(candidateFacts.contains("testfact3_base"));
+String hqlQuery = ctx.toHQL();
+String expected1 = getExpectedQuery(cubeName, "select sum(basecube.msr12)
as `msr12` FROM ", null, null,
+ getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
+String expected2 = getExpectedQuery(cubeName, "select
round(sum(basecube.msr2)/1000) as `roundedmsr2` FROM ", null,
+ null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
+String expected3 = getExpectedQuery(cubeName, "select
count((basecube.msr14)) as `msr14` FROM ", null, null,
+ getWhereForDailyAndHourly2days(cubeName, "C1_testFact3_BASE"));
+compareContains(expected1, hqlQuery);
+compareContains(expected2, hqlQuery);
+compareContains(expected3, hqlQuery);
+String lower = hqlQuery.toLowerCase();
+assertTrue(lower.startsWith("select mq1.roundedmsr2 roundedmsr2, mq3.msr14
msr14, mq2.msr12 msr12 from ") || lower
+ .startsWith("select mq3.roundedmsr2 roundedmsr2, mq1.msr14 msr14,
mq2.msr12 msr12 from ") || lower
+ .startsWith("select mq2.roundedmsr2 roundedmsr2, mq3.msr14 msr14,
mq1.msr12 msr12 from ") || lower
+ .startsWith("select mq3.roundedmsr2 roundedmsr2, mq2.msr14 msr14,
mq1.msr12 msr12 from ") || lower
+ .startsWith("select mq1.roundedmsr2 roundedmsr2, mq2.msr14 msr14,
mq3.msr12 msr12 from ") || lower
+ .startsWith("select mq2.roundedmsr2 roundedmsr2, mq1.msr14 msr14,
mq3.msr12 msr12 from "), hqlQuery);
+assertTrue(lower.contains("mq1 full outer join") && lower.endsWith("mq3"));
+assertFalse(lower.contains("mq3 on"), hqlQuery);
+assertFalse(lower.cont
lens git commit: LENS-1298 : Make the test event handler sync
Repository: lens
Updated Branches:
refs/heads/master d9911f618 -> 85cb0c59f
LENS-1298 : Make the test event handler sync
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/85cb0c59
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/85cb0c59
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/85cb0c59
Branch: refs/heads/master
Commit: 85cb0c59f4300d92e8512eefcea5f4cc16f6aea6
Parents: d9911f6
Author: Lavkesh Lahngir
Authored: Thu Sep 8 09:38:19 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Sep 8 09:38:19 2016 +0530
--
.../test/java/org/apache/lens/server/query/TestEventService.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/85cb0c59/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
--
diff --git
a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
index 31b6625..526accc 100644
---
a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
+++
b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
@@ -568,10 +568,10 @@ public class TestEventService {
}
}
- private static class TestEventHandler extends AsyncEventListener {
+ private static class TestEventHandler implements
LensEventListener {
@Override
-public void process(TestEvent event) {
+public void onEvent(TestEvent event) throws LensException {
event.processed = true;
}
}
lens git commit: LENS-1197 : Update doc for api getAllFacts of a cube
Repository: lens
Updated Branches:
refs/heads/master 85cb0c59f -> 051ad2d3e
LENS-1197 : Update doc for api getAllFacts of a cube
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/051ad2d3
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/051ad2d3
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/051ad2d3
Branch: refs/heads/master
Commit: 051ad2d3ecd8c3364745297f3c915f1520b26b4c
Parents: 85cb0c5
Author: Archana H
Authored: Thu Sep 8 09:43:09 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Sep 8 09:43:09 2016 +0530
--
.../java/org/apache/lens/server/metastore/MetastoreResource.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/051ad2d3/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
index f684247..9d823da 100644
---
a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
+++
b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
@@ -668,7 +668,7 @@ public class MetastoreResource {
*
* @param sessionid The sessionid in which user is working
* @param cubeName name of the base cube or derived cube
- * @return List of {@link XFactTable} objects
+ * @return StringList consisting of all the fact names in the given cube
*/
@GET
@Path("/cubes/{cubeName}/facts")
lens git commit: LENS-1197 : Update doc for api getAllFacts of a cube
Repository: lens
Updated Branches:
refs/heads/current-release-line be839122d -> ca19ead47
LENS-1197 : Update doc for api getAllFacts of a cube
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/ca19ead4
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/ca19ead4
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/ca19ead4
Branch: refs/heads/current-release-line
Commit: ca19ead475e4f929a7154de4ca9c92dc034b0f2c
Parents: be83912
Author: Archana H
Authored: Thu Sep 8 09:43:09 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Sep 8 09:43:58 2016 +0530
--
.../java/org/apache/lens/server/metastore/MetastoreResource.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/ca19ead4/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
index f684247..9d823da 100644
---
a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
+++
b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
@@ -668,7 +668,7 @@ public class MetastoreResource {
*
* @param sessionid The sessionid in which user is working
* @param cubeName name of the base cube or derived cube
- * @return List of {@link XFactTable} objects
+ * @return StringList consisting of all the fact names in the given cube
*/
@GET
@Path("/cubes/{cubeName}/facts")
lens git commit: LENS-1304 : Log Driver query in lens finished_queries
Repository: lens
Updated Branches:
refs/heads/master 12774400d -> 7947acda7
LENS-1304 : Log Driver query in lens finished_queries
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7947acda
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7947acda
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7947acda
Branch: refs/heads/master
Commit: 7947acda7be47764f69ab32a08289bb8c9e7c9a1
Parents: 1277440
Author: Raghavendra Singh
Authored: Sat Sep 17 10:52:28 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Sat Sep 17 10:52:28 2016 +0530
--
.../server/api/query/FinishedLensQuery.java | 11 +
.../apache/lens/server/query/LensServerDAO.java | 11 +
.../apache/lens/server/query/TestLensDAO.java | 26 +---
3 files changed, 39 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/7947acda/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
index 1d56757..edd4284 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
@@ -175,6 +175,13 @@ public class FinishedLensQuery {
private String priority;
/**
+ * The selected driver's query.
+ */
+ @Getter
+ @Setter
+ private String driverQuery;
+
+ /**
* Instantiates a new finished lens query.
*/
public FinishedLensQuery() {
@@ -204,6 +211,7 @@ public class FinishedLensQuery {
this.selectedDriver = ctx.getSelectedDriver();
if (null != ctx.getSelectedDriver()) {
this.driverName = ctx.getSelectedDriver().getFullyQualifiedName();
+ this.driverQuery = ctx.getSelectedDriverQuery();
}
//Priority can be null in case no driver is fit to execute a query and
launch fails.
if (null != ctx.getPriority()) {
@@ -230,6 +238,9 @@ public class FinishedLensQuery {
qctx.getDriverStatus().setDriverFinishTime(getDriverEndTime());
qctx.setResultSetPath(getResult());
qctx.setQueryName(getQueryName());
+if (null != driverQuery){
+ qctx.setSelectedDriverQuery(driverQuery);
+}
if (getPriority() != null) {
qctx.setPriority(Priority.valueOf(getPriority()));
}
http://git-wip-us.apache.org/repos/asf/lens/blob/7947acda/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index 47159b5..2ec938e 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -83,11 +83,11 @@ public class LensServerDAO {
*/
public void createFinishedQueriesTable() throws Exception {
String sql = "CREATE TABLE if not exists finished_queries (handle
varchar(255) not null unique,"
- + "userquery varchar(1) not null," + "submitter varchar(255) not
null," + "priority varchar(255), "
+ + "userquery varchar(2) not null," + "submitter varchar(255) not
null," + "priority varchar(255), "
+ "starttime bigint, " + "endtime bigint," + "result varchar(255)," +
"status varchar(255), "
+ "metadata varchar(10), " + "rows int, " + "filesize bigint, " +
"errormessage varchar(1), "
+ "driverstarttime bigint, " + "driverendtime bigint, " + "drivername
varchar(1), "
- + "queryname varchar(255), " + "submissiontime bigint" + ")";
+ + "queryname varchar(255), " + "submissiontime bigint, " + "driverquery
varchar(100)" + ")";
try {
QueryRunner runner = new QueryRunner(ds);
runner.update(sql);
@@ -110,15 +110,16 @@ public class LensServerDAO {
Connection conn = null;
String sql = "insert into finished_queries (handle, userquery,
submitter, priority, "
+ "starttime,endtime,result,status,metadata,rows,filesize,"
-+ "errormessage,driverstarttime,driverendtime, drivername, queryname,
submissiontime)"
-+ " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?
lens git commit: LENS-1303 : Query lists on UI will use time filters
Repository: lens
Updated Branches:
refs/heads/master 7947acda7 -> b438aa2f2
LENS-1303 : Query lists on UI will use time filters
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b438aa2f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b438aa2f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b438aa2f
Branch: refs/heads/master
Commit: b438aa2f22992fc0e29f6ec10527959da0cefdf4
Parents: 7947acd
Author: Rajat Khandelwal
Authored: Sat Sep 17 10:57:37 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Sat Sep 17 10:57:37 2016 +0530
--
lens-ui/app/actions/AdhocQueryActions.js| 37 -
lens-ui/app/adapters/AdhocQueryAdapter.js | 23 +++
lens-ui/app/components/CubeSchemaComponent.js | 2 +-
.../app/components/QueryOperationsComponent.js | 12 +-
lens-ui/app/components/QueryPreviewComponent.js | 4 +
lens-ui/app/components/QueryResultsComponent.js | 153 +--
lens-ui/app/constants/AdhocQueryConstants.js| 3 +
lens-ui/app/stores/AdhocQueryStore.js | 68 +
8 files changed, 216 insertions(+), 86 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/b438aa2f/lens-ui/app/actions/AdhocQueryActions.js
--
diff --git a/lens-ui/app/actions/AdhocQueryActions.js
b/lens-ui/app/actions/AdhocQueryActions.js
index 38f2794..421f7ea 100644
--- a/lens-ui/app/actions/AdhocQueryActions.js
+++ b/lens-ui/app/actions/AdhocQueryActions.js
@@ -288,7 +288,42 @@ let AdhocQueryActions = {
});
});
},
-
+ getQueryHandles (secretToken, email, options) {
+AdhocQueryAdapter.getQueryHandles(secretToken, email, options)
+ .then(function (handles) {
+AppDispatcher.dispatch({
+ actionType: AdhocQueryConstants.RECEIVE_QUERY_HANDLES,
+ payload: { handles: handles }
+});
+ }, function (error) {
+AppDispatcher.dispatch({
+ actionType: AdhocQueryConstants.RECEIVE_QUERY_HANDLES_FAILED,
+ payload: {
+responseCode: error.status,
+responseMessage: error.statusText
+ }
+});
+ });
+ },
+ getQueriesDetails (secretToken, handles) {
+if (handles && handles.length) {
+ AdhocQueryAdapter.getQueriesDetails(secretToken, handles)
+.then(function (queries) {
+ AppDispatcher.dispatch({
+actionType: AdhocQueryConstants.RECEIVE_QUERIES,
+payload: {queries: queries}
+ });
+}, function (error) {
+ AppDispatcher.dispatch({
+actionType: AdhocQueryConstants.RECEIVE_QUERIES_FAILED,
+payload: {
+ responseCode: error.status,
+ responseMessage: error.statusText
+}
+ });
+});
+}
+ },
getQuery (secretToken, handle) {
AdhocQueryAdapter.getQuery(secretToken, handle)
.then(function (query) {
http://git-wip-us.apache.org/repos/asf/lens/blob/b438aa2f/lens-ui/app/adapters/AdhocQueryAdapter.js
--
diff --git a/lens-ui/app/adapters/AdhocQueryAdapter.js
b/lens-ui/app/adapters/AdhocQueryAdapter.js
index a54274f..9e3cc45 100644
--- a/lens-ui/app/adapters/AdhocQueryAdapter.js
+++ b/lens-ui/app/adapters/AdhocQueryAdapter.js
@@ -150,6 +150,29 @@ let AdhocQueryAdapter = {
}));
});
},
+ getQueryHandles (secretToken, email, options) {
+let queryOptions = {};
+queryOptions.sessionid = secretToken;
+queryOptions.user = email;
+var state;
+if (options && options.state) {
+ state = options.state.toUpperCase();
+}
+let handlesUrl = baseUrl + urls.query + '?sessionid=' + secretToken +
'&user=' +
+ email;
+if (state) handlesUrl += '&state=' + state;
+if (options.fromDate) handlesUrl += "&fromDate="+options.fromDate;
+if (options.toDate) handlesUrl += "&toDate="+options.toDate;
+return BaseAdapter.get(handlesUrl);
+ },
+ getQueriesDetails (secretToken, handles) {
+let url = baseUrl + urls.query + '?sessionid=' + secretToken;
+return Promise.all(handles.map((handle) => {
+ let queryUrl = baseUrl + urls.query + '/' + handle +
+'?sessionid=' + secretToken + '&queryHandle=' + handle;
+ return BaseAdapter.get(queryUrl);
+}));
+ },
getQueryResult (secretToken, handle, queryMode) {
// on page refresh, the store won't have queryMode so fetch query
http://git-wip-us.apache.org/repos/asf/lens/blob/b438aa2f/lens-ui/app/components/CubeSchemaComponent.js
lens git commit: LENS-1322 : Display join chain details as a tooltip on dim attributes using that join chain
Repository: lens
Updated Branches:
refs/heads/master b438aa2f2 -> d1a463552
LENS-1322 : Display join chain details as a tooltip on dim attributes using
that join chain
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d1a46355
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d1a46355
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d1a46355
Branch: refs/heads/master
Commit: d1a4635526f770287099c60ddc1e9665644b9cc9
Parents: b438aa2
Author: Rajat Khandelwal
Authored: Sat Sep 17 11:21:34 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Sat Sep 17 11:21:34 2016 +0530
--
lens-ui/app/components/CubeSchemaComponent.js | 44 --
lens-ui/app/stores/CubeStore.js | 5 +++
lens-ui/app/styles/css/global.css | 4 ++
lens-ui/app/styles/css/query-component.css| 2 +-
lens-ui/package.json | 2 +-
5 files changed, 43 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/d1a46355/lens-ui/app/components/CubeSchemaComponent.js
--
diff --git a/lens-ui/app/components/CubeSchemaComponent.js
b/lens-ui/app/components/CubeSchemaComponent.js
index 6a2b7af..27adfac 100644
--- a/lens-ui/app/components/CubeSchemaComponent.js
+++ b/lens-ui/app/components/CubeSchemaComponent.js
@@ -23,6 +23,7 @@ import CubeStore from '../stores/CubeStore';
import UserStore from '../stores/UserStore';
import AdhocQueryActions from '../actions/AdhocQueryActions';
import Loader from '../components/LoaderComponent';
+import {Tooltip, OverlayTrigger} from 'react-bootstrap'
function getCubes(database) {
return CubeStore.getCubes(database);
@@ -73,23 +74,41 @@ function constructMeasureTable(cubeName, measures) {
);
}
-function constructDimensionTable(cubeName, dimensions) {
+function constructDimensionTable(cubeName, dimensions, join_chains_by_name) {
let table = dimensions.map((dimension) => {
if (typeof(dimension) == "string") {
return (
- { dimension}
);
} else {
+ let join_chain_column;
+ if (dimension.chain_ref_column) {
+let inner = dimension.chain_ref_column.map((ref) => {
+ let join_chain = join_chains_by_name[ref.chain_name];
+ let tooltip = {
+join_chain.paths.path.map((path) => {
+ let paths = path.edges.edge.map((edge) => {
+return edge.from.table + "." + edge.from.column + "=" +
edge.to.table + "." + edge.to.column
+ }).join("->")
+ return ({paths})
+})
+ }
+ ;
+ return (
+
+ {ref.chain_name}
+{"."+ref.ref_col}
+ );
+});
+join_chain_column = ({inner});
+ }
return (
{ dimension.name }
{ dimension.display_string }
{ dimension.description }
- { dimension.chain_ref_column ?
dimension.chain_ref_column.map((ref) => {
-return ref.chain_name + "." + ref.ref_col
- }).join(" ") : ""}
+ {join_chain_column}
);
}
@@ -194,11 +213,11 @@ function constructExpressionTable(cubeName, expressions) {
class CubeSchema extends React.Component {
constructor(props) {
super(props);
-this.state = {cube: {}, database: props.params.databaseName};
+this.state = {cube: {}, database: UserStore.currentDatabase()};
this._onChange = this._onChange.bind(this);
AdhocQueryActions
- .getCubeDetails(UserStore.getUserDetails().secretToken,
props.params.databaseName, props.params.cubeName);
+ .getCubeDetails(UserStore.getUserDetails().secretToken,
UserStore.currentDatabase(), props.params.cubeName);
}
componentDidMount() {
@@ -212,7 +231,7 @@ class CubeSchema extends React.Component {
componentWillReceiveProps(props) {
// TODO are props updated automatically, unlike state?
let cubeName = props.params.cubeName;
-let cube = getCubes(props.params.databaseName)[cubeName];
+let cube = getCubes(UserStore.currentDatabase())[cubeName];
if (cube.isLoaded) {
this.setState({cube: cube, database: props.params.database});
@@ -220,10 +239,10 @@ class CubeSchema extends React.Component {
}
AdhocQueryActions
- .getCubeDetails(UserStore.getUserDetails().secretToken,
props.params.databaseName, cubeName);
+ .getCubeDetails(UserStore.getUserDetails().secretToken,
UserStore.currentDatabase(), cubeName);
// empty the previous state
-this
lens git commit: LENS-1300 : Query submission to be rejected on sessions marked for close
Repository: lens
Updated Branches:
refs/heads/master d1a463552 -> 4556773e3
LENS-1300 : Query submission to be rejected on sessions marked for close
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/4556773e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/4556773e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/4556773e
Branch: refs/heads/master
Commit: 4556773e3e4e2d80c1c68afd87338a4fe2de0e6d
Parents: d1a4635
Author: Rajat Khandelwal
Authored: Sat Sep 17 11:43:29 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Sat Sep 17 11:43:29 2016 +0530
--
.../java/org/apache/lens/api/APIResult.java | 3 +
.../lens/server/api/session/SessionService.java | 7 ++
.../org/apache/lens/server/BaseLensService.java | 8 +-
.../lens/server/session/HiveSessionService.java | 15 ++-
.../lens/server/session/LensSessionImpl.java| 4 +-
.../lens/server/session/SessionResource.java| 20 +++-
.../TestQueryIndependenceFromSessionClose.java | 105 +--
7 files changed, 121 insertions(+), 41 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/4556773e/lens-api/src/main/java/org/apache/lens/api/APIResult.java
--
diff --git a/lens-api/src/main/java/org/apache/lens/api/APIResult.java
b/lens-api/src/main/java/org/apache/lens/api/APIResult.java
index 26e9926..e147fec 100644
--- a/lens-api/src/main/java/org/apache/lens/api/APIResult.java
+++ b/lens-api/src/main/java/org/apache/lens/api/APIResult.java
@@ -98,6 +98,9 @@ public class APIResult extends ToYAMLString {
public static APIResult success() {
return SUCCESS;
}
+ public static APIResult success(String message) {
+return new APIResult(Status.SUCCEEDED, message);
+ }
public static APIResult failure(Exception e) {
String cause = extractCause(e);
http://git-wip-us.apache.org/repos/asf/lens/blob/4556773e/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionService.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionService.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionService.java
index 80ce030..20ec686 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionService.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionService.java
@@ -67,6 +67,13 @@ public interface SessionService {
void closeSession(LensSessionHandle sessionHandle) throws LensException;
/**
+ * Close idle sessions.
+ *
+ * @throws LensException the lens exception
+ */
+
+ void cleanupIdleSessions() throws LensException;
+ /**
* Adds the resource.
*
* @param sessionHandle the session handle
http://git-wip-us.apache.org/repos/asf/lens/blob/4556773e/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java
b/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java
index e0e4bf8..d7ce434 100644
--- a/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java
+++ b/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java
@@ -547,7 +547,13 @@ public abstract class BaseLensService extends
CompositeService implements Extern
if (handle == null) {
throw new LensException(SESSION_ID_NOT_PROVIDED.getLensErrorInfo());
}
-if (!getSession(handle).isActive()) {
+LensSessionImpl session;
+try {
+ session = getSession(handle);
+} catch (ClientErrorException e) {
+ throw new LensException(SESSION_CLOSED.getLensErrorInfo(), handle, e);
+}
+if (!session.isActive() || session.isMarkedForClose()) {
throw new LensException(SESSION_CLOSED.getLensErrorInfo(), handle);
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/4556773e/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
index 8909098..21e2a62 100644
---
a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
+++
b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
@@ -22,9 +22,7 @@ import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.*;
-import java.util.concurrent.Executors;
-imp
lens git commit: LENS-1323 : Fix TestRemoteHiveDriver#testMultiThreadClient test failure
Repository: lens
Updated Branches:
refs/heads/master 4556773e3 -> fe4ea6bd6
LENS-1323 : Fix TestRemoteHiveDriver#testMultiThreadClient test failure
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/fe4ea6bd
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/fe4ea6bd
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/fe4ea6bd
Branch: refs/heads/master
Commit: fe4ea6bd60596f7fb62239d9011050b338acced0
Parents: 4556773
Author: Rajat Khandelwal
Authored: Sat Sep 17 11:57:39 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Sat Sep 17 11:57:39 2016 +0530
--
.../org/apache/lens/driver/hive/HiveDriver.java | 87 +---
.../lens/driver/hive/TestRemoteHiveDriver.java | 6 +-
2 files changed, 44 insertions(+), 49 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/fe4ea6bd/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
--
diff --git
a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 0218be3..84d9933 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -583,7 +583,7 @@ public class HiveDriver extends AbstractLensDriver {
}
private DriverQueryStatus
updateDriverStateFromOperationStatus(OperationHandle handle, DriverQueryStatus
status)
-throws LensException, HiveSQLException {
+throws LensException, HiveSQLException, IOException {
if (status == null) {
status = new DriverQueryStatus();
}
@@ -626,34 +626,13 @@ public class HiveDriver extends AbstractLensDriver {
default:
throw new LensException("Query is in unknown state at HiveServer");
}
-return status;
- }
- /*
- * (non-Javadoc)
- *
- * @see
org.apache.lens.server.api.driver.LensDriver#updateStatus(org.apache.lens.server.api.query.QueryContext)
- */
- @Override
- public void updateStatus(QueryContext context) throws LensException {
-log.debug("GetStatus: {}", context.getQueryHandle());
-if (context.getDriverStatus().isFinished()) {
- return;
-}
-OperationHandle hiveHandle = getHiveHandle(context.getQueryHandle());
-ByteArrayInputStream in = null;
-try {
- // Get operation status from hive server
- log.debug("GetStatus hiveHandle: {}", hiveHandle);
- fetchLogs(hiveHandle);
- OperationStatus opStatus = getClient().getOperationStatus(hiveHandle);
- updateDriverStateFromOperationStatus(hiveHandle,
context.getDriverStatus());
- float progress = 0f;
- String jsonTaskStatus = opStatus.getTaskStatus();
- String errorMsg = null;
- if (StringUtils.isNotBlank(jsonTaskStatus)) {
+float progress = 0f;
+String jsonTaskStatus = opStatus.getTaskStatus();
+String errorMsg = null;
+if (StringUtils.isNotBlank(jsonTaskStatus)) {
+ try (ByteArrayInputStream in = new
ByteArrayInputStream(jsonTaskStatus.getBytes("UTF-8"))) {
ObjectMapper mapper = new ObjectMapper();
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES,
false);
-in = new ByteArrayInputStream(jsonTaskStatus.getBytes("UTF-8"));
List taskStatuses = mapper.readValue(in, new
TypeReference>() {
});
int completedTasks = 0;
@@ -670,32 +649,44 @@ public class HiveDriver extends AbstractLensDriver {
}
progress = taskStatuses.size() == 0 ? 0 : (float) completedTasks /
taskStatuses.size();
errorMsg = errorMessage.toString();
- } else {
-log.warn("Empty task statuses");
- }
- String error = null;
- if (StringUtils.isNotBlank(errorMsg)) {
-error = errorMsg;
- } else if (opStatus.getState().equals(OperationState.ERROR)) {
-error = context.getDriverStatus().getErrorMessage();
}
- context.getDriverStatus().setErrorMessage(error);
- context.getDriverStatus().setProgressMessage(jsonTaskStatus);
- context.getDriverStatus().setProgress(progress);
-
context.getDriverStatus().setDriverStartTime(opStatus.getOperationStarted());
-
context.getDriverStatus().setDriverFinishTime(opStatus.getOperationCompleted());
+} else {
+ log.warn("Empty task statuses");
+}
+String error = null;
+if (StringUtils.isNotBlank(errorMsg)) {
+ error = errorMsg;
+} else if (status.getState().equals(DriverQueryState.FAILED)) {
+ error = status.getErrorMessage();
+}
+status.setErrorMessage(error);
+status.setProgressMessage(jsonTa
lens git commit: LENS-1317 : Lens Query Conf to be persisted in Finished_Queries store
Repository: lens
Updated Branches:
refs/heads/master fe4ea6bd6 -> 4fa61d994
LENS-1317 : Lens Query Conf to be persisted in Finished_Queries store
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/4fa61d99
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/4fa61d99
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/4fa61d99
Branch: refs/heads/master
Commit: 4fa61d9940af6ede89401d5837f1f7796810f254
Parents: fe4ea6b
Author: Srikanth Sundarrajan
Authored: Tue Sep 20 10:59:28 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Sep 20 10:59:28 2016 +0530
--
.../server/api/query/FinishedLensQuery.java | 9 +++--
.../apache/lens/server/query/LensServerDAO.java | 38 +---
.../apache/lens/server/query/TestLensDAO.java | 5 +++
3 files changed, 45 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/4fa61d99/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
index edd4284..b58fcf9 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
@@ -46,7 +46,7 @@ import lombok.ToString;
*
* @see java.lang.Object#hashCode()
*/
-@EqualsAndHashCode(exclude = "selectedDriver")
+@EqualsAndHashCode(exclude = {"selectedDriver", "conf"})
/*
* (non-Javadoc)
*
@@ -174,6 +174,10 @@ public class FinishedLensQuery {
@Setter
private String priority;
+ @Getter
+ @Setter
+ private LensConf conf;
+
/**
* The selected driver's query.
*/
@@ -217,6 +221,7 @@ public class FinishedLensQuery {
if (null != ctx.getPriority()) {
this.priority = ctx.getPriority().toString();
}
+this.conf = ctx.getLensConf();
}
public QueryContext toQueryContext(Configuration conf,
Collection drivers) {
@@ -226,7 +231,7 @@ public class FinishedLensQuery {
}
QueryContext qctx =
- new QueryContext(userQuery, submitter, new LensConf(), conf, drivers,
selectedDriver, submissionTime,
+ new QueryContext(userQuery, submitter, this.conf, conf, drivers,
selectedDriver, submissionTime,
false);
qctx.setQueryHandle(QueryHandle.fromString(handle));
http://git-wip-us.apache.org/repos/asf/lens/blob/4fa61d99/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index 2ec938e..997b5a8 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.server.query;
+import java.nio.charset.Charset;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -26,12 +27,16 @@ import java.util.List;
import javax.sql.DataSource;
+import org.apache.lens.api.LensConf;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.FinishedLensQuery;
import org.apache.lens.server.util.UtilityMethods;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.dbutils.BasicRowProcessor;
+import org.apache.commons.dbutils.BeanProcessor;
import org.apache.commons.dbutils.DbUtils;
import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.ResultSetHandler;
@@ -87,7 +92,8 @@ public class LensServerDAO {
+ "starttime bigint, " + "endtime bigint," + "result varchar(255)," +
"status varchar(255), "
+ "metadata varchar(10), " + "rows int, " + "filesize bigint, " +
"errormessage varchar(1), "
+ "driverstarttime bigint, " + "driverendtime bigint, " + "drivername
varchar(1), "
- + "queryname varchar(255), " + "submissiontime bigint, " + "driverquery
varchar(100)" + ")";
+ + "queryname varchar(255), " + "submissiontime bigint, " + "driverquery
varchar(100), "
+ + "conf varchar(10))";
try {
QueryRunner
lens git commit: LENS-1283 : Map failure in APIResult to exception in LensClient
Repository: lens
Updated Branches:
refs/heads/master 4fa61d994 -> 8108ba062
LENS-1283 : Map failure in APIResult to exception in LensClient
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/8108ba06
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/8108ba06
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/8108ba06
Branch: refs/heads/master
Commit: 8108ba0625a477a15a7c7cbc65d73157ad83be14
Parents: 4fa61d9
Author: Rajat Khandelwal
Authored: Wed Sep 21 09:39:16 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed Sep 21 09:39:16 2016 +0530
--
.../java/org/apache/lens/api/APIResult.java | 27 ++-
.../java/org/apache/lens/api/ToYAMLString.java | 5 +-
.../lens/api/jaxb/YAMLToStringStrategyTest.java | 14 +-
...lens.api.query.QueryHandleWithResultSet.yaml | 2 +-
.../apache/lens/cli/LensCliApplicationTest.java | 2 +-
.../lens/cli/TestLensDatabaseCommands.java | 9 +-
.../apache/lens/client/LensMetadataClient.java | 203 ++-
.../exceptions/LensBriefErrorException.java | 5 +
.../lens/client/model/IdBriefErrorTemplate.java | 22 +-
.../client/model/IdBriefErrorTemplateTest.java | 4 +-
.../examples/ExampleSchemaToStringTest.java | 8 +-
11 files changed, 163 insertions(+), 138 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/8108ba06/lens-api/src/main/java/org/apache/lens/api/APIResult.java
--
diff --git a/lens-api/src/main/java/org/apache/lens/api/APIResult.java
b/lens-api/src/main/java/org/apache/lens/api/APIResult.java
index e147fec..cba564e 100644
--- a/lens-api/src/main/java/org/apache/lens/api/APIResult.java
+++ b/lens-api/src/main/java/org/apache/lens/api/APIResult.java
@@ -20,6 +20,8 @@ package org.apache.lens.api;
import javax.xml.bind.annotation.*;
+import org.apache.commons.lang.StringUtils;
+
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Getter;
@@ -74,7 +76,7 @@ public class APIResult extends ToYAMLString {
FAILED
}
- private static final APIResult SUCCESS = new APIResult(Status.SUCCEEDED, "");
+ private static final APIResult SUCCESS = new APIResult(Status.SUCCEEDED,
null);
public static APIResult partial(int actual, int expected) {
return new APIResult(Status.PARTIAL, actual + " out of " + expected);
@@ -113,13 +115,22 @@ public class APIResult extends ToYAMLString {
}
private static String extractCause(Throwable e) {
-StringBuilder cause = new StringBuilder();
-String sep = "";
-while (e != null) {
- cause.append(sep).append(e.getMessage());
- e = e.getCause();
- sep = ": ";
+if (e == null) {
+ return null;
+}
+if (e.getCause() != null) {
+ String causeMessage = extractCause(e.getCause());
+ if (StringUtils.isBlank(e.getMessage())) {
+return causeMessage;
+ } else {
+if (e.getMessage().contains(causeMessage)) {
+ return causeMessage;
+} else {
+ return e.getMessage() + ": " + causeMessage;
+}
+ }
+} else {
+ return e.getMessage();
}
-return cause.toString();
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/8108ba06/lens-api/src/main/java/org/apache/lens/api/ToYAMLString.java
--
diff --git a/lens-api/src/main/java/org/apache/lens/api/ToYAMLString.java
b/lens-api/src/main/java/org/apache/lens/api/ToYAMLString.java
index 65a0c44..ea8e8d1 100644
--- a/lens-api/src/main/java/org/apache/lens/api/ToYAMLString.java
+++ b/lens-api/src/main/java/org/apache/lens/api/ToYAMLString.java
@@ -68,7 +68,10 @@ public abstract class ToYAMLString implements ToString {
for (Field field : this.getClass().getDeclaredFields()) {
try {
Method getter = getGetter(field);
- fieldNameAndValueList.add(new
FieldNameAndValue(getReadableName(getter), getter.invoke(this)));
+ Object value = getter.invoke(this);
+ if (value != null) {
+fieldNameAndValueList.add(new
FieldNameAndValue(getReadableName(getter), value));
+ }
} catch (IllegalAccessException | InvocationTargetException e) {
throw new RuntimeException(e);
} catch (NoSuchMethodException e) {
http://git-wip-us.apache.org/repos/asf/lens/blob/8108ba06/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java
--
diff --git
a/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java
b/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java
index 3a78
lens git commit: LENS-1314 : Update 2.6 release documentation
Repository: lens
Updated Branches:
refs/heads/master 8108ba062 -> 0d31b74b3
LENS-1314 : Update 2.6 release documentation
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/0d31b74b
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/0d31b74b
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/0d31b74b
Branch: refs/heads/master
Commit: 0d31b74b39019b3f7370084e7a8e1f9b647afb5b
Parents: 8108ba0
Author: Puneet Gupta
Authored: Wed Sep 21 09:57:56 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed Sep 21 09:57:56 2016 +0530
--
src/site/apt/releases/download.apt| 2 +-
src/site/apt/releases/release-history.apt | 35 ++
2 files changed, 36 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/0d31b74b/src/site/apt/releases/download.apt
--
diff --git a/src/site/apt/releases/download.apt
b/src/site/apt/releases/download.apt
index d0b313f..7e6895d 100644
--- a/src/site/apt/releases/download.apt
+++ b/src/site/apt/releases/download.apt
@@ -20,7 +20,7 @@
Download
The latest release of Apache Lens can be
- {{{http://www.apache.org/dyn/closer.lua/lens/2.5-beta}downloaded from the
ASF}}.
+ {{{http://www.apache.org/dyn/closer.lua/lens/2.6}downloaded from the ASF}}.
Older releases may be found {{{http://archive.apache.org/dist/lens}in the
archives}}.
Releases prior to graduation may be found
{{{http://archive.apache.org/dist/incubator/lens}in the incubator archives}}.
http://git-wip-us.apache.org/repos/asf/lens/blob/0d31b74b/src/site/apt/releases/release-history.apt
--
diff --git a/src/site/apt/releases/release-history.apt
b/src/site/apt/releases/release-history.apt
index 95891be..5ed0af8 100644
--- a/src/site/apt/releases/release-history.apt
+++ b/src/site/apt/releases/release-history.apt
@@ -23,6 +23,8 @@ All Apache Lens releases
*--+--+---+--+--+--+--+
|Release version|Major features|Release documentation|Release
Notes|Download|Incompatibilities|More Info |
*--+--+---+--+--+--+--+
+|2.6.0| GA release, Apache Hive 2 support, Query Scheduler, Lens UI
enhancements, Python client, Streaming results, New regression module, Weighted
Driver selector, Limit number of open sessions, Exponential back off for hive
status updates, Single execution for duplicate queries, Auto kill queries on
timeout, Convert dimension filter to fact filters for performance improvement,
Code cleanup and refactor, Doc/javadoc improvements, Bug fixes|
{{{../versions/2.6.0/index.html} 2.6.x docs}}
|{{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12333846&styleName=&projectId=12315923}
2.6 release notes}} |{{{http://archive.apache.org/dist/lens/2.6/} Download}}
|{{{#a2.6.x_from_2.5.x} 2.6.x-incompatibilities}} | - |
+*--+--+---+--+--+--+--+
|2.5.0-beta| Support for multiple instances of a driver, REST API
enhancements, Derived cube validation, Per-queue driver max launched queries
constraint, New error codes for Hive and JDBC Driver, Lens CLI improvements,
Query result retention policy, UI Enhancements, Code cleanup and refactor,
Doc/javadoc improvements, Bug fixes| {{{../versions/2.5.0-beta/index.html}
2.5.x docs}}
|{{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=1226&projectId=12315923}
2.5 release notes}} |{{{http://archive.apache.org/dist/lens/2.5-beta/}
Download}} |{{{#a2.5.x_from_2.4.x} 2.5.x-incompatibilities}} | - |
*--+--+---+--+--+--+--+
|2.4.0-beta| JDK8 support, new web client, Saved Query and Parameterization,
Support flattening of columns selected through bridge-tables(many-to-many
relationships), column name mapping for columns in underlying storage tables,
Enhancements cube.xsd and jdbc driver, Improvements in Error codes, Doc
improvements, bug fixes| {{{../versions/2.4.0-beta/index.html} 2.4.x docs}}
|{{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12315923&version=12333087}
2.4 release notes}} |{{{http://archive.apache.org/dist/lens/2.4-beta/}
Download}} |{{{#a2.4.x_from_2.3.x} 2.4.x-incompatibilities}} | - |
@@ -44,6 +46,39 @@ All Apache Lens releases
* Incompatibilities
+** 2.6.x from 2.5.x
+
+ * LENS-919: For older deployments the schema for "finished_queries" table
needs to be modified to include "priority"
+
+ * LENS-1212: Saved query service is enabled by default now. "saved_query"
table will be created to store information
+pertaining to saved queries
+
+ * LENS-833: Number of open sessions allowed for each user is capped at 10 by
default now
+
+ * LENS-1244: Dropping beta from lens releases. Lens 2.6 will be a GA release
+
+ * LENS-
lens git commit: LENS-1314 : Update 2.6 release documentation
Repository: lens
Updated Branches:
refs/heads/current-release-line ca19ead47 -> 035959184
LENS-1314 : Update 2.6 release documentation
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/03595918
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/03595918
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/03595918
Branch: refs/heads/current-release-line
Commit: 03595918430cf9fd3944539a7c8f26b724a3e773
Parents: ca19ead
Author: Puneet Gupta
Authored: Wed Sep 21 09:57:56 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed Sep 21 09:58:46 2016 +0530
--
src/site/apt/releases/download.apt| 2 +-
src/site/apt/releases/release-history.apt | 35 ++
2 files changed, 36 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/03595918/src/site/apt/releases/download.apt
--
diff --git a/src/site/apt/releases/download.apt
b/src/site/apt/releases/download.apt
index d0b313f..7e6895d 100644
--- a/src/site/apt/releases/download.apt
+++ b/src/site/apt/releases/download.apt
@@ -20,7 +20,7 @@
Download
The latest release of Apache Lens can be
- {{{http://www.apache.org/dyn/closer.lua/lens/2.5-beta}downloaded from the
ASF}}.
+ {{{http://www.apache.org/dyn/closer.lua/lens/2.6}downloaded from the ASF}}.
Older releases may be found {{{http://archive.apache.org/dist/lens}in the
archives}}.
Releases prior to graduation may be found
{{{http://archive.apache.org/dist/incubator/lens}in the incubator archives}}.
http://git-wip-us.apache.org/repos/asf/lens/blob/03595918/src/site/apt/releases/release-history.apt
--
diff --git a/src/site/apt/releases/release-history.apt
b/src/site/apt/releases/release-history.apt
index 95891be..5ed0af8 100644
--- a/src/site/apt/releases/release-history.apt
+++ b/src/site/apt/releases/release-history.apt
@@ -23,6 +23,8 @@ All Apache Lens releases
*--+--+---+--+--+--+--+
|Release version|Major features|Release documentation|Release
Notes|Download|Incompatibilities|More Info |
*--+--+---+--+--+--+--+
+|2.6.0| GA release, Apache Hive 2 support, Query Scheduler, Lens UI
enhancements, Python client, Streaming results, New regression module, Weighted
Driver selector, Limit number of open sessions, Exponential back off for hive
status updates, Single execution for duplicate queries, Auto kill queries on
timeout, Convert dimension filter to fact filters for performance improvement,
Code cleanup and refactor, Doc/javadoc improvements, Bug fixes|
{{{../versions/2.6.0/index.html} 2.6.x docs}}
|{{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12333846&styleName=&projectId=12315923}
2.6 release notes}} |{{{http://archive.apache.org/dist/lens/2.6/} Download}}
|{{{#a2.6.x_from_2.5.x} 2.6.x-incompatibilities}} | - |
+*--+--+---+--+--+--+--+
|2.5.0-beta| Support for multiple instances of a driver, REST API
enhancements, Derived cube validation, Per-queue driver max launched queries
constraint, New error codes for Hive and JDBC Driver, Lens CLI improvements,
Query result retention policy, UI Enhancements, Code cleanup and refactor,
Doc/javadoc improvements, Bug fixes| {{{../versions/2.5.0-beta/index.html}
2.5.x docs}}
|{{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=1226&projectId=12315923}
2.5 release notes}} |{{{http://archive.apache.org/dist/lens/2.5-beta/}
Download}} |{{{#a2.5.x_from_2.4.x} 2.5.x-incompatibilities}} | - |
*--+--+---+--+--+--+--+
|2.4.0-beta| JDK8 support, new web client, Saved Query and Parameterization,
Support flattening of columns selected through bridge-tables(many-to-many
relationships), column name mapping for columns in underlying storage tables,
Enhancements cube.xsd and jdbc driver, Improvements in Error codes, Doc
improvements, bug fixes| {{{../versions/2.4.0-beta/index.html} 2.4.x docs}}
|{{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12315923&version=12333087}
2.4 release notes}} |{{{http://archive.apache.org/dist/lens/2.4-beta/}
Download}} |{{{#a2.4.x_from_2.3.x} 2.4.x-incompatibilities}} | - |
@@ -44,6 +46,39 @@ All Apache Lens releases
* Incompatibilities
+** 2.6.x from 2.5.x
+
+ * LENS-919: For older deployments the schema for "finished_queries" table
needs to be modified to include "priority"
+
+ * LENS-1212: Saved query service is enabled by default now. "saved_query"
table will be created to store information
+pertaining to saved queries
+
+ * LENS-833: Number of open sessions allowed for each user is capped at 10 by
default now
+
+ * LENS-1244: Dropping beta from lens releases. Lens 2.6 will be
lens git commit: LENS-1337 : Update docs link in release-history
Repository: lens
Updated Branches:
refs/heads/master d4225f645 -> faf721f05
LENS-1337 : Update docs link in release-history
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/faf721f0
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/faf721f0
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/faf721f0
Branch: refs/heads/master
Commit: faf721f054c939aadb2cf11565f1d192d83cfd94
Parents: d4225f6
Author: Puneet Gupta
Authored: Sun Sep 25 07:51:41 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Sun Sep 25 07:51:41 2016 +0530
--
src/site/apt/releases/release-history.apt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/faf721f0/src/site/apt/releases/release-history.apt
--
diff --git a/src/site/apt/releases/release-history.apt
b/src/site/apt/releases/release-history.apt
index f919ca5..7057c74 100644
--- a/src/site/apt/releases/release-history.apt
+++ b/src/site/apt/releases/release-history.apt
@@ -23,7 +23,7 @@ All Apache Lens releases
*--+--+---+--+--+--+--+
|Release version|Major features|Release documentation|Release
Notes|Download|Incompatibilities|More Info |
*--+--+---+--+--+--+--+
-|2.6.1| GA release, Apache Hive 2 support, Query Scheduler, Lens UI
enhancements, Python client, Streaming results, New regression module, Weighted
Driver selector, Limit number of open sessions, Exponential back off for hive
status updates, Single execution for duplicate queries, Auto kill queries on
timeout, Convert dimension filter to fact filters for performance improvement,
Code cleanup and refactor, Doc/javadoc improvements, Bug fixes|
{{{../versions/2.6.0/index.html} 2.6.x docs}}
|{{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12333846&styleName=&projectId=12315923}
2.6 release notes}} |{{{http://archive.apache.org/dist/lens/2.6/} Download}}
|{{{#a2.6.x_from_2.5.x} 2.6.x-incompatibilities}} | - |
+|2.6.1| GA release, Apache Hive 2 support, Query Scheduler, Lens UI
enhancements, Python client, Streaming results, New regression module, Weighted
Driver selector, Limit number of open sessions, Exponential back off for hive
status updates, Single execution for duplicate queries, Auto kill queries on
timeout, Convert dimension filter to fact filters for performance improvement,
Code cleanup and refactor, Doc/javadoc improvements, Bug fixes|
{{{../versions/2.6.1/index.html} 2.6.x docs}}
|{{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12333846&styleName=&projectId=12315923}
2.6 release notes}} |{{{http://archive.apache.org/dist/lens/2.6/} Download}}
|{{{#a2.6.x_from_2.5.x} 2.6.x-incompatibilities}} | - |
*--+--+---+--+--+--+--+
|2.5.0-beta| Support for multiple instances of a driver, REST API
enhancements, Derived cube validation, Per-queue driver max launched queries
constraint, New error codes for Hive and JDBC Driver, Lens CLI improvements,
Query result retention policy, UI Enhancements, Code cleanup and refactor,
Doc/javadoc improvements, Bug fixes| {{{../versions/2.5.0-beta/index.html}
2.5.x docs}}
|{{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=1226&projectId=12315923}
2.5 release notes}} |{{{http://archive.apache.org/dist/lens/2.5-beta/}
Download}} |{{{#a2.5.x_from_2.4.x} 2.5.x-incompatibilities}} | - |
*--+--+---+--+--+--+--+
lens git commit: LENS-1337 : Update docs link in release-history
Repository: lens
Updated Branches:
refs/heads/current-release-line 0f7a1d6d9 -> 7efc6436a
LENS-1337 : Update docs link in release-history
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7efc6436
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7efc6436
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7efc6436
Branch: refs/heads/current-release-line
Commit: 7efc6436a2e8fbde77e1c3feb181f6aa2b989d82
Parents: 0f7a1d6
Author: Puneet Gupta
Authored: Sun Sep 25 07:51:41 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Sun Sep 25 07:52:27 2016 +0530
--
src/site/apt/releases/release-history.apt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/7efc6436/src/site/apt/releases/release-history.apt
--
diff --git a/src/site/apt/releases/release-history.apt
b/src/site/apt/releases/release-history.apt
index f919ca5..7057c74 100644
--- a/src/site/apt/releases/release-history.apt
+++ b/src/site/apt/releases/release-history.apt
@@ -23,7 +23,7 @@ All Apache Lens releases
*--+--+---+--+--+--+--+
|Release version|Major features|Release documentation|Release
Notes|Download|Incompatibilities|More Info |
*--+--+---+--+--+--+--+
-|2.6.1| GA release, Apache Hive 2 support, Query Scheduler, Lens UI
enhancements, Python client, Streaming results, New regression module, Weighted
Driver selector, Limit number of open sessions, Exponential back off for hive
status updates, Single execution for duplicate queries, Auto kill queries on
timeout, Convert dimension filter to fact filters for performance improvement,
Code cleanup and refactor, Doc/javadoc improvements, Bug fixes|
{{{../versions/2.6.0/index.html} 2.6.x docs}}
|{{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12333846&styleName=&projectId=12315923}
2.6 release notes}} |{{{http://archive.apache.org/dist/lens/2.6/} Download}}
|{{{#a2.6.x_from_2.5.x} 2.6.x-incompatibilities}} | - |
+|2.6.1| GA release, Apache Hive 2 support, Query Scheduler, Lens UI
enhancements, Python client, Streaming results, New regression module, Weighted
Driver selector, Limit number of open sessions, Exponential back off for hive
status updates, Single execution for duplicate queries, Auto kill queries on
timeout, Convert dimension filter to fact filters for performance improvement,
Code cleanup and refactor, Doc/javadoc improvements, Bug fixes|
{{{../versions/2.6.1/index.html} 2.6.x docs}}
|{{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12333846&styleName=&projectId=12315923}
2.6 release notes}} |{{{http://archive.apache.org/dist/lens/2.6/} Download}}
|{{{#a2.6.x_from_2.5.x} 2.6.x-incompatibilities}} | - |
*--+--+---+--+--+--+--+
|2.5.0-beta| Support for multiple instances of a driver, REST API
enhancements, Derived cube validation, Per-queue driver max launched queries
constraint, New error codes for Hive and JDBC Driver, Lens CLI improvements,
Query result retention policy, UI Enhancements, Code cleanup and refactor,
Doc/javadoc improvements, Bug fixes| {{{../versions/2.5.0-beta/index.html}
2.5.x docs}}
|{{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=1226&projectId=12315923}
2.5 release notes}} |{{{http://archive.apache.org/dist/lens/2.5-beta/}
Download}} |{{{#a2.5.x_from_2.4.x} 2.5.x-incompatibilities}} | - |
*--+--+---+--+--+--+--+
lens git commit: LENS-1334 : Fix full outer join condition for multi fact queries with more than two tables
Repository: lens
Updated Branches:
refs/heads/master fa865bfb6 -> f1a959bdd
LENS-1334 : Fix full outer join condition for multi fact queries with more than
two tables
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/f1a959bd
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/f1a959bd
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/f1a959bd
Branch: refs/heads/master
Commit: f1a959bdd78e4161b7883734776e4dc4c464bb49
Parents: fa865bf
Author: Sushil Mohanty
Authored: Tue Sep 27 16:38:28 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Sep 27 16:38:28 2016 +0530
--
.../lens/cube/parse/MultiFactHQLContext.java| 36 ---
.../lens/cube/parse/TestBaseCubeQueries.java| 46 +++-
2 files changed, 45 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/f1a959bd/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
index 7fbcd7e..b3547db 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
@@ -129,6 +129,17 @@ class MultiFactHQLContext extends SimpleHQLContext {
return select.toString();
}
+ private String getMultiFactJoinCondition(int i, String dim) {
+StringBuilder joinCondition = new StringBuilder();
+if (i <= 1) {
+ return "".toString();
+} else {
+ joinCondition.append("mq").append(i -
2).append(".").append(dim).append(" <=> ").
+ append("mq").append(i - 1).append(".").append(dim);
+}
+return joinCondition.toString();
+ }
+
private String getFromString() throws LensException {
StringBuilder fromBuilder = new StringBuilder();
int aliasCount = 1;
@@ -137,24 +148,17 @@ class MultiFactHQLContext extends SimpleHQLContext {
SimpleHQLContext facthql = factHQLContextMap.get(fact);
fromBuilder.append(sep).append("(").append(facthql.toHQL()).append(")").append("
mq").append(aliasCount++);
sep = " full outer join ";
-}
-CandidateFact firstFact = facts.iterator().next();
-if (!firstFact.getDimFieldIndices().isEmpty()) {
- fromBuilder.append(" on ");
-}
-for (int i = 2; i <= facts.size(); i++) {
- Iterator dimIter = firstFact.getDimFieldIndices().iterator();
- while (dimIter.hasNext()) {
-String dim = query.getSelectAlias(dimIter.next());
-fromBuilder.append("mq1").append(".").append(dim).append(" <=>
").append("mq").append(i).append(".")
- .append(dim);
-if (dimIter.hasNext()) {
- fromBuilder.append(" AND ");
+ if (!fact.getDimFieldIndices().isEmpty() && aliasCount > 2) {
+fromBuilder.append(" on ");
+Iterator dimIter = fact.getDimFieldIndices().iterator();
+while (dimIter.hasNext()) {
+ String dim = query.getSelectAlias(dimIter.next());
+ fromBuilder.append(getMultiFactJoinCondition(aliasCount, dim));
+ if (dimIter.hasNext()) {
+fromBuilder.append(" AND ");
+ }
}
}
- if (i != facts.size() && firstFact.getDimFieldIndices().size() > 0) {
-fromBuilder.append(" AND ");
- }
}
return fromBuilder.toString();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/f1a959bd/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
index 6fb027a..053cad3 100644
---
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
+++
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
@@ -331,42 +331,46 @@ public class TestBaseCubeQueries extends TestQueryRewrite
{
@Test
public void testMultiFactQueryInvolvingThreeFactTables() throws Exception {
// query with 3 fact tables
-String hqlQuery = rewrite("select dim1, msr12, roundedmsr2, msr13, msr3
from basecube where " + TWO_DAYS_RANGE,
- conf);
+String hqlQuery = rewrite("select dim1, d_time, msr12, roundedmsr2, msr13,
msr3 from basecube where "
+
lens git commit: LENS-1324 : Support more hooks in DriverQueryHook
Repository: lens
Updated Branches:
refs/heads/master 241603cf0 -> 08ce29785
LENS-1324 : Support more hooks in DriverQueryHook
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/08ce2978
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/08ce2978
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/08ce2978
Branch: refs/heads/master
Commit: 08ce29785f886b82b490f269a1ca4a6c6eff9eed
Parents: 241603c
Author: Puneet Gupta
Authored: Mon Oct 3 16:55:58 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Mon Oct 3 16:55:58 2016 +0530
--
.../org/apache/lens/driver/hive/HiveDriver.java | 1 +
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 1 +
.../server/api/driver/AbstractLensDriver.java | 1 +
.../lens/server/api/driver/DriverQueryHook.java | 54 +---
.../server/api/driver/NoOpDriverQueryHook.java | 44 ++--
.../query/cost/FactPartitionBasedQueryCost.java | 5 ++
.../lens/server/api/driver/MockDriver.java | 1 +
.../server/api/user/MockDriverQueryHook.java| 34 +++-
.../server/query/QueryExecutionServiceImpl.java | 12 -
.../lens/server/query/TestQueryService.java | 4 ++
10 files changed, 144 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
--
diff --git
a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 84d9933..0d8810f 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -383,6 +383,7 @@ public class HiveDriver extends AbstractLensDriver {
queryHook = driverConf.getClass(
HIVE_QUERY_HOOK_CLASS, NoOpDriverQueryHook.class, DriverQueryHook.class
).newInstance();
+ queryHook.setDriver(this);
} catch (InstantiationException | IllegalAccessException e) {
throw new LensException("Can't instantiate driver query hook for
hivedriver with given class", e);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
--
diff --git
a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 3bf5e8f..f805ec6 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -407,6 +407,7 @@ public class JDBCDriver extends AbstractLensDriver {
queryHook = this.conf.getClass(
JDBC_QUERY_HOOK_CLASS, NoOpDriverQueryHook.class, DriverQueryHook.class
).newInstance();
+ queryHook.setDriver(this);
} catch (InstantiationException | IllegalAccessException e) {
throw new LensException("Can't instantiate driver query hook for
hivedriver with given class", e);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
index 03079e2..e498479 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
+++
b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
@@ -55,6 +55,7 @@ public abstract class AbstractLensDriver implements
LensDriver {
throw new LensException("Driver Type and Name can not be null or empty");
}
fullyQualifiedName = new
StringBuilder(driverType).append(SEPARATOR).append(driverName).toString();
+noOpDriverQueryHook.setDriver(this);
}
/**
http://git-wip-us.apache.org/repos/asf/lens/blob/08ce2978/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java
--
diff --git
a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java
b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java
index da125bd..f8a9ee0 100644
---
a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java
+++
b/lens-server-api/src/
lens git commit: LENS-1340 : Add DataCompletenessChecker
Repository: lens
Updated Branches:
refs/heads/master 08ce29785 -> 47a3012ed
LENS-1340 : Add DataCompletenessChecker
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/47a3012e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/47a3012e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/47a3012e
Branch: refs/heads/master
Commit: 47a3012edc9604e94795d1453a77ac4ee2b1b936
Parents: 08ce297
Author: Narayan Periwal
Authored: Tue Oct 4 12:44:15 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Oct 4 12:44:15 2016 +0530
--
.../cube/parse/DataCompletenessChecker.java | 55
1 file changed, 55 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/47a3012e/lens-cube/src/main/java/org/apache/lens/cube/parse/DataCompletenessChecker.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/DataCompletenessChecker.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/DataCompletenessChecker.java
new file mode 100644
index 000..026c9a9
--- /dev/null
+++
b/lens-cube/src/main/java/org/apache/lens/cube/parse/DataCompletenessChecker.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.parse;
+
+import org.apache.lens.server.api.error.LensException;
+
+import java.util.Date;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * DataCompletenessChecker is for identifying the completeness of data in a
fact for the given set of measures, start
+ * and end date. A fact will have a dataCompletenessTag, multiple facts can
have the same dataCompletenessTag.
+ * Similarly, measures will have a dataCompletenessTag, multiple measures can
have the same dataCompletenessTag.
+ * The api will take the dataCompletenessTag for the facts and measures and
compute the completeness based on these
+ * tags. The utility of having tags is that the similar kind of measures or
facts which will have the same level of
+ * completeness can use the same tag, thus we avoid the redundant completeness
computation for similar measures
+ * and facts.
+ * The implementations of the interface can truncate the start and end date.
+ */
+public interface DataCompletenessChecker {
+
+/**
+ * Get completeness of the set of measures in a fact based on the
dataCompletenessTag for the given starttime and
+ * endtime.
+ *
+ * @param factTag This is the dataCompletenessTag for a fact. The tag can
be specified by setting the property
+ *named dataCompletenessTag for the fact. Mutltiple facts
can have the same dataCompletenessTag.
+ * @param start Start time of the query (Inclusive).
+ * @param end End time of the query (Exclusive).
+ * @param measureTag List of distinct tag of the measures in the query.
Multiple measures can have the same
+ * dataCompletenessTag.
+ * @return map; key is the name of the dataCompletenessTag which refers to
one or more measures. Value is the map
+ * of date and %completeness.
+ */
+Map> getCompleteness(String factTag, Date start,
Date end, Set measureTag)
+throws LensException;
+
+}
lens git commit: LENS-1326 : Disable hive session and operation timeouts
Repository: lens Updated Branches: refs/heads/master 47a3012ed -> bf25ac578 LENS-1326 : Disable hive session and operation timeouts Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/bf25ac57 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/bf25ac57 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/bf25ac57 Branch: refs/heads/master Commit: bf25ac578d0643e664daafb2d19f95dda158ea18 Parents: 47a3012 Author: Lavkesh Lahngir Authored: Tue Oct 4 12:49:35 2016 +0530 Committer: Amareshwari Sriramadasu Committed: Tue Oct 4 12:49:35 2016 +0530 -- .../src/main/resources/lensserver-default.xml | 12 + src/site/apt/admin/config.apt | 240 ++- 2 files changed, 134 insertions(+), 118 deletions(-) -- http://git-wip-us.apache.org/repos/asf/lens/blob/bf25ac57/lens-server/src/main/resources/lensserver-default.xml -- diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml index d88ab3c..8e5fe8a 100644 --- a/lens-server/src/main/resources/lensserver-default.xml +++ b/lens-server/src/main/resources/lensserver-default.xml @@ -491,6 +491,18 @@ +hive.server2.idle.session.timeout +0 +Session should be closed by lens. There would be a mismatch when hive has closed session due to timeout +and lens is not aware of that. + + +hive.server2.idle.operation.timeout +0 +Closing of operations should be handled by lens. Hive should not invalidate operations upon timeout + + + hive.server2.async.exec.threads 5 Number of threads required in async thread pool for query execution in the lens directly. Lens uses http://git-wip-us.apache.org/repos/asf/lens/blob/bf25ac57/src/site/apt/admin/config.apt -- diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt index a1d4d1a..f22fc0c 100644 --- a/src/site/apt/admin/config.apt +++ b/src/site/apt/admin/config.apt @@ -36,245 +36,249 @@ Lens server configuration *--+--+---+--+ |6|hive.server2.authentication.ldap.url| |LDAP connection URL| *--+--+---+--+ -|7|hive.server2.log.redirection.enabled|false|Disable the log direction on the lens server sessions. There are no logs required for each session in lens sessions. This will decrease number of file handles associated to lens server.| +|7|hive.server2.idle.operation.timeout|0|Closing of operations should be handled by lens. Hive should not invalidate operations upon timeout| *--+--+---+--+ -|8|hive.server2.logging.operation.enabled|false|Disable the operation logging, since there are no execution operation happening in lens session. Keeping it true might conflict with Hive Server if both are running on the same machine causing disappearance of the operation log directory.| +|8|hive.server2.idle.session.timeout|0|Session should be closed by lens. There would be a mismatch when hive has closed session due to timeout and lens is not aware of that.| *--+--+---+--+ -|9|lens.query.current.time.millis|0|Query current time in millis. This is used to resolve 'now'. If value is set to zero, 'now' is resolved to current value| +|9|hive.server2.log.redirection.enabled|false|Disable the log direction on the lens server sessions. There are no logs required for each session in lens sessions. This will decrease number of file handles associated to lens server.| *--+--+---+--+ -|10|lens.server.alarm.service.impl|org.apache.lens.server.scheduler.AlarmService|Implementation class for alarm service| +|10|hive.server2.logging.operation.enabled|false|Disable the operation logging, since there are no execution operation happening in lens session. Keeping it true might conflict with Hive Server if both are running on the same machine causing disappearance of the operation log directory.| *--+--+---+--+ -|11|lens.server.appevent.ws.listener.impl|org.apache.lens.server.LensApplicationListener|Implementation class for Lens Application Listener| +|11|lens.query.current.time.millis|0|Query current time in millis. This is used to resolve 'now'. If value is set to zero, 'now' is resolved to current value| *--+--+---+--+ -|12|lens.server.base.url|http://0.0.0.0:/lensapi/|The base url for the lens server| +|12|lens.server.alarm.service.impl|org.apache.lens.server.scheduler.AlarmService|Implementation class for alarm service| *--+--+---+--+ -|13|lens.server.consistentState.ws.filter.impl|org.apache.lens.server.ConsistentStateFilter|Implementation class for ConsistentState Filter| +|13|lens.server.appevent.ws.listener.impl|org.apache.lens.server.LensApplicationListener
lens git commit: Revert "LENS-1340 : Add DataCompletenessChecker"
Repository: lens
Updated Branches:
refs/heads/master 5a18cce85 -> 1a57cdd82
Revert "LENS-1340 : Add DataCompletenessChecker"
This reverts commit 47a3012edc9604e94795d1453a77ac4ee2b1b936.
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/1a57cdd8
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/1a57cdd8
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/1a57cdd8
Branch: refs/heads/master
Commit: 1a57cdd82f6d89f37673bfced9fc1a2fb0c65f07
Parents: 5a18cce
Author: Amareshwari Sriramadasu
Authored: Tue Oct 4 13:28:43 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Tue Oct 4 13:28:43 2016 +0530
--
.../cube/parse/DataCompletenessChecker.java | 55
1 file changed, 55 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/1a57cdd8/lens-cube/src/main/java/org/apache/lens/cube/parse/DataCompletenessChecker.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/DataCompletenessChecker.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/DataCompletenessChecker.java
deleted file mode 100644
index 026c9a9..000
---
a/lens-cube/src/main/java/org/apache/lens/cube/parse/DataCompletenessChecker.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.cube.parse;
-
-import org.apache.lens.server.api.error.LensException;
-
-import java.util.Date;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * DataCompletenessChecker is for identifying the completeness of data in a
fact for the given set of measures, start
- * and end date. A fact will have a dataCompletenessTag, multiple facts can
have the same dataCompletenessTag.
- * Similarly, measures will have a dataCompletenessTag, multiple measures can
have the same dataCompletenessTag.
- * The api will take the dataCompletenessTag for the facts and measures and
compute the completeness based on these
- * tags. The utility of having tags is that the similar kind of measures or
facts which will have the same level of
- * completeness can use the same tag, thus we avoid the redundant completeness
computation for similar measures
- * and facts.
- * The implementations of the interface can truncate the start and end date.
- */
-public interface DataCompletenessChecker {
-
-/**
- * Get completeness of the set of measures in a fact based on the
dataCompletenessTag for the given starttime and
- * endtime.
- *
- * @param factTag This is the dataCompletenessTag for a fact. The tag can
be specified by setting the property
- *named dataCompletenessTag for the fact. Mutltiple facts
can have the same dataCompletenessTag.
- * @param start Start time of the query (Inclusive).
- * @param end End time of the query (Exclusive).
- * @param measureTag List of distinct tag of the measures in the query.
Multiple measures can have the same
- * dataCompletenessTag.
- * @return map; key is the name of the dataCompletenessTag which refers to
one or more measures. Value is the map
- * of date and %completeness.
- */
-Map> getCompleteness(String factTag, Date start,
Date end, Set measureTag)
-throws LensException;
-
-}
lens git commit: LENS-1344 : Make individual component headers on cube schema page on UI left aligned
Repository: lens
Updated Branches:
refs/heads/master 1a57cdd82 -> e6a59b739
LENS-1344 : Make individual component headers on cube schema page on UI left
aligned
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/e6a59b73
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/e6a59b73
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/e6a59b73
Branch: refs/heads/master
Commit: e6a59b7391364f1c7ad2b2894efcaf731dca119b
Parents: 1a57cdd
Author: Rajat Khandelwal
Authored: Wed Oct 5 14:57:58 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed Oct 5 14:57:58 2016 +0530
--
lens-ui/app/components/CubeSchemaComponent.js | 8
1 file changed, 4 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/e6a59b73/lens-ui/app/components/CubeSchemaComponent.js
--
diff --git a/lens-ui/app/components/CubeSchemaComponent.js
b/lens-ui/app/components/CubeSchemaComponent.js
index 27adfac..d7492eb 100644
--- a/lens-ui/app/components/CubeSchemaComponent.js
+++ b/lens-ui/app/components/CubeSchemaComponent.js
@@ -66,7 +66,7 @@ function constructMeasureTable(cubeName, measures) {
return (
-Measures
+Measures
{header}
{table}
@@ -131,7 +131,7 @@ function constructDimensionTable(cubeName, dimensions,
join_chains_by_name) {
return (
-Dim-Attributes
+Dim-Attributes
{header}
{table}
@@ -165,7 +165,7 @@ function constructJoinChainTable(cubeName, join_chains) {
return (
-Join Chains
+Join Chains
{header}
{table}
@@ -201,7 +201,7 @@ function constructExpressionTable(cubeName, expressions) {
return (
-Expressions
+Expressions
{header}
{table}
lens git commit: LENS-1343 : Fix expression not being propagated to error message in COULD_NOT_PARSE_EXPRESSION error
Repository: lens
Updated Branches:
refs/heads/master e6a59b739 -> 1bb189b30
LENS-1343 : Fix expression not being propagated to error message in
COULD_NOT_PARSE_EXPRESSION error
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/1bb189b3
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/1bb189b3
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/1bb189b3
Branch: refs/heads/master
Commit: 1bb189b30f99413fffd5ee978b7fbd728a03c268
Parents: e6a59b7
Author: Rajat Khandelwal
Authored: Thu Oct 6 10:21:12 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Oct 6 10:21:12 2016 +0530
--
.../org/apache/lens/cube/parse/HQLParser.java | 2 +-
.../apache/lens/cube/parse/TestHQLParser.java | 21
lens-cube/src/test/resources/hive-site.xml | 10 ++
3 files changed, 32 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/1bb189b3/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
--
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
index 6e52d32..1976248 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
@@ -201,7 +201,7 @@ public final class HQLParser {
try {
tree = driver.parseExpression(expr);
} catch (ParseException e) {
- throw new LensException(COULD_NOT_PARSE_EXPRESSION.getLensErrorInfo(),
e, e.getMessage());
+ throw new LensException(COULD_NOT_PARSE_EXPRESSION.getLensErrorInfo(),
e, expr);
}
return ParseUtils.findRootNonNullToken(tree);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/1bb189b3/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
--
diff --git
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
index 861d4f7..2619cb0 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
+import org.apache.lens.cube.error.LensCubeErrorCode;
import org.apache.lens.server.api.error.LensException;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -489,4 +490,24 @@ public class TestHQLParser {
ASTNode tree = HQLParser.parseHQL(expr, conf);
Assert.assertEquals(HQLParser.getString((ASTNode)
tree.getChild(1).getChild(0)), dirString);
}
+
+ @DataProvider
+ public Object[][] exprDataProvider() {
+return new Object[][] {
+ {"a.b", true},
+ {"a.date", false},
+};
+ }
+
+ @Test(dataProvider = "exprDataProvider")
+ public void testParseExpr(String expr, boolean success) {
+try {
+ HQLParser.parseExpr(expr);
+ Assert.assertTrue(success);
+} catch (LensException e) {
+ Assert.assertFalse(success);
+ Assert.assertTrue(e.getMessage().contains(expr));
+
Assert.assertTrue(e.getMessage().contains(LensCubeErrorCode.COULD_NOT_PARSE_EXPRESSION.name()));
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/1bb189b3/lens-cube/src/test/resources/hive-site.xml
--
diff --git a/lens-cube/src/test/resources/hive-site.xml
b/lens-cube/src/test/resources/hive-site.xml
index d21e61c..68dedfc 100644
--- a/lens-cube/src/test/resources/hive-site.xml
+++ b/lens-cube/src/test/resources/hive-site.xml
@@ -57,4 +57,14 @@
true
+
+hive.metastore.schema.verification
+false
+
+
+
+hive.support.sql11.reserved.keywords
+false
+
+
lens git commit: LENS-1355 : Fix parsing fields named 'date'
Repository: lens
Updated Branches:
refs/heads/master 39252f9af -> 182f6dcac
LENS-1355 : Fix parsing fields named 'date'
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/182f6dca
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/182f6dca
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/182f6dca
Branch: refs/heads/master
Commit: 182f6dcacc0226dc0e070edaa3fc8ba4aa3e6146
Parents: 39252f9
Author: Rajat Khandelwal
Authored: Thu Oct 13 12:14:30 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Oct 13 12:14:30 2016 +0530
--
.../lens/cube/parse/AggregateResolver.java | 2 +-
.../lens/cube/parse/CubeQueryContext.java | 4 +--
.../apache/lens/cube/parse/GroupbyResolver.java | 4 +--
.../org/apache/lens/cube/parse/HQLParser.java | 26 +---
.../apache/lens/cube/parse/TestHQLParser.java | 9 ---
5 files changed, 33 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/182f6dca/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
index c522061..9658100 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
@@ -119,7 +119,7 @@ class AggregateResolver implements ContextRewriter {
cubeql.getExprCtx().getAllExprsQueried().get(expr).iterator();
itrContext.hasNext();) {
for (Iterator itrCol =
itrContext.next().getExprCol().getExpressionSpecs().iterator();
itrCol.hasNext();) {
-ASTNode exprAST = HQLParser.parseExpr(itrCol.next().getExpr());
+ASTNode exprAST = HQLParser.parseExpr(itrCol.next().getExpr(),
cubeql.getConf());
if (HQLParser.isAggregateAST(exprAST)) {
return true;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/182f6dca/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 6d53d00..e83ae76 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -886,10 +886,10 @@ public class CubeQueryContext extends
TracksQueriedColumns implements QueryAST {
String table = entry.getKey();
String rangeWhere = entry.getValue();
if (!StringUtils.isBlank(rangeWhere)) {
-ASTNode rangeAST = HQLParser.parseExpr(rangeWhere);
+ASTNode rangeAST = HQLParser.parseExpr(rangeWhere, conf);
range.getParent().setChild(range.getChildIndex(), rangeAST);
}
- fact.getStorgeWhereClauseMap().put(table,
HQLParser.parseExpr(getWhereString()));
+ fact.getStorgeWhereClauseMap().put(table,
HQLParser.parseExpr(getWhereString(), conf));
}
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/182f6dca/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
index 26ae1e7..216ae52 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
@@ -71,7 +71,7 @@ class GroupbyResolver implements ContextRewriter {
String expr = sel.getExprWithoutAlias();
if (!groupByExprs.contains(expr)) {
if (!sel.isAggregate()) {
-ASTNode exprAST = HQLParser.parseExpr(expr);
+ASTNode exprAST = HQLParser.parseExpr(expr, cubeql.getConf());
ASTNode groupbyAST = cubeql.getGroupByAST();
if (!isConstantsUsed(exprAST)) {
if (groupbyAST != null) {
@@ -130,7 +130,7 @@ class GroupbyResolver implements ContextRewriter {
int index = 0;
for (String expr : groupByExprs) {
if (!contains(selectExprs, expr)) {
-ASTNode exprAST = HQLParser.parseExpr(expr);
+ASTNode exprAST = HQLParser.parseExpr(expr, cubeql.getConf());
addChildAtIndex(index, cubeql.getSelectAST(), exprAST);
index++;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/182f6
lens git commit: LENS-1325 : Add tests and document passing encoded urls for query http notifications
Repository: lens
Updated Branches:
refs/heads/master 70855ea8c -> 54961853f
LENS-1325 : Add tests and document passing encoded urls for query http
notifications
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/54961853
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/54961853
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/54961853
Branch: refs/heads/master
Commit: 54961853fb6762431f5e20d0af95be3bb00d2afa
Parents: 70855ea
Author: Puneet Gupta
Authored: Thu Oct 20 09:53:24 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Oct 20 09:53:24 2016 +0530
--
.../server/query/QueryEventHttpNotifier.java| 6 ++---
.../src/main/resources/lenssession-default.xml | 3 ++-
.../query/TestQueryNotifictaionResource.java| 23 ++--
.../lens/server/query/TestQueryService.java | 15 ++---
src/site/apt/admin/session-config.apt | 2 +-
5 files changed, 39 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/54961853/lens-server/src/main/java/org/apache/lens/server/query/QueryEventHttpNotifier.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/QueryEventHttpNotifier.java
b/lens-server/src/main/java/org/apache/lens/server/query/QueryEventHttpNotifier.java
index 1760bec..6e7bd3b 100644
---
a/lens-server/src/main/java/org/apache/lens/server/query/QueryEventHttpNotifier.java
+++
b/lens-server/src/main/java/org/apache/lens/server/query/QueryEventHttpNotifier.java
@@ -146,7 +146,7 @@ public abstract class QueryEventHttpNotifier extends Async
queryContext.getQueryHandleString(), httpEndPoint, responseCode);
} catch (LensException e) {
log.error("Error while sending {} HTTP Notification for Query {} to
{}", getNotificationType(),
- queryContext.getQueryHandleString(), httpEndPoint, e);
+ queryContext.getQueryHandleString(), httpEndPoint, e); // continue
to try other httpEndPoints..
}
}
}
@@ -187,7 +187,6 @@ public abstract class QueryEventHttpNotifier extends Async
*/
private int notifyEvent(String httpEndPoint, Map
eventDetails, MediaType mediaType)
throws LensException {
-
final WebTarget target = buildClient().target(httpEndPoint);
FormDataMultiPart mp = new FormDataMultiPart();
for (Map.Entry eventDetail : eventDetails.entrySet()) {
@@ -203,8 +202,9 @@ public abstract class QueryEventHttpNotifier extends Async
} catch (Exception e) {
throw new LensException("Error while publishing Http notification", e);
}
+
//2XX = SUCCESS
-if (!(response.getStatus() >= 200 && response.getStatus() < 300)) {
+if
(!response.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL))
{
throw new LensException("Error while publishing Http notification.
Response code " + response.getStatus());
}
return response.getStatus();
http://git-wip-us.apache.org/repos/asf/lens/blob/54961853/lens-server/src/main/resources/lenssession-default.xml
--
diff --git a/lens-server/src/main/resources/lenssession-default.xml
b/lens-server/src/main/resources/lenssession-default.xml
index b6bfa2e..9a8f9da 100644
--- a/lens-server/src/main/resources/lenssession-default.xml
+++ b/lens-server/src/main/resources/lenssession-default.xml
@@ -263,7 +263,8 @@
lens.query.http.notification.urls
These are the http end points for Query http notifications.
Users can specify more than one comma
- separated end points for a query. If this property is not set, no http
notification will be sent out by lens
+ separated end points for a query. Url parameter values that include
special characters should be
+ encoded. Please note that if this property is not set, no http
notification will be sent out by lens
server for the query.
http://git-wip-us.apache.org/repos/asf/lens/blob/54961853/lens-server/src/test/java/org/apache/lens/server/query/TestQueryNotifictaionResource.java
--
diff --git
a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryNotifictaionResource.java
b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryNotifictaionResource.java
index 6a57b20..da1a1c6 100644
---
a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryNotifictaionResource.java
+++
b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryNotifictaionResource.java
@@ -45,6 +45,10 @@ public class TestQueryNotifictaionResource {
private static int failedCount = 0;
@
lens git commit: LENS-1361 : Ignore DB failure while scheduler startup
Repository: lens
Updated Branches:
refs/heads/master 54961853f -> f274e29d5
LENS-1361 : Ignore DB failure while scheduler startup
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/f274e29d
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/f274e29d
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/f274e29d
Branch: refs/heads/master
Commit: f274e29d53998f482697bfd8c57a3841b1779abc
Parents: 5496185
Author: Lavkesh Lahngir
Authored: Fri Oct 21 10:33:22 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Fri Oct 21 10:33:22 2016 +0530
--
.../org/apache/lens/server/scheduler/SchedulerDAO.java | 11 ++-
.../lens/server/scheduler/SchedulerServiceImpl.java | 6 +-
2 files changed, 11 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/f274e29d/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java
b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java
index 69e81c1..5f472fc 100644
---
a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java
+++
b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java
@@ -60,8 +60,9 @@ public class SchedulerDAO {
this.store.createJobInstanceTable();
this.store.createJobInstanceRunTable();
} catch (SQLException e) {
+ // If tables are not created, the DAO operations will fail at runtime.
+ // The APIs will fail with Internal Server Error.
log.error("Error creating job tables", e);
- throw new LensException("Error creating job tables ", e);
} catch (ClassNotFoundException e) {
log.error("No class found ", e);
throw new LensException("No class found ", e);
@@ -239,7 +240,7 @@ public class SchedulerDAO {
return store.getAllJobInstances(id.getHandleIdString());
} catch (SQLException e) {
log.error("Error while getting instances of a job with id {}",
id.getHandleIdString(), e);
- return null;
+ return new ArrayList<>();
}
}
@@ -258,7 +259,7 @@ public class SchedulerDAO {
return store.getJobs(username, jobStates == null ? new
SchedulerJobState[] {} : jobStates, startTime, endTime);
} catch (SQLException e) {
log.error("Error while getting jobs ", e);
- return null;
+ return new ArrayList<>();
}
}
@@ -273,7 +274,7 @@ public class SchedulerDAO {
return store.getJobsByName(jobName);
} catch (SQLException e) {
log.error("Error while getting jobs ", e);
- return null;
+ return new ArrayList<>();
}
}
@@ -288,7 +289,7 @@ public class SchedulerDAO {
return store.getInstanceRuns(states);
} catch (SQLException e) {
log.error("Error while getting jobs ", e);
- return null;
+ return new ArrayList<>();
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/f274e29d/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
index 00130d0..73277de 100644
---
a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
+++
b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
@@ -81,6 +81,7 @@ public class SchedulerServiceImpl extends BaseLensService
implements SchedulerSe
private AlarmService alarmService;
private int maxJobsPerUser =
LensConfConstants.DEFAULT_MAX_SCHEDULED_JOB_PER_USER;
+ private boolean healthy = true;
/**
* Instantiates a new scheduler service.
@@ -105,6 +106,7 @@ public class SchedulerServiceImpl extends BaseLensService
implements SchedulerSe
getEventService().addListenerForType(schedulerQueryEventListener,
QueryEnded.class);
} catch (LensException e) {
log.error("Error Initialising Scheduler-service", e);
+ healthy = false;
}
}
@@ -126,7 +128,6 @@ public class SchedulerServiceImpl extends BaseLensService
implements SchedulerSe
*/
@Override
public synchronized void start() {
-super.start();
List instanceRuns = schedulerDAO
.getInstanceRuns(SchedulerJobInstanceState.WAITING,
SchedulerJobInstanceState.LAUNCHED,
SchedulerJobInstanceState.RUNNING);
@@ -166,6 +167,9 @@ public class SchedulerServiceImpl extends BaseLensService
implements SchedulerSe
}
}
}
+if (healthy) {
+ super.start();
+}
}
/**
lens git commit: LENS-1327 : Handle data delay for scheduled jobs
Repository: lens
Updated Branches:
refs/heads/master d09e5e2c8 -> 95a4f596c
LENS-1327 : Handle data delay for scheduled jobs
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/95a4f596
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/95a4f596
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/95a4f596
Branch: refs/heads/master
Commit: 95a4f596c42f57647b2eb12c0419175c9c87de8e
Parents: d09e5e2
Author: Lavkesh Lahngir
Authored: Wed Nov 2 13:58:36 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Wed Nov 2 13:58:36 2016 +0530
--
.../scheduler/SchedulerJobInstanceEvent.java| 3 +-
.../scheduler/SchedulerJobInstanceState.java| 39 ++---
.../lens/server/api/LensConfConstants.java | 15 +++-
.../lens/server/scheduler/SchedulerDAO.java | 5 +-
.../scheduler/SchedulerEventListener.java | 46 +++---
.../server/scheduler/SchedulerServiceImpl.java | 77 +++--
.../src/main/resources/lensserver-default.xml | 10 ++-
.../lens/server/scheduler/SchedulerDAOTest.java | 2 +-
.../server/scheduler/SchedulerRestartTest.java | 12 +++
src/site/apt/admin/config.apt | 88 ++--
10 files changed, 217 insertions(+), 80 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/95a4f596/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceEvent.java
--
diff --git
a/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceEvent.java
b/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceEvent.java
index 4f3409c..1bcbeca 100644
---
a/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceEvent.java
+++
b/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceEvent.java
@@ -22,9 +22,10 @@ package org.apache.lens.api.scheduler;
* All events(actions) which can happen on an instance of
SchedulerJob.
*/
public enum SchedulerJobInstanceEvent {
- ON_CREATION, // an instance is first considered by the scheduler.
+ ON_PREPARE, // an instance is first considered by the scheduler.
ON_TIME_OUT,
ON_CONDITIONS_MET,
+ ON_CONDITIONS_NOT_MET,
ON_RUN,
ON_SUCCESS,
ON_FAILURE,
http://git-wip-us.apache.org/repos/asf/lens/blob/95a4f596/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceState.java
--
diff --git
a/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceState.java
b/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceState.java
index 3d2605e..52b3aec 100644
---
a/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceState.java
+++
b/lens-api/src/main/java/org/apache/lens/api/scheduler/SchedulerJobInstanceState.java
@@ -18,7 +18,7 @@
*/
package org.apache.lens.api.scheduler;
-import javax.xml.bind.annotation.*;
+import javax.xml.bind.annotation.XmlRootElement;
import org.apache.lens.api.error.InvalidStateTransitionException;
@@ -31,10 +31,8 @@ public enum SchedulerJobInstanceState
public SchedulerJobInstanceState nextTransition(SchedulerJobInstanceEvent
event)
throws InvalidStateTransitionException {
switch (event) {
- case ON_CREATION:
-return this;
- case ON_CONDITIONS_MET:
-return SchedulerJobInstanceState.LAUNCHED;
+ case ON_PREPARE:
+return SchedulerJobInstanceState.LAUNCHING;
case ON_TIME_OUT:
return SchedulerJobInstanceState.TIMED_OUT;
case ON_RUN:
@@ -52,13 +50,15 @@ public enum SchedulerJobInstanceState
}
},
- LAUNCHED {
+ LAUNCHING {
@Override
public SchedulerJobInstanceState nextTransition(SchedulerJobInstanceEvent
event)
throws InvalidStateTransitionException {
switch (event) {
case ON_CONDITIONS_MET:
-return this;
+return SchedulerJobInstanceState.LAUNCHED;
+ case ON_CONDITIONS_NOT_MET:
+return SchedulerJobInstanceState.WAITING;
case ON_RUN:
return SchedulerJobInstanceState.RUNNING;
case ON_SUCCESS:
@@ -74,6 +74,25 @@ public enum SchedulerJobInstanceState
}
},
+ LAUNCHED {
+@Override
+public SchedulerJobInstanceState nextTransition(SchedulerJobInstanceEvent
event)
+ throws InvalidStateTransitionException {
+ switch (event) {
+ case ON_RUN:
+return SchedulerJobInstanceState.RUNNING;
+ case ON_SUCCESS:
+return SchedulerJobInstanceState.SUCCEEDED;
+ case ON_FAILURE:
+return SchedulerJobInstanceState.FAILED;
+ case ON_KILL:
+return SchedulerJobInstanceState.KILLED;
+ default:
+throw
lens git commit: LENS-1342 : Close InputStreamReader in PropertyBasedUserConfigLoader
Repository: lens
Updated Branches:
refs/heads/master 95a4f596c -> d1a0e0e26
LENS-1342 : Close InputStreamReader in PropertyBasedUserConfigLoader
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d1a0e0e2
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d1a0e0e2
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d1a0e0e2
Branch: refs/heads/master
Commit: d1a0e0e266b5f7ffd0ad1d5b742fb92e4a91eeb3
Parents: 95a4f59
Author: Ted Yu
Authored: Thu Nov 3 10:40:13 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Nov 3 10:40:13 2016 +0530
--
.../apache/lens/server/user/PropertyBasedUserConfigLoader.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/d1a0e0e2/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java
b/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java
index 1c025dd..632f71c 100644
---
a/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java
+++
b/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java
@@ -52,8 +52,8 @@ public class PropertyBasedUserConfigLoader implements
UserConfigLoader {
throw new UserConfigLoaderException("property file path not provided for
property based resolver."
+ "Please set property " +
LensConfConstants.USER_RESOLVER_PROPERTYBASED_FILENAME);
}
-try {
- properties.load(new InputStreamReader(new FileInputStream(new
File(filename)), "UTF-8"));
+try (InputStreamReader reader = new InputStreamReader(new
FileInputStream(new File(filename)), "UTF-8")) {
+ properties.load(reader);
} catch (IOException e) {
throw new UserConfigLoaderException("property file not found. Provided
path was: " + filename);
}
lens git commit: LENS-1357 : Fix NPE in stop lens services after a failed attempt of start
Repository: lens
Updated Branches:
refs/heads/master d1a0e0e26 -> 9b0a453de
LENS-1357 : Fix NPE in stop lens services after a failed attempt of start
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/9b0a453d
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/9b0a453d
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/9b0a453d
Branch: refs/heads/master
Commit: 9b0a453de7059bf453a84f2c5945095151a6a9d7
Parents: d1a0e0e
Author: Rajat Khandelwal
Authored: Thu Nov 3 10:44:30 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Nov 3 10:44:30 2016 +0530
--
.../org/apache/lens/server/LensServices.java | 19 +++
1 file changed, 11 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/9b0a453d/lens-server/src/main/java/org/apache/lens/server/LensServices.java
--
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
index 7ff5cbb..a3e272a 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
@@ -411,16 +411,19 @@ public class LensServices extends CompositeService
implements ServiceProvider {
if (isServerStatePersistenceEnabled) {
try {
//1. shutdown serverSnapshotScheduler gracefully by allowing already
triggered task (if any) to finish
- serverSnapshotScheduler.shutdown();
- try { //Wait for shutdown. Shutdown should be immediate in case no
task is running at this point
-while (!serverSnapshotScheduler.awaitTermination(1,
TimeUnit.MINUTES)) {
- log.info("Waiting for Lens-server-snapshotter to shutdown
gracefully...");
+ if (serverSnapshotScheduler != null) {
+serverSnapshotScheduler.shutdown();
+try { //Wait for shutdown. Shutdown should be immediate in case no
task is running at this point
+ while (!serverSnapshotScheduler.awaitTermination(1,
TimeUnit.MINUTES)) {
+log.info("Waiting for Lens-server-snapshotter to shutdown
gracefully...");
+ }
+} catch (InterruptedException e) {
+ log.error("Lens-server-snapshotter interrupted while shutting
down", e);
}
- } catch (InterruptedException e) {
-log.error("Lens-server-snapshotter interrupted while shutting
down" , e);
+log.info("Lens-server-snapshotter was shutdown");
+ } else {
+log.info("Lens-server-snapshotter wasn't started, so no need to
shutdown");
}
- log.info("Lens-server-snapshotter was shutdown");
-
//2. persist the latest state of all the services
persistLensServiceState();
} finally {
lens git commit: LENS-1370 : Fix Queries persisted before retry framework upgrade are not getting purged
Repository: lens
Updated Branches:
refs/heads/master 9b0a453de -> 3c23f18c2
LENS-1370 : Fix Queries persisted before retry framework upgrade are not
getting purged
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3c23f18c
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3c23f18c
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3c23f18c
Branch: refs/heads/master
Commit: 3c23f18c22aeacd891acd9ef7b7900e53c0015f1
Parents: 9b0a453
Author: Rajat Khandelwal
Authored: Thu Nov 3 11:50:04 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Nov 3 11:50:04 2016 +0530
--
.../src/main/java/org/apache/lens/server/query/LensServerDAO.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/3c23f18c/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
--
diff --git
a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index 3ab3aef..dd489e8 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -137,7 +137,8 @@ public class LensServerDAO {
query.getStartTime(), query.getEndTime(), query.getResult(),
query.getStatus(), query.getMetadata(),
query.getRows(), query.getFileSize(), query.getErrorMessage(),
query.getDriverStartTime(),
query.getDriverEndTime(), query.getDriverName(),
query.getQueryName(), query.getSubmissionTime(),
-query.getDriverQuery(), serializeConf(query.getConf()),
query.getFailedAttempts().size());
+query.getDriverQuery(), serializeConf(query.getConf()),
+query.getFailedAttempts() == null ? 0 :
query.getFailedAttempts().size());
for (int i = 0; i < query.getFailedAttempts().size(); i++) {
insertFailedAttempt(runner, conn, query.getHandle(),
query.getFailedAttempts().get(i), i);
}
lens git commit: LENS-1374 : Add more logs in FieldValidator
Repository: lens
Updated Branches:
refs/heads/master 08d79631b -> 666fc776a
LENS-1374 : Add more logs in FieldValidator
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/666fc776
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/666fc776
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/666fc776
Branch: refs/heads/master
Commit: 666fc776a1f22c5b308b6eb1ea054a52d6ec141d
Parents: 08d7963
Author: Rajat Khandelwal
Authored: Thu Nov 10 11:11:51 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Thu Nov 10 11:11:51 2016 +0530
--
.../main/java/org/apache/lens/cube/parse/FieldValidator.java | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/666fc776/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java
b/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java
index 7f4aa93..48af0c9 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java
@@ -33,10 +33,12 @@ import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import com.google.common.collect.ImmutableSet;
+import lombok.extern.slf4j.Slf4j;
/**
* Validate fields based on cube queryability
*/
+@Slf4j
public class FieldValidator implements ContextRewriter {
@Override
@@ -73,12 +75,13 @@ public class FieldValidator implements ContextRewriter {
// do validation
// Find atleast one derived cube which contains all the dimensions
// queried.
-
+ log.info("Chained columns: {}, Queried dim attributes: {}",
chainedSrcColumns, queriedDimAttrs);
boolean derivedCubeFound = false;
for (DerivedCube dcube : dcubes) {
if (dcube.getDimAttributeNames().containsAll(chainedSrcColumns)
&& dcube.getDimAttributeNames().containsAll(queriedDimAttrs)) {
+ log.info("Derived cube found: {}", dcube.getName());
// remove all the measures that are covered
queriedMsrs.removeAll(dcube.getMeasureNames());
derivedCubeFound = true;
@@ -93,9 +96,10 @@ public class FieldValidator implements ContextRewriter {
}
if (!queriedMsrs.isEmpty()) {
+log.info("Uncovered queried Measures: {}", queriedMsrs);
// Add appropriate message to know which fields are not queryable
together
if (!nonQueryableFields.isEmpty()) {
-
+ log.info("Non queryable fields: {}", nonQueryableFields);
conflictingFields.addAll(nonQueryableFields);
conflictingFields.addAll(queriedMsrs);
throw new FieldsCannotBeQueriedTogetherException(new
ConflictingFields(conflictingFields));
[1/2] lens git commit: LENS-1333: Add data completeness checks on query writing path
Repository: lens Updated Branches: refs/heads/master 0cce226f7 -> 078555c1a http://git-wip-us.apache.org/repos/asf/lens/blob/078555c1/src/site/apt/admin/config.apt -- diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt index 21b23f9..f89caab 100644 --- a/src/site/apt/admin/config.apt +++ b/src/site/apt/admin/config.apt @@ -44,245 +44,249 @@ Lens server configuration *--+--+---+--+ |10|hive.server2.logging.operation.enabled|false|Disable the operation logging, since there are no execution operation happening in lens session. Keeping it true might conflict with Hive Server if both are running on the same machine causing disappearance of the operation log directory.| *--+--+---+--+ -|11|lens.query.current.time.millis|0|Query current time in millis. This is used to resolve 'now'. If value is set to zero, 'now' is resolved to current value| +|11|lens.cube.metastore.completeness.checker.class|org.apache.lens.server.api.metastore.DefaultChecker|The class that implements the DataCompletenessChecker Interface. This will take effect if the flag "lens.cube.metastore.enable.datacompleteness.check" is set.| *--+--+---+--+ -|12|lens.server.alarm.service.impl|org.apache.lens.server.scheduler.AlarmService|Implementation class for alarm service| +|12|lens.cube.metastore.enable.datacompleteness.check|false|This property is to enable Data Completeness Checks while resolving partitions.| *--+--+---+--+ -|13|lens.server.appevent.ws.listener.impl|org.apache.lens.server.LensApplicationListener|Implementation class for Lens Application Listener| +|13|lens.query.current.time.millis|0|Query current time in millis. This is used to resolve 'now'. If value is set to zero, 'now' is resolved to current value| *--+--+---+--+ -|14|lens.server.base.url|http://0.0.0.0:/lensapi/|The base url for the lens server| +|14|lens.server.alarm.service.impl|org.apache.lens.server.scheduler.AlarmService|Implementation class for alarm service| *--+--+---+--+ -|15|lens.server.consistentState.ws.filter.impl|org.apache.lens.server.ConsistentStateFilter|Implementation class for ConsistentState Filter| +|15|lens.server.appevent.ws.listener.impl|org.apache.lens.server.LensApplicationListener|Implementation class for Lens Application Listener| *--+--+---+--+ -|16|lens.server.database.resource.dir|/tmp/lens/resources|Top level directory to store database specific static resources which are added to each database. To add a static resource for a DB, create a directory with the same name as that of the DB under this directory, and place the resources inside the DB directory. Any subdirectories under the DB directory and the resources under them will be ignored| +|16|lens.server.base.url|http://0.0.0.0:/lensapi/|The base url for the lens server| *--+--+---+--+ -|17|lens.server.db.driver.name|org.hsqldb.jdbcDriver|Database driver for database where Finished queries have to be stored. Also used for database based user config loaders| +|17|lens.server.consistentState.ws.filter.impl|org.apache.lens.server.ConsistentStateFilter|Implementation class for ConsistentState Filter| *--+--+---+--+ -|18|lens.server.db.jdbc.pass| |JDBC Password for Finished queries table. Also used for database based user config loaders| +|18|lens.server.database.resource.dir|/tmp/lens/resources|Top level directory to store database specific static resources which are added to each database. To add a static resource for a DB, create a directory with the same name as that of the DB under this directory, and place the resources inside the DB directory. Any subdirectories under the DB directory and the resources under them will be ignored| *--+--+---+--+ -|19|lens.server.db.jdbc.url|jdbc:hsqldb:/tmp/lensserver/queries.db|JDBC URL where the database for storing finished queries is located. Also used for database based user config loaders| +|19|lens.server.db.driver.name|org.hsqldb.jdbcDriver|Database driver for database where Finished queries have to be stored. Also used for database based user config loaders| *--+--+---+--+ -|20|lens.server.db.jdbc.user|SA|JDBC User for Finished queries table. Also used for database based user config loaders| +|20|lens.server.db.jdbc.pass| |JDBC Password for Finished queries table. Also used for database based user config loaders| *--+--+---+--+ -|21|lens.server.db.validation.query|select 1 from INFORMATION_SCHEMA.SYSTEM_USERS|BasicDatasource has a config parameter called validationQuery. This query is run before passing the Datasource to the application to verify if connection is valid.\ | +|21|lens.server.db.jdbc.url|jdbc:hsqldb:/tmp/lensserver/queries.db|JDBC URL where the database for storing finished queries is located. Also used for database based user config loaders| +*--+--+---+--+ +|22|lens.server.db.jdbc.user|SA|JDBC User for Finished queries table. Also used for database based user config loaders|
[2/2] lens git commit: LENS-1333: Add data completeness checks on query writing path
LENS-1333: Add data completeness checks on query writing path
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/078555c1
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/078555c1
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/078555c1
Branch: refs/heads/master
Commit: 078555c1a92cecef19f53962703d7dd00eb44633
Parents: 0cce226
Author: Narayan Periwal
Authored: Fri Nov 11 09:05:49 2016 +0530
Committer: Amareshwari Sriramadasu
Committed: Fri Nov 11 09:05:49 2016 +0530
--
.../lens/cube/metadata/CubeFactTable.java | 4 +
.../lens/cube/metadata/CubeMetastoreClient.java | 22 ++
.../lens/cube/metadata/MetastoreConstants.java | 2 +
.../apache/lens/cube/parse/CandidateFact.java | 3 +
.../cube/parse/CandidateTablePruneCause.java| 21 ++
.../lens/cube/parse/CubeQueryConfUtil.java | 4 +-
.../cube/parse/DataCompletenessChecker.java | 55 -
.../cube/parse/MaxCoveringFactResolver.java | 55 -
.../lens/cube/parse/StorageTableResolver.java | 154 +++-
.../src/main/resources/olap-query-conf.xml | 13 +
.../apache/lens/cube/parse/CubeTestSetup.java | 42 +++-
.../FieldsCannotBeQueriedTogetherTest.java | 1 -
.../cube/parse/MockCompletenessChecker.java | 46
.../lens/cube/parse/TestBaseCubeQueries.java| 3 +-
.../lens/cube/parse/TestCubeRewriter.java | 48
.../lens/cube/parse/TestQueryRewrite.java | 5 +
.../lens/server/api/LensConfConstants.java | 23 ++
.../api/metastore/DataCompletenessChecker.java | 55 +
.../server/api/metastore/DefaultChecker.java| 34 +++
.../src/main/resources/lensserver-default.xml | 12 +
.../src/main/resources/lenssession-default.xml | 7 +
src/site/apt/admin/config.apt | 240 ++-
src/site/apt/admin/session-config.apt | 78 +++---
src/site/apt/user/olap-query-conf.apt | 60 ++---
24 files changed, 737 insertions(+), 250 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/lens/blob/078555c1/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
index fb958c3..adb6c92 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
@@ -314,6 +314,10 @@ public class CubeFactTable extends AbstractCubeTable {
addCubeNames(getName(), getProperties(), cubeName);
}
+ public String getDataCompletenessTag() {
+return getProperties().get(MetastoreConstants.FACT_DATA_COMPLETENESS_TAG);
+ }
+
public boolean isAggregated() {
// It's aggregate table unless explicitly set to false
return
!"false".equalsIgnoreCase(getProperties().get(MetastoreConstants.FACT_AGGREGATED_PROPERTY));
http://git-wip-us.apache.org/repos/asf/lens/blob/078555c1/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
--
diff --git
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index e14c43f..6c9cde2 100644
---
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -31,7 +31,9 @@ import org.apache.lens.cube.metadata.Storage.LatestInfo;
import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo;
import org.apache.lens.cube.metadata.timeline.PartitionTimeline;
import org.apache.lens.cube.metadata.timeline.PartitionTimelineFactory;
+import org.apache.lens.server.api.*;
import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.metastore.DataCompletenessChecker;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.commons.lang.StringUtils;
@@ -45,6 +47,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.thrift.TException;
import com.google.common.collect.Lists;
@@ -92,6 +95,25 @@ public class CubeMetastoreClient {
private static final Map CLIENT_MAPPING =
Maps.newConcurrentMap();
// Set of all storage table names for which latest partitions exist
private final Set latestLookupCache = Sets.newSetFromMap(new
Co
