Repository: lens Updated Branches: refs/heads/master fe4ea6bd6 -> 4fa61d994
LENS-1317 : Lens Query Conf to be persisted in Finished_Queries store Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/4fa61d99 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/4fa61d99 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/4fa61d99 Branch: refs/heads/master Commit: 4fa61d9940af6ede89401d5837f1f7796810f254 Parents: fe4ea6b Author: Srikanth Sundarrajan <srik...@apache.org> Authored: Tue Sep 20 10:59:28 2016 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Tue Sep 20 10:59:28 2016 +0530 ---------------------------------------------------------------------- .../server/api/query/FinishedLensQuery.java | 9 +++-- .../apache/lens/server/query/LensServerDAO.java | 38 +++++++++++++++++--- .../apache/lens/server/query/TestLensDAO.java | 5 +++ 3 files changed, 45 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/4fa61d99/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java index edd4284..b58fcf9 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java @@ -46,7 +46,7 @@ import lombok.ToString; * * @see java.lang.Object#hashCode() */ -@EqualsAndHashCode(exclude = "selectedDriver") +@EqualsAndHashCode(exclude = {"selectedDriver", "conf"}) /* * (non-Javadoc) * @@ -174,6 +174,10 @@ public class FinishedLensQuery { @Setter private String priority; + @Getter + @Setter + private LensConf conf; + /** * The selected driver's query. */ @@ -217,6 +221,7 @@ public class FinishedLensQuery { if (null != ctx.getPriority()) { this.priority = ctx.getPriority().toString(); } + this.conf = ctx.getLensConf(); } public QueryContext toQueryContext(Configuration conf, Collection<LensDriver> drivers) { @@ -226,7 +231,7 @@ public class FinishedLensQuery { } QueryContext qctx = - new QueryContext(userQuery, submitter, new LensConf(), conf, drivers, selectedDriver, submissionTime, + new QueryContext(userQuery, submitter, this.conf, conf, drivers, selectedDriver, submissionTime, false); qctx.setQueryHandle(QueryHandle.fromString(handle)); http://git-wip-us.apache.org/repos/asf/lens/blob/4fa61d99/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java index 2ec938e..997b5a8 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java @@ -18,6 +18,7 @@ */ package org.apache.lens.server.query; +import java.nio.charset.Charset; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -26,12 +27,16 @@ import java.util.List; import javax.sql.DataSource; +import org.apache.lens.api.LensConf; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.api.query.QueryStatus; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.FinishedLensQuery; import org.apache.lens.server.util.UtilityMethods; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.dbutils.BasicRowProcessor; +import org.apache.commons.dbutils.BeanProcessor; import org.apache.commons.dbutils.DbUtils; import org.apache.commons.dbutils.QueryRunner; import org.apache.commons.dbutils.ResultSetHandler; @@ -87,7 +92,8 @@ public class LensServerDAO { + "starttime bigint, " + "endtime bigint," + "result varchar(255)," + "status varchar(255), " + "metadata varchar(100000), " + "rows int, " + "filesize bigint, " + "errormessage varchar(10000), " + "driverstarttime bigint, " + "driverendtime bigint, " + "drivername varchar(10000), " - + "queryname varchar(255), " + "submissiontime bigint, " + "driverquery varchar(1000000)" + ")"; + + "queryname varchar(255), " + "submissiontime bigint, " + "driverquery varchar(1000000), " + + "conf varchar(100000))"; try { QueryRunner runner = new QueryRunner(ds); runner.update(sql); @@ -110,8 +116,8 @@ public class LensServerDAO { Connection conn = null; String sql = "insert into finished_queries (handle, userquery, submitter, priority, " + "starttime,endtime,result,status,metadata,rows,filesize," - + "errormessage,driverstarttime,driverendtime, drivername, queryname, submissiontime, driverquery)" - + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + + "errormessage,driverstarttime,driverendtime, drivername, queryname, submissiontime, driverquery, conf)" + + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; try { conn = getConnection(); QueryRunner runner = new QueryRunner(); @@ -119,7 +125,7 @@ public class LensServerDAO { query.getStartTime(), query.getEndTime(), query.getResult(), query.getStatus(), query.getMetadata(), query.getRows(), query.getFileSize(), query.getErrorMessage(), query.getDriverStartTime(), query.getDriverEndTime(), query.getDriverName(), query.getQueryName(), query.getSubmissionTime(), - query.getDriverQuery()); + query.getDriverQuery(), serializeConf(query.getConf())); conn.commit(); } finally { DbUtils.closeQuietly(conn); @@ -137,6 +143,15 @@ public class LensServerDAO { } } + private String serializeConf(LensConf conf) { + return Base64.encodeBase64String(conf.toXMLString().getBytes(Charset.defaultCharset())); + } + + private LensConf deserializeConf(String serializedConf) { + return LensConf.fromXMLString(new String(Base64.decodeBase64(serializedConf), + Charset.defaultCharset()), LensConf.class); + } + /** * Fetch Finished query from Database. * @@ -144,7 +159,8 @@ public class LensServerDAO { * @return Finished query. */ public FinishedLensQuery getQuery(String handle) { - ResultSetHandler<FinishedLensQuery> rsh = new BeanHandler<FinishedLensQuery>(FinishedLensQuery.class); + ResultSetHandler<FinishedLensQuery> rsh = new BeanHandler<>(FinishedLensQuery.class, + new BasicRowProcessor(new FinishedLensQueryBeanProcessor())); String sql = "select * from finished_queries where handle=?"; QueryRunner runner = new QueryRunner(ds); try { @@ -155,6 +171,18 @@ public class LensServerDAO { return null; } + private class FinishedLensQueryBeanProcessor extends BeanProcessor { + + @Override + protected Object processColumn(ResultSet rs, int index, Class<?> propType) throws SQLException { + Object obj = super.processColumn(rs, index, propType); + if (obj != null && propType.equals(LensConf.class) && obj instanceof String) { + return deserializeConf((String) obj); + } + return obj; + } + } + /** * Find finished queries. * http://git-wip-us.apache.org/repos/asf/lens/blob/4fa61d99/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java index 2a820ec..e620dc5 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java @@ -72,6 +72,8 @@ public class TestLensDAO { new Configuration(), 0); long submissionTime = queryContext.getSubmissionTime(); queryContext.setQueryName("daoTestQuery1"); + queryContext.getDriverContext().setSelectedDriver(new MockDriver()); + queryContext.getLensConf().addProperty("prop", "value"); LensDriver mockDriver = new MockDriver(); DriverSelectorQueryContext mockDriverContext = new DriverSelectorQueryContext(userQuery, @@ -127,6 +129,9 @@ public class TestLensDAO { FinishedLensQuery actual = service.lensServerDao.getQuery(finishedHandle); + Assert.assertEquals(finishedLensQuery, actual); + Assert.assertEquals(actual.getConf().getProperty("prop"), "value"); + // Try to read back result set metadata class, should not throw deserialize exception JDBCResultSet.JDBCResultSetMetadata actualRsMeta = MAPPER.readValue(actual.getMetadata(), JDBCResultSet.JDBCResultSetMetadata.class);