Repository: lens Updated Branches: refs/heads/lens-1381 4d493597e -> 2aaf6e0a0
http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java index 21e2a62..b480d14 100644 --- a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java +++ b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java @@ -276,17 +276,19 @@ public class HiveSessionService extends BaseLensService implements SessionServic */ @Override public void setSessionParameter(LensSessionHandle sessionid, String key, String value) { - setSessionParameter(sessionid, key, value, true); + HashMap<String, String> config = Maps.newHashMap(); + config.put(key, value); + setSessionParameters(sessionid, config); } + /** * Sets the session parameter. * * @param sessionid the sessionid * @param config map of string-string. each entry represents key and the value to be set for that key - * @param addToSession the add to session */ - protected void setSessionParameters(LensSessionHandle sessionid, Map<String, String> config, boolean addToSession) { + protected void setSessionParameters(LensSessionHandle sessionid, Map<String, String> config) { log.info("Request to Set params:" + config); try { acquire(sessionid); @@ -297,17 +299,11 @@ public class HiveSessionService extends BaseLensService implements SessionServic var = var.substring(SystemVariables.HIVECONF_PREFIX.length()); } getSession(sessionid).getSessionConf().set(var, entry.getValue()); - if (addToSession) { - String command = "set" + " " + entry.getKey() + "= " + entry.getValue(); - closeCliServiceOp(getCliService().executeStatement(getHiveSessionHandle(sessionid), command, null)); - } else { - getSession(sessionid).getHiveConf().set(entry.getKey(), entry.getValue()); - } + String command = "set" + " " + entry.getKey() + "= " + entry.getValue(); + closeCliServiceOp(getCliService().executeStatement(getHiveSessionHandle(sessionid), command, null)); } // add to persist - if (addToSession) { - getSession(sessionid).setConfig(config); - } + getSession(sessionid).setConfig(config); log.info("Set params:" + config); } catch (HiveSQLException e) { throw new WebApplicationException(e); @@ -315,18 +311,18 @@ public class HiveSessionService extends BaseLensService implements SessionServic release(sessionid); } } - /** - * Sets the session parameter. - * - * @param sessionid the sessionid - * @param key the key - * @param value the value - * @param addToSession the add to session - */ - protected void setSessionParameter(LensSessionHandle sessionid, String key, String value, boolean addToSession) { - HashMap<String, String> config = Maps.newHashMap(); - config.put(key, value); - setSessionParameters(sessionid, config, addToSession); + + private void setSessionParametersOnRestore(LensSessionHandle sessionid, Map<String, String> config) { + // set in session conf + for(Map.Entry<String, String> entry: config.entrySet()) { + String var = entry.getKey(); + if (var.indexOf(SystemVariables.HIVECONF_PREFIX) == 0) { + var = var.substring(SystemVariables.HIVECONF_PREFIX.length()); + } + getSession(sessionid).getSessionConf().set(var, entry.getValue()); + getSession(sessionid).getHiveConf().set(entry.getKey(), entry.getValue()); + } + log.info("Set params on restart:" + config); } /* @@ -367,7 +363,7 @@ public class HiveSessionService extends BaseLensService implements SessionServic LensSessionHandle sessionHandle = persistInfo.getSessionHandle(); restoreSession(sessionHandle, persistInfo.getUsername(), persistInfo.getPassword()); LensSessionImpl session = getSession(sessionHandle); - session.setLastAccessTime(persistInfo.getLastAccessTime()); + session.getLensSessionPersistInfo().setLastAccessTime(persistInfo.getLastAccessTime()); session.getLensSessionPersistInfo().setConfig(persistInfo.getConfig()); session.getLensSessionPersistInfo().setResources(persistInfo.getResources()); session.setCurrentDatabase(persistInfo.getDatabase()); @@ -384,7 +380,7 @@ public class HiveSessionService extends BaseLensService implements SessionServic // Add config for restored sessions try{ - setSessionParameters(sessionHandle, session.getConfig(), false); + setSessionParametersOnRestore(sessionHandle, session.getConfig()); } catch (Exception e) { log.error("Error setting parameters " + session.getConfig() + " for session: " + session, e); @@ -504,7 +500,7 @@ public class HiveSessionService extends BaseLensService implements SessionServic } } - Runnable getSessionExpiryRunnable() { + public Runnable getSessionExpiryRunnable() { return sessionExpiryRunnable; } @@ -517,7 +513,7 @@ public class HiveSessionService extends BaseLensService implements SessionServic * Run internal. */ public void runInternal() { - List<LensSessionHandle> sessionsToRemove = new ArrayList<LensSessionHandle>(SESSION_MAP.values()); + List<LensSessionHandle> sessionsToRemove = new ArrayList<>(SESSION_MAP.values()); Iterator<LensSessionHandle> itr = sessionsToRemove.iterator(); while (itr.hasNext()) { LensSessionHandle sessionHandle = itr.next(); @@ -527,10 +523,12 @@ public class HiveSessionService extends BaseLensService implements SessionServic itr.remove(); } } catch (ClientErrorException nfe) { + log.error("Error getting session " + sessionHandle.getPublicId(), nfe); itr.remove(); } } + log.info("Sessions to remove : {} out of {} all sessions", sessionsToRemove.size(), SESSION_MAP.size()); // Now close all inactive sessions for (LensSessionHandle sessionHandle : sessionsToRemove) { try { @@ -540,6 +538,7 @@ public class HiveSessionService extends BaseLensService implements SessionServic + new Date(lastAccessTime)); notifyEvent(new SessionExpired(System.currentTimeMillis(), sessionHandle)); } catch (ClientErrorException nfe) { + log.error("Error getting session " + sessionHandle.getPublicId(), nfe); // Do nothing } catch (LensException e) { log.error("Error closing session " + sessionHandle.getPublicId() + " reason " + e.getMessage(), e); @@ -555,9 +554,10 @@ public class HiveSessionService extends BaseLensService implements SessionServic @Override public void run() { try { + log.info("Running session expiry run"); runInternal(); } catch (Exception e) { - log.warn("Unknown error while checking for inactive sessions - " + e.getMessage()); + log.warn("Unknown error while checking for inactive sessions - ", e); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java index 34c901c..08a5cff 100644 --- a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java @@ -62,9 +62,6 @@ public class LensSessionImpl extends HiveSessionImpl implements AutoCloseable { /** The persist info. */ private LensSessionPersistInfo persistInfo = new LensSessionPersistInfo(); - /** The last access time. */ - private long lastAccessTime = System.currentTimeMillis(); - /** The session timeout. */ private long sessionTimeout; private static class IntegerThreadLocal extends ThreadLocal<Integer> { @@ -116,7 +113,7 @@ public class LensSessionImpl extends HiveSessionImpl implements AutoCloseable { getSessionHandle().getHandleIdentifier().getSecretId())); persistInfo.setUsername(getUserName()); persistInfo.setPassword(getPassword()); - persistInfo.setLastAccessTime(lastAccessTime); + persistInfo.setLastAccessTime(System.currentTimeMillis()); persistInfo.setSessionConf(sessionConf); if (sessionConf != null) { for (Map.Entry<String, String> entry : sessionConf.entrySet()) { @@ -280,12 +277,17 @@ public class LensSessionImpl extends HiveSessionImpl implements AutoCloseable { } public boolean isActive() { - return System.currentTimeMillis() - lastAccessTime < sessionTimeout - && (!persistInfo.markedForClose|| activeOperationsPresent()); + // session is active, if any active operations are present. + // If no active operations are present, session is active if timeout is not reached and session is not + // marked for close + return activeOperationsPresent() || ((System.currentTimeMillis() - persistInfo.lastAccessTime < sessionTimeout) + && !persistInfo.markedForClose); } + public boolean isMarkedForClose() { return persistInfo.isMarkedForClose(); } + public synchronized void setActive() { setLastAccessTime(System.currentTimeMillis()); } @@ -468,12 +470,12 @@ public class LensSessionImpl extends HiveSessionImpl implements AutoCloseable { return persistInfo; } - void setLastAccessTime(long lastAccessTime) { - this.lastAccessTime = lastAccessTime; + public void setLastAccessTime(long lastAccessTime) { + persistInfo.lastAccessTime = lastAccessTime; } public long getLastAccessTime() { - return lastAccessTime; + return persistInfo.lastAccessTime; } /** http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-server/src/main/resources/lensserver-default.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml index 261fa52..28b1db2 100644 --- a/lens-server/src/main/resources/lensserver-default.xml +++ b/lens-server/src/main/resources/lensserver-default.xml @@ -942,6 +942,17 @@ <value>org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost$Parser</value> <description>The Query cost parser class. Default query cost class used is FactPartitionBasedQueryCost</description> </property> + + <property> + <name>lens.server.query.comparator.classes</name> + <value>org.apache.lens.server.api.query.comparators.MoreRetriesFirstComparator,org.apache.lens.server.api.query.comparators.QueryPriorityComparator,org.apache.lens.server.api.query.comparators.FIFOQueryComparator</value> + <description>The Query cost comparator chain. Queries are compared in this order. To compare queries q1 and q2, + first number of retries are considered. The one with more retries is placed first in the queue. If those are same, + then their priorities are considered, with higher priorities coming before lower ones. If those are also same, + then their submission times are considered. The query that was submitted first is placed first. + </description> + </property> + <property> <name>lens.cube.metastore.enable.datacompleteness.check</name> <value>false</value> http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java index 0e6a4a1..f6f6e77 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java @@ -62,7 +62,6 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.glassfish.jersey.test.TestProperties; - import org.testng.Assert; import org.testng.annotations.*; @@ -974,7 +973,29 @@ public class TestMetastoreService extends LensJerseyTest { final String[] timePartColNames = {"dt"}; return createStorageTblElement(storageName, table, timePartColNames, updatePeriod); } + private XStorageTableElement createStorageTblWithMultipleTableDescriptors(String storageName, String[] tables, + String [] updatePeriods) { + String [][] timePartColNames = new String[updatePeriods.length][]; + for (int i = 0; i < updatePeriods.length; i++) { + timePartColNames[i] = new String[]{ "dt" }; + } + return createStorageTblWithMultipleTableDescriptors(storageName, tables, timePartColNames, updatePeriods); + } + private XStorageTableElement createStorageTblWithMultipleTableDescriptors(String storageName, String[] tables, + String[][] timePartColNames, String [] updatePeriods) { + XStorageTableElement tbl = cubeObjectFactory.createXStorageTableElement(); + tbl.setStorageName(storageName); + XUpdatePeriods xUpdatePeriods = new XUpdatePeriods(); + tbl.setUpdatePeriods(xUpdatePeriods); + for (int i = 0; i < updatePeriods.length; i++) { + XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor(); + updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(updatePeriods[i])); + updatePeriodTableDescriptor.setTableDesc(createStorageTableDesc(tables[i], timePartColNames[i])); + xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor); + } + return tbl; + } private XStorageTableElement createStorageTblElement(String storageName, String table, final String[] timePartColNames, String... updatePeriod) { XStorageTableElement tbl = cubeObjectFactory.createXStorageTableElement(); @@ -1836,6 +1857,169 @@ public class TestMetastoreService extends LensJerseyTest { } @Test(dataProvider = "mediaTypeData") + public void testCreateFactTableWithMultipleUpdatePeriods(MediaType mediaType) throws Exception { + + final String table = "testCreateFactTableWithMultipleUpdatePeriods"; + String prevDb = getCurrentDatabase(mediaType); + final String DB = dbPFX + "testCreateFactTableWithMultipleUpdatePeriods_DB" + mediaType.getSubtype(); + createDatabase(DB, mediaType); + setCurrentDatabase(DB, mediaType); + createStorage("S1", mediaType); + try { + final XCube cube = createTestCube("testCube"); + target().path("metastore").path("cubes").queryParam("sessionid", lensSessionId).request(mediaType) + .post(Entity.entity(new GenericEntity<JAXBElement<XCube>>(cubeObjectFactory.createXCube(cube)) { + }, mediaType), APIResult.class); + XFactTable f = createFactTable(table); + String[] tables = new String[] { "testTable1", "testTable2", "testTable3" }; + String[] updatePeriods = new String[] { "HOURLY", "DAILY", "MONTHLY" }; + f.getStorageTables().getStorageTable() + .add(createStorageTblWithMultipleTableDescriptors("S1", tables, updatePeriods)); + APIResult result = target().path("metastore").path("facts").queryParam("sessionid", lensSessionId) + .request(mediaType) + .post(Entity.entity(new GenericEntity<JAXBElement<XFactTable>>(cubeObjectFactory.createXFactTable(f)) { + }, mediaType), APIResult.class); + assertSuccess(result); + + StringList factNames = target().path("metastore/facts").queryParam("sessionid", lensSessionId).request(mediaType) + .get(StringList.class); + assertTrue(factNames.getElements().contains(table.toLowerCase())); + + // Get the created tables + JAXBElement<XFactTable> gotFactElement = target().path("metastore/facts").path(table) + .queryParam("sessionid", lensSessionId).request(mediaType).get(new GenericType<JAXBElement<XFactTable>>() { + }); + XFactTable gotFact = gotFactElement.getValue(); + assertTrue(gotFact.getName().equalsIgnoreCase(table)); + assertEquals(gotFact.getWeight(), 10.0); + + // Check for the created tables per update period. + List<XUpdatePeriodTableDescriptor> updatePeriodTableDescriptor = gotFact.getStorageTables().getStorageTable() + .get(0).getUpdatePeriods().getUpdatePeriodTableDescriptor(); + assertEquals(updatePeriodTableDescriptor.size(), 3); + + CubeFactTable cf = JAXBUtils.cubeFactFromFactTable(gotFact); + + Map<UpdatePeriod, String> updatePeriodTablePrefixMap = cf.getStoragePrefixUpdatePeriodMap().get("S1"); + for (Map.Entry entry : updatePeriodTablePrefixMap.entrySet()) { + assertEquals(entry.getValue(), entry.getKey() + "_S1"); + } + // Do some changes to test update + cf.alterWeight(20.0); + cf.alterColumn(new FieldSchema("c2", "double", "changed to double")); + + XFactTable update = JAXBUtils.factTableFromCubeFactTable(cf); + XStorageTableElement s1Tbl = createStorageTblWithMultipleTableDescriptors("S1", + new String[] { tables[0], tables[1] }, new String[] { updatePeriods[0], updatePeriods[1] }); + update.getStorageTables().getStorageTable().add(s1Tbl); + // Update + result = target().path("metastore").path("facts").path(table).queryParam("sessionid", lensSessionId) + .request(mediaType) + .put(Entity.entity(new GenericEntity<JAXBElement<XFactTable>>(cubeObjectFactory.createXFactTable(update)) { + }, mediaType), APIResult.class); + assertSuccess(result); + + // Get the updated table + gotFactElement = target().path("metastore/facts").path(table).queryParam("sessionid", lensSessionId) + .request(mediaType).get(new GenericType<JAXBElement<XFactTable>>() { + }); + gotFact = gotFactElement.getValue(); + CubeFactTable ucf = JAXBUtils.cubeFactFromFactTable(gotFact); + assertEquals(ucf.weight(), 20.0); + assertTrue(ucf.getUpdatePeriods().get("S1").contains(HOURLY)); + assertTrue(ucf.getUpdatePeriods().get("S1").contains(DAILY)); + assertFalse(ucf.getUpdatePeriods().get("S1").contains(MONTHLY)); + + // Add partitions + final Date partDate = new Date(); + XPartition xp = createPartition(table, partDate); + APIResult partAddResult = target().path("metastore/facts/").path(table).path("storages/S1/partition") + .queryParam("sessionid", lensSessionId).request(mediaType) + .post(Entity.entity(new GenericEntity<JAXBElement<XPartition>>(cubeObjectFactory.createXPartition(xp)) { + }, mediaType), APIResult.class); + assertSuccess(partAddResult); + + // add same should fail + partAddResult = target().path("metastore/facts/").path(table).path("storages/S1/partition") + .queryParam("sessionid", lensSessionId).request(mediaType) + .post(Entity.entity(new GenericEntity<JAXBElement<XPartition>>(cubeObjectFactory.createXPartition(xp)) { + }, mediaType), APIResult.class); + assertEquals(partAddResult.getStatus(), Status.FAILED); + + xp.setLocation(xp.getLocation() + "/a/b/c"); + APIResult partUpdateResult = target().path("metastore/facts/").path(table).path("storages/S1/partition") + .queryParam("sessionid", lensSessionId).request(mediaType) + .put(Entity.entity(new GenericEntity<JAXBElement<XPartition>>(cubeObjectFactory.createXPartition(xp)) { + }, mediaType), APIResult.class); + assertSuccess(partUpdateResult); + + JAXBElement<XPartitionList> partitionsElement = target().path("metastore/facts").path(table) + .path("storages/S1/partitions").queryParam("sessionid", lensSessionId).request(mediaType) + .get(new GenericType<JAXBElement<XPartitionList>>() { + }); + + XPartitionList partitions = partitionsElement.getValue(); + assertNotNull(partitions); + assertEquals(partitions.getPartition().size(), 1); + XPartition readPartition = partitions.getPartition().get(0); + assertEquals(readPartition.getLocation(), xp.getLocation()); + assertEquals(readPartition.getTimePartitionSpec(), xp.getTimePartitionSpec()); + assertEquals(readPartition.getNonTimePartitionSpec(), xp.getNonTimePartitionSpec()); + assertNotNull(readPartition.getFullPartitionSpec()); + XTimePartSpecElement timePartSpec = readPartition.getTimePartitionSpec().getPartSpecElement().iterator().next(); + XPartSpecElement fullPartSpec = readPartition.getFullPartitionSpec().getPartSpecElement().iterator().next(); + assertEquals(timePartSpec.getKey(), fullPartSpec.getKey()); + assertEquals( + UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).format(JAXBUtils.getDateFromXML(timePartSpec.getValue())), + fullPartSpec.getValue()); + DateTime date = target().path("metastore/cubes").path("testCube").path("latestdate") + .queryParam("timeDimension", "dt").queryParam("sessionid", lensSessionId).request(mediaType) + .get(DateTime.class); + + partDate.setMinutes(0); + partDate.setSeconds(0); + partDate.setTime(partDate.getTime() - partDate.getTime() % 1000); + assertEquals(date.getDate(), partDate); + // add two partitions, one of them already added. result should be partial + XPartitionList parts = new XPartitionList(); + parts.getPartition().add(xp); + parts.getPartition().add(createPartition(table, DateUtils.addHours(partDate, 1))); + partAddResult = target().path("metastore/facts/").path(table).path("storages/S1/partitions") + .queryParam("sessionid", lensSessionId).request(mediaType).post( + Entity.entity(new GenericEntity<JAXBElement<XPartitionList>>(cubeObjectFactory.createXPartitionList(parts)) { + }, mediaType), APIResult.class); + assertEquals(partAddResult.getStatus(), Status.PARTIAL); + + // Drop the partitions + APIResult dropResult = target().path("metastore/facts").path(table).path("storages/S1/partitions") + .queryParam("sessionid", lensSessionId).request(mediaType).delete(APIResult.class); + + assertSuccess(dropResult); + + // Verify partition was dropped + partitionsElement = target().path("metastore/facts").path(table).path("storages/S1/partitions") + .queryParam("sessionid", lensSessionId).request(mediaType).get(new GenericType<JAXBElement<XPartitionList>>() { + }); + + partitions = partitionsElement.getValue(); + assertNotNull(partitions); + assertEquals(partitions.getPartition().size(), 0); + // add null in batch + Response resp = target().path("metastore/facts/").path(table).path("storages/S1/partitions") + .queryParam("sessionid", lensSessionId).request(mediaType).post(null); + Assert.assertEquals(resp.getStatus(), 400); + + // Drop the cube + WebTarget target = target().path("metastore").path("cubes").path("testCube"); + result = target.queryParam("sessionid", lensSessionId).request(mediaType).delete(APIResult.class); + assertSuccess(result); + } finally { + setCurrentDatabase(prevDb, mediaType); + dropDatabase(DB, mediaType); + } + } + + @Test(dataProvider = "mediaTypeData") public void testCreateFactTable(MediaType mediaType) throws Exception { final String table = "testCreateFactTable"; final String DB = dbPFX + "testCreateFactTable_DB" + mediaType.getSubtype(); http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java index 8c1bb7b..017584c 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java @@ -22,10 +22,7 @@ import static org.apache.lens.server.api.LensConfConstants.*; import static org.testng.Assert.*; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import javax.ws.rs.core.Application; import javax.ws.rs.core.MediaType; @@ -33,6 +30,7 @@ import javax.ws.rs.core.Response; import org.apache.lens.api.LensConf; import org.apache.lens.api.LensSessionHandle; +import org.apache.lens.api.query.LensQuery; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.api.query.QueryStatus; import org.apache.lens.api.result.LensAPIResult; @@ -41,6 +39,7 @@ import org.apache.lens.driver.hive.HiveDriver; import org.apache.lens.server.LensJerseyTest; import org.apache.lens.server.LensServerTestUtil; import org.apache.lens.server.LensServices; +import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.LensServerAPITestUtil; import org.apache.lens.server.api.driver.LensDriver; import org.apache.lens.server.api.error.LensException; @@ -51,6 +50,9 @@ import org.apache.lens.server.common.RestAPITestUtil; import org.apache.lens.server.common.TestResourceFile; import org.apache.lens.server.error.LensServerErrorCode; import org.apache.lens.server.session.HiveSessionService; +import org.apache.lens.server.session.LensSessionImpl; + +import org.apache.hadoop.hive.conf.HiveConf; import org.glassfish.jersey.test.TestProperties; import org.testng.annotations.*; @@ -111,10 +113,6 @@ public class TestQueryIndependenceFromSessionClose extends LensJerseyTest { QUERY_PERSISTENT_RESULT_INDRIVER, true, QUERY_OUTPUT_FORMATTER, TestQueryService.DeferredPersistentResultFormatter.class.getName()); } - @AfterClass - public void restart() { - restartLensServer(); - } @Override public Map<String, String> getServerConfOverWrites() { @@ -150,10 +148,17 @@ public class TestQueryIndependenceFromSessionClose extends LensJerseyTest { private void customRestartLensServer() { queryService = null; - super.restartLensServer(getServerConf(), false); + super.restartLensServer(getServerConf()); getQueryService(); } + private void restartLensServerWithLowerExpiry() { + sessionService = null; + HiveConf hconf = new HiveConf(getServerConf()); + hconf.setLong(LensConfConstants.SESSION_TIMEOUT_SECONDS, 1L); + super.restartLensServer(hconf); + getSessionService(); + } /* * (non-Javadoc) * @@ -277,6 +282,54 @@ public class TestQueryIndependenceFromSessionClose extends LensJerseyTest { return sessions; } + @Test + public void testSessionExpiryWithActiveOperation() throws Exception { + LensSessionHandle oldSession = getSession(); + assertTrue(sessionService.getSession(oldSession).isActive()); + restartLensServerWithLowerExpiry(); + assertFalse(sessionService.getSession(oldSession).isActive()); + // create a new session and launch a query + LensSessionHandle sessionHandle = getSession(); + LensSessionImpl session = sessionService.getSession(sessionHandle); + QueryHandle handle = RestAPITestUtil.executeAndGetHandle(target(), + Optional.of(sessionHandle), Optional.of("select * from " + TEST_TABLE), Optional.of(conf), defaultMT); + assertTrue(session.isActive()); + session.setLastAccessTime( + session.getLastAccessTime() - 2000 * getServerConf().getLong(LensConfConstants.SESSION_TIMEOUT_SECONDS, + LensConfConstants.SESSION_TIMEOUT_SECONDS_DEFAULT)); + assertTrue(session.isActive()); + assertFalse(session.isMarkedForClose()); + + LensSessionHandle sessionHandle2 = getSession(); + LensQuery ctx = RestAPITestUtil.getLensQuery(target(), sessionHandle2, handle, defaultMT); + while (!ctx.getStatus().finished()) { + ctx = RestAPITestUtil.getLensQuery(target(), sessionHandle2, handle, defaultMT); + Thread.sleep(1000); + sessionHandle2 = getSession(); + } + assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL, String.valueOf(ctx)); + assertFalse(session.isActive()); + assertFalse(session.isMarkedForClose()); + + // run the expiry thread + sessionService.getSessionExpiryRunnable().run(); + try { + sessionService.getSession(sessionHandle); + // should throw exception since session should be expired by now + fail("Expected get session to fail for session " + sessionHandle.getPublicId()); + } catch (Exception e) { + // pass + } + try { + sessionService.getSession(oldSession); + // should throw exception since session should be expired by now + fail("Expected get session to fail for session " + oldSession.getPublicId()); + } catch (Exception e) { + // pass + } + restartLensServer(); + lensSessionId = getSession(); + } @AfterMethod private void waitForPurge() throws InterruptedException { waitForPurge(0, getQueryService().finishedQueries); http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 8ea64b7..12cd9cc 100644 --- a/pom.xml +++ b/pom.xml @@ -34,8 +34,8 @@ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <!-- java and javax --> - <java.source.version>1.7</java.source.version> - <java.target.version>1.7</java.target.version> + <java.source.version>1.8</java.source.version> + <java.target.version>1.8</java.target.version> <servlet3.version>3.0.1</servlet3.version> <servlet.api.version>2.5</servlet.api.version> <ws.rs.version>2.0.1</ws.rs.version> http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/src/site/apt/admin/config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt index eb35ae3..8720c82 100644 --- a/src/site/apt/admin/config.apt +++ b/src/site/apt/admin/config.apt @@ -163,132 +163,134 @@ Lens server configuration *--+--+---+--+ |68|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors are consulted first, before anything happens for the given query. They can either return null or return a messaging indicating why the given query shouldn't be accepted. These can be used to filter out queries at the earliest.| *--+--+---+--+ -|69|lens.server.query.cost.parser.class|org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost$Parser|The Query cost parser class. Default query cost class used is FactPartitionBasedQueryCost| +|69|lens.server.query.comparator.classes|org.apache.lens.server.api.query.comparators.MoreRetriesFirstComparator,org.apache.lens.server.api.query.comparators.QueryPriorityComparator,org.apache.lens.server.api.query.comparators.FIFOQueryComparator|The Query cost comparator chain. Queries are compared in this order. To compare queries q1 and q2, first number of retries are considered. The one with more retries is placed first in the queue. If those are same, then their priorities are considered, with higher priorities coming before lower ones. If those are also same, then their submission times are considered. The query that was submitted first is placed first.| *--+--+---+--+ -|70|lens.server.query.expiry.check.interval.millis|60000|The interval(milliseconds) with which query expiry will run periodically. Default is 1 minute. The value needs to be much lower than lens.query.timeout.millis. If the final deployment values of query timeout can be smaller, then reduce this value to be much lower.| +|70|lens.server.query.cost.parser.class|org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost$Parser|The Query cost parser class. Default query cost class used is FactPartitionBasedQueryCost| *--+--+---+--+ -|71|lens.server.query.launching.constraint.factories|org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory|Factories used to instantiate constraints enforced on queries by lens. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint. A query will be launched only if all constraints pass.| +|71|lens.server.query.expiry.check.interval.millis|60000|The interval(milliseconds) with which query expiry will run periodically. Default is 1 minute. The value needs to be much lower than lens.query.timeout.millis. If the final deployment values of query timeout can be smaller, then reduce this value to be much lower.| *--+--+---+--+ -|72|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ | +|72|lens.server.query.launching.constraint.factories|org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory|Factories used to instantiate constraints enforced on queries by lens. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint. A query will be launched only if all constraints pass.| +*--+--+---+--+ +|73|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ | | | | |Use cases will be to use extra intelligence to convert user query to optimized cube query. \ | | | | |Or define shortcuts for certain frequently used queries :) | *--+--+---+--+ -|73|lens.server.query.resultset.retention|1 day|Lens query resultset retention period. Default 1 day| +|74|lens.server.query.resultset.retention|1 day|Lens query resultset retention period. Default 1 day| *--+--+---+--+ -|74|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service| +|75|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service| *--+--+---+--+ -|75|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in logback xml for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger| +|76|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in logback xml for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger| *--+--+---+--+ -|76|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource| +|77|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource| *--+--+---+--+ -|77|lens.server.querypurger.sleep.interval|10000|The interval(milliseconds) with which purger to run periodically. Default 10 sec.| +|78|lens.server.querypurger.sleep.interval|10000|The interval(milliseconds) with which purger to run periodically. Default 10 sec.| *--+--+---+--+ -|78|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service| +|79|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service| *--+--+---+--+ -|79|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource| +|80|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource| *--+--+---+--+ -|80|lens.server.requestlogger.ws.filter.impl|org.apache.lens.server.LensRequestLoggingFilter|Implementation class for Request logging Filter| +|81|lens.server.requestlogger.ws.filter.impl|org.apache.lens.server.LensRequestLoggingFilter|Implementation class for Request logging Filter| *--+--+---+--+ -|81|lens.server.resultset.purge.enabled|false|Whether to purge the query results| +|82|lens.server.resultset.purge.enabled|false|Whether to purge the query results| *--+--+---+--+ -|82|lens.server.resultsetpurger.sleep.interval.secs|3600|Periodicity for Query result purger runs. Default 1 hour.| +|83|lens.server.resultsetpurger.sleep.interval.secs|3600|Periodicity for Query result purger runs. Default 1 hour.| *--+--+---+--+ -|83|lens.server.savedquery.jdbc.dialectclass|org.apache.lens.server.query.save.SavedQueryDao$HSQLDialect|Dialect of the target DB, Default is HSQL. Override with the target DB used.| +|84|lens.server.savedquery.jdbc.dialectclass|org.apache.lens.server.query.save.SavedQueryDao$HSQLDialect|Dialect of the target DB, Default is HSQL. Override with the target DB used.| *--+--+---+--+ -|84|lens.server.savedquery.list.default.count|20|Key denoting the default fetch value of saved query list api.| +|85|lens.server.savedquery.list.default.count|20|Key denoting the default fetch value of saved query list api.| *--+--+---+--+ -|85|lens.server.savedquery.list.default.offset|0|Key denoting the default start value of saved query list api.| +|86|lens.server.savedquery.list.default.offset|0|Key denoting the default start value of saved query list api.| *--+--+---+--+ -|86|lens.server.savedquery.service.impl|org.apache.lens.server.query.save.SavedQueryServiceImpl|Implementation class for saved query service| +|87|lens.server.savedquery.service.impl|org.apache.lens.server.query.save.SavedQueryServiceImpl|Implementation class for saved query service| *--+--+---+--+ -|87|lens.server.savedquery.ws.resource.impl|org.apache.lens.server.query.save.SavedQueryResource|Implementation class for Saved query Resource| +|88|lens.server.savedquery.ws.resource.impl|org.apache.lens.server.query.save.SavedQueryResource|Implementation class for Saved query Resource| *--+--+---+--+ -|88|lens.server.scheduler.instance.waiting.thread.interval.millis|300000|Thread interval for checking the waiting instances in milliseconds| +|89|lens.server.scheduler.instance.waiting.thread.interval.millis|300000|Thread interval for checking the waiting instances in milliseconds| *--+--+---+--+ -|89|lens.server.scheduler.max.job.per.user|-1|Maximum number of jobs that can be scheduled by a single user. If the number is less than zero, then there is no restriction on the number of jobs scheduled.| +|90|lens.server.scheduler.max.job.per.user|-1|Maximum number of jobs that can be scheduled by a single user. If the number is less than zero, then there is no restriction on the number of jobs scheduled.| *--+--+---+--+ -|90|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.SchedulerServiceImpl|Implementation class for query scheduler service| +|91|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.SchedulerServiceImpl|Implementation class for query scheduler service| *--+--+---+--+ -|91|lens.server.scheduler.store.class|org.apache.lens.server.scheduler.SchedulerDAO$SchedulerHsqlDBStore|A subclass of SchedulerDBStore class used for storing scheduler related information.| +|92|lens.server.scheduler.store.class|org.apache.lens.server.scheduler.SchedulerDAO$SchedulerHsqlDBStore|A subclass of SchedulerDBStore class used for storing scheduler related information.| *--+--+---+--+ -|92|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource| +|93|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource| *--+--+---+--+ -|93|lens.server.scheduling.queue.poll.interval.millisec|2000|The interval at which submission thread will poll scheduling queue to fetch the next query for submission. If value is less than equal to 0, then it would mean that thread will continuosly poll without sleeping. The interval has to be given in milliseconds.| +|94|lens.server.scheduling.queue.poll.interval.millisec|2000|The interval at which submission thread will poll scheduling queue to fetch the next query for submission. If value is less than equal to 0, then it would mean that thread will continuosly poll without sleeping. The interval has to be given in milliseconds.| *--+--+---+--+ -|94|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter| +|95|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter| *--+--+---+--+ -|95|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.| +|96|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.| *--+--+---+--+ -|96|lens.server.servicenames|session,alarm,query,savedquery,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up| +|97|lens.server.servicenames|session,alarm,query,savedquery,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up| *--+--+---+--+ -|97|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs| +|98|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs| *--+--+---+--+ -|98|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service| +|99|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service| *--+--+---+--+ -|99|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.| +|100|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.| *--+--+---+--+ -|100|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource| +|101|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource| *--+--+---+--+ -|101|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.| +|102|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.| *--+--+---+--+ -|102|lens.server.state.persistence.enabled|true|If flag is enabled, state of all the services will be persisted periodically to a location specified by lens.server.persist.location and on server restart all the services will be started from last saved state.| +|103|lens.server.state.persistence.enabled|true|If flag is enabled, state of all the services will be persisted periodically to a location specified by lens.server.persist.location and on server restart all the services will be started from last saved state.| *--+--+---+--+ -|103|lens.server.state.persistence.interval.millis|300000|Lens server state persistence time interval in milliseconds| +|104|lens.server.state.persistence.interval.millis|300000|Lens server state persistence time interval in milliseconds| *--+--+---+--+ -|104|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.| +|105|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.| *--+--+---+--+ -|105|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.| +|106|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.| *--+--+---+--+ -|106|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.| +|107|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.| *--+--+---+--+ -|107|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.| +|108|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.| *--+--+---+--+ -|108|lens.server.status.update.exponential.wait.millis|30000|Number of millis that would grow exponentially for next update, incase of transient failures.| +|109|lens.server.status.update.exponential.wait.millis|30000|Number of millis that would grow exponentially for next update, incase of transient failures.| *--+--+---+--+ -|109|lens.server.status.update.maximum.delay.secs|1800|The maximum delay in seconds for next status update to happen after any transient failure. This will be used a maximum delay sothat exponential wait times not to grow to bigger value.| +|110|lens.server.status.update.maximum.delay.secs|1800|The maximum delay in seconds for next status update to happen after any transient failure. This will be used a maximum delay sothat exponential wait times not to grow to bigger value.| *--+--+---+--+ -|110|lens.server.status.update.num.retries|10|The number of retries a status update will tried with exponentital back off, in case of transient issues, upon which query will be marked FAILED.| +|111|lens.server.status.update.num.retries|10|The number of retries a status update will tried with exponentital back off, in case of transient issues, upon which query will be marked FAILED.| *--+--+---+--+ -|111|lens.server.total.query.cost.ceiling.per.user|-1.0|A query submitted by user will be launched only if total query cost of all current launched queries of user is less than or equal to total query cost ceiling defined by this property. This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total query cost of launched queries submitted by a user.| +|112|lens.server.total.query.cost.ceiling.per.user|-1.0|A query submitted by user will be launched only if total query cost of all current launched queries of user is less than or equal to total query cost ceiling defined by this property. This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total query cost of launched queries submitted by a user.| *--+--+---+--+ -|112|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader| +|113|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader| *--+--+---+--+ -|113|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.| +|114|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.| *--+--+---+--+ -|114|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.| +|115|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.| *--+--+---+--+ -|115|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.| +|116|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.| *--+--+---+--+ -|116|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...| +|117|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...| *--+--+---+--+ -|117|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above| +|118|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above| *--+--+---+--+ -|118|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.| +|119|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.| *--+--+---+--+ -|119|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.| +|120|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.| *--+--+---+--+ -|120|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time| +|121|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time| *--+--+---+--+ -|121|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.| +|122|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.| *--+--+---+--+ -|122|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...| +|123|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...| *--+--+---+--+ -|123|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search| +|124|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search| *--+--+---+--+ -|124|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.| +|125|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.| *--+--+---+--+ -|125|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default| +|126|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default| *--+--+---+--+ -|126|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.| +|127|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.| *--+--+---+--+ -|127|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| +|128|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| *--+--+---+--+ -|128|lens.server.ws.featurenames|multipart,moxyjson,moxyjsonconfigresovler|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up| +|129|lens.server.ws.featurenames|multipart,moxyjson,moxyjsonconfigresovler|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up| *--+--+---+--+ -|129|lens.server.ws.filternames|requestlogger,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up| +|130|lens.server.ws.filternames|requestlogger,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up| *--+--+---+--+ -|130|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up| +|131|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up| *--+--+---+--+ -|131|lens.server.ws.resourcenames|session,metastore,query,savedquery,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up| +|132|lens.server.ws.resourcenames|session,metastore,query,savedquery,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up| *--+--+---+--+ The configuration parameters and their default values http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/tools/conf/server/logback.xml ---------------------------------------------------------------------- diff --git a/tools/conf/server/logback.xml b/tools/conf/server/logback.xml index 63ab23c..5173321 100644 --- a/tools/conf/server/logback.xml +++ b/tools/conf/server/logback.xml @@ -41,7 +41,7 @@ <maxHistory>30</maxHistory> </rollingPolicy> <encoder> - <pattern>%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c - %m%n</pattern> + <pattern>%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %L - %m%n</pattern> </encoder> </appender> @@ -107,7 +107,7 @@ <file>${lens.log.dir}/${queryLogId}.log</file> <append>true</append> <layout class="ch.qos.logback.classic.PatternLayout"> - <pattern>%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c - %m%n</pattern> + <pattern>%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %L - %m%n</pattern> </layout> </appender> </sift>
