Repository: lens Updated Branches: refs/heads/master 11885b981 -> e39dec5f8
http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java index da774fa..a803109 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java @@ -204,6 +204,21 @@ public interface QueryExecutionService extends LensService, SessionValidator { */ Response getHttpResultSet(LensSessionHandle sessionHandle, QueryHandle queryHandle) throws LensException; + + /** + * Get the secure http end point for the result set. + * + * @param sessionHandle The lens session handle + * @param queryHandle The query handle + * @param userPrincipalName the principal name + * @return returns javax.ws.rs.core.Response object + * @throws LensException the lens exception + */ + Response getAuthorizedHttpResultSet(LensSessionHandle sessionHandle, QueryHandle queryHandle, + String userPrincipalName) + throws LensException; + + /** * Closes result set by releasing any resources used in serving the resultset. * http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server-api/src/main/java/org/apache/lens/server/api/query/save/exception/PrivilegeException.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/save/exception/PrivilegeException.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/save/exception/PrivilegeException.java index 0d85b8c..427e735 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/save/exception/PrivilegeException.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/save/exception/PrivilegeException.java @@ -23,12 +23,16 @@ import static org.apache.lens.api.error.LensCommonErrorCode.NOT_AUTHORIZED; import org.apache.lens.server.api.LensErrorInfo; import org.apache.lens.server.api.error.LensException; +import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.ToString; /** * The class PrivilegeException. Thrown when the user is * not having the required privileges to complete the action. */ +@EqualsAndHashCode(callSuper = true) +@ToString public class PrivilegeException extends LensException { @Getter http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserGroupConfigLoader.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserGroupConfigLoader.java b/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserGroupConfigLoader.java new file mode 100644 index 0000000..a999e69 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserGroupConfigLoader.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api.user; + +import java.util.Map; + +public interface UserGroupConfigLoader { + + /** + * Gets the user groups config. + * + * @param loggedInUser the logged in user + * @return the user group config + * @throws UserGroupLoaderException the user group loader exception + */ + Map<String, String> getUserConfig(String loggedInUser) throws UserGroupLoaderException; +} http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserGroupLoaderException.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserGroupLoaderException.java b/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserGroupLoaderException.java new file mode 100644 index 0000000..54c47a8 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserGroupLoaderException.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api.user; + +public class UserGroupLoaderException extends RuntimeException { + + /** + * Instantiates a new user group loader exception. + */ + public UserGroupLoaderException() { + super(); + } + + /** + * Instantiates a new user group loader exception. + * + * @param s the s + */ + public UserGroupLoaderException(String s) { + super(s); + } + + /** + * Instantiates a new user group loader exception. + * + * @param e the e + */ + public UserGroupLoaderException(Throwable e) { + super(e); + } + + /** + * Instantiates a new user grouploader exception. + * + * @param message the message + * @param cause the cause + */ + public UserGroupLoaderException(String message, Throwable cause) { + super(message, cause); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java b/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java index ebb3a95..9364872 100644 --- a/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java +++ b/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java @@ -20,6 +20,7 @@ package org.apache.lens.server; import static org.apache.lens.server.error.LensServerErrorCode.SESSION_CLOSED; import static org.apache.lens.server.error.LensServerErrorCode.SESSION_ID_NOT_PROVIDED; +import static org.apache.lens.server.error.LensServerErrorCode.SESSION_UNAUTHORIZED; import java.io.*; import java.util.ArrayList; @@ -52,6 +53,7 @@ import org.apache.lens.server.error.LensServerErrorCode; import org.apache.lens.server.query.QueryExecutionServiceImpl; import org.apache.lens.server.session.LensSessionImpl; import org.apache.lens.server.user.UserConfigLoaderFactory; +import org.apache.lens.server.user.usergroup.UserGroupLoaderFactory; import org.apache.lens.server.util.UtilityMethods; import org.apache.commons.lang3.StringUtils; @@ -207,6 +209,9 @@ public abstract class BaseLensService extends CompositeService implements Extern log.info("Got user config: {}", userConfig); UtilityMethods.mergeMaps(sessionConf, userConfig, false); sessionConf.put(LensConfConstants.SESSION_LOGGEDIN_USER, username); + + Map<String, String> userGroupConfig = UserGroupLoaderFactory.getUserGroupConfig(username); + UtilityMethods.mergeMaps(sessionConf, userGroupConfig, false); if (sessionConf.get(LensConfConstants.SESSION_CLUSTER_USER) == null) { log.info("Didn't get cluster user from user config loader. Setting same as logged in user: {}", username); sessionConf.put(LensConfConstants.SESSION_CLUSTER_USER, username); @@ -226,11 +231,18 @@ public abstract class BaseLensService extends CompositeService implements Extern } private void updateSessionsPerUser(String userName) { - Integer numOfSessions = SESSIONS_PER_USER.get(userName); - if (null == numOfSessions) { - SESSIONS_PER_USER.put(userName, 1); - } else { - SESSIONS_PER_USER.put(userName, ++numOfSessions); + SessionUser sessionUser = SESSION_USER_INSTANCE_MAP.get(userName); + if (sessionUser == null) { + log.info("Trying to update invalid session {} for user {}", userName); + return; + } + synchronized (sessionUser) { + Integer numOfSessions = SESSIONS_PER_USER.get(userName); + if (null == numOfSessions) { + SESSIONS_PER_USER.put(userName, 1); + } else { + SESSIONS_PER_USER.put(userName, ++numOfSessions); + } } } @@ -572,6 +584,16 @@ public abstract class BaseLensService extends CompositeService implements Extern } } + + @Override + public void validateAndAuthorizeSession(LensSessionHandle handle, String userPrincipalName) throws LensException { + validateSession(handle); + LensSessionImpl session = getSession(handle); + if (!session.getLoggedInUser().equals(userPrincipalName)) { + throw new LensException(SESSION_UNAUTHORIZED.getLensErrorInfo(), handle); + } + } + public class SessionContext implements AutoCloseable { private LensSessionHandle sessionHandle; http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server/src/main/java/org/apache/lens/server/LensServices.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java index 5f4a699..60dc561 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java @@ -42,6 +42,7 @@ import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext; import org.apache.lens.server.session.LensSessionImpl; import org.apache.lens.server.stats.StatisticsService; import org.apache.lens.server.user.UserConfigLoaderFactory; +import org.apache.lens.server.user.usergroup.UserGroupLoaderFactory; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.concurrent.BasicThreadFactory; @@ -212,6 +213,7 @@ public class LensServices extends CompositeService implements ServiceProvider { ServiceMode.valueOf(DEFAULT_SERVER_MODE)); cliService = new CLIService(null); UserConfigLoaderFactory.init(conf); + UserGroupLoaderFactory.init(conf); // Add default services addService(cliService); addService(new EventServiceImpl(LensEventService.NAME)); http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server/src/main/java/org/apache/lens/server/error/LensServerErrorCode.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/error/LensServerErrorCode.java b/lens-server/src/main/java/org/apache/lens/server/error/LensServerErrorCode.java index b9150e9..ef43371 100644 --- a/lens-server/src/main/java/org/apache/lens/server/error/LensServerErrorCode.java +++ b/lens-server/src/main/java/org/apache/lens/server/error/LensServerErrorCode.java @@ -29,7 +29,8 @@ public enum LensServerErrorCode { SESSION_CLOSED(2005, 0), INVALID_HANDLE(2006, 0), NULL_OR_EMPTY_ARGUMENT(2007, 0), - SERVER_OVERLOADED(2008, 0); + SERVER_OVERLOADED(2008, 0), + SESSION_UNAUTHORIZED(2009, 0); public LensErrorInfo getLensErrorInfo() { return this.errorInfo; http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java b/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java index 443ad9d..0352151 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java @@ -69,8 +69,7 @@ public class LensPersistentResult extends PersistentResultSet { * @param conf the lens server conf */ public LensPersistentResult(QueryHandle queryHandle, LensResultSetMetadata metadata, String outputPath, Integer - numRows, Long fileSize, - Configuration conf, LensConf qconf) { + numRows, Long fileSize, Configuration conf, LensConf qconf) { this.metadata = metadata; this.outputPath = outputPath; this.numRows = numRows; http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 4023a24..07a2107 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -66,6 +66,7 @@ import org.apache.lens.server.api.query.comparators.*; import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint; import org.apache.lens.server.api.query.cost.QueryCost; import org.apache.lens.server.api.query.events.*; +import org.apache.lens.server.api.query.save.exception.PrivilegeException; import org.apache.lens.server.api.retry.BackOffRetryHandler; import org.apache.lens.server.api.retry.ChainedRetryPolicyDecider; import org.apache.lens.server.api.retry.OperationRetryHandlerFactory; @@ -3298,6 +3299,31 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE ? new HealthStatus(isHealthy, "QueryExecution service is healthy.") : new HealthStatus(isHealthy, details.toString()); } + /* + * (non-Javadoc) + * + * @see org.apache.lens.server.api.query.QueryExecutionService#getHttpResultSet(org.apache.lens.api.LensSessionHandle, + * org.apache.lens.api.query.QueryHandle) + */ + + @Override + public Response getAuthorizedHttpResultSet(LensSessionHandle sessionHandle, QueryHandle queryHandle, + String userPrincipalName) throws LensException { + + String loggedInUser; + if (sessionHandle != null) { + //@TODO this check can be introduced in other api calls as well if required + validateAndAuthorizeSession(sessionHandle, userPrincipalName); + loggedInUser = getSession(sessionHandle).getLoggedInUser(); + } else { + loggedInUser = userPrincipalName; + } + final QueryContext ctx = getUpdatedQueryContext(sessionHandle, queryHandle); + if (!loggedInUser.equals(ctx.getSubmittedUser())) { + throw new PrivilegeException("Query", queryHandle.toString(), "download"); + } + return getResponse(sessionHandle, queryHandle, ctx); + } /* * (non-Javadoc) * @@ -3307,10 +3333,18 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE @Override public Response getHttpResultSet(LensSessionHandle sessionHandle, QueryHandle queryHandle) throws LensException { + final QueryContext ctx = getUpdatedQueryContext(sessionHandle, queryHandle); + return getResponse(sessionHandle, queryHandle, ctx); + } + + private Response getResponse(LensSessionHandle sessionHandle, final QueryHandle queryHandle, final QueryContext ctx) + throws LensException { + LensResultSet resultSet = getResultset(queryHandle); if (!resultSet.isHttpResultAvailable()) { throw new NotFoundException("http result not available"); } + final Path resultPath = new Path(resultSet.getOutputPath()); try { FileSystem fs = resultPath.getFileSystem(conf); @@ -3320,7 +3354,6 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE } catch (IOException e) { throw new LensException(e); } - final QueryContext ctx = getUpdatedQueryContext(sessionHandle, queryHandle); String resultFSReadUrl = conf.get(RESULT_FS_READ_URL); if (resultFSReadUrl != null) { try { http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java index d00087a..47b40a8 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java @@ -19,22 +19,30 @@ package org.apache.lens.server.query; import static org.apache.lens.api.query.SubmitOp.*; +import static org.apache.lens.server.api.LensConfConstants.ENABLE_RESULT_DOWNLOAD_AUTHORIZATION_CHECK; import static org.apache.lens.server.error.LensServerErrorCode.INVALID_HANDLE; import static org.apache.lens.server.error.LensServerErrorCode.NULL_OR_EMPTY_OR_BLANK_QUERY; +import java.security.Principal; import java.util.List; +import java.util.Optional; import javax.ws.rs.*; +import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; import org.apache.lens.api.APIResult; import org.apache.lens.api.APIResult.Status; import org.apache.lens.api.LensConf; import org.apache.lens.api.LensSessionHandle; +import org.apache.lens.api.auth.AuthScheme; import org.apache.lens.api.query.*; import org.apache.lens.api.result.LensAPIResult; +import org.apache.lens.server.LensServerConf; import org.apache.lens.server.LensServices; +import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.annotations.MultiPurposeResource; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.query.QueryExecutionService; @@ -45,6 +53,7 @@ import org.apache.lens.server.model.LogSegregationContext; import org.apache.lens.server.util.UtilityMethods; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; import org.glassfish.jersey.media.multipart.FormDataParam; @@ -65,6 +74,15 @@ public class QueryServiceResource { private final LogSegregationContext logSegregationContext; + @Context + private SecurityContext securityContext; + + + public static final Configuration CONF = LensServerConf.getHiveConf(); + public static final Optional<AuthScheme> AUTH_SCHEME = + AuthScheme.getFromString(CONF.get(LensConfConstants.AUTH_SCHEME)); + + private void validateSessionId(final LensSessionHandle sessionHandle) throws LensException { queryServer.validateSession(sessionHandle); } @@ -633,7 +651,17 @@ public class QueryServiceResource { @Produces({MediaType.APPLICATION_OCTET_STREAM}) public Response getHttpResultSet(@QueryParam("sessionid") LensSessionHandle sessionid, @PathParam("queryHandle") String queryHandle) throws LensException { + + if (AUTH_SCHEME.isPresent()) { + Principal userPrincipal = securityContext.getUserPrincipal(); + String userPrincipalName = userPrincipal.getName(); + if (CONF.getBoolean(ENABLE_RESULT_DOWNLOAD_AUTHORIZATION_CHECK, + LensConfConstants.DEFAULT_ENABLE_RESULT_DOWNLOAD_AUTHORIZATION_CHECK)) { + return queryServer.getAuthorizedHttpResultSet(sessionid, getQueryHandle(queryHandle), userPrincipalName); + } + } return queryServer.getHttpResultSet(sessionid, getQueryHandle(queryHandle)); + } /** http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server/src/main/java/org/apache/lens/server/user/usergroup/FixedUserGroupConfigLoader.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/user/usergroup/FixedUserGroupConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/usergroup/FixedUserGroupConfigLoader.java new file mode 100644 index 0000000..46c21e1 --- /dev/null +++ b/lens-server/src/main/java/org/apache/lens/server/user/usergroup/FixedUserGroupConfigLoader.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.user.usergroup; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.user.UserGroupConfigLoader; + +import org.apache.hadoop.hive.conf.HiveConf; + +import com.google.common.collect.Maps; + +public class FixedUserGroupConfigLoader implements UserGroupConfigLoader { + + private final String fixedValue; + + public FixedUserGroupConfigLoader(HiveConf conf) { + fixedValue = conf.get(LensConfConstants.USER_GROUP_FIXED_VALUE); + } + + /* + * (non-Javadoc) + * + * @see org.apache.lens.server.user.UserGroupConfigLoader#getUserConfig(java.lang.String) + */ + @Override + public Map<String, String> getUserConfig(String loggedInUser) { + HashMap<String, String> userConfig = Maps.newHashMap(); + userConfig.put(LensConfConstants.SESSION_USER_GROUPS, fixedValue); + return userConfig; + } +} + http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server/src/main/java/org/apache/lens/server/user/usergroup/UserGroupLoaderFactory.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/user/usergroup/UserGroupLoaderFactory.java b/lens-server/src/main/java/org/apache/lens/server/user/usergroup/UserGroupLoaderFactory.java new file mode 100644 index 0000000..f4024a4 --- /dev/null +++ b/lens-server/src/main/java/org/apache/lens/server/user/usergroup/UserGroupLoaderFactory.java @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.user.usergroup; + +import static org.apache.lens.server.api.LensConfConstants.USER_GROUP_CUSTOM_CLASS; + +import java.lang.reflect.InvocationTargetException; +import java.util.Map; + +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.user.UserGroupConfigLoader; +import org.apache.lens.server.api.user.UserGroupLoaderException; + +import org.apache.hadoop.hive.conf.HiveConf; + +import lombok.extern.slf4j.Slf4j; + +/** + * A factory for creating UserGroupConfigLoader objects. + */ +@Slf4j +public final class UserGroupLoaderFactory { + private UserGroupLoaderFactory() { + + } + + /** The conf. */ + private static HiveConf conf; + + /** The user group config loader. */ + private static UserGroupConfigLoader userGroupConfigLoader; + + /** + * Inits the. + * + * @param c the c + */ + public static void init(HiveConf c) { + conf = c; + userGroupConfigLoader = null; + } + + /** + * The Enum GroupType. + */ + + public enum GroupType { + + /** The custom. */ + CUSTOM, + + /** The fixed. */ + FIXED + } + + public static UserGroupConfigLoader getUserGroupConfigLoader() { + if (userGroupConfigLoader == null) { + userGroupConfigLoader = initializeUserGroupConfigLoader(); + } + return userGroupConfigLoader; + } + + /** + * Initialize user config loader. + * + * @return the user config loader + */ + public static UserGroupConfigLoader initializeUserGroupConfigLoader() { + String groupType = conf.get(LensConfConstants.USER_GROUP_TYPE); + if (groupType == null || groupType.length() == 0) { + throw new UserGroupLoaderException("user group type not determined. value was not provided in conf"); + } + for (GroupType type : GroupType.values()) { + if (type.name().equals(groupType)) { + return createUserGroupConfigLoader(type); + } + } + throw new UserGroupLoaderException("user resolver type not determined. provided value: " + groupType); + } + + /** + * Gets the query user resolver. + * + * @param groupType the resolver type + * @return the query user resolver + */ + public static UserGroupConfigLoader createUserGroupConfigLoader(GroupType groupType) { + switch (groupType) { + case CUSTOM: + try { + return (conf.getClass(USER_GROUP_CUSTOM_CLASS, UserGroupConfigLoader.class, UserGroupConfigLoader.class)) + .getConstructor(HiveConf.class).newInstance(conf); + } catch (InvocationTargetException | NoSuchMethodException | IllegalAccessException | InstantiationException e) { + throw new UserGroupLoaderException(e); + } + case FIXED: + default: + return new FixedUserGroupConfigLoader(conf); + } + } + + /** + * Gets the user group grconfig. + * + * @param loggedInUser the logged in user + * @return the user config + */ + public static Map<String, String> getUserGroupConfig(String loggedInUser) { + try { + Map<String, String> config = getUserGroupConfigLoader().getUserConfig(loggedInUser); + if (config == null) { + throw new UserGroupLoaderException("Got null User Group config for: " + loggedInUser); + } + return config; + } catch (RuntimeException e) { + log.error("Couldn't get user Group config for user: " + loggedInUser, e); + throw e; + } + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server/src/main/resources/lensserver-default.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml index dd81f62..2ea73a3 100644 --- a/lens-server/src/main/resources/lensserver-default.xml +++ b/lens-server/src/main/resources/lensserver-default.xml @@ -593,6 +593,13 @@ </property> <property> + <name>lens.server.user.group.type</name> + <value>FIXED</value> + <description>Type of user group config resolver. allowed values are FIXED, CUSTOM. + </description> + </property> + + <property> <name>lens.server.user.resolver.fixed.value</name> <value></value> <description>Required for FIXED user resolver. @@ -600,6 +607,15 @@ </description> </property> + + <property> + <name>lens.server.user.group.fixed.value</name> + <value>test</value> + <description>Required for FIXED user group resolver. + when lens.server.user.group.type=FIXED, This will be the value user groups will resolve to. + </description> + </property> + <property> <name>lens.server.user.resolver.propertybased.filename</name> <value>/path/to/propertyfile</value> http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server/src/main/resources/ranger-lens-audit.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/main/resources/ranger-lens-audit.xml b/lens-server/src/main/resources/ranger-lens-audit.xml new file mode 100644 index 0000000..31d91d7 --- /dev/null +++ b/lens-server/src/main/resources/ranger-lens-audit.xml @@ -0,0 +1,20 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> +</configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server/src/main/resources/ranger-lens-security.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/main/resources/ranger-lens-security.xml b/lens-server/src/main/resources/ranger-lens-security.xml new file mode 100644 index 0000000..7274c44 --- /dev/null +++ b/lens-server/src/main/resources/ranger-lens-security.xml @@ -0,0 +1,85 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + + <property> + <name>ranger.plugin.lens.service.name</name> + <value>lensservice</value> + <description> + Name of the Ranger service containing policies for this YARN instance + </description> + </property> + + <property> + <name>ranger.plugin.lens.policy.source.impl</name> + <value>org.apache.ranger.admin.client.RangerAdminRESTClient</value> + <description> + Class to retrieve policies from the source + </description> + </property> + + <property> + <name>ranger.plugin.lens.policy.rest.url</name> + <value>http://localhost:6080</value> + <description> + URL to Ranger Admin + </description> + </property> + + <property> + <name>ranger.plugin.lens.policy.rest.ssl.config.file</name> + <value>/tmp/ranger-policymgr-ssl.xml</value> + <description> + Path to the file containing SSL details to contact Ranger Admin + </description> + </property> + + <property> + <name>ranger.plugin.lens.policy.pollIntervalMs</name> + <value>30000</value> + <description> + How often to poll for changes in policies? + </description> + </property> + + <property> + <name>ranger.plugin.lens.policy.cache.dir</name> + <value>/tmp/policycache</value> + <description> + Directory where Ranger policies are cached after successful retrieval from the source + </description> + </property> + + <property> + <name>ranger.plugin.lens.policy.rest.client.connection.timeoutMs</name> + <value>120000</value> + <description> + Hdfs Plugin RangerRestClient Connection Timeout in Milli Seconds + </description> + </property> + + <property> + <name>ranger.plugin.lens.policy.rest.client.read.timeoutMs</name> + <value>30000</value> + <description> + Hdfs Plugin RangerRestClient read Timeout in Milli Seconds + </description> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server/src/main/resources/ranger-policymgr-ssl.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/main/resources/ranger-policymgr-ssl.xml b/lens-server/src/main/resources/ranger-policymgr-ssl.xml new file mode 100644 index 0000000..31d91d7 --- /dev/null +++ b/lens-server/src/main/resources/ranger-policymgr-ssl.xml @@ -0,0 +1,20 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> +</configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 1b1831e..562e06e 100644 --- a/pom.xml +++ b/pom.xml @@ -75,6 +75,9 @@ <spring.shell.version>1.1.0.RELEASE</spring.shell.version> <javax.api.version>1.3</javax.api.version> + <!--Apache Ranger version--> + <ranger.version>1.0.0</ranger.version> + <c3p0.version>0.9.5</c3p0.version> <hsqldb.version>2.2.9</hsqldb.version> <dbcp.version>1.4</dbcp.version> @@ -1302,6 +1305,101 @@ </exclusion> </exclusions> </dependency> + + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + <version>1.19.4</version> + <exclusions> + <exclusion> + <groupId>javax.ws.rs</groupId> + <artifactId>jsr311-api</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-bundle</artifactId> + <version>1.19.3</version> + <exclusions> + <exclusion> + <groupId>javax.ws.rs</groupId> + <artifactId>jsr311-api</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger-plugins-common</artifactId> + <version>${ranger.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger-plugins-audit</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-bundle</artifactId> + </exclusion> + </exclusions> + + </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger-plugins-audit</artifactId> + <version>${ranger.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-bundle</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.persistence</groupId> + <artifactId>javax.persistence</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.persistence</groupId> + <artifactId>eclipselink</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger-plugins-cred</artifactId> + <version>${ranger.version}</version> + <exclusions> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-bundle</artifactId> + </exclusion> + <exclusion> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-jaxrs</artifactId> + </exclusion> + <exclusion> + <groupId>javax.ws.rs</groupId> + <artifactId>javax.ws.rs-api</artifactId> + </exclusion> + <exclusion> + <groupId>javax.ws.rs</groupId> + <artifactId>jsr311-api</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-exec</artifactId> @@ -1439,6 +1537,13 @@ <artifactId>jersey-container-grizzly2-servlet</artifactId> <version>${jersey.version}</version> </dependency> + + <dependency> + <groupId>org.glassfish.jersey.containers</groupId> + <artifactId>jersey-container-servlet-core</artifactId> + <version>${jersey.version}</version> + </dependency> + <dependency> <groupId>org.glassfish.jersey.media</groupId> <artifactId>jersey-media-jaxb</artifactId> http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/src/site/apt/admin/config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt index b3dff28..e900f98 100644 --- a/src/site/apt/admin/config.apt +++ b/src/site/apt/admin/config.apt @@ -263,44 +263,48 @@ Lens server configuration *--+--+---+--+ |117|lens.server.total.query.cost.ceiling.per.user|-1.0|A query submitted by user will be launched only if total query cost of all current launched queries of user is less than or equal to total query cost ceiling defined by this property. This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total query cost of launched queries submitted by a user.| *--+--+---+--+ -|118|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader| +|118|lens.server.user.group.fixed.value|test|Required for FIXED user group resolver. when lens.server.user.group.type=FIXED, This will be the value user groups will resolve to.| *--+--+---+--+ -|119|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.| +|119|lens.server.user.group.type|FIXED|Type of user group config resolver. allowed values are FIXED, CUSTOM.| *--+--+---+--+ -|120|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.| +|120|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader| *--+--+---+--+ -|121|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.| +|121|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.| *--+--+---+--+ -|122|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...| +|122|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.| *--+--+---+--+ -|123|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above| +|123|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.| *--+--+---+--+ -|124|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.| +|124|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...| *--+--+---+--+ -|125|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.| +|125|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above| *--+--+---+--+ -|126|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time| +|126|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.| *--+--+---+--+ -|127|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.| +|127|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.| *--+--+---+--+ -|128|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...| +|128|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time| *--+--+---+--+ -|129|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search| +|129|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.| *--+--+---+--+ -|130|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.| +|130|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...| *--+--+---+--+ -|131|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default| +|131|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search| *--+--+---+--+ -|132|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.| +|132|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.| *--+--+---+--+ -|133|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| +|133|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default| *--+--+---+--+ -|134|lens.server.ws.featurenames|multipart,moxyjson,moxyjsonconfigresovler|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up| +|134|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.| *--+--+---+--+ -|135|lens.server.ws.filternames|requestlogger,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up| +|135|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| *--+--+---+--+ -|136|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up| +|136|lens.server.ws.featurenames|multipart,moxyjson,moxyjsonconfigresovler|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up| *--+--+---+--+ -|137|lens.server.ws.resourcenames|session,metastore,query,savedquery,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up| +|137|lens.server.ws.filternames|requestlogger,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up| +*--+--+---+--+ +|138|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up| +*--+--+---+--+ +|139|lens.server.ws.resourcenames|session,metastore,query,savedquery,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up| *--+--+---+--+ The configuration parameters and their default values http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/src/site/apt/admin/hivedriver-config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/hivedriver-config.apt b/src/site/apt/admin/hivedriver-config.apt index 5d99892..1532a27 100644 --- a/src/site/apt/admin/hivedriver-config.apt +++ b/src/site/apt/admin/hivedriver-config.apt @@ -42,21 +42,25 @@ Hive driver configuration *--+--+---+--+ |9|lens.cube.query.replace.timedim|true|Tells whether timedim attribute queried in the time range should be replaced with its corresponding partition column name.| *--+--+---+--+ -|10|lens.driver.cost.query.decider.class|org.apache.lens.server.api.query.cost.RangeBasedQueryCostTypeDecider|Decider class which looks at ranges passed in config and decides the querycosttype| +|10|lens.cube.query.user.groups.authorization.enable|false|true if authorization is based on User Groups, false otherwise| *--+--+---+--+ -|11|lens.driver.cost.type.ranges|LOW,0.0,HIGH|Cost based Query type mapping| +|11|lens.cube.query.user.name.authorization.enable|false|true if authorization is based on User Name, false otherwise| *--+--+---+--+ -|12|lens.driver.hive.calculate.priority|true|Whether priority should be calculated for hive mr jobs or not| +|12|lens.driver.cost.query.decider.class|org.apache.lens.server.api.query.cost.RangeBasedQueryCostTypeDecider|Decider class which looks at ranges passed in config and decides the querycosttype| *--+--+---+--+ -|13|lens.driver.hive.connection.class|org.apache.lens.driver.hive.EmbeddedThriftConnection|The connection class from HiveDriver to HiveServer. The default is an embedded connection which does not require a remote hive server. For connecting to a hiveserver end point, remote connection should be used. The possible values are org.apache.lens.driver.hive.EmbeddedThriftConnection and org.apache.lens.driver.hive.RemoteThriftConnection.| +|13|lens.driver.cost.type.ranges|LOW,0.0,HIGH|Cost based Query type mapping| *--+--+---+--+ -|14|lens.driver.hive.cost.calculator.class|org.apache.lens.cube.query.cost.FactPartitionBasedQueryCostCalculator|Cost calculator class. By default calculating cost through fact partitions.| +|14|lens.driver.hive.calculate.priority|true|Whether priority should be calculated for hive mr jobs or not| *--+--+---+--+ -|15|lens.driver.hive.hs2.connection.expiry.delay|600000|The idle time (in milliseconds) for expiring connection from hivedriver to HiveServer2| +|15|lens.driver.hive.connection.class|org.apache.lens.driver.hive.EmbeddedThriftConnection|The connection class from HiveDriver to HiveServer. The default is an embedded connection which does not require a remote hive server. For connecting to a hiveserver end point, remote connection should be used. The possible values are org.apache.lens.driver.hive.EmbeddedThriftConnection and org.apache.lens.driver.hive.RemoteThriftConnection.| *--+--+---+--+ -|16|lens.driver.hive.kerberos.principal|hive/_h...@apache.com|Set principal name to be used for hive server.| +|16|lens.driver.hive.cost.calculator.class|org.apache.lens.cube.query.cost.FactPartitionBasedQueryCostCalculator|Cost calculator class. By default calculating cost through fact partitions.| *--+--+---+--+ -|17|lens.driver.hive.priority.ranges|VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW|Priority Ranges. The numbers are the costs of the query. \ | +|17|lens.driver.hive.hs2.connection.expiry.delay|600000|The idle time (in milliseconds) for expiring connection from hivedriver to HiveServer2| +*--+--+---+--+ +|18|lens.driver.hive.kerberos.principal|hive/_h...@apache.com|Set principal name to be used for hive server.| +*--+--+---+--+ +|19|lens.driver.hive.priority.ranges|VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW|Priority Ranges. The numbers are the costs of the query. \ | | | | |The cost is calculated based on partition weights and fact weights. The interpretation of the default config is: \ | | | | | \ | | | | |cost \<= 7\ \ \ \ \ \ \ \ \ \ \ :\ \ \ \ \ Priority = VERY_HIGH \ | @@ -72,14 +76,14 @@ Hive driver configuration | | | |One use case in range tuning can be that you never want queries to run with VERY_HIGH, assuming no other changes, you'll modify the value of this param in hivedriver-site.xml to be HIGH,30.0,NORMAL,90,LOW\ | | | | |via the configs, you can tune both the ranges and partition weights. this would give the end user more control. | *--+--+---+--+ -|18|lens.driver.hive.query.hook.classes| |The query hook classes for hive driver. By default there are no hooks. To add a hook, you should look at the default implementation and from there it'll be easy to derive what value can be added through a new hook. Multiple hooks can be provided by providing comma seperated name of classes.| +|20|lens.driver.hive.query.hook.classes| |The query hook classes for hive driver. By default there are no hooks. To add a hook, you should look at the default implementation and from there it'll be easy to derive what value can be added through a new hook. Multiple hooks can be provided by providing comma seperated name of classes.| *--+--+---+--+ -|19|lens.driver.hive.query.launching.constraint.factories| |Factories used to instantiate constraints enforced on queries by driver. A query will be launched only if all constraints pass. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint.| +|21|lens.driver.hive.query.launching.constraint.factories| |Factories used to instantiate constraints enforced on queries by driver. A query will be launched only if all constraints pass. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint.| *--+--+---+--+ -|20|lens.driver.hive.waiting.queries.selection.policy.factories| |Factories used to instantiate driver specific waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| +|22|lens.driver.hive.waiting.queries.selection.policy.factories| |Factories used to instantiate driver specific waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| *--+--+---+--+ -|21|query.retry.policy.classes|org.apache.lens.server.api.retry.SubstringMessagePolicyDecider|List of policy decider classes| +|23|query.retry.policy.classes|org.apache.lens.server.api.retry.SubstringMessagePolicyDecider|List of policy decider classes| *--+--+---+--+ -|22|retry.messages.contains.map|Session handle not found=org.apache.lens.server.api.retry.ImmediateRetryHandler(2)|Comma separated error messages and retry policy| +|24|retry.messages.contains.map|Session handle not found=org.apache.lens.server.api.retry.ImmediateRetryHandler(2)|Comma separated error messages and retry policy| *--+--+---+--+ The configuration parameters and their default values http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/src/site/apt/admin/jdbcdriver-config.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/admin/jdbcdriver-config.apt b/src/site/apt/admin/jdbcdriver-config.apt index 792278e..2455ac6 100644 --- a/src/site/apt/admin/jdbcdriver-config.apt +++ b/src/site/apt/admin/jdbcdriver-config.apt @@ -34,75 +34,79 @@ Jdbc driver configuration *--+--+---+--+ |5|lens.cube.query.time.range.writer.class|org.apache.lens.cube.parse.BetweenTimeRangeWriter|The timerange writer class which specifies how the resolved partitions in timeranges should be written in final query. Available writers are org.apache.lens.cube.parse.ORTimeRangeWriter and org.apache.lens.cube.parse.BetweenTimeRangeWriter| *--+--+---+--+ -|6|lens.driver.cost.query.decider.class|org.apache.lens.server.api.query.cost.RangeBasedQueryCostTypeDecider|Decider class which looks at ranges passed in config and decides the querycosttype| +|6|lens.cube.query.user.groups.authorization.enable|false|true if authorization is based on User Groups, false otherwise| *--+--+---+--+ -|7|lens.driver.cost.type.ranges|LOW,0.0,HIGH|Cost based Query type mapping| +|7|lens.cube.query.user.name.authorization.enable|false|true if authorization is based on User Name, false otherwise| *--+--+---+--+ -|8|lens.driver.jdbc.connection.properties| |Connection properties for jdbc connection.| +|8|lens.driver.cost.query.decider.class|org.apache.lens.server.api.query.cost.RangeBasedQueryCostTypeDecider|Decider class which looks at ranges passed in config and decides the querycosttype| *--+--+---+--+ -|9|lens.driver.jdbc.connection.provider| |A contract for obtaining JDBC connections| +|9|lens.driver.cost.type.ranges|LOW,0.0,HIGH|Cost based Query type mapping| *--+--+---+--+ -|10|lens.driver.jdbc.cost.calculator.class|org.apache.lens.cube.query.cost.StaticCostCalculator|Cost calculator class. By default calculating cost through static values| +|10|lens.driver.jdbc.connection.properties| |Connection properties for jdbc connection.| *--+--+---+--+ -|11|lens.driver.jdbc.db.password| |The database user's password| +|11|lens.driver.jdbc.connection.provider| |A contract for obtaining JDBC connections| *--+--+---+--+ -|12|lens.driver.jdbc.db.uri| |JDBC connection URL in the format jdbc:dbms://host:port/dbname| +|12|lens.driver.jdbc.cost.calculator.class|org.apache.lens.cube.query.cost.StaticCostCalculator|Cost calculator class. By default calculating cost through static values| *--+--+---+--+ -|13|lens.driver.jdbc.db.user| |The database user on whose behalf the connection is being made| +|13|lens.driver.jdbc.db.password| |The database user's password| *--+--+---+--+ -|14|lens.driver.jdbc.driver.class|com.mysql.jdbc.Driver|Type of JDBC driver used to connect backend database| +|14|lens.driver.jdbc.db.uri| |JDBC connection URL in the format jdbc:dbms://host:port/dbname| *--+--+---+--+ -|15|lens.driver.jdbc.enable.resultset.streaming.retrieval|false|Flag to enable row by row retrieval of result set from the database server. This is used to enable streaming result sets for MySQL. This is set to false by default.| +|15|lens.driver.jdbc.db.user| |The database user on whose behalf the connection is being made| *--+--+---+--+ -|16|lens.driver.jdbc.estimate.connection.properties| |Connection properties for jdbc estimate connection.| +|16|lens.driver.jdbc.driver.class|com.mysql.jdbc.Driver|Type of JDBC driver used to connect backend database| *--+--+---+--+ -|17|lens.driver.jdbc.estimate.db.password| |The database user's password, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.password would be used. Override this property to tune estimate connection pool| +|17|lens.driver.jdbc.enable.resultset.streaming.retrieval|false|Flag to enable row by row retrieval of result set from the database server. This is used to enable streaming result sets for MySQL. This is set to false by default.| *--+--+---+--+ -|18|lens.driver.jdbc.estimate.db.uri| |JDBC connection URL in the format jdbc:dbms://host:port/dbname for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.uri will be used.| +|18|lens.driver.jdbc.estimate.connection.properties| |Connection properties for jdbc estimate connection.| *--+--+---+--+ -|19|lens.driver.jdbc.estimate.db.user| |The database user on whose behalf the connection is being made, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.user would be used. Override this property to tune estimate connection pool| +|19|lens.driver.jdbc.estimate.db.password| |The database user's password, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.password would be used. Override this property to tune estimate connection pool| *--+--+---+--+ -|20|lens.driver.jdbc.estimate.driver.class| |Type of JDBC driver used to connect backend database for estimate queries. If This property is not specified, value for lens.driver.jdbc.driver.class will be used. Override this property to tune estimate connection pool| +|20|lens.driver.jdbc.estimate.db.uri| |JDBC connection URL in the format jdbc:dbms://host:port/dbname for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.uri will be used.| *--+--+---+--+ -|21|lens.driver.jdbc.estimate.get.connection.timeout| |Response timeout in milliseconds of any JDBC call invoking data transmission over a connection socket , for estimate queries. If this property is not specified, value for lens.driver.jdbc.get.connection.timeout would be used. Override this property to tune estimate connection pool.| +|21|lens.driver.jdbc.estimate.db.user| |The database user on whose behalf the connection is being made, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.user would be used. Override this property to tune estimate connection pool| *--+--+---+--+ -|22|lens.driver.jdbc.estimate.pool.idle.time| |Maximum idle time in sec before a connection is closed, for estimate queries. If this property is not specified, value for lens.driver.jdbc.pool.idle.time would be used. Override this property to tune estimate connection pool.| +|22|lens.driver.jdbc.estimate.driver.class| |Type of JDBC driver used to connect backend database for estimate queries. If This property is not specified, value for lens.driver.jdbc.driver.class will be used. Override this property to tune estimate connection pool| *--+--+---+--+ -|23|lens.driver.jdbc.estimate.pool.max.size| |Maximum number of concurrent connections allowed in pool, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.pool.max.size would be used. Override this property to tune estimate connection pool| +|23|lens.driver.jdbc.estimate.get.connection.timeout| |Response timeout in milliseconds of any JDBC call invoking data transmission over a connection socket , for estimate queries. If this property is not specified, value for lens.driver.jdbc.get.connection.timeout would be used. Override this property to tune estimate connection pool.| *--+--+---+--+ -|24|lens.driver.jdbc.estimate.pool.max.statements| |Maximum number of prepared statements to cache per connection, for estimate queries. If this property is not specified, value for lens.driver.jdbc.pool.max.statements would be used.| +|24|lens.driver.jdbc.estimate.pool.idle.time| |Maximum idle time in sec before a connection is closed, for estimate queries. If this property is not specified, value for lens.driver.jdbc.pool.idle.time would be used. Override this property to tune estimate connection pool.| *--+--+---+--+ -|25|lens.driver.jdbc.explain.keyword|Explain|Explain keyword used to get the query plan of underlying database| +|25|lens.driver.jdbc.estimate.pool.max.size| |Maximum number of concurrent connections allowed in pool, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.pool.max.size would be used. Override this property to tune estimate connection pool| *--+--+---+--+ -|26|lens.driver.jdbc.fetch.size|1000|Fetch size for JDBC result set| +|26|lens.driver.jdbc.estimate.pool.max.statements| |Maximum number of prepared statements to cache per connection, for estimate queries. If this property is not specified, value for lens.driver.jdbc.pool.max.statements would be used.| *--+--+---+--+ -|27|lens.driver.jdbc.get.connection.timeout|10000|The number of milliseconds a client calling getConnection() will wait for a Connection to be checked-in or acquired when the pool is exhausted. Zero means wait indefinitely. Setting any positive value will cause the getConnection () call to time-out and break with an SQLException after the specified number of milliseconds. The default value of this property is 10 secs.| +|27|lens.driver.jdbc.explain.keyword|Explain|Explain keyword used to get the query plan of underlying database| *--+--+---+--+ -|28|lens.driver.jdbc.pool.idle.time|600|Maximum idle time in sec before a connection is closed| +|28|lens.driver.jdbc.fetch.size|1000|Fetch size for JDBC result set| *--+--+---+--+ -|29|lens.driver.jdbc.pool.max.size|15|Maximum number of concurrent connections allowed in pool| +|29|lens.driver.jdbc.get.connection.timeout|10000|The number of milliseconds a client calling getConnection() will wait for a Connection to be checked-in or acquired when the pool is exhausted. Zero means wait indefinitely. Setting any positive value will cause the getConnection () call to time-out and break with an SQLException after the specified number of milliseconds. The default value of this property is 10 secs.| *--+--+---+--+ -|30|lens.driver.jdbc.pool.max.statements|20|Maximum number of prepared statements to cache per connection| +|30|lens.driver.jdbc.pool.idle.time|600|Maximum idle time in sec before a connection is closed| *--+--+---+--+ -|31|lens.driver.jdbc.query.launching.constraint.factories|org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory, +|31|lens.driver.jdbc.pool.max.size|15|Maximum number of concurrent connections allowed in pool| +*--+--+---+--+ +|32|lens.driver.jdbc.pool.max.statements|20|Maximum number of prepared statements to cache per connection| +*--+--+---+--+ +|33|lens.driver.jdbc.query.launching.constraint.factories|org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory, org.apache.lens.driver.jdbc.MaxJDBCConnectionCheckConstraintFactory|Factories used to instantiate constraints enforced on queries by driver. A query will be launched only if all constraints pass. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint.| *--+--+---+--+ -|32|lens.driver.jdbc.query.rewriter|org.apache.lens.driver.jdbc.ColumnarSQLRewriter|Rewriting the HQL to optimized sql queries| +|34|lens.driver.jdbc.query.rewriter|org.apache.lens.driver.jdbc.ColumnarSQLRewriter|Rewriting the HQL to optimized sql queries| *--+--+---+--+ -|33|lens.driver.jdbc.regex.replacement.values|to_date=date, format_number=format, date_sub\((.*?)\,\s*([0-9]+\s*)\)=date_sub($1\, interval $2 day), date_add\((.*?)\,\s*([0-9]+\s*)\)=date_add($1\, interval $2 day)|Rewriting the HQL to optimized sql queries| +|35|lens.driver.jdbc.regex.replacement.values|to_date=date, format_number=format, date_sub\((.*?)\,\s*([0-9]+\s*)\)=date_sub($1\, interval $2 day), date_add\((.*?)\,\s*([0-9]+\s*)\)=date_add($1\, interval $2 day)|Rewriting the HQL to optimized sql queries| *--+--+---+--+ -|34|lens.driver.jdbc.statement.cancel.supported|true|Flag to indicate Whether cancel on JDBC statement is supported. If not supported, framework wont call cancel on JDBC statement.| +|36|lens.driver.jdbc.statement.cancel.supported|true|Flag to indicate Whether cancel on JDBC statement is supported. If not supported, framework wont call cancel on JDBC statement.| *--+--+---+--+ -|35|lens.driver.jdbc.validate.through.prepare|true|Flag to enable query syntactic and semantic validation using prepared statement.| +|37|lens.driver.jdbc.validate.through.prepare|true|Flag to enable query syntactic and semantic validation using prepared statement.| *--+--+---+--+ -|36|lens.driver.jdbc.waiting.queries.selection.policy.factories|org.apache.lens.server.api.query.collect.DriverSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate driver specific waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| +|38|lens.driver.jdbc.waiting.queries.selection.policy.factories|org.apache.lens.server.api.query.collect.DriverSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate driver specific waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.| *--+--+---+--+ -|37|lens.driver.query.cost|0.0|Jdbc driver static cost value| +|39|lens.driver.query.cost|0.0|Jdbc driver static cost value| *--+--+---+--+ -|38|lens.query.timeout.millis|3600000|The runtime(millis) of the query after which query will be timedout and cancelled. Default is 1 hour for jdbc queries.| +|40|lens.query.timeout.millis|3600000|The runtime(millis) of the query after which query will be timedout and cancelled. Default is 1 hour for jdbc queries.| *--+--+---+--+ -|39|query.retry.policy.classes|org.apache.lens.server.api.retry.SubstringMessagePolicyDecider|List of classes to decide policies| +|41|query.retry.policy.classes|org.apache.lens.server.api.retry.SubstringMessagePolicyDecider|List of classes to decide policies| *--+--+---+--+ -|40|retry.messages.contains.map|Query not found=org.apache.lens.server.api.retry.ImmediateRetryHandler(2)|Comma separated error messages and retry policy| +|42|retry.messages.contains.map|Query not found=org.apache.lens.server.api.retry.ImmediateRetryHandler(2)|Comma separated error messages and retry policy| *--+--+---+--+ The configuration parameters and their default values
