Repository: sentry Updated Branches: refs/heads/master 5d4465284 -> d4c622845
SENTRY-2329: Integrate sentry with Hadoop 3.1.1 (kalyan kumar kalvagadda, reviewed by Sergio Pena) This integration is compatible with old Hadoop 2.x versions. You can compile with Hadoop 2.x by changing the pom.xml or you can add the Sentry/HDFS binding jars built with Hadoop3 in the Hadoop 2 classpath. Both ways are verified and working. Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/d4c62284 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/d4c62284 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/d4c62284 Branch: refs/heads/master Commit: d4c622845b9d32f214d1704e81fe443ce8c63290 Parents: 5d44652 Author: Sergio Pena <sergio.p...@cloudera.com> Authored: Mon Nov 5 13:58:48 2018 -0600 Committer: Sergio Pena <sergio.p...@cloudera.com> Committed: Mon Nov 5 13:58:48 2018 -0600 ---------------------------------------------------------------------- pom.xml | 22 +++++++++++++++++++- sentry-binding/sentry-binding-solr/pom.xml | 16 ++++++++++++++ sentry-hdfs/sentry-hdfs-common/pom.xml | 6 ++++++ .../org/apache/sentry/hdfs/PathsUpdate.java | 8 +++---- .../hdfs/SentryINodeAttributesProvider.java | 5 +++++ .../thrift/TestSentryWebServerWithKerberos.java | 21 ++++++++++++++++--- 6 files changed, 69 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/d4c62284/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index acbdcc2..46ca38e 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,6 @@ limitations under the License. <easymock.version>3.0</easymock.version> <fest.reflect.version>1.4.1</fest.reflect.version> <guava.version>14.0.1</guava.version> - <hadoop.version>2.7.5</hadoop.version> <hamcrest.version>1.3</hamcrest.version> <hive.version>2.3.3</hive.version> <jackson.version>1.9.13</jackson.version> @@ -101,6 +100,10 @@ limitations under the License. <test.sentry.hadoop.classpath>${maven.test.classpath}</test.sentry.hadoop.classpath> <zookeeper.version>3.4.5</zookeeper.version> <maven.jar.plugin.version>3.0.2</maven.jar.plugin.version> + <httpcomponents.version>4.5.3</httpcomponents.version> + + <!-- Package versions for Sentry binding components --> + <hadoop.version>3.1.1</hadoop.version> <!-- Datanucleus package versions --> <datanucleus.maven.plugin.version>4.0.5</datanucleus.maven.plugin.version> @@ -197,6 +200,7 @@ limitations under the License. <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> <version>${hadoop.version}</version> + <scope>test</scope> <exclusions> <exclusion> <artifactId>curator-client</artifactId> @@ -410,6 +414,10 @@ limitations under the License. <artifactId>jackson-mapper-asl</artifactId> <groupId>org.codehaus.jackson</groupId> </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-archives</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -521,6 +529,10 @@ limitations under the License. <artifactId>jackson-xc</artifactId> <groupId>org.codehaus.jackson</groupId> </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-resourcemanager</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -608,6 +620,10 @@ limitations under the License. <artifactId>jackson-xc</artifactId> <groupId>org.codehaus.jackson</groupId> </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-registry</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -945,6 +961,10 @@ limitations under the License. <artifactId>jackson-xc</artifactId> <groupId>org.codehaus.jackson</groupId> </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-archives</artifactId> + </exclusion> </exclusions> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/sentry/blob/d4c62284/sentry-binding/sentry-binding-solr/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-binding/sentry-binding-solr/pom.xml b/sentry-binding/sentry-binding-solr/pom.xml index f2a5fca..97b1879 100644 --- a/sentry-binding/sentry-binding-solr/pom.xml +++ b/sentry-binding/sentry-binding-solr/pom.xml @@ -29,6 +29,22 @@ limitations under the License. <name>Sentry Binding for Solr</name> <dependencies> + <!-- This jetty dependency must be added before the solr-core dependency to avoid conflicting + other jetty dependencies --> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> + <!-- This jetty dependency must be added before the solr-core dependency to avoid conflicting + other jetty dependencies --> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-webapp</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> <dependency> <groupId>org.apache.solr</groupId> <artifactId>solr-core</artifactId> http://git-wip-us.apache.org/repos/asf/sentry/blob/d4c62284/sentry-hdfs/sentry-hdfs-common/pom.xml ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/pom.xml b/sentry-hdfs/sentry-hdfs-common/pom.xml index df6f04c..4d70d13 100644 --- a/sentry-hdfs/sentry-hdfs-common/pom.xml +++ b/sentry-hdfs/sentry-hdfs-common/pom.xml @@ -80,6 +80,12 @@ limitations under the License. <artifactId>mockito-all</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <version>${httpcomponents.version}</version> + <scope>provided</scope> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/sentry/blob/d4c62284/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java index c9ecc40..edc3afc 100644 --- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java +++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java @@ -26,11 +26,10 @@ import java.util.List; import com.google.common.annotations.VisibleForTesting; import org.apache.sentry.hdfs.service.thrift.TPathChanges; import org.apache.sentry.hdfs.service.thrift.TPathsUpdate; -import org.apache.commons.httpclient.util.URIUtil; -import org.apache.commons.httpclient.URIException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.conf.Configuration; +import org.apache.http.client.utils.URIBuilder; import org.apache.thrift.TException; @@ -137,11 +136,10 @@ public class PathsUpdate implements Updateable.Update { URI uri; try { - uri = new URI(URIUtil.encodePath(path)); + // Below converts unescaped path to escaped string and the constructs a URI from it. + uri = new URI(StringUtils.stripStart(new URIBuilder().setPath(path).toString(), "/")); } catch (URISyntaxException e) { throw new SentryMalformedPathException("Incomprehensible path [" + path + "]", e); - } catch (URIException e) { - throw new SentryMalformedPathException("Unable to create URI from path[" + path + "]", e); } String scheme = uri.getScheme(); http://git-wip-us.apache.org/repos/asf/sentry/blob/d4c62284/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java index 18b6265..4d2ef00 100644 --- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java +++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java @@ -331,6 +331,11 @@ public class SentryINodeAttributesProvider extends INodeAttributeProvider public INodeAttributes getAttributes(String[] pathElements, INodeAttributes inode) { Preconditions.checkNotNull(pathElements); + + if (pathElements.length == 0) { + return inode; + } + pathElements = "".equals(pathElements[0]) && pathElements.length > 1 ? Arrays.copyOfRange(pathElements, 1, pathElements.length) : pathElements; http://git-wip-us.apache.org/repos/asf/sentry/blob/d4c62284/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/api/service/thrift/TestSentryWebServerWithKerberos.java ---------------------------------------------------------------------- diff --git a/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/api/service/thrift/TestSentryWebServerWithKerberos.java b/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/api/service/thrift/TestSentryWebServerWithKerberos.java index 5d94d4b..add2c28 100644 --- a/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/api/service/thrift/TestSentryWebServerWithKerberos.java +++ b/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/api/service/thrift/TestSentryWebServerWithKerberos.java @@ -88,7 +88,7 @@ public class TestSentryWebServerWithKerberos extends SentryServiceIntegrationBas new AuthenticatedURL(new KerberosAuthenticator()).openConnection(url, new AuthenticatedURL.Token()); fail("Here should fail."); } catch (Exception e) { - boolean isExpectError = e.getMessage().contains("No valid credentials provided"); + boolean isExpectError = exceptionContainsMessage(e,"No valid credentials provided"); Assert.assertTrue("Here should fail by 'No valid credentials provided'," + " but the exception is:" + e, isExpectError); } @@ -124,7 +124,7 @@ public class TestSentryWebServerWithKerberos extends SentryServiceIntegrationBas fail("Here should fail."); } catch (AuthenticationException e) { String expectedError = "status code: 403"; - if (!e.getMessage().contains(expectedError)) { + if (!exceptionContainsMessage(e, expectedError)) { LOG.error("UnexpectedError: " + e.getMessage(), e); fail("UnexpectedError: " + e.getMessage()); } @@ -155,7 +155,7 @@ public class TestSentryWebServerWithKerberos extends SentryServiceIntegrationBas fail("Login with user1 should fail"); } catch (AuthenticationException e) { String expectedError = "status code: 403"; - if (!e.getMessage().contains(expectedError)) { + if (!exceptionContainsMessage(e, expectedError)) { LOG.error("UnexpectedError: " + e.getMessage(), e); fail("UnexpectedError: " + e.getMessage()); } @@ -172,4 +172,19 @@ public class TestSentryWebServerWithKerberos extends SentryServiceIntegrationBas conn.setRequestMethod("TRACE"); Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN, conn.getResponseCode()); } + + /* + * Check if the exception contains the specified message in any of the exception message + * or cause message. + * + * <p/>i.e. Hadoop 2.x has a 'no valid privileges' message in the e.getMessage() whereas + * Hadoop 3.x has the 'no valid privileges' message in the e.getCause().getMessage(). + */ + private boolean exceptionContainsMessage(Exception e, String message) { + if (e.getMessage().contains(message)) { + return true; + } + + return e.getCause().getMessage().contains(message); + } }
