[03/50] [abbrv] hadoop git commit: HADOOP-14627. Support MSI and DeviceCode token provider in ADLS. Contributed by Atul Sikaria.

2017-08-21 Thread aengineer 
HADOOP-14627. Support MSI and DeviceCode token provider in ADLS. Contributed by 
Atul Sikaria.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7769e961
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7769e961
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7769e961

Branch: refs/heads/HDFS-7240
Commit: 7769e9614956283a86eda9e4e69aaa592c0ca960
Parents: 8b242f0
Author: John Zhuge 
Authored: Thu Aug 10 00:43:40 2017 -0700
Committer: John Zhuge 
Committed: Sun Aug 13 00:22:34 2017 -0700

--
 .../src/main/resources/core-default.xml | 37 +++-
 hadoop-tools/hadoop-azure-datalake/pom.xml  |  2 +-
 .../org/apache/hadoop/fs/adl/AdlConfKeys.java   |  8 ++
 .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 21 +
 .../apache/hadoop/fs/adl/TokenProviderType.java |  2 +
 .../src/site/markdown/index.md  | 98 ++--
 .../hadoop/fs/adl/TestAzureADTokenProvider.java | 40 
 7 files changed, 198 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7769e961/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml 
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index ffcab2c..7c4b0f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2586,11 +2586,16 @@
 ClientCredential
 
   Defines Azure Active Directory OAuth2 access token provider type.
-  Supported types are ClientCredential, RefreshToken, and Custom.
+  Supported types are ClientCredential, RefreshToken, MSI, DeviceCode,
+  and Custom.
   The ClientCredential type requires property fs.adl.oauth2.client.id,
   fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url.
   The RefreshToken type requires property fs.adl.oauth2.client.id and
   fs.adl.oauth2.refresh.token.
+  The MSI type requires properties fs.adl.oauth2.msi.port and
+  fs.adl.oauth2.msi.tenantguid.
+  The DeviceCode type requires property
+  fs.adl.oauth2.devicecode.clientapp.id.
   The Custom type requires property fs.adl.oauth2.access.token.provider.
 
   
@@ -2627,6 +2632,36 @@
 
   
 
+  
+fs.adl.oauth2.msi.port
+
+
+  The localhost port for the MSI token service. This is the port specified
+  when creating the Azure VM.
+  Used by MSI token provider.
+
+  
+
+  
+fs.adl.oauth2.msi.tenantguid
+
+
+  The tenant guid for the Azure AAD tenant under which the azure data lake
+  store account is created.
+  Used by MSI token provider.
+
+  
+
+  
+fs.adl.oauth2.devicecode.clientapp.id
+
+
+  The app id of the AAD native app in whose context the auth request
+  should be made.
+  Used by DeviceCode token provider.
+
+  
+
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7769e961/hadoop-tools/hadoop-azure-datalake/pom.xml
--
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml 
b/hadoop-tools/hadoop-azure-datalake/pom.xml
index 3aed5e1..47f12df 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -110,7 +110,7 @@
 
   com.microsoft.azure
   azure-data-lake-store-sdk
-  2.1.4
+  2.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7769e961/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
--
diff --git 
a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
 
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
index 31df222..f77d981 100644
--- 
a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
+++ 
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -54,6 +54,14 @@ public final class AdlConfKeys {
   public static final String TOKEN_PROVIDER_TYPE_CLIENT_CRED =
   "ClientCredential";
 
+  // MSI Auth Configuration
+  public static final String MSI_PORT = "fs.adl.oauth2.msi.port";
+  public static final String MSI_TENANT_GUID = "fs.adl.oauth2.msi.tenantguid";
+
+  // DeviceCode Auth configuration
+  public static final String DEVICE_CODE_CLIENT_APP_ID =
+  "fs.adl.oauth2.devicecode.clientapp.id";
+
   public static final String READ_AHEAD_BUFFER_SIZE_KEY =
   "adl.feature.client.cache.

[03/50] [abbrv] hadoop git commit: HADOOP-14627. Support MSI and DeviceCode token provider in ADLS. Contributed by Atul Sikaria.

2017-08-16 Thread sunilg 
HADOOP-14627. Support MSI and DeviceCode token provider in ADLS. Contributed by 
Atul Sikaria.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7769e961
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7769e961
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7769e961

Branch: refs/heads/YARN-3926
Commit: 7769e9614956283a86eda9e4e69aaa592c0ca960
Parents: 8b242f0
Author: John Zhuge 
Authored: Thu Aug 10 00:43:40 2017 -0700
Committer: John Zhuge 
Committed: Sun Aug 13 00:22:34 2017 -0700

--
 .../src/main/resources/core-default.xml | 37 +++-
 hadoop-tools/hadoop-azure-datalake/pom.xml  |  2 +-
 .../org/apache/hadoop/fs/adl/AdlConfKeys.java   |  8 ++
 .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 21 +
 .../apache/hadoop/fs/adl/TokenProviderType.java |  2 +
 .../src/site/markdown/index.md  | 98 ++--
 .../hadoop/fs/adl/TestAzureADTokenProvider.java | 40 
 7 files changed, 198 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7769e961/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml 
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index ffcab2c..7c4b0f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2586,11 +2586,16 @@
 ClientCredential
 
   Defines Azure Active Directory OAuth2 access token provider type.
-  Supported types are ClientCredential, RefreshToken, and Custom.
+  Supported types are ClientCredential, RefreshToken, MSI, DeviceCode,
+  and Custom.
   The ClientCredential type requires property fs.adl.oauth2.client.id,
   fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url.
   The RefreshToken type requires property fs.adl.oauth2.client.id and
   fs.adl.oauth2.refresh.token.
+  The MSI type requires properties fs.adl.oauth2.msi.port and
+  fs.adl.oauth2.msi.tenantguid.
+  The DeviceCode type requires property
+  fs.adl.oauth2.devicecode.clientapp.id.
   The Custom type requires property fs.adl.oauth2.access.token.provider.
 
   
@@ -2627,6 +2632,36 @@
 
   
 
+  
+fs.adl.oauth2.msi.port
+
+
+  The localhost port for the MSI token service. This is the port specified
+  when creating the Azure VM.
+  Used by MSI token provider.
+
+  
+
+  
+fs.adl.oauth2.msi.tenantguid
+
+
+  The tenant guid for the Azure AAD tenant under which the azure data lake
+  store account is created.
+  Used by MSI token provider.
+
+  
+
+  
+fs.adl.oauth2.devicecode.clientapp.id
+
+
+  The app id of the AAD native app in whose context the auth request
+  should be made.
+  Used by DeviceCode token provider.
+
+  
+
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7769e961/hadoop-tools/hadoop-azure-datalake/pom.xml
--
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml 
b/hadoop-tools/hadoop-azure-datalake/pom.xml
index 3aed5e1..47f12df 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -110,7 +110,7 @@
 
   com.microsoft.azure
   azure-data-lake-store-sdk
-  2.1.4
+  2.2.1
 
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7769e961/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
--
diff --git 
a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
 
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
index 31df222..f77d981 100644
--- 
a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
+++ 
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -54,6 +54,14 @@ public final class AdlConfKeys {
   public static final String TOKEN_PROVIDER_TYPE_CLIENT_CRED =
   "ClientCredential";
 
+  // MSI Auth Configuration
+  public static final String MSI_PORT = "fs.adl.oauth2.msi.port";
+  public static final String MSI_TENANT_GUID = "fs.adl.oauth2.msi.tenantguid";
+
+  // DeviceCode Auth configuration
+  public static final String DEVICE_CODE_CLIENT_APP_ID =
+  "fs.adl.oauth2.devicecode.clientapp.id";
+
   public static final String READ_AHEAD_BUFFER_SIZE_KEY =
   "adl.feature.client.cache.