Repository: zeppelin Updated Branches: refs/heads/master 9812e26bc -> 791619d61
[ZEPPELIN-2950] Support Ceph as a notebook storage ### What is this PR for? Make Zeppelin support Ceph as a notebook storage. Ceph has APIs which are compatible with AWS S3 APIs. However, it supports only AWS Signature V2 and GetObject requests of aws-sdk-java use V4 by default: https://github.com/aws/aws-sdk-java/issues/372 According to https://github.com/aws/aws-sdk-java/issues/372#issuecomment-137299691 , the Zeppelin configuration of `zeppelin.notebook.s3.signerOverride` is added to make the `signerOverride` field of a `ClientConfiguration` instance configurable. ### What type of PR is it? [Improvement] ### Todos ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2950 ### How should this be tested? Tested manually. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Keiji Yoshida <kjmrk...@gmail.com> Closes #2598 from kjmrknsn/feature/ceph-support and squashes the following commits: c66cb63 [Keiji Yoshida] Support Ceph as a notebook storage Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/791619d6 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/791619d6 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/791619d6 Branch: refs/heads/master Commit: 791619d61165f849de36c89701cad964f77e859c Parents: 9812e26 Author: Keiji Yoshida <kjmrk...@gmail.com> Authored: Mon Sep 25 21:03:54 2017 +0900 Committer: Lee moon soo <m...@apache.org> Committed: Sat Oct 14 15:41:49 2017 -0700 ---------------------------------------------------------------------- conf/zeppelin-site.xml.template | 10 ++++++ docs/setup/operation/configuration.md | 6 ++++ .../zeppelin/conf/ZeppelinConfiguration.java | 5 +++ .../zeppelin/notebook/repo/S3NotebookRepo.java | 33 ++++++++++++++------ 4 files changed, 45 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/791619d6/conf/zeppelin-site.xml.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index b25ba19..4c31669 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -138,6 +138,16 @@ </property> --> +<!-- Optional override to control which signature algorithm should be used to sign AWS requests --> +<!-- Set this property to "S3SignerType" if your AWS S3 compatible APIs support only AWS Signature Version 2 such as Ceph. --> +<!-- +<property> + <name>zeppelin.notebook.s3.signerOverride</name> + <value>S3SignerType</value> + <description>optional override to control which signature algorithm should be used to sign AWS requests</description> +</property> +--> + <!-- If using Azure for storage use the following settings --> <!-- <property> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/791619d6/docs/setup/operation/configuration.md ---------------------------------------------------------------------- diff --git a/docs/setup/operation/configuration.md b/docs/setup/operation/configuration.md index 21ae5b3..458affc 100644 --- a/docs/setup/operation/configuration.md +++ b/docs/setup/operation/configuration.md @@ -216,6 +216,12 @@ If both are defined, then the **environment variables** will take priority. <td>Save notebooks to S3 with server-side encryption enabled</td> </tr> <tr> + <td><h6 class="properties">ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE</h6></td> + <td><h6 class="properties">zeppelin.notebook.s3.signerOverride</h6></td> + <td></td> + <td>Optional override to control which signature algorithm should be used to sign AWS requests</td> + </tr> + <tr> <td><h6 class="properties">ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</h6></td> <td><h6 class="properties">zeppelin.notebook.azure.connectionString</h6></td> <td></td> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/791619d6/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 3a82bc5..720d6ec 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -383,6 +383,10 @@ public class ZeppelinConfiguration extends XMLConfiguration { return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_S3_SSE); } + public String getS3SignerOverride() { + return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE); + } + public String getMongoUri() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_URI); } @@ -654,6 +658,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID("zeppelin.notebook.s3.kmsKeyID", null), ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION("zeppelin.notebook.s3.kmsKeyRegion", null), ZEPPELIN_NOTEBOOK_S3_SSE("zeppelin.notebook.s3.sse", false), + ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE("zeppelin.notebook.s3.signerOverride", null), ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING("zeppelin.notebook.azure.connectionString", null), ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"), ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"), http://git-wip-us.apache.org/repos/asf/zeppelin/blob/791619d6/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java index 16b270c..8828985 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java @@ -42,6 +42,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.amazonaws.AmazonClientException; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.ClientConfigurationFactory; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.services.s3.AmazonS3; @@ -94,33 +96,30 @@ public class S3NotebookRepo implements NotebookRepo { // always use the default provider chain AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); - CryptoConfiguration cryptoConf = null; + CryptoConfiguration cryptoConf = new CryptoConfiguration(); String keyRegion = conf.getS3KMSKeyRegion(); if (StringUtils.isNotBlank(keyRegion)) { - cryptoConf = new CryptoConfiguration(); cryptoConf.setAwsKmsRegion(Region.getRegion(Regions.fromName(keyRegion))); } + + ClientConfiguration cliConf = createClientConfiguration(); // see if we should be encrypting data in S3 String kmsKeyID = conf.getS3KMSKeyID(); if (kmsKeyID != null) { // use the AWS KMS to encrypt data KMSEncryptionMaterialsProvider emp = new KMSEncryptionMaterialsProvider(kmsKeyID); - if (cryptoConf != null) { - this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cryptoConf); - } else { - this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp); - } + this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf); } else if (conf.getS3EncryptionMaterialsProviderClass() != null) { // use a custom encryption materials provider class EncryptionMaterialsProvider emp = createCustomProvider(conf); - this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp); + this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf); } else { // regular S3 - this.s3client = new AmazonS3Client(credentialsProvider); + this.s3client = new AmazonS3Client(credentialsProvider, cliConf); } // set S3 endpoint to use @@ -154,6 +153,22 @@ public class S3NotebookRepo implements NotebookRepo { return emp; } + /** + * Create AWS client configuration and return it. + * @return AWS client configuration + */ + private ClientConfiguration createClientConfiguration() { + ClientConfigurationFactory configFactory = new ClientConfigurationFactory(); + ClientConfiguration config = configFactory.getConfig(); + + String s3SignerOverride = conf.getS3SignerOverride(); + if (StringUtils.isNotBlank(s3SignerOverride)) { + config.setSignerOverride(s3SignerOverride); + } + + return config; + } + @Override public List<NoteInfo> list(AuthenticationInfo subject) throws IOException { List<NoteInfo> infos = new LinkedList<>();
