Repository: incubator-atlas Updated Branches: refs/heads/master b832faf0c -> 266d7cc00
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java index 174dafb..ca71293 100755 --- a/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java +++ b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java @@ -23,8 +23,14 @@ import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; -import org.mortbay.jetty.Connector; -import org.mortbay.jetty.security.SslSocketConnector; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.eclipse.jetty.http.HttpVersion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,18 +60,38 @@ public class SecureEmbeddedServer extends EmbeddedServer { protected Connector getConnector(int port) throws IOException { PropertiesConfiguration config = getConfiguration(); - SslSocketConnector connector = new SslSocketConnector(); - connector.setPort(port); - connector.setHost("0.0.0.0"); - connector.setKeystore(config.getString(KEYSTORE_FILE_KEY, - System.getProperty(KEYSTORE_FILE_KEY, DEFAULT_KEYSTORE_FILE_LOCATION))); - connector.setKeyPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY)); - connector.setTruststore(config.getString(TRUSTSTORE_FILE_KEY, - System.getProperty(TRUSTSTORE_FILE_KEY, DEFATULT_TRUSTORE_FILE_LOCATION))); - connector.setTrustPassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY)); - connector.setPassword(getPassword(config, KEYSTORE_PASSWORD_KEY)); - connector.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY))); - return connector; + SslContextFactory sslContextFactory = new SslContextFactory(); + sslContextFactory.setKeyStorePath(config.getString(KEYSTORE_FILE_KEY, + System.getProperty(KEYSTORE_FILE_KEY, DEFAULT_KEYSTORE_FILE_LOCATION))); + sslContextFactory.setKeyStorePassword(getPassword(config, KEYSTORE_PASSWORD_KEY)); + sslContextFactory.setKeyManagerPassword(getPassword(config, SERVER_CERT_PASSWORD_KEY)); + sslContextFactory.setTrustStorePath(config.getString(TRUSTSTORE_FILE_KEY, + System.getProperty(TRUSTSTORE_FILE_KEY, DEFATULT_TRUSTORE_FILE_LOCATION))); + sslContextFactory.setTrustStorePassword(getPassword(config, TRUSTSTORE_PASSWORD_KEY)); + sslContextFactory.setWantClientAuth(config.getBoolean(CLIENT_AUTH_KEY, Boolean.getBoolean(CLIENT_AUTH_KEY))); + + // SSL HTTP Configuration + // HTTP Configuration + HttpConfiguration http_config = new HttpConfiguration(); + http_config.setSecureScheme("https"); + final int bufferSize = getBufferSize(); + http_config.setSecurePort(port); + http_config.setRequestHeaderSize(bufferSize); + http_config.setResponseHeaderSize(bufferSize); + http_config.setSendServerVersion(true); + http_config.setSendDateHeader(false); + + HttpConfiguration https_config = new HttpConfiguration(http_config); + https_config.addCustomizer(new SecureRequestCustomizer()); + + // SSL Connector + ServerConnector sslConnector = new ServerConnector(server, + new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()), + new HttpConnectionFactory(https_config)); + sslConnector.setPort(port); + server.addConnector(sslConnector); + + return sslConnector; } /** http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/main/resources/application.properties ---------------------------------------------------------------------- diff --git a/webapp/src/main/resources/application.properties b/webapp/src/main/resources/application.properties index c9b8408..ecfdc38 100755 --- a/webapp/src/main/resources/application.properties +++ b/webapp/src/main/resources/application.properties @@ -18,16 +18,29 @@ ######### Graph Database Configs ######### # Graph Storage -atlas.graph.storage.backend=berkeleyje +atlas.graph.storage.backend=${titan.storage.backend} + +# Graph Search Index Backend +atlas.graph.index.search.backend=${titan.index.backend} + +#Berkeley storage directory atlas.graph.storage.directory=target/data/berkley -# Graph Search Index -atlas.graph.index.search.backend=elasticsearch +#hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 +atlas.graph.storage.hostname=${titan.storage.hostname} + +#ElasticSearch atlas.graph.index.search.directory=target/data/es atlas.graph.index.search.elasticsearch.client-only=false atlas.graph.index.search.elasticsearch.local-mode=true atlas.graph.index.search.elasticsearch.create.sleep=2000 +# Solr cloud mode properties +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address} + ######### Hive Lineage Configs ######### # This models reflects the base super types for Data and Process #atlas.lineage.hive.table.type.name=DataSet http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java b/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java index 95acc57..bb42dcf 100644 --- a/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationKerberosFilterIT.java @@ -16,12 +16,12 @@ */ package org.apache.atlas.web.filters; -import org.apache.atlas.security.BaseSecurityTest; +import org.apache.atlas.web.security.BaseSecurityTest; import org.apache.atlas.web.service.EmbeddedServer; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hdfs.web.URLConnectionFactory; -import org.mortbay.jetty.Server; +import org.eclipse.jetty.server.Server; import org.testng.Assert; import org.testng.annotations.Test; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java b/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java index 5ea2e15..477ac4e 100644 --- a/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/filters/MetadataAuthenticationSimpleFilterIT.java @@ -16,10 +16,10 @@ */ package org.apache.atlas.web.filters; -import org.apache.atlas.security.BaseSecurityTest; +import org.apache.atlas.web.security.BaseSecurityTest; import org.apache.atlas.web.service.EmbeddedServer; import org.apache.commons.configuration.ConfigurationException; -import org.mortbay.jetty.Server; +import org.eclipse.jetty.server.Server; import org.testng.Assert; import org.testng.annotations.Test; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java b/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java index 54a7a7b..2148fe2 100644 --- a/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/listeners/LoginProcessorIT.java @@ -16,7 +16,7 @@ */ package org.apache.atlas.web.listeners; -import org.apache.atlas.security.BaseSecurityTest; +import org.apache.atlas.web.security.BaseSecurityTest; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.hadoop.conf.Configuration; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/java/org/apache/atlas/web/listeners/TestGuiceServletConfig.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/listeners/TestGuiceServletConfig.java b/webapp/src/test/java/org/apache/atlas/web/listeners/TestGuiceServletConfig.java new file mode 100644 index 0000000..3ba6387 --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/web/listeners/TestGuiceServletConfig.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.web.listeners; + +import com.google.inject.Key; +import com.google.inject.Provider; +import com.google.inject.TypeLiteral; +import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.util.TitanCleanup; +import com.tinkerpop.blueprints.Graph; +import org.apache.atlas.repository.graph.GraphProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.ServletContextEvent; + +public class TestGuiceServletConfig extends GuiceServletConfig { + + private static final Logger LOG = LoggerFactory.getLogger(TestGuiceServletConfig.class); + + @Override + public void contextInitialized(ServletContextEvent servletContextEvent) { + LOG.info("Initializing test servlet listener"); + super.contextInitialized(servletContextEvent); + } + + @Override + public void contextDestroyed(ServletContextEvent servletContextEvent) { + if(injector != null) { + TypeLiteral<GraphProvider<TitanGraph>> graphProviderType = new TypeLiteral<GraphProvider<TitanGraph>>() {}; + Provider<GraphProvider<TitanGraph>> graphProvider = injector.getProvider(Key.get(graphProviderType)); + TitanGraph graph = graphProvider.get().get(); + + LOG.info("Clearing graph store"); + try { + graph.shutdown(); + TitanCleanup.clear(graph); + } catch (Exception e) { + LOG.warn("Clearing graph store failed ", e); + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/java/org/apache/atlas/web/security/BaseSecurityTest.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/security/BaseSecurityTest.java b/webapp/src/test/java/org/apache/atlas/web/security/BaseSecurityTest.java new file mode 100644 index 0000000..67f3901 --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/web/security/BaseSecurityTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.web.security; + +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.zookeeper.Environment; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.webapp.WebAppContext; +import org.testng.Assert; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Writer; +import java.nio.file.Files; +import java.util.Locale; +import java.util.Properties; + +/** + * + */ +public class BaseSecurityTest { + private static final String JAAS_ENTRY = "%s { \n" + " %s required\n" + // kerberos module + + " keyTab=\"%s\"\n" + " debug=true\n" + " principal=\"%s\"\n" + " useKeyTab=true\n" + + " useTicketCache=false\n" + " doNotPrompt=true\n" + " storeKey=true;\n" + "}; \n"; + protected MiniKdc kdc; + + protected String getWarPath() { + return String.format("/target/atlas-webapp-%s.war", + System.getProperty("release.version")); + } + + protected void generateTestProperties(Properties props) throws ConfigurationException, IOException { + PropertiesConfiguration config = + new PropertiesConfiguration(System.getProperty("user.dir") + "/../src/conf/application.properties"); + for (String propName : props.stringPropertyNames()) { + config.setProperty(propName, props.getProperty(propName)); + } + File file = new File(System.getProperty("user.dir"), "application.properties"); + file.deleteOnExit(); + Writer fileWriter = new FileWriter(file); + config.save(fileWriter); + } + + protected void startEmbeddedServer(Server server) throws Exception { + WebAppContext webapp = new WebAppContext(); + webapp.setContextPath("/"); + webapp.setWar(System.getProperty("user.dir") + getWarPath()); + server.setHandler(webapp); + + server.start(); + } + + protected File startKDC() throws Exception { + File target = Files.createTempDirectory("sectest").toFile(); + File kdcWorkDir = new File(target, "kdc"); + Properties kdcConf = MiniKdc.createConf(); + kdcConf.setProperty(MiniKdc.DEBUG, "true"); + kdc = new MiniKdc(kdcConf, kdcWorkDir); + kdc.start(); + + Assert.assertNotNull(kdc.getRealm()); + return kdcWorkDir; + } + + public String createJAASEntry(String context, String principal, File keytab) { + String keytabpath = keytab.getAbsolutePath(); + // fix up for windows; no-op on unix + keytabpath = keytabpath.replace('\\', '/'); + return String.format(Locale.ENGLISH, JAAS_ENTRY, context, getKerberosAuthModuleForJVM(), keytabpath, principal); + } + + protected String getKerberosAuthModuleForJVM() { + if (System.getProperty("java.vendor").contains("IBM")) { + return "com.ibm.security.auth.module.Krb5LoginModule"; + } else { + return "com.sun.security.auth.module.Krb5LoginModule"; + } + } + + protected void bindJVMtoJAASFile(File jaasFile) { + String path = jaasFile.getAbsolutePath(); + System.setProperty(Environment.JAAS_CONF_KEY, path); + } + + protected File createKeytab(MiniKdc kdc, File kdcWorkDir, String principal, String filename) throws Exception { + File keytab = new File(kdcWorkDir, filename); + kdc.createPrincipal(keytab, principal, principal + "/localhost", principal + "/127.0.0.1"); + return keytab; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java index 3e27532..7eb36d8 100644 --- a/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java @@ -38,6 +38,7 @@ public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase { SecureEmbeddedServer secureEmbeddedServer = null; try { String appPath = System.getProperty("user.dir") + getWarPath(); + secureEmbeddedServer = new SecureEmbeddedServer(21443, appPath) { @Override protected PropertiesConfiguration getConfiguration() { @@ -53,6 +54,8 @@ public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase { // test to see whether server is up and root page can be served Assert.assertEquals(connection.getResponseCode(), 200); + } catch(Throwable e) { + Assert.fail("War deploy failed", e); } finally { secureEmbeddedServer.server.stop(); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java index db110ca..9a5b8ad 100755 --- a/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java +++ b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java @@ -31,7 +31,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.security.alias.JavaKeyStoreProvider; -import org.mortbay.jetty.webapp.WebAppContext; +import org.eclipse.jetty.webapp.WebAppContext; import org.testng.Assert; import org.testng.TestListenerAdapter; import org.testng.TestNG; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/266d7cc0/webapp/src/test/webapp/WEB-INF/web.xml ---------------------------------------------------------------------- diff --git a/webapp/src/test/webapp/WEB-INF/web.xml b/webapp/src/test/webapp/WEB-INF/web.xml new file mode 100755 index 0000000..0562643 --- /dev/null +++ b/webapp/src/test/webapp/WEB-INF/web.xml @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<!DOCTYPE web-app PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN" + "http://java.sun.com/dtd/web-app_2_3.dtd";> + +<web-app> + <display-name>Apache Atlas</display-name> + <description>Metadata Management and Data Governance Platform over Hadoop</description> + + <context-param> + <param-name>guice.packages</param-name> + <param-value> + org.apache.atlas.web.resources,org.apache.atlas.web.params + </param-value> + </context-param> + + <!-- + More information can be found here: + + https://jersey.java.net/nonav/apidocs/1.11/contribs/jersey-guice/com/sun/jersey/guice/spi/container/servlet/package-summary.html + --> + + <filter> + <filter-name>guiceFilter</filter-name> + <filter-class>com.google.inject.servlet.GuiceFilter</filter-class> + </filter> + + <filter-mapping> + <filter-name>guiceFilter</filter-name> + <url-pattern>/*</url-pattern> + </filter-mapping> + + <listener> + <listener-class>org.apache.atlas.web.listeners.TestGuiceServletConfig</listener-class> + </listener> +</web-app>
