Repository: incubator-atlas Updated Branches: refs/heads/master d95775161 -> 43a1e25bd
ATLAS-1081 Atlas jetty server configuration (shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/43a1e25b Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/43a1e25b Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/43a1e25b Branch: refs/heads/master Commit: 43a1e25bdd1d3f7700c905a29c21dcd601591872 Parents: d957751 Author: Shwetha GS <[email protected]> Authored: Tue Nov 15 13:54:16 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Tue Nov 15 13:54:16 2016 +0530 ---------------------------------------------------------------------- .../org/apache/atlas/AtlasConfiguration.java | 72 ++++++++++++++++++++ .../java/org/apache/atlas/AtlasProperties.java | 64 ----------------- distro/src/bin/atlas_start.py | 2 +- docs/src/site/twiki/Configuration.twiki | 12 ++++ release-log.txt | 1 + .../atlas/discovery/DataSetLineageService.java | 4 +- .../resources/MetadataDiscoveryResource.java | 6 +- .../atlas/web/service/EmbeddedServer.java | 35 +++++----- .../atlas/web/service/SecureEmbeddedServer.java | 9 +-- 9 files changed, 112 insertions(+), 93 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/common/src/main/java/org/apache/atlas/AtlasConfiguration.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/AtlasConfiguration.java b/common/src/main/java/org/apache/atlas/AtlasConfiguration.java new file mode 100644 index 0000000..f5a648d --- /dev/null +++ b/common/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas; + +import org.apache.commons.configuration.Configuration; + +/** + * Enum that encapsulated each property name and its default value. + */ +public enum AtlasConfiguration { + //web server configuration + WEBSERVER_MIN_THREADS("atlas.webserver.minthreads", 10), + WEBSERVER_MAX_THREADS("atlas.webserver.maxthreads", 100), + WEBSERVER_KEEPALIVE_SECONDS("atlas.webserver.keepalivetimesecs", 60), + WEBSERVER_QUEUE_SIZE("atlas.webserver.queuesize", 100), + WEBSERVER_REQUEST_BUFFER_SIZE("atlas.jetty.request.buffer.size", 16192), + + //search configuration + SEARCH_MAX_LIMIT("atlas.search.maxlimit", 10000), + SEARCH_DEFAULT_LIMIT("atlas.search.defaultlimit", 100); + + private static final Configuration APPLICATION_PROPERTIES; + + static { + try { + APPLICATION_PROPERTIES = ApplicationProperties.get(); + } catch (AtlasException e) { + throw new RuntimeException(e); + } + } + + private final String propertyName; + private final Object defaultValue; + + AtlasConfiguration(String propertyName, Object defaultValue) { + this.propertyName = propertyName; + this.defaultValue = defaultValue; + } + + public int getInt() { + return APPLICATION_PROPERTIES.getInt(propertyName, Integer.valueOf(defaultValue.toString()).intValue()); + } + + public long getLong() { + return APPLICATION_PROPERTIES.getLong(propertyName, Long.valueOf(defaultValue.toString()).longValue()); + } + + public String getString() { + return APPLICATION_PROPERTIES.getString(propertyName, defaultValue.toString()); + } + + public Object get() { + Object value = APPLICATION_PROPERTIES.getProperty(propertyName); + return value == null ? defaultValue : value; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/common/src/main/java/org/apache/atlas/AtlasProperties.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/AtlasProperties.java b/common/src/main/java/org/apache/atlas/AtlasProperties.java deleted file mode 100644 index df1bccb..0000000 --- a/common/src/main/java/org/apache/atlas/AtlasProperties.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas; - -import org.apache.commons.configuration.Configuration; - -/** - * Utility for reading properties in atlas-application.properties. - */ -public final class AtlasProperties { - private static final Configuration APPLICATION_PROPERTIES; - - private AtlasProperties() { } - - static { - try { - APPLICATION_PROPERTIES = ApplicationProperties.get(); - } catch (AtlasException e) { - throw new RuntimeException(e); - } - } - - /** - * Enum that encapsulated each property name and its default value. - */ - public enum AtlasProperty { - SEARCH_MAX_LIMIT("atlas.search.maxlimit", 10000), - SEARCH_DEFAULT_LIMIT("atlas.search.defaultlimit", 100); - - private final String propertyName; - private final Object defaultValue; - - AtlasProperty(String propertyName, Object defaultValue) { - this.propertyName = propertyName; - this.defaultValue = defaultValue; - } - } - - public static <T> T getProperty(AtlasProperty property) { - Object value = APPLICATION_PROPERTIES.getProperty(property.propertyName); - if (value == null) { - return (T) property.defaultValue; - } else { - return (T) value; - - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/distro/src/bin/atlas_start.py ---------------------------------------------------------------------- diff --git a/distro/src/bin/atlas_start.py b/distro/src/bin/atlas_start.py index 4590548..a6a3455 100755 --- a/distro/src/bin/atlas_start.py +++ b/distro/src/bin/atlas_start.py @@ -25,7 +25,7 @@ ATLAS_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=%s.log" ATLAS_COMMAND_OPTS="-Datlas.home=%s" ATLAS_CONFIG_OPTS="-Datlas.conf=%s" DEFAULT_JVM_HEAP_OPTS="-Xmx1024m -XX:MaxPermSize=512m" -DEFAULT_JVM_OPTS="-Dlog4j.configuration=atlas-log4j.xml -Djava.net.preferIPv4Stack=true" +DEFAULT_JVM_OPTS="-Dlog4j.configuration=atlas-log4j.xml -Djava.net.preferIPv4Stack=true -server" def main(): http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/docs/src/site/twiki/Configuration.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/Configuration.twiki b/docs/src/site/twiki/Configuration.twiki index bbe56d2..7f24f88 100644 --- a/docs/src/site/twiki/Configuration.twiki +++ b/docs/src/site/twiki/Configuration.twiki @@ -271,4 +271,16 @@ atlas.graph.storage.lock.retries=10 # Milliseconds to wait before evicting a cached entry. This should be > atlas.graph.storage.lock.wait-time x atlas.graph.storage.lock.retries # If this is set to a low value (default is 10000), warnings on transactions taking too long will occur in the Atlas application log. atlas.graph.storage.cache.db-cache-time=120000 + +# Minimum number of threads in the atlas web server +atlas.webserver.minthreads=10 + +# Maximum number of threads in the atlas web server +atlas.webserver.maxthreads=100 + +# Keepalive time in secs for the thread pool of the atlas web server +atlas.webserver.keepalivetimesecs=60 + +# Queue size for the requests(when max threads are busy) for the atlas web server +atlas.webserver.queuesize=100 </verbatim> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index da45215..4451222 100644 --- a/release-log.txt +++ b/release-log.txt @@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai) ALL CHANGES: +ATLAS-1081 Atlas jetty server configuration (shwethags) ATLAS-1257 Map Entity REST APIs to ATLAS v1 backend (sumasai) ATLAS-1279 Remove QueryPlan attribute from Hive Process entity (svimal2106) ATLAS-1234 Lineage REST API - v2 ([email protected] via mneethiraj) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java index c3fd72b..4d3177c 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java @@ -20,8 +20,8 @@ package org.apache.atlas.discovery; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; -import org.apache.atlas.AtlasProperties; import org.apache.atlas.GraphTransaction; import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy; import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService; @@ -178,7 +178,7 @@ public class DataSetLineageService implements LineageService { if (propertiesConf.getString(configName) != null) { final String schemaQuery = String.format(propertiesConf.getString(configName), guid); - int limit = AtlasProperties.getProperty(AtlasProperties.AtlasProperty.SEARCH_MAX_LIMIT); + int limit = AtlasConfiguration.SEARCH_MAX_LIMIT.getInt(); return discoveryService.searchByDSL(schemaQuery, new QueryParams(limit, 0)); } throw new SchemaNotFoundException("Schema is not configured for type " + typeName + ". Configure " + configName); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java index 64344da..ac89869 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java @@ -20,7 +20,7 @@ package org.apache.atlas.web.resources; import com.google.common.base.Preconditions; import org.apache.atlas.AtlasClient; -import org.apache.atlas.AtlasProperties; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.classification.InterfaceAudience; import org.apache.atlas.discovery.DiscoveryException; import org.apache.atlas.discovery.DiscoveryService; @@ -157,8 +157,8 @@ public class MetadataDiscoveryResource { } private QueryParams validateQueryParams(int limitParam, int offsetParam) { - int maxLimit = AtlasProperties.getProperty(AtlasProperties.AtlasProperty.SEARCH_MAX_LIMIT); - int defaultLimit = AtlasProperties.getProperty(AtlasProperties.AtlasProperty.SEARCH_DEFAULT_LIMIT); + int maxLimit = AtlasConfiguration.SEARCH_MAX_LIMIT.getInt(); + int defaultLimit = AtlasConfiguration.SEARCH_DEFAULT_LIMIT.getInt(); int limit = defaultLimit; boolean limitSet = (limitParam != Integer.valueOf(LIMIT_OFFSET_DEFAULT)); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java index 1ee13fb..e615a5b 100755 --- a/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java +++ b/webapp/src/main/java/org/apache/atlas/web/service/EmbeddedServer.java @@ -18,18 +18,20 @@ package org.apache.atlas.web.service; -import org.apache.atlas.ApplicationProperties; -import org.apache.commons.configuration.Configuration; +import org.apache.atlas.AtlasConfiguration; import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.util.thread.ExecutorThreadPool; import org.eclipse.jetty.webapp.WebAppContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; /** * This class embeds a Jetty server and a connector. @@ -37,11 +39,19 @@ import java.io.IOException; public class EmbeddedServer { public static final Logger LOG = LoggerFactory.getLogger(EmbeddedServer.class); - private static final int DEFAULT_BUFFER_SIZE = 16192; - - protected final Server server = new Server(); + protected final Server server; public EmbeddedServer(int port, String path) throws IOException { + int queueSize = AtlasConfiguration.WEBSERVER_QUEUE_SIZE.getInt(); + LinkedBlockingQueue<Runnable> queue = new LinkedBlockingQueue<>(queueSize); + + int minThreads = AtlasConfiguration.WEBSERVER_MIN_THREADS.getInt(); + int maxThreads = AtlasConfiguration.WEBSERVER_MAX_THREADS.getInt(); + long keepAliveTime = AtlasConfiguration.WEBSERVER_KEEPALIVE_SECONDS.getLong(); + ExecutorThreadPool pool = + new ExecutorThreadPool(minThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, queue); + server = new Server(pool); + Connector connector = getConnector(port); server.addConnector(connector); @@ -66,31 +76,18 @@ public class EmbeddedServer { } protected Connector getConnector(int port) throws IOException { - HttpConfiguration http_config = new HttpConfiguration(); // this is to enable large header sizes when Kerberos is enabled with AD - final int bufferSize = getBufferSize(); + final int bufferSize = AtlasConfiguration.WEBSERVER_REQUEST_BUFFER_SIZE.getInt();; http_config.setResponseHeaderSize(bufferSize); http_config.setRequestHeaderSize(bufferSize); ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory(http_config)); connector.setPort(port); connector.setHost("0.0.0.0"); - server.addConnector(connector); return connector; } - protected Integer getBufferSize() { - try { - Configuration configuration = ApplicationProperties.get(); - return configuration.getInt("atlas.jetty.request.buffer.size", DEFAULT_BUFFER_SIZE); - } catch (Exception e) { - // do nothing - } - - return DEFAULT_BUFFER_SIZE; - } - public void start() throws Exception { server.start(); server.join(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/43a1e25b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java index b433e71..a6af5a2 100755 --- a/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java +++ b/webapp/src/main/java/org/apache/atlas/web/service/SecureEmbeddedServer.java @@ -19,10 +19,12 @@ package org.apache.atlas.web.service; import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; @@ -30,24 +32,23 @@ import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.util.ssl.SslContextFactory; -import org.eclipse.jetty.http.HttpVersion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; +import static org.apache.atlas.security.SecurityProperties.ATLAS_SSL_EXCLUDE_CIPHER_SUITES; import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH; import static org.apache.atlas.security.SecurityProperties.CLIENT_AUTH_KEY; import static org.apache.atlas.security.SecurityProperties.DEFATULT_TRUSTORE_FILE_LOCATION; +import static org.apache.atlas.security.SecurityProperties.DEFAULT_CIPHER_SUITES; import static org.apache.atlas.security.SecurityProperties.DEFAULT_KEYSTORE_FILE_LOCATION; import static org.apache.atlas.security.SecurityProperties.KEYSTORE_FILE_KEY; import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY; import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY; import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_FILE_KEY; import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY; -import static org.apache.atlas.security.SecurityProperties.ATLAS_SSL_EXCLUDE_CIPHER_SUITES; -import static org.apache.atlas.security.SecurityProperties.DEFAULT_CIPHER_SUITES; /** * This is a jetty server which requires client auth via certificates. @@ -81,7 +82,7 @@ public class SecureEmbeddedServer extends EmbeddedServer { // HTTP Configuration HttpConfiguration http_config = new HttpConfiguration(); http_config.setSecureScheme("https"); - final int bufferSize = getBufferSize(); + final int bufferSize = AtlasConfiguration.WEBSERVER_REQUEST_BUFFER_SIZE.getInt(); http_config.setSecurePort(port); http_config.setRequestHeaderSize(bufferSize); http_config.setResponseHeaderSize(bufferSize);
