Repository: incubator-atlas Updated Branches: refs/heads/master 25c2adf36 -> f312a577d
ATLAS-616 Resolve OOM - Zookeeper throws exceptions when trying to fire DSL queries at Atlas at large scale. (yhemanth via sumasai) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/f312a577 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/f312a577 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/f312a577 Branch: refs/heads/master Commit: f312a577df7a7bb12bc1774f10e0b92b377a53b7 Parents: 25c2adf Author: Suma Shivaprasad <[email protected]> Authored: Fri Apr 22 17:27:21 2016 -0700 Committer: Suma Shivaprasad <[email protected]> Committed: Fri Apr 22 17:27:21 2016 -0700 ---------------------------------------------------------------------- distro/src/bin/atlas_config.py | 5 ++++- distro/src/bin/atlas_start.py | 13 ++++++++++--- distro/src/conf/atlas-env.sh | 8 ++++++++ docs/src/site/twiki/InstallationSteps.twiki | 22 ++++++++++++++++++++++ release-log.txt | 1 + 5 files changed, 45 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f312a577/distro/src/bin/atlas_config.py ---------------------------------------------------------------------- diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py index 858337e..2b6bc82 100755 --- a/distro/src/bin/atlas_config.py +++ b/distro/src/bin/atlas_config.py @@ -32,15 +32,18 @@ CONF = "conf" LOG = "logs" WEBAPP = "server" + os.sep + "webapp" DATA = "data" -ENV_KEYS = ["JAVA_HOME", "ATLAS_OPTS", "ATLAS_LOG_DIR", "ATLAS_PID_DIR", "ATLAS_CONF", "ATLASCPPATH", "ATLAS_DATA_DIR", "ATLAS_HOME_DIR", "ATLAS_EXPANDED_WEBAPP_DIR", "HBASE_CONF_DIR"] ATLAS_CONF = "ATLAS_CONF" ATLAS_LOG = "ATLAS_LOG_DIR" ATLAS_PID = "ATLAS_PID_DIR" ATLAS_WEBAPP = "ATLAS_EXPANDED_WEBAPP_DIR" +ATLAS_SERVER_OPTS = "ATLAS_SERVER_OPTS" ATLAS_OPTS = "ATLAS_OPTS" +ATLAS_SERVER_HEAP = "ATLAS_SERVER_HEAP" ATLAS_DATA = "ATLAS_DATA_DIR" ATLAS_HOME = "ATLAS_HOME_DIR" HBASE_CONF_DIR = "HBASE_CONF_DIR" +ENV_KEYS = ["JAVA_HOME", ATLAS_OPTS, ATLAS_SERVER_OPTS, ATLAS_SERVER_HEAP, ATLAS_LOG, ATLAS_PID, ATLAS_CONF, + "ATLASCPPATH", ATLAS_DATA, ATLAS_HOME, ATLAS_WEBAPP, HBASE_CONF_DIR] IS_WINDOWS = platform.system() == "Windows" ON_POSIX = 'posix' in sys.builtin_module_names CONF_FILE="atlas-application.properties" http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f312a577/distro/src/bin/atlas_start.py ---------------------------------------------------------------------- diff --git a/distro/src/bin/atlas_start.py b/distro/src/bin/atlas_start.py index 3e632f4..4199e37 100755 --- a/distro/src/bin/atlas_start.py +++ b/distro/src/bin/atlas_start.py @@ -24,7 +24,8 @@ import atlas_config as mc ATLAS_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=%s.log" ATLAS_COMMAND_OPTS="-Datlas.home=%s" ATLAS_CONFIG_OPTS="-Datlas.conf=%s" -DEFAULT_JVM_OPTS="-Xmx1024m -XX:MaxPermSize=512m -Dlog4j.configuration=atlas-log4j.xml -Djava.net.preferIPv4Stack=true" +DEFAULT_JVM_HEAP_OPTS="-Xmx1024m -XX:MaxPermSize=512m" +DEFAULT_JVM_OPTS="-Dlog4j.configuration=atlas-log4j.xml -Djava.net.preferIPv4Stack=true" def main(): @@ -56,8 +57,14 @@ def main(): config_opts = (ATLAS_CONFIG_OPTS % jvm_confdir) jvm_opts_list.extend(config_opts.split()) - default_jvm_opts = DEFAULT_JVM_OPTS - atlas_jvm_opts = os.environ.get(mc.ATLAS_OPTS, default_jvm_opts) + atlas_server_heap_opts = os.environ.get(mc.ATLAS_SERVER_HEAP, DEFAULT_JVM_HEAP_OPTS) + jvm_opts_list.extend(atlas_server_heap_opts.split()) + + atlas_server_jvm_opts = os.environ.get(mc.ATLAS_SERVER_OPTS) + if atlas_server_jvm_opts: + jvm_opts_list.extend(atlas_server_jvm_opts.split()) + + atlas_jvm_opts = os.environ.get(mc.ATLAS_OPTS, DEFAULT_JVM_OPTS) jvm_opts_list.extend(atlas_jvm_opts.split()) #expand web app dir http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f312a577/distro/src/conf/atlas-env.sh ---------------------------------------------------------------------- diff --git a/distro/src/conf/atlas-env.sh b/distro/src/conf/atlas-env.sh index 7029ba1..4f1f46c 100644 --- a/distro/src/conf/atlas-env.sh +++ b/distro/src/conf/atlas-env.sh @@ -30,9 +30,17 @@ # any additional opts you want to set for atlas service. #export ATLAS_SERVER_OPTS= +# indicative values for large number of metadata entities (equal or more than 10,000s) +#export ATLAS_SERVER_OPTS="-server -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+PrintTenuringDistribution -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dumps/atlas_server.hprof -Xloggc:logs/gc-worker.log -verbose:gc -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1m -XX:+PrintGCDetails -XX:+PrintHeapAtGC -XX:+PrintGCTimeStamps" + # java heap size we want to set for the atlas server. Default is 1024MB #export ATLAS_SERVER_HEAP= +# indicative values for large number of metadata entities (equal or more than 10,000s) for JDK 7 +#export ATLAS_SERVER_HEAP="-Xms15360m -Xmx15360m -XX:MaxNewSize=3072m -XX:PermSize=100M -XX:MaxPermSize=512m" +# indicative values for large number of metadata entities (equal or more than 10,000s) for JDK 8 +#export ATLAS_SERVER_HEAP="-Xms15360m -Xmx15360m -XX:MaxNewSize=5120m -XX:MetaspaceSize=100M -XX:MaxMetaspaceSize=512m" + # What is is considered as atlas home dir. Default is the base locaion of the installed software #export ATLAS_HOME_DIR= http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f312a577/docs/src/site/twiki/InstallationSteps.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki index 952cc90..270d1c4 100644 --- a/docs/src/site/twiki/InstallationSteps.twiki +++ b/docs/src/site/twiki/InstallationSteps.twiki @@ -121,6 +121,28 @@ executed. The following environment variables are available to set. #export ATLAS_EXPANDED_WEBAPP_DIR= </verbatim> +*Settings to support large number of metadata objects* + +If you plan to store several tens of thousands of metadata objects, it is recommended that you use values +tuned for better GC performance of the JVM. + +The following values are common server side options: +<verbatim> +export ATLAS_SERVER_OPTS="-server -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+PrintTenuringDistribution -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dumps/atlas_server.hprof -Xloggc:logs/gc-worker.log -verbose:gc -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1m -XX:+PrintGCDetails -XX:+PrintHeapAtGC -XX:+PrintGCTimeStamps" +</verbatim> + +The =-XX:SoftRefLRUPolicyMSPerMB= option was found to be particularly helpful to regulate GC performance for +query heavy workloads with many concurrent users. + +The following values are recommended for JDK 7: +<verbatim> +export ATLAS_SERVER_HEAP="-Xms15360m -Xmx15360m -XX:MaxNewSize=3072m -XX:PermSize=100M -XX:MaxPermSize=512m" +</verbatim> + +The following values are recommended for JDK 8: +<verbatim> +export ATLAS_SERVER_HEAP="-Xms15360m -Xmx15360m -XX:MaxNewSize=5120m -XX:MetaspaceSize=100M -XX:MaxMetaspaceSize=512m" +</verbatim> *NOTE for Mac OS users* If you are using a Mac OS, you will need to configure the ATLAS_SERVER_OPTS (explained above). http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f312a577/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index b475bb9..f199915 100644 --- a/release-log.txt +++ b/release-log.txt @@ -17,6 +17,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-616 Resolve OOM - Zookeeper throws exceptions when trying to fire DSL queries at Atlas at large scale. (yhemanth via sumasai) ATLAS-530 Add table information to column class (sumasai) ATLAS-538 Rename table should retain traits/tags assigned to columns/storage descriptors (sumasai) ATLAS-628 Starting two Atlas instances at the same time causes exceptions in HA mode (yhemanth via sumasai)
