Repository: incubator-atlas Updated Branches: refs/heads/master 42d5e37e3 -> 208e90b1a
ATLAS-833 Make default build profile for External HBase and Solr (tbeerbower via yhemanth) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/208e90b1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/208e90b1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/208e90b1 Branch: refs/heads/master Commit: 208e90b1ae33074aeb89c79e48a0a3489d5cc415 Parents: 42d5e37 Author: Hemanth Yamijala <[email protected]> Authored: Tue May 31 23:03:14 2016 +0530 Committer: Hemanth Yamijala <[email protected]> Committed: Tue May 31 23:03:14 2016 +0530 ---------------------------------------------------------------------- distro/pom.xml | 81 ++++++++++++++++++++--- distro/src/conf/atlas-application.properties | 26 +------- distro/src/conf/atlas-env.sh | 3 + docs/src/site/twiki/Configuration.twiki | 12 ++-- docs/src/site/twiki/InstallationSteps.twiki | 22 ++++-- release-log.txt | 1 + 6 files changed, 103 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/208e90b1/distro/pom.xml ---------------------------------------------------------------------- diff --git a/distro/pom.xml b/distro/pom.xml index 22f70ae..131a488 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -28,9 +28,28 @@ <artifactId>atlas-distro</artifactId> <description>Apache Atlas Distribution</description> <name>Apache Atlas Distribution</name> + + <!-- by default configure hbase and solr with the distribution --> <properties> - <titan.storage.backend>berkeleyje</titan.storage.backend> - <titan.index.backend>elasticsearch</titan.index.backend> + + <titan.storage.backend>hbase</titan.storage.backend> + <titan.storage.properties>#Hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 +atlas.graph.storage.hostname= +atlas.graph.storage.hbase.regions-per-server=1 +atlas.graph.storage.lock.wait-time=10000 + </titan.storage.properties> + <titan.index.backend>solr5</titan.index.backend> + <titan.index.properties>#Solr +#Solr cloud mode properties +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url= + +#Solr http mode properties +#atlas.graph.index.search.solr.mode=http +#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr + </titan.index.properties> <hbase.embedded>false</hbase.embedded> <solr.embedded>false</solr.embedded> </properties> @@ -93,15 +112,59 @@ </plugins> </build> </profile> - <!-- embedded_services profile to package and configure embedded hbase and solr with the distribution --> + <!-- profile to configure berkeley and elasticsearch with the distribution --> + <profile> + <id>berkeley-elasticsearch</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <properties> + <titan.storage.backend>berkeleyje</titan.storage.backend> + <titan.storage.properties>#Berkeley +atlas.graph.storage.directory=${sys:atlas.home}/data/berkley + </titan.storage.properties> + <titan.index.backend>elasticsearch</titan.index.backend> + <titan.index.properties>#ElasticSearch +atlas.graph.index.search.directory=${sys:atlas.home}/data/es +atlas.graph.index.search.elasticsearch.client-only=false +atlas.graph.index.search.elasticsearch.local-mode=true +atlas.graph.index.search.elasticsearch.create.sleep=2000 + </titan.index.properties> + </properties> + </profile> + + <!-- profile to configure berkeley and elasticsearch with the distribution --> <profile> - <id>embedded_services</id> + <id>external-hbase-solr</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + </profile> + + <!-- profile to package and configure embedded hbase and solr with the distribution --> + <profile> + <id>embedded-hbase-solr</id> <activation> <activeByDefault>false</activeByDefault> </activation> <properties> - <titan.storage.backend>hbase</titan.storage.backend> - <titan.index.backend>solr5</titan.index.backend> + <titan.storage.properties>#Hbase +#For standalone mode , specify localhost +#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 +atlas.graph.storage.hostname=localhost +atlas.graph.storage.hbase.regions-per-server=1 +atlas.graph.storage.lock.wait-time=10000 + </titan.storage.properties> + <titan.index.properties>#Solr +#Solr cloud mode properties +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url=localhost:2181 + +#Solr http mode properties +#atlas.graph.index.search.solr.mode=http +#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr + </titan.index.properties> + <hbase.embedded>true</hbase.embedded> <solr.embedded>true</solr.embedded> @@ -193,9 +256,9 @@ <resource> <directory>src/bin</directory> <filtering>true</filtering> - <includes> - <include>**/*.py</include> - </includes> + <includes> + <include>**/*.py</include> + </includes> <targetPath>bin</targetPath> </resource> <resource> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/208e90b1/distro/src/conf/atlas-application.properties ---------------------------------------------------------------------- diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties index bb12538..bfa40e8 100755 --- a/distro/src/conf/atlas-application.properties +++ b/distro/src/conf/atlas-application.properties @@ -19,33 +19,13 @@ ######### Graph Database Configs ######### # Graph Storage atlas.graph.storage.backend=${titan.storage.backend} -atlas.graph.storage.directory=${sys:atlas.home}/data/berkley -#Hbase as storage backend -#atlas.graph.storage.backend=hbase -#For standalone mode , specify localhost -#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2 -atlas.graph.storage.hostname=localhost -atlas.graph.storage.hbase.regions-per-server=1 -atlas.graph.storage.lock.wait-time=10000 +${titan.storage.properties} -#Solr +# Graph Search Index atlas.graph.index.search.backend=${titan.index.backend} -# Solr cloud mode properties -atlas.graph.index.search.solr.mode=cloud -atlas.graph.index.search.solr.zookeeper-url=localhost:2181 - -#Solr http mode properties -#atlas.graph.index.search.solr.mode=http -#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr - -# Graph Search Index -#ElasticSearch -atlas.graph.index.search.directory=${sys:atlas.home}/data/es -atlas.graph.index.search.elasticsearch.client-only=false -atlas.graph.index.search.elasticsearch.local-mode=true -atlas.graph.index.search.elasticsearch.create.sleep=2000 +${titan.index.properties} ######### Notification Configs ######### http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/208e90b1/distro/src/conf/atlas-env.sh ---------------------------------------------------------------------- diff --git a/distro/src/conf/atlas-env.sh b/distro/src/conf/atlas-env.sh index c671c45..6a5d5e4 100644 --- a/distro/src/conf/atlas-env.sh +++ b/distro/src/conf/atlas-env.sh @@ -57,5 +57,8 @@ # Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir. #export ATLAS_EXPANDED_WEBAPP_DIR= +# indicates whether or not a local instance of HBase should be started for Atlas export MANAGE_LOCAL_HBASE=${hbase.embedded} + +# indicates whether or not a local instance of Solr should be started for Atlas export MANAGE_LOCAL_SOLR=${solr.embedded} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/208e90b1/docs/src/site/twiki/Configuration.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/Configuration.twiki b/docs/src/site/twiki/Configuration.twiki index f532a73..2884f42 100644 --- a/docs/src/site/twiki/Configuration.twiki +++ b/docs/src/site/twiki/Configuration.twiki @@ -53,10 +53,10 @@ Without Ranger, HBase shell can be used to set the permissions. echo "grant 'atlas', 'RWXCA', 'titan'" | hbase shell </verbatim> -Note that if the embedded_services profile is used then HBase is included in the distribution so that a standalone -instance of HBase can be started as the default storage backend for the graph repository. Using the embedded_services +Note that if the embedded-hbase-solr profile is used then HBase is included in the distribution so that a standalone +instance of HBase can be started as the default storage backend for the graph repository. Using the embedded-hbase-solr profile will configure Atlas so that HBase instance will be started and stopped along with the Atlas server by default. -To use the embedded_services profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]] +To use the embedded-hbase-solr profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]] section. ---+++ Graph Search Index @@ -80,10 +80,10 @@ Please note that Solr installation in Cloud mode is a prerequisite before config atlas.graph.index.search.solr.zookeeper-url=<the ZK quorum setup for solr as comma separated value> eg: 10.1.6.4:2181,10.1.6.5:2181 </verbatim> -Also note that if the embedded_services profile is used then Solr is included in the distribution so that a standalone -instance of Solr can be started as the default search indexing backend. Using the embedded_services profile will +Also note that if the embedded-hbase-solr profile is used then Solr is included in the distribution so that a standalone +instance of Solr can be started as the default search indexing backend. Using the embedded-hbase-solr profile will configure Atlas so that the standalone Solr instance will be started and stopped along with the Atlas server by default. -To use the embedded_services profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]] +To use the embedded-hbase-solr profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]] section. ---+++ Choosing between Persistence and Indexing Backends http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/208e90b1/docs/src/site/twiki/InstallationSteps.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki index fe6b4a7..9699a94 100644 --- a/docs/src/site/twiki/InstallationSteps.twiki +++ b/docs/src/site/twiki/InstallationSteps.twiki @@ -18,15 +18,29 @@ mvn clean package -Pdist </verbatim> -To build a distribution that packages HBase and Solr, build with the embedded_services profile. +To build a distribution that configures Atlas for external HBase and Solr, build with the external-hbase-solr profile. <verbatim> -mvn clean package -Pdist,embedded_services +mvn clean package -Pdist,external-hbase-solr </verbatim> -Using the embedded_services profile will configure Atlas so that an HBase instance and a Solr instance will be started +Note that when the external-hbase-solr profile is used the following steps need to be completed to make Atlas functional. + * Configure atlas.graph.storage.hostname (see "Graph persistence engine - HBase" in the [[Configuration][Configuration]] section). + * Configure atlas.graph.index.search.solr.zookeeper-url (see "Graph Search Index - Solr" in the [[Configuration][Configuration]] section). + * Set HBASE_CONF_DIR to point to a valid HBase config directory (see "Graph persistence engine - HBase" in the [[Configuration][Configuration]] section). + * Create the SOLR indices (see "Graph Search Index - Solr" in the [[Configuration][Configuration]] section). + +To build a distribution that packages HBase and Solr, build with the embedded-hbase-solr profile. + +<verbatim> + +mvn clean package -Pdist,embedded-hbase-solr + +</verbatim> + +Using the embedded-hbase-solr profile will configure Atlas so that an HBase instance and a Solr instance will be started and stopped along with the Atlas server by default. Tar can be found in atlas/distro/target/apache-atlas-${project.version}-bin.tar.gz @@ -75,7 +89,7 @@ Tar is structured as follows </verbatim> -Note that if the embedded_services profile is specified for the build then HBase and Solr are included in the +Note that if the embedded-hbase-solr profile is specified for the build then HBase and Solr are included in the distribution. In this case, a standalone instance of HBase can be started as the default storage backend for the graph repository. http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/208e90b1/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 4dbe0d4..7f25604 100644 --- a/release-log.txt +++ b/release-log.txt @@ -22,6 +22,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-833 Make default build profile for External HBase and Solr (tbeerbower via yhemanth) ATLAS-841 mvn clean install of Atlas is failing python unit tests (yhemanth) ATLAS-834 Handle exceptions from HiveHook executor.submit() (sumasai) ATLAS-635 Process showing old entity name where as actual entity is renamed ( svimal2106 via sumasai )
