Repository: incubator-atlas Updated Branches: refs/heads/master 98769871e -> af63bb887
ATLAS-663,ATLAS-673 Install Setup: SOLR (tbeerbower via sumasai) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/af63bb88 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/af63bb88 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/af63bb88 Branch: refs/heads/master Commit: af63bb887e817a64d55f75a37e815cecc4fe0c26 Parents: 9876987 Author: Suma Shivaprasad <[email protected]> Authored: Thu May 12 22:56:30 2016 -0700 Committer: Suma Shivaprasad <[email protected]> Committed: Thu May 12 22:56:30 2016 -0700 ---------------------------------------------------------------------- .gitignore | 4 + distro/pom.xml | 38 ++++++++- distro/src/bin/atlas_config.py | 90 +++++++++++++++++++- distro/src/bin/atlas_start.py | 16 +++- distro/src/bin/atlas_stop.py | 4 + distro/src/conf/atlas-application.properties | 16 ++-- distro/src/conf/hbase/hbase-site.xml.template | 4 +- .../src/main/assemblies/standalone-package.xml | 7 ++ distro/src/test/python/scripts/TestMetadata.py | 23 ++++- docs/src/site/twiki/Configuration.twiki | 11 ++- docs/src/site/twiki/InstallationSteps.twiki | 18 +++- release-log.txt | 1 + 12 files changed, 210 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 20db282..f5899e2 100755 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,10 @@ test-output #hbase package downloaded distro/hbase/*.tar.gz + +#solr package downloaded +distro/solr/*.tgz + .cache-main # emacs files http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/pom.xml ---------------------------------------------------------------------- diff --git a/distro/pom.xml b/distro/pom.xml index ee9add0..1ab092d 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -31,8 +31,12 @@ <properties> <hbase.dir>${project.build.directory}/hbase</hbase.dir> - <hbase.tar>http://apache.mirrors.pair.com/hbase/stable/hbase-1.1.4-bin.tar.gz</hbase.tar> - <hbase.folder>hbase-1.1.4</hbase.folder> + <hbase.tar>http://archive.apache.org/dist/hbase/${hbase.version}/hbase-${hbase.version}-bin.tar.gz</hbase.tar> + <hbase.folder>hbase-${hbase.version}</hbase.folder> + + <solr.dir>${project.build.directory}/solr</solr.dir> + <solr.tar>http://archive.apache.org/dist/lucene/solr/${solr.version}/solr-${solr.version}.tgz</solr.tar> + <solr.folder>solr-${solr.version}</solr.folder> </properties> <profiles> @@ -77,6 +81,7 @@ <version>1.7</version> <executions> <execution> + <id>hbase</id> <phase>generate-resources</phase> <goals> <goal>run</goal> @@ -104,6 +109,35 @@ </target> </configuration> </execution> + <execution> + <id>solr</id> + <phase>generate-resources</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target name="Download SOLR"> + <mkdir dir="${solr.dir}"/> + <mkdir dir="${project.basedir}/solr"/> + <get + src="${solr.tar}" + dest="${project.basedir}/solr/${solr.folder}.tgz" + usetimestamp="true" + verbose="true" skipexisting="true" + /> + <untar + src="${project.basedir}/solr/${solr.folder}.tgz" + dest="${project.build.directory}/solr.temp" + compression="gzip" + /> + <copy todir="${solr.dir}"> + <fileset dir="${project.build.directory}/solr.temp/${solr.folder}"> + <include name="**/*"/> + </fileset> + </copy> + </target> + </configuration> + </execution> </executions> </plugin> <plugin> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/src/bin/atlas_config.py ---------------------------------------------------------------------- diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py index dbe1191..8f3479c 100755 --- a/distro/src/bin/atlas_config.py +++ b/distro/src/bin/atlas_config.py @@ -31,6 +31,7 @@ LIB = "lib" CONF = "conf" LOG = "logs" WEBAPP = "server" + os.sep + "webapp" +CONFIG_SETS_CONF = "server" + os.sep + "solr" + os.sep + "configsets" + os.sep + "basic_configs" + os.sep + "conf" DATA = "data" ATLAS_CONF = "ATLAS_CONF" ATLAS_LOG = "ATLAS_LOG_DIR" @@ -42,13 +43,28 @@ ATLAS_SERVER_HEAP = "ATLAS_SERVER_HEAP" ATLAS_DATA = "ATLAS_DATA_DIR" ATLAS_HOME = "ATLAS_HOME_DIR" HBASE_CONF_DIR = "HBASE_CONF_DIR" +MANAGE_LOCAL_HBASE = "MANAGE_LOCAL_HBASE" +MANAGE_LOCAL_SOLR = "MANAGE_LOCAL_SOLR" +SOLR_BIN = "SOLR_BIN" +SOLR_CONF = "SOLR_CONF" +SOLR_PORT = "SOLR_PORT" +DEFAULT_SOLR_PORT = "9838" +SOLR_SHARDS = "SOLR_SHARDS" +DEFAULT_SOLR_SHARDS = "1" +SOLR_REPLICATION_FACTOR = "SOLR_REPLICATION_FACTOR" +DEFAULT_SOLR_REPLICATION_FACTOR = "1" + ENV_KEYS = ["JAVA_HOME", ATLAS_OPTS, ATLAS_SERVER_OPTS, ATLAS_SERVER_HEAP, ATLAS_LOG, ATLAS_PID, ATLAS_CONF, - "ATLASCPPATH", ATLAS_DATA, ATLAS_HOME, ATLAS_WEBAPP, HBASE_CONF_DIR] + "ATLASCPPATH", ATLAS_DATA, ATLAS_HOME, ATLAS_WEBAPP, HBASE_CONF_DIR, SOLR_PORT] IS_WINDOWS = platform.system() == "Windows" ON_POSIX = 'posix' in sys.builtin_module_names CONF_FILE="atlas-application.properties" HBASE_STORAGE_CONF_ENTRY="atlas.graph.storage.backend\s*=\s*hbase" HBASE_STORAGE_LOCAL_CONF_ENTRY="atlas.graph.storage.hostname\s*=\s*localhost" +SOLR_INDEX_CONF_ENTRY="atlas.graph.index.search.backend\s*=\s*solr5" +SOLR_INDEX_LOCAL_CONF_ENTRY="atlas.graph.index.search.solr.zookeeper-url\s*=\s*localhost" +SOLR_INDEX_ZK_URL="atlas.graph.index.search.solr.zookeeper-url" + DEBUG = False def scriptDir(): @@ -74,6 +90,21 @@ def hbaseBinDir(dir): def hbaseConfDir(dir): return os.environ.get(HBASE_CONF_DIR, os.path.join(dir, "hbase", CONF)) +def solrBinDir(dir): + return os.environ.get(SOLR_BIN, os.path.join(dir, "solr", BIN)) + +def solrConfDir(dir): + return os.environ.get(SOLR_CONF, os.path.join(dir, "solr", CONFIG_SETS_CONF)) + +def solrPort(): + return os.environ.get(SOLR_PORT, DEFAULT_SOLR_PORT) + +def solrShards(): + return os.environ.get(SOLR_SHARDS, DEFAULT_SOLR_SHARDS) + +def solrReplicationFactor(): + return os.environ.get(SOLR_REPLICATION_FACTOR, DEFAULT_SOLR_REPLICATION_FACTOR) + def logDir(dir): localLog = os.path.join(dir, LOG) return os.environ.get(ATLAS_LOG, localLog) @@ -357,6 +388,9 @@ def is_hbase(confdir): return grep(confdir, HBASE_STORAGE_CONF_ENTRY) is not None def is_hbase_local(confdir): + if os.environ.get(MANAGE_LOCAL_HBASE, "True").lower() == 'false': + return False + confdir = os.path.join(confdir, CONF_FILE) return grep(confdir, HBASE_STORAGE_CONF_ENTRY) is not None and grep(confdir, HBASE_STORAGE_LOCAL_CONF_ENTRY) is not None @@ -380,10 +414,56 @@ def run_hbase_action(dir, action, hbase_conf_dir = None, logdir = None, wait=Tru return runProcess(cmd, logdir, False, wait) +def is_solr(confdir): + confdir = os.path.join(confdir, CONF_FILE) + return grep(confdir, SOLR_INDEX_CONF_ENTRY) is not None + +def is_solr_local(confdir): + if os.environ.get(MANAGE_LOCAL_SOLR, "True").lower() == 'false': + return False + + confdir = os.path.join(confdir, CONF_FILE) + return grep(confdir, SOLR_INDEX_CONF_ENTRY) is not None and grep(confdir, SOLR_INDEX_LOCAL_CONF_ENTRY) is not None + +def get_solr_zk_url(confdir): + confdir = os.path.join(confdir, CONF_FILE) + return getConfig(confdir, SOLR_INDEX_ZK_URL) + +def run_solr(dir, action, zk_url = None, port = None, logdir = None, wait=True): + + solrScript = "solr" + + if IS_WINDOWS: + solrScript = "solr.cmd" + + if zk_url is None: + if port is None: + cmd = [os.path.join(dir, solrScript), action] + else: + cmd = [os.path.join(dir, solrScript), action, '-p', str(port)] + else: + if port is None: + cmd = [os.path.join(dir, solrScript), action, '-z', zk_url] + else: + cmd = [os.path.join(dir, solrScript), action, '-z', zk_url, '-p', port] + + return runProcess(cmd, logdir, False, wait) + +def create_solr_collection(dir, confdir, index, logdir = None, wait=True): + solrScript = "solr" + + if IS_WINDOWS: + solrScript = "solr.cmd" + + cmd = [os.path.join(dir, solrScript), 'create', '-c', index, '-d', confdir, '-shards', solrShards(), '-replicationFactor', solrReplicationFactor()] + + return runProcess(cmd, logdir, False, wait) + def configure_hbase(dir): env_conf_dir = os.environ.get(HBASE_CONF_DIR) conf_dir = os.path.join(dir, "hbase", CONF) tmpl_dir = os.path.join(dir, CONF, "hbase") + data_dir = dataDir(atlasDir()) if env_conf_dir is None or env_conf_dir == conf_dir: hbase_conf_file = "hbase-site.xml" @@ -403,6 +483,7 @@ def configure_hbase(dir): f.close() config = template.replace("${hbase_home}", dir) + config = config.replace("${atlas_data}", data_dir) config = config.replace("${url_prefix}", url_prefix) f = open(conf_file,'w') @@ -423,6 +504,13 @@ def grep(file, value): return line return None +def getConfig(file, key): + key = key + "\s*=" + for line in open(file).readlines(): + if re.match(key, line): + return line.split('=')[1].strip() + return None + def isCygwin(): return platform.system().startswith("CYGWIN") http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/src/bin/atlas_start.py ---------------------------------------------------------------------- diff --git a/distro/src/bin/atlas_start.py b/distro/src/bin/atlas_start.py index 73b8920..cb869bb 100755 --- a/distro/src/bin/atlas_start.py +++ b/distro/src/bin/atlas_start.py @@ -35,6 +35,7 @@ def main(): confdir = mc.dirMustExist(mc.confDir(atlas_home)) mc.executeEnvSh(confdir) logdir = mc.dirMustExist(mc.logDir(atlas_home)) + mc.dirMustExist(mc.dataDir(atlas_home)) if mc.isCygwin(): # Pathnames that are passed to JVM must be converted to Windows format. jvm_atlas_home = mc.convertCygwinPath(atlas_home) @@ -86,7 +87,7 @@ def main(): else: if mc.is_hbase(confdir): raise Exception("Could not find hbase-site.xml in %s. Please set env var HBASE_CONF_DIR to the hbase client conf dir", hbase_conf_dir) - + if mc.isCygwin(): atlas_classpath = mc.convertCygwinPath(atlas_classpath, True) @@ -111,6 +112,19 @@ def main(): mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "start", hbase_conf_dir, logdir) print "hbase started." + #solr setup + if mc.is_solr_local(confdir): + print "configured for local solr." + mc.run_solr(mc.solrBinDir(atlas_home), "start", mc.get_solr_zk_url(confdir), mc.solrPort(), logdir) + print "solr started." + + #solr indexes + if mc.is_solr(confdir): + print "setting up solr collections..." + mc.create_solr_collection(mc.solrBinDir(atlas_home), mc.solrConfDir(atlas_home), "vertex_index", logdir) + mc.create_solr_collection(mc.solrBinDir(atlas_home), mc.solrConfDir(atlas_home), "edge_index", logdir) + mc.create_solr_collection(mc.solrBinDir(atlas_home), mc.solrConfDir(atlas_home), "fulltext_index", logdir) + web_app_path = os.path.join(web_app_dir, "atlas") if (mc.isCygwin()): web_app_path = mc.convertCygwinPath(web_app_path) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/src/bin/atlas_stop.py ---------------------------------------------------------------------- diff --git a/distro/src/bin/atlas_stop.py b/distro/src/bin/atlas_stop.py index 265084e..a72c724 100755 --- a/distro/src/bin/atlas_stop.py +++ b/distro/src/bin/atlas_stop.py @@ -54,6 +54,10 @@ def main(): if os.path.exists(atlas_pid_file): os.remove(atlas_pid_file) + # stop solr + if mc.is_solr_local(confdir): + mc.run_solr(mc.solrBinDir(atlas_home), "stop", None, mc.solrPort(), None, True) + # stop hbase if mc.is_hbase_local(confdir): mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "stop", None, None, True) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/src/conf/atlas-application.properties ---------------------------------------------------------------------- diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties index b2b62aa..68a0021 100755 --- a/distro/src/conf/atlas-application.properties +++ b/distro/src/conf/atlas-application.properties @@ -30,11 +30,11 @@ atlas.graph.storage.hbase.regions-per-server=1 atlas.graph.storage.lock.wait-time=10000 #Solr -#atlas.graph.index.search.backend=solr +atlas.graph.index.search.backend=solr5 # Solr cloud mode properties -#atlas.graph.index.search.solr.mode=cloud -#atlas.graph.index.search.solr.zookeeper-url=localhost:2181 +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.zookeeper-url=localhost:2181 #Solr http mode properties #atlas.graph.index.search.solr.mode=http @@ -42,11 +42,11 @@ atlas.graph.storage.lock.wait-time=10000 # Graph Search Index #ElasticSearch -atlas.graph.index.search.backend=elasticsearch -atlas.graph.index.search.directory=${sys:atlas.home}/data/es -atlas.graph.index.search.elasticsearch.client-only=false -atlas.graph.index.search.elasticsearch.local-mode=true -atlas.graph.index.search.elasticsearch.create.sleep=2000 +#atlas.graph.index.search.backend=elasticsearch +#atlas.graph.index.search.directory=${sys:atlas.home}/data/es +#atlas.graph.index.search.elasticsearch.client-only=false +#atlas.graph.index.search.elasticsearch.local-mode=true +#atlas.graph.index.search.elasticsearch.create.sleep=2000 ######### Notification Configs ######### http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/src/conf/hbase/hbase-site.xml.template ---------------------------------------------------------------------- diff --git a/distro/src/conf/hbase/hbase-site.xml.template b/distro/src/conf/hbase/hbase-site.xml.template index f5c4aad..d599c20 100644 --- a/distro/src/conf/hbase/hbase-site.xml.template +++ b/distro/src/conf/hbase/hbase-site.xml.template @@ -19,11 +19,11 @@ <configuration> <property> <name>hbase.rootdir</name> - <value>${url_prefix}${hbase_home}/root</value> + <value>${url_prefix}${atlas_data}/hbase-root</value> </property> <property> <name>hbase.zookeeper.property.dataDir</name> - <value>${hbase_home}/zookeeper-data</value> + <value>${atlas_data}/hbase-zookeeper-data</value> </property> <property> <name>hbase.master.info.port</name> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/src/main/assemblies/standalone-package.xml ---------------------------------------------------------------------- diff --git a/distro/src/main/assemblies/standalone-package.xml b/distro/src/main/assemblies/standalone-package.xml index 527f604..09ff47d 100755 --- a/distro/src/main/assemblies/standalone-package.xml +++ b/distro/src/main/assemblies/standalone-package.xml @@ -62,6 +62,13 @@ </fileSet> <fileSet> + <directory>target/solr</directory> + <outputDirectory>solr</outputDirectory> + <fileMode>0755</fileMode> + <directoryMode>0755</directoryMode> + </fileSet> + + <fileSet> <directory>../logs</directory> <outputDirectory>logs</outputDirectory> <directoryMode>0777</directoryMode> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/src/test/python/scripts/TestMetadata.py ---------------------------------------------------------------------- diff --git a/distro/src/test/python/scripts/TestMetadata.py b/distro/src/test/python/scripts/TestMetadata.py index 9e55869..00ca3e2 100644 --- a/distro/src/test/python/scripts/TestMetadata.py +++ b/distro/src/test/python/scripts/TestMetadata.py @@ -20,6 +20,7 @@ limitations under the License. import sys from os import environ from mock import patch +from mock import call import unittest import logging import atlas_config as mc @@ -32,6 +33,7 @@ logger = logging.getLogger() class TestMetadata(unittest.TestCase): @patch.object(mc,"runProcess") @patch.object(mc,"configure_hbase") + @patch.object(mc,"getConfig") @patch.object(mc,"grep") @patch.object(mc,"exist_pid") @patch.object(mc,"writePid") @@ -41,7 +43,7 @@ class TestMetadata(unittest.TestCase): @patch("os.path.exists") @patch.object(mc, "java") - def test_main(self, java_mock, exists_mock, expandWebApp_mock, atlasDir_mock, executeEnvSh_mock, writePid_mock, exist_pid_mock, grep_mock, configure_hbase_mock, runProcess_mock): + def test_main(self, java_mock, exists_mock, expandWebApp_mock, atlasDir_mock, executeEnvSh_mock, writePid_mock, exist_pid_mock, grep_mock, getConfig_mock, configure_hbase_mock, runProcess_mock): sys.argv = [] exists_mock.return_value = True expandWebApp_mock.return_value = "webapp" @@ -50,13 +52,28 @@ class TestMetadata(unittest.TestCase): exist_pid_mock(789) exist_pid_mock.assert_called_with(789) grep_mock.return_value = "hbase" + getConfig_mock.return_value = "localhost:9838" atlas.main() self.assertTrue(configure_hbase_mock.called) + if IS_WINDOWS: - runProcess_mock.assert_called_with(['atlas_home\\hbase\\bin\\start-hbase.cmd', '--config', 'atlas_home\\hbase\\conf'], 'atlas_home\\logs', False, True) + calls = [call(['atlas_home\\hbase\\bin\\start-hbase.cmd', '--config', 'atlas_home\\hbase\\conf', 'start', 'master'], 'atlas_home\\logs', False, True), + call(['atlas_home\\solr\\bin\\solr.cmd', 'start', '-z', 'localhost:9838', '-p', '9838'], 'atlas_home\\logs', False, True), + call(['atlas_home\\solr\\bin\\solr.cmd', 'create', '-c', 'vertex_index', '-d', 'atlas_home\\solr\\server\\solr\\configsets\\basic_configs\\conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home\\logs', False, True), + call(['atlas_home\\solr\\bin\\solr.cmd', 'create', '-c', 'edge_index', '-d', 'atlas_home\\solr\\server\\solr\\configsets\\basic_configs\\conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home\\logs', False, True), + call(['atlas_home\\solr\\bin\\solr.cmd', 'create', '-c', 'fulltext_index', '-d', 'atlas_home\\solr\\server\\solr\\configsets\\basic_configs\\conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home\\logs', False, True)] + + runProcess_mock.assert_has_calls(calls) else: - runProcess_mock.assert_called_with(['atlas_home/hbase/bin/hbase-daemon.sh', '--config', 'atlas_home/hbase/conf', 'start', 'master'], 'atlas_home/logs', False, True) + calls = [call(['atlas_home/hbase/bin/hbase-daemon.sh', '--config', 'atlas_home/hbase/conf', 'start', 'master'], 'atlas_home/logs', False, True), + call(['atlas_home/solr/bin/solr', 'start', '-z', 'localhost:9838', '-p', '9838'], 'atlas_home/logs', False, True), + call(['atlas_home/solr/bin/solr', 'create', '-c', 'vertex_index', '-d', 'atlas_home/solr/server/solr/configsets/basic_configs/conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home/logs', False, True), + call(['atlas_home/solr/bin/solr', 'create', '-c', 'edge_index', '-d', 'atlas_home/solr/server/solr/configsets/basic_configs/conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home/logs', False, True), + call(['atlas_home/solr/bin/solr', 'create', '-c', 'fulltext_index', '-d', 'atlas_home/solr/server/solr/configsets/basic_configs/conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home/logs', False, True)] + + runProcess_mock.assert_has_calls(calls) + self.assertTrue(java_mock.called) if IS_WINDOWS: http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/docs/src/site/twiki/Configuration.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/Configuration.twiki b/docs/src/site/twiki/Configuration.twiki index 0793c82..879c06c 100644 --- a/docs/src/site/twiki/Configuration.twiki +++ b/docs/src/site/twiki/Configuration.twiki @@ -54,7 +54,10 @@ Without Ranger, HBase shell can be used to set the permissions. </verbatim> Note that HBase is included in the distribution so that a standalone instance of HBase can be started as the default -storage backend for the graph repository. +storage backend for the graph repository. By default, the standalone HBase instance will be started and stopped along +with the Atlas server when the configuraton 'atlas.graph.storage.backend' is set to 'hbase' and +'atlas.graph.storage.hostname' is set to 'localhost'. Set the MANAGE_LOCAL_HBASE environment variable to 'False' to +override this behavior. ---+++ Graph Search Index This section sets up the graph db - titan - to use an search indexing system. The example @@ -77,6 +80,12 @@ Please note that Solr installation in Cloud mode is a prerequisite before config atlas.graph.index.search.solr.zookeeper-url=<the ZK quorum setup for solr as comma separated value> eg: 10.1.6.4:2181,10.1.6.5:2181 </verbatim> +Also note that Solr is included in the distribution so that a standalone instance of Solr can be started as the default +search indexing backend. By default, the standalone Solr instance will be started and stopped along +with the Atlas server when the configuraton 'atlas.graph.index.search.backend' is set to 'solr5' and +'atlas.graph.index.search.solr.zookeeper-url' is set to 'localhost'. Set the MANAGE_LOCAL_SOLR environment variable to +'False' to override this behavior. + ---+++ Choosing between Persistence and Indexing Backends Refer http://s3.thinkaurelius.com/docs/titan/0.5.4/bdb.html and http://s3.thinkaurelius.com/docs/titan/0.5.4/hbase.html for choosing between the persistence backends. http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/docs/src/site/twiki/InstallationSteps.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki index 270d1c4..006e1e8 100644 --- a/docs/src/site/twiki/InstallationSteps.twiki +++ b/docs/src/site/twiki/InstallationSteps.twiki @@ -53,6 +53,9 @@ Tar is structured as follows |- server |- webapp |- atlas.war +|- solr + |- bin + ... |- README |- NOTICE.txt |- LICENSE.txt @@ -67,6 +70,10 @@ expanded and moved to hbase/conf/hbase-site.xml for the initial standalone HBase graph persistence for a different HBase instance, please see "Graph persistence engine - HBase" in the [[Configuration][Configuration]] section. +Also note that Solr is included in the distribution so that a standalone instance of Solr can be started as the default +search indexing backend. To configure ATLAS search indexing for a different Solr instance, please see +"Graph Search Index - Solr" in the [[Configuration][Configuration]] section. + ---+++ Installing & Running Atlas ---++++ Installing Atlas @@ -184,14 +191,14 @@ For configuring Titan to work with Solr, please follow the instructions below $SOLR_HOME/bin/solr start -c -z <zookeeper_host:port> -p 8983 </verbatim> - * Run the following commands from SOLR_HOME directory to create collections in Solr corresponding to the indexes that Atlas uses. In the case that the ATLAS and SOLR instance are on 2 different hosts, + * Run the following commands from SOLR_BIN (e.g. $SOLR_HOME/bin) directory to create collections in Solr corresponding to the indexes that Atlas uses. In the case that the ATLAS and SOLR instance are on 2 different hosts, first copy the required configuration files from ATLAS_HOME/conf/solr on the ATLAS instance host to the Solr instance host. SOLR_CONF in the below mentioned commands refer to the directory where the solr configuration files have been copied to on Solr host: <verbatim> - bin/solr create -c vertex_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor - bin/solr create -c edge_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor - bin/solr create -c fulltext_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor + $SOLR_BIN/solr create -c vertex_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor + $SOLR_BIN/solr create -c edge_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor + $SOLR_BIN/solr create -c fulltext_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor </verbatim> Note: If numShards and replicationFactor are not specified, they default to 1 which suffices if you are trying out solr with ATLAS on a single node instance. @@ -200,6 +207,9 @@ For configuring Titan to work with Solr, please follow the instructions below The number of replicas (replicationFactor) can be set according to the redundancy required. + Also note that solr will automatically be called to create the indexes when the Atlas server is started if the + SOLR_BIN and SOLR_CONF environment variables are set and the search indexing backend is set to 'solr5'. + * Change ATLAS configuration to point to the Solr instance setup. Please make sure the following configurations are set to the below values in ATLAS_HOME/conf/atlas-application.properties <verbatim> atlas.graph.index.search.backend=solr5 http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 5d37d07..a067fe9 100644 --- a/release-log.txt +++ b/release-log.txt @@ -20,6 +20,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-663,ATLAS-673 Install Setup: SOLR (tbeerbower via sumasai) ATLAS-629 Kafka messages in ATLAS_HOOK might be lost in HA mode at the instant of failover. (yhemanth) ATLAS-758 hdfs location of hive table is pointing to old location even after rename ( sumasai ) ATLAS-667 Entity delete should check for required reverse references ( dkantor via sumasai )
