Repository: ambari Updated Branches: refs/heads/trunk 742f3f863 -> 36f92c70b
AMBARI-10720: [WinTP2] Create chocolatey packages for ambari artifacts and update agent bootstrap (jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/36f92c70 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/36f92c70 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/36f92c70 Branch: refs/heads/trunk Commit: 36f92c70b628ff8f0f71635f0af8edff13ceee06 Parents: 742f3f8 Author: Jayush Luniya <jlun...@hortonworks.com> Authored: Wed Apr 29 14:17:01 2015 -0700 Committer: Jayush Luniya <jlun...@hortonworks.com> Committed: Wed Apr 29 14:17:01 2015 -0700 ---------------------------------------------------------------------- ambari-agent/pom.xml | 34 ++- .../src/main/package/choco/ambari-agent.nuspec | 25 ++ .../main/package/choco/chocolateyinstall.ps1 | 30 +++ .../main/package/choco/chocolateyuninstall.ps1 | 23 ++ ambari-agent/src/packages/windows-choco.xml | 42 +++ ambari-agent/src/packages/windows.xml | 1 + ambari-server/pom.xml | 34 ++- .../main/assemblies/server-windows-choco.xml | 42 +++ .../src/main/assemblies/server-windows.xml | 1 + .../src/main/package/choco/ambari-server.nuspec | 25 ++ .../main/package/choco/chocolateyinstall.ps1 | 30 +++ .../main/package/choco/chocolateyuninstall.ps1 | 23 ++ ambari-server/src/main/python/bootstrap.py | 5 +- ambari-server/src/main/python/setupAgent.py | 266 ++++--------------- ambari-server/src/test/python/TestSetupAgent.py | 28 +- 15 files changed, 377 insertions(+), 232 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-agent/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-agent/pom.xml b/ambari-agent/pom.xml index a57e760..a08e81c 100644 --- a/ambari-agent/pom.xml +++ b/ambari-agent/pom.xml @@ -703,7 +703,7 @@ <artifactId>apache-rat-plugin</artifactId> <configuration> <excludes> - <exclude>src/test/python/tmp_hostcheck.result</exclude> + <exclude>src/test/python/tmp_hostcheck.result</exclude> <exclude>src/examples/*</exclude> <exclude>src/test/python/dummy*.txt</exclude> <exclude>src/test/python/ambari_agent/dummy_files/*</exclude> @@ -715,6 +715,7 @@ <exclude>**/*.json</exclude> <exclude>**/*.pydevproject</exclude> <exclude>**/*.wxs</exclude> + <exclude>src/main/package/choco/ambari-agent.nuspec</exclude> </excludes> </configuration> <executions> @@ -815,6 +816,21 @@ <goal>single</goal> </goals> </execution> + <execution> + <id>build-choco-dir</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <attach>false</attach> + <appendAssemblyId>true</appendAssemblyId> + <tarLongFileMode>gnu</tarLongFileMode> + <descriptors> + <descriptor>src/packages/windows-choco.xml</descriptor> + </descriptors> + </configuration> + </execution> </executions> </plugin> <!-- msi creation --> @@ -849,6 +865,22 @@ <workingDirectory>${basedir}/target/${final.name}-windows-dist</workingDirectory> </configuration> </execution> + <execution> + <id>build-choco-package</id> + <phase>package</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>choco.exe</executable> + <arguments> + <argument>pack</argument> + <argument>--version=${project.version}</argument> + <argument>${basedir}/target/ambari-agent-${project.version}-choco/ambari-agent.nuspec</argument> + </arguments> + <workingDirectory>target/ambari-agent-${project.version}-choco</workingDirectory> + </configuration> + </execution> </executions> </plugin> <plugin> http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-agent/src/main/package/choco/ambari-agent.nuspec ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/package/choco/ambari-agent.nuspec b/ambari-agent/src/main/package/choco/ambari-agent.nuspec new file mode 100644 index 0000000..d7efb4d --- /dev/null +++ b/ambari-agent/src/main/package/choco/ambari-agent.nuspec @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Do not remove this test for UTF-8: if âΩâ doesnât appear as greek uppercase omega letter enclosed in quotation marks, you should use an editor that supports UTF-8, not this one. --> +<package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd"> + <metadata> + <!-- Read this before publishing packages to chocolatey.org: https://github.com/chocolatey/chocolatey/wiki/CreatePackages --> + <id>ambari-agent</id> + <title>Ambari Agent</title> + <version>1.0</version> + <authors>Apache Ambari</authors> + <owners>Apache Ambari</owners> + <summary>Ambari Agent</summary> + <description>Ambari Agent + </description> + <projectUrl>http://ambari.apache.org</projectUrl> + <tags>ambari-agent</tags> + <copyright>https://github.com/apache/ambari/blob/trunk/NOTICE.txt</copyright> + <licenseUrl>https://github.com/apache/ambari/blob/trunk/LICENSE.txt</licenseUrl> + <requireLicenseAcceptance>false</requireLicenseAcceptance> + <releaseNotes></releaseNotes> + </metadata> + <files> + <file src="tools\**" target="tools" /> + <file src="content\**" target="content" /> + </files> +</package> http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-agent/src/main/package/choco/chocolateyinstall.ps1 ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/package/choco/chocolateyinstall.ps1 b/ambari-agent/src/main/package/choco/chocolateyinstall.ps1 new file mode 100644 index 0000000..3d95e72 --- /dev/null +++ b/ambari-agent/src/main/package/choco/chocolateyinstall.ps1 @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# stop on all errors +$ErrorActionPreference = 'Stop'; + +$packageName = $Env:chocolateyPackageName +$packageVersion = $Env:chocolateyPackageVersion +$toolsDir = "$(Split-Path -parent $MyInvocation.MyCommand.Definition)" +$rootDir = "$(Split-Path -parent $toolsDir)" +$contentDir = "$(Join-Path $rootDir content)" + +$zipFile = "$(Join-Path $contentDir $packageName-$packageVersion-windows-dist.zip)" +$ambariRoot = "C:\ambari" +$specificFolder = "" + +Get-ChocolateyUnzip "$zipFile" "$ambariRoot\$packageName-$packageVersion" $specificFolder $packageName +cmd /c mklink /D "$ambariRoot\$packageName" "$ambariRoot\$packageName-$packageVersion" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-agent/src/main/package/choco/chocolateyuninstall.ps1 ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/package/choco/chocolateyuninstall.ps1 b/ambari-agent/src/main/package/choco/chocolateyuninstall.ps1 new file mode 100644 index 0000000..493da40 --- /dev/null +++ b/ambari-agent/src/main/package/choco/chocolateyuninstall.ps1 @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# stop on all errors +$ErrorActionPreference = 'Stop'; +$packageName = $Env:chocolateyPackageName +$packageVersion = $Env:chocolateyPackageVersion +$ambariRoot = "C:\ambari" + +cmd /c rmdir "$ambariRoot\$packageName" +cmd /c rmdir /s/q "$ambariRoot\$packageName-$packageVersion" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-agent/src/packages/windows-choco.xml ---------------------------------------------------------------------- diff --git a/ambari-agent/src/packages/windows-choco.xml b/ambari-agent/src/packages/windows-choco.xml new file mode 100644 index 0000000..20edba9 --- /dev/null +++ b/ambari-agent/src/packages/windows-choco.xml @@ -0,0 +1,42 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly> + <id>choco</id> + <formats> + <format>dir</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <files> + <file> + <source>${project.build.directory}/${artifact.artifactId}-${artifact.version}-windows-dist.zip</source> + <outputDirectory>content</outputDirectory> + </file> + <file> + <source>${basedir}/src/main/package/choco/ambari-agent.nuspec</source> + </file> + <file> + <source>${basedir}/src/main/package/choco/chocolateyinstall.ps1</source> + <outputDirectory>tools</outputDirectory> + </file> + <file> + <source>${basedir}/src/main/package/choco/chocolateyuninstall.ps1</source> + <outputDirectory>tools</outputDirectory> + </file> + </files> +</assembly> http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-agent/src/packages/windows.xml ---------------------------------------------------------------------- diff --git a/ambari-agent/src/packages/windows.xml b/ambari-agent/src/packages/windows.xml index 6b395fb..ba18ef0 100644 --- a/ambari-agent/src/packages/windows.xml +++ b/ambari-agent/src/packages/windows.xml @@ -24,6 +24,7 @@ <id>windows-dist</id> <formats> <format>dir</format> + <format>zip</format> </formats> <includeBaseDirectory>false</includeBaseDirectory> <fileSets> http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index 626a364..0d37429 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -167,6 +167,7 @@ <exclude>src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2</exclude> <exclude>src/main/windows/ambari-server.cmd</exclude> <exclude>src/main/windows/ambari-server.ps1</exclude> + <exclude>src/main/package/choco/ambari-server.nuspec</exclude> <exclude>src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/smoketest_metrics.json.j2</exclude> <exclude>src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/balancer-emulator/balancer-err.log</exclude> <exclude>src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/balancer-emulator/balancer.log</exclude> @@ -187,7 +188,6 @@ <exclude>**/cluster.properties.j2</exclude> <exclude>**/repo_ubuntu.j2</exclude> <exclude>**/.pydev*</exclude> - <!--gitignore content --> <exclude>src/main/resources/db/newcerts/**</exclude> @@ -1310,6 +1310,7 @@ <path.python.1>${project.basedir}\..\ambari-common\src\main\python;${project.basedir}\..\ambari-agent\src\main\python;${project.basedir}\..\ambari-common\src\main\python\ambari_jinja2;${project.basedir}\..\ambari-common\src\main\python\ambari_commons;${project.basedir}\..\ambari-common\src\test\python;${project.basedir}\src\main\python;${project.basedir}\src\main\python\ambari-server-state;${project.basedir}\src\main\resources\custom_actions;${project.basedir}\src\main\resources\scripts;${project.basedir}\src\test\python</path.python.1> <assemblydescriptor>src/main/assemblies/server-windows.xml</assemblydescriptor> <assemblybootstrap>src/main/assemblies/bootstrap-windows.xml</assemblybootstrap> + <assemblychocodescriptor>src/main/assemblies/server-windows-choco.xml</assemblychocodescriptor> <packagingFormat>jar</packagingFormat> </properties> <build> @@ -1353,6 +1354,21 @@ </descriptors> </configuration> </execution> + <execution> + <id>build-choco-dir</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <attach>false</attach> + <appendAssemblyId>true</appendAssemblyId> + <tarLongFileMode>gnu</tarLongFileMode> + <descriptors> + <descriptor>${assemblychocodescriptor}</descriptor> + </descriptors> + </configuration> + </execution> </executions> </plugin> <plugin> @@ -1386,6 +1402,22 @@ <workingDirectory>target/ambari-server-${project.version}-dist/ambari-server-${project.version}</workingDirectory> </configuration> </execution> + <execution> + <id>build-choco-package</id> + <phase>package</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>choco.exe</executable> + <arguments> + <argument>pack</argument> + <argument>--version=${project.version}</argument> + <argument>${basedir}/target/ambari-server-${project.version}-choco/ambari-server.nuspec</argument> + </arguments> + <workingDirectory>target/ambari-server-${project.version}-choco</workingDirectory> + </configuration> + </execution> </executions> </plugin> <plugin> http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-server/src/main/assemblies/server-windows-choco.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/assemblies/server-windows-choco.xml b/ambari-server/src/main/assemblies/server-windows-choco.xml new file mode 100644 index 0000000..8aec023 --- /dev/null +++ b/ambari-server/src/main/assemblies/server-windows-choco.xml @@ -0,0 +1,42 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly> + <id>choco</id> + <formats> + <format>dir</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <files> + <file> + <source>${project.build.directory}/${artifact.artifactId}-${artifact.version}-dist.zip</source> + <outputDirectory>content</outputDirectory> + </file> + <file> + <source>${basedir}/src/main/package/choco/ambari-server.nuspec</source> + </file> + <file> + <source>${basedir}/src/main/package/choco/chocolateyinstall.ps1</source> + <outputDirectory>tools</outputDirectory> + </file> + <file> + <source>${basedir}/src/main/package/choco/chocolateyuninstall.ps1</source> + <outputDirectory>tools</outputDirectory> + </file> + </files> +</assembly> http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-server/src/main/assemblies/server-windows.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/assemblies/server-windows.xml b/ambari-server/src/main/assemblies/server-windows.xml index 1fcd191..4f9c874 100644 --- a/ambari-server/src/main/assemblies/server-windows.xml +++ b/ambari-server/src/main/assemblies/server-windows.xml @@ -20,6 +20,7 @@ <id>dist</id> <formats> <format>dir</format> + <format>zip</format> </formats> <includeBaseDirectory>false</includeBaseDirectory> <files> http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-server/src/main/package/choco/ambari-server.nuspec ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/package/choco/ambari-server.nuspec b/ambari-server/src/main/package/choco/ambari-server.nuspec new file mode 100644 index 0000000..851e644 --- /dev/null +++ b/ambari-server/src/main/package/choco/ambari-server.nuspec @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Do not remove this test for UTF-8: if âΩâ doesnât appear as greek uppercase omega letter enclosed in quotation marks, you should use an editor that supports UTF-8, not this one. --> +<package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd"> + <metadata> + <!-- Read this before publishing packages to chocolatey.org: https://github.com/chocolatey/chocolatey/wiki/CreatePackages --> + <id>ambari-server</id> + <title>Ambari Server</title> + <version>1.0</version> + <authors>Apache Ambari</authors> + <owners>Apache Ambari</owners> + <summary>Ambari Server</summary> + <description>Ambari Server + </description> + <projectUrl>http://ambari.apache.org</projectUrl> + <tags>ambari-server</tags> + <copyright>https://github.com/apache/ambari/blob/trunk/NOTICE.txt</copyright> + <licenseUrl>https://github.com/apache/ambari/blob/trunk/LICENSE.txt</licenseUrl> + <requireLicenseAcceptance>false</requireLicenseAcceptance> + <releaseNotes></releaseNotes> + </metadata> + <files> + <file src="tools\**" target="tools" /> + <file src="content\**" target="content" /> + </files> +</package> http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-server/src/main/package/choco/chocolateyinstall.ps1 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/package/choco/chocolateyinstall.ps1 b/ambari-server/src/main/package/choco/chocolateyinstall.ps1 new file mode 100644 index 0000000..becf079 --- /dev/null +++ b/ambari-server/src/main/package/choco/chocolateyinstall.ps1 @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# stop on all errors +$ErrorActionPreference = 'Stop'; + +$packageName = $Env:chocolateyPackageName +$packageVersion = $Env:chocolateyPackageVersion +$toolsDir = "$(Split-Path -parent $MyInvocation.MyCommand.Definition)" +$rootDir = "$(Split-Path -parent $toolsDir)" +$contentDir = "$(Join-Path $rootDir content)" + +$zipFile = "$(Join-Path $contentDir $packageName-$packageVersion-dist.zip)" +$ambariRoot = "C:\ambari" +$specificFolder = "" + +Get-ChocolateyUnzip "$zipFile" $ambariRoot $specificFolder $packageName +cmd /c mklink /D "$ambariRoot\$packageName" "$ambariRoot\$packageName-$packageVersion" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-server/src/main/package/choco/chocolateyuninstall.ps1 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/package/choco/chocolateyuninstall.ps1 b/ambari-server/src/main/package/choco/chocolateyuninstall.ps1 new file mode 100644 index 0000000..503fb72 --- /dev/null +++ b/ambari-server/src/main/package/choco/chocolateyuninstall.ps1 @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# stop on all errors +$ErrorActionPreference = 'Stop'; +$packageName = $Env:chocolateyPackageName +$packageVersion = $Env:chocolateyPackageVersion +$unzipLocation = "C:\ambari" + +cmd /c rmdir "$unzipLocation\$packageName" +cmd /c rmdir /s/q "$unzipLocation\$packageName-$packageVersion" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-server/src/main/python/bootstrap.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/bootstrap.py b/ambari-server/src/main/python/bootstrap.py index a3b561d..b22a10d 100755 --- a/ambari-server/src/main/python/bootstrap.py +++ b/ambari-server/src/main/python/bootstrap.py @@ -305,10 +305,11 @@ class BootstrapWindows(Bootstrap): def getRunSetupCommand(self, expected_hostname): setupFile = os.path.join(self.getTempFolder(), self.SETUP_SCRIPT_FILENAME) - msi_url = 'http://{0}:{1}/resources/ambari-agent.msi'.format(self.shared_state.ambari_server, self.shared_state.server_port) + passphrase = os.environ[AMBARI_PASSPHRASE_VAR_NAME] + user_run_as = self.shared_state.user_run_as server = self.shared_state.ambari_server version = self.getAmbariVersion() - return ' '.join(['python', setupFile, msi_url, server, version]) + return ' '.join(['python', setupFile, expected_hostname, passphrase, server, user_run_as, version]) def runSetupAgent(self): self.host_log.write("==========================\n") http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-server/src/main/python/setupAgent.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/setupAgent.py b/ambari-server/src/main/python/setupAgent.py index b3825ad..ff2c5e9 100755 --- a/ambari-server/src/main/python/setupAgent.py +++ b/ambari-server/src/main/python/setupAgent.py @@ -32,19 +32,13 @@ if OSCheck.is_windows_family(): import urllib2 from ambari_commons.exceptions import FatalException - from ambari_commons.inet_utils import force_download_file from ambari_commons.os_utils import run_os_command AMBARI_PASSPHRASE_VAR = "AMBARI_PASSPHRASE" PROJECT_VERSION_DEFAULT = "DEFAULT" -def _init_ambari_agent_symlink(): - installationDrive = os.path.splitdrive(__file__.replace('/', os.sep))[0] - return os.path.join(installationDrive, os.sep, "ambari", "ambari-agent") - -AMBARI_AGENT_INSTALL_SYMLINK = _init_ambari_agent_symlink() -INSTALL_MARKER_OK = "ambari-agent.installed" +AMBARI_AGENT_INSTALL_SYMLINK = "C:\\ambari\\ambari-agent" def _ret_init(ret): if not ret: @@ -101,7 +95,7 @@ def execOsCommand(osCommand, tries=1, try_sleep=0, ret=None, cwd=None): if retcode == 0: break - _ret_append_stdout("\nRetrying " + str(osCommand)) + _ret_append_stdout(ret, "\nRetrying " + str(osCommand)) return ret @@ -122,103 +116,7 @@ def execOsCommand(osCommand, tries=1, try_sleep=0, ret=None, cwd=None): return ret -def _download_file(url, destFilePath, progress_function=None, ret=None): - ret = _ret_init(ret) - - if os.path.exists(destFilePath): - _ret_append_stdout(ret, "\nFile {0} already exists, assuming it was downloaded before".format(destFilePath)) - else: - try: - #Intrinsically reliable and resumable. Downloads to a temp file and renames the tem file to the destination file - # upon successful termination. - force_download_file(url, destFilePath, 16 * 1024, progress_function) - except FatalException, e: - _ret_merge(ret, e.code, None, "Failed to download {0} -> {1} : {2}".format(url, destFilePath, e.reason)) - except urllib2.URLError, ue: - _ret_merge(ret, 2, None, "Failed to download {0} -> {1} : {2}".format(url, destFilePath, ue.reason)) - return ret - - -def _create_agent_symlink(symlinkPath, agentInstallDir, ret): - ret = _ret_init(ret) - - symLinkCreationAttempts = 0 - while (symLinkCreationAttempts < 1000): - # Handle contention from other bootstrap processes - try: - os.rmdir(symlinkPath) - except OSError: - #It's ok to attempt to delete a non-existing link - pass - - try: - os.symlink(agentInstallDir, symlinkPath) - if os.readlink(symlinkPath) == agentInstallDir: - break - except OSError: - pass - - symLinkCreationAttempts += 1 - if symLinkCreationAttempts == 1000: - _ret_merge(ret, 1000, '', - 'Failed creating the symbolic link {0} because of contention.'.format(AMBARI_AGENT_INSTALL_SYMLINK)) - return ret - - -@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) -def installAgent(url, downloadDir, projectVersion, ret=None): - """ Download the agent msi and install it - :param url: - :param projectVersion: - :return: {"exitstatus": exit code, "log": log records string} - """ - ret = _ret_init(ret) - - installationDrive = os.path.splitdrive(__file__.replace('/', os.sep))[0] - - agentInstallDir = os.path.join(installationDrive, os.sep, "ambari", "ambari-agent-" + projectVersion) - agentInstallMarkerFile = os.path.join(agentInstallDir, INSTALL_MARKER_OK) - - if not os.path.exists(agentInstallMarkerFile): - destMsiFilePath = os.path.join(downloadDir, "ambari-agent-{0}.msi".format(projectVersion)) - ret = _download_file(url, destMsiFilePath, ret=ret) - if ret['exitstatus'] != 0: - return ret - - #ambari-agent-<version>.msi downloaded, proceed to the installation - installLogPath = os.path.join(downloadDir, "ambari-agent-{0}.install.log".format(projectVersion)) - installCmd = [ - "cmd", - "/c", - "start", - "/wait", - "msiexec", - "/i", destMsiFilePath, - "AGENT_INSTALL_DIRECTORY=" + agentInstallDir, - "/qn", - "/Lv", installLogPath] - ret = execOsCommand(installCmd, tries=3, try_sleep=10, ret=ret) - if ret['exitstatus'] != 0: - #TODO Check if the product was already installed. Only machine reimage can repair a broken installation. - return ret - - try: - if os.readlink(AMBARI_AGENT_INSTALL_SYMLINK) != agentInstallDir: - ret = _create_agent_symlink(AMBARI_AGENT_INSTALL_SYMLINK, agentInstallDir, ret) - except OSError: - ret = _create_agent_symlink(AMBARI_AGENT_INSTALL_SYMLINK, agentInstallDir, ret) - if ret['exitstatus'] != 0: - return ret - - try: - open(agentInstallMarkerFile, "w+").close() - except IOError: - pass - - return ret - -@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) -def installAgent(projectVersion): +def installAgent(projectVersion, ret=None): """ Run install and make sure the agent install alright """ # The command doesn't work with file mask ambari-agent*.rpm, so rename it on agent host if OSCheck.is_suse_family(): @@ -226,41 +124,42 @@ def installAgent(projectVersion): elif OSCheck.is_ubuntu_family(): # add * to end of version in case of some test releases Command = ["apt-get", "install", "-y", "--allow-unauthenticated", "ambari-agent=" + projectVersion + "*"] + elif OSCheck.is_windows_family(): + Command = ["cmd", "/c", "choco", "install", "-y", "ambari-agent", "--version=" + projectVersion] else: Command = ["yum", "-y", "install", "--nogpgcheck", "ambari-agent-" + projectVersion] - return execOsCommand(Command, tries=3, try_sleep=10) + return execOsCommand(Command, tries=3, try_sleep=10, ret=ret) @OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) -def configureAgent(server_hostname, cwd, ret=None): +def configureAgent(server_hostname, user_run_as, ret=None): #Customize ambari-agent.ini & register the Ambari Agent service agentSetupCmd = ["cmd", "/c", "ambari-agent.cmd", "setup", "--hostname=" + server_hostname] return execOsCommand(agentSetupCmd, tries=3, try_sleep=10, cwd=AMBARI_AGENT_INSTALL_SYMLINK, ret=ret) @OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) -def configureAgent(server_hostname, user_run_as): +def configureAgent(server_hostname, user_run_as, ret=None): """ Configure the agent so that it has all the configs knobs properly installed """ osCommand = ["sed", "-i.bak", "s/hostname=localhost/hostname=" + server_hostname + "/g", "/etc/ambari-agent/conf/ambari-agent.ini"] - ret = execOsCommand(osCommand) + ret = execOsCommand(osCommand, ret=ret) if ret['exitstatus'] != 0: return ret osCommand = ["sed", "-i.bak", "s/run_as_user=.*$/run_as_user=" + user_run_as + "/g", "/etc/ambari-agent/conf/ambari-agent.ini"] - ret = execOsCommand(osCommand) + ret = execOsCommand(osCommand, ret=ret) return ret - -#Windows-specific -def runAgentService(ret=None): +@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) +def runAgent(passPhrase, expected_hostname, user_run_as, verbose, ret=None): ret = _ret_init(ret) #Invoke ambari-agent restart as a child process agentRestartCmd = ["cmd", "/c", "ambari-agent.cmd", "restart"] return execOsCommand(agentRestartCmd, tries=3, try_sleep=10, cwd=AMBARI_AGENT_INSTALL_SYMLINK, ret=ret) -#Linux-specific -def runAgent(passPhrase, expected_hostname, user_run_as, verbose): +@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) +def runAgent(passPhrase, expected_hostname, user_run_as, verbose, ret=None): os.environ[AMBARI_PASSPHRASE_VAR] = passPhrase vo = "" if verbose: @@ -272,7 +171,7 @@ def runAgent(passPhrase, expected_hostname, user_run_as, verbose): agent_retcode = p.returncode for i in range(3): time.sleep(1) - ret = execOsCommand(["tail", "-20", "/var/log/ambari-agent/ambari-agent.log"]) + ret = execOsCommand(["tail", "-20", "/var/log/ambari-agent/ambari-agent.log"], ret=ret) if (0 == ret['exitstatus']): try: log = ret['log'] @@ -281,7 +180,16 @@ def runAgent(passPhrase, expected_hostname, user_run_as, verbose): print log break return {"exitstatus": agent_retcode, "log": log} - + +@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) +def tryStopAgent(): + verbose = False + if os.path.exists(AMBARI_AGENT_INSTALL_SYMLINK): + agentStopCmd = ["cmd", "/c", "ambari-agent.cmd", "stop"] + execOsCommand(agentStopCmd, tries=3, try_sleep=10, cwd=AMBARI_AGENT_INSTALL_SYMLINK) + return verbose + +@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) def tryStopAgent(): verbose = False cmds = ["bash", "-c", "ps aux | grep 'AmbariAgent.py' | grep ' \-v'"] @@ -293,26 +201,29 @@ def tryStopAgent(): @OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) def getOptimalVersion(initialProjectVersion): - if initialProjectVersion == "null" or initialProjectVersion == "{ambariVersion}" or \ - initialProjectVersion == PROJECT_VERSION_DEFAULT or not initialProjectVersion: - #Extract the project version form the current script path - scriptPath = os.path.dirname(__file__.replace('/', os.sep)) - optimalVersion = os.path.split(scriptPath)[1] + optimalVersion = initialProjectVersion + ret = findNearestAgentPackageVersion(optimalVersion) + if ret["exitstatus"] == 0 and ret["log"][0].strip() != "" \ + and ret["log"][0].strip().startswith(initialProjectVersion): + optimalVersion = ret["log"][0].strip() + retcode = 0 else: - optimalVersion = initialProjectVersion - return optimalVersion + ret = getAvailableAgentPackageVersions() + retcode = 1 + optimalVersion = ret["log"] + + return {"exitstatus": retcode, "log": optimalVersion} @OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) def getOptimalVersion(initialProjectVersion): optimalVersion = initialProjectVersion ret = findNearestAgentPackageVersion(optimalVersion) - if ret["exitstatus"] == 0 and ret["log"][0].strip() != "" \ and ret["log"][0].strip() == initialProjectVersion: optimalVersion = ret["log"][0].strip() retcode = 0 else: - ret = getAvaliableAgentPackageVersions() + ret = getAvailableAgentPackageVersions() retcode = 1 optimalVersion = ret["log"] @@ -325,6 +236,10 @@ def findNearestAgentPackageVersion(projectVersion): if OSCheck.is_suse_family(): Command = ["bash", "-c", "zypper --no-gpg-checks -q search -s --match-exact ambari-agent | grep '" + projectVersion + "' | cut -d '|' -f 4 | head -n1 | sed -e 's/-\w[^:]*//1' "] + elif OSCheck.is_windows_family(): + listPackagesCommand = ["cmd", "/c", "choco list ambari-agent --pre --all | findstr " + projectVersion + " > agentPackages.list"] + execOsCommand(listPackagesCommand) + Command = ["cmd", "/c", "powershell", "get-content agentPackages.list | select-object -last 1 | foreach-object {$_ -replace 'ambari-agent ', ''}"] elif OSCheck.is_ubuntu_family(): if projectVersion == " ": Command = ["bash", "-c", "apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"] @@ -336,10 +251,11 @@ def findNearestAgentPackageVersion(projectVersion): "' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | head -n1 | sed -e 's/-\w[^:]*//1' "] return execOsCommand(Command) - def isAgentPackageAlreadyInstalled(projectVersion): if OSCheck.is_ubuntu_family(): Command = ["bash", "-c", "dpkg-query -W -f='${Status} ${Version}\n' ambari-agent | grep -v deinstall | grep " + projectVersion] + elif OSCheck.is_windows_family(): + Command = ["cmd", "/c", "choco list ambari-agent --local-only | findstr ambari-agent"] else: Command = ["bash", "-c", "rpm -qa | grep ambari-agent-"+projectVersion] ret = execOsCommand(Command) @@ -348,11 +264,12 @@ def isAgentPackageAlreadyInstalled(projectVersion): res = True return res - -def getAvaliableAgentPackageVersions(): +def getAvailableAgentPackageVersions(): if OSCheck.is_suse_family(): Command = ["bash", "-c", "zypper --no-gpg-checks -q search -s --match-exact ambari-agent | grep ambari-agent | sed -re 's/\s+/ /g' | cut -d '|' -f 4 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"] + elif OSCheck.is_windows_family(): + Command = ["cmd", "/c", "choco list ambari-agent --pre --all | findstr ambari-agent"] elif OSCheck.is_ubuntu_family(): Command = ["bash", "-c", "apt-cache -q show ambari-agent|grep 'Version\:'|cut -d ' ' -f 2| tr '\\n' ', '|sed -s 's/[-|~][A-Za-z0-9]*//g'"] @@ -361,7 +278,6 @@ def getAvaliableAgentPackageVersions(): "yum -q list all ambari-agent | grep -E '^ambari-agent' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"] return execOsCommand(Command) - def checkServerReachability(host, port): ret = {} s = socket.socket() @@ -375,31 +291,7 @@ def checkServerReachability(host, port): "Please check the network connectivity between the Ambari Agent host and the Ambari Server" return ret -# Command line syntax help - Windows -# IsOptional Index Description -# 0 ambari-agent.msi URL -# 1 Server host name -# X 2 Project version (Ambari) - -@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) -def parseArguments(argv=None): - if argv is None: # make sure that arguments was passed - return {"exitstatus": 2, "log": "No arguments were passed"} - args = argv[1:] # shift path to script - if len(args) < 2: - return {"exitstatus": 1, "log": "Not all required arguments were passed"} - - agentUrl = args[0] - serverHostname = args[1] - projectVersion = PROJECT_VERSION_DEFAULT - - if len(args) > 2: - projectVersion = args[2] - - parsed_args = (agentUrl, serverHostname, projectVersion) - return {"exitstatus": 0, "log": ("", ""), "parsed_args": parsed_args} - -# Command line syntax help - Linux +# Command line syntax help # IsOptional Index Description # 0 Expected host name # 1 Password @@ -407,9 +299,6 @@ def parseArguments(argv=None): # 3 User to run agent as # X 4 Project Version (Ambari) # X 5 Server port - - -@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) def parseArguments(argv=None): if argv is None: # make sure that arguments was passed return {"exitstatus": 2, "log": "No arguments were passed"} @@ -436,47 +325,6 @@ def parseArguments(argv=None): parsed_args = (expected_hostname, passPhrase, hostname, user_run_as, projectVersion, server_port) return {"exitstatus": 0, "log": "", "parsed_args": parsed_args} - -@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) -def run_setup(argv=None): - """ - if the Agent is not downloaded or the download was interrupted - download the Agent msi package - install the Agent from the msi package - customize the Agent configuration - register the Ambari Agent Windows service - if JDK is not installed on the local machine - download and install JDK - set the machine-wide JAVA_HOME environment variable - if the Agent service is running from a previous session - stop the Agent service - create/switch the Agent dir symbolic link to the new version - start the Agent service - """ - - # Parse passed arguments - retcode = parseArguments(argv) - if (retcode["exitstatus"] != 0): - return retcode - - (agent_url, server_hostname, projectVersion) = retcode["parsed_args"] - - availableProjectVersion = getOptimalVersion(projectVersion) - - retcode = installAgent(agent_url, os.getcwd(), availableProjectVersion, retcode) - if (not retcode["exitstatus"] == 0): - return retcode - - retcode = configureAgent(server_hostname, retcode) - if retcode['exitstatus'] != 0: - return retcode - - #TODO Install the JDK - #install_jdk(jdk_url, java_home_dir, jdk_name, ret) - - return runAgentService(retcode) - -@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) def run_setup(argv=None): # Parse passed arguments retcode = parseArguments(argv) @@ -493,10 +341,11 @@ def run_setup(argv=None): retcode = getOptimalVersion("") else: retcode = getOptimalVersion(projectVersion) + if retcode["exitstatus"] == 0 and retcode["log"] != None and retcode["log"] != "" and retcode["log"][0].strip() != "": - availiableProjectVersion = retcode["log"].strip() - if not isAgentPackageAlreadyInstalled(availiableProjectVersion): - retcode = installAgent(availiableProjectVersion) + availableProjectVersion = retcode["log"].strip() + if not isAgentPackageAlreadyInstalled(availableProjectVersion): + retcode = installAgent(availableProjectVersion) if (not retcode["exitstatus"] == 0): return retcode elif retcode["exitstatus"] == 1 and retcode["log"][0].strip() != "": @@ -512,16 +361,6 @@ def run_setup(argv=None): return retcode return runAgent(passPhrase, expected_hostname, user_run_as, verbose) - -@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY) -def main(argv=None): - try: - exitcode = run_setup(argv) - except Exception, e: - exitcode = {"exitstatus": -1, "log": str(e)} - return exitcode - -@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT) def main(argv=None): #Try stop agent and check --verbose option if agent already run global verbose @@ -539,6 +378,5 @@ if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) ret = main(sys.argv) retcode = ret["exitstatus"] - if 0 != retcode: - print ret["log"] + print ret["log"] sys.exit(retcode) http://git-wip-us.apache.org/repos/asf/ambari/blob/36f92c70/ambari-server/src/test/python/TestSetupAgent.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestSetupAgent.py b/ambari-server/src/test/python/TestSetupAgent.py index 380893f..80178ba 100644 --- a/ambari-server/src/test/python/TestSetupAgent.py +++ b/ambari-server/src/test/python/TestSetupAgent.py @@ -128,13 +128,13 @@ class TestSetupAgent(TestCase): pass @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) - @patch.object(setup_agent, 'getAvaliableAgentPackageVersions') + @patch.object(setup_agent, 'getAvailableAgentPackageVersions') @patch('ambari_commons.OSCheck.is_suse_family') @patch('ambari_commons.OSCheck.is_ubuntu_family') @patch.object(setup_agent, 'findNearestAgentPackageVersion') def test_returned_optimal_version_is_initial_on_suse(self, findNearestAgentPackageVersion_method, is_ubuntu_family_method, - is_suse_family_method, getAvaliableAgentPackageVersions_method): - getAvaliableAgentPackageVersions_method.return_value = {"exitstatus": 0, "log": "1.1.1"} + is_suse_family_method, getAvailableAgentPackageVersions_method): + getAvailableAgentPackageVersions_method.return_value = {"exitstatus": 0, "log": "1.1.1"} is_suse_family_method.return_value = True is_ubuntu_family_method.return_value = False @@ -145,13 +145,13 @@ class TestSetupAgent(TestCase): pass @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) - @patch.object(setup_agent, 'getAvaliableAgentPackageVersions') + @patch.object(setup_agent, 'getAvailableAgentPackageVersions') @patch('ambari_commons.OSCheck.is_suse_family') @patch('ambari_commons.OSCheck.is_ubuntu_family') @patch.object(setup_agent, 'findNearestAgentPackageVersion') def test_returned_optimal_version_is_initial_on_ubuntu(self, findNearestAgentPackageVersion_method, is_ubuntu_family_method, - is_suse_family_method, getAvaliableAgentPackageVersions_method): - getAvaliableAgentPackageVersions_method.return_value = {"exitstatus": 0, "log": "1.1.1"} + is_suse_family_method, getAvailableAgentPackageVersions_method): + getAvailableAgentPackageVersions_method.return_value = {"exitstatus": 0, "log": "1.1.1"} is_suse_family_method.return_value = False is_ubuntu_family_method.return_value = True @@ -206,14 +206,14 @@ class TestSetupAgent(TestCase): pass @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) - @patch.object(setup_agent, 'getAvaliableAgentPackageVersions') + @patch.object(setup_agent, 'getAvailableAgentPackageVersions') @patch('ambari_commons.OSCheck.is_suse_family') @patch('ambari_commons.OSCheck.is_ubuntu_family') @patch.object(setup_agent, 'findNearestAgentPackageVersion') def test_returned_optimal_version_is_initial(self, findNearestAgentPackageVersion_method, is_ubuntu_family_method, - is_suse_family_method, getAvaliableAgentPackageVersions_method): - getAvaliableAgentPackageVersions_method.return_value = {"exitstatus": 0, "log": "1.1.1"} + is_suse_family_method, getAvailableAgentPackageVersions_method): + getAvailableAgentPackageVersions_method.return_value = {"exitstatus": 0, "log": "1.1.1"} is_suse_family_method.return_value = False is_ubuntu_family_method.return_value = False @@ -224,14 +224,14 @@ class TestSetupAgent(TestCase): pass @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value)) - @patch.object(setup_agent, 'getAvaliableAgentPackageVersions') + @patch.object(setup_agent, 'getAvailableAgentPackageVersions') @patch('ambari_commons.OSCheck.is_suse_family') @patch('ambari_commons.OSCheck.is_ubuntu_family') @patch.object(setup_agent, 'findNearestAgentPackageVersion') def test_returned_optimal_version_is_default(self, findNearestAgentPackageVersion_method, is_ubuntu_family_method, - is_suse_family_method, getAvaliableAgentPackageVersions_method): - getAvaliableAgentPackageVersions_method.return_value = {"exitstatus": 0, "log": "1.1.1"} + is_suse_family_method, getAvailableAgentPackageVersions_method): + getAvailableAgentPackageVersions_method.return_value = {"exitstatus": 0, "log": "1.1.1"} is_suse_family_method.return_value = False is_ubuntu_family_method.return_value = False findNearestAgentPackageVersion_method.return_value = { @@ -419,8 +419,8 @@ class TestSetupAgent(TestCase): pass @patch.object(setup_agent, 'execOsCommand') - def test_getAvaliableAgentPackageVersions(self, execOsCommand_mock): - setup_agent.getAvaliableAgentPackageVersions() + def test_getAvailableAgentPackageVersions(self, execOsCommand_mock): + setup_agent.getAvailableAgentPackageVersions() self.assertTrue(execOsCommand_mock.called) pass