YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)
Change-Id: I46e8d9fd32c7745c313030da62da41486a77b3ea Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46d6e001 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46d6e001 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46d6e001 Branch: refs/heads/HDDS-4 Commit: 46d6e0016610ced51a76189daeb3ad0e3dbbf94c Parents: ed08dd3 Author: Wangda Tan <wan...@apache.org> Authored: Tue Oct 16 13:36:59 2018 -0700 Committer: Wangda Tan <wan...@apache.org> Committed: Tue Oct 16 13:51:02 2018 -0700 ---------------------------------------------------------------------- .../resources/assemblies/hadoop-yarn-dist.xml | 8 + .../installation/install.conf | 74 ++++ .../installation/install.sh | 116 +++++ .../package/calico/calico-node.service | 50 +++ .../installation/package/calico/calicoctl.cfg | 22 + .../installation/package/docker/daemon.json | 23 + .../installation/package/docker/docker.service | 35 ++ .../installation/package/etcd/etcd.service | 40 ++ .../package/hadoop/container-executor.cfg | 41 ++ .../installation/package/submarine/submarine.sh | 25 ++ .../installation/scripts/calico.sh | 224 ++++++++++ .../installation/scripts/docker.sh | 166 +++++++ .../installation/scripts/download-server.sh | 42 ++ .../installation/scripts/environment.sh | 213 +++++++++ .../installation/scripts/etcd.sh | 152 +++++++ .../installation/scripts/hadoop.sh | 117 +++++ .../installation/scripts/menu.sh | 444 +++++++++++++++++++ .../installation/scripts/nvidia-docker.sh | 99 +++++ .../installation/scripts/nvidia.sh | 120 +++++ .../installation/scripts/submarine.sh | 38 ++ .../installation/scripts/utils.sh | 123 +++++ 21 files changed, 2172 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index a2ea08c..9c401e8 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -66,6 +66,14 @@ <fileMode>0755</fileMode> </fileSet> <fileSet> + <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation</directory> + <outputDirectory>/share/hadoop/yarn/submarine-installer</outputDirectory> + <includes> + <include>**/*</include> + </includes> + <fileMode>0755</fileMode> + </fileSet> + <fileSet> <directory>hadoop-yarn/conf</directory> <outputDirectory>etc/hadoop</outputDirectory> <includes> http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf new file mode 100644 index 0000000..82dcf61 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### install config ##### + +# DNS +LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf +YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address + +# etcd hosts list +ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) + +# docker registry ip:port +DOCKER_REGISTRY="10.120.196.232:5000" + +# Start the http download service on the specified server, +# Will download all the dependencies in the http server, +# Run the install script on other servers. +# Automatically download dependencies from http, +# Solve the problem that all servers are slow to download online. +# At the same time, you can also manually download the dependencies to the downloads directory. +# Offline installation of the system +DOWNLOAD_SERVER_IP="10.120.196.236" +DOWNLOAD_SERVER_PORT="19000" + +# yarn container-executor config +# How to compile container-executor: +# Go to the hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager +# path and enter the command: mvn package -Pnative -DskipTests +# Only the nodemanager is compiled. The path of the compiled container-executor is: +# ./target/native/target/usr/local/bin/container-executor +YARN_CONTAINER_EXECUTOR_PATH="Please enter the full path to the container-executor here" + +# Keep the same configuration as 'yarn.nodemanager.linux-container-executor.cgroups.hierarchy' +# in yarn-site.xml, default '/hadoop-yarn' +YARN_HIERARCHY="/hadoop-yarn" + +# Keep the same configuration as 'yarn.nodemanager.local-dirs' in yarn-site.xml +YARN_NODEMANAGER_LOCAL_DIRS="/home/hadoop/disk/1/yarn/local,/home/hadoop/disk/2/yarn/local,/home/hadoop/disk/3/yarn/local" + +# Keep the same configuration as 'yarn.nodemanager.log-dirs' in yarn-site.xml +YARN_NODEMANAGER_LOG_DIRS="/home/hadoop/disk/1/yarn/logs,/home/hadoop/disk/2/yarn/logs,/home/hadoop/disk/3/yarn/logs" + +##### System component download url address, Generally do not need to be modified ##### +DOCKER_REPO="https://yum.dockerproject.org/repo/main/centos/7/Packages"; +DOCKER_ENGINE_RPM="docker-engine-1.12.5-1.el7.centos.x86_64.rpm" +DOCKER_ENGINE_SELINUX_RPM="docker-engine-selinux-1.12.5-1.el7.centos.noarch.rpm" + +NVIDIA_DETECT_URL="https://raw.githubusercontent.com/liuxunorg/submarine-installer/packages/nvidia/nvidia-detect"; +NVIDIA_DOCKER_RPM="nvidia-docker-1.0.1-1.x86_64.rpm" +NVIDIA_DOCKER_RPM_URL="https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/${NVIDIA_DOCKER_RPM}"; + +ETCD_TAR_GZ="etcd-v3.3.9-linux-amd64.tar.gz" +ETCD_DOWNLOAD_URL="https://github.com/etcd-io/etcd/releases/download/v3.3.9/${ETCD_TAR_GZ}"; + +CALICO_IPV4POOL_CIDR="192.20.0.0" +CALICO_NETWORK_NAME="calico-network" +CALICO_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico"; +CALICO_IPAM_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico-ipam"; +CALICOCTL_DOWNLOAD_URL="https://github.com/projectcalico/calicoctl/releases/download/v3.2.3/calicoctl"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh new file mode 100755 index 0000000..90b12c2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# description: sumbarine install scripts. + +ROOT=$(cd "$(dirname "$0")"; pwd) +SUBMARINE_INSTALLER_VERSION="v0.7" +PACKAGE_DIR=${ROOT}/package +SCRIPTS_DIR=${ROOT}/scripts +INSTALL_TEMP_DIR=${ROOT}/temp +DOWNLOAD_DIR=${ROOT}/downloads +DATE=`date +%Y%m%d-%H:%M:%S` +LOG=${ROOT}/logs/install.log.`date +%Y%m%d%H%M%S` +LOCAL_HOST_IP_LIST=() +LOCAL_HOST_IP="" +OPERATING_SYSTEM="" +DOWNLOAD_HTTP="" + +# import shell script +. ${ROOT}/install.conf +. ${ROOT}/scripts/calico.sh +. ${ROOT}/scripts/docker.sh +. ${ROOT}/scripts/download-server.sh +. ${ROOT}/scripts/environment.sh +. ${ROOT}/scripts/etcd.sh +. ${ROOT}/scripts/hadoop.sh +. ${ROOT}/scripts/menu.sh +. ${ROOT}/scripts/nvidia.sh +. ${ROOT}/scripts/nvidia-docker.sh +. ${ROOT}/scripts/submarine.sh +. ${ROOT}/scripts/utils.sh + +#================================= Main ======================================== +mkdir $ROOT/logs/ -p +mkdir $INSTALL_TEMP_DIR -p +mkdir $DOWNLOAD_DIR -p + +source /etc/os-release +OPERATING_SYSTEM=$ID + +get_ip_list +ipCount=${#LOCAL_HOST_IP_LIST[@]} +if [[ $ipCount -eq 1 ]]; then + LOCAL_HOST_IP=${LOCAL_HOST_IP_LIST[0]} +else + echo -e "Detect the network card IP in the server, \e[31m[${LOCAL_HOST_IP_LIST[@]}]\e[0m" + echo -n -e "please enter a valid IP address: " + + read ipInput + if ! valid_ip $ipInput; then + echo -e "you input \e[31m$ipInput\e[0m address format is incorrect! " | tee -a $LOG + exit_install + else + LOCAL_HOST_IP=$ipInput + fi +fi + +echo -n -e "Please confirm whether the IP address of this machine is \e[31m${LOCAL_HOST_IP}\e[0m?[y|n]" +read myselect +if [[ "$myselect" != "y" && "$myselect" != "Y" ]]; then + exit_install +fi + +check_install_conf + +if [[ -n "$DOWNLOAD_SERVER_IP" && -n "$DOWNLOAD_SERVER_PORT" && "$DOWNLOAD_SERVER_IP" != "$LOCAL_HOST_IP" ]]; then + DOWNLOAD_HTTP="http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}"; +fi + +check_install_user + +# æ¸ çå®è£ 临æ¶ç®å½ +rm $INSTALL_TEMP_DIR/* -rf >>$LOG 2>&1 + +menu_index="0" +for ((j=1;;j++)) +do + menu + case "$menu_index" in + "0") + menu_index="$menu_choice" + ;; + "1"|"2"|"3"|"4"|"5") +# echo "aaaa=$menu_index-$menu_choice" + menu_process + if [[ $? = 1 ]]; then + echo "Press any key to return menu!" + read + fi + ;; + "a") + exit_install + ;; + "q") + exit_install + ;; + *) + menu_index="0" + menu_choice="0" + menu + ;; + esac +done http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service new file mode 100644 index 0000000..744c419 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=calico node +After=docker.service +Requires=docker.service + +[Service] +User=root +PermissionsStartOnly=true +ExecStart=/usr/bin/docker run --net=host --privileged --name=calico-node \ + -e ETCD_ENDPOINTS=ETCD_ENDPOINTS_REPLACE \ + -e CALICO_LIBNETWORK_ENABLED=true \ + -e CALICO_NETWORKING_BACKEND=bird \ + -e CALICO_DISABLE_FILE_LOGGING=true \ + -e CALICO_IPV4POOL_CIDR=CALICO_IPV4POOL_CIDR_REPLACE/16 \ + -e CALICO_IPV4POOL_IPIP=always \ + -e FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT \ + -e FELIX_IPV6SUPPORT=false \ + -e FELIX_LOGSEVERITYSCREEN=info \ + -e FELIX_IPINIPMTU=1440 \ + -e FELIX_HEALTHENABLED=true \ + -e IP= \ + -v /etc/calico/ssl:/etc/calico/ssl \ + -v /var/run/calico:/var/run/calico \ + -v /lib/modules:/lib/modules \ + -v /run/docker/plugins:/run/docker/plugins \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/log/calico:/var/log/calico \ + calico/node:v2.6.2 +ExecStop=/usr/bin/docker rm -f calico-node +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg new file mode 100644 index 0000000..44ada86 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: calicoApiConfig +metadata: +spec: + datastoreType: "etcdv2" + etcdEndpoints: ETCD_ENDPOINTS_REPLACE \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json new file mode 100644 index 0000000..052fe73 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{ + "insecure-registries": ["DOCKER_REGISTRY_REPLACE"], + "cluster-store":"CLUSTER_STORE_REPLACE", + "cluster-advertise":"LOCAL_HOST_IP_REPLACE:2375", + "dns": ["YARN_DNS_HOST_REPLACE", "LOCAL_DNS_HOST_REPLACE"], + "hosts": ["tcp://LOCAL_HOST_IP_REPLACE:2375","unix:///var/run/docker.sock"] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service new file mode 100644 index 0000000..f444f99 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=Docker Application Container Engine +Documentation=http://docs.docker.io + +[Service] +Environment="PATH=/usr/bin:/bin:/sbin:/usr/bin:/usr/sbin" +ExecStart=/usr/bin/dockerd --log-level=error +ExecStartPost=/sbin/iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT +ExecReload=/bin/kill -s HUP $MAINPID +Restart=on-failure +RestartSec=5 +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +Delegate=yes +KillMode=process + +[Install] +WantedBy=multi-user.target \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service new file mode 100644 index 0000000..5a44ce2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=etcd +Documentation=https://github.com/coreos/etcd + +[Service] +Type=notify +Restart=always +RestartSec=5s +LimitNOFILE=40000 +TimeoutStartSec=0 + +ExecStart=/usr/bin/etcd \ + --name=ETCD_NODE_NAME_REPLACE \ + --data-dir=/var/lib/etcd \ + --listen-client-urls=http://LOCAL_HOST_REPLACE:2379,http://127.0.0.1:2379 \ + --listen-peer-urls=http://LOCAL_HOST_REPLACE:2380 \ + --advertise-client-urls=http://LOCAL_HOST_REPLACE:2379 \ + --initial-advertise-peer-urls=http://LOCAL_HOST_REPLACE:2380 \ + --initial-cluster=INITIAL_CLUSTER_REPLACE \ + --initial-cluster-token=etcd-token \ + --initial-cluster-state=new + +[Install] +WantedBy=multi-user.target \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg new file mode 100644 index 0000000..9d9f0ca --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +yarn.nodemanager.local-dirs=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE +yarn.nodemanager.linux-container-executor.group=yarn +yarn.nodemanager.log-dirs=YARN_NODEMANAGER_LOG_DIRS_REPLACE +banned.users=root +allowed.system.users=yarn +min.user.id=500 + +[docker] +module.enabled=true +docker.binary=/usr/bin/docker +docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE,DAC_READ_SEARCH,SYS_PTRACE,SYS_ADMIN +docker.allowed.networks=bridge,host,none,CALICO_NETWORK_NAME_REPLACE +docker.allowed.ro-mounts=/etc/group,/etc/passwd,/etc/krb5.conf,YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,regex:^nvidia_driver_.*$ +docker.allowed.rw-mounts=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,YARN_NODEMANAGER_LOG_DIRS_REPLACE +docker.privileged-containers.enabled=false +docker.trusted.registries=local,centos,hortonworks,DOCKER_REGISTRY_REPLACE +docker.allowed.volume-drivers=nvidia-docker +docker.allowed.devices=regex:^/dev/nvidia.*$ + +[gpu] +module.enabled=true + +[cgroups] +root=/sys/fs/cgroup +yarn-hierarchy=YARN_HIERARCHY_REPLACE \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh new file mode 100644 index 0000000..ffbc38f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Solution: Grant user yarn the access to /sys/fs/cgroup/cpu,cpuacct, +# which is the subfolder of cgroup mount destination. +chown :yarn -R /sys/fs/cgroup/cpu,cpuacct +chmod g+rwx -R /sys/fs/cgroup/cpu,cpuacct + +# If GPUs are usedï¼the access to cgroup devices folder is neede as well +chown :yarn -R /sys/fs/cgroup/devices +chmod g+rwx -R /sys/fs/cgroup/devices http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh new file mode 100644 index 0000000..bd40fcf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description download calico bin +## @audience public +## @stability stable +function download_calico_bin() +{ + # submarin http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_CALICOCTL_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calicoctl + MY_CALICO_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calico + MY_CALICO_IPAM_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calico-ipam + else + MY_CALICOCTL_DOWNLOAD_URL=${CALICOCTL_DOWNLOAD_URL} + MY_CALICO_DOWNLOAD_URL=${CALICO_DOWNLOAD_URL} + MY_CALICO_IPAM_DOWNLOAD_URL=${CALICO_IPAM_DOWNLOAD_URL} + fi + + mkdir -p ${DOWNLOAD_DIR}/calico + + if [[ -f ${DOWNLOAD_DIR}/calico/calico ]]; then + echo "${DOWNLOAD_DIR}/calico/calico is exist." + else + echo "download ${MY_CALICO_DOWNLOAD_URL} ..." + wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICO_DOWNLOAD_URL} + fi + + if [[ -f ${DOWNLOAD_DIR}/calico/calicoctl ]]; then + echo "${DOWNLOAD_DIR}/calico is exist." + else + echo "download ${MY_CALICOCTL_DOWNLOAD_URL} ..." + wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICOCTL_DOWNLOAD_URL} + fi + + if [[ -f ${DOWNLOAD_DIR}/calico/calico-ipam ]]; then + echo "${DOWNLOAD_DIR}/calico/calico-ipam is exist." + else + echo "download ${MY_CALICO_IPAM_DOWNLOAD_URL} ..." + wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICO_IPAM_DOWNLOAD_URL} + fi +} + +## @description install calico bin +## @audience public +## @stability stable +function install_calico_bin() +{ + download_calico_bin + + cp -f ${DOWNLOAD_DIR}/calico/calico /usr/bin/calico + cp -f ${DOWNLOAD_DIR}/calico/calicoctl /usr/bin/calicoctl + cp -f ${DOWNLOAD_DIR}/calico/calico-ipam /usr/bin/calico-ipam + + chmod +x /usr/bin/calico + chmod +x /usr/bin/calicoctl + chmod +x /usr/bin/calico-ipam +} + +## @description install calico config +## @audience public +## @stability stable +function install_calico_config() +{ + mkdir -p /etc/calico + + cp -rf ${PACKAGE_DIR}/calico ${INSTALL_TEMP_DIR}/ + + # 1. replace etcdEndpoints + # etcdEndpoints: https://10.196.69.173:2379,https://10.196.69.174:2379,https://10.196.69.175:2379 + etcdEndpoints='' + index=0 + etcdHostsSize=${#ETCD_HOSTS[@]} + for item in ${ETCD_HOSTS[@]} + do + index=$(($index+1)) + etcdEndpoints="${etcdEndpoints}http:\/\/${item}:2379" + if [[ ${index} -lt ${etcdHostsSize} ]]; then + etcdEndpoints=${etcdEndpoints}"," + fi + done + # echo "etcdEndpoints=${etcdEndpoints}" + sed -i "s/ETCD_ENDPOINTS_REPLACE/${etcdEndpoints}/g" $INSTALL_TEMP_DIR/calico/calicoctl.cfg >>$LOG + + if [[ ! -d /etc/calico ]]; then + mkdir /etc/calico + else + rm -rf /etc/calico/* + fi + + cp -f $INSTALL_TEMP_DIR/calico/calicoctl.cfg /etc/calico/calicoctl.cfg + + sed -i "s/ETCD_ENDPOINTS_REPLACE/${etcdEndpoints}/g" $INSTALL_TEMP_DIR/calico/calico-node.service >>$LOG + sed -i "s/CALICO_IPV4POOL_CIDR_REPLACE/${CALICO_IPV4POOL_CIDR}/g" $INSTALL_TEMP_DIR/calico/calico-node.service >>$LOG + cp $INSTALL_TEMP_DIR/calico/calico-node.service /etc/systemd/system/ >>$LOG + + systemctl daemon-reload + systemctl enable calico-node.service +} + +## @description modify kernel network config +## @audience public +## @stability stable +function kernel_network_config() +{ + if [ `grep -c "net.ipv4.conf.all.rp_filter=1" /etc/sysctl.conf` -eq '0' ]; then + echo "net.ipv4.conf.all.rp_filter=1" >>/etc/sysctl.conf + fi + + if [ `grep -c "net.ipv4.ip_forward=1" /etc/sysctl.conf` -eq '0' ]; then + echo "net.ipv4.ip_forward=1" >>/etc/sysctl.conf + fi + + sysctl -p +} + +## @description check if the calico-network exist +## @audience public +## @stability stable +function calico_network_exist() +{ + local dockerNetwokInfo=`docker network ls --filter NAME=${CALICO_NETWORK_NAME}` + echo ${dockerNetwokInfo} | grep ${CALICO_NETWORK_NAME} +} + +## @description verification calico +## @audience public +## @stability stable +function verification_calico() +{ + echo " ===== Check if the network between 2 containers can be connected =====" + local claicoNetworkExist=`calico_network_exist` + if [[ "$claicoNetworkExist" = "" ]]; then + echo "Create a calico network" + docker network create --driver calico --ipam-driver calico-ipam ${CALICO_NETWORK_NAME} + else + echo "calico network ${CALICO_NETWORK_NAME} is exist." + fi + + local verifyA="verify-calico-network-A" + local verifyAInfo=`containers_exist ${verifyA}` + if [[ -n "$verifyAInfo" ]]; then + echo "Delete existing container ${verifyA}." + docker stop ${verifyA} + docker rm ${verifyA} + fi + echo "Create containers verify-calico-network-A" + docker run --net ${CALICO_NETWORK_NAME} --name ${verifyA} -tid busybox + + local verifyB="verify-calico-network-B" + local verifyBInfo=`containers_exist ${verifyB}` + if [[ -n "$verifyBInfo" ]]; then + echo "Delete existing container ${verifyB}." + docker stop ${verifyB} + docker rm ${verifyB} + fi + echo "Create containers verify-calico-network-B" + docker run --net ${CALICO_NETWORK_NAME} --name ${verifyB} -tid busybox + + echo -e "\033[33m${verifyA} ping ${verifyB}\033[0m" + docker exec ${verifyA} ping ${verifyB} -c 5 +} + +## @description install calico +## @audience public +## @stability stable +function install_calico() +{ + kernel_network_config + install_calico_bin + install_calico_config + start_calico + verification_calico +} + +## @description uninstall calico +## @audience public +## @stability stable +function uninstall_calico() +{ + echo "stop calico-node.service" + systemctl stop calico-node.service + + echo "rm /usr/bin/calico ..." + rm /usr/bin/calicoctl + rm /usr/bin/calico + rm /usr/bin/calico-ipam + + rm -rf /etc/calico/ + rm /etc/systemd/system/calico-node.service + systemctl daemon-reload +} + +## @description start calico +## @audience public +## @stability stable +function start_calico() +{ + systemctl restart calico-node.service + systemctl status calico-node.service +} + +## @description stop calico +## @audience public +## @stability stable +function stop_calico() +{ + systemctl stop calico-node.service + systemctl status calico-node.service +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh new file mode 100644 index 0000000..26f19cc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description download docker rmp +## @audience public +## @stability stable +function download_docker_rpm() +{ + # download http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_DOCKER_ENGINE_SELINUX_RPM="${DOWNLOAD_HTTP}/downloads/docker/${DOCKER_ENGINE_SELINUX_RPM}" + MY_DOCKER_ENGINE_RPM="${DOWNLOAD_HTTP}/downloads/docker/${DOCKER_ENGINE_RPM}" + else + MY_DOCKER_ENGINE_SELINUX_RPM=${DOCKER_REPO}/${DOCKER_ENGINE_SELINUX_RPM} + MY_DOCKER_ENGINE_RPM=${DOCKER_REPO}/${DOCKER_ENGINE_RPM} + fi + + # download docker rpm + if [[ -f ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} ]]; then + echo "${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} is exist." + else + echo "download ${MY_DOCKER_ENGINE_SELINUX_RPM} ..." + wget -P ${DOWNLOAD_DIR}/docker/ ${MY_DOCKER_ENGINE_SELINUX_RPM} + fi + + if [[ -f ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} ]]; then + echo "${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} is exist." + else + echo "download ${MY_DOCKER_ENGINE_RPM} ..." + wget -P ${DOWNLOAD_DIR}/docker/ ${MY_DOCKER_ENGINE_RPM} + fi +} + +## @description install docker bin +## @audience public +## @stability stable +function install_docker_bin() +{ + download_docker_rpm + + yum -y localinstall ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} + yum -y localinstall ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} +} + +## @description uninstall docker bin +## @audience public +## @stability stable +function uninstall_docker_bin() +{ + download_docker_rpm + + yum -y remove ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} + yum -y remove ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} +} + +## @description install docker config +## @audience public +## @stability stable +function install_docker_config() +{ + rm -rf ${INSTALL_TEMP_DIR}/docker + cp -rf ${PACKAGE_DIR}/docker ${INSTALL_TEMP_DIR}/ + + # replace cluster-store + # "cluster-store":"etcd://10.196.69.173:2379,10.196.69.174:2379,10.196.69.175:2379" + # char '/' need to escape '\/' + clusterStore="etcd:\/\/" + index=1 + etcdHostsSize=${#ETCD_HOSTS[@]} + for item in ${ETCD_HOSTS[@]} + do + clusterStore="${clusterStore}${item}:2379" + if [[ ${index} -lt ${etcdHostsSize}-1 ]]; then + clusterStore=${clusterStore}"," + fi + index=$(($index+1)) + done + echo "clusterStore=${clusterStore}" + sed -i "s/CLUSTER_STORE_REPLACE/${clusterStore}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG + + sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG + sed -i "s/LOCAL_HOST_IP_REPLACE/${LOCAL_HOST_IP}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG + sed -i "s/YARN_DNS_HOST_REPLACE/${YARN_DNS_HOST}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG + sed -i "s/LOCAL_DNS_HOST_REPLACE/${LOCAL_DNS_HOST}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG + + # Delete the ASF license comment in the daemon.json file, otherwise it will cause a json format error. + sed -i '1,16d' $INSTALL_TEMP_DIR/docker/daemon.json + + if [ ! -d "/etc/docker" ]; then + mkdir /etc/docker + fi + + cp $INSTALL_TEMP_DIR/docker/daemon.json /etc/docker/ + cp $INSTALL_TEMP_DIR/docker/docker.service /etc/systemd/system/ >>$LOG +} + +## @description install docker +## @audience public +## @stability stable +function install_docker() +{ + install_docker_bin + install_docker_config + + systemctl daemon-reload + systemctl enable docker.service +} + +## @description unstall docker +## @audience public +## @stability stable +function uninstall_docker() +{ + echo "stop docker service" + systemctl stop docker + + echo "remove docker" + uninstall_docker_bin + + rm /etc/docker/daemon.json >>$LOG + rm /etc/systemd/system/docker.service >>$LOG + + systemctl daemon-reload +} + +## @description start docker +## @audience public +## @stability stable +function start_docker() +{ + systemctl restart docker + systemctl status docker + docker info +} + +## @description stop docker +## @audience public +## @stability stable +function stop_docker() +{ + systemctl stop docker + systemctl status docker +} + +## @description check if the containers exist +## @audience public +## @stability stable +function containers_exist() +{ + local dockerContainersInfo=`docker ps -a --filter NAME=$1` + echo ${dockerContainersInfo} | grep $1 +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh new file mode 100644 index 0000000..62af827 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description start download server +## @audience public +## @stability stable +function start_download_server() +{ + if [[ "$DOWNLOAD_SERVER_IP" != "$LOCAL_HOST_IP" ]]; then + echo -e "\033[31mERROR: Only $DOWNLOAD_SERVER_IP can start the download service.\033[0m" + return 1 + fi + + echo -e "You can put the install package file in the \033[34m${DOWNLOAD_DIR}\033[0m folder first, Or automatic download." + echo -n "Do you want to start download http server?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + download_etcd_bin + download_calico_bin + download_docker_rpm + download_nvidia_driver + download_nvidia_docker_bin + download_yarn_container_executor + + python -m SimpleHTTPServer ${DOWNLOAD_SERVER_PORT} + fi +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh new file mode 100644 index 0000000..17f59b2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description check operation System +## @audience public +## @stability stable +function check_operationSystem() +{ + echo -e "The submarine assembly support \033[32m[centos-release-7-3.1611.el7.centos.x86_64]\033[0m or higher operating system version." + + case ${OPERATING_SYSTEM} in + centos) + local operationSystemVersion=`rpm --query centos-release` + echo -e "The current operating system version is \e[31m[${operationSystemVersion}]\e[0m" | tee -a $LOG + ;; + *) + echo -e "\033[31mWARN: The submarine assembly Unsupported [${OPERATING_SYSTEM}] operating system\033[0m" + ;; + esac +} + +## @description update operation System Kernel +## @audience public +## @stability stable +function update_operationSystemKernel() +{ + echo "If the server is unable to connect to the network, execute the following command yourself: + wget http://vault.centos.org/7.3.1611/os/x86_64/Packages/kernel-headers-3.10.0-514.el7.x86_64.rpm + rpm -ivh kernel-headers-3.10.0-514.el7.x86_64.rpm" + + echo -n "Do you want to kernel upgrades?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + echo "Now try to use the yum command for kernel upgrades ..." + yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r) + + local kernelVersion=`uname -r` + echo -e "After the upgrade, the operating system kernel version is \e[31m${kernelVersion}\e[0m" | tee -a $LOG + fi +} + +## @description check operation system kernel +## @audience public +## @stability stable +function check_operationSystemKernel() +{ +case ${OPERATING_SYSTEM} in +centos) + local kernelVersion=`uname -r` + + echo -e "Submarine support operating system kernel version is \033[32m 3.10.0-514.el7.x86_64 \033[0m" | tee -a $LOG + echo -e "Current operating system kernel version is \e[31m${kernelVersion}\e[0m" | tee -a $LOG + + update_operationSystemKernel + ;; +*) + echo -e "\033[31m WARN: The submarine assembly Unsupported operating system [${OPERATING_SYSTEM}] \033[0m" + ;; +esac +} + +## @description get gcc version +## @audience public +## @stability stable +function get_gcc_version() +{ + local gccVersion=`gcc --version` + version=${gccVersion%Copyright*} + echo $version +} + +## @description install gcc +## @audience public +## @stability stable +function install_gcc() +{ + echo -n "Do you want to install gcc?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]]; then + echo "Execute the yum install gcc make g++ command" + yum install gcc make g++ + + local gccVersion=`gcc --version` + echo -e "After the install, the gcc version is \e[31m${gccVersion}\e[0m" | tee -a $LOG + fi +} + +## @description check gcc Version +## @audience public +## @stability stable +function check_gccVersion() +{ + local gccVersionInfo=`gcc --version` + local gccVersion=${gccVersionInfo%Copyright*} + + if [[ "$gccVersion" = "" ]]; then + echo "The gcc was not installed on the system. Automated installation ..." + install_gcc + else + echo -e "Submarine gcc version need \033[34mgcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-11)\033[0m or higher." + echo -e "Current gcc version was \033[34m${gccVersion}\033[0m" + fi +} + +## @description check GPU +## @audience public +## @stability stable +function check_GPU() +{ + gpuInfo=`lspci | grep -i nvidia` + + if [[ "$gpuInfo" = "" ]]; then + echo -e "\033[31mERROR: The system did not detect the GPU graphics card.\033[0m" + else + echo -e "\033[32mINFO: The system detect the GPU graphics card.\033[0m" + fi +} + +## @description check user group +## @audience public +## @stability stable +function check_userGroup() +{ + echo -e "check hadoop user group ..." + + echo -e "Hadoop runs the required user [hdfs, mapred, yarn] and groups [hdfs, mapred, yarn, hadoop] installed by ambari." + echo -e "If you are not using ambari for hadoop installation, +then you can add the user and group by root by executing the following command: +\033[34madduser hdfs +adduser mapred +adduser yarn +addgroup hadoop +usermod -aG hdfs,hadoop hdfs +usermod -aG mapred,hadoop mapred +usermod -aG yarn,hadoop yarn +usermod -aG hdfs,hadoop hadoop +groupadd docker +usermod -aG docker yarn +usermod -aG docker hadoop\033[0m\n" + + echo -e "check docker user group ..." + # check user group + DOCKER_USER_GROUP='docker' + egrep "^${DOCKER_USER_GROUP}" /etc/group >& /dev/null + if [[ $? -ne 0 ]]; then + echo -e "user group ${DOCKER_USER_GROUP} does not exist, Please execute the following command:" + echo -e "\033[34mgroupadd $DOCKER_USER_GROUP\033[0m" + fi + + # check user + USER_GROUP=(yarn hadoop) + for user in ${USER_GROUP[@]} + do + egrep "^${user}" /etc/passwd >& /dev/null + if [[ $? -ne 0 ]]; then + echo -e "User ${user} does not exist, Please execute the following command:" + echo -e "\033[34madduser ${user}\033[0m" + echo -e "\033[34musermod -aG ${DOCKER_USER_GROUP} ${user}\033[0m" + fi + + echo -e "Please execute the following command:" + echo -e "\033[34musermod -aG ${DOCKER_USER_GROUP} ${user}\033[0m" + done +} + +## @description Some preparatory work for nvidia driver installation +## @audience public +## @stability stable +function prepare_nvidia_environment() +{ + echo "prepare nvidia environment ..." + + yum -y update + yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r) + + yum -y install epel-release + yum -y install dkms + + echo -e "\033[34m ===== Please manually execute the following command ===== +# 1. Disable nouveau +# Add the content 'rd.driver.blacklist=nouveau nouveau.modeset=0' +# to the 'GRUB_CMDLINE_LINUX' configuration item in the /etc/default/grub file. +root:> vi /etc/default/grub +vi:> GRUB_CMDLINE_LINUX=\"rd.driver.blacklist=nouveau nouveau.modeset=0 ...\" + +# 2. Generate configuration +root:> grub2-mkconfig -o /boot/grub2/grub.cfg + +# 3. Open (new) /etc/modprobe.d/blacklist.conf, add content 'blacklist nouveau' +root:> vi /etc/modprobe.d/blacklist.conf +vi:> blacklist nouveau + +# 4. Update configuration and reboot +root:> mv /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r)-nouveau.img +root:> dracut /boot/initramfs-$(uname -r).img $(uname -r) +root:> reboot +\033[0m" +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh new file mode 100644 index 0000000..48df80e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description download etcd bin +## @audience public +## @stability stable +function download_etcd_bin() +{ + # my download http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_ETCD_DOWNLOAD_URL="${DOWNLOAD_HTTP}/downloads/etcd/${ETCD_TAR_GZ}" + else + MY_ETCD_DOWNLOAD_URL=${ETCD_DOWNLOAD_URL} + fi + + if [[ -f "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ}" ]]; then + echo "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ} is exist." + else + echo "download ${MY_ETCD_DOWNLOAD_URL} ..." + wget -P ${DOWNLOAD_DIR}/etcd ${MY_ETCD_DOWNLOAD_URL} + fi +} + +## @description install etcd bin +## @audience public +## @stability stable +function install_etcd_bin() +{ + download_etcd_bin + + # install etcd bin + mkdir -p ${INSTALL_TEMP_DIR} + rm -rf ${INSTALL_TEMP_DIR}/etcd-*-linux-amd6 + tar zxvf ${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ} -C ${INSTALL_TEMP_DIR} + + cp -f ${INSTALL_TEMP_DIR}/etcd-*-linux-amd64/etcd /usr/bin + cp -f ${INSTALL_TEMP_DIR}/etcd-*-linux-amd64/etcdctl /usr/bin + + mkdir -p /var/lib/etcd + chmod -R a+rw /var/lib/etcd +} + +## @description install etcd config +## @audience public +## @stability stable +function install_etcd_config() +{ + # config etcd.service + rm -rf ${INSTALL_TEMP_DIR}/etcd + cp -rf ${PACKAGE_DIR}/etcd ${INSTALL_TEMP_DIR}/ + + # 1. Replace name with ETCD_NODE_NAME_REPLACE based on the location of the local IP in $ETCD_HOSTS + indexEtcdList=$(indexByEtcdHosts ${LOCAL_HOST_IP}) + # echo ${indexEtcdList} + etcdNodeName="etcdnode${indexEtcdList}" + # echo ${etcdNodeName} + sed -i "s/ETCD_NODE_NAME_REPLACE/${etcdNodeName}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG + + # 2. Replace local IP address + sed -i "s/LOCAL_HOST_REPLACE/${LOCAL_HOST_IP}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG + + # 3. Replace the initial-cluster parameter + # --initial-cluster=etcdnode1=http://10.196.69.173:2380,etcdnode2=http://10.196.69.174:2380,etcdnode3=http://10.196.69.175:2380 \ + initialCluster='' + index=0 + etcdHostsSize=${#ETCD_HOSTS[@]} + for item in ${ETCD_HOSTS[@]} + do + # char '/' need to escape '\/' + initialCluster="${initialCluster}etcdnode${index}=http:\/\/${item}:2380" + if [[ ${index} -lt ${etcdHostsSize}-1 ]]; then + initialCluster=${initialCluster}"," + fi + index=$(($index+1)) + done + #echo "initialCluster=${initialCluster}" + sed -i "s/INITIAL_CLUSTER_REPLACE/${initialCluster}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG + + cp $INSTALL_TEMP_DIR/etcd/etcd.service /etc/systemd/system/ >>$LOG +} + +## @description install etcd +## @audience public +## @stability stable +function install_etcd() +{ + index=$(indexByEtcdHosts ${LOCAL_HOST_IP}) + if [ -z "$index" ]; then + echo -e "STOP: This host\033[31m[${LOCAL_HOST_IP}]\033[0m is not in the ETCD server list\033[31m[${ETCD_HOSTS[@]}]\033[0m" + return 1 + fi + + install_etcd_bin + + install_etcd_config + + systemctl daemon-reload + systemctl enable etcd.service +} + +## @description uninstall etcd +## @audience public +## @stability stable +function uninstall_etcd() +{ + echo "stop etcd.service" + systemctl stop etcd.service + + echo "rm etcd ..." + rm /usr/bin/etcd + rm /usr/bin/etcdctl + rm -rf /var/lib/etcd + rm /etc/systemd/system/etcd.service + + systemctl daemon-reload +} + +## @description start etcd +## @audience public +## @stability stable +function start_etcd() +{ + systemctl restart etcd.service + + echo " ===== Check the status of the etcd service =====" + echo " exec etcdctl cluster-health" + etcdctl cluster-health + echo " exec etcdctl cluster-health" + etcdctl member list +} + +## @description stop etcd +## @audience public +## @stability stable +function stop_etcd() +{ + systemctl stop etcd.service +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh new file mode 100644 index 0000000..9df5d7e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description install yarn +## @audience public +## @stability stable +function install_yarn() +{ + install_yarn_container_executor + install_yarn_config +} + +## @description uninstall yarn +## @audience public +## @stability stable +function uninstall_yarn() +{ + rm -rf /etc/yarn/sbin/Linux-amd64-64/* + rm -rf /etc/yarn/sbin/etc/hadoop/* +} + +## @description download yarn container executor +## @audience public +## @stability stable +function download_yarn_container_executor() +{ + # my download http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_YARN_CONTAINER_EXECUTOR_PATH="${DOWNLOAD_HTTP}/downloads/hadoop/container-executor" + else + MY_YARN_CONTAINER_EXECUTOR_PATH=${YARN_CONTAINER_EXECUTOR_PATH} + fi + + if [ ! -d "${DOWNLOAD_DIR}/hadoop" ]; then + mkdir -p ${DOWNLOAD_DIR}/hadoop + fi + + if [[ -f "${DOWNLOAD_DIR}/hadoop/container-executor" ]]; then + echo "${DOWNLOAD_DIR}/hadoop/container-executor is exist." + else + if [[ -n "$DOWNLOAD_HTTP" ]]; then + echo "download ${MY_YARN_CONTAINER_EXECUTOR_PATH} ..." + wget -P ${DOWNLOAD_DIR}/hadoop ${MY_YARN_CONTAINER_EXECUTOR_PATH} + else + echo "copy ${MY_YARN_CONTAINER_EXECUTOR_PATH} ..." + cp ${MY_YARN_CONTAINER_EXECUTOR_PATH} ${DOWNLOAD_DIR}/hadoop/ + fi + fi +} + +## @description install yarn container executor +## @audience public +## @stability stable +function install_yarn_container_executor() +{ + echo "install yarn container executor file ..." + + download_yarn_container_executor + + if [ ! -d "/etc/yarn/sbin/Linux-amd64-64" ]; then + mkdir -p /etc/yarn/sbin/Linux-amd64-64 + fi + if [ -f "/etc/yarn/sbin/Linux-amd64-64/container-executor" ]; then + rm /etc/yarn/sbin/Linux-amd64-64/container-executor + fi + + cp -f ${DOWNLOAD_DIR}/hadoop/container-executor /etc/yarn/sbin/Linux-amd64-64 + + sudo chmod 6755 /etc/yarn/sbin/Linux-amd64-64 + sudo chown :yarn /etc/yarn/sbin/Linux-amd64-64/container-executor + sudo chmod 6050 /etc/yarn/sbin/Linux-amd64-64/container-executor +} + +## @description install yarn config +## @audience public +## @stability stable +function install_yarn_config() +{ + echo "install yarn config file ..." + + cp -R ${PACKAGE_DIR}/hadoop ${INSTALL_TEMP_DIR}/ + + find="/" + replace="\/" + escape_yarn_nodemanager_local_dirs=${YARN_NODEMANAGER_LOCAL_DIRS//$find/$replace} + escape_yarn_nodemanager_log_dirs=${YARN_NODEMANAGER_LOG_DIRS//$find/$replace} + escape_yarn_hierarchy=${YARN_HIERARCHY//$find/$replace} + + sed -i "s/YARN_NODEMANAGER_LOCAL_DIRS_REPLACE/${escape_yarn_nodemanager_local_dirs}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG + sed -i "s/YARN_NODEMANAGER_LOG_DIRS_REPLACE/${escape_yarn_nodemanager_log_dirs}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG + sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG + sed -i "s/CALICO_NETWORK_NAME_REPLACE/${CALICO_NETWORK_NAME}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG + sed -i "s/YARN_HIERARCHY_REPLACE/${escape_yarn_hierarchy}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG + + # Delete the ASF license comment in the container-executor.cfg file, otherwise it will cause a cfg format error. + sed -i '1,16d' $INSTALL_TEMP_DIR/hadoop/container-executor.cfg + + if [ ! -d "/etc/yarn/sbin/etc/hadoop" ]; then + mkdir -p /etc/yarn/sbin/etc/hadoop + fi + + cp -f $INSTALL_TEMP_DIR/hadoop/container-executor.cfg /etc/yarn/sbin/etc/hadoop/ +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh new file mode 100644 index 0000000..68f91c1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh @@ -0,0 +1,444 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description main menu +## @audience public +## @stability stable +main_menu() +{ +cat<<MENULIST +==================================================================================== + SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION} + +HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT} +==================================================================================== +[Main menu] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \e[32m1.prepare system environment [..]\e[0m" +echo -e " \e[32m2.install component [..]\e[0m" +echo -e " \e[32m3.uninstall component [..]\e[0m" +echo -e " \e[32m4.start component [..]\e[0m" +echo -e " \e[32m5.stop component [..]\e[0m" +echo -e " \e[32m6.start download server [..]\e[0m" +echo -e "" +echo -e " \e[32mq.quit\e[0m" +cat<<MENULIST +==================================================================================== +MENULIST + +echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m6\e[0m,\e[32mq\e[0m(quit)]:" +} + +## @description check menu +## @audience public +## @stability stable +check_menu() +{ +cat<<MENULIST +==================================================================================== + SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION} + +HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT} +==================================================================================== +[Main menu] > [prepare system environment] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \e[32m1.prepare operation system\e[0m" +echo -e " \e[32m2.prepare operation system kernel\e[0m" +echo -e " \e[32m3.prepare GCC version\e[0m" +echo -e " \e[32m4.check GPU\e[0m" +echo -e " \e[32m5.prepare user&group\e[0m" +echo -e " \e[32m6.prepare nvidia environment\e[0m" +echo -e "" +echo -e " \e[32mb.back main menu\e[0m" +cat<<MENULIST +==================================================================================== +MENULIST + +echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m5\e[0m,\e[32mb\e[0m(back)]:" +} + +## @description install menu +## @audience public +## @stability stable +install_menu() +{ +cat<<MENULIST +==================================================================================== + SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION} + +HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT} +==================================================================================== +[Main menu] > [install component] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \e[32m1.instll etcd\e[0m" +echo -e " \e[32m2.instll docker\e[0m" +echo -e " \e[32m3.instll calico network\e[0m" +echo -e " \e[32m4.instll nvidia driver\e[0m" +echo -e " \e[32m5.instll nvidia docker\e[0m" +echo -e " \e[32m6.instll yarn container-executor\e[0m" +echo -e " \e[32m7.instll submarine autorun script\e[0m" +echo -e "" +echo -e " \e[32mb.back main menu\e[0m" +cat<<MENULIST +==================================================================================== +MENULIST + +echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m7\e[0m,\e[32mb\e[0m(back)]:" +} + +## @description unstall menu +## @audience public +## @stability stable +uninstall_menu() +{ +cat<<MENULIST +==================================================================================== + SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION} + +HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT} +==================================================================================== +[Main menu] > [uninstll component] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \e[32m1.uninstll etcd\e[0m" +echo -e " \e[32m2.uninstll docker\e[0m" +echo -e " \e[32m3.uninstll calico network\e[0m" +echo -e " \e[32m4.uninstll nvidia driver\e[0m" +echo -e " \e[32m5.uninstll nvidia docker\e[0m" +echo -e " \e[32m6.uninstll yarn container-executor\e[0m" +echo -e " \e[32m7.uninstll submarine autorun script\e[0m" +echo -e "" +echo -e " \e[32mb.back main menu\e[0m" +cat<<MENULIST +==================================================================================== +MENULIST + +echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m7\e[0m,\e[32mb\e[0m(back)]:" +} + +## @description start menu +## @audience public +## @stability stable +start_menu() +{ +cat<<MENULIST +==================================================================================== + SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION} + +HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT} +==================================================================================== +[Main menu] > [stop component] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \e[32m1.start etcd\e[0m" +echo -e " \e[32m2.start docker\e[0m" +echo -e " \e[32m3.start calico network\e[0m" +echo -e "" +echo -e " \e[32mb.back main menu\e[0m" +cat<<MENULIST +==================================================================================== +MENULIST + +echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m3\e[0m,\e[32mb\e[0m(back)]:" +} + +## @description stop menu +## @audience public +## @stability stable +stop_menu() +{ +cat<<MENULIST +==================================================================================== + SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION} + +HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT} +==================================================================================== +[Main menu] > [stop component] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \e[32m1.stop etcd\e[0m" +echo -e " \e[32m2.stop docker\e[0m" +echo -e " \e[32m3.stop calico network\e[0m" +echo -e "" +echo -e " \e[32mb.back main menu\e[0m" +cat<<MENULIST +==================================================================================== +MENULIST + +echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m3\e[0m,\e[32mb\e[0m(back)]:" +} + +## @description menu operation +## @audience public +## @stability stable +menu_index="0" +menu() +{ + clear + # echo "menu_index-menu_choice=$menu_index-$menu_choice" + case $menu_index in + "0") + main_menu + ;; + "1") + check_menu + ;; + "2") + install_menu + ;; + "3") + uninstall_menu + ;; + "4") + start_menu + ;; + "5") + stop_menu + ;; + "6") + start_download_server + ;; + "q") + exit 1 + ;; + *) + echo "error input!" + menu_index="0" + menu_choice="0" + main_menu + ;; + esac + + read menu_choice +} + +## @description menu process +## @audience public +## @stability stable +menu_process() +{ + process=0 + unset myselect + # echo "debug=$menu_index-$menu_choice" + case "$menu_index-$menu_choice" in + "1-b"|"2-b"|"3-b"|"4-b"|"5-b"|"6-b") + menu_index="0" + menu_choice="0" + ;; +# check system environment + "1-1") + myselect="y" + check_operationSystem + ;; + "1-2") + myselect="y" + check_operationSystemKernel + ;; + "1-3") + myselect="y" + check_gccVersion + ;; + "1-4") + myselect="y" + check_GPU + ;; + "1-5") + myselect="y" + check_userGroup + ;; + "1-6") + myselect="y" + prepare_nvidia_environment + ;; +# install component + "2-1") + echo -n "Do you want to install etcd?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + install_etcd + fi + ;; + "2-2") + echo -n "Do you want to install docker?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + install_docker + fi + ;; + "2-3") + echo -n "Do you want to install calico network?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + install_calico + fi + ;; + "2-4") + echo -n "Do you want to install nvidia driver?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + install_nvidia + fi + ;; + "2-5") + echo -n "Do you want to install nvidia docker?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + install_nvidia_docker + fi + ;; + "2-6") + echo -n "Do you want to install yarn container-executor?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + install_yarn + fi + ;; + "2-7") + echo -n "Do you want to install submarine auto start script?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + install_submarine + fi + ;; +# uninstall component + "3-1") + echo -n "Do you want to uninstall etcd?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + uninstall_etcd + fi + ;; + "3-2") + echo -n "Do you want to uninstall docker?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + uninstall_docker + fi + ;; + "3-3") + echo -n "Do you want to uninstall calico network?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + uninstall_calico + fi + ;; + "3-4") + echo -n "Do you want to uninstall nvidia driver?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + uninstall_nvidia + fi + ;; + "3-5") + echo -n "Do you want to uninstall nvidia docker?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + uninstall_nvidia_docker + fi + ;; + "3-6") + echo -n "Do you want to uninstall yarn container-executor?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + uninstall_yarn + fi + ;; + "3-7") + echo -n "Do you want to uninstall submarine autostart script?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + uninstall_submarine + fi + ;; +# startup component + "4-1") + echo -n "Do you want to startup etcd?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + start_etcd + fi + ;; + "4-2") + echo -n "Do you want to startup docker?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + start_docker + fi + ;; + "4-3") + echo -n "Do you want to startup calico network?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + start_calico + fi + ;; +# stop component + "5-1") + echo -n "Do you want to stop etcd?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + stop_etcd + fi + ;; + "5-2") + echo -n "Do you want to stop docker?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + stop_docker + fi + ;; + "5-3") + echo -n "Do you want to stop calico network?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + stop_calico + fi + ;; + esac + + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + process=1 + fi + +# echo "process=$process" + return $process +} + http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia-docker.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia-docker.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia-docker.sh new file mode 100644 index 0000000..3441f28 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia-docker.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description download nvidia docker bin +## @audience public +## @stability stable +function download_nvidia_docker_bin() +{ + # download http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_NVIDIA_DOCKER_RPM_URL="${DOWNLOAD_HTTP}/downloads/nvidia-docker/${NVIDIA_DOCKER_RPM}" + else + MY_NVIDIA_DOCKER_RPM_URL=${NVIDIA_DOCKER_RPM_URL} + fi + + if [[ -f "${DOWNLOAD_DIR}/nvidia-docker/${NVIDIA_DOCKER_RPM}" ]]; then + echo "${DOWNLOAD_DIR}/nvidia-docker/${NVIDIA_DOCKER_RPM} is exist." + else + echo "download ${MY_NVIDIA_DOCKER_RPM_URL} ..." + wget -P ${DOWNLOAD_DIR}/nvidia-docker/ ${MY_NVIDIA_DOCKER_RPM_URL} + fi +} + +## @description install nvidia docker +## @audience public +## @stability stable +function install_nvidia_docker() +{ + download_nvidia_docker_bin + + sudo rpm -i ${DOWNLOAD_DIR}/nvidia-docker/${NVIDIA_DOCKER_RPM} + + echo -e "\033[32m===== Start nvidia-docker =====\033[0m" + sudo systemctl start nvidia-docker + + echo -e "\033[32m===== Check nvidia-docker status =====\033[0m" + systemctl status nvidia-docker + + echo -e "\033[32m===== Check nvidia-docker log =====\033[0m" + journalctl -u nvidia-docker + + echo -e "\033[32m===== Test nvidia-docker-plugin =====\033[0m" + curl http://localhost:3476/v1.0/docker/cli + + # create nvidia driver library path + if [ ! -d "/var/lib/nvidia-docker/volumes/nvidia_driver" ]; then + echo "WARN: /var/lib/nvidia-docker/volumes/nvidia_driver folder path is not exist!" + mkdir -p /var/lib/nvidia-docker/volumes/nvidia_driver + fi + + local nvidiaVersion=`get_nvidia_version` + echo -e "\033[31m nvidia detect version is ${nvidiaVersion}\033[0m" + + mkdir /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion} + mkdir /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/bin + mkdir /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64 + + cp /usr/bin/nvidia* /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/bin + cp /usr/lib64/libcuda* /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64 + cp /usr/lib64/libnvidia* /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64 + + echo -e "\033[32m===== Please manually execute the following command =====\033[0m" + echo -e "\033[32mshell:> nvidia-docker run --rm ${DOCKER_REGISTRY}/nvidia/cuda:9.0-devel nvidia-smi +# If you don't see the list of graphics cards above, the NVIDIA driver installation failed. ===== +\033[0m" + + echo -e "\033[32m===== Please manually execute the following command =====\033[0m" + echo -e "\033[32m# Test with tf.test.is_gpu_available() +shell:> nvidia-docker run -it ${DOCKER_REGISTRY}/tensorflow/tensorflow:1.9.0-gpu bash +# In docker container +container:> python +python:> import tensorflow as tf +python:> tf.test.is_gpu_available() +python:> exit() +\033[0m" +} + +## @description uninstall nvidia docker +## @audience public +## @stability stable +function uninstall_nvidia_docker() +{ + echo "This method is not implemented." +} + http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh new file mode 100644 index 0000000..7b2b1de --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nvidia_run_file="" + +## @description get nvidia version +## @audience public +## @stability stable +function get_nvidia_version() +{ + chmod +x ${DOWNLOAD_DIR}/nvidia/nvidia-detect + local nvidia_detect_info=`${DOWNLOAD_DIR}/nvidia/nvidia-detect -v` + echo $nvidia_detect_info | sed "s/^.*This device requires the current \([0-9.]*\).*/\1/" +} + +## @description download nvidia driver +## @audience public +## @stability stable +function download_nvidia_driver() +{ + # submarin http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_NVIDIA_DETECT_URL="${DOWNLOAD_HTTP}/downloads/nvidia/nvidia-detect" + else + MY_NVIDIA_DETECT_URL=${NVIDIA_DETECT_URL} + fi + + # download nvidia_detect + if [[ -f "${DOWNLOAD_DIR}/nvidia/nvidia-detect" ]]; then + echo "${DOWNLOAD_DIR}/nvidia/nvidia-detect is exist." + else + echo "download ${MY_NVIDIA_DETECT_URL} ..." + wget -P ${DOWNLOAD_DIR}/nvidia/ ${MY_NVIDIA_DETECT_URL} + fi + + echo "execution nvidia-detect to check the graphics card ..." + local nvidiaVersion=`get_nvidia_version` + echo -e "detect nvidia version is \033[31m${nvidiaVersion}\033[0m" + + # download NVIDIA driver + if [[ "$nvidiaVersion" = "" ]]; then + echo -e "\033[31mERROR: No graphics card device detected.\033[0m" + return 1 + else + nvidia_run_file="NVIDIA-Linux-x86_64-${nvidiaVersion}.run" + + # submarin http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_NVIDIA_DRIVER_RUN_URL="${DOWNLOAD_HTTP}/downloads/nvidia/${nvidia_run_file}" + else + # http://us.download.nvidia.com/XFree86/Linux-x86_64/390.87/NVIDIA-Linux-x86_64-390.87.run + MY_NVIDIA_DRIVER_RUN_URL="http://us.download.nvidia.com/XFree86/Linux-x86_64/${nvidiaVersion}/${nvidia_run_file}"; + fi + + if [[ -f ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} ]]; then + echo "NVIDIA driver files already exist in the ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} directory." + echo "===== Please make sure the ${DOWNLOAD_DIR}/nvidia/nvidia/${nvidia_run_file} file is complete and can be used normally. =====" + else + echo "Download the NVIDIA driver from the ${MY_NVIDIA_DRIVER_RUN_URL}" + wget -P ${DOWNLOAD_DIR}/nvidia/ ${MY_NVIDIA_DRIVER_RUN_URL} + fi + fi +} + +## @description install nvidia +## @audience public +## @stability stable +function install_nvidia() +{ + download_nvidia_driver + + # Confirm that the system disables nouveau + local disable_nouveau_info=`lsmod | grep nouveau` + if [[ "$disable_nouveau_info" = "" ]]; then + echo "===== Start installing the NVIDIA driver =====" + echo -e "Some options during the installation +Would you like to register the kernel module sources with DKMS? + This will allow DKMS to automatically build a new module, if you install a different kernel later. \033[33m[Yes]\033[0m +Install NVIDIA's 32-bit compatibility libraries \033[33m[Yes]\033[0m +centos Install NVIDIA's 32-bit compatibility libraries \033[33m[Yes]\033[0m +Would you like to run the nvidia-xconfig utility to automatically update your X configuration file... \033[33m[No]\033[0m" + sleep 2 + sh ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} + else + echo -e "ERROR: Nouveau is not disabled" + return 1 + fi + + echo -e "\033[32m===== execute nvidia-smi. You should be able to see the list of graphics cards =====\033[0m" + sleep 1 + nvidia-smi +} + +## @description uninstall nvidia +## @audience public +## @stability stable +function uninstall_nvidia() +{ + if [ ! -f "/usr/bin/nvidia-uninstall" ]; then + echo -e "\033[31mERROR: /usr/bin/nvidia-uninstall file is not exist!\033[0m" + return 1 + fi + + echo -e "execute /usr/bin/nvidia-uninstall" + /usr/bin/nvidia-uninstall +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh new file mode 100644 index 0000000..b73d5e7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description install submarine autorun script +## @audience public +## @stability stable +function install_submarine() +{ + cp ${PACKAGE_DIR}/submarine/submarine.sh /etc/rc.d/init.d/submarine.sh + chmod +x /etc/rc.d/init.d/submarine.sh + chmod +x /etc/rc.d/rc.local + + if [ `grep -c "/etc/rc.d/init.d/submarine.sh" /etc/rc.d/rc.local` -eq '0' ]; then + echo "/etc/rc.d/init.d/submarine.sh">> /etc/rc.d/rc.local + fi +} + +## @description uninstall submarine autorun script +## @audience public +## @stability stable +function uninstall_submarine() +{ + rm /etc/rc.d/init.d/submarine.sh +} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
