HADOOP-14816. Update Dockerfile to use Xenial. Contributed by Allen Wittenauer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca8ddc6a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca8ddc6a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca8ddc6a Branch: refs/heads/YARN-1011 Commit: ca8ddc6aa413de347866ad9a0a3407356a280a1f Parents: cbd2b73 Author: Chris Douglas <cdoug...@apache.org> Authored: Thu Oct 19 16:33:47 2017 -0700 Committer: Chris Douglas <cdoug...@apache.org> Committed: Thu Oct 19 16:45:18 2017 -0700 ---------------------------------------------------------------------- dev-support/docker/Dockerfile | 161 +++++++++++++++------------ dev-support/docker/hadoop_env_checks.sh | 15 ++- 2 files changed, 98 insertions(+), 78 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca8ddc6a/dev-support/docker/Dockerfile ---------------------------------------------------------------------- diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 31ac611..8af002d 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -18,21 +18,28 @@ # Dockerfile for installing the necessary dependencies for building Hadoop. # See BUILDING.txt. - -FROM ubuntu:trusty +FROM ubuntu:xenial WORKDIR /root +##### +# Disable suggests/recommends +##### +RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras +RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras + ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_TERSE true ###### -# Install common dependencies from packages +# Install common dependencies from packages. Versions here are either +# sufficient or irrelevant. # # WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default # Ubuntu Java. See Java section below! ###### -RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ +RUN apt-get -q update && apt-get -q install -y \ + apt-utils \ build-essential \ bzip2 \ curl \ @@ -42,7 +49,6 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ gcc \ git \ gnupg-agent \ - make \ libbz2-dev \ libcurl4-openssl-dev \ libfuse-dev \ @@ -51,106 +57,110 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ libsnappy-dev \ libssl-dev \ libtool \ + locales \ + make \ pinentry-curses \ pkg-config \ - protobuf-compiler \ - protobuf-c-compiler \ python \ python2.7 \ - python2.7-dev \ python-pip \ + python-pkg-resources \ + python-setuptools \ + python-wheel \ rsync \ + software-properties-common \ snappy \ + sudo \ zlib1g-dev ####### -# Oracle Java +# OpenJDK 8 ####### +RUN apt-get -q install -y openjdk-8-jdk -RUN echo "dot_style = mega" > "/root/.wgetrc" -RUN echo "quiet = on" >> "/root/.wgetrc" - -RUN apt-get -q install --no-install-recommends -y software-properties-common -RUN add-apt-repository -y ppa:webupd8team/java -RUN apt-get -q update - -# Auto-accept the Oracle JDK license -RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections -RUN apt-get -q install --no-install-recommends -y oracle-java8-installer - -#### -# Apps that require Java -### -RUN apt-get -q update && apt-get -q install --no-install-recommends -y ant +####### +# OpenJDK 9 +# w/workaround for +# https://bugs.launchpad.net/ubuntu/+source/openjdk-9/+bug/1593191 +####### +RUN apt-get -o Dpkg::Options::="--force-overwrite" \ + -q install -y \ + openjdk-9-jdk-headless -###### -# Install Apache Maven -###### -RUN mkdir -p /opt/maven && \ - curl -L -s -S \ - https://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \ - -o /opt/maven.tar.gz && \ - tar xzf /opt/maven.tar.gz --strip-components 1 -C /opt/maven -ENV MAVEN_HOME /opt/maven -ENV PATH "${PATH}:/opt/maven/bin" +####### +# Set default Java +####### +# +# By default, OpenJDK sets the default Java to the highest version. +# We want the opposite, soooooo.... +# +RUN update-java-alternatives --set java-1.8.0-openjdk-amd64 +RUN update-alternatives --get-selections | grep -i jdk | \ + while read line; do \ + alternative=$(echo $line | awk '{print $1}'); \ + path=$(echo $line | awk '{print $3}'); \ + newpath=$(echo $path | sed -e 's/java-9/java-8/'); \ + update-alternatives --set $alternative $newpath; \ + done ###### -# Install cmake +# Install cmake 3.1.0 (3.5.1 ships with Xenial) ###### RUN mkdir -p /opt/cmake && \ curl -L -s -S \ - https://cmake.org/files/v3.1/cmake-3.1.0-Linux-x86_64.tar.gz \ - -o /opt/cmake.tar.gz && \ + https://cmake.org/files/v3.1/cmake-3.1.0-Linux-x86_64.tar.gz \ + -o /opt/cmake.tar.gz && \ tar xzf /opt/cmake.tar.gz --strip-components 1 -C /opt/cmake ENV CMAKE_HOME /opt/cmake ENV PATH "${PATH}:/opt/cmake/bin" ###### -# Install findbugs +# Install Google Protobuf 2.5.0 (2.6.0 ships with Xenial) ###### -RUN mkdir -p /opt/findbugs && \ +RUN mkdir -p /opt/protobuf-src && \ curl -L -s -S \ - https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \ - -o /opt/findbugs.tar.gz && \ - tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs -ENV FINDBUGS_HOME /opt/findbugs -ENV PATH "${PATH}:/opt/findbugs/bin" + https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \ + -o /opt/protobuf.tar.gz && \ + tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src +RUN cd /opt/protobuf-src && ./configure --prefix=/opt/protobuf && make install +ENV PROTOBUF_HOME /opt/protobuf +ENV PATH "${PATH}:/opt/protobuf/bin" + +###### +# Install Apache Maven 3.3.9 (3.3.9 ships with Xenial) +###### +RUN apt-get -q update && apt-get -q install -y maven +ENV MAVEN_HOME /usr + +###### +# Install findbugs 3.0.1 (3.0.1 ships with Xenial) +###### +RUN apt-get -q update && apt-get -q install -y findbugs +ENV FINDBUGS_HOME /usr #### -# Install shellcheck +# Install shellcheck (0.4.6, the latest as of 2017-09-26) #### -RUN apt-get -q install -y cabal-install -RUN mkdir /root/.cabal -RUN echo "remote-repo: hackage.fpcomplete.com:http://hackage.fpcomplete.com/"; >> /root/.cabal/config -#RUN echo "remote-repo: hackage.haskell.org:http://hackage.haskell.org/"; > /root/.cabal/config -RUN echo "remote-repo-cache: /root/.cabal/packages" >> /root/.cabal/config -RUN cabal update -RUN cabal install shellcheck --global +RUN add-apt-repository -y ppa:jonathonf/ghc-8.0.2 +RUN apt-get -q update && apt-get -q install -y shellcheck #### -# Install bats +# Install bats (0.4.0, the latest as of 2017-09-26, ships with Xenial) #### -RUN add-apt-repository -y ppa:duggan/bats -RUN apt-get -q update -RUN apt-get -q install --no-install-recommends -y bats +RUN apt-get -q update && apt-get -q install -y bats #### -# Install pylint +# Install pylint (always want latest) #### -RUN pip install pylint +RUN pip2 install pylint #### # Install dateutil.parser #### -RUN pip install python-dateutil +RUN pip2 install python-dateutil ### -# Avoid out of memory errors in builds -### -ENV MAVEN_OPTS -Xms256m -Xmx512m - -### -# Install node js tools for web UI frameowkr +# Install node.js for web UI framework (4.2.6 ships with Xenial) ### RUN apt-get -y install nodejs && \ ln -s /usr/bin/nodejs /usr/bin/node && \ @@ -160,18 +170,29 @@ RUN apt-get -y install nodejs && \ npm install -g ember-cli ### +# Avoid out of memory errors in builds +### +ENV MAVEN_OPTS -Xms256m -Xmx1g + + +### # Everything past this point is either not needed for testing or breaks Yetus. # So tell Yetus not to read the rest of the file: # YETUS CUT HERE ### #### -# Install Forrest (for Apache Hadoop website) +# Install svn, Ant, & Forrest (for Apache Hadoop website) ### -RUN mkdir -p /usr/local/apache-forrest ; \ - curl -s -S -O http://archive.apache.org/dist/forrest/0.8/apache-forrest-0.8.tar.gz ; \ - tar xzf *forrest* --strip-components 1 -C /usr/local/apache-forrest ; \ - echo 'forrest.home=/usr/local/apache-forrest' > build.properties +RUN apt-get -q update && apt-get -q install -y ant subversion + +RUN mkdir -p /opt/apache-forrest && \ + curl -L -s -S \ + https://archive.apache.org/dist/forrest/0.8/apache-forrest-0.8.tar.gz \ + -o /opt/forrest.tar.gz && \ + tar xzf /opt/forrest.tar.gz --strip-components 1 -C /opt/apache-forrest +RUN echo 'forrest.home=/opt/apache-forrest' > build.properties +ENV FORREST_HOME=/opt/apache-forrest # Add a welcome message and environment checks. ADD hadoop_env_checks.sh /root/hadoop_env_checks.sh http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca8ddc6a/dev-support/docker/hadoop_env_checks.sh ---------------------------------------------------------------------- diff --git a/dev-support/docker/hadoop_env_checks.sh b/dev-support/docker/hadoop_env_checks.sh index 5cb4b2b..c9420e7 100755 --- a/dev-support/docker/hadoop_env_checks.sh +++ b/dev-support/docker/hadoop_env_checks.sh @@ -16,6 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# SHELLDOC-IGNORE + # ------------------------------------------------------- function showWelcome { cat <<Welcome-message @@ -80,14 +82,10 @@ End-of-message # ------------------------------------------------------- -# Configurable low water mark in GiB -MINIMAL_MEMORY_GiB=2 - function warnIfLowMemory { - MINIMAL_MEMORY=$((MINIMAL_MEMORY_GiB*1024*1024)) # Convert to KiB - INSTALLED_MEMORY=$(fgrep MemTotal /proc/meminfo | awk '{print $2}') - if [ $((INSTALLED_MEMORY)) -le $((MINIMAL_MEMORY)) ]; - then + MINIMAL_MEMORY=2046755 + INSTALLED_MEMORY=$(grep -F MemTotal /proc/meminfo | awk '{print $2}') + if [[ $((INSTALLED_MEMORY)) -lt $((MINIMAL_MEMORY)) ]]; then cat <<End-of-message _ ___ ___ @@ -103,7 +101,8 @@ Your system is running on very little memory. This means it may work but it wil most likely be slower than needed. If you are running this via boot2docker you can simply increase -the available memory to atleast ${MINIMAL_MEMORY_GiB} GiB (you have $((INSTALLED_MEMORY/(1024*1024))) GiB ) +the available memory to at least ${MINIMAL_MEMORY}KiB +(you have ${INSTALLED_MEMORY}KiB ) End-of-message fi --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
