Repository: incubator-crail Updated Branches: refs/heads/master 312ccc32c -> 86f1bd350
Docker build: TCP and RDMA tier Docker build files for the TCP and RDMA tier. User can extend configuration to use NVMf or other tiers. https://issues.apache.org/jira/projects/CRAIL/issues/CRAIL-49 Signed-off-by: Jonas Pfefferle <peppe...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/incubator-crail/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crail/commit/31be5fed Tree: http://git-wip-us.apache.org/repos/asf/incubator-crail/tree/31be5fed Diff: http://git-wip-us.apache.org/repos/asf/incubator-crail/diff/31be5fed Branch: refs/heads/master Commit: 31be5fed423491afac680a409e9e267cfc4991af Parents: 312ccc3 Author: Jonas Pfefferle <peppe...@apache.org> Authored: Thu Jul 26 13:35:25 2018 +0200 Committer: Jonas Pfefferle <peppe...@apache.org> Committed: Tue Aug 14 13:19:17 2018 +0200 ---------------------------------------------------------------------- docker/Dockerfile | 59 ++++++++++++++++++++++++++++++++++++++++ docker/RDMA/Dockerfile | 36 ++++++++++++++++++++++++ docker/RDMA/crail-site.conf | 13 +++++++++ docker/core-site.xml | 36 ++++++++++++++++++++++++ docker/crail-env.sh | 3 ++ docker/crail-site.conf | 7 +++++ 6 files changed, 154 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/31be5fed/docker/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..910dd81 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.) + +FROM ubuntu:18.04 +MAINTAINER Apache Crail <d...@crail.apache.org> + +# TODO: automate update version on new release +ARG GIT_COMMIT="v1.0" +ENV LOG_COMMIT=$GIT_COMMIT + +RUN echo "Crail-$LOG_COMMIT install openjdk8, git and envsubst" && \ + apt-get update && apt-get install --no-install-recommends -y \ + openjdk-8-jdk-headless \ + git \ + gettext-base +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +ENV PATH=${PATH}:${JAVA_HOME}/bin + +# We need to install maven separately since it otherwise pulls in java 10 +RUN echo "Crail-$LOG_COMMIT install maven" && \ + apt-get install --no-install-recommends -y maven + + +RUN echo "Crail-$LOG_COMMIT clone & build Crail repo" && \ + git clone https://github.com/apache/incubator-crail.git && \ + cd incubator-crail && \ + git checkout ${GIT_COMMIT} && \ + mvn -DskipTests package +ENV CRAIL_HOME /incubator-crail/assembly/target/crail-1.0-bin +ENV PATH=${PATH}:${CRAIL_HOME}/bin + +RUN echo "Crail-$LOG_COMMIT update crail script to update core-site.xml" && \ + sed -i '$ienvsubst < $CRAIL_HOME/conf/core-site.xml > $CRAIL_HOME/conf/core-site.xml' $CRAIL_HOME/bin/crail + +ENV NAMENODE_HOST=localhost +ENV NAMENODE_PORT=9060 +ENV INTERFACE=eth0 +ENV DATAPATH=/dev/hugepages/data +ENV STORAGELIMIT=1073741824 +ENV CACHEPATH=/dev/hugepages/cache +ENV CACHELIMIT=1073741824 + +COPY ./crail-env.sh $CRAIL_HOME/conf +COPY ./core-site.xml $CRAIL_HOME/conf +COPY ./crail-site.conf $CRAIL_HOME/conf + +ENTRYPOINT ["crail"] http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/31be5fed/docker/RDMA/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/RDMA/Dockerfile b/docker/RDMA/Dockerfile new file mode 100644 index 0000000..0863d06 --- /dev/null +++ b/docker/RDMA/Dockerfile @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.) + +FROM crail:latest +MAINTAINER Apache Crail <d...@crail.apache.org> + +# TODO: automate update version +ARG DISNI_COMMIT="v1.5" + +RUN echo "Crail-$LOG_COMMIT install rdma libraries and autotools" && \ + apt-get install -y --no-install-recommends \ + autoconf autotools-dev automake libtool make g++ \ + librdmacm-dev libibverbs-dev ibverbs-providers + +RUN echo "Crail-$LOG_COMMIT clone and build disni native library" && \ + cd && git clone https://github.com/zrlio/disni.git && \ + cd ~/disni/libdisni && \ + git checkout $DISNI_COMMIT && \ + ./autoprepare.sh && \ + ./configure --with-jdk=$JAVA_HOME && \ + make && make install +ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib + +COPY ./crail-site.conf $CRAIL_HOME/conf http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/31be5fed/docker/RDMA/crail-site.conf ---------------------------------------------------------------------- diff --git a/docker/RDMA/crail-site.conf b/docker/RDMA/crail-site.conf new file mode 100644 index 0000000..ceeaaa3 --- /dev/null +++ b/docker/RDMA/crail-site.conf @@ -0,0 +1,13 @@ +crail.cachepath $CACHEPATH +crail.cachelimit $CACHELIMIT + + +crail.namenode.address crail://$NAMENODE_HOST:$NAMENODE_PORT +crail.namenode.rpctype org.apache.crail.namenode.rpc.darpc.DaRPCNameNode +crail.namenode.darpc.polling true + +crail.storage.rdma.interface $INTERFACE +crail.storage.rdma.datapath $DATAPATH +crail.storage.rdma.storagelimit $STORAGELIMIT + +crail.storage.types org.apache.crail.storage.rdma.RdmaStorageTier http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/31be5fed/docker/core-site.xml ---------------------------------------------------------------------- diff --git a/docker/core-site.xml b/docker/core-site.xml new file mode 100644 index 0000000..81e1c03 --- /dev/null +++ b/docker/core-site.xml @@ -0,0 +1,36 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + <property> + <name>fs.crail.impl</name> + <value>org.apache.crail.hdfs.CrailHadoopFileSystem</value> + </property> + <property> + <name>fs.defaultFS</name> + <value>crail://$NAMENODE_HOST:$NAMENODE_PORT</value> + </property> + <property> + <name>fs.AbstractFileSystem.crail.impl</name> + <value>org.apache.crail.hdfs.CrailHDFS</value> + </property> + <property> + <name>io.file.buffer.size</name> + <value>1048576</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/31be5fed/docker/crail-env.sh ---------------------------------------------------------------------- diff --git a/docker/crail-env.sh b/docker/crail-env.sh new file mode 100644 index 0000000..1bfb348 --- /dev/null +++ b/docker/crail-env.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# This env varibale allows setting additional java parameter +#CRAIL_EXTRA_JAVA_OPTIONS="-Xmx24G -Xmn16G" http://git-wip-us.apache.org/repos/asf/incubator-crail/blob/31be5fed/docker/crail-site.conf ---------------------------------------------------------------------- diff --git a/docker/crail-site.conf b/docker/crail-site.conf new file mode 100644 index 0000000..6b260b6 --- /dev/null +++ b/docker/crail-site.conf @@ -0,0 +1,7 @@ +crail.namenode.address crail://$NAMENODE_HOST:$NAMENODE_PORT +crail.cachepath $CACHEPATH +crail.cachelimit $CACHELIMIT +crail.storage.tcp.interface $INTERFACE +crail.storage.tcp.datapath $DATAPATH +crail.storage.tcp.storagelimit $STORAGELIMIT +