This is an automated email from the ASF dual-hosted git repository.
apalumbo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mahout.git
The following commit(s) were added to refs/heads/master by this push:
new 06866f6 (nojira) remove old or uinfinished files before release (#386)
06866f6 is described below
commit 06866f6807994ad6aebcbbbb6f98de0036ac553c
Author: Andrew Palumbo <[email protected]>
AuthorDate: Sat Dec 14 15:52:23 2019 -0800
(nojira) remove old or uinfinished files before release (#386)
* remove old or uinfinished files
* note about change-scala version script.
---
Dockerfile | 92 ---------------------
README.md | 4 +-
buildtools/change-scala-version.sh | 3 +
.../spark/src/main/assembly/dependency-reduced.xml | 4 +-
.../org/apache/mahout/sparkbindings/package.scala | 4 +-
entrypoint.sh | 94 ----------------------
pom.xml | 7 +-
.../docker/src/main/dockerfiles/.gitignore | 0
.../docker/src/main/dockerfiles/Dockerfile | 76 -----------------
.../docker/src/main/dockerfiles/entrypoint.sh | 93 ---------------------
10 files changed, 13 insertions(+), 364 deletions(-)
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index 0897f83..0000000
--- a/Dockerfile
+++ /dev/null
@@ -1,92 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-FROM openjdk:8-alpine
-
-ENV spark_uid=185
-
-ENV SCALA_MAJOR 2.12
-ENV HADOOP_MAJOR 2.7
-ENV SPARK_MAJOR_MINOR 2.4.4
-
-# Before building the mahout docker image, we must build a spark distrobution
following
-# the instructions in http://spark.apache.org/docs/latest/building-spark.html.
-# this Dockerfile will build Spark version 2.4.4 against Scala 2.12 by default.
-# docker build -t mahout:latest -f
resource_managers/docker/kubernetes/src/main/dockerfiles/Dockerfile .
-
-
-RUN set -ex && \
- apk upgrade --no-cache && \
- ln -s /lib /lib64 && \
- apk add --no-cache bash tini libc6-compat linux-pam krb5 krb5-libs nss
curl openssl && \
- mkdir -p /opt/mahout && \
- mkdir -p /opt/mahout/examples && \
- mkdir -p /opt/mahout/work-dir && \
- mkdir -p /opt/spark && \
- export MAHOUT_DOCKER_HOME=/opt/mahout && \
- export SPARK_VERSION=spark-${SPARK_MAJOR_MINOR} && \
- export SPARK_BASE=/opt/spark && \
- export SPARK_HOME=${SPARK_BASE}/${SPARK_VERSION}
- export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m" && \
- export
SPARK_SRC_URL="https://archive.apache.org/dist/spark/${SPARK_VERSION}/${SPARK_VERSION}.tgz";
&& \
- export
SPARK_SRC_SHA512_URL="https://archive.apache.org/dist/spark/${SPARK_VERSION}/${SPARK_VERSION}.tgz.sha512";
&& \
- export
SPARK_SRC_SHA512="D33096E7EFBC4B131004C85FB5833AC3BAB8F097644CBE68D89ADC81F5144B5535337FD0082FA04A19C2870BD7D84758E8AE9C6EC1C7F3DF9FED35325EEA8928"
&& \
- curl -LfsS $SPARK_SRC_URL -o ${SPARK_BASE}/${SPARK_VERSION}.tgz && \
- curl -LfsS $SPARK_SRC_SHA512_URL -o
${SPARK_BASE}/${SPARK_VERSION}.tgz.sha512
- #$SPARK_HOME/$SPARK_VERSION.sha512 ${SPARK_HOME}/$SPARK_VERSION.tgz |
shasum -a 512 -c - && \
- tar -xzvf ${SPARK_BASE}/${SPARK_VERSION}.tgz -C ${SPARK_BASE}/&& \
- echo ${SPARK_BASE}/${SPARK_VERSION}
- sh ${SPARK_HOME}/dev/change-scala-version.sh ${SCALA_MAJOR} && \
- sh ${SPARK_HOME}/dev/make-distribution.sh --name ${DATE}-${REVISION} --pip
--tgz -DzincPort=${ZINC_PORT} \
- -Phadoop-${HADOOP_MAJOR} -Pkubernetes -Pkinesis-asl -Phive
-Phive-thriftserver
-
- #sh ${SPARK_HOME}/build/mvn -Pkubernetes -Pscala-${scala_version}
-DskipTests clean package
- touch /opt/mahout/RELEASE && \
- # below is for nodes. for the moment lets get a master up
- # rm /bin/sh && \
- # ln -sv /bin/bash /bin/sh && \
- # echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd
-
-# build mahout
-RUN mvn clean package install
-
-ENV MAHOUT_HOME /opt/mahout
-COPY lib ${MAHPOUT_HOME}/lib
-COPY bin ${MAHPOUT_HOME}/bin
-COPY entrypoint.sh ${MAHPOUT_HOME}
-COPY Dockerfile ${MAHPOUT_HOME}
-COPY examples ${MAHPOUT_HOME}/examples
-
-ENV SPARK_HOME /opt/spark
-COPY spark-build/jars ${SPARK_HOME}/jars
-COPY spark-build/bin ${SPARK_HOME}/bin
-COPY spark-build/sbin ${SPARK_HOME}/sbin
-COPY spark-build/kubernetes/tests ${SPARK_HOME}/tests
-COPY spark-build/data ${SPARK_HOME}/data
-
-ENV MAHOUT_CLASSPATH ${MAHOUT_HOME}/lib
-ENV SPARK_CLASSPATH ${SPARK_HOME}/jars
-
-WORKDIR /opt/mahout/work-dir
-RUN chmod g+w /opt/mahout/work-dir
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
-
-# Specify the User that the actual main process will run as
-USER ${spark_uid}
\ No newline at end of file
diff --git a/README.md b/README.md
index b90c5a4..09ca2e1 100644
--- a/README.md
+++ b/README.md
@@ -44,7 +44,7 @@ To use the Samsara environment you'll need to include both
the engine neutral ma
```
<dependency>
<groupId>org.apache.mahout</groupId>
- <artifactId>mahout-math-scala_2.10</artifactId>
+ <artifactId>mahout-math-scala_2.11</artifactId>
<version>${mahout.version}</version>
</dependency>
```
@@ -52,7 +52,7 @@ and a dependency for back end engine translation, e.g:
```
<dependency>
<groupId>org.apache.mahout</groupId>
- <artifactId>mahout-spark_2.10</artifactId>
+ <artifactId>mahout-spark_2.11</artifactId>
<version>${mahout.version}</version>
</dependency>
```
diff --git a/buildtools/change-scala-version.sh
b/buildtools/change-scala-version.sh
index b4128ca..76a2255 100755
--- a/buildtools/change-scala-version.sh
+++ b/buildtools/change-scala-version.sh
@@ -19,6 +19,9 @@
# A BIG Shoutout to the Bros and Bro-ettes at Apache Spark for this
+## TODO post 14.1 fix this script ---- Idempotency is lost somewhere, and
+## this script will mess an entire IntelliJ project structure up !!######
+
set -ex
VALID_VERSIONS=( 2.10 2.11 2.12 )
diff --git a/engine/spark/src/main/assembly/dependency-reduced.xml
b/engine/spark/src/main/assembly/dependency-reduced.xml
index d87291b..78a03cd 100644
--- a/engine/spark/src/main/assembly/dependency-reduced.xml
+++ b/engine/spark/src/main/assembly/dependency-reduced.xml
@@ -42,8 +42,8 @@
<!-- <include>com.tdunning:t-digest</include>-->
<include>org.apache.commons:commons-math3</include>
<include>it.unimi.dsi:fastutil</include>
- <include>org.apache.mahout:mahout-native-viennacl_2.10</include>
- <include>org.apache.mahout:mahout-native-viennacl-omp_2.10</include>
+ <include>org.apache.mahout:mahout-native-viennacl_2.11</include>
+ <include>org.apache.mahout:mahout-native-viennacl-omp_2.11</include>
<include>org.bytedeco:javacpp</include>
</includes>
</dependencySet>
diff --git
a/engine/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala
b/engine/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala
index ef2bd52..b0b5a29 100644
--- a/engine/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala
+++ b/engine/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala
@@ -265,8 +265,8 @@ package object sparkbindings {
// no need for mapreduce jar in Spark
// j.matches(".*mahout-mr-\\d.*\\.jar") ||
j.matches(".*spark_\\d.*\\.jar") ||
- // vcl jars: mahout-native-viennacl_2.10.jar,
- // mahout-native-viennacl-omp_2.10.jar
+ // vcl jars: mahout-native-viennacl_2.11.jar,
+ // mahout-native-viennacl-omp_2.11.jar
// j.matches(".*mahout-native-viennacl_\\d.*\\\\.jar") ||
// j.matches(".*mahout-native-viennacl-omp_\\d.*\\.jar")||
j.matches(".*mahout-native-viennacl*.jar")||
diff --git a/entrypoint.sh b/entrypoint.sh
deleted file mode 100755
index bb40767..0000000
--- a/entrypoint.sh
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/usr/bin/env bash
-
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# echo commands to the terminal output
-set -ex
-
-# Check whether there is a passwd entry for the container UID
-myuid=$(id -u)
-mygid=$(id -g)
-# turn off -e for getent because it will return error code in anonymous uid
case
-set +e
-uidentry=$(getent passwd $myuid)
-set -e
-
-# If there is no passwd entry for the container UID, attempt to create one
-if [ -z "$uidentry" ] ; then
- if [ -w /etc/passwd ] ; then
- echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous
uid}:${SPARK_HOME}:/bin/false" >> /etc/passwd
- else
- echo "Container ENTRYPOINT failed to add passwd entry for anonymous
UID"
- fi
-fi
-
-###Todo: ensure that MAHOUT_HOME is properly established as an env var.
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*:${$MAHOUT_HOME}/lib/*"
-env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' >
/tmp/java_opts.txt
-readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-
-
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH:$MAHOUT_CLASSPATH";
-fi
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- --conf "spark.executor.extraLibraryPath"=\
- "${MAHOUT_HOME}/lib/mahout-spark_*-dependency-reduced.jar"
- --deploy-mode client
- "$@"
- )
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
- -cp "$SPARK_CLASSPATH:$MAHOUT_CLASSPATH"
- org.apache.spark.executor.CoarseGrainedExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --conf spark.executor.extraLibraryPath=\
- "$MAHOUT_HOME/lib/mahout-spark_*-dependency-reduced.jar"
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- )
- ;;
-
- *)
- echo "Non-spark-on-k8s command provided, proceeding in pass-through
mode..."
- CMD=("$@")
- ;;
-esac
-
-# Execute the container CMD under tini for better hygiene
-exec /sbin/tini -s -- "${CMD[@]}"
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index bfe49e6..ea1f714 100644
--- a/pom.xml
+++ b/pom.xml
@@ -768,11 +768,12 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-site-plugin</artifactId>
<version>3.7.1</version>
- <configuration>
+ <!-- below was not parsing -->
+ <!--<configuration>
<reportPlugins>
<plugin>
<artifactId>maven-project-info-reports-plugin</artifactId>
- <!--<version>2.2</version>-->
+ <!–<version>2.2</version>–>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
@@ -786,7 +787,7 @@
</configuration>
</plugin>
</reportPlugins>
- </configuration>
+ </configuration>-->
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
diff --git
a/resource-managers/kubernetes/docker/src/main/dockerfiles/.gitignore
b/resource-managers/kubernetes/docker/src/main/dockerfiles/.gitignore
deleted file mode 100644
index e69de29..0000000
diff --git
a/resource-managers/kubernetes/docker/src/main/dockerfiles/Dockerfile
b/resource-managers/kubernetes/docker/src/main/dockerfiles/Dockerfile
deleted file mode 100644
index 6f588ab..0000000
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/Dockerfile
+++ /dev/null
@@ -1,76 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-FROM openjdk:8-alpine
-
-ARG spark_uid=185
-
-# Before building the mahout docker image, we must build a spark distrobution
following
-# the instructions in http://spark.apache.org/docs/latest/building-spark.html.
-# this Dockerfile will build the Spark version 2.4.3 against Scala 2.12.
-# docker build -t mahout:latest -f
resource_managers/docker/kubernetes/src/main/dockerfiles/Dockerfile .
-
-
-RUN set -ex && \
- apk upgrade --no-cache && \
- ln -s /lib /lib64 && \
- apk add --no-cache bash tini libc6-compat linux-pam krb5 krb5-libs nss
curl openssl && \
- mkdir -p /opt/mahout && \
- mkdir -p /opt/mahout/examples && \
- mkdir -p /opt/mahout/work-dir && \
- mkdir -p /opt/spark && \
- export MAHOUT_HOME=. && \
- mkdir -p $MAHOUT_HOME/spark-build && \
- export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m" && \
- export SPARK_HOME=$MAHOUT_HOME/spark-build/ && \
- export
SPARK_SRC_URL="https://www.apache.org/dyn/closer.lua/spark/spark-2.4.3/spark-2.4.3.tgz";
&& \
- export
SPARK_SRC_SHA256="3EAEA3B0A81A717BB43CE6EE0BB2C3B8351EF080DB9499AF66F9F22C8A18D38C5E1426CBFEF04AFD2A4002ACE5B28A6BEACBCE4E5E42506F4FD270B05D0DB379"
&& \
- curl -LfsS $SPARK_SRC_URL -o $SPARK_HOME/spark-2.4.3.tgz && \
- echo "${SPARK_SRC_SHA256} ${SPARK_HOME}/spark-2.4.3.tgz" | sha512sum -c -
&& \
- $SPARK_HOME/dev/change-scala-version.sh 2.12 && \
- $SPARK_HOME/build/mvn -Pkubernetes -Pscala-2.12 -DskipTests clean package
&& \
- touch /opt/mahout/RELEASE && \
- rm /bin/sh && \
- ln -sv /bin/bash /bin/sh && \
- echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
- chgrp root /etc/passwd && chmod ug+rw /etc/passwd
-
-
-COPY lib /opt/mahout/lib
-COPY bin /opt/mahout/bin
-COPY resource-managers/kubernetes/docker/src/main/dockerfiles/entrypoint.sh
/opt/
-COPY examples /opt/mahout/examples
-
-COPY spark-build/jars /opt/spark/jars
-COPY spark-build/bin /opt/spark/bin
-COPY spark-build/sbin /opt/spark/sbin
-COPY spark-build/kubernetes/tests /opt/spark/tests
-COPY spark-build/data /opt/spark/data
-
-ENV MAHOUT_HOME /opt/mahout
-ENV SPARK_HOME /opt/spark
-
-
-
-WORKDIR /opt/mahout/work-dir
-RUN chmod g+w /opt/mahout/work-dir
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]
-
-# Specify the User that the actual main process will run as
-USER ${spark_uid}
\ No newline at end of file
diff --git
a/resource-managers/kubernetes/docker/src/main/dockerfiles/entrypoint.sh
b/resource-managers/kubernetes/docker/src/main/dockerfiles/entrypoint.sh
deleted file mode 100755
index 726b613..0000000
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/entrypoint.sh
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/usr/bin/env bash
-
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# echo commands to the terminal output
-set -ex
-
-# Check whether there is a passwd entry for the container UID
-myuid=$(id -u)
-mygid=$(id -g)
-# turn off -e for getent because it will return error code in anonymous uid
case
-set +e
-uidentry=$(getent passwd $myuid)
-set -e
-
-# If there is no passwd entry for the container UID, attempt to create one
-if [ -z "$uidentry" ] ; then
- if [ -w /etc/passwd ] ; then
- echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous
uid}:$SPARK_HOME:/bin/false" >> /etc/passwd
- else
- echo "Container ENTRYPOINT failed to add passwd entry for anonymous
UID"
- fi
-fi
-
-SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*"
-env | grep SPARK_JAVA_OPT_ | sort -t_ -k4 -n | sed 's/[^=]*=\(.*\)/\1/g' >
/tmp/java_opts.txt
-readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt
-
-if [ -n "$SPARK_EXTRA_CLASSPATH" ]; then
- SPARK_CLASSPATH="$SPARK_CLASSPATH:$SPARK_EXTRA_CLASSPATH"
-fi
-
-
-
-if ! [ -z ${HADOOP_CONF_DIR+x} ]; then
- SPARK_CLASSPATH="$HADOOP_CONF_DIR:$SPARK_CLASSPATH:$MAHOUT_CLASSPATH";
-fi
-
-case "$1" in
- driver)
- shift 1
- CMD=(
- "$SPARK_HOME/bin/spark-submit"
- --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS"
- -conf spark.executor.extraLibraryPath=\
- "$MAHOUT_HOME/lib/mahout-spark_*-dependency-reduced.jar"
- --deploy-mode client
- "$@"
- )
- ;;
- executor)
- shift 1
- CMD=(
- ${JAVA_HOME}/bin/java
- "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
- -Xms$SPARK_EXECUTOR_MEMORY
- -Xmx$SPARK_EXECUTOR_MEMORY
- -cp "$SPARK_CLASSPATH:$MAHOUT_CLASSPATH"
- org.apache.spark.executor.CoarseGrainedExecutorBackend
- --driver-url $SPARK_DRIVER_URL
- --executor-id $SPARK_EXECUTOR_ID
- --conf spark.executor.extraLibraryPath=\
- "$MAHOUT_HOME/lib/mahout-spark_*-dependency-reduced.jar"
- --cores $SPARK_EXECUTOR_CORES
- --app-id $SPARK_APPLICATION_ID
- --hostname $SPARK_EXECUTOR_POD_IP
- )
- ;;
-
- *)
- echo "Non-spark-on-k8s command provided, proceeding in pass-through
mode..."
- CMD=("$@")
- ;;
-esac
-
-# Execute the container CMD under tini for better hygiene
-exec /sbin/tini -s -- "${CMD[@]}"
\ No newline at end of file
