Repository: incubator-zeppelin Updated Branches: refs/heads/master 146b330d4 -> 8243140bd
ZEPPELIN-207: travis-ci build log is too long to be displayed Author: caofangkun <[email protected]> Closes #186 from caofangkun/zeppelin-207 and squashes the following commits: a2d155a [caofangkun] ZEPPELIN-207: travis-ci build log is too long to be displayed 713ce5a [caofangkun] ZEPPELIN-207: travis-ci build log is too long to be displayed 3ee6a15 [caofangkun] ZEPPELIN-207: travis-ci build log is too long to be displayed aa2568d [caofangkun] ZEPPELIN-207: travis-ci build log is too long to be displayed e5b0067 [caofangkun] ZEPPELIN-207: travis-ci build log is too long to be displayed Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/8243140b Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/8243140b Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/8243140b Branch: refs/heads/master Commit: 8243140bdcea19500b6b47455b1547a69f4f19aa Parents: 146b330 Author: caofangkun <[email protected]> Authored: Fri Aug 7 17:55:47 2015 +0800 Committer: Lee moon soo <[email protected]> Committed: Mon Aug 10 22:24:36 2015 -0700 ---------------------------------------------------------------------- .travis.yml | 22 ++++++++-------- dev/travis/save-logs.py | 54 +++++++++++++++++++++++++++++++++++++++ dev/travis/travis-install.sh | 31 ++++++++++++++++++++++ testing/startSparkCluster.sh | 2 +- 4 files changed, 97 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/8243140b/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index ac80e8a..e1b3c5d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,34 +22,34 @@ before_install: - "sh -e /etc/init.d/xvfb start" install: - - mvn package -DskipTests -Phadoop-2.3 -Ppyspark -B + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -DskipTests -Phadoop-2.3 -Ppyspark -B before_script: - script: # spark 1.4 - - mvn package -Pbuild-distr -Phadoop-2.3 -Ppyspark -B + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pbuild-distr -Phadoop-2.3 -Ppyspark -B - ./testing/startSparkCluster.sh 1.4.0 2.3 - - SPARK_HOME=`pwd`/spark-1.4.0-bin-hadoop2.3 mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B - ./testing/stopSparkCluster.sh 1.4.0 2.3 # spark 1.3 - - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - - mvn package -Pbuild-distr -Pspark-1.3 -Phadoop-2.3 -B + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pbuild-distr -Pspark-1.3 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.3.1 2.3 - - SPARK_HOME=./spark-1.3.1-bin-hadoop2.3 mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - - mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - - mvn package -Pbuild-distr -Pspark-1.2 -Phadoop-2.3 -B + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn package -Pbuild-distr -Pspark-1.2 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.2.1 2.3 - - SPARK_HOME=./spark-1.2.1-bin-hadoop2.3 mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - - mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - mvn package -Pbuild-distr -Pspark-1.1 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.1.1 2.3 - - SPARK_HOME=./spark-1.1.1-bin-hadoop2.3 mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - /bin/bash ./dev/travis/travis-install.sh `pwd` mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - ./testing/stopSparkCluster.sh 1.1.1 2.3 after_failure: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/8243140b/dev/travis/save-logs.py ---------------------------------------------------------------------- diff --git a/dev/travis/save-logs.py b/dev/travis/save-logs.py new file mode 100755 index 0000000..5f4ad28 --- /dev/null +++ b/dev/travis/save-logs.py @@ -0,0 +1,54 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import subprocess +from datetime import datetime, timedelta + +def main(file, cmd): + print cmd, "writing to", file + out = open(file, "w") + count = 0 + process = subprocess.Popen(cmd, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE) + + start = datetime.now() + nextPrint = datetime.now() + timedelta(seconds=1) + # wait for the process to terminate + pout = process.stdout + line = pout.readline() + while line: + count = count + 1 + if datetime.now() > nextPrint: + diff = datetime.now() - start + sys.stdout.write("\r%d seconds %d log lines"%(diff.seconds, count)) + sys.stdout.flush() + nextPrint = datetime.now() + timedelta(seconds=10) + out.write(line) + line = pout.readline() + out.close() + errcode = process.wait() + diff = datetime.now() - start + sys.stdout.write("\r%d seconds %d log lines"%(diff.seconds, count)) + print + print cmd, "done", errcode + return errcode + +if __name__ == "__main__": + if sys.argv < 1: + print "Usage: %s [file info]" % sys.argv[0] + sys.exit(1) + + sys.exit(main(sys.argv[1], sys.argv[2:])) http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/8243140b/dev/travis/travis-install.sh ---------------------------------------------------------------------- diff --git a/dev/travis/travis-install.sh b/dev/travis/travis-install.sh new file mode 100755 index 0000000..56a68b4 --- /dev/null +++ b/dev/travis/travis-install.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +ZEPPELIN_SRC_ROOT_DIR=$1 +shift +TRAVIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +cd ${ZEPPELIN_SRC_ROOT_DIR} + +python ${TRAVIS_SCRIPT_DIR}/save-logs.py "install.txt" "$@" +BUILD_RET_VAL=$? + +if [[ "$BUILD_RET_VAL" != "0" ]]; +then + cat "install.txt" +fi + +exit ${BUILD_RET_VAL} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/8243140b/testing/startSparkCluster.sh ---------------------------------------------------------------------- diff --git a/testing/startSparkCluster.sh b/testing/startSparkCluster.sh index 8ee0a32..7cdd996 100755 --- a/testing/startSparkCluster.sh +++ b/testing/startSparkCluster.sh @@ -29,7 +29,7 @@ HADOOP_VERSION="${2}" FWDIR=$(dirname "${BASH_SOURCE-$0}") ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} - +echo "SPARK_HOME is ${SPARK_HOME} " if [ ! -d "${SPARK_HOME}" ]; then wget -q http://www.us.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz tar zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
