This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new f0b6245 [SPARK-27158][BUILD] dev/mima and dev/scalastyle support dynamic profiles f0b6245 is described below commit f0b6245ea4cef0dc61d4277f1d6874ccf315e47a Author: Yuming Wang <yumw...@ebay.com> AuthorDate: Fri Mar 15 08:20:42 2019 +0900 [SPARK-27158][BUILD] dev/mima and dev/scalastyle support dynamic profiles ## What changes were proposed in this pull request? `dev/mima` and `dev/scalastyle` support dynamic reading profiles from `modules.py`. ## How was this patch tested? manual tests Closes #24089 from wangyum/SPARK-27158. Authored-by: Yuming Wang <yumw...@ebay.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- dev/lint-scala | 2 +- dev/mima | 2 +- dev/run-tests.py | 14 ++++++++++---- dev/sbt-checkstyle | 4 ++-- dev/scalastyle | 12 ++++-------- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/dev/lint-scala b/dev/lint-scala index c676dfd..9c701ab 100755 --- a/dev/lint-scala +++ b/dev/lint-scala @@ -20,4 +20,4 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" SPARK_ROOT_DIR="$(dirname $SCRIPT_DIR)" -"$SCRIPT_DIR/scalastyle" +"$SCRIPT_DIR/scalastyle" "$1" diff --git a/dev/mima b/dev/mima index dc7b085..f324c5c 100755 --- a/dev/mima +++ b/dev/mima @@ -24,7 +24,7 @@ set -e FWDIR="$(cd "`dirname "$0"`"/..; pwd)" cd "$FWDIR" -SPARK_PROFILES="-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive" +SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | tail -n1)" OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | tail -n1)" diff --git a/dev/run-tests.py b/dev/run-tests.py index aa106af..535c877 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -175,9 +175,11 @@ def run_apache_rat_checks(): run_cmd([os.path.join(SPARK_HOME, "dev", "check-license")]) -def run_scala_style_checks(): +def run_scala_style_checks(build_profiles): set_title_and_block("Running Scala style checks", "BLOCK_SCALA_STYLE") - run_cmd([os.path.join(SPARK_HOME, "dev", "lint-scala")]) + profiles = " ".join(build_profiles) + print("[info] Checking Scala style using SBT with these profiles: ", profiles) + run_cmd([os.path.join(SPARK_HOME, "dev", "lint-scala"), profiles]) def run_java_style_checks(build_profiles): @@ -359,7 +361,10 @@ def build_apache_spark(build_tool, hadoop_version): def detect_binary_inop_with_mima(hadoop_version): build_profiles = get_hadoop_profiles(hadoop_version) + modules.root.build_profile_flags set_title_and_block("Detecting binary incompatibilities with MiMa", "BLOCK_MIMA") - run_cmd([os.path.join(SPARK_HOME, "dev", "mima")] + build_profiles) + profiles = " ".join(build_profiles) + print("[info] Detecting binary incompatibilities with MiMa using SBT with these profiles: ", + profiles) + run_cmd([os.path.join(SPARK_HOME, "dev", "mima"), profiles]) def run_scala_tests_maven(test_profiles): @@ -582,7 +587,8 @@ def main(): if not changed_files or any(f.endswith(".scala") or f.endswith("scalastyle-config.xml") for f in changed_files): - run_scala_style_checks() + build_profiles = get_hadoop_profiles(hadoop_version) + modules.root.build_profile_flags + run_scala_style_checks(build_profiles) should_run_java_style_checks = False if not changed_files or any(f.endswith(".java") or f.endswith("checkstyle.xml") diff --git a/dev/sbt-checkstyle b/dev/sbt-checkstyle index 1ecad59..415155f 100755 --- a/dev/sbt-checkstyle +++ b/dev/sbt-checkstyle @@ -17,12 +17,12 @@ # limitations under the License. # -profiles=${1:-"-Pkinesis-asl -Pmesos -Pkubernetes -Pyarn -Phive -Phive-thriftserver"} +SPARK_PROFILES=${1:-"-Pkinesis-asl -Pmesos -Pkubernetes -Pyarn -Phive -Phive-thriftserver"} # NOTE: echo "q" is needed because SBT prompts the user for input on encountering a build file # with failure (either resolution or compilation); the "q" makes SBT quit. ERRORS=$(echo -e "q\n" \ - | build/sbt ${profiles} checkstyle test:checkstyle \ + | build/sbt ${SPARK_PROFILES} checkstyle test:checkstyle \ | awk '{if($1~/error/)print}' \ ) diff --git a/dev/scalastyle b/dev/scalastyle index d5cebb3..212ef90 100755 --- a/dev/scalastyle +++ b/dev/scalastyle @@ -17,19 +17,15 @@ # limitations under the License. # +SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"} + # NOTE: echo "q" is needed because SBT prompts the user for input on encountering a build file # with failure (either resolution or compilation); the "q" makes SBT quit. ERRORS=$(echo -e "q\n" \ | build/sbt \ - -Pkinesis-asl \ - -Pmesos \ - -Pkubernetes \ - -Pyarn \ - -Phive \ - -Phive-thriftserver \ - -Pspark-ganglia-lgpl \ + ${SPARK_PROFILES} \ -Pdocker-integration-tests \ - -Pkubernetes-integration-tests \ + -Pkubernetes-integration-tests \ scalastyle test:scalastyle \ | awk '{if($1~/error/)print}' \ ) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org